diff --git a/src/main/java/com/commafeed/CommaFeedApplication.java b/src/main/java/com/commafeed/CommaFeedApplication.java
index 41018307..decb0fc1 100644
--- a/src/main/java/com/commafeed/CommaFeedApplication.java
+++ b/src/main/java/com/commafeed/CommaFeedApplication.java
@@ -1,17 +1,9 @@
package com.commafeed;
-import io.dropwizard.Application;
-import io.dropwizard.assets.AssetsBundle;
-import io.dropwizard.db.DataSourceFactory;
-import io.dropwizard.hibernate.HibernateBundle;
-import io.dropwizard.migrations.MigrationsBundle;
-import io.dropwizard.servlets.CacheBustingFilter;
-import io.dropwizard.setup.Bootstrap;
-import io.dropwizard.setup.Environment;
-
import java.io.IOException;
import java.util.Date;
import java.util.EnumSet;
+import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import javax.servlet.DispatcherType;
@@ -22,6 +14,7 @@ import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import org.eclipse.jetty.server.session.SessionHandler;
+import org.hibernate.cfg.AvailableSettings;
import com.commafeed.backend.feed.FeedRefreshTaskGiver;
import com.commafeed.backend.feed.FeedRefreshUpdater;
@@ -39,10 +32,8 @@ import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserSettings;
import com.commafeed.backend.service.StartupService;
import com.commafeed.backend.service.UserService;
-import com.commafeed.backend.task.OldStatusesCleanupTask;
-import com.commafeed.backend.task.OrphansCleanupTask;
-import com.commafeed.frontend.auth.SecurityCheckProvider;
-import com.commafeed.frontend.auth.SecurityCheckProvider.SecurityCheckUserServiceProvider;
+import com.commafeed.backend.task.ScheduledTask;
+import com.commafeed.frontend.auth.SecurityCheckFactoryProvider;
import com.commafeed.frontend.resource.AdminREST;
import com.commafeed.frontend.resource.CategoryREST;
import com.commafeed.frontend.resource.EntryREST;
@@ -54,19 +45,22 @@ import com.commafeed.frontend.servlet.AnalyticsServlet;
import com.commafeed.frontend.servlet.CustomCssServlet;
import com.commafeed.frontend.servlet.LogoutServlet;
import com.commafeed.frontend.servlet.NextUnreadServlet;
-import com.commafeed.frontend.session.SessionHelperProvider;
+import com.commafeed.frontend.session.SessionHelperFactoryProvider;
import com.google.inject.Guice;
import com.google.inject.Injector;
-import com.sun.jersey.api.core.ResourceConfig;
-import com.wordnik.swagger.config.ConfigFactory;
-import com.wordnik.swagger.config.ScannerFactory;
-import com.wordnik.swagger.config.SwaggerConfig;
-import com.wordnik.swagger.jaxrs.config.DefaultJaxrsScanner;
-import com.wordnik.swagger.jaxrs.listing.ApiDeclarationProvider;
-import com.wordnik.swagger.jaxrs.listing.ApiListingResourceJSON;
-import com.wordnik.swagger.jaxrs.listing.ResourceListingProvider;
-import com.wordnik.swagger.jaxrs.reader.DefaultJaxrsApiReader;
-import com.wordnik.swagger.reader.ClassReaders;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
+
+import io.dropwizard.Application;
+import io.dropwizard.assets.AssetsBundle;
+import io.dropwizard.db.DataSourceFactory;
+import io.dropwizard.forms.MultiPartBundle;
+import io.dropwizard.hibernate.HibernateBundle;
+import io.dropwizard.migrations.MigrationsBundle;
+import io.dropwizard.server.DefaultServerFactory;
+import io.dropwizard.servlets.CacheBustingFilter;
+import io.dropwizard.setup.Bootstrap;
+import io.dropwizard.setup.Environment;
public class CommaFeedApplication extends Application
{
@@ -89,36 +83,44 @@ public class CommaFeedApplication extends Application {
FeedSubscription.class, User.class, UserRole.class, UserSettings.class) {
@Override
public DataSourceFactory getDataSourceFactory(CommaFeedConfiguration configuration) {
- return configuration.getDatabase();
+ DataSourceFactory factory = configuration.getDataSourceFactory();
+
+ // keep using old id generator for backward compatibility
+ factory.getProperties().put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false");
+
+ factory.getProperties().put(AvailableSettings.STATEMENT_BATCH_SIZE, "50");
+ factory.getProperties().put(AvailableSettings.BATCH_VERSIONED_DATA, "true");
+ return factory;
}
});
bootstrap.addBundle(new MigrationsBundle() {
@Override
public DataSourceFactory getDataSourceFactory(CommaFeedConfiguration configuration) {
- return configuration.getDatabase();
+ return configuration.getDataSourceFactory();
}
});
bootstrap.addBundle(new AssetsBundle("/assets/", "/", "index.html"));
+ bootstrap.addBundle(new MultiPartBundle());
}
@Override
public void run(CommaFeedConfiguration config, Environment environment) throws Exception {
- // configure context path
- environment.getApplicationContext().setContextPath(config.getApplicationSettings().getContextPath());
-
// guice init
Injector injector = Guice.createInjector(new CommaFeedModule(hibernateBundle.getSessionFactory(), config, environment.metrics()));
- // Auth/session management
+ // session management
environment.servlets().setSessionHandler(new SessionHandler(config.getSessionManagerFactory().build()));
- environment.jersey().register(new SecurityCheckUserServiceProvider(injector.getInstance(UserService.class)));
- environment.jersey().register(SecurityCheckProvider.class);
- environment.jersey().register(SessionHelperProvider.class);
+
+ // support for "@SecurityCheck User user" injection
+ environment.jersey().register(new SecurityCheckFactoryProvider.Binder(injector.getInstance(UserService.class)));
+ // support for "@Context SessionHelper sessionHelper" injection
+ environment.jersey().register(new SessionHelperFactoryProvider.Binder());
// REST resources
environment.jersey().setUrlPattern("/rest/*");
+ ((DefaultServerFactory) config.getServerFactory()).setJerseyRootPath("/rest/*");
environment.jersey().register(injector.getInstance(AdminREST.class));
environment.jersey().register(injector.getInstance(CategoryREST.class));
environment.jersey().register(injector.getInstance(EntryREST.class));
@@ -134,9 +136,13 @@ public class CommaFeedApplication extends Application {
environment.servlets().addServlet("analytics.js", injector.getInstance(AnalyticsServlet.class)).addMapping("/analytics.js");
// Scheduled tasks
- ScheduledExecutorService executor = environment.lifecycle().scheduledExecutorService("task-scheduler").build();
- injector.getInstance(OldStatusesCleanupTask.class).register(executor);
- injector.getInstance(OrphansCleanupTask.class).register(executor);
+ Set tasks = injector.getInstance(Key.get(new TypeLiteral>() {
+ }));
+ ScheduledExecutorService executor = environment.lifecycle().scheduledExecutorService("task-scheduler", true).threads(tasks.size())
+ .build();
+ for (ScheduledTask task : tasks) {
+ task.register(executor);
+ }
// database init/changelogs
environment.lifecycle().manage(injector.getInstance(StartupService.class));
@@ -146,16 +152,6 @@ public class CommaFeedApplication extends Application {
environment.lifecycle().manage(injector.getInstance(FeedRefreshWorker.class));
environment.lifecycle().manage(injector.getInstance(FeedRefreshUpdater.class));
- // Swagger
- environment.jersey().register(new ApiListingResourceJSON());
- environment.jersey().register(new ApiDeclarationProvider());
- environment.jersey().register(new ResourceListingProvider());
- ScannerFactory.setScanner(new DefaultJaxrsScanner());
- ClassReaders.setReader(new DefaultJaxrsApiReader());
- SwaggerConfig swaggerConfig = ConfigFactory.config();
- swaggerConfig.setApiVersion("1");
- swaggerConfig.setBasePath("/rest");
-
// cache configuration
// prevent caching on REST resources, except for favicons
environment.servlets().addFilter("cache-filter", new CacheBustingFilter() {
@@ -170,8 +166,6 @@ public class CommaFeedApplication extends Application {
}
}).addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/rest/*");
- // enable wadl
- environment.jersey().disable(ResourceConfig.FEATURE_DISABLE_WADL);
}
public static void main(String[] args) throws Exception {
diff --git a/src/main/java/com/commafeed/CommaFeedConfiguration.java b/src/main/java/com/commafeed/CommaFeedConfiguration.java
index 5e8fc163..018b991e 100644
--- a/src/main/java/com/commafeed/CommaFeedConfiguration.java
+++ b/src/main/java/com/commafeed/CommaFeedConfiguration.java
@@ -12,7 +12,7 @@ import javax.validation.constraints.NotNull;
import lombok.Getter;
-import org.apache.commons.lang.time.DateUtils;
+import org.apache.commons.lang3.time.DateUtils;
import org.hibernate.validator.constraints.NotBlank;
import com.commafeed.backend.cache.RedisPoolFactory;
@@ -35,7 +35,7 @@ public class CommaFeedConfiguration extends Configuration {
@Valid
@NotNull
@JsonProperty("database")
- private DataSourceFactory database = new DataSourceFactory();
+ private DataSourceFactory dataSourceFactory = new DataSourceFactory();
@Valid
@NotNull
@@ -64,60 +64,76 @@ public class CommaFeedConfiguration extends Configuration {
public static class ApplicationSettings {
@NotNull
@NotBlank
- private String contextPath;
-
- @NotNull
- @NotBlank
+ @Valid
private String publicUrl;
@NotNull
- private boolean allowRegistrations;
+ @Valid
+ private Boolean allowRegistrations;
+
+ @NotNull
+ @Valid
+ private Boolean createDemoAccount;
private String googleAnalyticsTrackingCode;
- @NotNull
- @Min(1)
- private int backgroundThreads;
+ private String googleAuthKey;
@NotNull
@Min(1)
- private int databaseUpdateThreads;
+ @Valid
+ private Integer backgroundThreads;
+
+ @NotNull
+ @Min(1)
+ @Valid
+ private Integer databaseUpdateThreads;
private String smtpHost;
-
private int smtpPort;
-
private boolean smtpTls;
-
private String smtpUserName;
-
private String smtpPassword;
+ private String smtpFromAddress;
@NotNull
- private boolean heavyLoad;
+ @Valid
+ private Boolean heavyLoad;
@NotNull
- private boolean pubsubhubbub;
+ @Valid
+ private Boolean pubsubhubbub;
@NotNull
- private boolean imageProxyEnabled;
+ @Valid
+ private Boolean imageProxyEnabled;
@NotNull
@Min(0)
- private int queryTimeout;
+ @Valid
+ private Integer queryTimeout;
@NotNull
@Min(0)
- private int keepStatusDays;
+ @Valid
+ private Integer keepStatusDays;
@NotNull
@Min(0)
- private int refreshIntervalMinutes;
+ @Valid
+ private Integer maxFeedCapacity;
@NotNull
+ @Min(0)
+ @Valid
+ private Integer refreshIntervalMinutes;
+
+ @NotNull
+ @Valid
private CacheType cache;
@NotNull
+ @Valid
private String announcement;
public Date getUnreadThreshold() {
diff --git a/src/main/java/com/commafeed/CommaFeedModule.java b/src/main/java/com/commafeed/CommaFeedModule.java
index a18b0be3..9cb83429 100644
--- a/src/main/java/com/commafeed/CommaFeedModule.java
+++ b/src/main/java/com/commafeed/CommaFeedModule.java
@@ -11,9 +11,15 @@ import com.commafeed.CommaFeedConfiguration.CacheType;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.cache.NoopCacheService;
import com.commafeed.backend.cache.RedisCacheService;
-import com.commafeed.backend.favicon.DefaultFaviconFetcher;
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
+import com.commafeed.backend.favicon.DefaultFaviconFetcher;
+import com.commafeed.backend.favicon.FacebookFaviconFetcher;
import com.commafeed.backend.favicon.YoutubeFaviconFetcher;
+import com.commafeed.backend.task.OldEntriesCleanupTask;
+import com.commafeed.backend.task.OldStatusesCleanupTask;
+import com.commafeed.backend.task.OrphanedContentsCleanupTask;
+import com.commafeed.backend.task.OrphanedFeedsCleanupTask;
+import com.commafeed.backend.task.ScheduledTask;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.multibindings.Multibinder;
@@ -38,8 +44,15 @@ public class CommaFeedModule extends AbstractModule {
log.info("using cache {}", cacheService.getClass());
bind(CacheService.class).toInstance(cacheService);
- Multibinder multibinder = Multibinder.newSetBinder(binder(), AbstractFaviconFetcher.class);
- multibinder.addBinding().to(YoutubeFaviconFetcher.class);
- multibinder.addBinding().to(DefaultFaviconFetcher.class);
+ Multibinder faviconMultibinder = Multibinder.newSetBinder(binder(), AbstractFaviconFetcher.class);
+ faviconMultibinder.addBinding().to(YoutubeFaviconFetcher.class);
+ faviconMultibinder.addBinding().to(FacebookFaviconFetcher.class);
+ faviconMultibinder.addBinding().to(DefaultFaviconFetcher.class);
+
+ Multibinder taskMultibinder = Multibinder.newSetBinder(binder(), ScheduledTask.class);
+ taskMultibinder.addBinding().to(OldStatusesCleanupTask.class);
+ taskMultibinder.addBinding().to(OldEntriesCleanupTask.class);
+ taskMultibinder.addBinding().to(OrphanedFeedsCleanupTask.class);
+ taskMultibinder.addBinding().to(OrphanedContentsCleanupTask.class);
}
}
diff --git a/src/main/java/com/commafeed/backend/ContentEncodingInterceptor.java b/src/main/java/com/commafeed/backend/ContentEncodingInterceptor.java
index 939ea478..7c4ad666 100644
--- a/src/main/java/com/commafeed/backend/ContentEncodingInterceptor.java
+++ b/src/main/java/com/commafeed/backend/ContentEncodingInterceptor.java
@@ -16,7 +16,7 @@ import org.apache.http.entity.HttpEntityWrapper;
import org.apache.http.protocol.HttpContext;
class ContentEncodingInterceptor implements HttpResponseInterceptor {
-
+
private static final Set ALLOWED_CONTENT_ENCODINGS = new HashSet<>(Arrays.asList("gzip", "x-gzip", "deflate", "identity"));
@Override
@@ -28,17 +28,17 @@ class ContentEncodingInterceptor implements HttpResponseInterceptor {
}
}
}
-
+
private boolean containsUnsupportedEncodings(Header contentEncodingHeader) {
HeaderElement[] codecs = contentEncodingHeader.getElements();
-
+
for (final HeaderElement codec : codecs) {
String codecName = codec.getName().toLowerCase(Locale.US);
if (!ALLOWED_CONTENT_ENCODINGS.contains(codecName)) {
return true;
}
}
-
+
return false;
}
@@ -47,9 +47,9 @@ class ContentEncodingInterceptor implements HttpResponseInterceptor {
@Override
public Header getContentEncoding() {
return null;
- };
+ }
};
-
+
response.setEntity(wrapped);
}
diff --git a/src/main/java/com/commafeed/backend/HttpGetter.java b/src/main/java/com/commafeed/backend/HttpGetter.java
index bca16f9f..a2ac67da 100644
--- a/src/main/java/com/commafeed/backend/HttpGetter.java
+++ b/src/main/java/com/commafeed/backend/HttpGetter.java
@@ -17,7 +17,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.http.Consts;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
@@ -34,7 +34,7 @@ import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.ConnectionConfig;
-import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
@@ -53,7 +53,7 @@ public class HttpGetter {
private static final String ACCEPT_LANGUAGE = "en";
private static final String PRAGMA_NO_CACHE = "No-cache";
private static final String CACHE_CONTROL_NO_CACHE = "no-cache";
-
+
private static final HttpResponseInterceptor REMOVE_INCORRECT_CONTENT_ENCODING = new ContentEncodingInterceptor();
private static SSLContext SSL_CONTEXT = null;
@@ -181,8 +181,8 @@ public class HttpGetter {
builder.addInterceptorFirst(REMOVE_INCORRECT_CONTENT_ENCODING);
builder.disableAutomaticRetries();
- builder.setSslcontext(SSL_CONTEXT);
- builder.setHostnameVerifier(SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
+ builder.setSSLContext(SSL_CONTEXT);
+ builder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
RequestConfig.Builder configBuilder = RequestConfig.custom();
configBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
diff --git a/src/main/java/com/commafeed/backend/cache/RedisCacheService.java b/src/main/java/com/commafeed/backend/cache/RedisCacheService.java
index 6a3eb460..8dd95134 100644
--- a/src/main/java/com/commafeed/backend/cache/RedisCacheService.java
+++ b/src/main/java/com/commafeed/backend/cache/RedisCacheService.java
@@ -1,5 +1,6 @@
package com.commafeed.backend.cache;
+import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -18,7 +19,6 @@ import com.commafeed.frontend.model.Category;
import com.commafeed.frontend.model.UnreadCount;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Lists;
@Slf4j
@RequiredArgsConstructor
@@ -30,7 +30,7 @@ public class RedisCacheService extends CacheService {
@Override
public List getLastEntries(Feed feed) {
- List list = Lists.newArrayList();
+ List list = new ArrayList<>();
try (Jedis jedis = pool.getResource()) {
String key = buildRedisEntryKey(feed);
Set members = jedis.smembers(key);
diff --git a/src/main/java/com/commafeed/backend/cache/RedisPoolFactory.java b/src/main/java/com/commafeed/backend/cache/RedisPoolFactory.java
index 302a2793..9cdb6671 100644
--- a/src/main/java/com/commafeed/backend/cache/RedisPoolFactory.java
+++ b/src/main/java/com/commafeed/backend/cache/RedisPoolFactory.java
@@ -2,7 +2,7 @@ package com.commafeed.backend.cache;
import lombok.Getter;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
diff --git a/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java b/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java
index d8e94817..6fc78806 100644
--- a/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java
+++ b/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java
@@ -1,19 +1,19 @@
package com.commafeed.backend.dao;
import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
-import org.apache.commons.lang.ObjectUtils;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.QFeedCategory;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
-import com.google.common.collect.Lists;
-import com.mysema.query.types.Predicate;
+import com.querydsl.core.types.Predicate;
@Singleton
public class FeedCategoryDAO extends GenericDAO {
@@ -26,11 +26,11 @@ public class FeedCategoryDAO extends GenericDAO {
}
public List findAll(User user) {
- return newQuery().from(category).where(category.user.eq(user)).join(category.user, QUser.user).fetch().list(category);
+ return query().selectFrom(category).where(category.user.eq(user)).join(category.user, QUser.user).fetchJoin().fetch();
}
public FeedCategory findById(User user, Long id) {
- return newQuery().from(category).where(category.user.eq(user), category.id.eq(id)).uniqueResult(category);
+ return query().selectFrom(category).where(category.user.eq(user), category.id.eq(id)).fetchOne();
}
public FeedCategory findByName(User user, String name, FeedCategory parent) {
@@ -40,7 +40,7 @@ public class FeedCategoryDAO extends GenericDAO {
} else {
parentPredicate = category.parent.eq(parent);
}
- return newQuery().from(category).where(category.user.eq(user), category.name.eq(name), parentPredicate).uniqueResult(category);
+ return query().selectFrom(category).where(category.user.eq(user), category.name.eq(name), parentPredicate).fetchOne();
}
public List findByParent(User user, FeedCategory parent) {
@@ -50,18 +50,11 @@ public class FeedCategoryDAO extends GenericDAO {
} else {
parentPredicate = category.parent.eq(parent);
}
- return newQuery().from(category).where(category.user.eq(user), parentPredicate).list(category);
+ return query().selectFrom(category).where(category.user.eq(user), parentPredicate).fetch();
}
public List findAllChildrenCategories(User user, FeedCategory parent) {
- List list = Lists.newArrayList();
- List all = findAll(user);
- for (FeedCategory cat : all) {
- if (isChild(cat, parent)) {
- list.add(cat);
- }
- }
- return list;
+ return findAll(user).stream().filter(c -> isChild(c, parent)).collect(Collectors.toList());
}
private boolean isChild(FeedCategory child, FeedCategory parent) {
@@ -70,7 +63,7 @@ public class FeedCategoryDAO extends GenericDAO {
}
boolean isChild = false;
while (child != null) {
- if (ObjectUtils.equals(child.getId(), parent.getId())) {
+ if (Objects.equals(child.getId(), parent.getId())) {
isChild = true;
break;
}
diff --git a/src/main/java/com/commafeed/backend/dao/FeedDAO.java b/src/main/java/com/commafeed/backend/dao/FeedDAO.java
index f8dad975..67f86cb8 100644
--- a/src/main/java/com/commafeed/backend/dao/FeedDAO.java
+++ b/src/main/java/com/commafeed/backend/dao/FeedDAO.java
@@ -7,7 +7,7 @@ import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.Feed;
@@ -15,8 +15,9 @@ import com.commafeed.backend.model.QFeed;
import com.commafeed.backend.model.QFeedSubscription;
import com.commafeed.backend.model.QUser;
import com.google.common.collect.Iterables;
-import com.mysema.query.BooleanBuilder;
-import com.mysema.query.jpa.hibernate.HibernateQuery;
+import com.querydsl.jpa.JPAExpressions;
+import com.querydsl.jpa.JPQLQuery;
+import com.querydsl.jpa.hibernate.HibernateQuery;
@Singleton
public class FeedDAO extends GenericDAO {
@@ -29,24 +30,23 @@ public class FeedDAO extends GenericDAO {
}
public List findNextUpdatable(int count, Date lastLoginThreshold) {
- BooleanBuilder disabledDatePredicate = new BooleanBuilder();
- disabledDatePredicate.or(feed.disabledUntil.isNull());
- disabledDatePredicate.or(feed.disabledUntil.lt(new Date()));
+ HibernateQuery query = query().selectFrom(feed);
+ query.where(feed.disabledUntil.isNull().or(feed.disabledUntil.lt(new Date())));
- HibernateQuery query = newQuery().from(feed);
if (lastLoginThreshold != null) {
QFeedSubscription subs = QFeedSubscription.feedSubscription;
QUser user = QUser.user;
- query.join(feed.subscriptions, subs).join(subs.user, user).where(disabledDatePredicate, user.lastLogin.gt(lastLoginThreshold));
- } else {
- query.where(disabledDatePredicate);
+
+ JPQLQuery subQuery = JPAExpressions.selectOne().from(subs);
+ subQuery.join(subs.user, user).where(user.lastLogin.gt(lastLoginThreshold));
+ query.where(subQuery.exists());
}
- return query.orderBy(feed.disabledUntil.asc()).limit(count).distinct().list(feed);
+ return query.orderBy(feed.disabledUntil.asc()).limit(count).distinct().fetch();
}
public Feed findByUrl(String normalizedUrl) {
- List feeds = newQuery().from(feed).where(feed.normalizedUrlHash.eq(DigestUtils.sha1Hex(normalizedUrl))).list(feed);
+ List feeds = query().selectFrom(feed).where(feed.normalizedUrlHash.eq(DigestUtils.sha1Hex(normalizedUrl))).fetch();
Feed feed = Iterables.getFirst(feeds, null);
if (feed != null && StringUtils.equals(normalizedUrl, feed.getNormalizedUrl())) {
return feed;
@@ -55,11 +55,12 @@ public class FeedDAO extends GenericDAO {
}
public List findByTopic(String topic) {
- return newQuery().from(feed).where(feed.pushTopicHash.eq(DigestUtils.sha1Hex(topic))).list(feed);
+ return query().selectFrom(feed).where(feed.pushTopicHash.eq(DigestUtils.sha1Hex(topic))).fetch();
}
public List findWithoutSubscriptions(int max) {
QFeedSubscription sub = QFeedSubscription.feedSubscription;
- return newQuery().from(feed).leftJoin(feed.subscriptions, sub).where(sub.id.isNull()).limit(max).list(feed);
+ return query().selectFrom(feed).where(JPAExpressions.selectOne().from(sub).where(sub.feed.eq(feed)).notExists()).limit(max)
+ .fetch();
}
}
diff --git a/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java b/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java
index b0239132..68eeb41e 100644
--- a/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java
+++ b/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java
@@ -10,12 +10,14 @@ import org.hibernate.SessionFactory;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
-import com.google.common.collect.Iterables;
+import com.querydsl.jpa.JPAExpressions;
+import com.querydsl.jpa.JPQLQuery;
@Singleton
public class FeedEntryContentDAO extends GenericDAO {
private QFeedEntryContent content = QFeedEntryContent.feedEntryContent;
+ private QFeedEntry entry = QFeedEntry.feedEntry;
@Inject
public FeedEntryContentDAO(SessionFactory sessionFactory) {
@@ -23,15 +25,15 @@ public class FeedEntryContentDAO extends GenericDAO {
}
public Long findExisting(String contentHash, String titleHash) {
- List list = newQuery().from(content).where(content.contentHash.eq(contentHash), content.titleHash.eq(titleHash)).limit(1)
- .list(content.id);
- return Iterables.getFirst(list, null);
+ return query().select(content.id).from(content).where(content.contentHash.eq(contentHash), content.titleHash.eq(titleHash))
+ .fetchFirst();
}
public int deleteWithoutEntries(int max) {
- QFeedEntry entry = QFeedEntry.feedEntry;
- List list = newQuery().from(content).leftJoin(content.entries, entry).where(entry.id.isNull()).limit(max)
- .list(content);
+
+ JPQLQuery subQuery = JPAExpressions.selectOne().from(entry).where(entry.content.id.eq(content.id));
+ List list = query().selectFrom(content).where(subQuery.notExists()).limit(max).fetch();
+
int deleted = list.size();
delete(list);
return deleted;
diff --git a/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java b/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
index 65f9997f..bbb3853f 100644
--- a/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
+++ b/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
@@ -1,7 +1,7 @@
package com.commafeed.backend.dao;
-import java.util.Date;
import java.util.List;
+import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -11,10 +11,12 @@ import org.hibernate.SessionFactory;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
-import com.commafeed.backend.model.QFeed;
import com.commafeed.backend.model.QFeedEntry;
-import com.commafeed.backend.model.QFeedSubscription;
-import com.google.common.collect.Iterables;
+import com.querydsl.core.Tuple;
+import com.querydsl.core.types.dsl.NumberExpression;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
@Singleton
public class FeedEntryDAO extends GenericDAO {
@@ -27,22 +29,32 @@ public class FeedEntryDAO extends GenericDAO {
}
public Long findExisting(String guid, Feed feed) {
- List list = newQuery().from(entry).where(entry.guidHash.eq(DigestUtils.sha1Hex(guid)), entry.feed.eq(feed)).limit(1)
- .list(entry.id);
- return Iterables.getFirst(list, null);
+ return query().select(entry.id).from(entry).where(entry.guidHash.eq(DigestUtils.sha1Hex(guid)), entry.feed.eq(feed)).limit(1)
+ .fetchOne();
}
- public List findWithoutSubscriptions(int max) {
- QFeed feed = QFeed.feed;
- QFeedSubscription sub = QFeedSubscription.feedSubscription;
- return newQuery().from(entry).join(entry.feed, feed).leftJoin(feed.subscriptions, sub).where(sub.id.isNull()).limit(max)
- .list(entry);
+ public List findFeedsExceedingCapacity(long maxCapacity, long max) {
+ NumberExpression count = entry.id.count();
+ List tuples = query().select(entry.feed.id, count).from(entry).groupBy(entry.feed).having(count.gt(maxCapacity)).limit(max)
+ .fetch();
+ return tuples.stream().map(t -> new FeedCapacity(t.get(entry.feed.id), t.get(count))).collect(Collectors.toList());
}
- public int delete(Date olderThan, int max) {
- List list = newQuery().from(entry).where(entry.inserted.lt(olderThan)).limit(max).list(entry);
- int deleted = list.size();
- delete(list);
- return deleted;
+ public int delete(Long feedId, long max) {
+
+ List list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).limit(max).fetch();
+ return delete(list);
+ }
+
+ public int deleteOldEntries(Long feedId, long max) {
+ List list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).orderBy(entry.updated.asc()).limit(max).fetch();
+ return delete(list);
+ }
+
+ @AllArgsConstructor
+ @Getter
+ public static class FeedCapacity {
+ private Long id;
+ private Long capacity;
}
}
diff --git a/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java b/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java
index a349c8ed..2f1ea425 100644
--- a/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java
+++ b/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java
@@ -1,5 +1,6 @@
package com.commafeed.backend.dao;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
@@ -7,12 +8,14 @@ import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.builder.CompareToBuilder;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.builder.CompareToBuilder;
import org.hibernate.SessionFactory;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.FixedSizeSortedSet;
+import com.commafeed.backend.feed.FeedEntryKeyword;
+import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
@@ -26,11 +29,10 @@ import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings.ReadingOrder;
import com.commafeed.frontend.model.UnreadCount;
import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
import com.google.common.collect.Ordering;
-import com.mysema.query.BooleanBuilder;
-import com.mysema.query.Tuple;
-import com.mysema.query.jpa.hibernate.HibernateQuery;
+import com.querydsl.core.BooleanBuilder;
+import com.querydsl.core.Tuple;
+import com.querydsl.jpa.hibernate.HibernateQuery;
@Singleton
public class FeedEntryStatusDAO extends GenericDAO {
@@ -60,13 +62,13 @@ public class FeedEntryStatusDAO extends GenericDAO {
builder.append(o2.getEntryUpdated(), o1.getEntryUpdated());
builder.append(o2.getId(), o1.getId());
return builder.toComparison();
- };
+ }
};
private static final Comparator STATUS_COMPARATOR_ASC = Ordering.from(STATUS_COMPARATOR_DESC).reverse();
public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) {
- List statuses = newQuery().from(status).where(status.entry.eq(entry), status.subscription.eq(sub)).list(status);
+ List statuses = query().selectFrom(status).where(status.entry.eq(entry), status.subscription.eq(sub)).fetch();
FeedEntryStatus status = Iterables.getFirst(statuses, null);
return handleStatus(user, status, sub, entry);
}
@@ -91,7 +93,7 @@ public class FeedEntryStatusDAO extends GenericDAO {
}
public List findStarred(User user, Date newerThan, int offset, int limit, ReadingOrder order, boolean includeContent) {
- HibernateQuery query = newQuery().from(status).where(status.user.eq(user), status.starred.isTrue());
+ HibernateQuery query = query().selectFrom(status).where(status.user.eq(user), status.starred.isTrue());
if (newerThan != null) {
query.where(status.entryInserted.gt(newerThan));
}
@@ -102,9 +104,13 @@ public class FeedEntryStatusDAO extends GenericDAO {
query.orderBy(status.entryUpdated.desc(), status.id.desc());
}
- query.offset(offset).limit(limit).setTimeout(config.getApplicationSettings().getQueryTimeout());
+ query.offset(offset).limit(limit);
+ int timeout = config.getApplicationSettings().getQueryTimeout();
+ if (timeout > 0) {
+ query.setTimeout(timeout / 1000);
+ }
- List statuses = query.list(status);
+ List statuses = query.fetch();
for (FeedEntryStatus status : statuses) {
status = handleStatus(user, status, status.getSubscription(), status.getEntry());
fetchTags(user, status);
@@ -112,18 +118,21 @@ public class FeedEntryStatusDAO extends GenericDAO {
return lazyLoadContent(includeContent, statuses);
}
- private HibernateQuery buildQuery(User user, FeedSubscription sub, boolean unreadOnly, String keywords, Date newerThan, int offset,
- int limit, ReadingOrder order, Date last, String tag) {
+ private HibernateQuery buildQuery(User user, FeedSubscription sub, boolean unreadOnly, List keywords,
+ Date newerThan, int offset, int limit, ReadingOrder order, Date last, String tag) {
- HibernateQuery query = newQuery().from(entry).where(entry.feed.eq(sub.getFeed()));
+ HibernateQuery query = query().selectFrom(entry).where(entry.feed.eq(sub.getFeed()));
- if (keywords != null) {
+ if (CollectionUtils.isNotEmpty(keywords)) {
query.join(entry.content, content);
- for (String keyword : StringUtils.split(keywords)) {
+ for (FeedEntryKeyword keyword : keywords) {
BooleanBuilder or = new BooleanBuilder();
- or.or(content.content.containsIgnoreCase(keyword));
- or.or(content.title.containsIgnoreCase(keyword));
+ or.or(content.content.containsIgnoreCase(keyword.getKeyword()));
+ or.or(content.title.containsIgnoreCase(keyword.getKeyword()));
+ if (keyword.getMode() == Mode.EXCLUDE) {
+ or.not();
+ }
query.where(or);
}
}
@@ -180,15 +189,16 @@ public class FeedEntryStatusDAO extends GenericDAO {
return query;
}
- public List findBySubscriptions(User user, List subs, boolean unreadOnly, String keywords,
- Date newerThan, int offset, int limit, ReadingOrder order, boolean includeContent, boolean onlyIds, String tag) {
+ public List findBySubscriptions(User user, List subs, boolean unreadOnly,
+ List keywords, Date newerThan, int offset, int limit, ReadingOrder order, boolean includeContent,
+ boolean onlyIds, String tag) {
int capacity = offset + limit;
Comparator comparator = order == ReadingOrder.desc ? STATUS_COMPARATOR_DESC : STATUS_COMPARATOR_ASC;
FixedSizeSortedSet set = new FixedSizeSortedSet(capacity, comparator);
for (FeedSubscription sub : subs) {
Date last = (order != null && set.isFull()) ? set.last().getEntryUpdated() : null;
- HibernateQuery query = buildQuery(user, sub, unreadOnly, keywords, newerThan, -1, capacity, order, last, tag);
- List tuples = query.list(entry.id, entry.updated, status.id);
+ HibernateQuery query = buildQuery(user, sub, unreadOnly, keywords, newerThan, -1, capacity, order, last, tag);
+ List tuples = query.select(entry.id, entry.updated, status.id).fetch();
for (Tuple tuple : tuples) {
Long id = tuple.get(entry.id);
Date updated = tuple.get(entry.updated);
@@ -211,7 +221,7 @@ public class FeedEntryStatusDAO extends GenericDAO {
List placeholders = set.asList();
int size = placeholders.size();
if (size < offset) {
- return Lists.newArrayList();
+ return new ArrayList<>();
}
placeholders = placeholders.subList(Math.max(offset, 0), size);
@@ -219,7 +229,7 @@ public class FeedEntryStatusDAO extends GenericDAO {
if (onlyIds) {
statuses = placeholders;
} else {
- statuses = Lists.newArrayList();
+ statuses = new ArrayList<>();
for (FeedEntryStatus placeholder : placeholders) {
Long statusId = placeholder.getId();
FeedEntry entry = feedEntryDAO.findById(placeholder.getEntry().getId());
@@ -235,8 +245,8 @@ public class FeedEntryStatusDAO extends GenericDAO {
public UnreadCount getUnreadCount(User user, FeedSubscription subscription) {
UnreadCount uc = null;
- HibernateQuery query = buildQuery(user, subscription, true, null, null, -1, -1, null, null, null);
- List tuples = query.list(entry.count(), entry.updated.max());
+ HibernateQuery query = buildQuery(user, subscription, true, null, null, -1, -1, null, null, null);
+ List tuples = query.select(entry.count(), entry.updated.max()).fetch();
for (Tuple tuple : tuples) {
Long count = tuple.get(entry.count());
Date updated = tuple.get(entry.updated.max());
@@ -256,7 +266,7 @@ public class FeedEntryStatusDAO extends GenericDAO {
}
public List getOldStatuses(Date olderThan, int limit) {
- return newQuery().from(status).where(status.entryInserted.lt(olderThan), status.starred.isFalse()).limit(limit).list(status);
+ return query().selectFrom(status).where(status.entryInserted.lt(olderThan), status.starred.isFalse()).limit(limit).fetch();
}
}
diff --git a/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java b/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java
index e2dc3ac9..f0ef9388 100644
--- a/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java
+++ b/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java
@@ -23,10 +23,10 @@ public class FeedEntryTagDAO extends GenericDAO {
}
public List findByUser(User user) {
- return newQuery().from(tag).where(tag.user.eq(user)).distinct().list(tag.name);
+ return query().selectDistinct(tag.name).from(tag).where(tag.user.eq(user)).fetch();
}
public List findByEntry(User user, FeedEntry entry) {
- return newQuery().from(tag).where(tag.user.eq(user), tag.entry.eq(entry)).list(tag);
+ return query().selectFrom(tag).where(tag.user.eq(user), tag.entry.eq(entry)).fetch();
}
}
diff --git a/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java b/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java
index efcddd62..bfdc39e0 100644
--- a/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java
+++ b/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java
@@ -1,6 +1,8 @@
package com.commafeed.backend.dao;
import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -13,10 +15,8 @@ import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.QFeedSubscription;
import com.commafeed.backend.model.User;
-import com.google.common.base.Function;
import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.mysema.query.jpa.hibernate.HibernateQuery;
+import com.querydsl.jpa.hibernate.HibernateQuery;
@Singleton
public class FeedSubscriptionDAO extends GenericDAO {
@@ -29,57 +29,44 @@ public class FeedSubscriptionDAO extends GenericDAO {
}
public FeedSubscription findById(User user, Long id) {
- List subs = newQuery().from(sub).where(sub.user.eq(user), sub.id.eq(id)).leftJoin(sub.feed).fetch()
- .leftJoin(sub.category).fetch().list(sub);
+ List subs = query().selectFrom(sub).where(sub.user.eq(user), sub.id.eq(id)).leftJoin(sub.feed).fetchJoin()
+ .leftJoin(sub.category).fetchJoin().fetch();
return initRelations(Iterables.getFirst(subs, null));
}
public List findByFeed(Feed feed) {
- return newQuery().from(sub).where(sub.feed.eq(feed)).list(sub);
+ return query().selectFrom(sub).where(sub.feed.eq(feed)).fetch();
}
public FeedSubscription findByFeed(User user, Feed feed) {
- List subs = newQuery().from(sub).where(sub.user.eq(user), sub.feed.eq(feed)).list(sub);
+ List subs = query().selectFrom(sub).where(sub.user.eq(user), sub.feed.eq(feed)).fetch();
return initRelations(Iterables.getFirst(subs, null));
}
public List findAll(User user) {
- List subs = newQuery().from(sub).where(sub.user.eq(user)).leftJoin(sub.feed).fetch().leftJoin(sub.category)
- .fetch().list(sub);
+ List subs = query().selectFrom(sub).where(sub.user.eq(user)).leftJoin(sub.feed).fetchJoin()
+ .leftJoin(sub.category).fetchJoin().fetch();
return initRelations(subs);
}
public List findByCategory(User user, FeedCategory category) {
- HibernateQuery query = newQuery().from(sub).where(sub.user.eq(user));
+ HibernateQuery query = query().selectFrom(sub).where(sub.user.eq(user));
if (category == null) {
query.where(sub.category.isNull());
} else {
query.where(sub.category.eq(category));
}
- return initRelations(query.list(sub));
+ return initRelations(query.fetch());
}
public List findByCategories(User user, List categories) {
- List categoryIds = Lists.transform(categories, new Function() {
- @Override
- public Long apply(FeedCategory input) {
- return input.getId();
- }
- });
-
- List subscriptions = Lists.newArrayList();
- for (FeedSubscription sub : findAll(user)) {
- if (sub.getCategory() != null && categoryIds.contains(sub.getCategory().getId())) {
- subscriptions.add(sub);
- }
- }
- return subscriptions;
+ Set categoryIds = categories.stream().map(c -> c.getId()).collect(Collectors.toSet());
+ return findAll(user).stream().filter(s -> s.getCategory() != null && categoryIds.contains(s.getCategory().getId()))
+ .collect(Collectors.toList());
}
private List initRelations(List list) {
- for (FeedSubscription sub : list) {
- initRelations(sub);
- }
+ list.forEach(s -> initRelations(s));
return list;
}
diff --git a/src/main/java/com/commafeed/backend/dao/GenericDAO.java b/src/main/java/com/commafeed/backend/dao/GenericDAO.java
index 338ada5f..3a6884ad 100644
--- a/src/main/java/com/commafeed/backend/dao/GenericDAO.java
+++ b/src/main/java/com/commafeed/backend/dao/GenericDAO.java
@@ -1,22 +1,25 @@
package com.commafeed.backend.dao;
-import io.dropwizard.hibernate.AbstractDAO;
-
import java.util.Collection;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.AbstractModel;
-import com.mysema.query.jpa.hibernate.HibernateQuery;
+import com.querydsl.jpa.hibernate.HibernateQueryFactory;
+
+import io.dropwizard.hibernate.AbstractDAO;
public abstract class GenericDAO extends AbstractDAO {
+ private HibernateQueryFactory factory;
+
protected GenericDAO(SessionFactory sessionFactory) {
super(sessionFactory);
+ this.factory = new HibernateQueryFactory(() -> currentSession());
}
- protected HibernateQuery newQuery() {
- return new HibernateQuery(currentSession());
+ protected HibernateQueryFactory query() {
+ return factory;
}
public void saveOrUpdate(T model) {
@@ -24,19 +27,7 @@ public abstract class GenericDAO extends AbstractDAO
}
public void saveOrUpdate(Collection models) {
- for (T model : models) {
- persist(model);
- }
- }
-
- public void merge(T model) {
- currentSession().merge(model);
- }
-
- public void merge(Collection models) {
- for (T model : models) {
- merge(model);
- }
+ models.forEach(m -> persist(m));
}
public T findById(Long id) {
@@ -50,9 +41,7 @@ public abstract class GenericDAO extends AbstractDAO
}
public int delete(Collection objects) {
- for (T object : objects) {
- delete(object);
- }
+ objects.forEach(o -> delete(o));
return objects.size();
}
diff --git a/src/main/java/com/commafeed/backend/dao/UnitOfWork.java b/src/main/java/com/commafeed/backend/dao/UnitOfWork.java
index 27f8d347..ae21c4a8 100644
--- a/src/main/java/com/commafeed/backend/dao/UnitOfWork.java
+++ b/src/main/java/com/commafeed/backend/dao/UnitOfWork.java
@@ -5,17 +5,26 @@ import org.hibernate.SessionFactory;
import org.hibernate.Transaction;
import org.hibernate.context.internal.ManagedSessionContext;
-public abstract class UnitOfWork {
+public class UnitOfWork {
- private SessionFactory sessionFactory;
-
- public UnitOfWork(SessionFactory sessionFactory) {
- this.sessionFactory = sessionFactory;
+ @FunctionalInterface
+ public static interface SessionRunner {
+ public void runInSession();
}
- protected abstract T runInSession() throws Exception;
+ @FunctionalInterface
+ public static interface SessionRunnerReturningValue {
+ public T runInSession();
+ }
- public T run() {
+ public static void run(SessionFactory sessionFactory, SessionRunner sessionRunner) {
+ call(sessionFactory, () -> {
+ sessionRunner.runInSession();
+ return null;
+ });
+ }
+
+ public static T call(SessionFactory sessionFactory, SessionRunnerReturningValue sessionRunner) {
final Session session = sessionFactory.openSession();
if (ManagedSessionContext.hasBind(sessionFactory)) {
throw new IllegalStateException("Already in a unit of work!");
@@ -25,11 +34,11 @@ public abstract class UnitOfWork {
ManagedSessionContext.bind(session);
session.beginTransaction();
try {
- t = runInSession();
+ t = sessionRunner.runInSession();
commitTransaction(session);
} catch (Exception e) {
rollbackTransaction(session);
- this. rethrow(e);
+ UnitOfWork. rethrow(e);
}
} finally {
session.close();
@@ -38,14 +47,14 @@ public abstract class UnitOfWork {
return t;
}
- private void rollbackTransaction(Session session) {
+ private static void rollbackTransaction(Session session) {
final Transaction txn = session.getTransaction();
if (txn != null && txn.isActive()) {
txn.rollback();
}
}
- private void commitTransaction(Session session) {
+ private static void commitTransaction(Session session) {
final Transaction txn = session.getTransaction();
if (txn != null && txn.isActive()) {
txn.commit();
@@ -53,7 +62,7 @@ public abstract class UnitOfWork {
}
@SuppressWarnings("unchecked")
- private void rethrow(Exception e) throws E {
+ private static void rethrow(Exception e) throws E {
throw (E) e;
}
diff --git a/src/main/java/com/commafeed/backend/dao/UserDAO.java b/src/main/java/com/commafeed/backend/dao/UserDAO.java
index 6159392f..8b51e95c 100644
--- a/src/main/java/com/commafeed/backend/dao/UserDAO.java
+++ b/src/main/java/com/commafeed/backend/dao/UserDAO.java
@@ -6,7 +6,6 @@ import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.QUser;
-import com.commafeed.backend.model.QUserRole;
import com.commafeed.backend.model.User;
@Singleton
@@ -20,21 +19,18 @@ public class UserDAO extends GenericDAO {
}
public User findByName(String name) {
- return newQuery().from(user).where(user.name.equalsIgnoreCase(name)).leftJoin(user.roles, QUserRole.userRole).fetch()
- .uniqueResult(user);
+ return query().selectFrom(user).where(user.name.equalsIgnoreCase(name)).fetchOne();
}
public User findByApiKey(String key) {
- return newQuery().from(user).where(user.apiKey.equalsIgnoreCase(key)).leftJoin(user.roles, QUserRole.userRole).fetch()
- .uniqueResult(user);
+ return query().selectFrom(user).where(user.apiKey.equalsIgnoreCase(key)).fetchOne();
}
public User findByEmail(String email) {
- return newQuery().from(user).where(user.email.equalsIgnoreCase(email)).leftJoin(user.roles, QUserRole.userRole).fetch()
- .uniqueResult(user);
+ return query().selectFrom(user).where(user.email.equalsIgnoreCase(email)).fetchOne();
}
public long count() {
- return newQuery().from(user).count();
+ return query().selectFrom(user).fetchCount();
}
}
diff --git a/src/main/java/com/commafeed/backend/dao/UserRoleDAO.java b/src/main/java/com/commafeed/backend/dao/UserRoleDAO.java
index 883cd3ce..303d454e 100644
--- a/src/main/java/com/commafeed/backend/dao/UserRoleDAO.java
+++ b/src/main/java/com/commafeed/backend/dao/UserRoleDAO.java
@@ -2,6 +2,7 @@ package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
+import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -12,7 +13,6 @@ import com.commafeed.backend.model.QUserRole;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
-import com.google.common.collect.Sets;
@Singleton
public class UserRoleDAO extends GenericDAO {
@@ -25,18 +25,14 @@ public class UserRoleDAO extends GenericDAO {
}
public List findAll() {
- return newQuery().from(role).leftJoin(role.user).fetch().distinct().list(role);
+ return query().selectFrom(role).leftJoin(role.user).fetchJoin().distinct().fetch();
}
public List findAll(User user) {
- return newQuery().from(role).where(role.user.eq(user)).distinct().list(role);
+ return query().selectFrom(role).where(role.user.eq(user)).distinct().fetch();
}
public Set findRoles(User user) {
- Set list = Sets.newHashSet();
- for (UserRole role : findAll(user)) {
- list.add(role.getRole());
- }
- return list;
+ return findAll(user).stream().map(r -> r.getRole()).collect(Collectors.toSet());
}
}
diff --git a/src/main/java/com/commafeed/backend/dao/UserSettingsDAO.java b/src/main/java/com/commafeed/backend/dao/UserSettingsDAO.java
index 92e8afd5..0553ef84 100644
--- a/src/main/java/com/commafeed/backend/dao/UserSettingsDAO.java
+++ b/src/main/java/com/commafeed/backend/dao/UserSettingsDAO.java
@@ -20,6 +20,6 @@ public class UserSettingsDAO extends GenericDAO {
}
public UserSettings findByUser(User user) {
- return newQuery().from(settings).where(settings.user.eq(user)).uniqueResult(settings);
+ return query().selectFrom(settings).where(settings.user.eq(user)).fetchFirst();
}
}
diff --git a/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java b/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java
index bd10f197..5d6b0dc0 100644
--- a/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java
+++ b/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java
@@ -3,9 +3,11 @@ package com.commafeed.backend.favicon;
import java.util.Arrays;
import java.util.List;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.model.Feed;
@@ -18,7 +20,7 @@ public abstract class AbstractFaviconFetcher {
protected static int TIMEOUT = 4000;
- public abstract byte[] fetch(Feed feed);
+ public abstract Favicon fetch(Feed feed);
protected boolean isValidIconResponse(byte[] content, String contentType) {
if (content == null) {
@@ -48,4 +50,11 @@ public abstract class AbstractFaviconFetcher {
return true;
}
+
+ @RequiredArgsConstructor
+ @Getter
+ public static class Favicon {
+ private final byte[] icon;
+ private final String mediaType;
+ }
}
diff --git a/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java b/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java
index d6b90f8b..65d68364 100644
--- a/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java
+++ b/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java
@@ -6,7 +6,7 @@ import javax.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
@@ -28,9 +28,15 @@ public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
- public byte[] fetch(Feed feed) {
- String url = feed.getLink() != null ? feed.getLink() : feed.getUrl();
+ public Favicon fetch(Feed feed) {
+ Favicon icon = fetch(feed.getLink());
+ if (icon == null) {
+ icon = fetch(feed.getUrl());
+ }
+ return icon;
+ }
+ private Favicon fetch(String url) {
if (url == null) {
log.debug("url is null");
return null;
@@ -47,7 +53,7 @@ public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
url = url.substring(0, firstSlash);
}
- byte[] icon = getIconAtRoot(url);
+ Favicon icon = getIconAtRoot(url);
if (icon == null) {
icon = getIconInPage(url);
@@ -56,7 +62,7 @@ public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
return icon;
}
- private byte[] getIconAtRoot(String url) {
+ private Favicon getIconAtRoot(String url) {
byte[] bytes = null;
String contentType = null;
@@ -67,23 +73,25 @@ public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
- log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
+ log.debug("Failed to retrieve iconAtRoot for url {}: ", url);
+ log.trace("Failed to retrieve iconAtRoot for url {}: ", url, e);
}
if (!isValidIconResponse(bytes, contentType)) {
- bytes = null;
+ return null;
}
- return bytes;
+ return new Favicon(bytes, contentType);
}
- private byte[] getIconInPage(String url) {
+ private Favicon getIconInPage(String url) {
Document doc = null;
try {
HttpResult result = getter.getBinary(url, TIMEOUT);
doc = Jsoup.parse(new String(result.getContent()), url);
} catch (Exception e) {
- log.debug("Failed to retrieve page to find icon", e);
+ log.debug("Failed to retrieve page to find icon");
+ log.trace("Failed to retrieve page to find icon", e);
return null;
}
@@ -109,7 +117,8 @@ public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
- log.debug("Failed to retrieve icon found in page {}", href, e);
+ log.debug("Failed to retrieve icon found in page {}", href);
+ log.trace("Failed to retrieve icon found in page {}", href, e);
return null;
}
@@ -118,6 +127,6 @@ public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
return null;
}
- return bytes;
+ return new Favicon(bytes, contentType);
}
}
diff --git a/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java b/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
new file mode 100644
index 00000000..6b1fe956
--- /dev/null
+++ b/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
@@ -0,0 +1,79 @@
+package com.commafeed.backend.favicon;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URLEncodedUtils;
+
+import com.commafeed.backend.HttpGetter;
+import com.commafeed.backend.HttpGetter.HttpResult;
+import com.commafeed.backend.model.Feed;
+
+@Slf4j
+@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
+@Singleton
+public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
+
+ private final HttpGetter getter;
+
+ @Override
+ public Favicon fetch(Feed feed) {
+ String url = feed.getUrl();
+
+ if (!url.toLowerCase().contains("www.facebook.com")) {
+ return null;
+ }
+
+ String userName = extractUserName(url);
+ if (userName == null) {
+ return null;
+ }
+
+ String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
+
+ byte[] bytes = null;
+ String contentType = null;
+
+ try {
+ log.debug("Getting Facebook user's icon, {}", url);
+
+ HttpResult iconResult = getter.getBinary(iconUrl, TIMEOUT);
+ bytes = iconResult.getContent();
+ contentType = iconResult.getContentType();
+ } catch (Exception e) {
+ log.debug("Failed to retrieve Facebook icon", e);
+ }
+
+ if (!isValidIconResponse(bytes, contentType)) {
+ return null;
+ }
+ return new Favicon(bytes, contentType);
+ }
+
+ private String extractUserName(String url) {
+ URI uri = null;
+ try {
+ uri = new URI(url);
+ } catch (URISyntaxException e) {
+ log.debug("could not parse url", e);
+ return null;
+ }
+ List params = URLEncodedUtils.parse(uri, StandardCharsets.UTF_8.name());
+ for (NameValuePair param : params) {
+ if ("id".equals(param.getName())) {
+ return param.getValue();
+ }
+ }
+ return null;
+ }
+
+}
diff --git a/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java b/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
index 7998d03d..5c52f896 100644
--- a/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
+++ b/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
@@ -1,66 +1,91 @@
package com.commafeed.backend.favicon;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Optional;
+
import javax.inject.Inject;
import javax.inject.Singleton;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.select.Elements;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URLEncodedUtils;
+import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpRequestInitializer;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.youtube.YouTube;
+import com.google.api.services.youtube.model.Channel;
+import com.google.api.services.youtube.model.ChannelListResponse;
+import com.google.api.services.youtube.model.Thumbnail;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
@Slf4j
-@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
+@RequiredArgsConstructor(onConstructor = @__({ @Inject }) )
@Singleton
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
+ private final CommaFeedConfiguration config;
@Override
- public byte[] fetch(Feed feed) {
+ public Favicon fetch(Feed feed) {
String url = feed.getUrl();
- if (!url.toLowerCase().contains("://gdata.youtube.com/")) {
+ if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
return null;
}
- String userName = extractUserName(url);
- if (userName == null) {
+ String googleAuthKey = config.getApplicationSettings().getGoogleAuthKey();
+ if (googleAuthKey == null) {
+ log.debug("no google auth key configured");
return null;
}
- String profileUrl = "https://gdata.youtube.com/feeds/users/" + userName;
-
byte[] bytes = null;
String contentType = null;
-
try {
- log.debug("Getting YouTube user's icon, {}", url);
-
- // initial get to translate username to obscure user thumbnail URL
- HttpResult profileResult = getter.getBinary(profileUrl, TIMEOUT);
- Document doc = Jsoup.parse(new String(profileResult.getContent()), profileUrl);
-
- Elements thumbnails = doc.select("media|thumbnail");
- if (thumbnails.isEmpty()) {
+ List params = URLEncodedUtils.parse(url.substring(url.indexOf("?") + 1), StandardCharsets.UTF_8);
+ Optional userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
+ Optional channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id")).findFirst();
+ if (!userId.isPresent() && !channelId.isPresent()) {
return null;
}
- String thumbnailUrl = thumbnails.get(0).attr("abs:url");
+ YouTube youtube = new YouTube.Builder(new NetHttpTransport(), JacksonFactory.getDefaultInstance(),
+ new HttpRequestInitializer() {
+ @Override
+ public void initialize(HttpRequest request) throws IOException {
+ }
+ }).setApplicationName("CommaFeed").build();
- int thumbnailStart = thumbnailUrl.indexOf(" fromQueryString(String keywords) {
+ List list = new ArrayList<>();
+ if (keywords != null) {
+ for (String keyword : StringUtils.split(keywords)) {
+ boolean not = false;
+ if (keyword.startsWith("-") || keyword.startsWith("!")) {
+ not = true;
+ keyword = keyword.substring(1);
+ }
+ list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
+ }
+ }
+ return list;
+ }
+}
diff --git a/src/main/java/com/commafeed/backend/feed/FeedFetcher.java b/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
index c492f7f3..8ba02971 100644
--- a/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
+++ b/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
@@ -41,16 +41,16 @@ public class FeedFetcher {
byte[] content = result.getContent();
try {
- fetchedFeed = parser.parse(feedUrl, content);
+ fetchedFeed = parser.parse(result.getUrlAfterRedirect(), content);
} catch (FeedException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(StringUtils.newStringUtf8(result.getContent()), feedUrl);
- if (org.apache.commons.lang.StringUtils.isNotBlank(extractedUrl)) {
+ if (org.apache.commons.lang3.StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
result = getter.getBinary(extractedUrl, lastModified, eTag, timeout);
content = result.getContent();
- fetchedFeed = parser.parse(feedUrl, content);
+ fetchedFeed = parser.parse(result.getUrlAfterRedirect(), content);
} else {
throw e;
}
diff --git a/src/main/java/com/commafeed/backend/feed/FeedParser.java b/src/main/java/com/commafeed/backend/feed/FeedParser.java
index baf87413..4b483af4 100644
--- a/src/main/java/com/commafeed/backend/feed/FeedParser.java
+++ b/src/main/java/com/commafeed/backend/feed/FeedParser.java
@@ -1,9 +1,11 @@
package com.commafeed.backend.feed;
import java.io.StringReader;
+import java.nio.charset.Charset;
import java.text.DateFormat;
import java.util.Date;
import java.util.List;
+import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -11,8 +13,7 @@ import javax.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.SystemUtils;
+import org.apache.commons.lang3.StringUtils;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
@@ -20,10 +21,7 @@ import org.xml.sax.InputSource;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
import com.google.common.collect.Iterables;
-import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
@@ -43,20 +41,13 @@ public class FeedParser {
private static final Date START = new Date(86400000);
private static final Date END = new Date(1000l * Integer.MAX_VALUE - 86400000);
- private static final Function CONTENT_TO_STRING = new Function() {
- @Override
- public String apply(SyndContent content) {
- return content.getValue();
- }
- };
-
public FetchedFeed parse(String feedUrl, byte[] xml) throws FeedException {
FetchedFeed fetchedFeed = new FetchedFeed();
Feed feed = fetchedFeed.getFeed();
List entries = fetchedFeed.getEntries();
try {
- String encoding = FeedUtils.guessEncoding(xml);
+ Charset encoding = FeedUtils.guessEncoding(xml);
String xmlString = FeedUtils.trimInvalidXmlCharacters(new String(xml, encoding));
if (xmlString == null) {
throw new FeedException("Input string is null for url " + feedUrl);
@@ -86,7 +77,7 @@ public class FeedParser {
}
entry.setGuid(FeedUtils.truncate(guid, 2048));
entry.setUpdated(validateDate(getEntryUpdateDate(item), true));
- entry.setUrl(FeedUtils.truncate(FeedUtils.toAbsoluteUrl(item.getLink(), feed.getLink(), feed.getUrlAfterRedirect()), 2048));
+ entry.setUrl(FeedUtils.truncate(FeedUtils.toAbsoluteUrl(item.getLink(), feed.getLink(), feedUrl), 2048));
// if link is empty but guid is used as url
if (StringUtils.isBlank(entry.getUrl()) && StringUtils.startsWith(entry.getGuid(), "http")) {
@@ -95,6 +86,8 @@ public class FeedParser {
FeedEntryContent content = new FeedEntryContent();
content.setContent(getContent(item));
+ content.setCategories(FeedUtils.truncate(
+ item.getCategories().stream().map(c -> c.getName()).collect(Collectors.joining(", ")), 4096));
content.setTitle(getTitle(item));
content.setAuthor(StringUtils.trimToNull(item.getAuthor()));
SyndEnclosure enclosure = Iterables.getFirst(item.getEnclosures(), null);
@@ -173,7 +166,7 @@ public class FeedParser {
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
- content = StringUtils.join(Collections2.transform(item.getContents(), CONTENT_TO_STRING), SystemUtils.LINE_SEPARATOR);
+ content = item.getContents().stream().map(c -> c.getValue()).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
diff --git a/src/main/java/com/commafeed/backend/feed/FeedQueues.java b/src/main/java/com/commafeed/backend/feed/FeedQueues.java
index c3c90bc8..f7b3a471 100644
--- a/src/main/java/com/commafeed/backend/feed/FeedQueues.java
+++ b/src/main/java/com/commafeed/backend/feed/FeedQueues.java
@@ -1,40 +1,45 @@
package com.commafeed.backend.feed;
+import java.util.ArrayList;
import java.util.Date;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang.time.DateUtils;
+import org.apache.commons.lang3.time.DateUtils;
+import org.hibernate.SessionFactory;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
+import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.Feed;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Queues;
@Singleton
public class FeedQueues {
+ private SessionFactory sessionFactory;
private final FeedDAO feedDAO;
private final CommaFeedConfiguration config;
- private Queue addQueue = Queues.newConcurrentLinkedQueue();
- private Queue takeQueue = Queues.newConcurrentLinkedQueue();
- private Queue giveBackQueue = Queues.newConcurrentLinkedQueue();
+ private Queue addQueue = new ConcurrentLinkedQueue<>();
+ private Queue takeQueue = new ConcurrentLinkedQueue<>();
+ private Queue giveBackQueue = new ConcurrentLinkedQueue<>();
private Meter refill;
@Inject
- public FeedQueues(FeedDAO feedDAO, CommaFeedConfiguration config, MetricRegistry metrics) {
+ public FeedQueues(SessionFactory sessionFactory, FeedDAO feedDAO, CommaFeedConfiguration config, MetricRegistry metrics) {
+ this.sessionFactory = sessionFactory;
this.config = config;
this.feedDAO = feedDAO;
@@ -78,13 +83,7 @@ public class FeedQueues {
public void add(Feed feed, boolean urgent) {
int refreshInterval = config.getApplicationSettings().getRefreshIntervalMinutes();
if (feed.getLastUpdated() == null || feed.getLastUpdated().before(DateUtils.addMinutes(new Date(), -1 * refreshInterval))) {
- boolean alreadyQueued = false;
- for (FeedRefreshContext context : addQueue) {
- if (context.getFeed().getId().equals(feed.getId())) {
- alreadyQueued = true;
- break;
- }
- }
+ boolean alreadyQueued = addQueue.stream().anyMatch(c -> c.getFeed().getId().equals(feed.getId()));
if (!alreadyQueued) {
addQueue.add(new FeedRefreshContext(feed, urgent));
}
@@ -97,7 +96,7 @@ public class FeedQueues {
private void refill() {
refill.mark();
- List contexts = Lists.newArrayList();
+ List contexts = new ArrayList<>();
int batchSize = Math.min(100, 3 * config.getApplicationSettings().getBackgroundThreads());
// add feeds we got from the add() method
@@ -109,7 +108,7 @@ public class FeedQueues {
// add feeds that are up to refresh from the database
int count = batchSize - contexts.size();
if (count > 0) {
- List feeds = feedDAO.findNextUpdatable(count, getLastLoginThreshold());
+ List feeds = UnitOfWork.call(sessionFactory, () -> feedDAO.findNextUpdatable(count, getLastLoginThreshold()));
for (Feed feed : feeds) {
contexts.add(new FeedRefreshContext(feed, false));
}
@@ -117,7 +116,7 @@ public class FeedQueues {
// set the disabledDate as we use it in feedDAO to decide what to refresh next. We also use a map to remove
// duplicates.
- Map map = Maps.newLinkedHashMap();
+ Map map = new LinkedHashMap<>();
for (FeedRefreshContext context : contexts) {
Feed feed = context.getFeed();
feed.setDisabledUntil(DateUtils.addMinutes(new Date(), config.getApplicationSettings().getRefreshIntervalMinutes()));
@@ -135,11 +134,8 @@ public class FeedQueues {
}
// update all feeds in the database
- List feeds = Lists.newArrayList();
- for (FeedRefreshContext context : map.values()) {
- feeds.add(context.getFeed());
- }
- feedDAO.merge(feeds);
+ List feeds = map.values().stream().map(c -> c.getFeed()).collect(Collectors.toList());
+ UnitOfWork.run(sessionFactory, () -> feedDAO.saveOrUpdate(feeds));
}
/**
@@ -154,7 +150,7 @@ public class FeedQueues {
}
private Date getLastLoginThreshold() {
- if (config.getApplicationSettings().isHeavyLoad()) {
+ if (config.getApplicationSettings().getHeavyLoad()) {
return DateUtils.addDays(new Date(), -30);
} else {
return null;
diff --git a/src/main/java/com/commafeed/backend/feed/FeedRefreshContext.java b/src/main/java/com/commafeed/backend/feed/FeedRefreshContext.java
index f2769b64..8408301d 100644
--- a/src/main/java/com/commafeed/backend/feed/FeedRefreshContext.java
+++ b/src/main/java/com/commafeed/backend/feed/FeedRefreshContext.java
@@ -2,9 +2,14 @@ package com.commafeed.backend.feed;
import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
+
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
+@Getter
+@Setter
public class FeedRefreshContext {
private Feed feed;
private List entries;
@@ -14,29 +19,4 @@ public class FeedRefreshContext {
this.feed = feed;
this.urgent = isUrgent;
}
-
- public Feed getFeed() {
- return feed;
- }
-
- public void setFeed(Feed feed) {
- this.feed = feed;
- }
-
- public boolean isUrgent() {
- return urgent;
- }
-
- public void setUrgent(boolean urgent) {
- this.urgent = urgent;
- }
-
- public List getEntries() {
- return entries;
- }
-
- public void setEntries(List entries) {
- this.entries = entries;
- }
-
}
diff --git a/src/main/java/com/commafeed/backend/feed/FeedRefreshExecutor.java b/src/main/java/com/commafeed/backend/feed/FeedRefreshExecutor.java
index b1f7f597..fa12c0af 100644
--- a/src/main/java/com/commafeed/backend/feed/FeedRefreshExecutor.java
+++ b/src/main/java/com/commafeed/backend/feed/FeedRefreshExecutor.java
@@ -5,11 +5,11 @@ import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import lombok.extern.slf4j.Slf4j;
-
import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
+import lombok.extern.slf4j.Slf4j;
+
/**
* Wraps a {@link ThreadPoolExecutor} instance. Blocks when queue is full instead of rejecting the task. Allow priority queueing by using
* {@link Task} instead of {@link Runnable}
@@ -37,7 +37,14 @@ public class FeedRefreshExecutor {
return offerLast(r);
}
}
- });
+ }) {
+ @Override
+ protected void afterExecute(Runnable r, Throwable t) {
+ if (t != null) {
+ log.error("thread from pool {} threw a runtime exception", poolName, t);
+ }
+ }
+ };
pool.setRejectedExecutionHandler(new RejectedExecutionHandler() {
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
diff --git a/src/main/java/com/commafeed/backend/feed/FeedRefreshTaskGiver.java b/src/main/java/com/commafeed/backend/feed/FeedRefreshTaskGiver.java
index 96e264da..f6235886 100644
--- a/src/main/java/com/commafeed/backend/feed/FeedRefreshTaskGiver.java
+++ b/src/main/java/com/commafeed/backend/feed/FeedRefreshTaskGiver.java
@@ -10,13 +10,10 @@ import javax.inject.Singleton;
import lombok.extern.slf4j.Slf4j;
-import org.hibernate.SessionFactory;
-
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
-import com.commafeed.backend.dao.UnitOfWork;
/**
* Infinite loop fetching feeds from @FeedQueues and queuing them to the {@link FeedRefreshWorker} pool.
@@ -26,7 +23,6 @@ import com.commafeed.backend.dao.UnitOfWork;
@Singleton
public class FeedRefreshTaskGiver implements Managed {
- private final SessionFactory sessionFactory;
private final FeedQueues queues;
private final FeedRefreshWorker worker;
@@ -36,9 +32,8 @@ public class FeedRefreshTaskGiver implements Managed {
private Meter threadWaited;
@Inject
- public FeedRefreshTaskGiver(SessionFactory sessionFactory, FeedQueues queues, FeedDAO feedDAO, FeedRefreshWorker worker,
- CommaFeedConfiguration config, MetricRegistry metrics) {
- this.sessionFactory = sessionFactory;
+ public FeedRefreshTaskGiver(FeedQueues queues, FeedDAO feedDAO, FeedRefreshWorker worker, CommaFeedConfiguration config,
+ MetricRegistry metrics) {
this.queues = queues;
this.worker = worker;
@@ -68,12 +63,7 @@ public class FeedRefreshTaskGiver implements Managed {
public void run() {
while (!executor.isShutdown()) {
try {
- FeedRefreshContext context = new UnitOfWork(sessionFactory) {
- @Override
- protected FeedRefreshContext runInSession() throws Exception {
- return queues.take();
- }
- }.run();
+ FeedRefreshContext context = queues.take();
if (context != null) {
feedRefreshed.mark();
worker.updateFeed(context);
diff --git a/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java b/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
index e526a0a1..08765228 100644
--- a/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
+++ b/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
@@ -1,23 +1,21 @@
package com.commafeed.backend.feed;
-import io.dropwizard.lifecycle.Managed;
-
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
+import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
-import lombok.extern.slf4j.Slf4j;
-
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.time.DateUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateUtils;
import org.hibernate.SessionFactory;
import com.codahale.metrics.Meter;
@@ -35,9 +33,11 @@ import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedUpdateService;
import com.commafeed.backend.service.PubSubService;
-import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Striped;
+import io.dropwizard.lifecycle.Managed;
+import lombok.extern.slf4j.Slf4j;
+
@Slf4j
@Singleton
public class FeedRefreshUpdater implements Managed {
@@ -112,7 +112,7 @@ public class FeedRefreshUpdater implements Managed {
feed.setMessage("Feed has no entries");
} else {
List lastEntries = cache.getLastEntries(feed);
- List currentEntries = Lists.newArrayList();
+ List currentEntries = new ArrayList<>();
List subscriptions = null;
for (FeedEntry entry : entries) {
@@ -120,12 +120,7 @@ public class FeedRefreshUpdater implements Managed {
if (!lastEntries.contains(cacheKey)) {
log.debug("cache miss for {}", entry.getUrl());
if (subscriptions == null) {
- subscriptions = new UnitOfWork>(sessionFactory) {
- @Override
- protected List runInSession() throws Exception {
- return feedSubscriptionDAO.findByFeed(feed);
- }
- }.run();
+ subscriptions = UnitOfWork.call(sessionFactory, () -> feedSubscriptionDAO.findByFeed(feed));
}
ok &= addEntry(feed, entry, subscriptions);
entryCacheMiss.mark();
@@ -143,16 +138,13 @@ public class FeedRefreshUpdater implements Managed {
}
if (CollectionUtils.isNotEmpty(subscriptions)) {
- List users = Lists.newArrayList();
- for (FeedSubscription sub : subscriptions) {
- users.add(sub.getUser());
- }
+ List users = subscriptions.stream().map(s -> s.getUser()).collect(Collectors.toList());
cache.invalidateUnreadCount(subscriptions.toArray(new FeedSubscription[0]));
cache.invalidateUserRootCategory(users.toArray(new User[0]));
}
}
- if (config.getApplicationSettings().isPubsubhubbub()) {
+ if (config.getApplicationSettings().getPubsubhubbub()) {
handlePubSub(feed);
}
if (!ok) {
@@ -190,12 +182,7 @@ public class FeedRefreshUpdater implements Managed {
locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
- boolean inserted = new UnitOfWork(sessionFactory) {
- @Override
- protected Boolean runInSession() throws Exception {
- return feedUpdateService.addEntry(feed, entry);
- }
- }.run();
+ boolean inserted = UnitOfWork.call(sessionFactory, () -> feedUpdateService.addEntry(feed, entry, subscriptions));
if (inserted) {
entryInserted.mark();
}
diff --git a/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java b/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
index 4192d3f1..e044aa1a 100644
--- a/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
+++ b/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
@@ -4,6 +4,8 @@ import io.dropwizard.lifecycle.Managed;
import java.util.Date;
import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -11,8 +13,8 @@ import javax.inject.Singleton;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.time.DateUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateUtils;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
@@ -20,7 +22,6 @@ import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.feed.FeedRefreshExecutor.Task;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
-import com.google.common.base.Optional;
/**
* Calls {@link FeedFetcher} and handles its outcome
@@ -84,13 +85,18 @@ public class FeedRefreshWorker implements Managed {
int refreshInterval = config.getApplicationSettings().getRefreshIntervalMinutes();
Date disabledUntil = DateUtils.addMinutes(new Date(), refreshInterval);
try {
- String url = Optional.fromNullable(feed.getUrlAfterRedirect()).or(feed.getUrl());
+ String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FetchedFeed fetchedFeed = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
List entries = fetchedFeed.getEntries();
- if (config.getApplicationSettings().isHeavyLoad()) {
+ Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
+ if (maxFeedCapacity > 0) {
+ entries = entries.stream().limit(maxFeedCapacity).collect(Collectors.toList());
+ }
+
+ if (config.getApplicationSettings().getHeavyLoad()) {
disabledUntil = FeedUtils.buildDisabledUntil(fetchedFeed.getFeed().getLastEntryDate(), fetchedFeed.getFeed()
.getAverageEntryInterval(), disabledUntil);
}
@@ -118,7 +124,7 @@ public class FeedRefreshWorker implements Managed {
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
- if (config.getApplicationSettings().isHeavyLoad()) {
+ if (config.getApplicationSettings().getHeavyLoad()) {
disabledUntil = FeedUtils.buildDisabledUntil(feed.getLastEntryDate(), feed.getAverageEntryInterval(), disabledUntil);
}
feed.setErrorCount(0);
diff --git a/src/main/java/com/commafeed/backend/feed/FeedUtils.java b/src/main/java/com/commafeed/backend/feed/FeedUtils.java
index c722a2b9..f6513ecf 100644
--- a/src/main/java/com/commafeed/backend/feed/FeedUtils.java
+++ b/src/main/java/com/commafeed/backend/feed/FeedUtils.java
@@ -3,19 +3,20 @@ package com.commafeed.backend.feed;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.net.URL;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
-
-import lombok.extern.slf4j.Slf4j;
+import java.util.stream.Collectors;
import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.time.DateUtils;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
@@ -25,17 +26,19 @@ import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Whitelist;
import org.jsoup.select.Elements;
-import org.mozilla.universalchardet.UniversalDetector;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
+import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry;
-import com.google.common.collect.Lists;
+import com.ibm.icu.text.CharsetDetector;
+import com.ibm.icu.text.CharsetMatch;
import com.steadystate.css.parser.CSSOMParser;
import edu.uci.ics.crawler4j.url.URLCanonicalizer;
+import lombok.extern.slf4j.Slf4j;
/**
* Utility methods related to feed handling
@@ -97,14 +100,14 @@ public class FeedUtils {
* feed
*
*/
- public static String guessEncoding(byte[] bytes) {
+ public static Charset guessEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (StringUtils.startsWithIgnoreCase(extracted, "iso-8859-")) {
if (StringUtils.endsWith(extracted, "1") == false) {
- return extracted;
+ return Charset.forName(extracted);
}
} else if (StringUtils.startsWithIgnoreCase(extracted, "windows-")) {
- return extracted;
+ return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
@@ -112,27 +115,23 @@ public class FeedUtils {
/**
* Detect encoding by analyzing characters in the array
*/
- public static String detectEncoding(byte[] bytes) {
- String DEFAULT_ENCODING = "UTF-8";
- UniversalDetector detector = new UniversalDetector(null);
- detector.handleData(bytes, 0, bytes.length);
- detector.dataEnd();
- String encoding = detector.getDetectedCharset();
- detector.reset();
- if (encoding == null) {
- encoding = DEFAULT_ENCODING;
- } else if (encoding.equalsIgnoreCase("ISO-8859-1")) {
+ public static Charset detectEncoding(byte[] bytes) {
+ String encoding = "UTF-8";
+
+ CharsetDetector detector = new CharsetDetector();
+ detector.setText(bytes);
+ CharsetMatch match = detector.detect();
+ if (match != null) {
+ encoding = match.getName();
+ }
+ if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
- return encoding;
+ return Charset.forName(encoding);
}
public static String replaceHtmlEntitiesWithNumericEntities(String source) {
- String result = source;
- for (String entity : HtmlEntities.NUMERIC_MAPPING.keySet()) {
- result = StringUtils.replace(result, entity, HtmlEntities.NUMERIC_MAPPING.get(entity));
- }
- return result;
+ return StringUtils.replaceEach(source, HtmlEntities.HTML_ENTITIES, HtmlEntities.NUMERIC_ENTITIES);
}
/**
@@ -182,7 +181,7 @@ public class FeedUtils {
return null;
}
- String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1));
+ String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = StringUtils.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
@@ -227,7 +226,7 @@ public class FeedUtils {
String rule = "";
CSSOMParser parser = new CSSOMParser();
try {
- List rules = Lists.newArrayList();
+ List rules = new ArrayList<>();
CSSStyleDeclaration decl = parser.parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
@@ -252,7 +251,7 @@ public class FeedUtils {
String rule = "";
CSSOMParser parser = new CSSOMParser();
try {
- List rules = Lists.newArrayList();
+ List rules = new ArrayList<>();
CSSStyleDeclaration decl = parser.parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
@@ -386,13 +385,7 @@ public class FeedUtils {
}
public static List getSortedTimestamps(List entries) {
- List timestamps = Lists.newArrayList();
- for (FeedEntry entry : entries) {
- timestamps.add(entry.getUpdated().getTime());
- }
- Collections.sort(timestamps);
- Collections.reverse(timestamps);
- return timestamps;
+ return entries.stream().map(t -> t.getUpdated().getTime()).sorted(Collections.reverseOrder()).collect(Collectors.toList());
}
public static String removeTrailingSlash(String url) {
@@ -436,22 +429,15 @@ public class FeedUtils {
}
public static boolean isRelative(final String url) {
- // the regex means "doesn't start with 'scheme://'"
- if ((url != null) && (url.startsWith("/") == false) && (!url.matches("^\\w+\\:\\/\\/.*")) && !(url.startsWith("#"))) {
- return true;
- } else {
- return false;
- }
+ // the regex means "start with 'scheme://'"
+ return url.startsWith("/") || url.startsWith("#") || !url.matches("^\\w+\\:\\/\\/.*");
}
public static String getFaviconUrl(FeedSubscription subscription, String publicUrl) {
return removeTrailingSlash(publicUrl) + "/rest/feed/favicon/" + subscription.getId();
}
- public static String proxyImages(String content, String publicUrl, boolean proxyImages) {
- if (!proxyImages) {
- return content;
- }
+ public static String proxyImages(String content, String publicUrl) {
if (StringUtils.isBlank(content)) {
return content;
}
@@ -461,7 +447,7 @@ public class FeedUtils {
for (Element element : elements) {
String href = element.attr("src");
if (href != null) {
- String proxy = removeTrailingSlash(publicUrl) + "/rest/server/proxy?u=" + imageProxyEncoder(href);
+ String proxy = proxyImage(href, publicUrl);
element.attr("src", proxy);
}
}
@@ -469,6 +455,13 @@ public class FeedUtils {
return doc.body().html();
}
+ public static String proxyImage(String url, String publicUrl) {
+ if (StringUtils.isBlank(url)) {
+ return url;
+ }
+ return removeTrailingSlash(publicUrl) + "/rest/server/proxy?u=" + imageProxyEncoder(url);
+ }
+
public static String rot13(String msg) {
StringBuilder message = new StringBuilder();
@@ -495,19 +488,20 @@ public class FeedUtils {
return rot13(new String(Base64.decodeBase64(code)));
}
- public static void removeUnwantedFromSearch(List entries, String keywords) {
- if (StringUtils.isBlank(keywords)) {
- return;
- }
-
+ public static void removeUnwantedFromSearch(List entries, List keywords) {
Iterator it = entries.iterator();
while (it.hasNext()) {
Entry entry = it.next();
boolean keep = true;
- for (String keyword : keywords.split(" ")) {
+ for (FeedEntryKeyword keyword : keywords) {
String title = Jsoup.parse(entry.getTitle()).text();
String content = Jsoup.parse(entry.getContent()).text();
- if (!StringUtils.containsIgnoreCase(content, keyword) && !StringUtils.containsIgnoreCase(title, keyword)) {
+ boolean condition = !StringUtils.containsIgnoreCase(content, keyword.getKeyword())
+ && !StringUtils.containsIgnoreCase(title, keyword.getKeyword());
+ if (keyword.getMode() == Mode.EXCLUDE) {
+ condition = !condition;
+ }
+ if (condition) {
keep = false;
break;
}
diff --git a/src/main/java/com/commafeed/backend/feed/FetchedFeed.java b/src/main/java/com/commafeed/backend/feed/FetchedFeed.java
index e538b686..e19928e0 100644
--- a/src/main/java/com/commafeed/backend/feed/FetchedFeed.java
+++ b/src/main/java/com/commafeed/backend/feed/FetchedFeed.java
@@ -1,58 +1,23 @@
package com.commafeed.backend.feed;
+import java.util.ArrayList;
import java.util.List;
+import lombok.Getter;
+import lombok.Setter;
+
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
-import com.google.common.collect.Lists;
+@Getter
+@Setter
public class FetchedFeed {
private Feed feed = new Feed();
- private List entries = Lists.newArrayList();
+ private List entries = new ArrayList<>();
private String title;
private String urlAfterRedirect;
private long fetchDuration;
- public Feed getFeed() {
- return feed;
- }
-
- public void setFeed(Feed feed) {
- this.feed = feed;
- }
-
- public List getEntries() {
- return entries;
- }
-
- public void setEntries(List entries) {
- this.entries = entries;
- }
-
- public String getTitle() {
- return title;
- }
-
- public void setTitle(String title) {
- this.title = title;
- }
-
- public long getFetchDuration() {
- return fetchDuration;
- }
-
- public void setFetchDuration(long fetchDuration) {
- this.fetchDuration = fetchDuration;
- }
-
- public String getUrlAfterRedirect() {
- return urlAfterRedirect;
- }
-
- public void setUrlAfterRedirect(String urlAfterRedirect) {
- this.urlAfterRedirect = urlAfterRedirect;
- }
-
}
diff --git a/src/main/java/com/commafeed/backend/feed/HtmlEntities.java b/src/main/java/com/commafeed/backend/feed/HtmlEntities.java
index 77d592ca..3e927c67 100644
--- a/src/main/java/com/commafeed/backend/feed/HtmlEntities.java
+++ b/src/main/java/com/commafeed/backend/feed/HtmlEntities.java
@@ -1,15 +1,14 @@
package com.commafeed.backend.feed;
-import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.Map;
-import com.google.common.collect.Maps;
-
public class HtmlEntities {
- public static final Map NUMERIC_MAPPING = Collections.unmodifiableMap(loadMap());
+ public static final String[] HTML_ENTITIES;
+ public static final String[] NUMERIC_ENTITIES;
- private static synchronized Map loadMap() {
- Map map = Maps.newLinkedHashMap();
+ static {
+ Map map = new LinkedHashMap<>();
map.put("Á", "Á");
map.put("á", "á");
map.put("Â", "Â");
@@ -261,6 +260,7 @@ public class HtmlEntities {
map.put("", "");
map.put("", "");
- return map;
+ HTML_ENTITIES = map.keySet().toArray(new String[map.size()]);
+ NUMERIC_ENTITIES = map.values().toArray(new String[map.size()]);
}
}
diff --git a/src/main/java/com/commafeed/backend/model/Feed.java b/src/main/java/com/commafeed/backend/model/Feed.java
index 8c15a670..9ac4e736 100644
--- a/src/main/java/com/commafeed/backend/model/Feed.java
+++ b/src/main/java/com/commafeed/backend/model/Feed.java
@@ -1,12 +1,9 @@
package com.commafeed.backend.model;
import java.util.Date;
-import java.util.Set;
-import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Entity;
-import javax.persistence.OneToMany;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
@@ -103,12 +100,6 @@ public class Feed extends AbstractModel {
@Column(length = 40)
private String lastContentHash;
- @OneToMany(mappedBy = "feed", cascade = CascadeType.REMOVE)
- private Set entries;
-
- @OneToMany(mappedBy = "feed")
- private Set subscriptions;
-
/**
* detected hub for pubsubhubbub
*/
diff --git a/src/main/java/com/commafeed/backend/model/FeedEntryContent.java b/src/main/java/com/commafeed/backend/model/FeedEntryContent.java
index a1398b7d..7b036d20 100644
--- a/src/main/java/com/commafeed/backend/model/FeedEntryContent.java
+++ b/src/main/java/com/commafeed/backend/model/FeedEntryContent.java
@@ -11,6 +11,8 @@ import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
+import org.hibernate.annotations.Type;
+
@Entity
@Table(name = "FEEDENTRYCONTENTS")
@SuppressWarnings("serial")
@@ -26,6 +28,7 @@ public class FeedEntryContent extends AbstractModel {
@Lob
@Column(length = Integer.MAX_VALUE)
+ @Type(type = "org.hibernate.type.StringClobType")
private String content;
@Column(length = 40)
@@ -40,6 +43,9 @@ public class FeedEntryContent extends AbstractModel {
@Column(length = 255)
private String enclosureType;
+ @Column(length = 4096)
+ private String categories;
+
@OneToMany(mappedBy = "content")
private Set entries;
diff --git a/src/main/java/com/commafeed/backend/model/FeedEntryStatus.java b/src/main/java/com/commafeed/backend/model/FeedEntryStatus.java
index 74cdb1c1..a891d4a1 100644
--- a/src/main/java/com/commafeed/backend/model/FeedEntryStatus.java
+++ b/src/main/java/com/commafeed/backend/model/FeedEntryStatus.java
@@ -1,5 +1,6 @@
package com.commafeed.backend.model;
+import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -16,8 +17,6 @@ import javax.persistence.Transient;
import lombok.Getter;
import lombok.Setter;
-import com.google.common.collect.Lists;
-
@Entity
@Table(name = "FEEDENTRYSTATUSES")
@SuppressWarnings("serial")
@@ -41,7 +40,7 @@ public class FeedEntryStatus extends AbstractModel {
private boolean markable;
@Transient
- private List tags = Lists.newArrayList();
+ private List tags = new ArrayList<>();
/**
* Denormalization starts here
diff --git a/src/main/java/com/commafeed/backend/model/FeedSubscription.java b/src/main/java/com/commafeed/backend/model/FeedSubscription.java
index 363665eb..92a08cec 100644
--- a/src/main/java/com/commafeed/backend/model/FeedSubscription.java
+++ b/src/main/java/com/commafeed/backend/model/FeedSubscription.java
@@ -40,4 +40,7 @@ public class FeedSubscription extends AbstractModel {
private Integer position;
+ @Column(length = 4096)
+ private String filter;
+
}
diff --git a/src/main/java/com/commafeed/backend/model/User.java b/src/main/java/com/commafeed/backend/model/User.java
index ddd3fab5..93963a3d 100644
--- a/src/main/java/com/commafeed/backend/model/User.java
+++ b/src/main/java/com/commafeed/backend/model/User.java
@@ -1,13 +1,9 @@
package com.commafeed.backend.model;
import java.util.Date;
-import java.util.Set;
-import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Entity;
-import javax.persistence.FetchType;
-import javax.persistence.OneToMany;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
@@ -15,11 +11,7 @@ import javax.persistence.TemporalType;
import lombok.Getter;
import lombok.Setter;
-import org.apache.commons.lang.time.DateUtils;
-import org.hibernate.annotations.Cascade;
-
-import com.commafeed.backend.model.UserRole.Role;
-import com.google.common.collect.Sets;
+import org.apache.commons.lang3.time.DateUtils;
@Entity
@Table(name = "USERS")
@@ -58,27 +50,10 @@ public class User extends AbstractModel {
@Temporal(TemporalType.TIMESTAMP)
private Date recoverPasswordTokenDate;
- @OneToMany(mappedBy = "user", cascade = { CascadeType.PERSIST, CascadeType.REMOVE })
- @Cascade({ org.hibernate.annotations.CascadeType.PERSIST, org.hibernate.annotations.CascadeType.SAVE_UPDATE,
- org.hibernate.annotations.CascadeType.REMOVE })
- private Set roles = Sets.newHashSet();
-
- @OneToMany(mappedBy = "user", fetch = FetchType.LAZY, cascade = CascadeType.REMOVE)
- private Set subscriptions;
-
@Column(name = "last_full_refresh")
@Temporal(TemporalType.TIMESTAMP)
private Date lastFullRefresh;
- public boolean hasRole(Role role) {
- for (UserRole userRole : getRoles()) {
- if (userRole.getRole() == role) {
- return true;
- }
- }
- return false;
- }
-
public boolean shouldRefreshFeedsAt(Date when) {
return (lastFullRefresh == null || lastFullRefreshMoreThan30MinutesBefore(when));
}
diff --git a/src/main/java/com/commafeed/backend/model/UserSettings.java b/src/main/java/com/commafeed/backend/model/UserSettings.java
index 70ee4418..4928ac2b 100644
--- a/src/main/java/com/commafeed/backend/model/UserSettings.java
+++ b/src/main/java/com/commafeed/backend/model/UserSettings.java
@@ -13,6 +13,8 @@ import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
+import org.hibernate.annotations.Type;
+
@Entity
@Table(name = "USERSETTINGS")
@SuppressWarnings("serial")
@@ -59,6 +61,7 @@ public class UserSettings extends AbstractModel {
@Lob
@Column(length = Integer.MAX_VALUE)
+ @Type(type = "org.hibernate.type.StringClobType")
private String customCss;
@Column(name = "scroll_speed")
diff --git a/src/main/java/com/commafeed/backend/opml/OPMLExporter.java b/src/main/java/com/commafeed/backend/opml/OPMLExporter.java
index 49e7011f..8dc733e5 100644
--- a/src/main/java/com/commafeed/backend/opml/OPMLExporter.java
+++ b/src/main/java/com/commafeed/backend/opml/OPMLExporter.java
@@ -1,7 +1,9 @@
package com.commafeed.backend.opml;
+import java.util.Collections;
import java.util.Date;
import java.util.List;
+import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -13,6 +15,7 @@ import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
+import com.google.common.base.MoreObjects;
import com.rometools.opml.feed.opml.Attribute;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
@@ -31,39 +34,40 @@ public class OPMLExporter {
opml.setCreated(new Date());
List categories = feedCategoryDAO.findAll(user);
+ Collections.sort(categories,
+ (e1, e2) -> MoreObjects.firstNonNull(e1.getPosition(), 0) - MoreObjects.firstNonNull(e2.getPosition(), 0));
+
List subscriptions = feedSubscriptionDAO.findAll(user);
+ Collections.sort(subscriptions,
+ (e1, e2) -> MoreObjects.firstNonNull(e1.getPosition(), 0) - MoreObjects.firstNonNull(e2.getPosition(), 0));
// export root categories
- for (FeedCategory cat : categories) {
- if (cat.getParent() == null) {
- opml.getOutlines().add(buildCategoryOutline(cat, subscriptions));
- }
+ for (FeedCategory cat : categories.stream().filter(c -> c.getParent() == null).collect(Collectors.toList())) {
+ opml.getOutlines().add(buildCategoryOutline(cat, categories, subscriptions));
}
// export root subscriptions
- for (FeedSubscription sub : subscriptions) {
- if (sub.getCategory() == null) {
- opml.getOutlines().add(buildSubscriptionOutline(sub));
- }
+ for (FeedSubscription sub : subscriptions.stream().filter(s -> s.getCategory() == null).collect(Collectors.toList())) {
+ opml.getOutlines().add(buildSubscriptionOutline(sub));
}
return opml;
}
- private Outline buildCategoryOutline(FeedCategory cat, List subscriptions) {
+ private Outline buildCategoryOutline(FeedCategory cat, List categories, List subscriptions) {
Outline outline = new Outline();
outline.setText(cat.getName());
outline.setTitle(cat.getName());
- for (FeedCategory child : cat.getChildren()) {
- outline.getChildren().add(buildCategoryOutline(child, subscriptions));
+ for (FeedCategory child : categories.stream().filter(c -> c.getParent() != null && c.getParent().getId().equals(cat.getId()))
+ .collect(Collectors.toList())) {
+ outline.getChildren().add(buildCategoryOutline(child, categories, subscriptions));
}
- for (FeedSubscription sub : subscriptions) {
- if (sub.getCategory() != null && sub.getCategory().getId().equals(cat.getId())) {
- outline.getChildren().add(buildSubscriptionOutline(sub));
- }
+ for (FeedSubscription sub : subscriptions.stream()
+ .filter(s -> s.getCategory() != null && s.getCategory().getId().equals(cat.getId())).collect(Collectors.toList())) {
+ outline.getChildren().add(buildSubscriptionOutline(sub));
}
return outline;
}
diff --git a/src/main/java/com/commafeed/backend/opml/OPMLImporter.java b/src/main/java/com/commafeed/backend/opml/OPMLImporter.java
index 6ff2ddff..9e910f3f 100644
--- a/src/main/java/com/commafeed/backend/opml/OPMLImporter.java
+++ b/src/main/java/com/commafeed/backend/opml/OPMLImporter.java
@@ -10,7 +10,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedCategoryDAO;
@@ -38,8 +38,8 @@ public class OPMLImporter {
try {
Opml feed = (Opml) input.build(new StringReader(xml));
List outlines = feed.getOutlines();
- for (Outline outline : outlines) {
- handleOutline(user, outline, null);
+ for (int i = 0; i < outlines.size(); i++) {
+ handleOutline(user, outlines.get(i), null, i);
}
} catch (Exception e) {
log.error(e.getMessage(), e);
@@ -47,7 +47,7 @@ public class OPMLImporter {
}
- private void handleOutline(User user, Outline outline, FeedCategory parent) {
+ private void handleOutline(User user, Outline outline, FeedCategory parent, int position) {
List children = outline.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
String name = FeedUtils.truncate(outline.getText(), 128);
@@ -64,11 +64,12 @@ public class OPMLImporter {
category.setName(name);
category.setParent(parent);
category.setUser(user);
+ category.setPosition(position);
feedCategoryDAO.saveOrUpdate(category);
}
- for (Outline child : children) {
- handleOutline(user, child, category);
+ for (int i = 0; i < children.size(); i++) {
+ handleOutline(user, children.get(i), category, i);
}
} else {
String name = FeedUtils.truncate(outline.getText(), 128);
@@ -80,7 +81,7 @@ public class OPMLImporter {
}
// make sure we continue with the import process even if a feed failed
try {
- feedSubscriptionService.subscribe(user, outline.getXmlUrl(), name, parent);
+ feedSubscriptionService.subscribe(user, outline.getXmlUrl(), name, parent, position);
} catch (FeedSubscriptionException e) {
throw e;
} catch (Exception e) {
diff --git a/src/main/java/com/commafeed/backend/rome/OPML11Parser.java b/src/main/java/com/commafeed/backend/rome/OPML11Parser.java
index 995baa1b..8a1eceab 100644
--- a/src/main/java/com/commafeed/backend/rome/OPML11Parser.java
+++ b/src/main/java/com/commafeed/backend/rome/OPML11Parser.java
@@ -1,9 +1,13 @@
package com.commafeed.backend.rome;
+import java.util.Locale;
+
import org.jdom2.Document;
import org.jdom2.Element;
import com.rometools.opml.io.impl.OPML10Parser;
+import com.rometools.rome.feed.WireFeed;
+import com.rometools.rome.io.FeedException;
/**
* Support for OPML 1.1 parsing
@@ -19,12 +23,17 @@ public class OPML11Parser extends OPML10Parser {
public boolean isMyType(Document document) {
Element e = document.getRootElement();
- if (e.getName().equals("opml") && (e.getChild("head") == null || e.getChild("head").getChild("docs") == null)
- && (e.getAttributeValue("version") == null || e.getAttributeValue("version").equals("1.1"))) {
+ if (e.getName().equals("opml")) {
return true;
}
return false;
- };
+ }
+
+ @Override
+ public WireFeed parse(Document document, boolean validate, Locale locale) throws IllegalArgumentException, FeedException {
+ document.getRootElement().getChildren().add(new Element("head"));
+ return super.parse(document, validate, locale);
+ }
}
diff --git a/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java b/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java
index 30418bbe..c79fbef0 100644
--- a/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java
+++ b/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java
@@ -1,32 +1,30 @@
package com.commafeed.backend.service;
-import java.util.Calendar;
import java.util.Date;
import java.util.List;
-import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Singleton;
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
import org.hibernate.SessionFactory;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.dao.FeedEntryDAO;
+import com.commafeed.backend.dao.FeedEntryDAO.FeedCapacity;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.Feed;
-import com.commafeed.backend.model.FeedEntryStatus;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
/**
* Contains utility methods for cleaning the database
*
*/
@Slf4j
-@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
+@RequiredArgsConstructor(onConstructor = @__({ @Inject }) )
@Singleton
public class DatabaseCleaningService {
@@ -42,14 +40,18 @@ public class DatabaseCleaningService {
log.info("cleaning feeds without subscriptions");
long total = 0;
int deleted = 0;
+ long entriesTotal = 0;
do {
- deleted = new UnitOfWork(sessionFactory) {
- @Override
- protected Integer runInSession() throws Exception {
- List feeds = feedDAO.findWithoutSubscriptions(1);
- return feedDAO.delete(feeds);
- };
- }.run();
+ List feeds = UnitOfWork.call(sessionFactory, () -> feedDAO.findWithoutSubscriptions(1));
+ for (Feed feed : feeds) {
+ int entriesDeleted = 0;
+ do {
+ entriesDeleted = UnitOfWork.call(sessionFactory, () -> feedEntryDAO.delete(feed.getId(), BATCH_SIZE));
+ entriesTotal += entriesDeleted;
+ log.info("removed {} entries for feeds without subscriptions", entriesTotal);
+ } while (entriesDeleted > 0);
+ }
+ deleted = UnitOfWork.call(sessionFactory, () -> feedDAO.delete(feeds));
total += deleted;
log.info("removed {} feeds without subscriptions", total);
} while (deleted != 0);
@@ -62,12 +64,7 @@ public class DatabaseCleaningService {
long total = 0;
int deleted = 0;
do {
- deleted = new UnitOfWork(sessionFactory) {
- @Override
- protected Integer runInSession() throws Exception {
- return feedEntryContentDAO.deleteWithoutEntries(BATCH_SIZE);
- }
- }.run();
+ deleted = UnitOfWork.call(sessionFactory, () -> feedEntryContentDAO.deleteWithoutEntries(BATCH_SIZE));
total += deleted;
log.info("removed {} contents without entries", total);
} while (deleted != 0);
@@ -75,23 +72,28 @@ public class DatabaseCleaningService {
return total;
}
- public long cleanEntriesOlderThan(long value, TimeUnit unit) {
- final Calendar cal = Calendar.getInstance();
- cal.add(Calendar.MINUTE, -1 * (int) unit.toMinutes(value));
-
+ public long cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
long total = 0;
- int deleted = 0;
- do {
- deleted = new UnitOfWork(sessionFactory) {
- @Override
- protected Integer runInSession() throws Exception {
- return feedEntryDAO.delete(cal.getTime(), BATCH_SIZE);
- }
- }.run();
- total += deleted;
- log.info("removed {} entries", total);
- } while (deleted != 0);
- log.info("cleanup done: {} entries deleted", total);
+ while (true) {
+ List feeds = UnitOfWork.call(sessionFactory,
+ () -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, BATCH_SIZE));
+ if (feeds.isEmpty()) {
+ break;
+ }
+
+ for (final FeedCapacity feed : feeds) {
+ long remaining = feed.getCapacity() - maxFeedCapacity;
+ do {
+ final long rem = remaining;
+ int deleted = UnitOfWork.call(sessionFactory,
+ () -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(BATCH_SIZE, rem)));
+ total += deleted;
+ remaining -= deleted;
+ log.info("removed {} entries for feeds exceeding capacity", total);
+ } while (remaining > 0);
+ }
+ }
+ log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
return total;
}
@@ -100,15 +102,10 @@ public class DatabaseCleaningService {
long total = 0;
int deleted = 0;
do {
- deleted = new UnitOfWork(sessionFactory) {
- @Override
- protected Integer runInSession() throws Exception {
- List list = feedEntryStatusDAO.getOldStatuses(olderThan, BATCH_SIZE);
- return feedEntryStatusDAO.delete(list);
- }
- }.run();
+ deleted = UnitOfWork.call(sessionFactory,
+ () -> feedEntryStatusDAO.delete(feedEntryStatusDAO.getOldStatuses(olderThan, BATCH_SIZE)));
total += deleted;
- log.info("cleaned {} old read statuses", total);
+ log.info("removed {} old read statuses", total);
} while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
return total;
diff --git a/src/main/java/com/commafeed/backend/service/FeedEntryContentService.java b/src/main/java/com/commafeed/backend/service/FeedEntryContentService.java
index 2a4e1011..d7bc7fdf 100644
--- a/src/main/java/com/commafeed/backend/service/FeedEntryContentService.java
+++ b/src/main/java/com/commafeed/backend/service/FeedEntryContentService.java
@@ -6,7 +6,7 @@ import javax.inject.Singleton;
import lombok.RequiredArgsConstructor;
import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils;
diff --git a/src/main/java/com/commafeed/backend/service/FeedEntryFilteringService.java b/src/main/java/com/commafeed/backend/service/FeedEntryFilteringService.java
new file mode 100644
index 00000000..c576ff45
--- /dev/null
+++ b/src/main/java/com/commafeed/backend/service/FeedEntryFilteringService.java
@@ -0,0 +1,119 @@
+package com.commafeed.backend.service;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import lombok.RequiredArgsConstructor;
+
+import org.apache.commons.jexl2.JexlContext;
+import org.apache.commons.jexl2.JexlEngine;
+import org.apache.commons.jexl2.JexlException;
+import org.apache.commons.jexl2.JexlInfo;
+import org.apache.commons.jexl2.MapContext;
+import org.apache.commons.jexl2.Script;
+import org.apache.commons.jexl2.introspection.JexlMethod;
+import org.apache.commons.jexl2.introspection.JexlPropertyGet;
+import org.apache.commons.jexl2.introspection.Uberspect;
+import org.apache.commons.jexl2.introspection.UberspectImpl;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.logging.LogFactory;
+import org.jsoup.Jsoup;
+
+import com.commafeed.backend.model.FeedEntry;
+
+@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
+@Singleton
+public class FeedEntryFilteringService {
+
+ private static final JexlEngine ENGINE = initEngine();
+
+ private static JexlEngine initEngine() {
+ // classloader that prevents object creation
+ ClassLoader cl = new ClassLoader() {
+ @Override
+ protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException {
+ return null;
+ }
+ };
+
+ // uberspect that prevents access to .class and .getClass()
+ Uberspect uberspect = new UberspectImpl(LogFactory.getLog(JexlEngine.class)) {
+ @Override
+ public JexlPropertyGet getPropertyGet(Object obj, Object identifier, JexlInfo info) {
+ if ("class".equals(identifier)) {
+ return null;
+ }
+ return super.getPropertyGet(obj, identifier, info);
+ }
+
+ @Override
+ public JexlMethod getMethod(Object obj, String method, Object[] args, JexlInfo info) {
+ if ("getClass".equals(method)) {
+ return null;
+ }
+ return super.getMethod(obj, method, args, info);
+ }
+ };
+
+ JexlEngine engine = new JexlEngine(uberspect, null, null, null);
+ engine.setStrict(true);
+ engine.setClassLoader(cl);
+ return engine;
+ }
+
+ private ExecutorService executor = Executors.newCachedThreadPool();
+
+ public boolean filterMatchesEntry(String filter, FeedEntry entry) throws FeedEntryFilterException {
+ if (StringUtils.isBlank(filter)) {
+ return true;
+ }
+
+ Script script = null;
+ try {
+ script = ENGINE.createScript(filter);
+ } catch (JexlException e) {
+ throw new FeedEntryFilterException("Exception while parsing expression " + filter, e);
+ }
+
+ JexlContext context = new MapContext();
+ context.set("title", entry.getContent().getTitle() == null ? "" : Jsoup.parse(entry.getContent().getTitle()).text().toLowerCase());
+ context.set("author", entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor().toLowerCase());
+ context.set("content", entry.getContent().getContent() == null ? "" : Jsoup.parse(entry.getContent().getContent()).text()
+ .toLowerCase());
+ context.set("url", entry.getUrl() == null ? "" : entry.getUrl().toLowerCase());
+ context.set("categories", entry.getContent().getCategories() == null ? "" : entry.getContent().getCategories().toLowerCase());
+
+ Callable callable = script.callable(context);
+ Future future = executor.submit(callable);
+ Object result = null;
+ try {
+ result = future.get(500, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e);
+ } catch (ExecutionException e) {
+ throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e);
+ } catch (TimeoutException e) {
+ throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
+ }
+ try {
+ return (boolean) result;
+ } catch (ClassCastException e) {
+ throw new FeedEntryFilterException(e.getMessage(), e);
+ }
+ }
+
+ @SuppressWarnings("serial")
+ public static class FeedEntryFilterException extends Exception {
+ public FeedEntryFilterException(String message, Throwable t) {
+ super(message, t);
+ }
+ }
+}
diff --git a/src/main/java/com/commafeed/backend/service/FeedEntryService.java b/src/main/java/com/commafeed/backend/service/FeedEntryService.java
index a3c5fbfc..96402d9d 100644
--- a/src/main/java/com/commafeed/backend/service/FeedEntryService.java
+++ b/src/main/java/com/commafeed/backend/service/FeedEntryService.java
@@ -1,5 +1,6 @@
package com.commafeed.backend.service;
+import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -12,11 +13,11 @@ import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
+import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
-import com.google.common.collect.Lists;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
@@ -65,9 +66,9 @@ public class FeedEntryService {
feedEntryStatusDAO.saveOrUpdate(status);
}
- public void markSubscriptionEntries(User user, List subscriptions, Date olderThan, String keywords) {
- List statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null, false,
- false, null);
+ public void markSubscriptionEntries(User user, List subscriptions, Date olderThan, List keywords) {
+ List statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null,
+ false, false, null);
markList(statuses, olderThan);
cache.invalidateUnreadCount(subscriptions.toArray(new FeedSubscription[0]));
cache.invalidateUserRootCategory(user);
@@ -79,7 +80,7 @@ public class FeedEntryService {
}
private void markList(List statuses, Date olderThan) {
- List list = Lists.newArrayList();
+ List list = new ArrayList<>();
for (FeedEntryStatus status : statuses) {
if (!status.isRead()) {
Date inserted = status.getEntry().getInserted();
diff --git a/src/main/java/com/commafeed/backend/service/FeedEntryTagService.java b/src/main/java/com/commafeed/backend/service/FeedEntryTagService.java
index f640c045..446d027b 100644
--- a/src/main/java/com/commafeed/backend/service/FeedEntryTagService.java
+++ b/src/main/java/com/commafeed/backend/service/FeedEntryTagService.java
@@ -1,7 +1,8 @@
package com.commafeed.backend.service;
import java.util.List;
-import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -13,9 +14,6 @@ import com.commafeed.backend.dao.FeedEntryTagDAO;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.User;
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
@@ -30,29 +28,12 @@ public class FeedEntryTagService {
return;
}
- List tags = feedEntryTagDAO.findByEntry(user, entry);
- Map tagMap = Maps.uniqueIndex(tags, new Function() {
- @Override
- public String apply(FeedEntryTag input) {
- return input.getName();
- }
- });
+ List existingTags = feedEntryTagDAO.findByEntry(user, entry);
+ Set existingTagNames = existingTags.stream().map(t -> t.getName()).collect(Collectors.toSet());
- List addList = Lists.newArrayList();
- List removeList = Lists.newArrayList();
-
- for (String tagName : tagNames) {
- FeedEntryTag tag = tagMap.get(tagName);
- if (tag == null) {
- addList.add(new FeedEntryTag(user, entry, tagName));
- }
- }
-
- for (FeedEntryTag tag : tags) {
- if (!tagNames.contains(tag.getName())) {
- removeList.add(tag);
- }
- }
+ List addList = tagNames.stream().filter(name -> !existingTagNames.contains(name))
+ .map(name -> new FeedEntryTag(user, entry, name)).collect(Collectors.toList());
+ List removeList = existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).collect(Collectors.toList());
feedEntryTagDAO.saveOrUpdate(addList);
feedEntryTagDAO.delete(removeList);
diff --git a/src/main/java/com/commafeed/backend/service/FeedService.java b/src/main/java/com/commafeed/backend/service/FeedService.java
index 59aaba30..2ee312af 100644
--- a/src/main/java/com/commafeed/backend/service/FeedService.java
+++ b/src/main/java/com/commafeed/backend/service/FeedService.java
@@ -12,6 +12,7 @@ import org.apache.commons.io.IOUtils;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
+import com.commafeed.backend.favicon.AbstractFaviconFetcher.Favicon;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
@@ -21,7 +22,7 @@ public class FeedService {
private final FeedDAO feedDAO;
private final Set faviconFetchers;
- private byte[] defaultFavicon;
+ private Favicon defaultFavicon;
@Inject
public FeedService(FeedDAO feedDAO, Set faviconFetchers) {
@@ -29,7 +30,7 @@ public class FeedService {
this.faviconFetchers = faviconFetchers;
try {
- defaultFavicon = IOUtils.toByteArray(getClass().getResource("/images/default_favicon.gif"));
+ defaultFavicon = new Favicon(IOUtils.toByteArray(getClass().getResource("/images/default_favicon.gif")), "image/gif");
} catch (IOException e) {
throw new RuntimeException("could not load default favicon", e);
}
@@ -49,9 +50,9 @@ public class FeedService {
return feed;
}
- public byte[] fetchFavicon(Feed feed) {
+ public Favicon fetchFavicon(Feed feed) {
- byte[] icon = null;
+ Favicon icon = null;
for (AbstractFaviconFetcher faviconFetcher : faviconFetchers) {
icon = faviconFetcher.fetch(feed);
if (icon != null) {
diff --git a/src/main/java/com/commafeed/backend/service/FeedSubscriptionService.java b/src/main/java/com/commafeed/backend/service/FeedSubscriptionService.java
index dbe6b542..f14c1328 100644
--- a/src/main/java/com/commafeed/backend/service/FeedSubscriptionService.java
+++ b/src/main/java/com/commafeed/backend/service/FeedSubscriptionService.java
@@ -2,6 +2,7 @@ package com.commafeed.backend.service;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -9,7 +10,7 @@ import javax.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.cache.CacheService;
@@ -23,7 +24,6 @@ import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.UnreadCount;
-import com.google.common.collect.Maps;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@@ -44,7 +44,15 @@ public class FeedSubscriptionService {
private final CacheService cache;
private final CommaFeedConfiguration config;
- public Feed subscribe(User user, String url, String title, FeedCategory category) {
+ public Feed subscribe(User user, String url, String title) {
+ return subscribe(user, url, title, null, 0);
+ }
+
+ public Feed subscribe(User user, String url, String title, FeedCategory parent) {
+ return subscribe(user, url, title, parent, 0);
+ }
+
+ public Feed subscribe(User user, String url, String title, FeedCategory category, int position) {
final String pubUrl = config.getApplicationSettings().getPublicUrl();
if (StringUtils.isBlank(pubUrl)) {
@@ -63,7 +71,7 @@ public class FeedSubscriptionService {
sub.setUser(user);
}
sub.setCategory(category);
- sub.setPosition(0);
+ sub.setPosition(position);
sub.setTitle(FeedUtils.truncate(title, 128));
feedSubscriptionDAO.saveOrUpdate(sub);
@@ -92,12 +100,7 @@ public class FeedSubscriptionService {
}
public Map getUnreadCount(User user) {
- Map