Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-03-21 21:37:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

split client and server into maven modules

Browse Source
This commit is contained in:
Athou
2022-08-13 10:34:59 +02:00
parent 4c4868a2b6
commit ac7b6eeb21
277 changed files with 645 additions and 521 deletions
Download Patch File Download Diff File Expand all files Collapse all files

182
commafeed-server/src/main/java/com/commafeed/CommaFeedApplication.java Normal file
View File

@@ -0,0 +1,182 @@
package com.commafeed;
import java.io.IOException;
import java.util.Date;
import java.util.EnumSet;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.servlet.DispatcherType;
import javax.servlet.FilterChain;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
import org.hibernate.cfg.AvailableSettings;
import com.codahale.metrics.json.MetricsModule;
import com.commafeed.backend.feed.FeedRefreshTaskGiver;
import com.commafeed.backend.feed.FeedRefreshUpdater;
import com.commafeed.backend.feed.FeedRefreshWorker;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserSettings;
import com.commafeed.backend.service.StartupService;
import com.commafeed.backend.service.UserService;
import com.commafeed.backend.task.ScheduledTask;
import com.commafeed.frontend.auth.SecurityCheckFactoryProvider;
import com.commafeed.frontend.resource.AdminREST;
import com.commafeed.frontend.resource.CategoryREST;
import com.commafeed.frontend.resource.EntryREST;
import com.commafeed.frontend.resource.FeedREST;
import com.commafeed.frontend.resource.PubSubHubbubCallbackREST;
import com.commafeed.frontend.resource.ServerREST;
import com.commafeed.frontend.resource.UserREST;
import com.commafeed.frontend.servlet.AnalyticsServlet;
import com.commafeed.frontend.servlet.CustomCssServlet;
import com.commafeed.frontend.servlet.LogoutServlet;
import com.commafeed.frontend.servlet.NextUnreadServlet;
import com.commafeed.frontend.session.SessionHelperFactoryProvider;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
import io.dropwizard.Application;
import io.dropwizard.assets.AssetsBundle;
import io.dropwizard.db.DataSourceFactory;
import io.dropwizard.forms.MultiPartBundle;
import io.dropwizard.hibernate.HibernateBundle;
import io.dropwizard.server.DefaultServerFactory;
import io.dropwizard.servlets.CacheBustingFilter;
import io.dropwizard.setup.Bootstrap;
import io.dropwizard.setup.Environment;
import io.dropwizard.web.WebBundle;
import io.dropwizard.web.conf.WebConfiguration;
public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
public static final String USERNAME_ADMIN = "admin";
public static final String USERNAME_DEMO = "demo";
public static final Date STARTUP_TIME = new Date();
private HibernateBundle<CommaFeedConfiguration> hibernateBundle;
@Override
public String getName() {
return "CommaFeed";
}
@Override
public void initialize(Bootstrap<CommaFeedConfiguration> bootstrap) {
bootstrap.getObjectMapper().registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
bootstrap.addBundle(hibernateBundle = new HibernateBundle<CommaFeedConfiguration>(AbstractModel.class, Feed.class,
FeedCategory.class, FeedEntry.class, FeedEntryContent.class, FeedEntryStatus.class, FeedEntryTag.class,
FeedSubscription.class, User.class, UserRole.class, UserSettings.class) {
@Override
public DataSourceFactory getDataSourceFactory(CommaFeedConfiguration configuration) {
DataSourceFactory factory = configuration.getDataSourceFactory();
// keep using old id generator for backward compatibility
factory.getProperties().put(AvailableSettings.USE_NEW_ID_GENERATOR_MAPPINGS, "false");
factory.getProperties().put(AvailableSettings.STATEMENT_BATCH_SIZE, "50");
factory.getProperties().put(AvailableSettings.BATCH_VERSIONED_DATA, "true");
return factory;
}
});
bootstrap.addBundle(new WebBundle<CommaFeedConfiguration>() {
@Override
public WebConfiguration getWebConfiguration(CommaFeedConfiguration configuration) {
WebConfiguration config = new WebConfiguration();
config.getFrameOptionsHeaderFactory().setEnabled(true);
return config;
}
});
bootstrap.addBundle(new AssetsBundle("/assets/", "/", "index.html"));
bootstrap.addBundle(new MultiPartBundle());
}
@Override
public void run(CommaFeedConfiguration config, Environment environment) throws Exception {
// guice init
Injector injector = Guice.createInjector(new CommaFeedModule(hibernateBundle.getSessionFactory(), config, environment.metrics()));
// session management
environment.servlets().setSessionHandler(config.getSessionHandlerFactory().build());
// support for "@SecurityCheck User user" injection
environment.jersey().register(new SecurityCheckFactoryProvider.Binder(injector.getInstance(UserService.class)));
// support for "@Context SessionHelper sessionHelper" injection
environment.jersey().register(new SessionHelperFactoryProvider.Binder());
// REST resources
environment.jersey().setUrlPattern("/rest/*");
((DefaultServerFactory) config.getServerFactory()).setJerseyRootPath("/rest/*");
environment.jersey().register(injector.getInstance(AdminREST.class));
environment.jersey().register(injector.getInstance(CategoryREST.class));
environment.jersey().register(injector.getInstance(EntryREST.class));
environment.jersey().register(injector.getInstance(FeedREST.class));
environment.jersey().register(injector.getInstance(PubSubHubbubCallbackREST.class));
environment.jersey().register(injector.getInstance(ServerREST.class));
environment.jersey().register(injector.getInstance(UserREST.class));
// Servlets
environment.servlets().addServlet("next", injector.getInstance(NextUnreadServlet.class)).addMapping("/next");
environment.servlets().addServlet("logout", injector.getInstance(LogoutServlet.class)).addMapping("/logout");
environment.servlets().addServlet("customCss", injector.getInstance(CustomCssServlet.class)).addMapping("/custom_css.css");
environment.servlets().addServlet("analytics.js", injector.getInstance(AnalyticsServlet.class)).addMapping("/analytics.js");
// Scheduled tasks
Set<ScheduledTask> tasks = injector.getInstance(Key.get(new TypeLiteral<Set<ScheduledTask>>() {
}));
ScheduledExecutorService executor = environment.lifecycle()
.scheduledExecutorService("task-scheduler", true)
.threads(tasks.size())
.build();
for (ScheduledTask task : tasks) {
task.register(executor);
}
// database init/changelogs
environment.lifecycle().manage(injector.getInstance(StartupService.class));
// background feed fetching
environment.lifecycle().manage(injector.getInstance(FeedRefreshTaskGiver.class));
environment.lifecycle().manage(injector.getInstance(FeedRefreshWorker.class));
environment.lifecycle().manage(injector.getInstance(FeedRefreshUpdater.class));
// cache configuration
// prevent caching on REST resources, except for favicons
environment.servlets().addFilter("cache-filter", new CacheBustingFilter() {
@Override
public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
String path = ((HttpServletRequest) request).getRequestURI();
if (path.contains("/feed/favicon")) {
chain.doFilter(request, response);
} else {
super.doFilter(request, response, chain);
}
}
}).addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/rest/*");
}
public static void main(String[] args) throws Exception {
new CommaFeedApplication().run(args);
}
}

152
commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java Normal file
View File

@@ -0,0 +1,152 @@
package com.commafeed;
import java.util.Date;
import java.util.ResourceBundle;
import javax.validation.Valid;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotBlank;
import javax.validation.constraints.NotNull;
import org.apache.commons.lang3.time.DateUtils;
import com.commafeed.backend.cache.RedisPoolFactory;
import com.commafeed.frontend.session.SessionHandlerFactory;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.dropwizard.Configuration;
import io.dropwizard.db.DataSourceFactory;
import lombok.Getter;
@Getter
public class CommaFeedConfiguration extends Configuration {
public enum CacheType {
NOOP, REDIS
}
@Valid
@NotNull
@JsonProperty("database")
private final DataSourceFactory dataSourceFactory = new DataSourceFactory();
@Valid
@NotNull
@JsonProperty("redis")
private final RedisPoolFactory redisPoolFactory = new RedisPoolFactory();
@Valid
@NotNull
@JsonProperty("session")
private final SessionHandlerFactory sessionHandlerFactory = new SessionHandlerFactory();
@Valid
@NotNull
@JsonProperty("app")
private ApplicationSettings applicationSettings;
private final ResourceBundle bundle;
public CommaFeedConfiguration() {
bundle = ResourceBundle.getBundle("application");
}
public String getVersion() {
return bundle.getString("version");
}
public String getGitCommit() {
return bundle.getString("git.commit");
}
@Getter
public static class ApplicationSettings {
@NotNull
@NotBlank
@Valid
private String publicUrl;
@NotNull
@Valid
private Boolean allowRegistrations;
@NotNull
@Valid
private Boolean createDemoAccount;
private String googleAnalyticsTrackingCode;
private String googleAuthKey;
@NotNull
@Min(1)
@Valid
private Integer backgroundThreads;
@NotNull
@Min(1)
@Valid
private Integer databaseUpdateThreads;
private String smtpHost;
private int smtpPort;
private boolean smtpTls;
private String smtpUserName;
private String smtpPassword;
private String smtpFromAddress;
private boolean graphiteEnabled;
private String graphitePrefix;
private String graphiteHost;
private int graphitePort;
private int graphiteInterval;
@NotNull
@Valid
private Boolean heavyLoad;
@NotNull
@Valid
private Boolean pubsubhubbub;
@NotNull
@Valid
private Boolean imageProxyEnabled;
@NotNull
@Min(0)
@Valid
private Integer queryTimeout;
@NotNull
@Min(0)
@Valid
private Integer keepStatusDays;
@NotNull
@Min(0)
@Valid
private Integer maxFeedCapacity;
@NotNull
@Min(0)
@Valid
private Integer refreshIntervalMinutes;
@NotNull
@Valid
private CacheType cache;
@Valid
private String announcement;
private String userAgent;
public Date getUnreadThreshold() {
int keepStatusDays = getKeepStatusDays();
return keepStatusDays > 0 ? DateUtils.addDays(new Date(), -1 * keepStatusDays) : null;
}
}
}

97
commafeed-server/src/main/java/com/commafeed/CommaFeedModule.java Normal file
View File

@@ -0,0 +1,97 @@
package com.commafeed;
import java.net.InetSocketAddress;
import java.util.concurrent.TimeUnit;
import org.hibernate.SessionFactory;
import com.codahale.metrics.MetricFilter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.graphite.Graphite;
import com.codahale.metrics.graphite.GraphiteReporter;
import com.commafeed.CommaFeedConfiguration.ApplicationSettings;
import com.commafeed.CommaFeedConfiguration.CacheType;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.cache.NoopCacheService;
import com.commafeed.backend.cache.RedisCacheService;
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
import com.commafeed.backend.favicon.DefaultFaviconFetcher;
import com.commafeed.backend.favicon.FacebookFaviconFetcher;
import com.commafeed.backend.favicon.YoutubeFaviconFetcher;
import com.commafeed.backend.task.DemoAccountCleanupTask;
import com.commafeed.backend.task.OldEntriesCleanupTask;
import com.commafeed.backend.task.OldStatusesCleanupTask;
import com.commafeed.backend.task.OrphanedContentsCleanupTask;
import com.commafeed.backend.task.OrphanedFeedsCleanupTask;
import com.commafeed.backend.task.ScheduledTask;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import com.commafeed.backend.urlprovider.InPageReferenceFeedURLProvider;
import com.commafeed.backend.urlprovider.YoutubeFeedURLProvider;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.multibindings.Multibinder;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
public class CommaFeedModule extends AbstractModule {
@Getter(onMethod = @__({ @Provides }))
private final SessionFactory sessionFactory;
@Getter(onMethod = @__({ @Provides }))
private final CommaFeedConfiguration config;
@Getter(onMethod = @__({ @Provides }))
private final MetricRegistry metrics;
@Override
protected void configure() {
CacheService cacheService = config.getApplicationSettings().getCache() == CacheType.NOOP ? new NoopCacheService()
: new RedisCacheService(config.getRedisPoolFactory().build());
log.info("using cache {}", cacheService.getClass());
bind(CacheService.class).toInstance(cacheService);
Multibinder<AbstractFaviconFetcher> faviconMultibinder = Multibinder.newSetBinder(binder(), AbstractFaviconFetcher.class);
faviconMultibinder.addBinding().to(YoutubeFaviconFetcher.class);
faviconMultibinder.addBinding().to(FacebookFaviconFetcher.class);
faviconMultibinder.addBinding().to(DefaultFaviconFetcher.class);
Multibinder<FeedURLProvider> urlProviderMultibinder = Multibinder.newSetBinder(binder(), FeedURLProvider.class);
urlProviderMultibinder.addBinding().to(InPageReferenceFeedURLProvider.class);
urlProviderMultibinder.addBinding().to(YoutubeFeedURLProvider.class);
Multibinder<ScheduledTask> taskMultibinder = Multibinder.newSetBinder(binder(), ScheduledTask.class);
taskMultibinder.addBinding().to(OldStatusesCleanupTask.class);
taskMultibinder.addBinding().to(OldEntriesCleanupTask.class);
taskMultibinder.addBinding().to(OrphanedFeedsCleanupTask.class);
taskMultibinder.addBinding().to(OrphanedContentsCleanupTask.class);
taskMultibinder.addBinding().to(DemoAccountCleanupTask.class);
ApplicationSettings settings = config.getApplicationSettings();
if (settings.isGraphiteEnabled()) {
final String graphitePrefix = settings.getGraphitePrefix();
final String graphiteHost = settings.getGraphiteHost();
final int graphitePort = settings.getGraphitePort();
final int graphiteInterval = settings.getGraphiteInterval();
log.info("Graphite Metrics will be sent to host={}, port={}, prefix={}, interval={}sec", graphiteHost, graphitePort,
graphitePrefix, graphiteInterval);
final Graphite graphite = new Graphite(new InetSocketAddress(graphiteHost, graphitePort));
final GraphiteReporter reporter = GraphiteReporter.forRegistry(metrics)
.prefixedWith(graphitePrefix)
.convertRatesTo(TimeUnit.SECONDS)
.convertDurationsTo(TimeUnit.MILLISECONDS)
.filter(MetricFilter.ALL)
.build(graphite);
reporter.start(graphiteInterval, TimeUnit.SECONDS);
} else {
log.info("Graphite Metrics Disabled. Metrics will not be sent.");
}
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/ContentEncodingInterceptor.java Normal file
View File

@@ -0,0 +1,60 @@
package com.commafeed.backend;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Locale;
import java.util.Set;
import org.apache.http.Header;
import org.apache.http.HeaderElement;
import org.apache.http.HttpEntity;
import org.apache.http.HttpException;
import org.apache.http.HttpResponse;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.entity.HttpEntityWrapper;
import org.apache.http.protocol.HttpContext;
class ContentEncodingInterceptor implements HttpResponseInterceptor {
private static final Set<String> ALLOWED_CONTENT_ENCODINGS = new HashSet<>(Arrays.asList("gzip", "x-gzip", "deflate", "identity"));
@Override
public void process(HttpResponse response, HttpContext context) throws HttpException, IOException {
if (hasContent(response)) {
Header contentEncodingHeader = response.getEntity().getContentEncoding();
if (contentEncodingHeader != null && containsUnsupportedEncodings(contentEncodingHeader)) {
overrideContentEncoding(response);
}
}
}
private boolean containsUnsupportedEncodings(Header contentEncodingHeader) {
HeaderElement[] codecs = contentEncodingHeader.getElements();
for (final HeaderElement codec : codecs) {
String codecName = codec.getName().toLowerCase(Locale.US);
if (!ALLOWED_CONTENT_ENCODINGS.contains(codecName)) {
return true;
}
}
return false;
}
private void overrideContentEncoding(HttpResponse response) {
HttpEntity wrapped = new HttpEntityWrapper(response.getEntity()) {
@Override
public Header getContentEncoding() {
return null;
}
};
response.setEntity(wrapped);
}
private boolean hasContent(HttpResponse response) {
return response.getEntity() != null && response.getEntity().getContentLength() != 0;
}
}

49
commafeed-server/src/main/java/com/commafeed/backend/FixedSizeSortedSet.java Normal file
View File

@@ -0,0 +1,49 @@
package com.commafeed.backend;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
/**
* List wrapper that sorts its elements in the order provided by given comparator and ensure a maximum capacity.
*
*
*/
public class FixedSizeSortedSet<E> {
private List<E> inner;
private final Comparator<? super E> comparator;
private final int capacity;
public FixedSizeSortedSet(int capacity, Comparator<? super E> comparator) {
this.inner = new ArrayList<E>(Math.max(0, capacity));
this.capacity = capacity < 0 ? Integer.MAX_VALUE : capacity;
this.comparator = comparator;
}
public void add(E e) {
int position = Math.abs(Collections.binarySearch(inner, e, comparator) + 1);
if (isFull()) {
if (position < inner.size()) {
inner.remove(inner.size() - 1);
inner.add(position, e);
}
} else {
inner.add(position, e);
}
}
public E last() {
return inner.get(inner.size() - 1);
}
public boolean isFull() {
return inner.size() == capacity;
}
public List<E> asList() {
return inner;
}
}

205
commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java Normal file
View File

@@ -0,0 +1,205 @@
package com.commafeed.backend;
import java.io.IOException;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.Consts;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
import org.apache.http.HttpResponseInterceptor;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpResponseException;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.ConnectionConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import com.commafeed.CommaFeedConfiguration;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import nl.altindag.ssl.SSLFactory;
/**
* Smart HTTP getter: handles gzip, ssl, last modified and etag headers
*
*/
@Singleton
public class HttpGetter {
private static final String ACCEPT_LANGUAGE = "en";
private static final String PRAGMA_NO_CACHE = "No-cache";
private static final String CACHE_CONTROL_NO_CACHE = "no-cache";
private static final HttpResponseInterceptor REMOVE_INCORRECT_CONTENT_ENCODING = new ContentEncodingInterceptor();
private static final SSLFactory SSL_FACTORY = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
private String userAgent;
@Inject
public HttpGetter(CommaFeedConfiguration config) {
this.userAgent = config.getApplicationSettings().getUserAgent();
if (this.userAgent == null) {
this.userAgent = String.format("CommaFeed/%s (https://github.com/Athou/commafeed)", config.getVersion());
}
}
public HttpResult getBinary(String url, int timeout) throws ClientProtocolException, IOException, NotModifiedException {
return getBinary(url, null, null, timeout);
}
/**
*
* @param url
* the url to retrive
* @param lastModified
* header we got last time we queried that url, or null
* @param eTag
* header we got last time we queried that url, or null
* @return
* @throws ClientProtocolException
* @throws IOException
* @throws NotModifiedException
* if the url hasn't changed since we asked for it last time
*/
public HttpResult getBinary(String url, String lastModified, String eTag, int timeout)
throws ClientProtocolException, IOException, NotModifiedException {
HttpResult result = null;
long start = System.currentTimeMillis();
CloseableHttpClient client = newClient(timeout);
CloseableHttpResponse response = null;
try {
HttpGet httpget = new HttpGet(url);
HttpClientContext context = HttpClientContext.create();
httpget.addHeader(HttpHeaders.ACCEPT_LANGUAGE, ACCEPT_LANGUAGE);
httpget.addHeader(HttpHeaders.PRAGMA, PRAGMA_NO_CACHE);
httpget.addHeader(HttpHeaders.CACHE_CONTROL, CACHE_CONTROL_NO_CACHE);
httpget.addHeader(HttpHeaders.USER_AGENT, userAgent);
if (lastModified != null) {
httpget.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
}
if (eTag != null) {
httpget.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
}
try {
response = client.execute(httpget, context);
int code = response.getStatusLine().getStatusCode();
if (code == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received");
} else if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);
}
} catch (HttpResponseException e) {
if (e.getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received");
} else {
throw e;
}
}
Header lastModifiedHeader = response.getFirstHeader(HttpHeaders.LAST_MODIFIED);
String lastModifiedHeaderValue = lastModifiedHeader == null ? null : StringUtils.trimToNull(lastModifiedHeader.getValue());
if (lastModifiedHeaderValue != null && StringUtils.equals(lastModified, lastModifiedHeaderValue)) {
throw new NotModifiedException("lastModifiedHeader is the same");
}
Header eTagHeader = response.getFirstHeader(HttpHeaders.ETAG);
String eTagHeaderValue = eTagHeader == null ? null : StringUtils.trimToNull(eTagHeader.getValue());
if (eTag != null && StringUtils.equals(eTag, eTagHeaderValue)) {
throw new NotModifiedException("eTagHeader is the same");
}
HttpEntity entity = response.getEntity();
byte[] content = null;
String contentType = null;
if (entity != null) {
content = EntityUtils.toByteArray(entity);
if (entity.getContentType() != null) {
contentType = entity.getContentType().getValue();
}
}
String urlAfterRedirect = url;
if (context.getRequest() instanceof HttpUriRequest) {
HttpUriRequest req = (HttpUriRequest) context.getRequest();
HttpHost host = context.getTargetHost();
urlAfterRedirect = req.getURI().isAbsolute() ? req.getURI().toString() : host.toURI() + req.getURI();
}
long duration = System.currentTimeMillis() - start;
result = new HttpResult(content, contentType, lastModifiedHeaderValue, eTagHeaderValue, duration, urlAfterRedirect);
} finally {
IOUtils.closeQuietly(response);
IOUtils.closeQuietly(client);
}
return result;
}
public static CloseableHttpClient newClient(int timeout) {
HttpClientBuilder builder = HttpClients.custom();
builder.useSystemProperties();
builder.addInterceptorFirst(REMOVE_INCORRECT_CONTENT_ENCODING);
builder.disableAutomaticRetries();
builder.setSSLContext(SSL_FACTORY.getSslContext());
builder.setSSLHostnameVerifier(SSL_FACTORY.getHostnameVerifier());
RequestConfig.Builder configBuilder = RequestConfig.custom();
configBuilder.setCookieSpec(CookieSpecs.IGNORE_COOKIES);
configBuilder.setSocketTimeout(timeout);
configBuilder.setConnectTimeout(timeout);
configBuilder.setConnectionRequestTimeout(timeout);
builder.setDefaultRequestConfig(configBuilder.build());
builder.setDefaultConnectionConfig(ConnectionConfig.custom().setCharset(Consts.ISO_8859_1).build());
return builder.build();
}
public static void main(String[] args) throws Exception {
CommaFeedConfiguration config = new CommaFeedConfiguration();
HttpGetter getter = new HttpGetter(config);
HttpResult result = getter.getBinary("https://sourceforge.net/projects/mpv-player-windows/rss", 30000);
System.out.println(new String(result.content));
}
public static class NotModifiedException extends Exception {
private static final long serialVersionUID = 1L;
public NotModifiedException(String message) {
super(message);
}
}
@Getter
@RequiredArgsConstructor
public static class HttpResult {
private final byte[] content;
private final String contentType;
private final String lastModifiedSince;
private final String eTag;
private final long duration;
private final String urlAfterRedirect;
}
}

39
commafeed-server/src/main/java/com/commafeed/backend/cache/CacheService.java vendored Normal file
View File

@@ -0,0 +1,39 @@
package com.commafeed.backend.cache;
import java.util.List;
import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.Category;
import com.commafeed.frontend.model.UnreadCount;
public abstract class CacheService {
// feed entries for faster refresh
public abstract List<String> getLastEntries(Feed feed);
public abstract void setLastEntries(Feed feed, List<String> entries);
public String buildUniqueEntryKey(Feed feed, FeedEntry entry) {
return DigestUtils.sha1Hex(entry.getGuid() + entry.getUrl());
}
// user categories
public abstract Category getUserRootCategory(User user);
public abstract void setUserRootCategory(User user, Category category);
public abstract void invalidateUserRootCategory(User... users);
// unread count
public abstract UnreadCount getUnreadCount(FeedSubscription sub);
public abstract void setUnreadCount(FeedSubscription sub, UnreadCount count);
public abstract void invalidateUnreadCount(FeedSubscription... subs);
}

53
commafeed-server/src/main/java/com/commafeed/backend/cache/NoopCacheService.java vendored Normal file
View File

@@ -0,0 +1,53 @@
package com.commafeed.backend.cache;
import java.util.Collections;
import java.util.List;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.Category;
import com.commafeed.frontend.model.UnreadCount;
public class NoopCacheService extends CacheService {
@Override
public List<String> getLastEntries(Feed feed) {
return Collections.emptyList();
}
@Override
public void setLastEntries(Feed feed, List<String> entries) {
}
@Override
public UnreadCount getUnreadCount(FeedSubscription sub) {
return null;
}
@Override
public void setUnreadCount(FeedSubscription sub, UnreadCount count) {
}
@Override
public void invalidateUnreadCount(FeedSubscription... subs) {
}
@Override
public Category getUserRootCategory(User user) {
return null;
}
@Override
public void setUserRootCategory(User user, Category category) {
}
@Override
public void invalidateUserRootCategory(User... users) {
}
}

159
commafeed-server/src/main/java/com/commafeed/backend/cache/RedisCacheService.java vendored Normal file
View File

@@ -0,0 +1,159 @@
package com.commafeed.backend.cache;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.Category;
import com.commafeed.frontend.model.UnreadCount;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.Pipeline;
@Slf4j
@RequiredArgsConstructor
public class RedisCacheService extends CacheService {
private static final ObjectMapper MAPPER = new ObjectMapper();
private final JedisPool pool;
@Override
public List<String> getLastEntries(Feed feed) {
List<String> list = new ArrayList<>();
try (Jedis jedis = pool.getResource()) {
String key = buildRedisEntryKey(feed);
Set<String> members = jedis.smembers(key);
for (String member : members) {
list.add(member);
}
}
return list;
}
@Override
public void setLastEntries(Feed feed, List<String> entries) {
try (Jedis jedis = pool.getResource()) {
String key = buildRedisEntryKey(feed);
Pipeline pipe = jedis.pipelined();
pipe.del(key);
for (String entry : entries) {
pipe.sadd(key, entry);
}
pipe.expire(key, (int) TimeUnit.DAYS.toSeconds(7));
pipe.sync();
}
}
@Override
public Category getUserRootCategory(User user) {
Category cat = null;
try (Jedis jedis = pool.getResource()) {
String key = buildRedisUserRootCategoryKey(user);
String json = jedis.get(key);
if (json != null) {
cat = MAPPER.readValue(json, Category.class);
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return cat;
}
@Override
public void setUserRootCategory(User user, Category category) {
try (Jedis jedis = pool.getResource()) {
String key = buildRedisUserRootCategoryKey(user);
Pipeline pipe = jedis.pipelined();
pipe.del(key);
pipe.set(key, MAPPER.writeValueAsString(category));
pipe.expire(key, (int) TimeUnit.MINUTES.toSeconds(30));
pipe.sync();
} catch (JsonProcessingException e) {
log.error(e.getMessage(), e);
}
}
@Override
public UnreadCount getUnreadCount(FeedSubscription sub) {
UnreadCount count = null;
try (Jedis jedis = pool.getResource()) {
String key = buildRedisUnreadCountKey(sub);
String json = jedis.get(key);
if (json != null) {
count = MAPPER.readValue(json, UnreadCount.class);
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return count;
}
@Override
public void setUnreadCount(FeedSubscription sub, UnreadCount count) {
try (Jedis jedis = pool.getResource()) {
String key = buildRedisUnreadCountKey(sub);
Pipeline pipe = jedis.pipelined();
pipe.del(key);
pipe.set(key, MAPPER.writeValueAsString(count));
pipe.expire(key, (int) TimeUnit.MINUTES.toSeconds(30));
pipe.sync();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
@Override
public void invalidateUserRootCategory(User... users) {
try (Jedis jedis = pool.getResource()) {
Pipeline pipe = jedis.pipelined();
if (users != null) {
for (User user : users) {
String key = buildRedisUserRootCategoryKey(user);
pipe.del(key);
}
}
pipe.sync();
}
}
@Override
public void invalidateUnreadCount(FeedSubscription... subs) {
try (Jedis jedis = pool.getResource()) {
Pipeline pipe = jedis.pipelined();
if (subs != null) {
for (FeedSubscription sub : subs) {
String key = buildRedisUnreadCountKey(sub);
pipe.del(key);
}
}
pipe.sync();
}
}
private String buildRedisEntryKey(Feed feed) {
return "f:" + Models.getId(feed);
}
private String buildRedisUserRootCategoryKey(User user) {
return "c:" + Models.getId(user);
}
private String buildRedisUnreadCountKey(FeedSubscription sub) {
return "u:" + Models.getId(sub);
}
}

27
commafeed-server/src/main/java/com/commafeed/backend/cache/RedisPoolFactory.java vendored Normal file
View File

@@ -0,0 +1,27 @@
package com.commafeed.backend.cache;
import org.apache.commons.lang3.StringUtils;
import lombok.Getter;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.Protocol;
@Getter
public class RedisPoolFactory {
private final String host = "localhost";
private final int port = Protocol.DEFAULT_PORT;
private String password;
private final int timeout = Protocol.DEFAULT_TIMEOUT;
private final int database = Protocol.DEFAULT_DATABASE;
private final int maxTotal = 500;
public JedisPool build() {
JedisPoolConfig config = new JedisPoolConfig();
config.setMaxTotal(maxTotal);
return new JedisPool(config, host, port, timeout, StringUtils.trimToNull(password), database);
}
}

75
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java Normal file
View File

@@ -0,0 +1,75 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.QFeedCategory;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
import com.querydsl.core.types.Predicate;
@Singleton
public class FeedCategoryDAO extends GenericDAO<FeedCategory> {
private QFeedCategory category = QFeedCategory.feedCategory;
@Inject
public FeedCategoryDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public List<FeedCategory> findAll(User user) {
return query().selectFrom(category).where(category.user.eq(user)).join(category.user, QUser.user).fetchJoin().fetch();
}
public FeedCategory findById(User user, Long id) {
return query().selectFrom(category).where(category.user.eq(user), category.id.eq(id)).fetchOne();
}
public FeedCategory findByName(User user, String name, FeedCategory parent) {
Predicate parentPredicate = null;
if (parent == null) {
parentPredicate = category.parent.isNull();
} else {
parentPredicate = category.parent.eq(parent);
}
return query().selectFrom(category).where(category.user.eq(user), category.name.eq(name), parentPredicate).fetchOne();
}
public List<FeedCategory> findByParent(User user, FeedCategory parent) {
Predicate parentPredicate = null;
if (parent == null) {
parentPredicate = category.parent.isNull();
} else {
parentPredicate = category.parent.eq(parent);
}
return query().selectFrom(category).where(category.user.eq(user), parentPredicate).fetch();
}
public List<FeedCategory> findAllChildrenCategories(User user, FeedCategory parent) {
return findAll(user).stream().filter(c -> isChild(c, parent)).collect(Collectors.toList());
}
private boolean isChild(FeedCategory child, FeedCategory parent) {
if (parent == null) {
return true;
}
boolean isChild = false;
while (child != null) {
if (Objects.equals(child.getId(), parent.getId())) {
isChild = true;
break;
}
child = child.getParent();
}
return isChild;
}
}

62
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedDAO.java Normal file
View File

@@ -0,0 +1,62 @@
package com.commafeed.backend.dao;
import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.QFeed;
import com.commafeed.backend.model.QFeedSubscription;
import com.commafeed.backend.model.QUser;
import com.google.common.collect.Iterables;
import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.JPQLQuery;
@Singleton
public class FeedDAO extends GenericDAO<Feed> {
private final QFeed feed = QFeed.feed;
@Inject
public FeedDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public List<Feed> findNextUpdatable(int count, Date lastLoginThreshold) {
JPQLQuery<Feed> query = query().selectFrom(feed);
query.where(feed.disabledUntil.isNull().or(feed.disabledUntil.lt(new Date())));
if (lastLoginThreshold != null) {
QFeedSubscription subs = QFeedSubscription.feedSubscription;
QUser user = QUser.user;
query.join(feed.subscriptions, subs).join(subs.user, user).where(user.lastLogin.gt(lastLoginThreshold));
}
return query.orderBy(feed.disabledUntil.asc()).limit(count).fetch();
}
public Feed findByUrl(String normalizedUrl) {
List<Feed> feeds = query().selectFrom(feed).where(feed.normalizedUrlHash.eq(DigestUtils.sha1Hex(normalizedUrl))).fetch();
Feed feed = Iterables.getFirst(feeds, null);
if (feed != null && StringUtils.equals(normalizedUrl, feed.getNormalizedUrl())) {
return feed;
}
return null;
}
public List<Feed> findByTopic(String topic) {
return query().selectFrom(feed).where(feed.pushTopicHash.eq(DigestUtils.sha1Hex(topic))).fetch();
}
public List<Feed> findWithoutSubscriptions(int max) {
QFeedSubscription sub = QFeedSubscription.feedSubscription;
return query().selectFrom(feed).where(JPAExpressions.selectOne().from(sub).where(sub.feed.eq(feed)).notExists()).limit(max).fetch();
}
}

40
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java Normal file
View File

@@ -0,0 +1,40 @@
package com.commafeed.backend.dao;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.JPQLQuery;
@Singleton
public class FeedEntryContentDAO extends GenericDAO<FeedEntryContent> {
private final QFeedEntryContent content = QFeedEntryContent.feedEntryContent;
private final QFeedEntry entry = QFeedEntry.feedEntry;
@Inject
public FeedEntryContentDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public List<FeedEntryContent> findExisting(String contentHash, String titleHash) {
return query().select(content).from(content).where(content.contentHash.eq(contentHash), content.titleHash.eq(titleHash)).fetch();
}
public int deleteWithoutEntries(int max) {
JPQLQuery<Integer> subQuery = JPAExpressions.selectOne().from(entry).where(entry.content.id.eq(content.id));
List<FeedEntryContent> list = query().selectFrom(content).where(subQuery.notExists()).limit(max).fetch();
int deleted = list.size();
delete(list);
return deleted;
}
}

67
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java Normal file
View File

@@ -0,0 +1,67 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.QFeedEntry;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.NumberExpression;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Singleton
public class FeedEntryDAO extends GenericDAO<FeedEntry> {
private QFeedEntry entry = QFeedEntry.feedEntry;
@Inject
public FeedEntryDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public Long findExisting(String guid, Feed feed) {
return query().select(entry.id)
.from(entry)
.where(entry.guidHash.eq(DigestUtils.sha1Hex(guid)), entry.feed.eq(feed))
.limit(1)
.fetchOne();
}
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max) {
NumberExpression<Long> count = entry.id.count();
List<Tuple> tuples = query().select(entry.feed.id, count)
.from(entry)
.groupBy(entry.feed)
.having(count.gt(maxCapacity))
.limit(max)
.fetch();
return tuples.stream().map(t -> new FeedCapacity(t.get(entry.feed.id), t.get(count))).collect(Collectors.toList());
}
public int delete(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).limit(max).fetch();
return delete(list);
}
public int deleteOldEntries(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(entry).where(entry.feed.id.eq(feedId)).orderBy(entry.updated.asc()).limit(max).fetch();
return delete(list);
}
@AllArgsConstructor
@Getter
public static class FeedCapacity {
private Long id;
private Long capacity;
}
}

277
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java Normal file
View File

@@ -0,0 +1,277 @@
package com.commafeed.backend.dao;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.builder.CompareToBuilder;
import org.hibernate.SessionFactory;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.FixedSizeSortedSet;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
import com.commafeed.backend.model.QFeedEntryStatus;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings.ReadingOrder;
import com.commafeed.frontend.model.UnreadCount;
import com.google.common.collect.Iterables;
import com.google.common.collect.Ordering;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.Tuple;
import com.querydsl.jpa.impl.JPAQuery;
@Singleton
public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
private static final Comparator<FeedEntryStatus> STATUS_COMPARATOR_DESC = new Comparator<FeedEntryStatus>() {
@Override
public int compare(FeedEntryStatus o1, FeedEntryStatus o2) {
CompareToBuilder builder = new CompareToBuilder();
builder.append(o2.getEntryUpdated(), o1.getEntryUpdated());
builder.append(o2.getId(), o1.getId());
return builder.toComparison();
}
};
private static final Comparator<FeedEntryStatus> STATUS_COMPARATOR_ASC = Ordering.from(STATUS_COMPARATOR_DESC).reverse();
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO;
private final CommaFeedConfiguration config;
private final QFeedEntryStatus status = QFeedEntryStatus.feedEntryStatus;
private final QFeedEntry entry = QFeedEntry.feedEntry;
private final QFeedEntryContent content = QFeedEntryContent.feedEntryContent;
private final QFeedEntryTag entryTag = QFeedEntryTag.feedEntryTag;
@Inject
public FeedEntryStatusDAO(SessionFactory sessionFactory, FeedEntryDAO feedEntryDAO, FeedEntryTagDAO feedEntryTagDAO,
CommaFeedConfiguration config) {
super(sessionFactory);
this.feedEntryDAO = feedEntryDAO;
this.feedEntryTagDAO = feedEntryTagDAO;
this.config = config;
}
public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) {
List<FeedEntryStatus> statuses = query().selectFrom(status).where(status.entry.eq(entry), status.subscription.eq(sub)).fetch();
FeedEntryStatus status = Iterables.getFirst(statuses, null);
return handleStatus(user, status, sub, entry);
}
private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
if (status == null) {
Date unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
boolean read = unreadThreshold == null ? false : entry.getUpdated().before(unreadThreshold);
status = new FeedEntryStatus(user, sub, entry);
status.setRead(read);
status.setMarkable(!read);
} else {
status.setMarkable(true);
}
return status;
}
private FeedEntryStatus fetchTags(User user, FeedEntryStatus status) {
List<FeedEntryTag> tags = feedEntryTagDAO.findByEntry(user, status.getEntry());
status.setTags(tags);
return status;
}
public List<FeedEntryStatus> findStarred(User user, Date newerThan, int offset, int limit, ReadingOrder order, boolean includeContent) {
JPAQuery<FeedEntryStatus> query = query().selectFrom(status).where(status.user.eq(user), status.starred.isTrue());
if (newerThan != null) {
query.where(status.entryInserted.gt(newerThan));
}
if (order == ReadingOrder.asc) {
query.orderBy(status.entryUpdated.asc(), status.id.asc());
} else {
query.orderBy(status.entryUpdated.desc(), status.id.desc());
}
query.offset(offset).limit(limit);
setTimeout(query, config.getApplicationSettings().getQueryTimeout());
List<FeedEntryStatus> statuses = query.fetch();
for (FeedEntryStatus status : statuses) {
status = handleStatus(user, status, status.getSubscription(), status.getEntry());
fetchTags(user, status);
}
return lazyLoadContent(includeContent, statuses);
}
private JPAQuery<FeedEntry> buildQuery(User user, FeedSubscription sub, boolean unreadOnly, List<FeedEntryKeyword> keywords,
Date newerThan, int offset, int limit, ReadingOrder order, FeedEntryStatus last, String tag) {
JPAQuery<FeedEntry> query = query().selectFrom(entry).where(entry.feed.eq(sub.getFeed()));
if (CollectionUtils.isNotEmpty(keywords)) {
query.join(entry.content, content);
for (FeedEntryKeyword keyword : keywords) {
BooleanBuilder or = new BooleanBuilder();
or.or(content.content.containsIgnoreCase(keyword.getKeyword()));
or.or(content.title.containsIgnoreCase(keyword.getKeyword()));
if (keyword.getMode() == Mode.EXCLUDE) {
or.not();
}
query.where(or);
}
}
query.leftJoin(entry.statuses, status).on(status.subscription.id.eq(sub.getId()));
if (unreadOnly && tag == null) {
BooleanBuilder or = new BooleanBuilder();
or.or(status.read.isNull());
or.or(status.read.isFalse());
query.where(or);
Date unreadThreshold = config.getApplicationSettings().getUnreadThreshold();
if (unreadThreshold != null) {
query.where(entry.updated.goe(unreadThreshold));
}
}
if (tag != null) {
BooleanBuilder and = new BooleanBuilder();
and.and(entryTag.user.id.eq(user.getId()));
and.and(entryTag.name.eq(tag));
query.join(entry.tags, entryTag).on(and);
}
if (newerThan != null) {
query.where(entry.inserted.goe(newerThan));
}
if (last != null) {
if (order == ReadingOrder.desc) {
query.where(entry.updated.gt(last.getEntryUpdated()));
} else {
query.where(entry.updated.lt(last.getEntryUpdated()));
}
}
if (order != null) {
if (order == ReadingOrder.asc) {
query.orderBy(entry.updated.asc(), entry.id.asc());
} else {
query.orderBy(entry.updated.desc(), entry.id.desc());
}
}
if (offset > -1) {
query.offset(offset);
}
if (limit > -1) {
query.limit(limit);
}
setTimeout(query, config.getApplicationSettings().getQueryTimeout());
return query;
}
public List<FeedEntryStatus> findBySubscriptions(User user, List<FeedSubscription> subs, boolean unreadOnly,
List<FeedEntryKeyword> keywords, Date newerThan, int offset, int limit, ReadingOrder order, boolean includeContent,
boolean onlyIds, String tag) {
int capacity = offset + limit;
Comparator<FeedEntryStatus> comparator = order == ReadingOrder.desc ? STATUS_COMPARATOR_DESC : STATUS_COMPARATOR_ASC;
FixedSizeSortedSet<FeedEntryStatus> set = new FixedSizeSortedSet<>(capacity, comparator);
for (FeedSubscription sub : subs) {
FeedEntryStatus last = (order != null && set.isFull()) ? set.last() : null;
JPAQuery<FeedEntry> query = buildQuery(user, sub, unreadOnly, keywords, newerThan, -1, capacity, order, last, tag);
List<Tuple> tuples = query.select(entry.id, entry.updated, status.id, entry.content.title).fetch();
for (Tuple tuple : tuples) {
Long id = tuple.get(entry.id);
Date updated = tuple.get(entry.updated);
Long statusId = tuple.get(status.id);
FeedEntryContent content = new FeedEntryContent();
content.setTitle(tuple.get(entry.content.title));
FeedEntry entry = new FeedEntry();
entry.setId(id);
entry.setUpdated(updated);
entry.setContent(content);
FeedEntryStatus status = new FeedEntryStatus();
status.setId(statusId);
status.setEntryUpdated(updated);
status.setEntry(entry);
status.setSubscription(sub);
set.add(status);
}
}
List<FeedEntryStatus> placeholders = set.asList();
int size = placeholders.size();
if (size < offset) {
return new ArrayList<>();
}
placeholders = placeholders.subList(Math.max(offset, 0), size);
List<FeedEntryStatus> statuses = null;
if (onlyIds) {
statuses = placeholders;
} else {
statuses = new ArrayList<>();
for (FeedEntryStatus placeholder : placeholders) {
Long statusId = placeholder.getId();
FeedEntry entry = feedEntryDAO.findById(placeholder.getEntry().getId());
FeedEntryStatus status = handleStatus(user, statusId == null ? null : findById(statusId), placeholder.getSubscription(),
entry);
status = fetchTags(user, status);
statuses.add(status);
}
statuses = lazyLoadContent(includeContent, statuses);
}
return statuses;
}
public UnreadCount getUnreadCount(User user, FeedSubscription subscription) {
UnreadCount uc = null;
JPAQuery<FeedEntry> query = buildQuery(user, subscription, true, null, null, -1, -1, null, null, null);
List<Tuple> tuples = query.select(entry.count(), entry.updated.max()).fetch();
for (Tuple tuple : tuples) {
Long count = tuple.get(entry.count());
Date updated = tuple.get(entry.updated.max());
uc = new UnreadCount(subscription.getId(), count, updated);
}
return uc;
}
private List<FeedEntryStatus> lazyLoadContent(boolean includeContent, List<FeedEntryStatus> results) {
if (includeContent) {
for (FeedEntryStatus status : results) {
Models.initialize(status.getSubscription().getFeed());
Models.initialize(status.getEntry().getContent());
}
}
return results;
}
public List<FeedEntryStatus> getOldStatuses(Date olderThan, int limit) {
return query().selectFrom(status).where(status.entryInserted.lt(olderThan), status.starred.isFalse()).limit(limit).fetch();
}
}

32
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java Normal file
View File

@@ -0,0 +1,32 @@
package com.commafeed.backend.dao;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
@Singleton
public class FeedEntryTagDAO extends GenericDAO<FeedEntryTag> {
private QFeedEntryTag tag = QFeedEntryTag.feedEntryTag;
@Inject
public FeedEntryTagDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public List<String> findByUser(User user) {
return query().selectDistinct(tag.name).from(tag).where(tag.user.eq(user)).fetch();
}
public List<FeedEntryTag> findByEntry(User user, FeedEntry entry) {
return query().selectFrom(tag).where(tag.user.eq(user), tag.entry.eq(entry)).fetch();
}
}

91
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java Normal file
View File

@@ -0,0 +1,91 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.QFeedSubscription;
import com.commafeed.backend.model.User;
import com.google.common.collect.Iterables;
import com.querydsl.jpa.JPQLQuery;
@Singleton
public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> {
private QFeedSubscription sub = QFeedSubscription.feedSubscription;
@Inject
public FeedSubscriptionDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public FeedSubscription findById(User user, Long id) {
List<FeedSubscription> subs = query().selectFrom(sub)
.where(sub.user.eq(user), sub.id.eq(id))
.leftJoin(sub.feed)
.fetchJoin()
.leftJoin(sub.category)
.fetchJoin()
.fetch();
return initRelations(Iterables.getFirst(subs, null));
}
public List<FeedSubscription> findByFeed(Feed feed) {
return query().selectFrom(sub).where(sub.feed.eq(feed)).fetch();
}
public FeedSubscription findByFeed(User user, Feed feed) {
List<FeedSubscription> subs = query().selectFrom(sub).where(sub.user.eq(user), sub.feed.eq(feed)).fetch();
return initRelations(Iterables.getFirst(subs, null));
}
public List<FeedSubscription> findAll(User user) {
List<FeedSubscription> subs = query().selectFrom(sub)
.where(sub.user.eq(user))
.leftJoin(sub.feed)
.fetchJoin()
.leftJoin(sub.category)
.fetchJoin()
.fetch();
return initRelations(subs);
}
public List<FeedSubscription> findByCategory(User user, FeedCategory category) {
JPQLQuery<FeedSubscription> query = query().selectFrom(sub).where(sub.user.eq(user));
if (category == null) {
query.where(sub.category.isNull());
} else {
query.where(sub.category.eq(category));
}
return initRelations(query.fetch());
}
public List<FeedSubscription> findByCategories(User user, List<FeedCategory> categories) {
Set<Long> categoryIds = categories.stream().map(c -> c.getId()).collect(Collectors.toSet());
return findAll(user).stream()
.filter(s -> s.getCategory() != null && categoryIds.contains(s.getCategory().getId()))
.collect(Collectors.toList());
}
private List<FeedSubscription> initRelations(List<FeedSubscription> list) {
list.forEach(s -> initRelations(s));
return list;
}
private FeedSubscription initRelations(FeedSubscription sub) {
if (sub != null) {
Models.initialize(sub.getFeed());
Models.initialize(sub.getCategory());
}
return sub;
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/dao/GenericDAO.java Normal file
View File

@@ -0,0 +1,60 @@
package com.commafeed.backend.dao;
import java.util.Collection;
import org.hibernate.SessionFactory;
import org.hibernate.annotations.QueryHints;
import com.commafeed.backend.model.AbstractModel;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
import io.dropwizard.hibernate.AbstractDAO;
public abstract class GenericDAO<T extends AbstractModel> extends AbstractDAO<T> {
private JPAQueryFactory factory;
protected GenericDAO(SessionFactory sessionFactory) {
super(sessionFactory);
this.factory = new JPAQueryFactory(() -> currentSession());
}
protected JPAQueryFactory query() {
return factory;
}
public void saveOrUpdate(T model) {
persist(model);
}
public void saveOrUpdate(Collection<T> models) {
models.forEach(m -> persist(m));
}
public void update(T model) {
currentSession().merge(model);
}
public T findById(Long id) {
return get(id);
}
public void delete(T object) {
if (object != null) {
currentSession().delete(object);
}
}
public int delete(Collection<T> objects) {
objects.forEach(o -> delete(o));
return objects.size();
}
protected void setTimeout(JPAQuery<?> query, int timeoutMs) {
if (timeoutMs > 0) {
query.setHint(QueryHints.TIMEOUT_JPA, timeoutMs);
}
}
}

69
commafeed-server/src/main/java/com/commafeed/backend/dao/UnitOfWork.java Normal file
View File

@@ -0,0 +1,69 @@
package com.commafeed.backend.dao;
import org.hibernate.Session;
import org.hibernate.SessionFactory;
import org.hibernate.Transaction;
import org.hibernate.context.internal.ManagedSessionContext;
public class UnitOfWork {
public static void run(SessionFactory sessionFactory, SessionRunner sessionRunner) {
call(sessionFactory, () -> {
sessionRunner.runInSession();
return null;
});
}
public static <T> T call(SessionFactory sessionFactory, SessionRunnerReturningValue<T> sessionRunner) {
final Session session = sessionFactory.openSession();
if (ManagedSessionContext.hasBind(sessionFactory)) {
throw new IllegalStateException("Already in a unit of work!");
}
T t = null;
try {
ManagedSessionContext.bind(session);
session.beginTransaction();
try {
t = sessionRunner.runInSession();
commitTransaction(session);
} catch (Exception e) {
rollbackTransaction(session);
UnitOfWork.<RuntimeException> rethrow(e);
}
} finally {
session.close();
ManagedSessionContext.unbind(sessionFactory);
}
return t;
}
private static void rollbackTransaction(Session session) {
final Transaction txn = session.getTransaction();
if (txn != null && txn.isActive()) {
txn.rollback();
}
}
private static void commitTransaction(Session session) {
final Transaction txn = session.getTransaction();
if (txn != null && txn.isActive()) {
txn.commit();
}
}
@SuppressWarnings("unchecked")
private static <E extends Exception> void rethrow(Exception e) throws E {
throw (E) e;
}
@FunctionalInterface
public interface SessionRunner {
void runInSession();
}
@FunctionalInterface
public interface SessionRunnerReturningValue<T> {
T runInSession();
}
}

36
commafeed-server/src/main/java/com/commafeed/backend/dao/UserDAO.java Normal file
View File

@@ -0,0 +1,36 @@
package com.commafeed.backend.dao;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
@Singleton
public class UserDAO extends GenericDAO<User> {
private QUser user = QUser.user;
@Inject
public UserDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public User findByName(String name) {
return query().selectFrom(user).where(user.name.equalsIgnoreCase(name)).fetchOne();
}
public User findByApiKey(String key) {
return query().selectFrom(user).where(user.apiKey.equalsIgnoreCase(key)).fetchOne();
}
public User findByEmail(String email) {
return query().selectFrom(user).where(user.email.equalsIgnoreCase(email)).fetchOne();
}
public long count() {
return query().selectFrom(user).fetchCount();
}
}

38
commafeed-server/src/main/java/com/commafeed/backend/dao/UserRoleDAO.java Normal file
View File

@@ -0,0 +1,38 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.QUserRole;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
@Singleton
public class UserRoleDAO extends GenericDAO<UserRole> {
private QUserRole role = QUserRole.userRole;
@Inject
public UserRoleDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public List<UserRole> findAll() {
return query().selectFrom(role).leftJoin(role.user).fetchJoin().distinct().fetch();
}
public List<UserRole> findAll(User user) {
return query().selectFrom(role).where(role.user.eq(user)).distinct().fetch();
}
public Set<Role> findRoles(User user) {
return findAll(user).stream().map(r -> r.getRole()).collect(Collectors.toSet());
}
}

25
commafeed-server/src/main/java/com/commafeed/backend/dao/UserSettingsDAO.java Normal file
View File

@@ -0,0 +1,25 @@
package com.commafeed.backend.dao;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.model.QUserSettings;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings;
@Singleton
public class UserSettingsDAO extends GenericDAO<UserSettings> {
private QUserSettings settings = QUserSettings.userSettings;
@Inject
public UserSettingsDAO(SessionFactory sessionFactory) {
super(sessionFactory);
}
public UserSettings findByUser(User user) {
return query().selectFrom(settings).where(settings.user.eq(user)).fetchFirst();
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java Normal file
View File

@@ -0,0 +1,60 @@
package com.commafeed.backend.favicon;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.model.Feed;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class AbstractFaviconFetcher {
protected static final int TIMEOUT = 4000;
private static final List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
public abstract Favicon fetch(Feed feed);
protected boolean isValidIconResponse(byte[] content, String contentType) {
if (content == null) {
return false;
}
long length = content.length;
if (StringUtils.isNotBlank(contentType)) {
contentType = contentType.split(";")[0];
}
if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
log.debug("Content-Type {} is blacklisted", contentType);
return false;
}
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
return true;
}
@RequiredArgsConstructor
@Getter
public static class Favicon {
private final byte[] icon;
private final String mediaType;
}
}

132
commafeed-server/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java Normal file
View File

@@ -0,0 +1,132 @@
package com.commafeed.backend.favicon;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Inspired/Ported from https://github.com/potatolondon/getfavicon
*
*/
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
Favicon icon = fetch(feed.getLink());
if (icon == null) {
icon = fetch(feed.getUrl());
}
return icon;
}
private Favicon fetch(String url) {
if (url == null) {
log.debug("url is null");
return null;
}
int doubleSlash = url.indexOf("//");
if (doubleSlash == -1) {
doubleSlash = 0;
} else {
doubleSlash += 2;
}
int firstSlash = url.indexOf('/', doubleSlash);
if (firstSlash != -1) {
url = url.substring(0, firstSlash);
}
Favicon icon = getIconAtRoot(url);
if (icon == null) {
icon = getIconInPage(url);
}
return icon;
}
private Favicon getIconAtRoot(String url) {
byte[] bytes = null;
String contentType = null;
try {
url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico";
log.debug("getting root icon at {}", url);
HttpResult result = getter.getBinary(url, TIMEOUT);
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url);
log.trace("Failed to retrieve iconAtRoot for url {}: ", url, e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private Favicon getIconInPage(String url) {
Document doc = null;
try {
HttpResult result = getter.getBinary(url, TIMEOUT);
doc = Jsoup.parse(new String(result.getContent()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon");
log.trace("Failed to retrieve page to find icon", e);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
String href = icons.get(0).attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
log.debug("Found unconfirmed iconInPage at {}", href);
byte[] bytes = null;
String contentType = null;
try {
HttpResult result = getter.getBinary(href, TIMEOUT);
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href);
log.trace("Failed to retrieve icon found in page {}", href, e);
return null;
}
if (!isValidIconResponse(bytes, contentType)) {
log.debug("Invalid icon found for {}", href);
return null;
}
return new Favicon(bytes, contentType);
}
}

79
commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java Normal file
View File

@@ -0,0 +1,79 @@
package com.commafeed.backend.favicon;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("www.facebook.com")) {
return null;
}
String userName = extractUserName(url);
if (userName == null) {
return null;
}
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
byte[] bytes = null;
String contentType = null;
try {
log.debug("Getting Facebook user's icon, {}", url);
HttpResult iconResult = getter.getBinary(iconUrl, TIMEOUT);
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private String extractUserName(String url) {
URI uri = null;
try {
uri = new URI(url);
} catch (URISyntaxException e) {
log.debug("could not parse url", e);
return null;
}
List<NameValuePair> params = URLEncodedUtils.parse(uri, StandardCharsets.UTF_8);
for (NameValuePair param : params) {
if ("id".equals(param.getName())) {
return param.getValue();
}
}
return null;
}
}

100
commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java Normal file
View File

@@ -0,0 +1,100 @@
package com.commafeed.backend.favicon;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Optional;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.youtube.YouTube;
import com.google.api.services.youtube.model.Channel;
import com.google.api.services.youtube.model.ChannelListResponse;
import com.google.api.services.youtube.model.Thumbnail;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
private final CommaFeedConfiguration config;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
return null;
}
String googleAuthKey = config.getApplicationSettings().getGoogleAuthKey();
if (googleAuthKey == null) {
log.debug("no google auth key configured");
return null;
}
byte[] bytes = null;
String contentType = null;
try {
List<NameValuePair> params = URLEncodedUtils.parse(url.substring(url.indexOf("?") + 1), StandardCharsets.UTF_8);
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
Optional<NameValuePair> channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id")).findFirst();
if (!userId.isPresent() && !channelId.isPresent()) {
return null;
}
YouTube youtube = new YouTube.Builder(new NetHttpTransport(), JacksonFactory.getDefaultInstance(),
new HttpRequestInitializer() {
@Override
public void initialize(HttpRequest request) throws IOException {
}
}).setApplicationName("CommaFeed").build();
YouTube.Channels.List list = youtube.channels().list("snippet");
list.setKey(googleAuthKey);
if (userId.isPresent()) {
list.setForUsername(userId.get().getValue());
} else {
list.setId(channelId.get().getValue());
}
log.debug("contacting youtube api");
ChannelListResponse response = list.execute();
if (response.getItems().isEmpty()) {
log.debug("youtube api returned no items");
return null;
}
Channel channel = response.getItems().get(0);
Thumbnail thumbnail = channel.getSnippet().getThumbnails().getDefault();
log.debug("fetching favicon");
HttpResult iconResult = getter.getBinary(thumbnail.getUrl(), TIMEOUT);
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve YouTube icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
}

39
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedEntryKeyword.java Normal file
View File

@@ -0,0 +1,39 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
/**
* A keyword used in a search query
*/
@Getter
@RequiredArgsConstructor
public class FeedEntryKeyword {
public enum Mode {
INCLUDE, EXCLUDE;
}
private final String keyword;
private final Mode mode;
public static List<FeedEntryKeyword> fromQueryString(String keywords) {
List<FeedEntryKeyword> list = new ArrayList<>();
if (keywords != null) {
for (String keyword : StringUtils.split(keywords)) {
boolean not = false;
if (keyword.startsWith("-") || keyword.startsWith("!")) {
not = true;
keyword = keyword.substring(1);
}
list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
}
}
return list;
}
}

96
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java Normal file
View File

@@ -0,0 +1,96 @@
package com.commafeed.backend.feed;
import java.io.IOException;
import java.util.Date;
import java.util.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.binary.StringUtils;
import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import com.rometools.rome.io.FeedException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedFetcher {
private final FeedParser parser;
private final HttpGetter getter;
private final Set<FeedURLProvider> urlProviders;
public FetchedFeed fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag, Date lastPublishedDate,
String lastContentHash) throws FeedException, IOException, NotModifiedException {
log.debug("Fetching feed {}", feedUrl);
FetchedFeed fetchedFeed = null;
int timeout = 20000;
HttpResult result = getter.getBinary(feedUrl, lastModified, eTag, timeout);
byte[] content = result.getContent();
try {
fetchedFeed = parser.parse(result.getUrlAfterRedirect(), content);
} catch (FeedException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, StringUtils.newStringUtf8(result.getContent()));
if (org.apache.commons.lang3.StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
result = getter.getBinary(extractedUrl, lastModified, eTag, timeout);
content = result.getContent();
fetchedFeed = parser.parse(result.getUrlAfterRedirect(), content);
} else {
throw e;
}
} else {
throw e;
}
}
if (content == null) {
throw new IOException("Feed content is empty.");
}
String hash = DigestUtils.sha1Hex(content);
if (lastContentHash != null && hash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException("content hash not modified");
}
if (lastPublishedDate != null && fetchedFeed.getFeed().getLastPublishedDate() != null
&& lastPublishedDate.getTime() == fetchedFeed.getFeed().getLastPublishedDate().getTime()) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified");
}
Feed feed = fetchedFeed.getFeed();
feed.setLastModifiedHeader(result.getLastModifiedSince());
feed.setEtagHeader(FeedUtils.truncate(result.getETag(), 255));
feed.setLastContentHash(hash);
fetchedFeed.setFetchDuration(result.getDuration());
fetchedFeed.setUrlAfterRedirect(result.getUrlAfterRedirect());
return fetchedFeed;
}
private static String extractFeedUrl(Set<FeedURLProvider> urlProviders, String url, String urlContent) {
for (FeedURLProvider urlProvider : urlProviders) {
String feedUrl = urlProvider.get(url, urlContent);
if (feedUrl != null) {
return feedUrl;
}
}
return null;
}
}

276
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedParser.java Normal file
View File

@@ -0,0 +1,276 @@
package com.commafeed.backend.feed;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.google.common.collect.Iterables;
import com.rometools.modules.mediarss.MediaEntryModule;
import com.rometools.modules.mediarss.MediaModule;
import com.rometools.modules.mediarss.types.MediaGroup;
import com.rometools.modules.mediarss.types.Metadata;
import com.rometools.modules.mediarss.types.Thumbnail;
import com.rometools.rome.feed.synd.SyndCategory;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.SyndFeedInput;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedParser {
private static final String ATOM_10_URI = "http://www.w3.org/2005/Atom";
private static final Namespace ATOM_10_NS = Namespace.getNamespace(ATOM_10_URI);
private static final Date START = new Date(86400000);
private static final Date END = new Date(1000L * Integer.MAX_VALUE - 86400000);
public FetchedFeed parse(String feedUrl, byte[] xml) throws FeedException {
FetchedFeed fetchedFeed = new FetchedFeed();
Feed feed = fetchedFeed.getFeed();
List<FeedEntry> entries = fetchedFeed.getEntries();
try {
Charset encoding = FeedUtils.guessEncoding(xml);
String xmlString = FeedUtils.trimInvalidXmlCharacters(new String(xml, encoding));
if (xmlString == null) {
throw new FeedException("Input string is null for url " + feedUrl);
}
xmlString = FeedUtils.replaceHtmlEntitiesWithNumericEntities(xmlString);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed rss = new SyndFeedInput().build(source);
handleForeignMarkup(rss);
fetchedFeed.setTitle(rss.getTitle());
feed.setPushHub(findHub(rss));
feed.setPushTopic(findSelf(rss));
feed.setUrl(feedUrl);
feed.setLink(rss.getLink());
List<SyndEntry> items = rss.getEntries();
for (SyndEntry item : items) {
FeedEntry entry = new FeedEntry();
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
entry.setGuid(FeedUtils.truncate(guid, 2048));
entry.setUpdated(validateDate(getEntryUpdateDate(item), true));
entry.setUrl(FeedUtils.truncate(FeedUtils.toAbsoluteUrl(item.getLink(), feed.getLink(), feedUrl), 2048));
// if link is empty but guid is used as url
if (StringUtils.isBlank(entry.getUrl()) && StringUtils.startsWith(entry.getGuid(), "http")) {
entry.setUrl(entry.getGuid());
}
FeedEntryContent content = new FeedEntryContent();
content.setContent(getContent(item));
content.setCategories(FeedUtils
.truncate(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")), 4096));
content.setTitle(getTitle(item));
content.setAuthor(StringUtils.trimToNull(item.getAuthor()));
SyndEnclosure enclosure = Iterables.getFirst(item.getEnclosures(), null);
if (enclosure != null) {
content.setEnclosureUrl(FeedUtils.truncate(enclosure.getUrl(), 2048));
content.setEnclosureType(enclosure.getType());
}
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module != null) {
Media media = getMedia(module);
if (media != null) {
content.setMediaDescription(media.getDescription());
content.setMediaThumbnailUrl(FeedUtils.truncate(media.getThumbnailUrl(), 2048));
content.setMediaThumbnailWidth(media.getThumbnailWidth());
content.setMediaThumbnailHeight(media.getThumbnailHeight());
}
}
entry.setContent(content);
entries.add(entry);
}
Date lastEntryDate = null;
Date publishedDate = validateDate(rss.getPublishedDate(), false);
if (!entries.isEmpty()) {
List<Long> sortedTimestamps = FeedUtils.getSortedTimestamps(entries);
Long timestamp = sortedTimestamps.get(0);
lastEntryDate = new Date(timestamp);
publishedDate = (publishedDate == null || publishedDate.before(lastEntryDate)) ? lastEntryDate : publishedDate;
}
feed.setLastPublishedDate(publishedDate);
feed.setAverageEntryInterval(FeedUtils.averageTimeBetweenEntries(entries));
feed.setLastEntryDate(lastEntryDate);
} catch (Exception e) {
throw new FeedException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
return fetchedFeed;
}
/**
* Adds atom links for rss feeds
*/
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
private Date getEntryUpdateDate(SyndEntry item) {
Date date = item.getUpdatedDate();
if (date == null) {
date = item.getPublishedDate();
}
if (date == null) {
date = new Date();
}
return date;
}
private Date validateDate(Date date, boolean nullToNow) {
Date now = new Date();
if (date == null) {
return nullToNow ? now : null;
}
if (date.before(START) || date.after(END)) {
return now;
}
if (date.after(now)) {
return now;
}
return date;
}
private String getContent(SyndEntry item) {
String content = null;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private Media getMedia(MediaEntryModule module) {
Media media = getMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = getMedia(group.getMetadata());
}
return media;
}
private Media getMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
Media media = new Media();
media.setDescription(metadata.getDescription());
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
media.setThumbnailWidth(thumbnail.getWidth());
media.setThumbnailHeight(thumbnail.getHeight());
if (thumbnail.getUrl() != null) {
media.setThumbnailUrl(thumbnail.getUrl().toString());
}
}
if (media.isEmpty()) {
return null;
}
return media;
}
private String findHub(SyndFeed feed) {
for (SyndLink l : feed.getLinks()) {
if ("hub".equalsIgnoreCase(l.getRel())) {
log.debug("found hub {} for feed {}", l.getHref(), feed.getLink());
return l.getHref();
}
}
return null;
}
private String findSelf(SyndFeed feed) {
for (SyndLink l : feed.getLinks()) {
if ("self".equalsIgnoreCase(l.getRel())) {
log.debug("found self {} for feed {}", l.getHref(), feed.getLink());
return l.getHref();
}
}
return null;
}
@Data
private static class Media {
private String description;
private String thumbnailUrl;
private Integer thumbnailWidth;
private Integer thumbnailHeight;
public boolean isEmpty() {
return description == null && thumbnailUrl == null;
}
}
}

160
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedQueues.java Normal file
View File

@@ -0,0 +1,160 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.hibernate.SessionFactory;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.Feed;
@Singleton
public class FeedQueues {
private SessionFactory sessionFactory;
private final FeedDAO feedDAO;
private final CommaFeedConfiguration config;
private Queue<FeedRefreshContext> addQueue = new ConcurrentLinkedQueue<>();
private Queue<FeedRefreshContext> takeQueue = new ConcurrentLinkedQueue<>();
private Queue<Feed> giveBackQueue = new ConcurrentLinkedQueue<>();
private Meter refill;
@Inject
public FeedQueues(SessionFactory sessionFactory, FeedDAO feedDAO, CommaFeedConfiguration config, MetricRegistry metrics) {
this.sessionFactory = sessionFactory;
this.config = config;
this.feedDAO = feedDAO;
refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
metrics.register(MetricRegistry.name(getClass(), "addQueue"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return addQueue.size();
}
});
metrics.register(MetricRegistry.name(getClass(), "takeQueue"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return takeQueue.size();
}
});
metrics.register(MetricRegistry.name(getClass(), "giveBackQueue"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return giveBackQueue.size();
}
});
}
/**
* take a feed from the refresh queue
*/
public synchronized FeedRefreshContext take() {
FeedRefreshContext context = takeQueue.poll();
if (context == null) {
refill();
context = takeQueue.poll();
}
return context;
}
/**
* add a feed to the refresh queue
*/
public void add(Feed feed, boolean urgent) {
int refreshInterval = config.getApplicationSettings().getRefreshIntervalMinutes();
if (feed.getLastUpdated() == null || feed.getLastUpdated().before(DateUtils.addMinutes(new Date(), -1 * refreshInterval))) {
boolean alreadyQueued = addQueue.stream().anyMatch(c -> c.getFeed().getId().equals(feed.getId()));
if (!alreadyQueued) {
addQueue.add(new FeedRefreshContext(feed, urgent));
}
}
}
/**
* refills the refresh queue and empties the giveBack queue while at it
*/
private void refill() {
refill.mark();
List<FeedRefreshContext> contexts = new ArrayList<>();
int batchSize = Math.min(100, 3 * config.getApplicationSettings().getBackgroundThreads());
// add feeds we got from the add() method
int addQueueSize = addQueue.size();
for (int i = 0; i < Math.min(batchSize, addQueueSize); i++) {
contexts.add(addQueue.poll());
}
// add feeds that are up to refresh from the database
int count = batchSize - contexts.size();
if (count > 0) {
List<Feed> feeds = UnitOfWork.call(sessionFactory, () -> feedDAO.findNextUpdatable(count, getLastLoginThreshold()));
for (Feed feed : feeds) {
contexts.add(new FeedRefreshContext(feed, false));
}
}
// set the disabledDate as we use it in feedDAO to decide what to refresh next. We also use a map to remove
// duplicates.
Map<Long, FeedRefreshContext> map = new LinkedHashMap<>();
for (FeedRefreshContext context : contexts) {
Feed feed = context.getFeed();
feed.setDisabledUntil(DateUtils.addMinutes(new Date(), config.getApplicationSettings().getRefreshIntervalMinutes()));
map.put(feed.getId(), context);
}
// refill the queue
takeQueue.addAll(map.values());
// add feeds from the giveBack queue to the map, overriding duplicates
int giveBackQueueSize = giveBackQueue.size();
for (int i = 0; i < giveBackQueueSize; i++) {
Feed feed = giveBackQueue.poll();
map.put(feed.getId(), new FeedRefreshContext(feed, false));
}
// update all feeds in the database
List<Feed> feeds = map.values().stream().map(c -> c.getFeed()).collect(Collectors.toList());
UnitOfWork.run(sessionFactory, () -> feedDAO.saveOrUpdate(feeds));
}
/**
* give a feed back, updating it to the database during the next refill()
*/
public void giveBack(Feed feed) {
String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
feed.setLastUpdated(new Date());
giveBackQueue.add(feed);
}
private Date getLastLoginThreshold() {
if (config.getApplicationSettings().getHeavyLoad()) {
return DateUtils.addDays(new Date(), -30);
} else {
return null;
}
}
}

22
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshContext.java Normal file
View File

@@ -0,0 +1,22 @@
package com.commafeed.backend.feed;
import java.util.List;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class FeedRefreshContext {
private Feed feed;
private List<FeedEntry> entries;
private boolean urgent;
public FeedRefreshContext(Feed feed, boolean isUrgent) {
this.feed = feed;
this.urgent = isUrgent;
}
}

99
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshExecutor.java Normal file
View File

@@ -0,0 +1,99 @@
package com.commafeed.backend.feed;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
import lombok.extern.slf4j.Slf4j;
/**
* Wraps a {@link ThreadPoolExecutor} instance. Blocks when queue is full instead of rejecting the task. Allow priority queueing by using
* {@link Task} instead of {@link Runnable}
*
*/
@Slf4j
public class FeedRefreshExecutor {
private String poolName;
private ThreadPoolExecutor pool;
private LinkedBlockingDeque<Runnable> queue;
public FeedRefreshExecutor(final String poolName, int threads, int queueCapacity, MetricRegistry metrics) {
log.info("Creating pool {} with {} threads", poolName, threads);
this.poolName = poolName;
pool = new ThreadPoolExecutor(threads, threads, 0, TimeUnit.MILLISECONDS, queue = new LinkedBlockingDeque<Runnable>(queueCapacity) {
private static final long serialVersionUID = 1L;
@Override
public boolean offer(Runnable r) {
Task task = (Task) r;
if (task.isUrgent()) {
return offerFirst(r);
} else {
return offerLast(r);
}
}
}) {
@Override
protected void afterExecute(Runnable r, Throwable t) {
if (t != null) {
log.error("thread from pool {} threw a runtime exception", poolName, t);
}
}
};
pool.setRejectedExecutionHandler(new RejectedExecutionHandler() {
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
log.debug("{} thread queue full, waiting...", poolName);
try {
Task task = (Task) r;
if (task.isUrgent()) {
queue.putFirst(r);
} else {
queue.put(r);
}
} catch (InterruptedException e1) {
log.error(poolName + " interrupted while waiting for queue.", e1);
}
}
});
metrics.register(MetricRegistry.name(getClass(), poolName, "active"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return pool.getActiveCount();
}
});
metrics.register(MetricRegistry.name(getClass(), poolName, "pending"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return queue.size();
}
});
}
public void execute(Task task) {
pool.execute(task);
}
public void shutdown() {
pool.shutdownNow();
while (!pool.isTerminated()) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
log.error("{} interrupted while waiting for threads to finish.", poolName);
}
}
}
public interface Task extends Runnable {
boolean isUrgent();
}
}

85
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshTaskGiver.java Normal file
View File

@@ -0,0 +1,85 @@
package com.commafeed.backend.feed;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import io.dropwizard.lifecycle.Managed;
import lombok.extern.slf4j.Slf4j;
/**
* Infinite loop fetching feeds from @FeedQueues and queuing them to the {@link FeedRefreshWorker} pool.
*
*/
@Slf4j
@Singleton
public class FeedRefreshTaskGiver implements Managed {
private final FeedQueues queues;
private final FeedRefreshWorker worker;
private final ExecutorService executor;
private final Meter feedRefreshed;
private final Meter threadWaited;
@Inject
public FeedRefreshTaskGiver(FeedQueues queues, FeedDAO feedDAO, FeedRefreshWorker worker, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.queues = queues;
this.worker = worker;
executor = Executors.newFixedThreadPool(1);
feedRefreshed = metrics.meter(MetricRegistry.name(getClass(), "feedRefreshed"));
threadWaited = metrics.meter(MetricRegistry.name(getClass(), "threadWaited"));
}
@Override
public void stop() {
log.info("shutting down feed refresh task giver");
executor.shutdownNow();
while (!executor.isTerminated()) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
log.error("interrupted while waiting for threads to finish.");
}
}
}
@Override
public void start() {
log.info("starting feed refresh task giver");
executor.execute(new Runnable() {
@Override
public void run() {
while (!executor.isShutdown()) {
try {
FeedRefreshContext context = queues.take();
if (context != null) {
feedRefreshed.mark();
worker.updateFeed(context);
} else {
log.debug("nothing to do, sleeping for 15s");
threadWaited.mark();
try {
Thread.sleep(15000);
} catch (InterruptedException e) {
log.debug("interrupted while sleeping");
}
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
}
});
}
}

226
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java Normal file
View File

@@ -0,0 +1,226 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.hibernate.SessionFactory;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.ApplicationSettings;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.FeedRefreshExecutor.Task;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedUpdateService;
import com.commafeed.backend.service.PubSubService;
import com.google.common.util.concurrent.Striped;
import io.dropwizard.lifecycle.Managed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedRefreshUpdater implements Managed {
private final SessionFactory sessionFactory;
private final FeedUpdateService feedUpdateService;
private final PubSubService pubSubService;
private final FeedQueues queues;
private final CommaFeedConfiguration config;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final CacheService cache;
private final FeedRefreshExecutor pool;
private final Striped<Lock> locks;
private final Meter entryCacheMiss;
private final Meter entryCacheHit;
private final Meter feedUpdated;
private final Meter entryInserted;
@Inject
public FeedRefreshUpdater(SessionFactory sessionFactory, FeedUpdateService feedUpdateService, PubSubService pubSubService,
FeedQueues queues, CommaFeedConfiguration config, MetricRegistry metrics, FeedSubscriptionDAO feedSubscriptionDAO,
CacheService cache) {
this.sessionFactory = sessionFactory;
this.feedUpdateService = feedUpdateService;
this.pubSubService = pubSubService;
this.queues = queues;
this.config = config;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.cache = cache;
ApplicationSettings settings = config.getApplicationSettings();
int threads = Math.max(settings.getDatabaseUpdateThreads(), 1);
pool = new FeedRefreshExecutor("feed-refresh-updater", threads, Math.min(50 * threads, 1000), metrics);
locks = Striped.lazyWeakLock(threads * 100000);
entryCacheMiss = metrics.meter(MetricRegistry.name(getClass(), "entryCacheMiss"));
entryCacheHit = metrics.meter(MetricRegistry.name(getClass(), "entryCacheHit"));
feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
@Override
public void start() throws Exception {
}
@Override
public void stop() throws Exception {
log.info("shutting down feed refresh updater");
pool.shutdown();
}
public void updateFeed(FeedRefreshContext context) {
pool.execute(new EntryTask(context));
}
private boolean addEntry(final Feed feed, final FeedEntry entry, final List<FeedSubscription> subscriptions) {
boolean success = false;
// lock on feed, make sure we are not updating the same feed twice at
// the same time
String key1 = StringUtils.trimToEmpty("" + feed.getId());
// lock on content, make sure we are not updating the same entry
// twice at the same time
FeedEntryContent content = entry.getContent();
String key2 = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getContent() + content.getTitle()));
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
Lock lock2 = iterator.next();
boolean locked1 = false;
boolean locked2 = false;
try {
locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
boolean inserted = UnitOfWork.call(sessionFactory, () -> feedUpdateService.addEntry(feed, entry, subscriptions));
if (inserted) {
entryInserted.mark();
}
success = true;
} else {
log.error("lock timeout for " + feed.getUrl() + " - " + key1);
}
} catch (InterruptedException e) {
log.error("interrupted while waiting for lock for " + feed.getUrl() + " : " + e.getMessage(), e);
} finally {
if (locked1) {
lock1.unlock();
}
if (locked2) {
lock2.unlock();
}
}
return success;
}
private void handlePubSub(final Feed feed) {
if (feed.getPushHub() != null && feed.getPushTopic() != null) {
Date lastPing = feed.getPushLastPing();
Date now = new Date();
if (lastPing == null || lastPing.before(DateUtils.addDays(now, -3))) {
new Thread() {
@Override
public void run() {
try {
// make sure the feed has been updated in the database so that the
// callback works
Thread.sleep(30000);
} catch (InterruptedException e1) {
// do nothing
}
pubSubService.subscribe(feed);
}
}.start();
}
}
}
private class EntryTask implements Task {
private final FeedRefreshContext context;
public EntryTask(FeedRefreshContext context) {
this.context = context;
}
@Override
public void run() {
boolean ok = true;
final Feed feed = context.getFeed();
List<FeedEntry> entries = context.getEntries();
if (entries.isEmpty()) {
feed.setMessage("Feed has no entries");
} else {
List<String> lastEntries = cache.getLastEntries(feed);
List<String> currentEntries = new ArrayList<>();
List<FeedSubscription> subscriptions = null;
for (FeedEntry entry : entries) {
String cacheKey = cache.buildUniqueEntryKey(feed, entry);
if (!lastEntries.contains(cacheKey)) {
log.debug("cache miss for {}", entry.getUrl());
if (subscriptions == null) {
subscriptions = UnitOfWork.call(sessionFactory, () -> feedSubscriptionDAO.findByFeed(feed));
}
ok &= addEntry(feed, entry, subscriptions);
entryCacheMiss.mark();
} else {
log.debug("cache hit for {}", entry.getUrl());
entryCacheHit.mark();
}
currentEntries.add(cacheKey);
}
cache.setLastEntries(feed, currentEntries);
if (subscriptions == null) {
feed.setMessage("No new entries found");
} else if (!subscriptions.isEmpty()) {
List<User> users = subscriptions.stream().map(FeedSubscription::getUser).collect(Collectors.toList());
cache.invalidateUnreadCount(subscriptions.toArray(new FeedSubscription[0]));
cache.invalidateUserRootCategory(users.toArray(new User[0]));
}
}
if (config.getApplicationSettings().getPubsubhubbub()) {
handlePubSub(feed);
}
if (!ok) {
// requeue asap
feed.setDisabledUntil(new Date(0));
}
feedUpdated.mark();
queues.giveBack(feed);
}
@Override
public boolean isUrgent() {
return context.isUrgent();
}
}
}

167
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java Normal file
View File

@@ -0,0 +1,167 @@
package com.commafeed.backend.feed;
import java.util.Date;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.feed.FeedRefreshExecutor.Task;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import io.dropwizard.lifecycle.Managed;
import lombok.extern.slf4j.Slf4j;
/**
* Calls {@link FeedFetcher} and handles its outcome
*
*/
@Slf4j
@Singleton
public class FeedRefreshWorker implements Managed {
private final FeedRefreshUpdater feedRefreshUpdater;
private final FeedFetcher fetcher;
private final FeedQueues queues;
private final CommaFeedConfiguration config;
private final FeedRefreshExecutor pool;
@Inject
public FeedRefreshWorker(FeedRefreshUpdater feedRefreshUpdater, FeedFetcher fetcher, FeedQueues queues, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.feedRefreshUpdater = feedRefreshUpdater;
this.fetcher = fetcher;
this.config = config;
this.queues = queues;
int threads = config.getApplicationSettings().getBackgroundThreads();
pool = new FeedRefreshExecutor("feed-refresh-worker", threads, Math.min(20 * threads, 1000), metrics);
}
@Override
public void start() throws Exception {
}
@Override
public void stop() throws Exception {
pool.shutdown();
}
public void updateFeed(FeedRefreshContext context) {
pool.execute(new FeedTask(context));
}
private void update(FeedRefreshContext context) {
Feed feed = context.getFeed();
int refreshInterval = config.getApplicationSettings().getRefreshIntervalMinutes();
Date disabledUntil = DateUtils.addMinutes(new Date(), refreshInterval);
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FetchedFeed fetchedFeed = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
List<FeedEntry> entries = fetchedFeed.getEntries();
Integer maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).collect(Collectors.toList());
}
if (config.getApplicationSettings().getHeavyLoad()) {
disabledUntil = FeedUtils.buildDisabledUntil(fetchedFeed.getFeed().getLastEntryDate(),
fetchedFeed.getFeed().getAverageEntryInterval(), disabledUntil);
}
String urlAfterRedirect = fetchedFeed.getUrlAfterRedirect();
if (StringUtils.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;
}
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(fetchedFeed.getFeed().getLink());
feed.setLastModifiedHeader(fetchedFeed.getFeed().getLastModifiedHeader());
feed.setEtagHeader(fetchedFeed.getFeed().getEtagHeader());
feed.setLastContentHash(fetchedFeed.getFeed().getLastContentHash());
feed.setLastPublishedDate(fetchedFeed.getFeed().getLastPublishedDate());
feed.setAverageEntryInterval(fetchedFeed.getFeed().getAverageEntryInterval());
feed.setLastEntryDate(fetchedFeed.getFeed().getLastEntryDate());
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(disabledUntil);
handlePubSub(feed, fetchedFeed.getFeed());
context.setEntries(entries);
feedRefreshUpdater.updateFeed(context);
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
if (config.getApplicationSettings().getHeavyLoad()) {
disabledUntil = FeedUtils.buildDisabledUntil(feed.getLastEntryDate(), feed.getAverageEntryInterval(), disabledUntil);
}
feed.setErrorCount(0);
feed.setMessage(e.getMessage());
feed.setDisabledUntil(disabledUntil);
queues.giveBack(feed);
} catch (Exception e) {
String message = "Unable to refresh feed " + feed.getUrl() + " : " + e.getMessage();
log.debug(e.getClass().getName() + " " + message, e);
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage(message);
feed.setDisabledUntil(FeedUtils.buildDisabledUntil(feed.getErrorCount()));
queues.giveBack(feed);
}
}
private void handlePubSub(Feed feed, Feed fetchedFeed) {
String hub = fetchedFeed.getPushHub();
String topic = fetchedFeed.getPushTopic();
if (hub != null && topic != null) {
if (hub.contains("hubbub.api.typepad.com")) {
// that hub does not exist anymore
return;
}
if (topic.startsWith("www.")) {
topic = "http://" + topic;
} else if (topic.startsWith("feed://")) {
topic = "http://" + topic.substring(7);
} else if (!topic.startsWith("http")) {
topic = "http://" + topic;
}
log.debug("feed {} has pubsub info: {}", feed.getUrl(), topic);
feed.setPushHub(hub);
feed.setPushTopic(topic);
feed.setPushTopicHash(DigestUtils.sha1Hex(topic));
}
}
private class FeedTask implements Task {
private final FeedRefreshContext context;
public FeedTask(FeedRefreshContext context) {
this.context = context;
}
@Override
public void run() {
update(context);
}
@Override
public boolean isUrgent() {
return context.isUrgent();
}
}
}

556
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java Normal file
View File

@@ -0,0 +1,556 @@
package com.commafeed.backend.feed;
import java.io.StringReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.ahocorasick.trie.Trie.TrieBuilder;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Document.OutputSettings;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Safelist;
import org.jsoup.select.Elements;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry;
import com.google.gwt.i18n.client.HasDirection.Direction;
import com.google.gwt.i18n.shared.BidiUtils;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import com.steadystate.css.parser.CSSOMParser;
import edu.uci.ics.crawler4j.url.URLCanonicalizer;
import lombok.extern.slf4j.Slf4j;
/**
* Utility methods related to feed handling
*
*/
@Slf4j
public class FeedUtils {
private static final String ESCAPED_QUESTION_MARK = Pattern.quote("?");
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
private static final Safelist WHITELIST = buildWhiteList();
public static String truncate(String string, int length) {
if (string != null) {
string = string.substring(0, Math.min(length, string.length()));
}
return string;
}
private static synchronized Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1",
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public static Charset guessEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (StringUtils.startsWithIgnoreCase(extracted, "iso-8859-")) {
if (!StringUtils.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (StringUtils.startsWithIgnoreCase(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Detect encoding by analyzing characters in the array
*/
public static Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
public static String replaceHtmlEntitiesWithNumericEntities(String source) {
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
TrieBuilder builder = Trie.builder();
builder.ignoreOverlaps();
for (String key : HtmlEntities.HTML_ENTITIES) {
builder.addKeyword(key);
}
Trie trie = builder.build();
Collection<Emit> emits = trie.parseText(source);
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
sb.append(source.substring(prevIndex, matchIndex));
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
return sb.toString();
}
public static boolean isHttp(String url) {
return url.startsWith("http://");
}
public static boolean isHttps(String url) {
return url.startsWith("https://");
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates
*/
public static String normalizeURL(String url) {
if (url == null) {
return null;
}
String normalized = URLCanonicalizer.getCanonicalURL(url);
if (normalized == null) {
normalized = url;
}
// convert to lower case, the url probably won't work in some cases
// after that but we don't care we just want to compare urls to avoid
// duplicates
normalized = normalized.toLowerCase();
// store all urls as http
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
// remove the www. part
normalized = normalized.replace("//www.", "//");
// feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedburner feeds have a special treatment
if (normalized.split(ESCAPED_QUESTION_MARK)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = normalized.split(ESCAPED_QUESTION_MARK)[0];
normalized = StringUtils.removeEnd(normalized, "/");
}
return normalized;
}
/**
* Extract the declared encoding from the xml
*/
public static String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = StringUtils.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10, pi.length());
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
public static String handleContent(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(WHITELIST);
Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
}
return content;
}
public static String escapeIFrameCss(String orig) {
String rule = "";
CSSOMParser parser = new CSSOMParser();
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = parser.parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
public static String escapeImgCss(String orig) {
String rule = "";
CSSOMParser parser = new CSSOMParser();
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = parser.parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
public static boolean isRTL(FeedEntry entry) {
String text = entry.getContent().getContent();
if (StringUtils.isBlank(text)) {
text = entry.getContent().getTitle();
}
if (StringUtils.isBlank(text)) {
return false;
}
text = Jsoup.parse(text).text();
if (StringUtils.isBlank(text)) {
return false;
}
Direction direction = BidiUtils.get().estimateDirection(text);
return direction == Direction.RTL;
}
public static String trimInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
StringBuilder sb = new StringBuilder();
boolean firstTagFound = false;
for (int i = 0; i < xml.length(); i++) {
char c = xml.charAt(i);
if (!firstTagFound) {
if (c == '<') {
firstTagFound = true;
} else {
continue;
}
}
if (c >= 32 || c == 9 || c == 10 || c == 13) {
if (!Character.isHighSurrogate(c) && !Character.isLowSurrogate(c)) {
sb.append(c);
}
}
}
return sb.toString();
}
/**
* When there was an error fetching the feed
*
*/
public static Date buildDisabledUntil(int errorCount) {
Date now = new Date();
int retriesBeforeDisable = 3;
if (errorCount >= retriesBeforeDisable) {
int disabledHours = errorCount - retriesBeforeDisable + 1;
disabledHours = Math.min(24 * 7, disabledHours);
return DateUtils.addHours(now, disabledHours);
}
return now;
}
/**
* When the feed was refreshed successfully
*/
public static Date buildDisabledUntil(Date publishedDate, Long averageEntryInterval, Date defaultRefreshInterval) {
Date now = new Date();
if (publishedDate == null) {
// feed with no entries, recheck in 24 hours
return DateUtils.addHours(now, 24);
} else if (publishedDate.before(DateUtils.addMonths(now, -1))) {
// older than a month, recheck in 24 hours
return DateUtils.addHours(now, 24);
} else if (publishedDate.before(DateUtils.addDays(now, -14))) {
// older than two weeks, recheck in 12 hours
return DateUtils.addHours(now, 12);
} else if (publishedDate.before(DateUtils.addDays(now, -7))) {
// older than a week, recheck in 6 hours
return DateUtils.addHours(now, 6);
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor
int factor = 2;
// not more than 6 hours
long date = Math.min(DateUtils.addHours(now, 6).getTime(), now.getTime() + averageEntryInterval / factor);
// not less than default refresh interval
date = Math.max(defaultRefreshInterval.getTime(), date);
return new Date(date);
} else {
// unknown case, recheck in 24 hours
return DateUtils.addHours(now, 24);
}
}
public static Long averageTimeBetweenEntries(List<FeedEntry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
List<Long> timestamps = getSortedTimestamps(entries);
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < timestamps.size() - 1; i++) {
long diff = Math.abs(timestamps.get(i) - timestamps.get(i + 1));
stats.addValue(diff);
}
return (long) stats.getMean();
}
public static List<Long> getSortedTimestamps(List<FeedEntry> entries) {
return entries.stream().map(t -> t.getUpdated().getTime()).sorted(Collections.reverseOrder()).collect(Collectors.toList());
}
public static String removeTrailingSlash(String url) {
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}
return url;
}
/**
*
* @param url
* the url of the entry
* @param feedLink
* the url of the feed as described in the feed
* @param feedUrl
* the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry
*/
public static String toAbsoluteUrl(String url, String feedLink, String feedUrl) {
url = StringUtils.trimToNull(StringUtils.normalizeSpace(url));
if (url == null || url.startsWith("http")) {
return url;
}
String baseUrl = (feedLink == null || isRelative(feedLink)) ? feedUrl : feedLink;
if (baseUrl == null) {
return url;
}
String result = null;
try {
result = new URL(new URL(baseUrl), url).toString();
} catch (MalformedURLException e) {
log.debug("could not parse url : " + e.getMessage(), e);
result = url;
}
return result;
}
public static boolean isRelative(final String url) {
// the regex means "start with 'scheme://'"
return url.startsWith("/") || url.startsWith("#") || !url.matches("^\\w+\\:\\/\\/.*");
}
public static String getFaviconUrl(FeedSubscription subscription, String publicUrl) {
return removeTrailingSlash(publicUrl) + "/rest/feed/favicon/" + subscription.getId();
}
public static String proxyImages(String content, String publicUrl) {
if (StringUtils.isBlank(content)) {
return content;
}
Document doc = Jsoup.parse(content);
Elements elements = doc.select("img");
for (Element element : elements) {
String href = element.attr("src");
if (href != null) {
String proxy = proxyImage(href, publicUrl);
element.attr("src", proxy);
}
}
return doc.body().html();
}
public static String proxyImage(String url, String publicUrl) {
if (StringUtils.isBlank(url)) {
return url;
}
return removeTrailingSlash(publicUrl) + "/rest/server/proxy?u=" + imageProxyEncoder(url);
}
public static String rot13(String msg) {
StringBuilder message = new StringBuilder();
for (char c : msg.toCharArray()) {
if (c >= 'a' && c <= 'm') {
c += 13;
} else if (c >= 'n' && c <= 'z') {
c -= 13;
} else if (c >= 'A' && c <= 'M') {
c += 13;
} else if (c >= 'N' && c <= 'Z') {
c -= 13;
}
message.append(c);
}
return message.toString();
}
public static String imageProxyEncoder(String url) {
return Base64.encodeBase64String(rot13(url).getBytes());
}
public static String imageProxyDecoder(String code) {
return rot13(new String(Base64.decodeBase64(code)));
}
public static void removeUnwantedFromSearch(List<Entry> entries, List<FeedEntryKeyword> keywords) {
Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
Entry entry = it.next();
boolean keep = true;
for (FeedEntryKeyword keyword : keywords) {
String title = entry.getTitle() == null ? null : Jsoup.parse(entry.getTitle()).text();
String content = entry.getContent() == null ? null : Jsoup.parse(entry.getContent()).text();
boolean condition = !StringUtils.containsIgnoreCase(content, keyword.getKeyword())
&& !StringUtils.containsIgnoreCase(title, keyword.getKeyword());
if (keyword.getMode() == Mode.EXCLUDE) {
condition = !condition;
}
if (condition) {
keep = false;
break;
}
}
if (!keep) {
it.remove();
}
}
}
}

23
commafeed-server/src/main/java/com/commafeed/backend/feed/FetchedFeed.java Normal file
View File

@@ -0,0 +1,23 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.List;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
public class FetchedFeed {
private Feed feed = new Feed();
private List<FeedEntry> entries = new ArrayList<>();
private String title;
private String urlAfterRedirect;
private long fetchDuration;
}

269
commafeed-server/src/main/java/com/commafeed/backend/feed/HtmlEntities.java Normal file
View File

@@ -0,0 +1,269 @@
package com.commafeed.backend.feed;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
public class HtmlEntities {
public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final String[] HTML_ENTITIES;
public static final String[] NUMERIC_ENTITIES;
static {
Map<String, String> map = new LinkedHashMap<>();
map.put("&Aacute;", "&#193;");
map.put("&aacute;", "&#225;");
map.put("&Acirc;", "&#194;");
map.put("&acirc;", "&#226;");
map.put("&acute;", "&#180;");
map.put("&AElig;", "&#198;");
map.put("&aelig;", "&#230;");
map.put("&Agrave;", "&#192;");
map.put("&agrave;", "&#224;");
map.put("&alefsym;", "&#8501;");
map.put("&Alpha;", "&#913;");
map.put("&alpha;", "&#945;");
map.put("&amp;", "&#38;");
map.put("&and;", "&#8743;");
map.put("&ang;", "&#8736;");
map.put("&Aring;", "&#197;");
map.put("&aring;", "&#229;");
map.put("&asymp;", "&#8776;");
map.put("&Atilde;", "&#195;");
map.put("&atilde;", "&#227;");
map.put("&Auml;", "&#196;");
map.put("&auml;", "&#228;");
map.put("&bdquo;", "&#8222;");
map.put("&Beta;", "&#914;");
map.put("&beta;", "&#946;");
map.put("&brvbar;", "&#166;");
map.put("&bull;", "&#8226;");
map.put("&cap;", "&#8745;");
map.put("&Ccedil;", "&#199;");
map.put("&ccedil;", "&#231;");
map.put("&cedil;", "&#184;");
map.put("&cent;", "&#162;");
map.put("&Chi;", "&#935;");
map.put("&chi;", "&#967;");
map.put("&circ;", "&#710;");
map.put("&clubs;", "&#9827;");
map.put("&cong;", "&#8773;");
map.put("&copy;", "&#169;");
map.put("&crarr;", "&#8629;");
map.put("&cup;", "&#8746;");
map.put("&curren;", "&#164;");
map.put("&dagger;", "&#8224;");
map.put("&Dagger;", "&#8225;");
map.put("&darr;", "&#8595;");
map.put("&dArr;", "&#8659;");
map.put("&deg;", "&#176;");
map.put("&Delta;", "&#916;");
map.put("&delta;", "&#948;");
map.put("&diams;", "&#9830;");
map.put("&divide;", "&#247;");
map.put("&Eacute;", "&#201;");
map.put("&eacute;", "&#233;");
map.put("&Ecirc;", "&#202;");
map.put("&ecirc;", "&#234;");
map.put("&Egrave;", "&#200;");
map.put("&egrave;", "&#232;");
map.put("&empty;", "&#8709;");
map.put("&emsp;", "&#8195;");
map.put("&ensp;", "&#8194;");
map.put("&Epsilon;", "&#917;");
map.put("&epsilon;", "&#949;");
map.put("&equiv;", "&#8801;");
map.put("&Eta;", "&#919;");
map.put("&eta;", "&#951;");
map.put("&ETH;", "&#208;");
map.put("&eth;", "&#240;");
map.put("&Euml;", "&#203;");
map.put("&euml;", "&#235;");
map.put("&euro;", "&#8364;");
map.put("&exist;", "&#8707;");
map.put("&fnof;", "&#402;");
map.put("&forall;", "&#8704;");
map.put("&frac12;", "&#189;");
map.put("&frac14;", "&#188;");
map.put("&frac34;", "&#190;");
map.put("&frasl;", "&#8260;");
map.put("&Gamma;", "&#915;");
map.put("&gamma;", "&#947;");
map.put("&ge;", "&#8805;");
map.put("&harr;", "&#8596;");
map.put("&hArr;", "&#8660;");
map.put("&hearts;", "&#9829;");
map.put("&hellip;", "&#8230;");
map.put("&Iacute;", "&#205;");
map.put("&iacute;", "&#237;");
map.put("&Icirc;", "&#206;");
map.put("&icirc;", "&#238;");
map.put("&iexcl;", "&#161;");
map.put("&Igrave;", "&#204;");
map.put("&igrave;", "&#236;");
map.put("&image;", "&#8465;");
map.put("&infin;", "&#8734;");
map.put("&int;", "&#8747;");
map.put("&Iota;", "&#921;");
map.put("&iota;", "&#953;");
map.put("&iquest;", "&#191;");
map.put("&isin;", "&#8712;");
map.put("&Iuml;", "&#207;");
map.put("&iuml;", "&#239;");
map.put("&Kappa;", "&#922;");
map.put("&kappa;", "&#954;");
map.put("&Lambda;", "&#923;");
map.put("&lambda;", "&#955;");
map.put("&lang;", "&#9001;");
map.put("&laquo;", "&#171;");
map.put("&larr;", "&#8592;");
map.put("&lArr;", "&#8656;");
map.put("&lceil;", "&#8968;");
map.put("&ldquo;", "&#8220;");
map.put("&le;", "&#8804;");
map.put("&lfloor;", "&#8970;");
map.put("&lowast;", "&#8727;");
map.put("&loz;", "&#9674;");
map.put("&lrm;", "&#8206;");
map.put("&lsaquo;", "&#8249;");
map.put("&lsquo;", "&#8216;");
map.put("&macr;", "&#175;");
map.put("&mdash;", "&#8212;");
map.put("&micro;", "&#181;");
map.put("&middot;", "&#183;");
map.put("&minus;", "&#8722;");
map.put("&Mu;", "&#924;");
map.put("&mu;", "&#956;");
map.put("&nabla;", "&#8711;");
map.put("&nbsp;", "&#160;");
map.put("&ndash;", "&#8211;");
map.put("&ne;", "&#8800;");
map.put("&ni;", "&#8715;");
map.put("&not;", "&#172;");
map.put("&notin;", "&#8713;");
map.put("&nsub;", "&#8836;");
map.put("&Ntilde;", "&#209;");
map.put("&ntilde;", "&#241;");
map.put("&Nu;", "&#925;");
map.put("&nu;", "&#957;");
map.put("&Oacute;", "&#211;");
map.put("&oacute;", "&#243;");
map.put("&Ocirc;", "&#212;");
map.put("&ocirc;", "&#244;");
map.put("&OElig;", "&#338;");
map.put("&oelig;", "&#339;");
map.put("&Ograve;", "&#210;");
map.put("&ograve;", "&#242;");
map.put("&oline;", "&#8254;");
map.put("&Omega;", "&#937;");
map.put("&omega;", "&#969;");
map.put("&Omicron;", "&#927;");
map.put("&omicron;", "&#959;");
map.put("&oplus;", "&#8853;");
map.put("&or;", "&#8744;");
map.put("&ordf;", "&#170;");
map.put("&ordm;", "&#186;");
map.put("&Oslash;", "&#216;");
map.put("&oslash;", "&#248;");
map.put("&Otilde;", "&#213;");
map.put("&otilde;", "&#245;");
map.put("&otimes;", "&#8855;");
map.put("&Ouml;", "&#214;");
map.put("&ouml;", "&#246;");
map.put("&para;", "&#182;");
map.put("&part;", "&#8706;");
map.put("&permil;", "&#8240;");
map.put("&perp;", "&#8869;");
map.put("&Phi;", "&#934;");
map.put("&phi;", "&#966;");
map.put("&Pi;", "&#928;");
map.put("&pi;", "&#960;");
map.put("&piv;", "&#982;");
map.put("&plusmn;", "&#177;");
map.put("&pound;", "&#163;");
map.put("&prime;", "&#8242;");
map.put("&Prime;", "&#8243;");
map.put("&prod;", "&#8719;");
map.put("&prop;", "&#8733;");
map.put("&Psi;", "&#936;");
map.put("&psi;", "&#968;");
map.put("&quot;", "&#34;");
map.put("&radic;", "&#8730;");
map.put("&rang;", "&#9002;");
map.put("&raquo;", "&#187;");
map.put("&rarr;", "&#8594;");
map.put("&rArr;", "&#8658;");
map.put("&rceil;", "&#8969;");
map.put("&rdquo;", "&#8221;");
map.put("&real;", "&#8476;");
map.put("&reg;", "&#174;");
map.put("&rfloor;", "&#8971;");
map.put("&Rho;", "&#929;");
map.put("&rho;", "&#961;");
map.put("&rlm;", "&#8207;");
map.put("&rsaquo;", "&#8250;");
map.put("&rsquo;", "&#8217;");
map.put("&sbquo;", "&#8218;");
map.put("&Scaron;", "&#352;");
map.put("&scaron;", "&#353;");
map.put("&sdot;", "&#8901;");
map.put("&sect;", "&#167;");
map.put("&shy;", "&#173;");
map.put("&Sigma;", "&#931;");
map.put("&sigma;", "&#963;");
map.put("&sigmaf;", "&#962;");
map.put("&sim;", "&#8764;");
map.put("&spades;", "&#9824;");
map.put("&sub;", "&#8834;");
map.put("&sube;", "&#8838;");
map.put("&sum;", "&#8721;");
map.put("&sup1;", "&#185;");
map.put("&sup2;", "&#178;");
map.put("&sup3;", "&#179;");
map.put("&sup;", "&#8835;");
map.put("&supe;", "&#8839;");
map.put("&szlig;", "&#223;");
map.put("&Tau;", "&#932;");
map.put("&tau;", "&#964;");
map.put("&there4;", "&#8756;");
map.put("&Theta;", "&#920;");
map.put("&theta;", "&#952;");
map.put("&thetasym;", "&#977;");
map.put("&thinsp;", "&#8201;");
map.put("&THORN;", "&#222;");
map.put("&thorn;", "&#254;");
map.put("&tilde;", "&#732;");
map.put("&times;", "&#215;");
map.put("&trade;", "&#8482;");
map.put("&Uacute;", "&#218;");
map.put("&uacute;", "&#250;");
map.put("&uarr;", "&#8593;");
map.put("&uArr;", "&#8657;");
map.put("&Ucirc;", "&#219;");
map.put("&ucirc;", "&#251;");
map.put("&Ugrave;", "&#217;");
map.put("&ugrave;", "&#249;");
map.put("&uml;", "&#168;");
map.put("&upsih;", "&#978;");
map.put("&Upsilon;", "&#933;");
map.put("&upsilon;", "&#965;");
map.put("&Uuml;", "&#220;");
map.put("&uuml;", "&#252;");
map.put("&weierp;", "&#8472;");
map.put("&Xi;", "&#926;");
map.put("&xi;", "&#958;");
map.put("&Yacute;", "&#221;");
map.put("&yacute;", "&#253;");
map.put("&yen;", "&#165;");
map.put("&yuml;", "&#255;");
map.put("&Yuml;", "&#376;");
map.put("&Zeta;", "&#918;");
map.put("&zeta;", "&#950;");
map.put("&zwj;", "&#8205;");
map.put("&zwnj;", "&#8204;");
HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map);
HTML_ENTITIES = map.keySet().toArray(new String[map.size()]);
NUMERIC_ENTITIES = map.values().toArray(new String[map.size()]);
}
}

33
commafeed-server/src/main/java/com/commafeed/backend/model/AbstractModel.java Normal file
View File

@@ -0,0 +1,33 @@
package com.commafeed.backend.model;
import java.io.Serializable;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.MappedSuperclass;
import javax.persistence.TableGenerator;
import lombok.Getter;
import lombok.Setter;
/**
* Abstract model for all entities, defining id and table generator
*
*/
@SuppressWarnings("serial")
@MappedSuperclass
@Getter
@Setter
public abstract class AbstractModel implements Serializable {
@Id
@GeneratedValue(strategy = GenerationType.TABLE, generator = "gen")
@TableGenerator(
name = "gen",
table = "hibernate_sequences",
pkColumnName = "sequence_name",
valueColumnName = "sequence_next_hi_value",
allocationSize = 1000)
private Long id;
}

129
commafeed-server/src/main/java/com/commafeed/backend/model/Feed.java Normal file
View File

@@ -0,0 +1,129 @@
package com.commafeed.backend.model;
import java.util.Date;
import java.util.Set;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDS")
@SuppressWarnings("serial")
@Getter
@Setter
public class Feed extends AbstractModel {
/**
* The url of the feed
*/
@Column(length = 2048, nullable = false)
private String url;
/**
* cache the url after potential http 30x redirects
*/
@Column(name = "url_after_redirect", length = 2048, nullable = false)
private String urlAfterRedirect;
@Column(length = 2048, nullable = false)
private String normalizedUrl;
@Column(length = 40, nullable = false)
private String normalizedUrlHash;
/**
* The url of the website, extracted from the feed
*/
@Column(length = 2048)
private String link;
/**
* Last time we tried to fetch the feed
*/
@Temporal(TemporalType.TIMESTAMP)
private Date lastUpdated;
/**
* Last publishedDate value in the feed
*/
@Temporal(TemporalType.TIMESTAMP)
private Date lastPublishedDate;
/**
* date of the last entry of the feed
*/
@Temporal(TemporalType.TIMESTAMP)
private Date lastEntryDate;
/**
* error message while retrieving the feed
*/
@Column(length = 1024)
private String message;
/**
* times we failed to retrieve the feed
*/
private int errorCount;
/**
* feed refresh is disabled until this date
*/
@Temporal(TemporalType.TIMESTAMP)
private Date disabledUntil;
/**
* http header returned by the feed
*/
@Column(length = 64)
private String lastModifiedHeader;
/**
* http header returned by the feed
*/
@Column(length = 255)
private String etagHeader;
/**
* average time between entries in the feed
*/
private Long averageEntryInterval;
/**
* last hash of the content of the feed xml
*/
@Column(length = 40)
private String lastContentHash;
/**
* detected hub for pubsubhubbub
*/
@Column(length = 2048)
private String pushHub;
/**
* detected topic for pubsubhubbub
*/
@Column(length = 2048)
private String pushTopic;
@Column(name = "push_topic_hash", length = 2048)
private String pushTopicHash;
/**
* last time we subscribed for that topic on that hub
*/
@Temporal(TemporalType.TIMESTAMP)
private Date pushLastPing;
@OneToMany(mappedBy = "feed")
private Set<FeedSubscription> subscriptions;
}

43
commafeed-server/src/main/java/com/commafeed/backend/model/FeedCategory.java Normal file
View File

@@ -0,0 +1,43 @@
package com.commafeed.backend.model;
import java.util.Set;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDCATEGORIES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedCategory extends AbstractModel {
@Column(length = 128, nullable = false)
private String name;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory parent;
@OneToMany(mappedBy = "parent")
private Set<FeedCategory> children;
@OneToMany(mappedBy = "category")
private Set<FeedSubscription> subscriptions;
private boolean collapsed;
private Integer position;
}

56
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntry.java Normal file
View File

@@ -0,0 +1,56 @@
package com.commafeed.backend.model;
import java.util.Date;
import java.util.Set;
import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.OneToOne;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRIES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntry extends AbstractModel {
@Column(length = 2048, nullable = false)
private String guid;
@Column(length = 40, nullable = false)
private String guidHash;
@ManyToOne(fetch = FetchType.LAZY)
private Feed feed;
@OneToOne(fetch = FetchType.LAZY, optional = false)
@JoinColumn(nullable = false, updatable = false)
private FeedEntryContent content;
@Column(length = 2048)
private String url;
@Temporal(TemporalType.TIMESTAMP)
private Date inserted;
@Temporal(TemporalType.TIMESTAMP)
private Date updated;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryTag> tags;
}

82
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryContent.java Normal file
View File

@@ -0,0 +1,82 @@
package com.commafeed.backend.model;
import java.util.Set;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Lob;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.hibernate.annotations.Type;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYCONTENTS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryContent extends AbstractModel {
@Column(length = 2048)
private String title;
@Column(length = 40)
private String titleHash;
@Lob
@Column(length = Integer.MAX_VALUE)
@Type(type = "org.hibernate.type.TextType")
private String content;
@Column(length = 40)
private String contentHash;
@Column(name = "author", length = 128)
private String author;
@Column(length = 2048)
private String enclosureUrl;
@Column(length = 255)
private String enclosureType;
@Lob
@Column(length = Integer.MAX_VALUE)
@Type(type = "org.hibernate.type.TextType")
private String mediaDescription;
@Column(length = 2048)
private String mediaThumbnailUrl;
private Integer mediaThumbnailWidth;
private Integer mediaThumbnailHeight;
@Column(length = 4096)
private String categories;
@OneToMany(mappedBy = "content")
private Set<FeedEntry> entries;
public boolean equivalentTo(FeedEntryContent c) {
if (c == null) {
return false;
}
return new EqualsBuilder().append(title, c.title)
.append(content, c.content)
.append(author, c.author)
.append(enclosureUrl, c.enclosureUrl)
.append(enclosureType, c.enclosureType)
.append(mediaDescription, c.mediaDescription)
.append(mediaThumbnailUrl, c.mediaThumbnailUrl)
.append(mediaThumbnailWidth, c.mediaThumbnailWidth)
.append(mediaThumbnailHeight, c.mediaThumbnailHeight)
.append(categories, c.categories)
.build();
}
}

71
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryStatus.java Normal file
View File

@@ -0,0 +1,71 @@
package com.commafeed.backend.model;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import javax.persistence.Transient;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYSTATUSES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryStatus extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedSubscription subscription;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedEntry entry;
@Column(name = "read_status")
private boolean read;
private boolean starred;
@Transient
private boolean markable;
@Transient
private List<FeedEntryTag> tags = new ArrayList<>();
/**
* Denormalization starts here
*/
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@Temporal(TemporalType.TIMESTAMP)
private Date entryInserted;
@Temporal(TemporalType.TIMESTAMP)
private Date entryUpdated;
public FeedEntryStatus() {
}
public FeedEntryStatus(User user, FeedSubscription subscription, FeedEntry entry) {
setUser(user);
setSubscription(subscription);
setEntry(entry);
setEntryInserted(entry.getInserted());
setEntryUpdated(entry.getUpdated());
}
}

40
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryTag.java Normal file
View File

@@ -0,0 +1,40 @@
package com.commafeed.backend.model;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYTAGS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryTag extends AbstractModel {
@JoinColumn(name = "user_id")
@ManyToOne(fetch = FetchType.LAZY)
private User user;
@JoinColumn(name = "entry_id")
@ManyToOne(fetch = FetchType.LAZY)
private FeedEntry entry;
@Column(name = "name", length = 40)
private String name;
public FeedEntryTag() {
}
public FeedEntryTag(User user, FeedEntry entry, String name) {
this.name = name;
this.entry = entry;
this.user = user;
}
}

46
commafeed-server/src/main/java/com/commafeed/backend/model/FeedSubscription.java Normal file
View File

@@ -0,0 +1,46 @@
package com.commafeed.backend.model;
import java.util.Set;
import javax.persistence.CascadeType;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.OneToMany;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDSUBSCRIPTIONS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedSubscription extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private Feed feed;
@Column(length = 128, nullable = false)
private String title;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory category;
@OneToMany(mappedBy = "subscription", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
private Integer position;
@Column(name = "filtering_expression", length = 4096)
private String filter;
}

29
commafeed-server/src/main/java/com/commafeed/backend/model/Models.java Normal file
View File

@@ -0,0 +1,29 @@
package com.commafeed.backend.model;
import org.hibernate.Hibernate;
import org.hibernate.HibernateException;
import org.hibernate.proxy.HibernateProxy;
import org.hibernate.proxy.LazyInitializer;
public class Models {
/**
* initialize a proxy
*/
public static void initialize(Object proxy) throws HibernateException {
Hibernate.initialize(proxy);
}
/**
* extract the id from the proxy without initializing it
*/
public static Long getId(AbstractModel model) {
if (model instanceof HibernateProxy) {
LazyInitializer lazyInitializer = ((HibernateProxy) model).getHibernateLazyInitializer();
if (lazyInitializer.isUninitialized()) {
return (Long) lazyInitializer.getIdentifier();
}
}
return model.getId();
}
}

65
commafeed-server/src/main/java/com/commafeed/backend/model/User.java Normal file
View File

@@ -0,0 +1,65 @@
package com.commafeed.backend.model;
import java.util.Date;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import javax.persistence.Temporal;
import javax.persistence.TemporalType;
import org.apache.commons.lang3.time.DateUtils;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERS")
@SuppressWarnings("serial")
@Getter
@Setter
public class User extends AbstractModel {
@Column(length = 32, nullable = false, unique = true)
private String name;
@Column(length = 255, unique = true)
private String email;
@Column(length = 256, nullable = false)
private byte[] password;
@Column(length = 40, unique = true)
private String apiKey;
@Column(length = 8, nullable = false)
private byte[] salt;
@Column(nullable = false)
private boolean disabled;
@Temporal(TemporalType.TIMESTAMP)
private Date lastLogin;
@Temporal(TemporalType.TIMESTAMP)
private Date created;
@Column(length = 40)
private String recoverPasswordToken;
@Temporal(TemporalType.TIMESTAMP)
private Date recoverPasswordTokenDate;
@Column(name = "last_full_refresh")
@Temporal(TemporalType.TIMESTAMP)
private Date lastFullRefresh;
public boolean shouldRefreshFeedsAt(Date when) {
return lastFullRefresh == null || lastFullRefreshMoreThan30MinutesBefore(when);
}
private boolean lastFullRefreshMoreThan30MinutesBefore(Date when) {
return lastFullRefresh.before(DateUtils.addMinutes(when, -30));
}
}

43
commafeed-server/src/main/java/com/commafeed/backend/model/UserRole.java Normal file
View File

@@ -0,0 +1,43 @@
package com.commafeed.backend.model;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.OneToOne;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERROLES")
@SuppressWarnings("serial")
@Getter
@Setter
public class UserRole extends AbstractModel {
public enum Role {
USER, ADMIN
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false)
private User user;
@Column(name = "roleName", nullable = false)
@Enumerated(EnumType.STRING)
private Role role;
public UserRole() {
}
public UserRole(User user, Role role) {
this.user = user;
this.role = role;
}
}

79
commafeed-server/src/main/java/com/commafeed/backend/model/UserSettings.java Normal file
View File

@@ -0,0 +1,79 @@
package com.commafeed.backend.model;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.FetchType;
import javax.persistence.JoinColumn;
import javax.persistence.Lob;
import javax.persistence.OneToOne;
import javax.persistence.Table;
import org.hibernate.annotations.Type;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERSETTINGS")
@SuppressWarnings("serial")
@Getter
@Setter
public class UserSettings extends AbstractModel {
public enum ReadingMode {
all, unread
}
public enum ReadingOrder {
asc, desc
}
public enum ViewMode {
title, expanded
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false, unique = true)
private User user;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingMode readingMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingOrder readingOrder;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ViewMode viewMode;
@Column(name = "user_lang", length = 4)
private String language;
private boolean showRead;
private boolean scrollMarks;
@Column(length = 32)
private String theme;
@Lob
@Column(length = Integer.MAX_VALUE)
@Type(type = "org.hibernate.type.TextType")
private String customCss;
@Column(name = "scroll_speed")
private int scrollSpeed;
private boolean email;
private boolean gmail;
private boolean facebook;
private boolean twitter;
private boolean tumblr;
private boolean pocket;
private boolean instapaper;
private boolean buffer;
}

89
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLExporter.java Normal file
View File

@@ -0,0 +1,89 @@
package com.commafeed.backend.opml;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.rometools.opml.feed.opml.Attribute;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class OPMLExporter {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
public Opml export(User user) {
Opml opml = new Opml();
opml.setFeedType("opml_1.1");
opml.setTitle(String.format("%s subscriptions in CommaFeed", user.getName()));
opml.setCreated(new Date());
List<FeedCategory> categories = feedCategoryDAO.findAll(user);
Collections.sort(categories,
(e1, e2) -> ObjectUtils.firstNonNull(e1.getPosition(), 0) - ObjectUtils.firstNonNull(e2.getPosition(), 0));
List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user);
Collections.sort(subscriptions,
(e1, e2) -> ObjectUtils.firstNonNull(e1.getPosition(), 0) - ObjectUtils.firstNonNull(e2.getPosition(), 0));
// export root categories
for (FeedCategory cat : categories.stream().filter(c -> c.getParent() == null).collect(Collectors.toList())) {
opml.getOutlines().add(buildCategoryOutline(cat, categories, subscriptions));
}
// export root subscriptions
for (FeedSubscription sub : subscriptions.stream().filter(s -> s.getCategory() == null).collect(Collectors.toList())) {
opml.getOutlines().add(buildSubscriptionOutline(sub));
}
return opml;
}
private Outline buildCategoryOutline(FeedCategory cat, List<FeedCategory> categories, List<FeedSubscription> subscriptions) {
Outline outline = new Outline();
outline.setText(cat.getName());
outline.setTitle(cat.getName());
for (FeedCategory child : categories.stream()
.filter(c -> c.getParent() != null && c.getParent().getId().equals(cat.getId()))
.collect(Collectors.toList())) {
outline.getChildren().add(buildCategoryOutline(child, categories, subscriptions));
}
for (FeedSubscription sub : subscriptions.stream()
.filter(s -> s.getCategory() != null && s.getCategory().getId().equals(cat.getId()))
.collect(Collectors.toList())) {
outline.getChildren().add(buildSubscriptionOutline(sub));
}
return outline;
}
private Outline buildSubscriptionOutline(FeedSubscription sub) {
Outline outline = new Outline();
outline.setText(sub.getTitle());
outline.setTitle(sub.getTitle());
outline.setType("rss");
outline.getAttributes().add(new Attribute("xmlUrl", sub.getFeed().getUrl()));
if (sub.getFeed().getLink() != null) {
outline.getAttributes().add(new Attribute("htmlUrl", sub.getFeed().getLink()));
}
return outline;
}
}

93
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLImporter.java Normal file
View File

@@ -0,0 +1,93 @@
package com.commafeed.backend.opml;
import java.io.StringReader;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedSubscriptionService;
import com.commafeed.backend.service.FeedSubscriptionService.FeedSubscriptionException;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import com.rometools.rome.io.WireFeedInput;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class OPMLImporter {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionService feedSubscriptionService;
private final CacheService cache;
public void importOpml(User user, String xml) {
xml = xml.substring(xml.indexOf('<'));
WireFeedInput input = new WireFeedInput();
try {
Opml feed = (Opml) input.build(new StringReader(xml));
List<Outline> outlines = feed.getOutlines();
for (int i = 0; i < outlines.size(); i++) {
handleOutline(user, outlines.get(i), null, i);
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
private void handleOutline(User user, Outline outline, FeedCategory parent, int position) {
List<Outline> children = outline.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
FeedCategory category = feedCategoryDAO.findByName(user, name, parent);
if (category == null) {
if (StringUtils.isBlank(name)) {
name = "Unnamed category";
}
category = new FeedCategory();
category.setName(name);
category.setParent(parent);
category.setUser(user);
category.setPosition(position);
feedCategoryDAO.saveOrUpdate(category);
}
for (int i = 0; i < children.size(); i++) {
handleOutline(user, children.get(i), category, i);
}
} else {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
if (StringUtils.isBlank(name)) {
name = "Unnamed subscription";
}
// make sure we continue with the import process even if a feed failed
try {
feedSubscriptionService.subscribe(user, outline.getXmlUrl(), name, parent, position);
} catch (FeedSubscriptionException e) {
throw e;
} catch (Exception e) {
log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage());
}
}
cache.invalidateUserRootCategory(user);
}
}

23
commafeed-server/src/main/java/com/commafeed/backend/rome/OPML11Generator.java Normal file
View File

@@ -0,0 +1,23 @@
package com.commafeed.backend.rome;
import org.jdom2.Element;
import com.rometools.opml.feed.opml.Opml;
/**
* Add missing title to the generated OPML
*
*/
public class OPML11Generator extends com.rometools.opml.io.impl.OPML10Generator {
public OPML11Generator() {
super("opml_1.1");
}
@Override
protected Element generateHead(Opml opml) {
Element head = new Element("head");
addNotNullSimpleElement(head, "title", opml.getTitle());
return head;
}
}

39
commafeed-server/src/main/java/com/commafeed/backend/rome/OPML11Parser.java Normal file
View File

@@ -0,0 +1,39 @@
package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Document;
import org.jdom2.Element;
import com.rometools.opml.io.impl.OPML10Parser;
import com.rometools.rome.feed.WireFeed;
import com.rometools.rome.io.FeedException;
/**
* Support for OPML 1.1 parsing
*
*/
public class OPML11Parser extends OPML10Parser {
public OPML11Parser() {
super("opml_1.1");
}
@Override
public boolean isMyType(Document document) {
Element e = document.getRootElement();
if (e.getName().equals("opml")) {
return true;
}
return false;
}
@Override
public WireFeed parse(Document document, boolean validate, Locale locale) throws IllegalArgumentException, FeedException {
document.getRootElement().getChildren().add(new Element("head"));
return super.parse(document, validate, locale);
}
}

27
commafeed-server/src/main/java/com/commafeed/backend/rome/RSS090DescriptionConverter.java Normal file
View File

@@ -0,0 +1,27 @@
package com.commafeed.backend.rome;
import com.rometools.rome.feed.rss.Description;
import com.rometools.rome.feed.rss.Item;
import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.impl.ConverterForRSS090;
/**
* Support description tag for RSS09
*
*/
public class RSS090DescriptionConverter extends ConverterForRSS090 {
@Override
protected SyndEntry createSyndEntry(Item item, boolean preserveWireItem) {
SyndEntry entry = super.createSyndEntry(item, preserveWireItem);
Description desc = item.getDescription();
if (desc != null) {
SyndContentImpl syndDesc = new SyndContentImpl();
syndDesc.setValue(desc.getValue());
entry.setDescription(syndDesc);
}
return entry;
}
}

29
commafeed-server/src/main/java/com/commafeed/backend/rome/RSS090DescriptionParser.java Normal file
View File

@@ -0,0 +1,29 @@
package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Element;
import com.rometools.rome.feed.rss.Description;
import com.rometools.rome.feed.rss.Item;
import com.rometools.rome.io.impl.RSS090Parser;
/**
* Support description tag for RSS09
*
*/
public class RSS090DescriptionParser extends RSS090Parser {
@Override
protected Item parseItem(Element rssRoot, Element eItem, Locale locale) {
Item item = super.parseItem(rssRoot, eItem, locale);
Element e = eItem.getChild("description", getRSSNamespace());
if (e != null) {
Description desc = new Description();
desc.setValue(e.getText());
item.setDescription(desc);
}
return item;
}
}

49
commafeed-server/src/main/java/com/commafeed/backend/rome/RSSRDF10Parser.java Normal file
View File

@@ -0,0 +1,49 @@
package com.commafeed.backend.rome;
import java.util.List;
import org.apache.commons.collections4.CollectionUtils;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.Namespace;
import com.google.common.collect.Lists;
import com.rometools.rome.io.impl.RSS10Parser;
public class RSSRDF10Parser extends RSS10Parser {
private static final String RSS_URI = "http://purl.org/rss/1.0/";
private static final Namespace RSS_NS = Namespace.getNamespace(RSS_URI);
public RSSRDF10Parser() {
super("rss_1.0", RSS_NS);
}
@Override
public boolean isMyType(Document document) {
boolean ok = false;
Element rssRoot = document.getRootElement();
Namespace defaultNS = rssRoot.getNamespace();
List<Namespace> additionalNSs = Lists.newArrayList(rssRoot.getAdditionalNamespaces());
List<Element> children = rssRoot.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
Element child = children.get(0);
additionalNSs.add(child.getNamespace());
additionalNSs.addAll(child.getAdditionalNamespaces());
}
ok = defaultNS != null && defaultNS.equals(getRDFNamespace());
if (ok) {
if (additionalNSs == null) {
ok = false;
} else {
ok = false;
for (int i = 0; !ok && i < additionalNSs.size(); i++) {
ok = getRSSNamespace().equals(additionalNSs.get(i));
}
}
}
return ok;
}
}

113
commafeed-server/src/main/java/com/commafeed/backend/service/DatabaseCleaningService.java Normal file
View File

@@ -0,0 +1,113 @@
package com.commafeed.backend.service;
import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryDAO.FeedCapacity;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Contains utility methods for cleaning the database
*
*/
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class DatabaseCleaningService {
private static final int BATCH_SIZE = 100;
private final SessionFactory sessionFactory;
private final FeedDAO feedDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
public long cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
int deleted = 0;
long entriesTotal = 0;
do {
List<Feed> feeds = UnitOfWork.call(sessionFactory, () -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) {
int entriesDeleted = 0;
do {
entriesDeleted = UnitOfWork.call(sessionFactory, () -> feedEntryDAO.delete(feed.getId(), BATCH_SIZE));
entriesTotal += entriesDeleted;
log.info("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
}
deleted = UnitOfWork.call(sessionFactory, () -> feedDAO.delete(feeds));
total += deleted;
log.info("removed {} feeds without subscriptions", total);
} while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total);
return total;
}
public long cleanContentsWithoutEntries() {
log.info("cleaning contents without entries");
long total = 0;
int deleted = 0;
do {
deleted = UnitOfWork.call(sessionFactory, () -> feedEntryContentDAO.deleteWithoutEntries(BATCH_SIZE));
total += deleted;
log.info("removed {} contents without entries", total);
} while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total);
return total;
}
public long cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
long total = 0;
while (true) {
List<FeedCapacity> feeds = UnitOfWork.call(sessionFactory,
() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, BATCH_SIZE));
if (feeds.isEmpty()) {
break;
}
for (final FeedCapacity feed : feeds) {
long remaining = feed.getCapacity() - maxFeedCapacity;
do {
final long rem = remaining;
int deleted = UnitOfWork.call(sessionFactory,
() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(BATCH_SIZE, rem)));
total += deleted;
remaining -= deleted;
log.info("removed {} entries for feeds exceeding capacity", total);
} while (remaining > 0);
}
}
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
return total;
}
public long cleanStatusesOlderThan(final Date olderThan) {
log.info("cleaning old read statuses");
long total = 0;
int deleted = 0;
do {
deleted = UnitOfWork.call(sessionFactory,
() -> feedEntryStatusDAO.delete(feedEntryStatusDAO.getOldStatuses(olderThan, BATCH_SIZE)));
total += deleted;
log.info("removed {} old read statuses", total);
} while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
return total;
}
}

48
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentService.java Normal file
View File

@@ -0,0 +1,48 @@
package com.commafeed.backend.service;
import java.util.List;
import java.util.Optional;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedEntryContent;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedEntryContentService {
private final FeedEntryContentDAO feedEntryContentDAO;
/**
* this is NOT thread-safe
*/
public FeedEntryContent findOrCreate(FeedEntryContent content, String baseUrl) {
content.setAuthor(FeedUtils.truncate(FeedUtils.handleContent(content.getAuthor(), baseUrl, true), 128));
content.setTitle(FeedUtils.truncate(FeedUtils.handleContent(content.getTitle(), baseUrl, true), 2048));
content.setContent(FeedUtils.handleContent(content.getContent(), baseUrl, false));
content.setMediaDescription(FeedUtils.handleContent(content.getMediaDescription(), baseUrl, false));
String contentHash = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getContent()));
content.setContentHash(contentHash);
String titleHash = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getTitle()));
content.setTitleHash(titleHash);
List<FeedEntryContent> existing = feedEntryContentDAO.findExisting(contentHash, titleHash);
Optional<FeedEntryContent> equivalentContent = existing.stream().filter(c -> content.equivalentTo(c)).findFirst();
if (equivalentContent.isPresent()) {
return equivalentContent.get();
}
feedEntryContentDAO.saveOrUpdate(content);
return content;
}
}

122
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryFilteringService.java Normal file
View File

@@ -0,0 +1,122 @@
package com.commafeed.backend.service;
import java.time.Year;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.jexl2.JexlContext;
import org.apache.commons.jexl2.JexlEngine;
import org.apache.commons.jexl2.JexlException;
import org.apache.commons.jexl2.JexlInfo;
import org.apache.commons.jexl2.MapContext;
import org.apache.commons.jexl2.Script;
import org.apache.commons.jexl2.introspection.JexlMethod;
import org.apache.commons.jexl2.introspection.JexlPropertyGet;
import org.apache.commons.jexl2.introspection.Uberspect;
import org.apache.commons.jexl2.introspection.UberspectImpl;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import com.commafeed.backend.model.FeedEntry;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedEntryFilteringService {
private static final JexlEngine ENGINE = initEngine();
private final ExecutorService executor = Executors.newCachedThreadPool();
private static JexlEngine initEngine() {
// classloader that prevents object creation
ClassLoader cl = new ClassLoader() {
@Override
protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
return null;
}
};
// uberspect that prevents access to .class and .getClass()
Uberspect uberspect = new UberspectImpl(LogFactory.getLog(JexlEngine.class)) {
@Override
public JexlPropertyGet getPropertyGet(Object obj, Object identifier, JexlInfo info) {
if ("class".equals(identifier)) {
return null;
}
return super.getPropertyGet(obj, identifier, info);
}
@Override
public JexlMethod getMethod(Object obj, String method, Object[] args, JexlInfo info) {
if ("getClass".equals(method)) {
return null;
}
return super.getMethod(obj, method, args, info);
}
};
JexlEngine engine = new JexlEngine(uberspect, null, null, null);
engine.setStrict(true);
engine.setClassLoader(cl);
return engine;
}
public boolean filterMatchesEntry(String filter, FeedEntry entry) throws FeedEntryFilterException {
if (StringUtils.isBlank(filter)) {
return true;
}
Script script = null;
try {
script = ENGINE.createScript(filter);
} catch (JexlException e) {
throw new FeedEntryFilterException("Exception while parsing expression " + filter, e);
}
JexlContext context = new MapContext();
context.set("title", entry.getContent().getTitle() == null ? "" : Jsoup.parse(entry.getContent().getTitle()).text().toLowerCase());
context.set("author", entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor().toLowerCase());
context.set("content",
entry.getContent().getContent() == null ? "" : Jsoup.parse(entry.getContent().getContent()).text().toLowerCase());
context.set("url", entry.getUrl() == null ? "" : entry.getUrl().toLowerCase());
context.set("categories", entry.getContent().getCategories() == null ? "" : entry.getContent().getCategories().toLowerCase());
context.set("year", Year.now().getValue());
Callable<Object> callable = script.callable(context);
Future<Object> future = executor.submit(callable);
Object result = null;
try {
result = future.get(500, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e);
} catch (ExecutionException e) {
throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}
try {
return (boolean) result;
} catch (ClassCastException e) {
throw new FeedEntryFilterException(e.getMessage(), e);
}
}
@SuppressWarnings("serial")
public static class FeedEntryFilterException extends Exception {
public FeedEntryFilterException(String message, Throwable t) {
super(message, t);
}
}
}

95
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java Normal file
View File

@@ -0,0 +1,95 @@
package com.commafeed.backend.service;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedEntryService {
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final CacheService cache;
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, entry.getFeed());
if (sub == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
if (status.isMarkable()) {
status.setRead(read);
feedEntryStatusDAO.saveOrUpdate(status);
cache.invalidateUnreadCount(sub);
cache.invalidateUserRootCategory(user);
}
}
public void starEntry(User user, Long entryId, Long subscriptionId, boolean starred) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subscriptionId);
if (sub == null) {
return;
}
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
status.setStarred(starred);
feedEntryStatusDAO.saveOrUpdate(status);
}
public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Date olderThan, List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null,
false, false, null);
markList(statuses, olderThan);
cache.invalidateUnreadCount(subscriptions.toArray(new FeedSubscription[0]));
cache.invalidateUserRootCategory(user);
}
public void markStarredEntries(User user, Date olderThan) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, -1, -1, null, false);
markList(statuses, olderThan);
}
private void markList(List<FeedEntryStatus> statuses, Date olderThan) {
List<FeedEntryStatus> list = new ArrayList<>();
for (FeedEntryStatus status : statuses) {
if (!status.isRead()) {
Date entryDate = status.getEntry().getUpdated();
if (olderThan == null || entryDate == null || olderThan.after(entryDate)) {
status.setRead(true);
list.add(status);
}
}
}
feedEntryStatusDAO.saveOrUpdate(list);
}
}

44
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryTagService.java Normal file
View File

@@ -0,0 +1,44 @@
package com.commafeed.backend.service;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryTagDAO;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedEntryTagService {
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO;
public void updateTags(User user, Long entryId, List<String> tagNames) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry);
Set<String> existingTagNames = existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
List<FeedEntryTag> addList = tagNames.stream()
.filter(name -> !existingTagNames.contains(name))
.map(name -> new FeedEntryTag(user, entry, name))
.collect(Collectors.toList());
List<FeedEntryTag> removeList = existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).collect(Collectors.toList());
feedEntryTagDAO.saveOrUpdate(addList);
feedEntryTagDAO.delete(removeList);
}
}

68
commafeed-server/src/main/java/com/commafeed/backend/service/FeedService.java Normal file
View File

@@ -0,0 +1,68 @@
package com.commafeed.backend.service;
import java.io.IOException;
import java.util.Date;
import java.util.Set;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
import com.commafeed.backend.favicon.AbstractFaviconFetcher.Favicon;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
@Singleton
public class FeedService {
private final FeedDAO feedDAO;
private final Set<AbstractFaviconFetcher> faviconFetchers;
private Favicon defaultFavicon;
@Inject
public FeedService(FeedDAO feedDAO, Set<AbstractFaviconFetcher> faviconFetchers) {
this.feedDAO = feedDAO;
this.faviconFetchers = faviconFetchers;
try {
defaultFavicon = new Favicon(IOUtils.toByteArray(getClass().getResource("/images/default_favicon.gif")), "image/gif");
} catch (IOException e) {
throw new RuntimeException("could not load default favicon", e);
}
}
public synchronized Feed findOrCreate(String url) {
String normalized = FeedUtils.normalizeURL(url);
Feed feed = feedDAO.findByUrl(normalized);
if (feed == null) {
feed = new Feed();
feed.setUrl(url);
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
feed.setDisabledUntil(new Date(0));
feedDAO.saveOrUpdate(feed);
}
return feed;
}
public Favicon fetchFavicon(Feed feed) {
Favicon icon = null;
for (AbstractFaviconFetcher faviconFetcher : faviconFetchers) {
icon = faviconFetcher.fetch(feed);
if (icon != null) {
break;
}
}
if (icon == null) {
icon = defaultFavicon;
}
return icon;
}
}

124
commafeed-server/src/main/java/com/commafeed/backend/service/FeedSubscriptionService.java Normal file
View File

@@ -0,0 +1,124 @@
package com.commafeed.backend.service;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedQueues;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.UnreadCount;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedSubscriptionService {
private final FeedDAO feedDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService;
private final FeedQueues queues;
private final CacheService cache;
private final CommaFeedConfiguration config;
public Feed subscribe(User user, String url, String title) {
return subscribe(user, url, title, null, 0);
}
public Feed subscribe(User user, String url, String title, FeedCategory parent) {
return subscribe(user, url, title, parent, 0);
}
public Feed subscribe(User user, String url, String title, FeedCategory category, int position) {
final String pubUrl = config.getApplicationSettings().getPublicUrl();
if (StringUtils.isBlank(pubUrl)) {
throw new FeedSubscriptionException("Public URL of this CommaFeed instance is not set");
}
if (url.startsWith(pubUrl)) {
throw new FeedSubscriptionException("Could not subscribe to a feed from this CommaFeed instance");
}
Feed feed = feedService.findOrCreate(url);
// upgrade feed to https if it was using http
if (FeedUtils.isHttp(feed.getUrl()) && FeedUtils.isHttps(url)) {
feed.setUrl(url);
feedDAO.saveOrUpdate(feed);
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
if (sub == null) {
sub = new FeedSubscription();
sub.setFeed(feed);
sub.setUser(user);
}
sub.setCategory(category);
sub.setPosition(position);
sub.setTitle(FeedUtils.truncate(title, 128));
feedSubscriptionDAO.saveOrUpdate(sub);
queues.add(feed, false);
cache.invalidateUserRootCategory(user);
return feed;
}
public boolean unsubscribe(User user, Long subId) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subId);
if (sub != null) {
feedSubscriptionDAO.delete(sub);
cache.invalidateUserRootCategory(user);
return true;
} else {
return false;
}
}
public void refreshAll(User user) {
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Feed feed = sub.getFeed();
queues.add(feed, true);
}
}
public Map<Long, UnreadCount> getUnreadCount(User user) {
return feedSubscriptionDAO.findAll(user).stream().collect(Collectors.toMap(FeedSubscription::getId, s -> getUnreadCount(user, s)));
}
private UnreadCount getUnreadCount(User user, FeedSubscription sub) {
UnreadCount count = cache.getUnreadCount(sub);
if (count == null) {
log.debug("unread count cache miss for {}", Models.getId(sub));
count = feedEntryStatusDAO.getUnreadCount(user, sub);
cache.setUnreadCount(sub, count);
}
return count;
}
@SuppressWarnings("serial")
public static class FeedSubscriptionException extends RuntimeException {
private FeedSubscriptionException(String msg) {
super(msg);
}
}
}

67
commafeed-server/src/main/java/com/commafeed/backend/service/FeedUpdateService.java Normal file
View File

@@ -0,0 +1,67 @@
package com.commafeed.backend.service;
import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedUpdateService {
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
/**
* this is NOT thread-safe
*/
public boolean addEntry(Feed feed, FeedEntry entry, List<FeedSubscription> subscriptions) {
Long existing = feedEntryDAO.findExisting(entry.getGuid(), feed);
if (existing != null) {
return false;
}
FeedEntryContent content = feedEntryContentService.findOrCreate(entry.getContent(), feed.getLink());
entry.setGuidHash(DigestUtils.sha1Hex(entry.getGuid()));
entry.setContent(content);
entry.setInserted(new Date());
entry.setFeed(feed);
feedEntryDAO.saveOrUpdate(entry);
// if filter does not match the entry, mark it as read
for (FeedSubscription sub : subscriptions) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.saveOrUpdate(status);
}
}
return true;
}
}

64
commafeed-server/src/main/java/com/commafeed/backend/service/MailService.java Normal file
View File

@@ -0,0 +1,64 @@
package com.commafeed.backend.service;
import java.util.Optional;
import java.util.Properties;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.mail.Authenticator;
import javax.mail.Message;
import javax.mail.PasswordAuthentication;
import javax.mail.Session;
import javax.mail.Transport;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeMessage;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.ApplicationSettings;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
/**
* Mailing service
*
*/
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class MailService {
private final CommaFeedConfiguration config;
public void sendMail(User user, String subject, String content) throws Exception {
ApplicationSettings settings = config.getApplicationSettings();
final String username = settings.getSmtpUserName();
final String password = settings.getSmtpPassword();
final String fromAddress = Optional.ofNullable(settings.getSmtpFromAddress()).orElse(settings.getSmtpUserName());
String dest = user.getEmail();
Properties props = new Properties();
props.put("mail.smtp.auth", "true");
props.put("mail.smtp.starttls.enable", "" + settings.isSmtpTls());
props.put("mail.smtp.host", settings.getSmtpHost());
props.put("mail.smtp.port", "" + settings.getSmtpPort());
Session session = Session.getInstance(props, new Authenticator() {
@Override
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(username, password);
}
});
Message message = new MimeMessage(session);
message.setFrom(new InternetAddress(fromAddress, "CommaFeed"));
message.setRecipients(Message.RecipientType.TO, InternetAddress.parse(dest));
message.setSubject("CommaFeed - " + subject);
message.setContent(content, "text/html; charset=utf-8");
Transport.send(message);
}
}

95
commafeed-server/src/main/java/com/commafeed/backend/service/PasswordEncryptionService.java Normal file
View File

@@ -0,0 +1,95 @@
package com.commafeed.backend.service;
import java.io.Serializable;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.spec.KeySpec;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
// taken from http://www.javacodegeeks.com/2012/05/secure-password-storage-donts-dos-and.html
@SuppressWarnings("serial")
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class PasswordEncryptionService implements Serializable {
public boolean authenticate(String attemptedPassword, byte[] encryptedPassword, byte[] salt) {
if (StringUtils.isBlank(attemptedPassword)) {
return false;
}
// Encrypt the clear-text password using the same salt that was used to
// encrypt the original password
byte[] encryptedAttemptedPassword = null;
try {
encryptedAttemptedPassword = getEncryptedPassword(attemptedPassword, salt);
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
if (encryptedAttemptedPassword == null) {
return false;
}
// Authentication succeeds if encrypted password that the user entered
// is equal to the stored hash
return MessageDigest.isEqual(encryptedPassword, encryptedAttemptedPassword);
}
public byte[] getEncryptedPassword(String password, byte[] salt) {
// PBKDF2 with SHA-1 as the hashing algorithm. Note that the NIST
// specifically names SHA-1 as an acceptable hashing algorithm for
// PBKDF2
String algorithm = "PBKDF2WithHmacSHA1";
// SHA-1 generates 160 bit hashes, so that's what makes sense here
int derivedKeyLength = 160;
// Pick an iteration count that works for you. The NIST recommends at
// least 1,000 iterations:
// http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf
// iOS 4.x reportedly uses 10,000:
// http://blog.crackpassword.com/2010/09/smartphone-forensics-cracking-blackberry-backup-passwords/
int iterations = 20000;
KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, derivedKeyLength);
byte[] bytes = null;
try {
SecretKeyFactory f = SecretKeyFactory.getInstance(algorithm);
SecretKey key = f.generateSecret(spec);
bytes = key.getEncoded();
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
return bytes;
}
public byte[] generateSalt() {
// VERY important to use SecureRandom instead of just Random
byte[] salt = null;
try {
SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
// Generate a 8 byte (64 bit) salt as recommended by RSA PKCS5
salt = new byte[8];
random.nextBytes(salt);
} catch (NoSuchAlgorithmException e) {
// should never happen
log.error(e.getMessage(), e);
}
return salt;
}
}

91
commafeed-server/src/main/java/com/commafeed/backend/service/PubSubService.java Normal file
View File

@@ -0,0 +1,91 @@
package com.commafeed.backend.service;
import java.util.ArrayList;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.ws.rs.core.MediaType;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpHeaders;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.feed.FeedQueues;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.frontend.resource.PubSubHubbubCallbackREST;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Sends push subscription requests. Callback is handled by {@link PubSubHubbubCallbackREST}
*
*/
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class PubSubService {
private final CommaFeedConfiguration config;
private final FeedQueues queues;
public void subscribe(Feed feed) {
String hub = feed.getPushHub();
String topic = feed.getPushTopic();
String publicUrl = FeedUtils.removeTrailingSlash(config.getApplicationSettings().getPublicUrl());
log.debug("sending new pubsub subscription to {} for {}", hub, topic);
HttpPost post = new HttpPost(hub);
List<NameValuePair> nvp = new ArrayList<>();
nvp.add(new BasicNameValuePair("hub.callback", publicUrl + "/rest/push/callback"));
nvp.add(new BasicNameValuePair("hub.topic", topic));
nvp.add(new BasicNameValuePair("hub.mode", "subscribe"));
nvp.add(new BasicNameValuePair("hub.verify", "async"));
nvp.add(new BasicNameValuePair("hub.secret", ""));
nvp.add(new BasicNameValuePair("hub.verify_token", ""));
nvp.add(new BasicNameValuePair("hub.lease_seconds", ""));
post.setHeader(HttpHeaders.USER_AGENT, "CommaFeed");
post.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_FORM_URLENCODED);
CloseableHttpClient client = HttpGetter.newClient(20000);
CloseableHttpResponse response = null;
try {
post.setEntity(new UrlEncodedFormEntity(nvp));
response = client.execute(post);
int code = response.getStatusLine().getStatusCode();
if (code != 204 && code != 202 && code != 200) {
String message = EntityUtils.toString(response.getEntity());
String pushpressError = " is value is not allowed. You may only subscribe to";
if (code == 400 && StringUtils.contains(message, pushpressError)) {
String[] tokens = message.split(" ");
feed.setPushTopic(tokens[tokens.length - 1]);
queues.giveBack(feed);
log.debug("handled pushpress subfeed {} : {}", topic, feed.getPushTopic());
} else {
throw new Exception(
"Unexpected response code: " + code + " " + response.getStatusLine().getReasonPhrase() + " - " + message);
}
}
log.debug("subscribed to {} for {}", hub, topic);
} catch (Exception e) {
log.error("Could not subscribe to {} for {} : " + e.getMessage(), hub, topic);
} finally {
IOUtils.closeQuietly(response);
IOUtils.closeQuietly(client);
}
}
}

89
commafeed-server/src/main/java/com/commafeed/backend/service/StartupService.java Normal file
View File

@@ -0,0 +1,89 @@
package com.commafeed.backend.service;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.Session;
import org.hibernate.SessionFactory;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import io.dropwizard.lifecycle.Managed;
import liquibase.Liquibase;
import liquibase.database.Database;
import liquibase.database.DatabaseFactory;
import liquibase.database.core.PostgresDatabase;
import liquibase.database.jvm.JdbcConnection;
import liquibase.resource.ClassLoaderResourceAccessor;
import liquibase.resource.ResourceAccessor;
import liquibase.structure.DatabaseObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class StartupService implements Managed {
private final SessionFactory sessionFactory;
private final UserDAO userDAO;
private final UserService userService;
private final CommaFeedConfiguration config;
@Override
public void start() throws Exception {
updateSchema();
long count = UnitOfWork.call(sessionFactory, () -> userDAO.count());
if (count == 0) {
UnitOfWork.run(sessionFactory, this::initialData);
}
}
private void updateSchema() {
Session session = sessionFactory.openSession();
session.doWork(connection -> {
try {
JdbcConnection jdbcConnection = new JdbcConnection(connection);
Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(jdbcConnection);
if (database instanceof PostgresDatabase) {
database = new PostgresDatabase() {
@Override
public String escapeObjectName(String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}
};
database.setConnection(jdbcConnection);
}
ResourceAccessor accessor = new ClassLoaderResourceAccessor(Thread.currentThread().getContextClassLoader());
try (Liquibase liq = new Liquibase("migrations.xml", accessor, database)) {
liq.update("prod");
}
} catch (Exception e) {
throw new RuntimeException(e);
}
});
session.close();
}
private void initialData() {
log.info("Populating database with default values");
try {
userService.createAdminUser();
if (config.getApplicationSettings().getCreateDemoAccount()) {
userService.createDemoUser();
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
@Override
public void stop() throws Exception {
}
}

145
commafeed-server/src/main/java/com/commafeed/backend/service/UserService.java Normal file
View File

@@ -0,0 +1,145 @@
package com.commafeed.backend.service;
import java.util.Arrays;
import java.util.Collection;
import java.util.Date;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.dao.UserRoleDAO;
import com.commafeed.backend.dao.UserSettingsDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.internal.PostLoginActivities;
import com.google.common.base.Preconditions;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class UserService {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UserDAO userDAO;
private final UserRoleDAO userRoleDAO;
private final UserSettingsDAO userSettingsDAO;
private final PasswordEncryptionService encryptionService;
private final CommaFeedConfiguration config;
private final PostLoginActivities postLoginActivities;
/**
* try to log in with given credentials
*/
public Optional<User> login(String nameOrEmail, String password) {
if (nameOrEmail == null || password == null) {
return Optional.empty();
}
User user = userDAO.findByName(nameOrEmail);
if (user == null) {
user = userDAO.findByEmail(nameOrEmail);
}
if (user != null && !user.isDisabled()) {
boolean authenticated = encryptionService.authenticate(password, user.getPassword(), user.getSalt());
if (authenticated) {
performPostLoginActivities(user);
return Optional.of(user);
}
}
return Optional.empty();
}
/**
* try to log in with given api key
*/
public Optional<User> login(String apiKey) {
if (apiKey == null) {
return Optional.empty();
}
User user = userDAO.findByApiKey(apiKey);
if (user != null && !user.isDisabled()) {
performPostLoginActivities(user);
return Optional.of(user);
}
return Optional.empty();
}
/**
* should triggers after successful login
*/
public void performPostLoginActivities(User user) {
postLoginActivities.executeFor(user);
}
public User register(String name, String password, String email, Collection<Role> roles) {
return register(name, password, email, roles, false);
}
public User register(String name, String password, String email, Collection<Role> roles, boolean forceRegistration) {
if (!forceRegistration) {
Preconditions.checkState(config.getApplicationSettings().getAllowRegistrations(),
"Registrations are closed on this CommaFeed instance");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");
if (StringUtils.isNotBlank(email)) {
Preconditions.checkArgument(userDAO.findByEmail(email) == null, "Email already taken");
}
User user = new User();
byte[] salt = encryptionService.generateSalt();
user.setName(name);
user.setEmail(email);
user.setCreated(new Date());
user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.saveOrUpdate(user);
for (Role role : roles) {
userRoleDAO.saveOrUpdate(new UserRole(user, role));
}
return user;
}
public void createAdminUser() {
register(CommaFeedApplication.USERNAME_ADMIN, "admin", "admin@commafeed.com", Arrays.asList(Role.ADMIN, Role.USER), true);
}
public void createDemoUser() {
register(CommaFeedApplication.USERNAME_DEMO, "demo", "demo@commafeed.com", Arrays.asList(Role.USER), true);
}
public void unregister(User user) {
userSettingsDAO.delete(userSettingsDAO.findByUser(user));
userRoleDAO.delete(userRoleDAO.findAll(user));
feedSubscriptionDAO.delete(feedSubscriptionDAO.findAll(user));
feedCategoryDAO.delete(feedCategoryDAO.findAll(user));
userDAO.delete(user);
}
public String generateApiKey(User user) {
byte[] key = encryptionService.getEncryptedPassword(UUID.randomUUID().toString(), user.getSalt());
return DigestUtils.sha1Hex(key);
}
public Set<Role> getRoles(User user) {
return userRoleDAO.findRoles(user);
}
}

46
commafeed-server/src/main/java/com/commafeed/backend/service/internal/PostLoginActivities.java Normal file
View File

@@ -0,0 +1,46 @@
package com.commafeed.backend.service.internal;
import java.util.Date;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.time.DateUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedSubscriptionService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class PostLoginActivities {
private final UserDAO userDAO;
private final FeedSubscriptionService feedSubscriptionService;
private final CommaFeedConfiguration config;
public void executeFor(User user) {
Date lastLogin = user.getLastLogin();
Date now = new Date();
boolean saveUser = false;
// only update lastLogin field every hour in order to not
// invalidate the cache every time someone logs in
if (lastLogin == null || lastLogin.before(DateUtils.addHours(now, -1))) {
user.setLastLogin(now);
saveUser = true;
}
if (config.getApplicationSettings().getHeavyLoad() && user.shouldRefreshFeedsAt(now)) {
feedSubscriptionService.refreshAll(user);
user.setLastFullRefresh(now);
saveUser = true;
}
if (saveUser) {
userDAO.saveOrUpdate(user);
}
}
}

64
commafeed-server/src/main/java/com/commafeed/backend/task/DemoAccountCleanupTask.java Normal file
View File

@@ -0,0 +1,64 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.hibernate.SessionFactory;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.UserService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
@Slf4j
public class DemoAccountCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final SessionFactory sessionFactory;
private final UserDAO userDAO;
private final UserService userService;
@Override
protected void run() {
if (!config.getApplicationSettings().getCreateDemoAccount()) {
return;
}
log.info("recreating demo user account");
UnitOfWork.run(sessionFactory, () -> {
User demoUser = userDAO.findByName(CommaFeedApplication.USERNAME_DEMO);
if (demoUser == null) {
return;
}
userService.unregister(demoUser);
userService.createDemoUser();
});
}
@Override
protected long getInitialDelay() {
return 1;
}
@Override
protected long getPeriod() {
return getTimeUnit().convert(24, TimeUnit.HOURS);
}
@Override
protected TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

43
commafeed-server/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java Normal file
View File

@@ -0,0 +1,43 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class OldEntriesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
int maxFeedCapacity = config.getApplicationSettings().getMaxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
}
}
@Override
public long getInitialDelay() {
return 5;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

44
commafeed-server/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java Normal file
View File

@@ -0,0 +1,44 @@
package com.commafeed.backend.task;
import java.util.Date;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class OldStatusesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
Date threshold = config.getApplicationSettings().getUnreadThreshold();
if (threshold != null) {
cleaner.cleanStatusesOlderThan(threshold);
}
}
@Override
public long getInitialDelay() {
return 10;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

38
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedContentsCleanupTask.java Normal file
View File

@@ -0,0 +1,38 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.commafeed.backend.service.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class OrphanedContentsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanContentsWithoutEntries();
}
@Override
public long getInitialDelay() {
return 20;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

38
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedFeedsCleanupTask.java Normal file
View File

@@ -0,0 +1,38 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Singleton;
import com.commafeed.backend.service.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class OrphanedFeedsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanFeedsWithoutSubscriptions();
}
@Override
public long getInitialDelay() {
return 15;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

33
commafeed-server/src/main/java/com/commafeed/backend/task/ScheduledTask.java Normal file
View File

@@ -0,0 +1,33 @@
package com.commafeed.backend.task;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class ScheduledTask {
protected abstract void run();
protected abstract long getInitialDelay();
protected abstract long getPeriod();
protected abstract TimeUnit getTimeUnit();
public void register(ScheduledExecutorService executor) {
Runnable runnable = new Runnable() {
@Override
public void run() {
try {
ScheduledTask.this.run();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
};
log.info("registering task {} for execution every {} {}, starting in {} {}", getClass().getSimpleName(), getPeriod(), getTimeUnit(),
getInitialDelay(), getTimeUnit());
executor.scheduleWithFixedDelay(runnable, getInitialDelay(), getPeriod(), getTimeUnit());
}
}

10
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/FeedURLProvider.java Normal file
View File

@@ -0,0 +1,10 @@
package com.commafeed.backend.urlprovider;
/**
* Tries to find a feed url given the url and page content
*/
public interface FeedURLProvider {
String get(String url, String urlContent);
}

28
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/InPageReferenceFeedURLProvider.java Normal file
View File

@@ -0,0 +1,28 @@
package com.commafeed.backend.urlprovider;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
public class InPageReferenceFeedURLProvider implements FeedURLProvider {
@Override
public String get(String url, String urlContent) {
String foundUrl = null;
Document doc = Jsoup.parse(urlContent, url);
String root = doc.children().get(0).tagName();
if ("html".equals(root)) {
Elements atom = doc.select("link[type=application/atom+xml]");
Elements rss = doc.select("link[type=application/rss+xml]");
if (!atom.isEmpty()) {
foundUrl = atom.get(0).attr("abs:href");
} else if (!rss.isEmpty()) {
foundUrl = rss.get(0).attr("abs:href");
}
}
return foundUrl;
}
}

22
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/YoutubeFeedURLProvider.java Normal file
View File

@@ -0,0 +1,22 @@
package com.commafeed.backend.urlprovider;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Workaround for Youtube channels
*
* converts the channel URL https://www.youtube.com/channel/CHANNEL_ID to the valid feed URL
* https://www.youtube.com/feeds/videos.xml?channel_id=CHANNEL_ID
*/
public class YoutubeFeedURLProvider implements FeedURLProvider {
private static final Pattern REGEXP = Pattern.compile("(.*\\byoutube\\.com)\\/channel\\/([^\\/]+)", Pattern.CASE_INSENSITIVE);
@Override
public String get(String url, String urlContent) {
Matcher matcher = REGEXP.matcher(url);
return matcher.find() ? matcher.group(1) + "/feeds/videos.xml?channel_id=" + matcher.group(2) : null;
}
}

59
commafeed-server/src/main/java/com/commafeed/frontend/auth/PasswordConstraintValidator.java Normal file
View File

@@ -0,0 +1,59 @@
package com.commafeed.frontend.auth;
import java.util.List;
import javax.validation.ConstraintValidator;
import javax.validation.ConstraintValidatorContext;
import org.apache.commons.lang3.StringUtils;
import org.passay.CharacterRule;
import org.passay.EnglishCharacterData;
import org.passay.LengthRule;
import org.passay.PasswordData;
import org.passay.PasswordValidator;
import org.passay.RuleResult;
import org.passay.WhitespaceRule;
public class PasswordConstraintValidator implements ConstraintValidator<ValidPassword, String> {
@Override
public void initialize(ValidPassword constraintAnnotation) {
// nothing to do
}
@Override
public boolean isValid(String value, ConstraintValidatorContext context) {
if (StringUtils.isBlank(value)) {
return true;
}
PasswordValidator validator = buildPasswordValidator();
RuleResult result = validator.validate(new PasswordData(value));
if (result.isValid()) {
return true;
}
List<String> messages = validator.getMessages(result);
String message = String.join(System.lineSeparator(), messages);
context.buildConstraintViolationWithTemplate(message).addConstraintViolation().disableDefaultConstraintViolation();
return false;
}
private PasswordValidator buildPasswordValidator() {
return new PasswordValidator(
// length
new LengthRule(8, 128),
// 1 uppercase char
new CharacterRule(EnglishCharacterData.UpperCase, 1),
// 1 lowercase char
new CharacterRule(EnglishCharacterData.LowerCase, 1),
// 1 digit
new CharacterRule(EnglishCharacterData.Digit, 1),
// 1 special char
new CharacterRule(EnglishCharacterData.Special, 1),
// no whitespace
new WhitespaceRule());
}
}

22
commafeed-server/src/main/java/com/commafeed/frontend/auth/SecurityCheck.java Normal file
View File

@@ -0,0 +1,22 @@
package com.commafeed.frontend.auth;
import java.lang.annotation.ElementType;
import java.lang.annotation.Inherited;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import com.commafeed.backend.model.UserRole.Role;
@Inherited
@Target({ ElementType.PARAMETER })
@Retention(RetentionPolicy.RUNTIME)
public @interface SecurityCheck {
/**
* Roles needed.
*/
Role value() default Role.USER;
boolean apiKeyAllowed() default false;
}

99
commafeed-server/src/main/java/com/commafeed/frontend/auth/SecurityCheckFactory.java Normal file
View File

@@ -0,0 +1,99 @@
package com.commafeed.frontend.auth;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.glassfish.jersey.server.ContainerRequest;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.UserService;
import com.commafeed.frontend.session.SessionHelper;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class SecurityCheckFactory implements Function<ContainerRequest, User> {
private static final String PREFIX = "Basic";
private final UserService userService;
private final HttpServletRequest request;
private final Role role;
private final boolean apiKeyAllowed;
@Override
public User apply(ContainerRequest req) {
Optional<User> user = apiKeyLogin();
if (!user.isPresent()) {
user = basicAuthenticationLogin();
}
if (!user.isPresent()) {
user = cookieSessionLogin(new SessionHelper(request));
}
if (user.isPresent()) {
Set<Role> roles = userService.getRoles(user.get());
if (roles.contains(role)) {
return user.get();
} else {
throw new WebApplicationException(Response.status(Response.Status.FORBIDDEN)
.entity("You don't have the required role to access this resource.")
.type(MediaType.TEXT_PLAIN_TYPE)
.build());
}
} else {
throw new WebApplicationException(Response.status(Response.Status.UNAUTHORIZED)
.entity("Credentials are required to access this resource.")
.type(MediaType.TEXT_PLAIN_TYPE)
.build());
}
}
Optional<User> cookieSessionLogin(SessionHelper sessionHelper) {
Optional<User> loggedInUser = sessionHelper.getLoggedInUser();
if (loggedInUser.isPresent()) {
userService.performPostLoginActivities(loggedInUser.get());
}
return loggedInUser;
}
private Optional<User> basicAuthenticationLogin() {
String header = request.getHeader(HttpHeaders.AUTHORIZATION);
if (header != null) {
int space = header.indexOf(' ');
if (space > 0) {
String method = header.substring(0, space);
if (PREFIX.equalsIgnoreCase(method)) {
byte[] decodedBytes = Base64.getDecoder().decode(header.substring(space + 1));
String decoded = new String(decodedBytes, StandardCharsets.ISO_8859_1);
int i = decoded.indexOf(':');
if (i > 0) {
String username = decoded.substring(0, i);
String password = decoded.substring(i + 1);
return userService.login(username, password);
}
}
}
}
return Optional.empty();
}
private Optional<User> apiKeyLogin() {
String apiKey = request.getParameter("apiKey");
if (apiKey != null && apiKeyAllowed) {
return userService.login(apiKey);
}
return Optional.empty();
}
}

63
commafeed-server/src/main/java/com/commafeed/frontend/auth/SecurityCheckFactoryProvider.java Normal file
View File

@@ -0,0 +1,63 @@
package com.commafeed.frontend.auth;
import java.util.function.Function;
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.servlet.http.HttpServletRequest;
import org.glassfish.hk2.utilities.binding.AbstractBinder;
import org.glassfish.jersey.server.ContainerRequest;
import org.glassfish.jersey.server.internal.inject.AbstractValueParamProvider;
import org.glassfish.jersey.server.internal.inject.MultivaluedParameterExtractorProvider;
import org.glassfish.jersey.server.model.Parameter;
import org.glassfish.jersey.server.spi.internal.ValueParamProvider;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.UserService;
import lombok.RequiredArgsConstructor;
@Singleton
public class SecurityCheckFactoryProvider extends AbstractValueParamProvider {
private UserService userService;
private HttpServletRequest request;
@Inject
public SecurityCheckFactoryProvider(final MultivaluedParameterExtractorProvider extractorProvider, UserService userService,
HttpServletRequest request) {
super(() -> extractorProvider, Parameter.Source.UNKNOWN);
this.userService = userService;
this.request = request;
}
@Override
protected Function<ContainerRequest, ?> createValueProvider(Parameter parameter) {
final Class<?> classType = parameter.getRawType();
SecurityCheck securityCheck = parameter.getAnnotation(SecurityCheck.class);
if (securityCheck == null) {
return null;
}
if (!classType.isAssignableFrom(User.class)) {
return null;
}
return new SecurityCheckFactory(userService, request, securityCheck.value(), securityCheck.apiKeyAllowed());
}
@RequiredArgsConstructor
public static class Binder extends AbstractBinder {
private final UserService userService;
@Override
protected void configure() {
bind(SecurityCheckFactoryProvider.class).to(ValueParamProvider.class).in(Singleton.class);
bind(userService).to(UserService.class);
}
}
}

23
commafeed-server/src/main/java/com/commafeed/frontend/auth/ValidPassword.java Normal file
View File

@@ -0,0 +1,23 @@
package com.commafeed.frontend.auth;
import java.lang.annotation.Documented;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import javax.validation.Constraint;
import javax.validation.Payload;
@Documented
@Constraint(validatedBy = PasswordConstraintValidator.class)
@Target({ ElementType.TYPE, ElementType.FIELD, ElementType.ANNOTATION_TYPE })
@Retention(RetentionPolicy.RUNTIME)
public @interface ValidPassword {
String message() default "Invalid Password";
Class<?>[] groups() default {};
Class<? extends Payload>[] payload() default {};
}

36
commafeed-server/src/main/java/com/commafeed/frontend/model/Category.java Normal file
View File

@@ -0,0 +1,36 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Entry details")
@Data
public class Category implements Serializable {
@ApiModelProperty(value = "category id", required = true)
private String id;
@ApiModelProperty(value = "parent category id")
private String parentId;
@ApiModelProperty(value = "category id", required = true)
private String name;
@ApiModelProperty(value = "category children categories", required = true)
private List<Category> children = new ArrayList<>();
@ApiModelProperty(value = "category feeds", required = true)
private List<Subscription> feeds = new ArrayList<>();
@ApiModelProperty(value = "wether the category is expanded or collapsed", required = true)
private boolean expanded;
@ApiModelProperty(value = "position of the category in the list", required = true)
private Integer position;
}

48
commafeed-server/src/main/java/com/commafeed/frontend/model/Entries.java Normal file
View File

@@ -0,0 +1,48 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "List of entries with some metadata")
@Data
public class Entries implements Serializable {
@ApiModelProperty(value = "name of the feed or the category requested", required = true)
private String name;
@ApiModelProperty(value = "error or warning message")
private String message;
@ApiModelProperty(value = "times the server tried to refresh the feed and failed", required = true)
private int errorCount;
@ApiModelProperty(value = "URL of the website, extracted from the feed", required = true)
private String feedLink;
@ApiModelProperty(value = "list generation timestamp", required = true)
private long timestamp;
@ApiModelProperty(value = "if the query has more elements", required = true)
private boolean hasMore;
@ApiModelProperty(value = "the requested offset")
private int offset;
@ApiModelProperty(value = "the requested limit")
private int limit;
@ApiModelProperty(value = "list of entries", required = true)
private List<Entry> entries = new ArrayList<>();
@ApiModelProperty(
value = "if true, the unread flag was ignored in the request, all entries are returned regardless of their read status",
required = true)
private boolean ignoredReadStatus;
}

175
commafeed-server/src/main/java/com/commafeed/frontend/model/Entry.java Normal file
View File

@@ -0,0 +1,175 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEnclosureImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndEntryImpl;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Entry details")
@Data
public class Entry implements Serializable {
@ApiModelProperty(value = "entry id", required = true)
private String id;
@ApiModelProperty(value = "entry guid", required = true)
private String guid;
@ApiModelProperty(value = "entry title", required = true)
private String title;
@ApiModelProperty(value = "entry content", required = true)
private String content;
@ApiModelProperty(value = "comma-separated list of categories")
private String categories;
@ApiModelProperty(value = "wether entry content and title are rtl", required = true)
private boolean rtl;
@ApiModelProperty(value = "entry author")
private String author;
@ApiModelProperty(value = "entry enclosure url, if any")
private String enclosureUrl;
@ApiModelProperty(value = "entry enclosure mime type, if any")
private String enclosureType;
@ApiModelProperty(value = "entry media description, if any")
private String mediaDescription;
@ApiModelProperty(value = "entry media thumbnail url, if any")
private String mediaThumbnailUrl;
@ApiModelProperty(value = "entry media thumbnail width, if any")
private Integer mediaThumbnailWidth;
@ApiModelProperty(value = "entry media thumbnail height, if any")
private Integer mediaThumbnailHeight;
@ApiModelProperty(value = "entry publication date", dataType = "number", required = true)
private Date date;
@ApiModelProperty(value = "entry insertion date in the database", dataType = "number", required = true)
private Date insertedDate;
@ApiModelProperty(value = "feed id", required = true)
private String feedId;
@ApiModelProperty(value = "feed name", required = true)
private String feedName;
@ApiModelProperty(value = "this entry's feed url", required = true)
private String feedUrl;
@ApiModelProperty(value = "this entry's website url", required = true)
private String feedLink;
@ApiModelProperty(value = "The favicon url to use for this feed", required = true)
private String iconUrl;
@ApiModelProperty(value = "entry url", required = true)
private String url;
@ApiModelProperty(value = "read status", required = true)
private boolean read;
@ApiModelProperty(value = "starred status", required = true)
private boolean starred;
@ApiModelProperty(value = "wether the entry is still markable (old entry statuses are discarded)", required = true)
private boolean markable;
@ApiModelProperty(value = "tags", required = true)
private List<String> tags;
public static Entry build(FeedEntryStatus status, String publicUrl, boolean proxyImages) {
Entry entry = new Entry();
FeedEntry feedEntry = status.getEntry();
FeedSubscription sub = status.getSubscription();
FeedEntryContent content = feedEntry.getContent();
entry.setId(String.valueOf(feedEntry.getId()));
entry.setGuid(feedEntry.getGuid());
entry.setRead(status.isRead());
entry.setStarred(status.isStarred());
entry.setMarkable(status.isMarkable());
entry.setDate(feedEntry.getUpdated());
entry.setInsertedDate(feedEntry.getInserted());
entry.setUrl(feedEntry.getUrl());
entry.setFeedName(sub.getTitle());
entry.setFeedId(String.valueOf(sub.getId()));
entry.setFeedUrl(sub.getFeed().getUrl());
entry.setFeedLink(sub.getFeed().getLink());
entry.setIconUrl(FeedUtils.getFaviconUrl(sub, publicUrl));
entry.setTags(status.getTags().stream().map(FeedEntryTag::getName).collect(Collectors.toList()));
if (content != null) {
entry.setRtl(FeedUtils.isRTL(feedEntry));
entry.setTitle(content.getTitle());
entry.setContent(proxyImages ? FeedUtils.proxyImages(content.getContent(), publicUrl) : content.getContent());
entry.setAuthor(content.getAuthor());
entry.setEnclosureType(content.getEnclosureType());
entry.setEnclosureUrl(proxyImages && StringUtils.contains(content.getEnclosureType(), "image")
? FeedUtils.proxyImage(content.getEnclosureUrl(), publicUrl)
: content.getEnclosureUrl());
entry.setMediaDescription(content.getMediaDescription());
entry.setMediaThumbnailUrl(
proxyImages ? FeedUtils.proxyImage(content.getMediaThumbnailUrl(), publicUrl) : content.getMediaThumbnailUrl());
entry.setMediaThumbnailWidth(content.getMediaThumbnailWidth());
entry.setMediaThumbnailHeight(content.getMediaThumbnailHeight());
entry.setCategories(content.getCategories());
}
return entry;
}
public SyndEntry asRss() {
SyndEntry entry = new SyndEntryImpl();
entry.setUri(getGuid());
entry.setTitle(getTitle());
entry.setAuthor(getAuthor());
SyndContentImpl content = new SyndContentImpl();
content.setValue(getContent());
entry.setContents(Arrays.<SyndContent> asList(content));
if (getEnclosureUrl() != null) {
SyndEnclosureImpl enclosure = new SyndEnclosureImpl();
enclosure.setType(getEnclosureType());
enclosure.setUrl(getEnclosureUrl());
entry.setEnclosures(Arrays.<SyndEnclosure> asList(enclosure));
}
entry.setLink(getUrl());
entry.setPublishedDate(getDate());
return entry;
}
}

20
commafeed-server/src/main/java/com/commafeed/frontend/model/FeedInfo.java Normal file
View File

@@ -0,0 +1,20 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Feed details")
@Data
public class FeedInfo implements Serializable {
@ApiModelProperty(value = "url", required = true)
private String url;
@ApiModelProperty(value = "title", required = true)
private String title;
}

32
commafeed-server/src/main/java/com/commafeed/frontend/model/ServerInfo.java Normal file
View File

@@ -0,0 +1,32 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Server infos")
@Data
public class ServerInfo implements Serializable {
@ApiModelProperty
private String announcement;
@ApiModelProperty(required = true)
private String version;
@ApiModelProperty(required = true)
private String gitCommit;
@ApiModelProperty(required = true)
private boolean allowRegistrations;
@ApiModelProperty
private String googleAnalyticsCode;
@ApiModelProperty(required = true)
private boolean smtpEnabled;
}

64
commafeed-server/src/main/java/com/commafeed/frontend/model/Settings.java Normal file
View File

@@ -0,0 +1,64 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "User settings")
@Data
public class Settings implements Serializable {
@ApiModelProperty(value = "user's preferred language, english if none", required = true)
private String language;
@ApiModelProperty(value = "user reads all entries or unread entries only", allowableValues = "all,unread", required = true)
private String readingMode;
@ApiModelProperty(value = "user reads entries in ascending or descending order", allowableValues = "asc,desc", required = true)
private String readingOrder;
@ApiModelProperty(value = "user viewing mode, either title-only or expande view", allowableValues = "title,expanded", required = true)
private String viewMode;
@ApiModelProperty(value = "user wants category and feeds with no unread entries shown", required = true)
private boolean showRead;
@ApiModelProperty(value = "In expanded view, scroll through entries mark them as read", required = true)
private boolean scrollMarks;
@ApiModelProperty(value = "user's selected theme")
private String theme;
@ApiModelProperty(value = "user's custom css for the website")
private String customCss;
@ApiModelProperty(value = "user's preferred scroll speed when navigating between entries", required = true)
private int scrollSpeed;
@ApiModelProperty(required = true)
private boolean email;
@ApiModelProperty(required = true)
private boolean gmail;
@ApiModelProperty(required = true)
private boolean facebook;
@ApiModelProperty(required = true)
private boolean twitter;
@ApiModelProperty(required = true)
private boolean tumblr;
@ApiModelProperty(required = true)
private boolean pocket;
@ApiModelProperty(required = true)
private boolean instapaper;
@ApiModelProperty(required = true)
private boolean buffer;
}

87
commafeed-server/src/main/java/com/commafeed/frontend/model/Subscription.java Normal file
View File

@@ -0,0 +1,87 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.Date;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "User information")
@Data
public class Subscription implements Serializable {
@ApiModelProperty(value = "subscription id", required = true)
private Long id;
@ApiModelProperty(value = "subscription name", required = true)
private String name;
@ApiModelProperty(value = "error message while fetching the feed", required = true)
private String message;
@ApiModelProperty(value = "error count", required = true)
private int errorCount;
@ApiModelProperty(value = "last time the feed was refreshed", dataType = "number", required = true)
private Date lastRefresh;
@ApiModelProperty(
value = "next time the feed refresh is planned, null if refresh is already queued",
dataType = "number",
required = true)
private Date nextRefresh;
@ApiModelProperty(value = "this subscription's feed url", required = true)
private String feedUrl;
@ApiModelProperty(value = "this subscription's website url", required = true)
private String feedLink;
@ApiModelProperty(value = "The favicon url to use for this feed", required = true)
private String iconUrl;
@ApiModelProperty(value = "unread count", required = true)
private long unread;
@ApiModelProperty(value = "category id")
private String categoryId;
@ApiModelProperty("position of the subscription's in the list")
private Integer position;
@ApiModelProperty(value = "date of the newest item", dataType = "number")
private Date newestItemTime;
@ApiModelProperty(value = "JEXL string evaluated on new entries to mark them as read if they do not match")
private String filter;
public static Subscription build(FeedSubscription subscription, String publicUrl, UnreadCount unreadCount) {
Date now = new Date();
FeedCategory category = subscription.getCategory();
Feed feed = subscription.getFeed();
Subscription sub = new Subscription();
sub.setId(subscription.getId());
sub.setName(subscription.getTitle());
sub.setPosition(subscription.getPosition());
sub.setMessage(feed.getMessage());
sub.setErrorCount(feed.getErrorCount());
sub.setFeedUrl(feed.getUrl());
sub.setFeedLink(feed.getLink());
sub.setIconUrl(FeedUtils.getFaviconUrl(subscription, publicUrl));
sub.setLastRefresh(feed.getLastUpdated());
sub.setNextRefresh((feed.getDisabledUntil() != null && feed.getDisabledUntil().before(now)) ? null : feed.getDisabledUntil());
sub.setUnread(unreadCount.getUnreadCount());
sub.setNewestItemTime(unreadCount.getNewestItemTime());
sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));
sub.setFilter(subscription.getFilter());
return sub;
}
}

33
commafeed-server/src/main/java/com/commafeed/frontend/model/UnreadCount.java Normal file
View File

@@ -0,0 +1,33 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.Date;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Unread count")
@Data
public class UnreadCount implements Serializable {
@ApiModelProperty
private long feedId;
@ApiModelProperty
private long unreadCount;
@ApiModelProperty(dataType = "number")
private Date newestItemTime;
public UnreadCount() {
}
public UnreadCount(long feedId, long unreadCount, Date newestItemTime) {
this.feedId = feedId;
this.unreadCount = unreadCount;
this.newestItemTime = newestItemTime;
}
}

42
commafeed-server/src/main/java/com/commafeed/frontend/model/UserModel.java Normal file
View File

@@ -0,0 +1,42 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.Date;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "User information")
@Data
public class UserModel implements Serializable {
@ApiModelProperty(value = "user id", required = true)
private Long id;
@ApiModelProperty(value = "user name", required = true)
private String name;
@ApiModelProperty("user email, if any")
private String email;
@ApiModelProperty("api key")
private String apiKey;
@ApiModelProperty(value = "user password, never returned by the api")
private String password;
@ApiModelProperty(value = "account status", required = true)
private boolean enabled;
@ApiModelProperty(value = "account creation date", dataType = "number", required = true)
private Date created;
@ApiModelProperty(value = "last login date", dataType = "number")
private Date lastLogin;
@ApiModelProperty(value = "user is admin", required = true)
private boolean admin;
}

26
commafeed-server/src/main/java/com/commafeed/frontend/model/request/AddCategoryRequest.java Normal file
View File

@@ -0,0 +1,26 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.Size;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Add Category Request")
@Data
public class AddCategoryRequest implements Serializable {
@ApiModelProperty(value = "name", required = true)
@NotEmpty
@Size(max = 128)
private String name;
@ApiModelProperty(value = "parent category id, if any")
@Size(max = 128)
private String parentId;
}

30
commafeed-server/src/main/java/com/commafeed/frontend/model/request/CategoryModificationRequest.java Normal file
View File

@@ -0,0 +1,30 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import javax.validation.constraints.Size;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Category modification request")
@Data
public class CategoryModificationRequest implements Serializable {
@ApiModelProperty(value = "id", required = true)
private Long id;
@ApiModelProperty(value = "new name, null if not changed")
@Size(max = 128)
private String name;
@ApiModelProperty(value = "new parent category id")
@Size(max = 128)
private String parentId;
@ApiModelProperty(value = "new display position, null if not changed")
private Integer position;
}

20
commafeed-server/src/main/java/com/commafeed/frontend/model/request/CollapseRequest.java Normal file
View File

@@ -0,0 +1,20 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Mark Request")
@Data
public class CollapseRequest implements Serializable {
@ApiModelProperty(value = "category id", required = true)
private Long id;
@ApiModelProperty(value = "collapse", required = true)
private boolean collapse;
}

22
commafeed-server/src/main/java/com/commafeed/frontend/model/request/FeedInfoRequest.java Normal file
View File

@@ -0,0 +1,22 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.Size;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Feed information request")
@Data
public class FeedInfoRequest implements Serializable {
@ApiModelProperty(value = "feed url", required = true)
@NotEmpty
@Size(max = 4096)
private String url;
}

34
commafeed-server/src/main/java/com/commafeed/frontend/model/request/FeedModificationRequest.java Normal file
View File

@@ -0,0 +1,34 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import javax.validation.constraints.Size;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel(description = "Feed modification request")
@Data
public class FeedModificationRequest implements Serializable {
@ApiModelProperty(value = "id", required = true)
private Long id;
@ApiModelProperty(value = "new name, null if not changed")
@Size(max = 128)
private String name;
@ApiModelProperty(value = "new parent category id")
@Size(max = 128)
private String categoryId;
@ApiModelProperty(value = "new display position, null if not changed")
private Integer position;
@ApiModelProperty(value = "JEXL string evaluated on new entries to mark them as read if they do not match")
@Size(max = 4096)
private String filter;
}

17
commafeed-server/src/main/java/com/commafeed/frontend/model/request/IDRequest.java Normal file
View File

@@ -0,0 +1,17 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
@SuppressWarnings("serial")
@ApiModel
@Data
public class IDRequest implements Serializable {
@ApiModelProperty(required = true)
private Long id;
}

Some files were not shown because too many files have changed in this diff Show More