Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-03-21 21:37:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

normalize line endings

Browse Source
This commit is contained in:
Athou
2025-03-10 08:38:21 +01:00
parent ec4554c76e
commit fb7f041454
223 changed files with 18091 additions and 18093 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
commafeed-server/src/main/java/com/commafeed/CommaFeedApplication.java
View File

@@ -1,41 +1,41 @@
package com.commafeed;
import jakarta.enterprise.event.Observes;
import jakarta.inject.Singleton;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.service.db.DatabaseStartupService;
import com.commafeed.backend.task.TaskScheduler;
import com.commafeed.security.password.PasswordConstraintValidator;
import io.quarkus.runtime.ShutdownEvent;
import io.quarkus.runtime.StartupEvent;
import lombok.RequiredArgsConstructor;
@Singleton
@RequiredArgsConstructor
public class CommaFeedApplication {
public static final String USERNAME_ADMIN = "admin";
public static final String USERNAME_DEMO = "demo";
private final DatabaseStartupService databaseStartupService;
private final FeedRefreshEngine feedRefreshEngine;
private final TaskScheduler taskScheduler;
private final CommaFeedConfiguration config;
public void start(@Observes StartupEvent ev) {
PasswordConstraintValidator.setStrict(config.users().strictPasswordPolicy());
databaseStartupService.populateInitialData();
feedRefreshEngine.start();
taskScheduler.start();
}
public void stop(@Observes ShutdownEvent ev) {
feedRefreshEngine.stop();
taskScheduler.stop();
}
}
package com.commafeed;
import jakarta.enterprise.event.Observes;
import jakarta.inject.Singleton;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.service.db.DatabaseStartupService;
import com.commafeed.backend.task.TaskScheduler;
import com.commafeed.security.password.PasswordConstraintValidator;
import io.quarkus.runtime.ShutdownEvent;
import io.quarkus.runtime.StartupEvent;
import lombok.RequiredArgsConstructor;
@Singleton
@RequiredArgsConstructor
public class CommaFeedApplication {
public static final String USERNAME_ADMIN = "admin";
public static final String USERNAME_DEMO = "demo";
private final DatabaseStartupService databaseStartupService;
private final FeedRefreshEngine feedRefreshEngine;
private final TaskScheduler taskScheduler;
private final CommaFeedConfiguration config;
public void start(@Observes StartupEvent ev) {
PasswordConstraintValidator.setStrict(config.users().strictPasswordPolicy());
databaseStartupService.populateInitialData();
feedRefreshEngine.start();
taskScheduler.start();
}
public void stop(@Observes ShutdownEvent ev) {
feedRefreshEngine.stop();
taskScheduler.stop();
}
}

732
commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java
View File

@@ -1,366 +1,366 @@
package com.commafeed;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.Positive;
import com.commafeed.backend.feed.FeedRefreshIntervalCalculator;
import io.quarkus.runtime.annotations.ConfigDocSection;
import io.quarkus.runtime.annotations.ConfigPhase;
import io.quarkus.runtime.annotations.ConfigRoot;
import io.quarkus.runtime.configuration.MemorySize;
import io.smallrye.config.ConfigMapping;
import io.smallrye.config.WithDefault;
/**
* CommaFeed configuration
*
* Default values are for production, they can be overridden in application.properties for other profiles
*/
@ConfigMapping(prefix = "commafeed")
@ConfigRoot(phase = ConfigPhase.RUN_TIME)
public interface CommaFeedConfiguration {
/**
* Whether to expose a robots.txt file that disallows web crawlers and search engine indexers.
*/
@WithDefault("true")
boolean hideFromWebCrawlers();
/**
* If enabled, images in feed entries will be proxied through the server instead of accessed directly by the browser.
*
* This is useful if commafeed is accessed through a restricting proxy that blocks some feeds that are followed.
*/
@WithDefault("false")
boolean imageProxyEnabled();
/**
* Enable password recovery via email.
*
* Quarkus mailer will need to be configured.
*/
@WithDefault("false")
boolean passwordRecoveryEnabled();
/**
* Message displayed in a notification at the bottom of the page.
*/
Optional<String> announcement();
/**
* Google Analytics tracking code.
*/
Optional<String> googleAnalyticsTrackingCode();
/**
* Google Auth key for fetching Youtube channel favicons.
*/
Optional<String> googleAuthKey();
/**
* HTTP client configuration
*/
@ConfigDocSection
HttpClient httpClient();
/**
* Feed refresh engine settings.
*/
@ConfigDocSection
FeedRefresh feedRefresh();
/**
* Database settings.
*/
@ConfigDocSection
Database database();
/**
* Users settings.
*/
@ConfigDocSection
Users users();
/**
* Websocket settings.
*/
@ConfigDocSection
Websocket websocket();
interface HttpClient {
/**
* User-Agent string that will be used by the http client, leave empty for the default one.
*/
Optional<String> userAgent();
/**
* Time to wait for a connection to be established.
*/
@WithDefault("5s")
Duration connectTimeout();
/**
* Time to wait for SSL handshake to complete.
*/
@WithDefault("5s")
Duration sslHandshakeTimeout();
/**
* Time to wait between two packets before timeout.
*/
@WithDefault("10s")
Duration socketTimeout();
/**
* Time to wait for the full response to be received.
*/
@WithDefault("10s")
Duration responseTimeout();
/**
* Time to live for a connection in the pool.
*/
@WithDefault("30s")
Duration connectionTimeToLive();
/**
* Time between eviction runs for idle connections.
*/
@WithDefault("1m")
Duration idleConnectionsEvictionInterval();
/**
* If a feed is larger than this, it will be discarded to prevent memory issues while parsing the feed.
*/
@WithDefault("5M")
MemorySize maxResponseSize();
/**
* Prevent access to local addresses to mitigate server-side request forgery (SSRF) attacks, which could potentially expose internal
* resources.
*
* You may want to disable this if you subscribe to feeds that are only available on your local network and you trust all users of
* your CommaFeed instance.
*/
@WithDefault("true")
boolean blockLocalAddresses();
/**
* HTTP client cache configuration
*/
@ConfigDocSection
HttpClientCache cache();
}
interface HttpClientCache {
/**
* Whether to enable the cache. This cache is used to avoid spamming feeds in short bursts (e.g. when subscribing to a feed for the
* first time or when clicking "fetch all my feeds now").
*/
@WithDefault("true")
boolean enabled();
/**
* Maximum amount of memory the cache can use.
*/
@WithDefault("10M")
MemorySize maximumMemorySize();
/**
* Duration after which an entry is removed from the cache.
*/
@WithDefault("1m")
Duration expiration();
}
interface FeedRefresh {
/**
* Default amount of time CommaFeed will wait before refreshing a feed.
*/
@WithDefault("5m")
Duration interval();
/**
* Maximum amount of time CommaFeed will wait before refreshing a feed. This is used as an upper bound when:
*
* <ul>
* <li>an error occurs while refreshing a feed and we're backing off exponentially</li>
* <li>we receive a Cache-Control header from the feed</li>
* <li>we receive a Retry-After header from the feed</li>
* </ul>
*/
@WithDefault("4h")
Duration maxInterval();
/**
* If enabled, CommaFeed will calculate the next refresh time based on the feed's average time between entries and the time since
* the last entry was published. The interval will be sometimes between the default refresh interval
* (`commafeed.feed-refresh.interval`) and the maximum refresh interval (`commafeed.feed-refresh.max-interval`).
*
* See {@link FeedRefreshIntervalCalculator} for details.
*/
@WithDefault("true")
boolean intervalEmpirical();
/**
* Feed refresh engine error handling settings.
*/
@ConfigDocSection
FeedRefreshErrorHandling errors();
/**
* Amount of http threads used to fetch feeds.
*/
@Min(1)
@WithDefault("3")
int httpThreads();
/**
* Amount of threads used to insert new entries in the database.
*/
@Min(1)
@WithDefault("1")
int databaseThreads();
/**
* Duration after which a user is considered inactive. Feeds for inactive users are not refreshed until they log in again.
*
* 0 to disable.
*/
@WithDefault("0")
Duration userInactivityPeriod();
/**
* Duration after which the evaluation of a filtering expresion to mark an entry as read is considered to have timed out.
*/
@WithDefault("500ms")
Duration filteringExpressionEvaluationTimeout();
/**
* Duration after which the "Fetch all my feeds now" action is available again after use to avoid spamming feeds.
*/
@WithDefault("0")
Duration forceRefreshCooldownDuration();
}
interface FeedRefreshErrorHandling {
/**
* Number of retries before backoff is applied.
*/
@Min(0)
@WithDefault("3")
int retriesBeforeBackoff();
/**
* Duration to wait before retrying after an error. Will be multiplied by the number of errors since the last successful fetch.
*/
@WithDefault("1h")
Duration backoffInterval();
}
interface Database {
/**
* Timeout applied to all database queries.
*
* 0 to disable.
*/
@WithDefault("0")
Duration queryTimeout();
/**
* Database cleanup settings.
*/
@ConfigDocSection
Cleanup cleanup();
interface Cleanup {
/**
* Maximum age of feed entries in the database. Older entries will be deleted.
*
* 0 to disable.
*/
@WithDefault("365d")
Duration entriesMaxAge();
/**
* Maximum age of feed entry statuses (read/unread) in the database. Older statuses will be deleted.
*
* 0 to disable.
*/
@WithDefault("0")
Duration statusesMaxAge();
/**
* Maximum number of entries per feed to keep in the database.
*
* 0 to disable.
*/
@WithDefault("500")
int maxFeedCapacity();
/**
* Limit the number of feeds a user can subscribe to.
*
* 0 to disable.
*/
@WithDefault("0")
int maxFeedsPerUser();
/**
* Rows to delete per query while cleaning up old entries.
*/
@Positive
@WithDefault("100")
int batchSize();
default Instant statusesInstantThreshold() {
return statusesMaxAge().toMillis() > 0 ? Instant.now().minus(statusesMaxAge()) : null;
}
}
}
interface Users {
/**
* Whether to let users create accounts for themselves.
*/
@WithDefault("false")
boolean allowRegistrations();
/**
* Whether to enable strict password validation (1 uppercase char, 1 lowercase char, 1 digit, 1 special char).
*/
@WithDefault("true")
boolean strictPasswordPolicy();
/**
* Whether to create a demo account the first time the app starts.
*/
@WithDefault("false")
boolean createDemoAccount();
}
interface Websocket {
/**
* Enable websocket connection so the server can notify web clients that there are new entries for feeds.
*/
@WithDefault("true")
boolean enabled();
/**
* Interval at which the client will send a ping message on the websocket to keep the connection alive.
*/
@WithDefault("15m")
Duration pingInterval();
/**
* If the websocket connection is disabled or the connection is lost, the client will reload the feed tree at this interval.
*/
@WithDefault("30s")
Duration treeReloadInterval();
}
}
package com.commafeed;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.Positive;
import com.commafeed.backend.feed.FeedRefreshIntervalCalculator;
import io.quarkus.runtime.annotations.ConfigDocSection;
import io.quarkus.runtime.annotations.ConfigPhase;
import io.quarkus.runtime.annotations.ConfigRoot;
import io.quarkus.runtime.configuration.MemorySize;
import io.smallrye.config.ConfigMapping;
import io.smallrye.config.WithDefault;
/**
* CommaFeed configuration
*
* Default values are for production, they can be overridden in application.properties for other profiles
*/
@ConfigMapping(prefix = "commafeed")
@ConfigRoot(phase = ConfigPhase.RUN_TIME)
public interface CommaFeedConfiguration {
/**
* Whether to expose a robots.txt file that disallows web crawlers and search engine indexers.
*/
@WithDefault("true")
boolean hideFromWebCrawlers();
/**
* If enabled, images in feed entries will be proxied through the server instead of accessed directly by the browser.
*
* This is useful if commafeed is accessed through a restricting proxy that blocks some feeds that are followed.
*/
@WithDefault("false")
boolean imageProxyEnabled();
/**
* Enable password recovery via email.
*
* Quarkus mailer will need to be configured.
*/
@WithDefault("false")
boolean passwordRecoveryEnabled();
/**
* Message displayed in a notification at the bottom of the page.
*/
Optional<String> announcement();
/**
* Google Analytics tracking code.
*/
Optional<String> googleAnalyticsTrackingCode();
/**
* Google Auth key for fetching Youtube channel favicons.
*/
Optional<String> googleAuthKey();
/**
* HTTP client configuration
*/
@ConfigDocSection
HttpClient httpClient();
/**
* Feed refresh engine settings.
*/
@ConfigDocSection
FeedRefresh feedRefresh();
/**
* Database settings.
*/
@ConfigDocSection
Database database();
/**
* Users settings.
*/
@ConfigDocSection
Users users();
/**
* Websocket settings.
*/
@ConfigDocSection
Websocket websocket();
interface HttpClient {
/**
* User-Agent string that will be used by the http client, leave empty for the default one.
*/
Optional<String> userAgent();
/**
* Time to wait for a connection to be established.
*/
@WithDefault("5s")
Duration connectTimeout();
/**
* Time to wait for SSL handshake to complete.
*/
@WithDefault("5s")
Duration sslHandshakeTimeout();
/**
* Time to wait between two packets before timeout.
*/
@WithDefault("10s")
Duration socketTimeout();
/**
* Time to wait for the full response to be received.
*/
@WithDefault("10s")
Duration responseTimeout();
/**
* Time to live for a connection in the pool.
*/
@WithDefault("30s")
Duration connectionTimeToLive();
/**
* Time between eviction runs for idle connections.
*/
@WithDefault("1m")
Duration idleConnectionsEvictionInterval();
/**
* If a feed is larger than this, it will be discarded to prevent memory issues while parsing the feed.
*/
@WithDefault("5M")
MemorySize maxResponseSize();
/**
* Prevent access to local addresses to mitigate server-side request forgery (SSRF) attacks, which could potentially expose internal
* resources.
*
* You may want to disable this if you subscribe to feeds that are only available on your local network and you trust all users of
* your CommaFeed instance.
*/
@WithDefault("true")
boolean blockLocalAddresses();
/**
* HTTP client cache configuration
*/
@ConfigDocSection
HttpClientCache cache();
}
interface HttpClientCache {
/**
* Whether to enable the cache. This cache is used to avoid spamming feeds in short bursts (e.g. when subscribing to a feed for the
* first time or when clicking "fetch all my feeds now").
*/
@WithDefault("true")
boolean enabled();
/**
* Maximum amount of memory the cache can use.
*/
@WithDefault("10M")
MemorySize maximumMemorySize();
/**
* Duration after which an entry is removed from the cache.
*/
@WithDefault("1m")
Duration expiration();
}
interface FeedRefresh {
/**
* Default amount of time CommaFeed will wait before refreshing a feed.
*/
@WithDefault("5m")
Duration interval();
/**
* Maximum amount of time CommaFeed will wait before refreshing a feed. This is used as an upper bound when:
*
* <ul>
* <li>an error occurs while refreshing a feed and we're backing off exponentially</li>
* <li>we receive a Cache-Control header from the feed</li>
* <li>we receive a Retry-After header from the feed</li>
* </ul>
*/
@WithDefault("4h")
Duration maxInterval();
/**
* If enabled, CommaFeed will calculate the next refresh time based on the feed's average time between entries and the time since
* the last entry was published. The interval will be sometimes between the default refresh interval
* (`commafeed.feed-refresh.interval`) and the maximum refresh interval (`commafeed.feed-refresh.max-interval`).
*
* See {@link FeedRefreshIntervalCalculator} for details.
*/
@WithDefault("true")
boolean intervalEmpirical();
/**
* Feed refresh engine error handling settings.
*/
@ConfigDocSection
FeedRefreshErrorHandling errors();
/**
* Amount of http threads used to fetch feeds.
*/
@Min(1)
@WithDefault("3")
int httpThreads();
/**
* Amount of threads used to insert new entries in the database.
*/
@Min(1)
@WithDefault("1")
int databaseThreads();
/**
* Duration after which a user is considered inactive. Feeds for inactive users are not refreshed until they log in again.
*
* 0 to disable.
*/
@WithDefault("0")
Duration userInactivityPeriod();
/**
* Duration after which the evaluation of a filtering expresion to mark an entry as read is considered to have timed out.
*/
@WithDefault("500ms")
Duration filteringExpressionEvaluationTimeout();
/**
* Duration after which the "Fetch all my feeds now" action is available again after use to avoid spamming feeds.
*/
@WithDefault("0")
Duration forceRefreshCooldownDuration();
}
interface FeedRefreshErrorHandling {
/**
* Number of retries before backoff is applied.
*/
@Min(0)
@WithDefault("3")
int retriesBeforeBackoff();
/**
* Duration to wait before retrying after an error. Will be multiplied by the number of errors since the last successful fetch.
*/
@WithDefault("1h")
Duration backoffInterval();
}
interface Database {
/**
* Timeout applied to all database queries.
*
* 0 to disable.
*/
@WithDefault("0")
Duration queryTimeout();
/**
* Database cleanup settings.
*/
@ConfigDocSection
Cleanup cleanup();
interface Cleanup {
/**
* Maximum age of feed entries in the database. Older entries will be deleted.
*
* 0 to disable.
*/
@WithDefault("365d")
Duration entriesMaxAge();
/**
* Maximum age of feed entry statuses (read/unread) in the database. Older statuses will be deleted.
*
* 0 to disable.
*/
@WithDefault("0")
Duration statusesMaxAge();
/**
* Maximum number of entries per feed to keep in the database.
*
* 0 to disable.
*/
@WithDefault("500")
int maxFeedCapacity();
/**
* Limit the number of feeds a user can subscribe to.
*
* 0 to disable.
*/
@WithDefault("0")
int maxFeedsPerUser();
/**
* Rows to delete per query while cleaning up old entries.
*/
@Positive
@WithDefault("100")
int batchSize();
default Instant statusesInstantThreshold() {
return statusesMaxAge().toMillis() > 0 ? Instant.now().minus(statusesMaxAge()) : null;
}
}
}
interface Users {
/**
* Whether to let users create accounts for themselves.
*/
@WithDefault("false")
boolean allowRegistrations();
/**
* Whether to enable strict password validation (1 uppercase char, 1 lowercase char, 1 digit, 1 special char).
*/
@WithDefault("true")
boolean strictPasswordPolicy();
/**
* Whether to create a demo account the first time the app starts.
*/
@WithDefault("false")
boolean createDemoAccount();
}
interface Websocket {
/**
* Enable websocket connection so the server can notify web clients that there are new entries for feeds.
*/
@WithDefault("true")
boolean enabled();
/**
* Interval at which the client will send a ping message on the websocket to keep the connection alive.
*/
@WithDefault("15m")
Duration pingInterval();
/**
* If the websocket connection is disabled or the connection is lost, the client will reload the feed tree at this interval.
*/
@WithDefault("30s")
Duration treeReloadInterval();
}
}

50
commafeed-server/src/main/java/com/commafeed/CommaFeedProducers.java
View File

@@ -1,25 +1,25 @@
package com.commafeed;
import java.time.InstantSource;
import jakarta.enterprise.inject.Produces;
import jakarta.inject.Singleton;
import com.codahale.metrics.MetricRegistry;
@Singleton
public class CommaFeedProducers {
@Produces
@Singleton
public InstantSource instantSource() {
return InstantSource.system();
}
@Produces
@Singleton
public MetricRegistry metricRegistry() {
return new MetricRegistry();
}
}
package com.commafeed;
import java.time.InstantSource;
import jakarta.enterprise.inject.Produces;
import jakarta.inject.Singleton;
import com.codahale.metrics.MetricRegistry;
@Singleton
public class CommaFeedProducers {
@Produces
@Singleton
public InstantSource instantSource() {
return InstantSource.system();
}
@Produces
@Singleton
public MetricRegistry metricRegistry() {
return new MetricRegistry();
}
}

64
commafeed-server/src/main/java/com/commafeed/CommaFeedVersion.java
View File

@@ -1,32 +1,32 @@
package com.commafeed;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import jakarta.inject.Singleton;
import lombok.Getter;
@Singleton
@Getter
public class CommaFeedVersion {
private final String version;
private final String gitCommit;
public CommaFeedVersion() {
Properties properties = new Properties();
try (InputStream stream = getClass().getResourceAsStream("/git.properties")) {
if (stream != null) {
properties.load(stream);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
this.version = properties.getProperty("git.build.version", "unknown");
this.gitCommit = properties.getProperty("git.commit.id.abbrev", "unknown");
}
}
package com.commafeed;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import jakarta.inject.Singleton;
import lombok.Getter;
@Singleton
@Getter
public class CommaFeedVersion {
private final String version;
private final String gitCommit;
public CommaFeedVersion() {
Properties properties = new Properties();
try (InputStream stream = getClass().getResourceAsStream("/git.properties")) {
if (stream != null) {
properties.load(stream);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
this.version = properties.getProperty("git.build.version", "unknown");
this.gitCommit = properties.getProperty("git.commit.id.abbrev", "unknown");
}
}

100
commafeed-server/src/main/java/com/commafeed/ExceptionMappers.java
View File

@@ -1,50 +1,50 @@
package com.commafeed;
import jakarta.annotation.Priority;
import jakarta.validation.ValidationException;
import jakarta.ws.rs.ext.Provider;
import org.jboss.resteasy.reactive.RestResponse;
import org.jboss.resteasy.reactive.RestResponse.Status;
import org.jboss.resteasy.reactive.server.ServerExceptionMapper;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.quarkus.security.AuthenticationFailedException;
import io.quarkus.security.UnauthorizedException;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Provider
@Priority(1)
public class ExceptionMappers {
private final CommaFeedConfiguration config;
@ServerExceptionMapper(UnauthorizedException.class)
public RestResponse<UnauthorizedResponse> unauthorized(UnauthorizedException e) {
return RestResponse.status(RestResponse.Status.UNAUTHORIZED,
new UnauthorizedResponse(e.getMessage(), config.users().allowRegistrations()));
}
@ServerExceptionMapper(AuthenticationFailedException.class)
public RestResponse<AuthenticationFailed> authenticationFailed(AuthenticationFailedException e) {
return RestResponse.status(RestResponse.Status.UNAUTHORIZED, new AuthenticationFailed(e.getMessage()));
}
@ServerExceptionMapper(ValidationException.class)
public RestResponse<ValidationFailed> validationFailed(ValidationException e) {
return RestResponse.status(Status.BAD_REQUEST, new ValidationFailed(e.getMessage()));
}
@RegisterForReflection
public record UnauthorizedResponse(String message, boolean allowRegistrations) {
}
@RegisterForReflection
public record AuthenticationFailed(String message) {
}
@RegisterForReflection
public record ValidationFailed(String message) {
}
}
package com.commafeed;
import jakarta.annotation.Priority;
import jakarta.validation.ValidationException;
import jakarta.ws.rs.ext.Provider;
import org.jboss.resteasy.reactive.RestResponse;
import org.jboss.resteasy.reactive.RestResponse.Status;
import org.jboss.resteasy.reactive.server.ServerExceptionMapper;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.quarkus.security.AuthenticationFailedException;
import io.quarkus.security.UnauthorizedException;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Provider
@Priority(1)
public class ExceptionMappers {
private final CommaFeedConfiguration config;
@ServerExceptionMapper(UnauthorizedException.class)
public RestResponse<UnauthorizedResponse> unauthorized(UnauthorizedException e) {
return RestResponse.status(RestResponse.Status.UNAUTHORIZED,
new UnauthorizedResponse(e.getMessage(), config.users().allowRegistrations()));
}
@ServerExceptionMapper(AuthenticationFailedException.class)
public RestResponse<AuthenticationFailed> authenticationFailed(AuthenticationFailedException e) {
return RestResponse.status(RestResponse.Status.UNAUTHORIZED, new AuthenticationFailed(e.getMessage()));
}
@ServerExceptionMapper(ValidationException.class)
public RestResponse<ValidationFailed> validationFailed(ValidationException e) {
return RestResponse.status(Status.BAD_REQUEST, new ValidationFailed(e.getMessage()));
}
@RegisterForReflection
public record UnauthorizedResponse(String message, boolean allowRegistrations) {
}
@RegisterForReflection
public record AuthenticationFailed(String message) {
}
@RegisterForReflection
public record ValidationFailed(String message) {
}
}

58
commafeed-server/src/main/java/com/commafeed/JacksonCustomizer.java
View File

@@ -1,29 +1,29 @@
package com.commafeed;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.json.MetricsModule;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.quarkus.jackson.ObjectMapperCustomizer;
@Singleton
public class JacksonCustomizer implements ObjectMapperCustomizer {
@Override
public void customize(ObjectMapper objectMapper) {
objectMapper.registerModule(new JavaTimeModule());
// read and write instants as milliseconds instead of nanoseconds
objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true)
.configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false)
.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false);
// add support for serializing metrics
objectMapper.registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
}
}
package com.commafeed;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.json.MetricsModule;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.quarkus.jackson.ObjectMapperCustomizer;
@Singleton
public class JacksonCustomizer implements ObjectMapperCustomizer {
@Override
public void customize(ObjectMapper objectMapper) {
objectMapper.registerModule(new JavaTimeModule());
// read and write instants as milliseconds instead of nanoseconds
objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true)
.configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false)
.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false);
// add support for serializing metrics
objectMapper.registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
}
}

452
commafeed-server/src/main/java/com/commafeed/NativeImageClasses.java
View File

@@ -1,226 +1,226 @@
package com.commafeed;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import io.quarkus.runtime.annotations.RegisterForReflection;
@RegisterForReflection(
targets = {
// metrics
MetricRegistry.class, Meter.class, Gauge.class, Counter.class, Timer.class, Histogram.class,
// rome
java.util.Date.class, com.rometools.opml.feed.synd.impl.TreeCategoryImpl.class,
com.rometools.rome.feed.synd.SyndFeedImpl.class, com.rometools.rome.feed.module.DCSubjectImpl.class,
com.rometools.rome.feed.synd.SyndEntryImpl.class, com.rometools.modules.psc.types.SimpleChapter.class,
com.rometools.rome.feed.synd.SyndCategoryImpl.class, com.rometools.rome.feed.synd.SyndImageImpl.class,
com.rometools.rome.feed.synd.SyndContentImpl.class, com.rometools.rome.feed.synd.SyndEnclosureImpl.class,
// rome cloneable
com.rometools.modules.activitystreams.types.Article.class, com.rometools.modules.activitystreams.types.Audio.class,
com.rometools.modules.activitystreams.types.Bookmark.class, com.rometools.modules.activitystreams.types.Comment.class,
com.rometools.modules.activitystreams.types.Event.class, com.rometools.modules.activitystreams.types.File.class,
com.rometools.modules.activitystreams.types.Folder.class, com.rometools.modules.activitystreams.types.List.class,
com.rometools.modules.activitystreams.types.Note.class, com.rometools.modules.activitystreams.types.Person.class,
com.rometools.modules.activitystreams.types.Photo.class, com.rometools.modules.activitystreams.types.PhotoAlbum.class,
com.rometools.modules.activitystreams.types.Place.class, com.rometools.modules.activitystreams.types.Playlist.class,
com.rometools.modules.activitystreams.types.Product.class, com.rometools.modules.activitystreams.types.Review.class,
com.rometools.modules.activitystreams.types.Service.class, com.rometools.modules.activitystreams.types.Song.class,
com.rometools.modules.activitystreams.types.Status.class, com.rometools.modules.base.types.DateTimeRange.class,
com.rometools.modules.base.types.FloatUnit.class, com.rometools.modules.base.types.GenderEnumeration.class,
com.rometools.modules.base.types.IntUnit.class, com.rometools.modules.base.types.PriceTypeEnumeration.class,
com.rometools.modules.base.types.ShippingType.class, com.rometools.modules.base.types.ShortDate.class,
com.rometools.modules.base.types.Size.class, com.rometools.modules.base.types.YearType.class,
com.rometools.modules.content.ContentItem.class, com.rometools.modules.georss.GeoRSSPoint.class,
com.rometools.modules.georss.geometries.Envelope.class, com.rometools.modules.georss.geometries.LineString.class,
com.rometools.modules.georss.geometries.LinearRing.class, com.rometools.modules.georss.geometries.Point.class,
com.rometools.modules.georss.geometries.Polygon.class, com.rometools.modules.georss.geometries.Position.class,
com.rometools.modules.georss.geometries.PositionList.class, com.rometools.modules.mediarss.types.MediaGroup.class,
com.rometools.modules.mediarss.types.Metadata.class, com.rometools.modules.mediarss.types.Thumbnail.class,
com.rometools.modules.opensearch.entity.OSQuery.class, com.rometools.modules.photocast.types.PhotoDate.class,
com.rometools.modules.sle.types.DateValue.class, com.rometools.modules.sle.types.Group.class,
com.rometools.modules.sle.types.NumberValue.class, com.rometools.modules.sle.types.Sort.class,
com.rometools.modules.sle.types.StringValue.class, com.rometools.modules.yahooweather.types.Astronomy.class,
com.rometools.modules.yahooweather.types.Atmosphere.class, com.rometools.modules.yahooweather.types.Condition.class,
com.rometools.modules.yahooweather.types.Forecast.class, com.rometools.modules.yahooweather.types.Location.class,
com.rometools.modules.yahooweather.types.Units.class, com.rometools.modules.yahooweather.types.Wind.class,
com.rometools.opml.feed.opml.Attribute.class, com.rometools.opml.feed.opml.Opml.class,
com.rometools.opml.feed.opml.Outline.class, com.rometools.rome.feed.atom.Category.class,
com.rometools.rome.feed.atom.Content.class, com.rometools.rome.feed.atom.Entry.class,
com.rometools.rome.feed.atom.Feed.class, com.rometools.rome.feed.atom.Generator.class,
com.rometools.rome.feed.atom.Link.class, com.rometools.rome.feed.atom.Person.class,
com.rometools.rome.feed.rss.Category.class, com.rometools.rome.feed.rss.Channel.class,
com.rometools.rome.feed.rss.Cloud.class, com.rometools.rome.feed.rss.Content.class,
com.rometools.rome.feed.rss.Description.class, com.rometools.rome.feed.rss.Enclosure.class,
com.rometools.rome.feed.rss.Guid.class, com.rometools.rome.feed.rss.Image.class, com.rometools.rome.feed.rss.Item.class,
com.rometools.rome.feed.rss.Source.class, com.rometools.rome.feed.rss.TextInput.class,
com.rometools.rome.feed.synd.SyndLinkImpl.class, com.rometools.rome.feed.synd.SyndPersonImpl.class,
java.util.ArrayList.class,
// rome modules
com.rometools.modules.sse.modules.Conflict.class, com.rometools.modules.sse.modules.Conflicts.class,
com.rometools.modules.cc.CreativeCommonsImpl.class, com.rometools.modules.feedpress.modules.FeedpressModuleImpl.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleImpl.class, com.rometools.modules.sse.modules.Sharing.class,
com.rometools.modules.georss.SimpleModuleImpl.class, com.rometools.modules.atom.modules.AtomLinkModuleImpl.class,
com.rometools.modules.itunes.EntryInformationImpl.class, com.rometools.modules.sse.modules.Update.class,
com.rometools.modules.photocast.PhotocastModuleImpl.class, com.rometools.modules.itunes.FeedInformationImpl.class,
com.rometools.modules.yahooweather.YWeatherModuleImpl.class, com.rometools.modules.feedburner.FeedBurnerImpl.class,
com.rometools.modules.sse.modules.Related.class, com.rometools.modules.fyyd.modules.FyydModuleImpl.class,
com.rometools.modules.psc.modules.PodloveSimpleChapterModuleImpl.class, com.rometools.modules.thr.ThreadingModuleImpl.class,
com.rometools.modules.sse.modules.Sync.class, com.rometools.modules.sle.SimpleListExtensionImpl.class,
com.rometools.modules.slash.SlashImpl.class, com.rometools.modules.sse.modules.History.class,
com.rometools.modules.georss.GMLModuleImpl.class, com.rometools.modules.base.CustomTagsImpl.class,
com.rometools.modules.base.GoogleBaseImpl.class, com.rometools.modules.sle.SleEntryImpl.class,
com.rometools.modules.mediarss.MediaEntryModuleImpl.class, com.rometools.modules.content.ContentModuleImpl.class,
com.rometools.modules.georss.W3CGeoModuleImpl.class, com.rometools.rome.feed.module.DCModuleImpl.class,
com.rometools.modules.mediarss.MediaModuleImpl.class, com.rometools.rome.feed.module.SyModuleImpl.class,
// extracted from all 3 rome.properties files of rome library
com.rometools.rome.io.impl.RSS090Parser.class, com.rometools.rome.io.impl.RSS091NetscapeParser.class,
com.rometools.rome.io.impl.RSS091UserlandParser.class, com.rometools.rome.io.impl.RSS092Parser.class,
com.rometools.rome.io.impl.RSS093Parser.class, com.rometools.rome.io.impl.RSS094Parser.class,
com.rometools.rome.io.impl.RSS10Parser.class, com.rometools.rome.io.impl.RSS20wNSParser.class,
com.rometools.rome.io.impl.RSS20Parser.class, com.rometools.rome.io.impl.Atom10Parser.class,
com.rometools.rome.io.impl.Atom03Parser.class,
com.rometools.rome.io.impl.SyModuleParser.class, com.rometools.rome.io.impl.DCModuleParser.class,
com.rometools.rome.io.impl.RSS090Generator.class, com.rometools.rome.io.impl.RSS091NetscapeGenerator.class,
com.rometools.rome.io.impl.RSS091UserlandGenerator.class, com.rometools.rome.io.impl.RSS092Generator.class,
com.rometools.rome.io.impl.RSS093Generator.class, com.rometools.rome.io.impl.RSS094Generator.class,
com.rometools.rome.io.impl.RSS10Generator.class, com.rometools.rome.io.impl.RSS20Generator.class,
com.rometools.rome.io.impl.Atom10Generator.class, com.rometools.rome.io.impl.Atom03Generator.class,
com.rometools.rome.feed.synd.impl.ConverterForAtom10.class, com.rometools.rome.feed.synd.impl.ConverterForAtom03.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS090.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Netscape.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Userland.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS092.class, com.rometools.rome.feed.synd.impl.ConverterForRSS093.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS094.class, com.rometools.rome.feed.synd.impl.ConverterForRSS10.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS20.class,
com.rometools.modules.mediarss.io.RSS20YahooParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.feedpress.io.FeedpressParser.class, com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.content.io.ContentModuleParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.atom.io.AtomModuleParser.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ItemParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.thr.io.ThreadingModuleParser.class, com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.sle.io.ModuleGenerator.class, com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class, com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class, com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class, com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.thr.io.ThreadingModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.opml.io.impl.OPML10Generator.class, com.rometools.opml.io.impl.OPML20Generator.class,
com.rometools.opml.io.impl.OPML10Parser.class, com.rometools.opml.io.impl.OPML20Parser.class,
com.rometools.opml.feed.synd.impl.ConverterForOPML10.class, com.rometools.opml.feed.synd.impl.ConverterForOPML20.class, })
public class NativeImageClasses {
}
package com.commafeed;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import io.quarkus.runtime.annotations.RegisterForReflection;
@RegisterForReflection(
targets = {
// metrics
MetricRegistry.class, Meter.class, Gauge.class, Counter.class, Timer.class, Histogram.class,
// rome
java.util.Date.class, com.rometools.opml.feed.synd.impl.TreeCategoryImpl.class,
com.rometools.rome.feed.synd.SyndFeedImpl.class, com.rometools.rome.feed.module.DCSubjectImpl.class,
com.rometools.rome.feed.synd.SyndEntryImpl.class, com.rometools.modules.psc.types.SimpleChapter.class,
com.rometools.rome.feed.synd.SyndCategoryImpl.class, com.rometools.rome.feed.synd.SyndImageImpl.class,
com.rometools.rome.feed.synd.SyndContentImpl.class, com.rometools.rome.feed.synd.SyndEnclosureImpl.class,
// rome cloneable
com.rometools.modules.activitystreams.types.Article.class, com.rometools.modules.activitystreams.types.Audio.class,
com.rometools.modules.activitystreams.types.Bookmark.class, com.rometools.modules.activitystreams.types.Comment.class,
com.rometools.modules.activitystreams.types.Event.class, com.rometools.modules.activitystreams.types.File.class,
com.rometools.modules.activitystreams.types.Folder.class, com.rometools.modules.activitystreams.types.List.class,
com.rometools.modules.activitystreams.types.Note.class, com.rometools.modules.activitystreams.types.Person.class,
com.rometools.modules.activitystreams.types.Photo.class, com.rometools.modules.activitystreams.types.PhotoAlbum.class,
com.rometools.modules.activitystreams.types.Place.class, com.rometools.modules.activitystreams.types.Playlist.class,
com.rometools.modules.activitystreams.types.Product.class, com.rometools.modules.activitystreams.types.Review.class,
com.rometools.modules.activitystreams.types.Service.class, com.rometools.modules.activitystreams.types.Song.class,
com.rometools.modules.activitystreams.types.Status.class, com.rometools.modules.base.types.DateTimeRange.class,
com.rometools.modules.base.types.FloatUnit.class, com.rometools.modules.base.types.GenderEnumeration.class,
com.rometools.modules.base.types.IntUnit.class, com.rometools.modules.base.types.PriceTypeEnumeration.class,
com.rometools.modules.base.types.ShippingType.class, com.rometools.modules.base.types.ShortDate.class,
com.rometools.modules.base.types.Size.class, com.rometools.modules.base.types.YearType.class,
com.rometools.modules.content.ContentItem.class, com.rometools.modules.georss.GeoRSSPoint.class,
com.rometools.modules.georss.geometries.Envelope.class, com.rometools.modules.georss.geometries.LineString.class,
com.rometools.modules.georss.geometries.LinearRing.class, com.rometools.modules.georss.geometries.Point.class,
com.rometools.modules.georss.geometries.Polygon.class, com.rometools.modules.georss.geometries.Position.class,
com.rometools.modules.georss.geometries.PositionList.class, com.rometools.modules.mediarss.types.MediaGroup.class,
com.rometools.modules.mediarss.types.Metadata.class, com.rometools.modules.mediarss.types.Thumbnail.class,
com.rometools.modules.opensearch.entity.OSQuery.class, com.rometools.modules.photocast.types.PhotoDate.class,
com.rometools.modules.sle.types.DateValue.class, com.rometools.modules.sle.types.Group.class,
com.rometools.modules.sle.types.NumberValue.class, com.rometools.modules.sle.types.Sort.class,
com.rometools.modules.sle.types.StringValue.class, com.rometools.modules.yahooweather.types.Astronomy.class,
com.rometools.modules.yahooweather.types.Atmosphere.class, com.rometools.modules.yahooweather.types.Condition.class,
com.rometools.modules.yahooweather.types.Forecast.class, com.rometools.modules.yahooweather.types.Location.class,
com.rometools.modules.yahooweather.types.Units.class, com.rometools.modules.yahooweather.types.Wind.class,
com.rometools.opml.feed.opml.Attribute.class, com.rometools.opml.feed.opml.Opml.class,
com.rometools.opml.feed.opml.Outline.class, com.rometools.rome.feed.atom.Category.class,
com.rometools.rome.feed.atom.Content.class, com.rometools.rome.feed.atom.Entry.class,
com.rometools.rome.feed.atom.Feed.class, com.rometools.rome.feed.atom.Generator.class,
com.rometools.rome.feed.atom.Link.class, com.rometools.rome.feed.atom.Person.class,
com.rometools.rome.feed.rss.Category.class, com.rometools.rome.feed.rss.Channel.class,
com.rometools.rome.feed.rss.Cloud.class, com.rometools.rome.feed.rss.Content.class,
com.rometools.rome.feed.rss.Description.class, com.rometools.rome.feed.rss.Enclosure.class,
com.rometools.rome.feed.rss.Guid.class, com.rometools.rome.feed.rss.Image.class, com.rometools.rome.feed.rss.Item.class,
com.rometools.rome.feed.rss.Source.class, com.rometools.rome.feed.rss.TextInput.class,
com.rometools.rome.feed.synd.SyndLinkImpl.class, com.rometools.rome.feed.synd.SyndPersonImpl.class,
java.util.ArrayList.class,
// rome modules
com.rometools.modules.sse.modules.Conflict.class, com.rometools.modules.sse.modules.Conflicts.class,
com.rometools.modules.cc.CreativeCommonsImpl.class, com.rometools.modules.feedpress.modules.FeedpressModuleImpl.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleImpl.class, com.rometools.modules.sse.modules.Sharing.class,
com.rometools.modules.georss.SimpleModuleImpl.class, com.rometools.modules.atom.modules.AtomLinkModuleImpl.class,
com.rometools.modules.itunes.EntryInformationImpl.class, com.rometools.modules.sse.modules.Update.class,
com.rometools.modules.photocast.PhotocastModuleImpl.class, com.rometools.modules.itunes.FeedInformationImpl.class,
com.rometools.modules.yahooweather.YWeatherModuleImpl.class, com.rometools.modules.feedburner.FeedBurnerImpl.class,
com.rometools.modules.sse.modules.Related.class, com.rometools.modules.fyyd.modules.FyydModuleImpl.class,
com.rometools.modules.psc.modules.PodloveSimpleChapterModuleImpl.class, com.rometools.modules.thr.ThreadingModuleImpl.class,
com.rometools.modules.sse.modules.Sync.class, com.rometools.modules.sle.SimpleListExtensionImpl.class,
com.rometools.modules.slash.SlashImpl.class, com.rometools.modules.sse.modules.History.class,
com.rometools.modules.georss.GMLModuleImpl.class, com.rometools.modules.base.CustomTagsImpl.class,
com.rometools.modules.base.GoogleBaseImpl.class, com.rometools.modules.sle.SleEntryImpl.class,
com.rometools.modules.mediarss.MediaEntryModuleImpl.class, com.rometools.modules.content.ContentModuleImpl.class,
com.rometools.modules.georss.W3CGeoModuleImpl.class, com.rometools.rome.feed.module.DCModuleImpl.class,
com.rometools.modules.mediarss.MediaModuleImpl.class, com.rometools.rome.feed.module.SyModuleImpl.class,
// extracted from all 3 rome.properties files of rome library
com.rometools.rome.io.impl.RSS090Parser.class, com.rometools.rome.io.impl.RSS091NetscapeParser.class,
com.rometools.rome.io.impl.RSS091UserlandParser.class, com.rometools.rome.io.impl.RSS092Parser.class,
com.rometools.rome.io.impl.RSS093Parser.class, com.rometools.rome.io.impl.RSS094Parser.class,
com.rometools.rome.io.impl.RSS10Parser.class, com.rometools.rome.io.impl.RSS20wNSParser.class,
com.rometools.rome.io.impl.RSS20Parser.class, com.rometools.rome.io.impl.Atom10Parser.class,
com.rometools.rome.io.impl.Atom03Parser.class,
com.rometools.rome.io.impl.SyModuleParser.class, com.rometools.rome.io.impl.DCModuleParser.class,
com.rometools.rome.io.impl.RSS090Generator.class, com.rometools.rome.io.impl.RSS091NetscapeGenerator.class,
com.rometools.rome.io.impl.RSS091UserlandGenerator.class, com.rometools.rome.io.impl.RSS092Generator.class,
com.rometools.rome.io.impl.RSS093Generator.class, com.rometools.rome.io.impl.RSS094Generator.class,
com.rometools.rome.io.impl.RSS10Generator.class, com.rometools.rome.io.impl.RSS20Generator.class,
com.rometools.rome.io.impl.Atom10Generator.class, com.rometools.rome.io.impl.Atom03Generator.class,
com.rometools.rome.feed.synd.impl.ConverterForAtom10.class, com.rometools.rome.feed.synd.impl.ConverterForAtom03.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS090.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Netscape.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Userland.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS092.class, com.rometools.rome.feed.synd.impl.ConverterForRSS093.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS094.class, com.rometools.rome.feed.synd.impl.ConverterForRSS10.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS20.class,
com.rometools.modules.mediarss.io.RSS20YahooParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.feedpress.io.FeedpressParser.class, com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.content.io.ContentModuleParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.atom.io.AtomModuleParser.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ItemParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.thr.io.ThreadingModuleParser.class, com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.sle.io.ModuleGenerator.class, com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class, com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class, com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class, com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.thr.io.ThreadingModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.opml.io.impl.OPML10Generator.class, com.rometools.opml.io.impl.OPML20Generator.class,
com.rometools.opml.io.impl.OPML10Parser.class, com.rometools.opml.io.impl.OPML20Parser.class,
com.rometools.opml.feed.synd.impl.ConverterForOPML10.class, com.rometools.opml.feed.synd.impl.ConverterForOPML20.class, })
public class NativeImageClasses {
}

58
commafeed-server/src/main/java/com/commafeed/backend/Digests.java
View File

@@ -1,29 +1,29 @@
package com.commafeed.backend;
import java.nio.charset.StandardCharsets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import lombok.experimental.UtilityClass;
@UtilityClass
@SuppressWarnings("deprecation")
public class Digests {
public static String sha1Hex(byte[] input) {
return hashBytesToHex(Hashing.sha1(), input);
}
public static String sha1Hex(String input) {
return hashBytesToHex(Hashing.sha1(), input.getBytes(StandardCharsets.UTF_8));
}
public static String md5Hex(String input) {
return hashBytesToHex(Hashing.md5(), input.getBytes(StandardCharsets.UTF_8));
}
private static String hashBytesToHex(HashFunction function, byte[] input) {
return function.hashBytes(input).toString();
}
}
package com.commafeed.backend;
import java.nio.charset.StandardCharsets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import lombok.experimental.UtilityClass;
@UtilityClass
@SuppressWarnings("deprecation")
public class Digests {
public static String sha1Hex(byte[] input) {
return hashBytesToHex(Hashing.sha1(), input);
}
public static String sha1Hex(String input) {
return hashBytesToHex(Hashing.sha1(), input.getBytes(StandardCharsets.UTF_8));
}
public static String md5Hex(String input) {
return hashBytesToHex(Hashing.md5(), input.getBytes(StandardCharsets.UTF_8));
}
private static String hashBytesToHex(HashFunction function, byte[] input) {
return function.hashBytes(input).toString();
}
}

856
commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java
View File

@@ -1,428 +1,428 @@
package com.commafeed.backend;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.stream.Stream;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.CacheControl;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.DnsResolver;
import org.apache.hc.client5.http.SystemDefaultDnsResolver;
import org.apache.hc.client5.http.config.ConnectionConfig;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.config.TlsConfig;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.io.HttpClientConnectionManager;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.client5.http.protocol.RedirectLocations;
import org.apache.hc.client5.http.utils.DateUtils;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.http.message.BasicHeader;
import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
import org.jboss.resteasy.reactive.common.headers.CacheControlDelegate;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.HttpClientCache;
import com.commafeed.CommaFeedVersion;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Iterables;
import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.apache5.util.Apache5SslUtils;
/**
* Smart HTTP getter: handles gzip, ssl, last modified and etag headers
*/
@Singleton
@Slf4j
public class HttpGetter {
private final CommaFeedConfiguration config;
private final InstantSource instantSource;
private final CloseableHttpClient client;
private final Cache<HttpRequest, HttpResponse> cache;
private final DnsResolver dnsResolver = SystemDefaultDnsResolver.INSTANCE;
public HttpGetter(CommaFeedConfiguration config, InstantSource instantSource, CommaFeedVersion version, MetricRegistry metrics) {
this.config = config;
this.instantSource = instantSource;
PoolingHttpClientConnectionManager connectionManager = newConnectionManager(config);
String userAgent = config.httpClient()
.userAgent()
.orElseGet(() -> String.format("CommaFeed/%s (https://github.com/Athou/commafeed)", version.getVersion()));
this.client = newClient(connectionManager, userAgent, config.httpClient().idleConnectionsEvictionInterval());
this.cache = newCache(config);
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "max"), () -> connectionManager.getTotalStats().getMax());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "size"),
() -> connectionManager.getTotalStats().getAvailable() + connectionManager.getTotalStats().getLeased());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "leased"), () -> connectionManager.getTotalStats().getLeased());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "pending"), () -> connectionManager.getTotalStats().getPending());
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "size"), () -> cache == null ? 0 : cache.size());
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "memoryUsage"),
() -> cache == null ? 0 : cache.asMap().values().stream().mapToInt(e -> e.content != null ? e.content.length : 0).sum());
}
public HttpResult get(String url)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException {
return get(HttpRequest.builder(url).build());
}
public HttpResult get(HttpRequest request)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException {
URI uri = URI.create(request.getUrl());
ensureHttpScheme(uri.getScheme());
if (config.httpClient().blockLocalAddresses()) {
ensurePublicAddress(uri.getHost());
}
final HttpResponse response;
if (cache == null) {
response = invoke(request);
} else {
try {
response = cache.get(request, () -> invoke(request));
} catch (ExecutionException e) {
if (e.getCause() instanceof IOException ioe) {
throw ioe;
} else {
throw new RuntimeException(e);
}
}
}
int code = response.getCode();
if (code == HttpStatus.SC_TOO_MANY_REQUESTS || code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.getRetryAfter() != null) {
throw new TooManyRequestsException(response.getRetryAfter());
}
if (code == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received");
}
if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);
}
String lastModifiedHeader = response.getLastModifiedHeader();
if (lastModifiedHeader != null && lastModifiedHeader.equals(request.getLastModified())) {
throw new NotModifiedException("lastModifiedHeader is the same");
}
String eTagHeader = response.getETagHeader();
if (eTagHeader != null && eTagHeader.equals(request.getETag())) {
throw new NotModifiedException("eTagHeader is the same");
}
Duration validFor = Optional.ofNullable(response.getCacheControl())
.filter(cc -> cc.getMaxAge() >= 0)
.map(cc -> Duration.ofSeconds(cc.getMaxAge()))
.orElse(Duration.ZERO);
return new HttpResult(response.getContent(), response.getContentType(), lastModifiedHeader, eTagHeader,
response.getUrlAfterRedirect(), validFor);
}
private void ensureHttpScheme(String scheme) throws SchemeNotAllowedException {
if (!"http".equals(scheme) && !"https".equals(scheme)) {
throw new SchemeNotAllowedException(scheme);
}
}
private void ensurePublicAddress(String host) throws HostNotAllowedException, UnknownHostException {
if (host == null) {
throw new HostNotAllowedException(null);
}
InetAddress[] addresses = dnsResolver.resolve(host);
if (Stream.of(addresses).anyMatch(this::isPrivateAddress)) {
throw new HostNotAllowedException(host);
}
}
private boolean isPrivateAddress(InetAddress address) {
return address.isSiteLocalAddress() || address.isAnyLocalAddress() || address.isLinkLocalAddress() || address.isLoopbackAddress()
|| address.isMulticastAddress();
}
private HttpResponse invoke(HttpRequest request) throws IOException {
log.debug("fetching {}", request.getUrl());
HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
// causes issues with some feeds
// see https://github.com/Athou/commafeed/issues/1572
// and https://issues.apache.org/jira/browse/HTTPCLIENT-2344
.setProtocolUpgradeEnabled(false)
.build());
return client.execute(request.toClassicHttpRequest(), context, resp -> {
byte[] content = resp.getEntity() == null ? null
: toByteArray(resp.getEntity(), config.httpClient().maxResponseSize().asLongValue());
int code = resp.getCode();
String lastModifiedHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.LAST_MODIFIED))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
String eTagHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.ETAG))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
CacheControl cacheControl = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.CACHE_CONTROL))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(HttpGetter::toCacheControl)
.orElse(null);
Instant retryAfter = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(this::toInstant)
.orElse(null);
String contentType = Optional.ofNullable(resp.getEntity()).map(HttpEntity::getContentType).orElse(null);
String urlAfterRedirect = Optional.ofNullable(context.getRedirectLocations())
.map(RedirectLocations::getAll)
.map(l -> Iterables.getLast(l, null))
.map(URI::toString)
.orElse(request.getUrl());
return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, retryAfter, content, contentType, urlAfterRedirect);
});
}
private static CacheControl toCacheControl(String headerValue) {
try {
return CacheControlDelegate.INSTANCE.fromString(headerValue);
} catch (Exception e) {
log.debug("Invalid Cache-Control header: {}", headerValue);
return null;
}
}
private Instant toInstant(String headerValue) {
if (headerValue == null) {
return null;
}
if (StringUtils.isNumeric(headerValue)) {
return instantSource.instant().plusSeconds(Long.parseLong(headerValue));
}
return DateUtils.parseStandardDate(headerValue);
}
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
if (entity.getContentLength() > maxBytes) {
throw new IOException(
"Response size (%s bytes) exceeds the maximum allowed size (%s bytes)".formatted(entity.getContentLength(), maxBytes));
}
try (InputStream input = entity.getContent()) {
if (input == null) {
return null;
}
byte[] bytes = ByteStreams.limit(input, maxBytes).readAllBytes();
if (bytes.length == maxBytes) {
throw new IOException("Response size exceeds the maximum allowed size (%s bytes)".formatted(maxBytes));
}
return bytes;
}
}
private PoolingHttpClientConnectionManager newConnectionManager(CommaFeedConfiguration config) {
SSLFactory sslFactory = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
int poolSize = config.feedRefresh().httpThreads();
return PoolingHttpClientConnectionManagerBuilder.create()
.setTlsSocketStrategy(Apache5SslUtils.toTlsSocketStrategy(sslFactory))
.setDefaultConnectionConfig(ConnectionConfig.custom()
.setConnectTimeout(Timeout.of(config.httpClient().connectTimeout()))
.setSocketTimeout(Timeout.of(config.httpClient().socketTimeout()))
.setTimeToLive(Timeout.of(config.httpClient().connectionTimeToLive()))
.build())
.setDefaultTlsConfig(TlsConfig.custom().setHandshakeTimeout(Timeout.of(config.httpClient().sslHandshakeTimeout())).build())
.setMaxConnPerRoute(poolSize)
.setMaxConnTotal(poolSize)
.setDnsResolver(dnsResolver)
.build();
}
private static CloseableHttpClient newClient(HttpClientConnectionManager connectionManager, String userAgent,
Duration idleConnectionsEvictionInterval) {
List<Header> headers = new ArrayList<>();
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
return HttpClientBuilder.create()
.useSystemProperties()
.disableAutomaticRetries()
.disableCookieManagement()
.setUserAgent(userAgent)
.setDefaultHeaders(headers)
.setConnectionManager(connectionManager)
.evictExpiredConnections()
.evictIdleConnections(TimeValue.of(idleConnectionsEvictionInterval))
.build();
}
private static Cache<HttpRequest, HttpResponse> newCache(CommaFeedConfiguration config) {
HttpClientCache cacheConfig = config.httpClient().cache();
if (!cacheConfig.enabled()) {
return null;
}
return CacheBuilder.newBuilder()
.weigher((HttpRequest key, HttpResponse value) -> value.getContent() != null ? value.getContent().length : 0)
.maximumWeight(cacheConfig.maximumMemorySize().asLongValue())
.expireAfterWrite(cacheConfig.expiration())
.build();
}
public static class SchemeNotAllowedException extends Exception {
private static final long serialVersionUID = 1L;
public SchemeNotAllowedException(String scheme) {
super("Scheme not allowed: " + scheme);
}
}
public static class HostNotAllowedException extends Exception {
private static final long serialVersionUID = 1L;
public HostNotAllowedException(String host) {
super("Host not allowed: " + host);
}
}
@Getter
public static class NotModifiedException extends Exception {
private static final long serialVersionUID = 1L;
/**
* if the value of this header changed, this is its new value
*/
private final String newLastModifiedHeader;
/**
* if the value of this header changed, this is its new value
*/
private final String newEtagHeader;
public NotModifiedException(String message) {
this(message, null, null);
}
public NotModifiedException(String message, String newLastModifiedHeader, String newEtagHeader) {
super(message);
this.newLastModifiedHeader = newLastModifiedHeader;
this.newEtagHeader = newEtagHeader;
}
}
@RequiredArgsConstructor
@Getter
public static class TooManyRequestsException extends Exception {
private static final long serialVersionUID = 1L;
private final Instant retryAfter;
}
@Getter
public static class HttpResponseException extends IOException {
private static final long serialVersionUID = 1L;
private final int code;
public HttpResponseException(int code, String message) {
super(message);
this.code = code;
}
}
@Builder(builderMethodName = "")
@EqualsAndHashCode
@Getter
public static class HttpRequest {
private String url;
private String lastModified;
private String eTag;
public static HttpRequestBuilder builder(String url) {
return new HttpRequestBuilder().url(url);
}
public ClassicHttpRequest toClassicHttpRequest() {
ClassicHttpRequest req = ClassicRequestBuilder.get(url).build();
if (lastModified != null) {
req.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
}
if (eTag != null) {
req.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
}
return req;
}
}
@Value
private static class HttpResponse {
int code;
String lastModifiedHeader;
String eTagHeader;
CacheControl cacheControl;
Instant retryAfter;
byte[] content;
String contentType;
String urlAfterRedirect;
}
@Value
public static class HttpResult {
byte[] content;
String contentType;
String lastModifiedSince;
String eTag;
String urlAfterRedirect;
Duration validFor;
}
}
package com.commafeed.backend;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.stream.Stream;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.CacheControl;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.DnsResolver;
import org.apache.hc.client5.http.SystemDefaultDnsResolver;
import org.apache.hc.client5.http.config.ConnectionConfig;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.config.TlsConfig;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.io.HttpClientConnectionManager;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.client5.http.protocol.RedirectLocations;
import org.apache.hc.client5.http.utils.DateUtils;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.http.message.BasicHeader;
import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
import org.jboss.resteasy.reactive.common.headers.CacheControlDelegate;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.HttpClientCache;
import com.commafeed.CommaFeedVersion;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Iterables;
import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.apache5.util.Apache5SslUtils;
/**
* Smart HTTP getter: handles gzip, ssl, last modified and etag headers
*/
@Singleton
@Slf4j
public class HttpGetter {
private final CommaFeedConfiguration config;
private final InstantSource instantSource;
private final CloseableHttpClient client;
private final Cache<HttpRequest, HttpResponse> cache;
private final DnsResolver dnsResolver = SystemDefaultDnsResolver.INSTANCE;
public HttpGetter(CommaFeedConfiguration config, InstantSource instantSource, CommaFeedVersion version, MetricRegistry metrics) {
this.config = config;
this.instantSource = instantSource;
PoolingHttpClientConnectionManager connectionManager = newConnectionManager(config);
String userAgent = config.httpClient()
.userAgent()
.orElseGet(() -> String.format("CommaFeed/%s (https://github.com/Athou/commafeed)", version.getVersion()));
this.client = newClient(connectionManager, userAgent, config.httpClient().idleConnectionsEvictionInterval());
this.cache = newCache(config);
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "max"), () -> connectionManager.getTotalStats().getMax());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "size"),
() -> connectionManager.getTotalStats().getAvailable() + connectionManager.getTotalStats().getLeased());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "leased"), () -> connectionManager.getTotalStats().getLeased());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "pending"), () -> connectionManager.getTotalStats().getPending());
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "size"), () -> cache == null ? 0 : cache.size());
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "memoryUsage"),
() -> cache == null ? 0 : cache.asMap().values().stream().mapToInt(e -> e.content != null ? e.content.length : 0).sum());
}
public HttpResult get(String url)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException {
return get(HttpRequest.builder(url).build());
}
public HttpResult get(HttpRequest request)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException {
URI uri = URI.create(request.getUrl());
ensureHttpScheme(uri.getScheme());
if (config.httpClient().blockLocalAddresses()) {
ensurePublicAddress(uri.getHost());
}
final HttpResponse response;
if (cache == null) {
response = invoke(request);
} else {
try {
response = cache.get(request, () -> invoke(request));
} catch (ExecutionException e) {
if (e.getCause() instanceof IOException ioe) {
throw ioe;
} else {
throw new RuntimeException(e);
}
}
}
int code = response.getCode();
if (code == HttpStatus.SC_TOO_MANY_REQUESTS || code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.getRetryAfter() != null) {
throw new TooManyRequestsException(response.getRetryAfter());
}
if (code == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received");
}
if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);
}
String lastModifiedHeader = response.getLastModifiedHeader();
if (lastModifiedHeader != null && lastModifiedHeader.equals(request.getLastModified())) {
throw new NotModifiedException("lastModifiedHeader is the same");
}
String eTagHeader = response.getETagHeader();
if (eTagHeader != null && eTagHeader.equals(request.getETag())) {
throw new NotModifiedException("eTagHeader is the same");
}
Duration validFor = Optional.ofNullable(response.getCacheControl())
.filter(cc -> cc.getMaxAge() >= 0)
.map(cc -> Duration.ofSeconds(cc.getMaxAge()))
.orElse(Duration.ZERO);
return new HttpResult(response.getContent(), response.getContentType(), lastModifiedHeader, eTagHeader,
response.getUrlAfterRedirect(), validFor);
}
private void ensureHttpScheme(String scheme) throws SchemeNotAllowedException {
if (!"http".equals(scheme) && !"https".equals(scheme)) {
throw new SchemeNotAllowedException(scheme);
}
}
private void ensurePublicAddress(String host) throws HostNotAllowedException, UnknownHostException {
if (host == null) {
throw new HostNotAllowedException(null);
}
InetAddress[] addresses = dnsResolver.resolve(host);
if (Stream.of(addresses).anyMatch(this::isPrivateAddress)) {
throw new HostNotAllowedException(host);
}
}
private boolean isPrivateAddress(InetAddress address) {
return address.isSiteLocalAddress() || address.isAnyLocalAddress() || address.isLinkLocalAddress() || address.isLoopbackAddress()
|| address.isMulticastAddress();
}
private HttpResponse invoke(HttpRequest request) throws IOException {
log.debug("fetching {}", request.getUrl());
HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
// causes issues with some feeds
// see https://github.com/Athou/commafeed/issues/1572
// and https://issues.apache.org/jira/browse/HTTPCLIENT-2344
.setProtocolUpgradeEnabled(false)
.build());
return client.execute(request.toClassicHttpRequest(), context, resp -> {
byte[] content = resp.getEntity() == null ? null
: toByteArray(resp.getEntity(), config.httpClient().maxResponseSize().asLongValue());
int code = resp.getCode();
String lastModifiedHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.LAST_MODIFIED))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
String eTagHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.ETAG))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
CacheControl cacheControl = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.CACHE_CONTROL))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(HttpGetter::toCacheControl)
.orElse(null);
Instant retryAfter = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(this::toInstant)
.orElse(null);
String contentType = Optional.ofNullable(resp.getEntity()).map(HttpEntity::getContentType).orElse(null);
String urlAfterRedirect = Optional.ofNullable(context.getRedirectLocations())
.map(RedirectLocations::getAll)
.map(l -> Iterables.getLast(l, null))
.map(URI::toString)
.orElse(request.getUrl());
return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, retryAfter, content, contentType, urlAfterRedirect);
});
}
private static CacheControl toCacheControl(String headerValue) {
try {
return CacheControlDelegate.INSTANCE.fromString(headerValue);
} catch (Exception e) {
log.debug("Invalid Cache-Control header: {}", headerValue);
return null;
}
}
private Instant toInstant(String headerValue) {
if (headerValue == null) {
return null;
}
if (StringUtils.isNumeric(headerValue)) {
return instantSource.instant().plusSeconds(Long.parseLong(headerValue));
}
return DateUtils.parseStandardDate(headerValue);
}
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
if (entity.getContentLength() > maxBytes) {
throw new IOException(
"Response size (%s bytes) exceeds the maximum allowed size (%s bytes)".formatted(entity.getContentLength(), maxBytes));
}
try (InputStream input = entity.getContent()) {
if (input == null) {
return null;
}
byte[] bytes = ByteStreams.limit(input, maxBytes).readAllBytes();
if (bytes.length == maxBytes) {
throw new IOException("Response size exceeds the maximum allowed size (%s bytes)".formatted(maxBytes));
}
return bytes;
}
}
private PoolingHttpClientConnectionManager newConnectionManager(CommaFeedConfiguration config) {
SSLFactory sslFactory = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
int poolSize = config.feedRefresh().httpThreads();
return PoolingHttpClientConnectionManagerBuilder.create()
.setTlsSocketStrategy(Apache5SslUtils.toTlsSocketStrategy(sslFactory))
.setDefaultConnectionConfig(ConnectionConfig.custom()
.setConnectTimeout(Timeout.of(config.httpClient().connectTimeout()))
.setSocketTimeout(Timeout.of(config.httpClient().socketTimeout()))
.setTimeToLive(Timeout.of(config.httpClient().connectionTimeToLive()))
.build())
.setDefaultTlsConfig(TlsConfig.custom().setHandshakeTimeout(Timeout.of(config.httpClient().sslHandshakeTimeout())).build())
.setMaxConnPerRoute(poolSize)
.setMaxConnTotal(poolSize)
.setDnsResolver(dnsResolver)
.build();
}
private static CloseableHttpClient newClient(HttpClientConnectionManager connectionManager, String userAgent,
Duration idleConnectionsEvictionInterval) {
List<Header> headers = new ArrayList<>();
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
return HttpClientBuilder.create()
.useSystemProperties()
.disableAutomaticRetries()
.disableCookieManagement()
.setUserAgent(userAgent)
.setDefaultHeaders(headers)
.setConnectionManager(connectionManager)
.evictExpiredConnections()
.evictIdleConnections(TimeValue.of(idleConnectionsEvictionInterval))
.build();
}
private static Cache<HttpRequest, HttpResponse> newCache(CommaFeedConfiguration config) {
HttpClientCache cacheConfig = config.httpClient().cache();
if (!cacheConfig.enabled()) {
return null;
}
return CacheBuilder.newBuilder()
.weigher((HttpRequest key, HttpResponse value) -> value.getContent() != null ? value.getContent().length : 0)
.maximumWeight(cacheConfig.maximumMemorySize().asLongValue())
.expireAfterWrite(cacheConfig.expiration())
.build();
}
public static class SchemeNotAllowedException extends Exception {
private static final long serialVersionUID = 1L;
public SchemeNotAllowedException(String scheme) {
super("Scheme not allowed: " + scheme);
}
}
public static class HostNotAllowedException extends Exception {
private static final long serialVersionUID = 1L;
public HostNotAllowedException(String host) {
super("Host not allowed: " + host);
}
}
@Getter
public static class NotModifiedException extends Exception {
private static final long serialVersionUID = 1L;
/**
* if the value of this header changed, this is its new value
*/
private final String newLastModifiedHeader;
/**
* if the value of this header changed, this is its new value
*/
private final String newEtagHeader;
public NotModifiedException(String message) {
this(message, null, null);
}
public NotModifiedException(String message, String newLastModifiedHeader, String newEtagHeader) {
super(message);
this.newLastModifiedHeader = newLastModifiedHeader;
this.newEtagHeader = newEtagHeader;
}
}
@RequiredArgsConstructor
@Getter
public static class TooManyRequestsException extends Exception {
private static final long serialVersionUID = 1L;
private final Instant retryAfter;
}
@Getter
public static class HttpResponseException extends IOException {
private static final long serialVersionUID = 1L;
private final int code;
public HttpResponseException(int code, String message) {
super(message);
this.code = code;
}
}
@Builder(builderMethodName = "")
@EqualsAndHashCode
@Getter
public static class HttpRequest {
private String url;
private String lastModified;
private String eTag;
public static HttpRequestBuilder builder(String url) {
return new HttpRequestBuilder().url(url);
}
public ClassicHttpRequest toClassicHttpRequest() {
ClassicHttpRequest req = ClassicRequestBuilder.get(url).build();
if (lastModified != null) {
req.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
}
if (eTag != null) {
req.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
}
return req;
}
}
@Value
private static class HttpResponse {
int code;
String lastModifiedHeader;
String eTagHeader;
CacheControl cacheControl;
Instant retryAfter;
byte[] content;
String contentType;
String urlAfterRedirect;
}
@Value
public static class HttpResult {
byte[] content;
String contentType;
String lastModifiedSince;
String eTag;
String urlAfterRedirect;
Duration validFor;
}
}

142
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java
View File

@@ -1,71 +1,71 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.QFeedCategory;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
import com.querydsl.core.types.Predicate;
@Singleton
public class FeedCategoryDAO extends GenericDAO<FeedCategory> {
private static final QFeedCategory CATEGORY = QFeedCategory.feedCategory;
public FeedCategoryDAO(EntityManager entityManager) {
super(entityManager, FeedCategory.class);
}
public List<FeedCategory> findAll(User user) {
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user)).join(CATEGORY.user, QUser.user).fetchJoin().fetch();
}
public FeedCategory findById(User user, Long id) {
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.id.eq(id)).fetchOne();
}
public FeedCategory findByName(User user, String name, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.name.eq(name), parentPredicate).fetchOne();
}
public List<FeedCategory> findByParent(User user, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), parentPredicate).fetch();
}
public List<FeedCategory> findAllChildrenCategories(User user, FeedCategory parent) {
return findAll(user).stream().filter(c -> isChild(c, parent)).toList();
}
private boolean isChild(FeedCategory child, FeedCategory parent) {
if (parent == null) {
return true;
}
boolean isChild = false;
while (child != null) {
if (Objects.equals(child.getId(), parent.getId())) {
isChild = true;
break;
}
child = child.getParent();
}
return isChild;
}
}
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.QFeedCategory;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
import com.querydsl.core.types.Predicate;
@Singleton
public class FeedCategoryDAO extends GenericDAO<FeedCategory> {
private static final QFeedCategory CATEGORY = QFeedCategory.feedCategory;
public FeedCategoryDAO(EntityManager entityManager) {
super(entityManager, FeedCategory.class);
}
public List<FeedCategory> findAll(User user) {
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user)).join(CATEGORY.user, QUser.user).fetchJoin().fetch();
}
public FeedCategory findById(User user, Long id) {
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.id.eq(id)).fetchOne();
}
public FeedCategory findByName(User user, String name, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.name.eq(name), parentPredicate).fetchOne();
}
public List<FeedCategory> findByParent(User user, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), parentPredicate).fetch();
}
public List<FeedCategory> findAllChildrenCategories(User user, FeedCategory parent) {
return findAll(user).stream().filter(c -> isChild(c, parent)).toList();
}
private boolean isChild(FeedCategory child, FeedCategory parent) {
if (parent == null) {
return true;
}
boolean isChild = false;
while (child != null) {
if (Objects.equals(child.getId(), parent.getId())) {
isChild = true;
break;
}
child = child.getParent();
}
return isChild;
}
}

128
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedDAO.java
View File

@@ -1,64 +1,64 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.QFeed;
import com.commafeed.backend.model.QFeedSubscription;
import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.impl.JPAQuery;
@Singleton
public class FeedDAO extends GenericDAO<Feed> {
private static final QFeed FEED = QFeed.feed;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
public FeedDAO(EntityManager entityManager) {
super(entityManager, Feed.class);
}
public List<Feed> findByIds(List<Long> id) {
return query().selectFrom(FEED).where(FEED.id.in(id)).fetch();
}
public List<Feed> findNextUpdatable(int count, Instant lastLoginThreshold) {
JPAQuery<Feed> query = query().selectFrom(FEED)
.distinct()
// join on subscriptions to only refresh feeds that have subscribers
.join(SUBSCRIPTION)
.on(SUBSCRIPTION.feed.eq(FEED))
.where(FEED.disabledUntil.isNull().or(FEED.disabledUntil.lt(Instant.now())));
if (lastLoginThreshold != null) {
query.join(SUBSCRIPTION.user).where(SUBSCRIPTION.user.lastLogin.gt(lastLoginThreshold));
}
return query.orderBy(FEED.disabledUntil.asc()).limit(count).fetch();
}
public void setDisabledUntil(List<Long> feedIds, Instant date) {
updateQuery(FEED).set(FEED.disabledUntil, date).where(FEED.id.in(feedIds)).execute();
}
public Feed findByUrl(String normalizedUrl, String normalizedUrlHash) {
return query().selectFrom(FEED)
.where(FEED.normalizedUrlHash.eq(normalizedUrlHash))
.fetch()
.stream()
.filter(f -> StringUtils.equals(normalizedUrl, f.getNormalizedUrl()))
.findFirst()
.orElse(null);
}
public List<Feed> findWithoutSubscriptions(int max) {
QFeedSubscription sub = QFeedSubscription.feedSubscription;
return query().selectFrom(FEED).where(JPAExpressions.selectOne().from(sub).where(sub.feed.eq(FEED)).notExists()).limit(max).fetch();
}
}
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.QFeed;
import com.commafeed.backend.model.QFeedSubscription;
import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.impl.JPAQuery;
@Singleton
public class FeedDAO extends GenericDAO<Feed> {
private static final QFeed FEED = QFeed.feed;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
public FeedDAO(EntityManager entityManager) {
super(entityManager, Feed.class);
}
public List<Feed> findByIds(List<Long> id) {
return query().selectFrom(FEED).where(FEED.id.in(id)).fetch();
}
public List<Feed> findNextUpdatable(int count, Instant lastLoginThreshold) {
JPAQuery<Feed> query = query().selectFrom(FEED)
.distinct()
// join on subscriptions to only refresh feeds that have subscribers
.join(SUBSCRIPTION)
.on(SUBSCRIPTION.feed.eq(FEED))
.where(FEED.disabledUntil.isNull().or(FEED.disabledUntil.lt(Instant.now())));
if (lastLoginThreshold != null) {
query.join(SUBSCRIPTION.user).where(SUBSCRIPTION.user.lastLogin.gt(lastLoginThreshold));
}
return query.orderBy(FEED.disabledUntil.asc()).limit(count).fetch();
}
public void setDisabledUntil(List<Long> feedIds, Instant date) {
updateQuery(FEED).set(FEED.disabledUntil, date).where(FEED.id.in(feedIds)).execute();
}
public Feed findByUrl(String normalizedUrl, String normalizedUrlHash) {
return query().selectFrom(FEED)
.where(FEED.normalizedUrlHash.eq(normalizedUrlHash))
.fetch()
.stream()
.filter(f -> StringUtils.equals(normalizedUrl, f.getNormalizedUrl()))
.findFirst()
.orElse(null);
}
public List<Feed> findWithoutSubscriptions(int max) {
QFeedSubscription sub = QFeedSubscription.feedSubscription;
return query().selectFrom(FEED).where(JPAExpressions.selectOne().from(sub).where(sub.feed.eq(FEED)).notExists()).limit(max).fetch();
}
}

72
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java
View File

@@ -1,36 +1,36 @@
package com.commafeed.backend.dao;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
@Singleton
public class FeedEntryContentDAO extends GenericDAO<FeedEntryContent> {
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
public FeedEntryContentDAO(EntityManager entityManager) {
super(entityManager, FeedEntryContent.class);
}
public List<FeedEntryContent> findExisting(String contentHash, String titleHash) {
return query().select(CONTENT).from(CONTENT).where(CONTENT.contentHash.eq(contentHash), CONTENT.titleHash.eq(titleHash)).fetch();
}
public long deleteWithoutEntries(int max) {
List<Long> ids = query().select(CONTENT.id)
.from(CONTENT)
.leftJoin(ENTRY)
.on(ENTRY.content.id.eq(CONTENT.id))
.where(ENTRY.id.isNull())
.limit(max)
.fetch();
return deleteQuery(CONTENT).where(CONTENT.id.in(ids)).execute();
}
}
package com.commafeed.backend.dao;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
@Singleton
public class FeedEntryContentDAO extends GenericDAO<FeedEntryContent> {
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
public FeedEntryContentDAO(EntityManager entityManager) {
super(entityManager, FeedEntryContent.class);
}
public List<FeedEntryContent> findExisting(String contentHash, String titleHash) {
return query().select(CONTENT).from(CONTENT).where(CONTENT.contentHash.eq(contentHash), CONTENT.titleHash.eq(titleHash)).fetch();
}
public long deleteWithoutEntries(int max) {
List<Long> ids = query().select(CONTENT.id)
.from(CONTENT)
.leftJoin(ENTRY)
.on(ENTRY.content.id.eq(CONTENT.id))
.where(ENTRY.id.isNull())
.limit(max)
.fetch();
return deleteQuery(CONTENT).where(CONTENT.id.in(ids)).execute();
}
}

146
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
View File

@@ -1,73 +1,73 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.QFeedEntry;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.NumberExpression;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Singleton
public class FeedEntryDAO extends GenericDAO<FeedEntry> {
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
public FeedEntryDAO(EntityManager entityManager) {
super(entityManager, FeedEntry.class);
}
public FeedEntry findExisting(String guidHash, Feed feed) {
return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne();
}
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max) {
NumberExpression<Long> count = ENTRY.id.count();
List<Tuple> tuples = query().select(ENTRY.feed.id, count)
.from(ENTRY)
.groupBy(ENTRY.feed)
.having(count.gt(maxCapacity))
.limit(max)
.fetch();
return tuples.stream().map(t -> new FeedCapacity(t.get(ENTRY.feed.id), t.get(count))).toList();
}
public int delete(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).limit(max).fetch();
return delete(list);
}
/**
* Delete entries older than a certain date
*/
public int deleteEntriesOlderThan(Instant olderThan, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY)
.where(ENTRY.published.lt(olderThan))
.orderBy(ENTRY.published.asc())
.limit(max)
.fetch();
return delete(list);
}
/**
* Delete the oldest entries of a feed
*/
public int deleteOldEntries(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).orderBy(ENTRY.published.asc()).limit(max).fetch();
return delete(list);
}
@AllArgsConstructor
@Getter
public static class FeedCapacity {
private Long id;
private Long capacity;
}
}
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.QFeedEntry;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.NumberExpression;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Singleton
public class FeedEntryDAO extends GenericDAO<FeedEntry> {
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
public FeedEntryDAO(EntityManager entityManager) {
super(entityManager, FeedEntry.class);
}
public FeedEntry findExisting(String guidHash, Feed feed) {
return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne();
}
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max) {
NumberExpression<Long> count = ENTRY.id.count();
List<Tuple> tuples = query().select(ENTRY.feed.id, count)
.from(ENTRY)
.groupBy(ENTRY.feed)
.having(count.gt(maxCapacity))
.limit(max)
.fetch();
return tuples.stream().map(t -> new FeedCapacity(t.get(ENTRY.feed.id), t.get(count))).toList();
}
public int delete(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).limit(max).fetch();
return delete(list);
}
/**
* Delete entries older than a certain date
*/
public int deleteEntriesOlderThan(Instant olderThan, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY)
.where(ENTRY.published.lt(olderThan))
.orderBy(ENTRY.published.asc())
.limit(max)
.fetch();
return delete(list);
}
/**
* Delete the oldest entries of a feed
*/
public int deleteOldEntries(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).orderBy(ENTRY.published.asc()).limit(max).fetch();
return delete(list);
}
@AllArgsConstructor
@Getter
public static class FeedCapacity {
private Long id;
private Long capacity;
}
}

472
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java
View File

@@ -1,236 +1,236 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.apache.commons.collections4.CollectionUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
import com.commafeed.backend.model.QFeedEntryStatus;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings.ReadingOrder;
import com.commafeed.frontend.model.UnreadCount;
import com.google.common.collect.Iterables;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.Tuple;
import com.querydsl.jpa.impl.JPAQuery;
@Singleton
public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
private final FeedEntryTagDAO feedEntryTagDAO;
private final CommaFeedConfiguration config;
public FeedEntryStatusDAO(EntityManager entityManager, FeedEntryTagDAO feedEntryTagDAO, CommaFeedConfiguration config) {
super(entityManager, FeedEntryStatus.class);
this.feedEntryTagDAO = feedEntryTagDAO;
this.config = config;
}
public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) {
List<FeedEntryStatus> statuses = query().selectFrom(STATUS).where(STATUS.entry.eq(entry), STATUS.subscription.eq(sub)).fetch();
FeedEntryStatus status = Iterables.getFirst(statuses, null);
return handleStatus(user, status, sub, entry);
}
/**
* creates an artificial "unread" status if status is null
*/
private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
if (status == null) {
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
boolean read = statusesInstantThreshold != null && entry.getPublished().isBefore(statusesInstantThreshold);
status = new FeedEntryStatus(user, sub, entry);
status.setRead(read);
status.setMarkable(!read);
} else {
status.setMarkable(true);
}
return status;
}
private void fetchTags(User user, List<FeedEntryStatus> statuses) {
Map<Long, List<FeedEntryTag>> tagsByEntryIds = feedEntryTagDAO.findByEntries(user,
statuses.stream().map(FeedEntryStatus::getEntry).toList());
for (FeedEntryStatus status : statuses) {
List<FeedEntryTag> tags = tagsByEntryIds.get(status.getEntry().getId());
status.setTags(tags == null ? List.of() : tags);
}
}
public List<FeedEntryStatus> findStarred(User user, Instant newerThan, int offset, int limit, ReadingOrder order,
boolean includeContent) {
JPAQuery<FeedEntryStatus> query = query().selectFrom(STATUS).where(STATUS.user.eq(user), STATUS.starred.isTrue());
if (includeContent) {
query.join(STATUS.entry).fetchJoin();
query.join(STATUS.entry.content).fetchJoin();
}
if (newerThan != null) {
query.where(STATUS.entryInserted.gt(newerThan));
}
if (order == ReadingOrder.asc) {
query.orderBy(STATUS.entryPublished.asc(), STATUS.id.asc());
} else {
query.orderBy(STATUS.entryPublished.desc(), STATUS.id.desc());
}
if (offset > -1) {
query.offset(offset);
}
if (limit > -1) {
query.limit(limit);
}
setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = query.fetch();
statuses.forEach(s -> s.setMarkable(true));
if (includeContent) {
fetchTags(user, statuses);
}
return statuses;
}
public List<FeedEntryStatus> findBySubscriptions(User user, List<FeedSubscription> subs, boolean unreadOnly,
List<FeedEntryKeyword> keywords, Instant newerThan, int offset, int limit, ReadingOrder order, boolean includeContent,
String tag, Long minEntryId, Long maxEntryId) {
Map<Long, List<FeedSubscription>> subsByFeedId = subs.stream().collect(Collectors.groupingBy(s -> s.getFeed().getId()));
JPAQuery<Tuple> query = query().select(ENTRY, STATUS).from(ENTRY);
query.leftJoin(ENTRY.statuses, STATUS).on(STATUS.subscription.in(subs));
query.where(ENTRY.feed.id.in(subsByFeedId.keySet()));
if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
query.join(ENTRY.content, CONTENT).fetchJoin();
}
if (CollectionUtils.isNotEmpty(keywords)) {
for (FeedEntryKeyword keyword : keywords) {
BooleanBuilder or = new BooleanBuilder();
or.or(CONTENT.content.containsIgnoreCase(keyword.getKeyword()));
or.or(CONTENT.title.containsIgnoreCase(keyword.getKeyword()));
if (keyword.getMode() == Mode.EXCLUDE) {
or.not();
}
query.where(or);
}
}
if (unreadOnly && tag == null) {
query.where(buildUnreadPredicate());
}
if (tag != null) {
BooleanBuilder and = new BooleanBuilder();
and.and(TAG.user.id.eq(user.getId()));
and.and(TAG.name.eq(tag));
query.join(ENTRY.tags, TAG).on(and);
}
if (newerThan != null) {
query.where(ENTRY.inserted.goe(newerThan));
}
if (minEntryId != null) {
query.where(ENTRY.id.gt(minEntryId));
}
if (maxEntryId != null) {
query.where(ENTRY.id.lt(maxEntryId));
}
if (order != null) {
if (order == ReadingOrder.asc) {
query.orderBy(ENTRY.published.asc(), ENTRY.id.asc());
} else {
query.orderBy(ENTRY.published.desc(), ENTRY.id.desc());
}
}
if (offset > -1) {
query.offset(offset);
}
if (limit > -1) {
query.limit(limit);
}
setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = new ArrayList<>();
List<Tuple> tuples = query.fetch();
for (Tuple tuple : tuples) {
FeedEntry e = tuple.get(ENTRY);
FeedEntryStatus s = tuple.get(STATUS);
for (FeedSubscription sub : subsByFeedId.get(e.getFeed().getId())) {
statuses.add(handleStatus(user, s, sub, e));
}
}
if (includeContent) {
fetchTags(user, statuses);
}
return statuses;
}
public UnreadCount getUnreadCount(FeedSubscription sub) {
JPAQuery<Tuple> query = query().select(ENTRY.count(), ENTRY.published.max())
.from(ENTRY)
.leftJoin(ENTRY.statuses, STATUS)
.on(STATUS.subscription.eq(sub))
.where(ENTRY.feed.eq(sub.getFeed()))
.where(buildUnreadPredicate());
Tuple tuple = query.fetchOne();
Long count = tuple.get(ENTRY.count());
Instant published = tuple.get(ENTRY.published.max());
return new UnreadCount(sub.getId(), count == null ? 0 : count, published);
}
private BooleanBuilder buildUnreadPredicate() {
BooleanBuilder or = new BooleanBuilder();
or.or(STATUS.read.isNull());
or.or(STATUS.read.isFalse());
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
if (statusesInstantThreshold != null) {
return or.and(ENTRY.published.goe(statusesInstantThreshold));
} else {
return or;
}
}
public long deleteOldStatuses(Instant olderThan, int limit) {
List<Long> ids = query().select(STATUS.id)
.from(STATUS)
.where(STATUS.entryInserted.lt(olderThan), STATUS.starred.isFalse())
.limit(limit)
.fetch();
return deleteQuery(STATUS).where(STATUS.id.in(ids)).execute();
}
}
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.apache.commons.collections4.CollectionUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
import com.commafeed.backend.model.QFeedEntryStatus;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings.ReadingOrder;
import com.commafeed.frontend.model.UnreadCount;
import com.google.common.collect.Iterables;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.Tuple;
import com.querydsl.jpa.impl.JPAQuery;
@Singleton
public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
private final FeedEntryTagDAO feedEntryTagDAO;
private final CommaFeedConfiguration config;
public FeedEntryStatusDAO(EntityManager entityManager, FeedEntryTagDAO feedEntryTagDAO, CommaFeedConfiguration config) {
super(entityManager, FeedEntryStatus.class);
this.feedEntryTagDAO = feedEntryTagDAO;
this.config = config;
}
public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) {
List<FeedEntryStatus> statuses = query().selectFrom(STATUS).where(STATUS.entry.eq(entry), STATUS.subscription.eq(sub)).fetch();
FeedEntryStatus status = Iterables.getFirst(statuses, null);
return handleStatus(user, status, sub, entry);
}
/**
* creates an artificial "unread" status if status is null
*/
private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
if (status == null) {
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
boolean read = statusesInstantThreshold != null && entry.getPublished().isBefore(statusesInstantThreshold);
status = new FeedEntryStatus(user, sub, entry);
status.setRead(read);
status.setMarkable(!read);
} else {
status.setMarkable(true);
}
return status;
}
private void fetchTags(User user, List<FeedEntryStatus> statuses) {
Map<Long, List<FeedEntryTag>> tagsByEntryIds = feedEntryTagDAO.findByEntries(user,
statuses.stream().map(FeedEntryStatus::getEntry).toList());
for (FeedEntryStatus status : statuses) {
List<FeedEntryTag> tags = tagsByEntryIds.get(status.getEntry().getId());
status.setTags(tags == null ? List.of() : tags);
}
}
public List<FeedEntryStatus> findStarred(User user, Instant newerThan, int offset, int limit, ReadingOrder order,
boolean includeContent) {
JPAQuery<FeedEntryStatus> query = query().selectFrom(STATUS).where(STATUS.user.eq(user), STATUS.starred.isTrue());
if (includeContent) {
query.join(STATUS.entry).fetchJoin();
query.join(STATUS.entry.content).fetchJoin();
}
if (newerThan != null) {
query.where(STATUS.entryInserted.gt(newerThan));
}
if (order == ReadingOrder.asc) {
query.orderBy(STATUS.entryPublished.asc(), STATUS.id.asc());
} else {
query.orderBy(STATUS.entryPublished.desc(), STATUS.id.desc());
}
if (offset > -1) {
query.offset(offset);
}
if (limit > -1) {
query.limit(limit);
}
setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = query.fetch();
statuses.forEach(s -> s.setMarkable(true));
if (includeContent) {
fetchTags(user, statuses);
}
return statuses;
}
public List<FeedEntryStatus> findBySubscriptions(User user, List<FeedSubscription> subs, boolean unreadOnly,
List<FeedEntryKeyword> keywords, Instant newerThan, int offset, int limit, ReadingOrder order, boolean includeContent,
String tag, Long minEntryId, Long maxEntryId) {
Map<Long, List<FeedSubscription>> subsByFeedId = subs.stream().collect(Collectors.groupingBy(s -> s.getFeed().getId()));
JPAQuery<Tuple> query = query().select(ENTRY, STATUS).from(ENTRY);
query.leftJoin(ENTRY.statuses, STATUS).on(STATUS.subscription.in(subs));
query.where(ENTRY.feed.id.in(subsByFeedId.keySet()));
if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
query.join(ENTRY.content, CONTENT).fetchJoin();
}
if (CollectionUtils.isNotEmpty(keywords)) {
for (FeedEntryKeyword keyword : keywords) {
BooleanBuilder or = new BooleanBuilder();
or.or(CONTENT.content.containsIgnoreCase(keyword.getKeyword()));
or.or(CONTENT.title.containsIgnoreCase(keyword.getKeyword()));
if (keyword.getMode() == Mode.EXCLUDE) {
or.not();
}
query.where(or);
}
}
if (unreadOnly && tag == null) {
query.where(buildUnreadPredicate());
}
if (tag != null) {
BooleanBuilder and = new BooleanBuilder();
and.and(TAG.user.id.eq(user.getId()));
and.and(TAG.name.eq(tag));
query.join(ENTRY.tags, TAG).on(and);
}
if (newerThan != null) {
query.where(ENTRY.inserted.goe(newerThan));
}
if (minEntryId != null) {
query.where(ENTRY.id.gt(minEntryId));
}
if (maxEntryId != null) {
query.where(ENTRY.id.lt(maxEntryId));
}
if (order != null) {
if (order == ReadingOrder.asc) {
query.orderBy(ENTRY.published.asc(), ENTRY.id.asc());
} else {
query.orderBy(ENTRY.published.desc(), ENTRY.id.desc());
}
}
if (offset > -1) {
query.offset(offset);
}
if (limit > -1) {
query.limit(limit);
}
setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = new ArrayList<>();
List<Tuple> tuples = query.fetch();
for (Tuple tuple : tuples) {
FeedEntry e = tuple.get(ENTRY);
FeedEntryStatus s = tuple.get(STATUS);
for (FeedSubscription sub : subsByFeedId.get(e.getFeed().getId())) {
statuses.add(handleStatus(user, s, sub, e));
}
}
if (includeContent) {
fetchTags(user, statuses);
}
return statuses;
}
public UnreadCount getUnreadCount(FeedSubscription sub) {
JPAQuery<Tuple> query = query().select(ENTRY.count(), ENTRY.published.max())
.from(ENTRY)
.leftJoin(ENTRY.statuses, STATUS)
.on(STATUS.subscription.eq(sub))
.where(ENTRY.feed.eq(sub.getFeed()))
.where(buildUnreadPredicate());
Tuple tuple = query.fetchOne();
Long count = tuple.get(ENTRY.count());
Instant published = tuple.get(ENTRY.published.max());
return new UnreadCount(sub.getId(), count == null ? 0 : count, published);
}
private BooleanBuilder buildUnreadPredicate() {
BooleanBuilder or = new BooleanBuilder();
or.or(STATUS.read.isNull());
or.or(STATUS.read.isFalse());
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
if (statusesInstantThreshold != null) {
return or.and(ENTRY.published.goe(statusesInstantThreshold));
} else {
return or;
}
}
public long deleteOldStatuses(Instant olderThan, int limit) {
List<Long> ids = query().select(STATUS.id)
.from(STATUS)
.where(STATUS.entryInserted.lt(olderThan), STATUS.starred.isFalse())
.limit(limit)
.fetch();
return deleteQuery(STATUS).where(STATUS.id.in(ids)).execute();
}
}

78
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java
View File

@@ -1,39 +1,39 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
@Singleton
public class FeedEntryTagDAO extends GenericDAO<FeedEntryTag> {
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
public FeedEntryTagDAO(EntityManager entityManager) {
super(entityManager, FeedEntryTag.class);
}
public List<String> findByUser(User user) {
return query().selectDistinct(TAG.name).from(TAG).where(TAG.user.eq(user)).fetch();
}
public List<FeedEntryTag> findByEntry(User user, FeedEntry entry) {
return query().selectFrom(TAG).where(TAG.user.eq(user), TAG.entry.eq(entry)).fetch();
}
public Map<Long, List<FeedEntryTag>> findByEntries(User user, List<FeedEntry> entries) {
return query().selectFrom(TAG)
.where(TAG.user.eq(user), TAG.entry.in(entries))
.fetch()
.stream()
.collect(Collectors.groupingBy(t -> t.getEntry().getId()));
}
}
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
@Singleton
public class FeedEntryTagDAO extends GenericDAO<FeedEntryTag> {
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
public FeedEntryTagDAO(EntityManager entityManager) {
super(entityManager, FeedEntryTag.class);
}
public List<String> findByUser(User user) {
return query().selectDistinct(TAG.name).from(TAG).where(TAG.user.eq(user)).fetch();
}
public List<FeedEntryTag> findByEntry(User user, FeedEntry entry) {
return query().selectFrom(TAG).where(TAG.user.eq(user), TAG.entry.eq(entry)).fetch();
}
public Map<Long, List<FeedEntryTag>> findByEntries(User user, List<FeedEntry> entries) {
return query().selectFrom(TAG)
.where(TAG.user.eq(user), TAG.entry.in(entries))
.fetch()
.stream()
.collect(Collectors.groupingBy(t -> t.getEntry().getId()));
}
}

260
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java
View File

@@ -1,130 +1,130 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.event.service.spi.EventListenerRegistry;
import org.hibernate.event.spi.EventType;
import org.hibernate.event.spi.PostCommitInsertEventListener;
import org.hibernate.event.spi.PostInsertEvent;
import org.hibernate.persister.entity.EntityPersister;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.QFeedSubscription;
import com.commafeed.backend.model.User;
import com.google.common.collect.Iterables;
import com.querydsl.jpa.JPQLQuery;
@Singleton
public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> {
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private final EntityManager entityManager;
public FeedSubscriptionDAO(EntityManager entityManager) {
super(entityManager, FeedSubscription.class);
this.entityManager = entityManager;
}
public void onPostCommitInsert(Consumer<FeedSubscription> consumer) {
entityManager.unwrap(SharedSessionContractImplementor.class)
.getFactory()
.getServiceRegistry()
.getService(EventListenerRegistry.class)
.getEventListenerGroup(EventType.POST_COMMIT_INSERT)
.appendListener(new PostCommitInsertEventListener() {
@Override
public void onPostInsert(PostInsertEvent event) {
if (event.getEntity() instanceof FeedSubscription s) {
consumer.accept(s);
}
}
@Override
public boolean requiresPostCommitHandling(EntityPersister persister) {
return true;
}
@Override
public void onPostInsertCommitFailed(PostInsertEvent event) {
// do nothing
}
});
}
public FeedSubscription findById(User user, Long id) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.id.eq(id))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
return initRelations(Iterables.getFirst(subs, null));
}
public List<FeedSubscription> findByFeed(Feed feed) {
return query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.feed.eq(feed)).fetch();
}
public FeedSubscription findByFeed(User user, Feed feed) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.feed.eq(feed))
.fetch();
return initRelations(Iterables.getFirst(subs, null));
}
public List<FeedSubscription> findAll(User user) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
return initRelations(subs);
}
public Long count(User user) {
return query().select(SUBSCRIPTION.count()).from(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user)).fetchOne();
}
public List<FeedSubscription> findByCategory(User user, FeedCategory category) {
JPQLQuery<FeedSubscription> query = query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user));
if (category == null) {
query.where(SUBSCRIPTION.category.isNull());
} else {
query.where(SUBSCRIPTION.category.eq(category));
}
return initRelations(query.fetch());
}
public List<FeedSubscription> findByCategories(User user, List<FeedCategory> categories) {
Set<Long> categoryIds = categories.stream().map(AbstractModel::getId).collect(Collectors.toSet());
return findAll(user).stream().filter(s -> s.getCategory() != null && categoryIds.contains(s.getCategory().getId())).toList();
}
private List<FeedSubscription> initRelations(List<FeedSubscription> list) {
list.forEach(this::initRelations);
return list;
}
private FeedSubscription initRelations(FeedSubscription sub) {
if (sub != null) {
Models.initialize(sub.getFeed());
Models.initialize(sub.getCategory());
}
return sub;
}
}
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.event.service.spi.EventListenerRegistry;
import org.hibernate.event.spi.EventType;
import org.hibernate.event.spi.PostCommitInsertEventListener;
import org.hibernate.event.spi.PostInsertEvent;
import org.hibernate.persister.entity.EntityPersister;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.QFeedSubscription;
import com.commafeed.backend.model.User;
import com.google.common.collect.Iterables;
import com.querydsl.jpa.JPQLQuery;
@Singleton
public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> {
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private final EntityManager entityManager;
public FeedSubscriptionDAO(EntityManager entityManager) {
super(entityManager, FeedSubscription.class);
this.entityManager = entityManager;
}
public void onPostCommitInsert(Consumer<FeedSubscription> consumer) {
entityManager.unwrap(SharedSessionContractImplementor.class)
.getFactory()
.getServiceRegistry()
.getService(EventListenerRegistry.class)
.getEventListenerGroup(EventType.POST_COMMIT_INSERT)
.appendListener(new PostCommitInsertEventListener() {
@Override
public void onPostInsert(PostInsertEvent event) {
if (event.getEntity() instanceof FeedSubscription s) {
consumer.accept(s);
}
}
@Override
public boolean requiresPostCommitHandling(EntityPersister persister) {
return true;
}
@Override
public void onPostInsertCommitFailed(PostInsertEvent event) {
// do nothing
}
});
}
public FeedSubscription findById(User user, Long id) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.id.eq(id))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
return initRelations(Iterables.getFirst(subs, null));
}
public List<FeedSubscription> findByFeed(Feed feed) {
return query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.feed.eq(feed)).fetch();
}
public FeedSubscription findByFeed(User user, Feed feed) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.feed.eq(feed))
.fetch();
return initRelations(Iterables.getFirst(subs, null));
}
public List<FeedSubscription> findAll(User user) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
return initRelations(subs);
}
public Long count(User user) {
return query().select(SUBSCRIPTION.count()).from(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user)).fetchOne();
}
public List<FeedSubscription> findByCategory(User user, FeedCategory category) {
JPQLQuery<FeedSubscription> query = query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user));
if (category == null) {
query.where(SUBSCRIPTION.category.isNull());
} else {
query.where(SUBSCRIPTION.category.eq(category));
}
return initRelations(query.fetch());
}
public List<FeedSubscription> findByCategories(User user, List<FeedCategory> categories) {
Set<Long> categoryIds = categories.stream().map(AbstractModel::getId).collect(Collectors.toSet());
return findAll(user).stream().filter(s -> s.getCategory() != null && categoryIds.contains(s.getCategory().getId())).toList();
}
private List<FeedSubscription> initRelations(List<FeedSubscription> list) {
list.forEach(this::initRelations);
return list;
}
private FeedSubscription initRelations(FeedSubscription sub) {
if (sub != null) {
Models.initialize(sub.getFeed());
Models.initialize(sub.getCategory());
}
return sub;
}
}

152
commafeed-server/src/main/java/com/commafeed/backend/dao/GenericDAO.java
View File

@@ -1,76 +1,76 @@
package com.commafeed.backend.dao;
import java.time.Duration;
import java.util.Collection;
import jakarta.persistence.EntityManager;
import org.hibernate.Session;
import org.hibernate.jpa.SpecHints;
import com.commafeed.backend.model.AbstractModel;
import com.querydsl.core.types.EntityPath;
import com.querydsl.jpa.impl.JPADeleteClause;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
import com.querydsl.jpa.impl.JPAUpdateClause;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public abstract class GenericDAO<T extends AbstractModel> {
private final EntityManager entityManager;
private final Class<T> entityClass;
protected JPAQueryFactory query() {
return new JPAQueryFactory(entityManager);
}
protected JPAUpdateClause updateQuery(EntityPath<T> entityPath) {
return new JPAUpdateClause(entityManager, entityPath);
}
protected JPADeleteClause deleteQuery(EntityPath<T> entityPath) {
return new JPADeleteClause(entityManager, entityPath);
}
@SuppressWarnings("deprecation")
public void saveOrUpdate(T model) {
entityManager.unwrap(Session.class).saveOrUpdate(model);
}
public void saveOrUpdate(Collection<T> models) {
models.forEach(this::saveOrUpdate);
}
public void persist(T model) {
entityManager.persist(model);
}
public T merge(T model) {
return entityManager.merge(model);
}
public T findById(Long id) {
return entityManager.find(entityClass, id);
}
public void delete(T object) {
if (object != null) {
entityManager.remove(object);
}
}
public int delete(Collection<T> objects) {
objects.forEach(this::delete);
return objects.size();
}
protected void setTimeout(JPAQuery<?> query, Duration timeout) {
if (!timeout.isZero()) {
query.setHint(SpecHints.HINT_SPEC_QUERY_TIMEOUT, Math.toIntExact(timeout.toMillis()));
}
}
}
package com.commafeed.backend.dao;
import java.time.Duration;
import java.util.Collection;
import jakarta.persistence.EntityManager;
import org.hibernate.Session;
import org.hibernate.jpa.SpecHints;
import com.commafeed.backend.model.AbstractModel;
import com.querydsl.core.types.EntityPath;
import com.querydsl.jpa.impl.JPADeleteClause;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
import com.querydsl.jpa.impl.JPAUpdateClause;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public abstract class GenericDAO<T extends AbstractModel> {
private final EntityManager entityManager;
private final Class<T> entityClass;
protected JPAQueryFactory query() {
return new JPAQueryFactory(entityManager);
}
protected JPAUpdateClause updateQuery(EntityPath<T> entityPath) {
return new JPAUpdateClause(entityManager, entityPath);
}
protected JPADeleteClause deleteQuery(EntityPath<T> entityPath) {
return new JPADeleteClause(entityManager, entityPath);
}
@SuppressWarnings("deprecation")
public void saveOrUpdate(T model) {
entityManager.unwrap(Session.class).saveOrUpdate(model);
}
public void saveOrUpdate(Collection<T> models) {
models.forEach(this::saveOrUpdate);
}
public void persist(T model) {
entityManager.persist(model);
}
public T merge(T model) {
return entityManager.merge(model);
}
public T findById(Long id) {
return entityManager.find(entityClass, id);
}
public void delete(T object) {
if (object != null) {
entityManager.remove(object);
}
}
public int delete(Collection<T> objects) {
objects.forEach(this::delete);
return objects.size();
}
protected void setTimeout(JPAQuery<?> query, Duration timeout) {
if (!timeout.isZero()) {
query.setHint(SpecHints.HINT_SPEC_QUERY_TIMEOUT, Math.toIntExact(timeout.toMillis()));
}
}
}

38
commafeed-server/src/main/java/com/commafeed/backend/dao/UnitOfWork.java
View File

@@ -1,19 +1,19 @@
package com.commafeed.backend.dao;
import java.util.concurrent.Callable;
import jakarta.inject.Singleton;
import io.quarkus.narayana.jta.QuarkusTransaction;
@Singleton
public class UnitOfWork {
public void run(Runnable runnable) {
QuarkusTransaction.joiningExisting().run(runnable);
}
public <T> T call(Callable<T> callable) {
return QuarkusTransaction.joiningExisting().call(callable);
}
}
package com.commafeed.backend.dao;
import java.util.concurrent.Callable;
import jakarta.inject.Singleton;
import io.quarkus.narayana.jta.QuarkusTransaction;
@Singleton
public class UnitOfWork {
public void run(Runnable runnable) {
QuarkusTransaction.joiningExisting().run(runnable);
}
public <T> T call(Callable<T> callable) {
return QuarkusTransaction.joiningExisting().call(callable);
}
}

66
commafeed-server/src/main/java/com/commafeed/backend/dao/UserDAO.java
View File

@@ -1,33 +1,33 @@
package com.commafeed.backend.dao;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
@Singleton
public class UserDAO extends GenericDAO<User> {
private static final QUser USER = QUser.user;
public UserDAO(EntityManager entityManager) {
super(entityManager, User.class);
}
public User findByName(String name) {
return query().selectFrom(USER).where(USER.name.equalsIgnoreCase(name)).fetchOne();
}
public User findByApiKey(String key) {
return query().selectFrom(USER).where(USER.apiKey.equalsIgnoreCase(key)).fetchOne();
}
public User findByEmail(String email) {
return query().selectFrom(USER).where(USER.email.equalsIgnoreCase(email)).fetchOne();
}
public long count() {
return query().select(USER.count()).from(USER).fetchOne();
}
}
package com.commafeed.backend.dao;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
@Singleton
public class UserDAO extends GenericDAO<User> {
private static final QUser USER = QUser.user;
public UserDAO(EntityManager entityManager) {
super(entityManager, User.class);
}
public User findByName(String name) {
return query().selectFrom(USER).where(USER.name.equalsIgnoreCase(name)).fetchOne();
}
public User findByApiKey(String key) {
return query().selectFrom(USER).where(USER.apiKey.equalsIgnoreCase(key)).fetchOne();
}
public User findByEmail(String email) {
return query().selectFrom(USER).where(USER.email.equalsIgnoreCase(email)).fetchOne();
}
public long count() {
return query().select(USER.count()).from(USER).fetchOne();
}
}

70
commafeed-server/src/main/java/com/commafeed/backend/dao/UserRoleDAO.java
View File

@@ -1,35 +1,35 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUserRole;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
@Singleton
public class UserRoleDAO extends GenericDAO<UserRole> {
private static final QUserRole ROLE = QUserRole.userRole;
public UserRoleDAO(EntityManager entityManager) {
super(entityManager, UserRole.class);
}
public List<UserRole> findAll() {
return query().selectFrom(ROLE).leftJoin(ROLE.user).fetchJoin().distinct().fetch();
}
public List<UserRole> findAll(User user) {
return query().selectFrom(ROLE).where(ROLE.user.eq(user)).distinct().fetch();
}
public Set<Role> findRoles(User user) {
return findAll(user).stream().map(UserRole::getRole).collect(Collectors.toSet());
}
}
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUserRole;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
@Singleton
public class UserRoleDAO extends GenericDAO<UserRole> {
private static final QUserRole ROLE = QUserRole.userRole;
public UserRoleDAO(EntityManager entityManager) {
super(entityManager, UserRole.class);
}
public List<UserRole> findAll() {
return query().selectFrom(ROLE).leftJoin(ROLE.user).fetchJoin().distinct().fetch();
}
public List<UserRole> findAll(User user) {
return query().selectFrom(ROLE).where(ROLE.user.eq(user)).distinct().fetch();
}
public Set<Role> findRoles(User user) {
return findAll(user).stream().map(UserRole::getRole).collect(Collectors.toSet());
}
}

44
commafeed-server/src/main/java/com/commafeed/backend/dao/UserSettingsDAO.java
View File

@@ -1,22 +1,22 @@
package com.commafeed.backend.dao;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUserSettings;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings;
@Singleton
public class UserSettingsDAO extends GenericDAO<UserSettings> {
private static final QUserSettings SETTINGS = QUserSettings.userSettings;
public UserSettingsDAO(EntityManager entityManager) {
super(entityManager, UserSettings.class);
}
public UserSettings findByUser(User user) {
return query().selectFrom(SETTINGS).where(SETTINGS.user.eq(user)).fetchFirst();
}
}
package com.commafeed.backend.dao;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUserSettings;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings;
@Singleton
public class UserSettingsDAO extends GenericDAO<UserSettings> {
private static final QUserSettings SETTINGS = QUserSettings.userSettings;
public UserSettingsDAO(EntityManager entityManager) {
super(entityManager, UserSettings.class);
}
public UserSettings findByUser(User user) {
return query().selectFrom(SETTINGS).where(SETTINGS.user.eq(user)).fetchFirst();
}
}

98
commafeed-server/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java
View File

@@ -1,49 +1,49 @@
package com.commafeed.backend.favicon;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class AbstractFaviconFetcher {
private static final List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
public abstract Favicon fetch(Feed feed);
protected boolean isValidIconResponse(byte[] content, String contentType) {
if (content == null) {
return false;
}
long length = content.length;
if (StringUtils.isNotBlank(contentType)) {
contentType = contentType.split(";")[0];
}
if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
log.debug("Content-Type {} is blacklisted", contentType);
return false;
}
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
return true;
}
}
package com.commafeed.backend.favicon;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class AbstractFaviconFetcher {
private static final List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
public abstract Favicon fetch(Feed feed);
protected boolean isValidIconResponse(byte[] content, String contentType) {
if (content == null) {
return false;
}
long length = content.length;
if (StringUtils.isNotBlank(contentType)) {
contentType = contentType.split(";")[0];
}
if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
log.debug("Content-Type {} is blacklisted", contentType);
return false;
}
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
return true;
}
}

266
commafeed-server/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java
View File

@@ -1,133 +1,133 @@
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Inspired/Ported from https://github.com/potatolondon/getfavicon
*
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(Integer.MIN_VALUE)
public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
Favicon icon = fetch(feed.getLink());
if (icon == null) {
icon = fetch(feed.getUrl());
}
return icon;
}
private Favicon fetch(String url) {
if (url == null) {
log.debug("url is null");
return null;
}
int doubleSlash = url.indexOf("//");
if (doubleSlash == -1) {
doubleSlash = 0;
} else {
doubleSlash += 2;
}
int firstSlash = url.indexOf('/', doubleSlash);
if (firstSlash != -1) {
url = url.substring(0, firstSlash);
}
Favicon icon = getIconAtRoot(url);
if (icon == null) {
icon = getIconInPage(url);
}
return icon;
}
private Favicon getIconAtRoot(String url) {
byte[] bytes = null;
String contentType = null;
try {
url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico";
log.debug("getting root icon at {}", url);
HttpResult result = getter.get(url);
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url);
log.trace("Failed to retrieve iconAtRoot for url {}: ", url, e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private Favicon getIconInPage(String url) {
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.getContent()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon");
log.trace("Failed to retrieve page to find icon", e);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
String href = icons.get(0).attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
log.debug("Found unconfirmed iconInPage at {}", href);
byte[] bytes;
String contentType;
try {
HttpResult result = getter.get(href);
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href);
log.trace("Failed to retrieve icon found in page {}", href, e);
return null;
}
if (!isValidIconResponse(bytes, contentType)) {
log.debug("Invalid icon found for {}", href);
return null;
}
return new Favicon(bytes, contentType);
}
}
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Inspired/Ported from https://github.com/potatolondon/getfavicon
*
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(Integer.MIN_VALUE)
public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
Favicon icon = fetch(feed.getLink());
if (icon == null) {
icon = fetch(feed.getUrl());
}
return icon;
}
private Favicon fetch(String url) {
if (url == null) {
log.debug("url is null");
return null;
}
int doubleSlash = url.indexOf("//");
if (doubleSlash == -1) {
doubleSlash = 0;
} else {
doubleSlash += 2;
}
int firstSlash = url.indexOf('/', doubleSlash);
if (firstSlash != -1) {
url = url.substring(0, firstSlash);
}
Favicon icon = getIconAtRoot(url);
if (icon == null) {
icon = getIconInPage(url);
}
return icon;
}
private Favicon getIconAtRoot(String url) {
byte[] bytes = null;
String contentType = null;
try {
url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico";
log.debug("getting root icon at {}", url);
HttpResult result = getter.get(url);
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url);
log.trace("Failed to retrieve iconAtRoot for url {}: ", url, e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private Favicon getIconInPage(String url) {
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.getContent()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon");
log.trace("Failed to retrieve page to find icon", e);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
String href = icons.get(0).attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
log.debug("Found unconfirmed iconInPage at {}", href);
byte[] bytes;
String contentType;
try {
HttpResult result = getter.get(href);
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href);
log.trace("Failed to retrieve icon found in page {}", href, e);
return null;
}
if (!isValidIconResponse(bytes, contentType)) {
log.debug("Invalid icon found for {}", href);
return null;
}
return new Favicon(bytes, contentType);
}
}

146
commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
View File

@@ -1,73 +1,73 @@
package com.commafeed.backend.favicon;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("www.facebook.com")) {
return null;
}
String userName = extractUserName(url);
if (userName == null) {
return null;
}
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
byte[] bytes = null;
String contentType = null;
try {
log.debug("Getting Facebook user's icon, {}", url);
HttpResult iconResult = getter.get(iconUrl);
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private String extractUserName(String url) {
URI uri;
try {
uri = new URI(url);
} catch (URISyntaxException e) {
log.debug("could not parse url", e);
return null;
}
List<NameValuePair> params = new URIBuilder(uri).getQueryParams();
return params.stream().filter(p -> "id".equals(p.getName())).map(NameValuePair::getValue).findFirst().orElse(null);
}
}
package com.commafeed.backend.favicon;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("www.facebook.com")) {
return null;
}
String userName = extractUserName(url);
if (userName == null) {
return null;
}
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
byte[] bytes = null;
String contentType = null;
try {
log.debug("Getting Facebook user's icon, {}", url);
HttpResult iconResult = getter.get(iconUrl);
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private String extractUserName(String url) {
URI uri;
try {
uri = new URI(url);
} catch (URISyntaxException e) {
log.debug("could not parse url", e);
return null;
}
List<NameValuePair> params = new URIBuilder(uri).getQueryParams();
return params.stream().filter(p -> "id".equals(p.getName())).map(NameValuePair::getValue).findFirst().orElse(null);
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/favicon/Favicon.java
View File

@@ -1,31 +1,31 @@
package com.commafeed.backend.favicon;
import jakarta.ws.rs.core.MediaType;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Getter
@Slf4j
public class Favicon {
private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.valueOf("image/x-icon");
private final byte[] icon;
private final MediaType mediaType;
public Favicon(byte[] icon, String contentType) {
this(icon, parseMediaType(contentType));
}
private static MediaType parseMediaType(String contentType) {
try {
return MediaType.valueOf(contentType);
} catch (Exception e) {
log.debug("invalid content type '{}' received, returning default value", contentType);
return DEFAULT_MEDIA_TYPE;
}
}
package com.commafeed.backend.favicon;
import jakarta.ws.rs.core.MediaType;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Getter
@Slf4j
public class Favicon {
private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.valueOf("image/x-icon");
private final byte[] icon;
private final MediaType mediaType;
public Favicon(byte[] icon, String contentType) {
this(icon, parseMediaType(contentType));
}
private static MediaType parseMediaType(String contentType) {
try {
return MediaType.valueOf(contentType);
} catch (Exception e) {
log.debug("invalid content type '{}' received, returning default value", contentType);
return DEFAULT_MEDIA_TYPE;
}
}
}

270
commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
View File

@@ -1,135 +1,135 @@
package com.commafeed.backend.favicon;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Optional;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.UriBuilder;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HostNotAllowedException;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.SchemeNotAllowedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.model.Feed;
import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
private static final JsonPointer CHANNEL_THUMBNAIL_URL = JsonPointer.compile("/items/0/snippet/thumbnails/default/url");
private static final JsonPointer PLAYLIST_CHANNEL_ID = JsonPointer.compile("/items/0/snippet/channelId");
private final HttpGetter getter;
private final CommaFeedConfiguration config;
private final ObjectMapper objectMapper;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
return null;
}
Optional<String> googleAuthKey = config.googleAuthKey();
if (googleAuthKey.isEmpty()) {
log.debug("no google auth key configured");
return null;
}
byte[] bytes = null;
String contentType = null;
try {
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
Optional<NameValuePair> channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id")).findFirst();
Optional<NameValuePair> playlistId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id")).findFirst();
byte[] response = null;
if (userId.isPresent()) {
log.debug("contacting youtube api for user {}", userId.get().getValue());
response = fetchForUser(googleAuthKey.get(), userId.get().getValue());
} else if (channelId.isPresent()) {
log.debug("contacting youtube api for channel {}", channelId.get().getValue());
response = fetchForChannel(googleAuthKey.get(), channelId.get().getValue());
} else if (playlistId.isPresent()) {
log.debug("contacting youtube api for playlist {}", playlistId.get().getValue());
response = fetchForPlaylist(googleAuthKey.get(), playlistId.get().getValue());
}
if (ArrayUtils.isEmpty(response)) {
log.debug("youtube api returned empty response");
return null;
}
JsonNode thumbnailUrl = objectMapper.readTree(response).at(CHANNEL_THUMBNAIL_URL);
if (thumbnailUrl.isMissingNode()) {
log.debug("youtube api returned invalid response");
return null;
}
HttpResult iconResult = getter.get(thumbnailUrl.asText());
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
} catch (Exception e) {
log.error("Failed to retrieve YouTube icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private byte[] fetchForUser(String googleAuthKey, String userId)
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)
.queryParam("forUsername", userId)
.build();
return getter.get(uri.toString()).getContent();
}
private byte[] fetchForChannel(String googleAuthKey, String channelId)
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)
.queryParam("id", channelId)
.build();
return getter.get(uri.toString()).getContent();
}
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)
.queryParam("id", playlistId)
.build();
byte[] playlistBytes = getter.get(uri.toString()).getContent();
JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID);
if (channelId.isMissingNode()) {
return null;
}
return fetchForChannel(googleAuthKey, channelId.asText());
}
}
package com.commafeed.backend.favicon;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Optional;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.UriBuilder;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HostNotAllowedException;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.SchemeNotAllowedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.model.Feed;
import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
private static final JsonPointer CHANNEL_THUMBNAIL_URL = JsonPointer.compile("/items/0/snippet/thumbnails/default/url");
private static final JsonPointer PLAYLIST_CHANNEL_ID = JsonPointer.compile("/items/0/snippet/channelId");
private final HttpGetter getter;
private final CommaFeedConfiguration config;
private final ObjectMapper objectMapper;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
return null;
}
Optional<String> googleAuthKey = config.googleAuthKey();
if (googleAuthKey.isEmpty()) {
log.debug("no google auth key configured");
return null;
}
byte[] bytes = null;
String contentType = null;
try {
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
Optional<NameValuePair> channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id")).findFirst();
Optional<NameValuePair> playlistId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id")).findFirst();
byte[] response = null;
if (userId.isPresent()) {
log.debug("contacting youtube api for user {}", userId.get().getValue());
response = fetchForUser(googleAuthKey.get(), userId.get().getValue());
} else if (channelId.isPresent()) {
log.debug("contacting youtube api for channel {}", channelId.get().getValue());
response = fetchForChannel(googleAuthKey.get(), channelId.get().getValue());
} else if (playlistId.isPresent()) {
log.debug("contacting youtube api for playlist {}", playlistId.get().getValue());
response = fetchForPlaylist(googleAuthKey.get(), playlistId.get().getValue());
}
if (ArrayUtils.isEmpty(response)) {
log.debug("youtube api returned empty response");
return null;
}
JsonNode thumbnailUrl = objectMapper.readTree(response).at(CHANNEL_THUMBNAIL_URL);
if (thumbnailUrl.isMissingNode()) {
log.debug("youtube api returned invalid response");
return null;
}
HttpResult iconResult = getter.get(thumbnailUrl.asText());
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
} catch (Exception e) {
log.error("Failed to retrieve YouTube icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private byte[] fetchForUser(String googleAuthKey, String userId)
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)
.queryParam("forUsername", userId)
.build();
return getter.get(uri.toString()).getContent();
}
private byte[] fetchForChannel(String googleAuthKey, String channelId)
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)
.queryParam("id", channelId)
.build();
return getter.get(uri.toString()).getContent();
}
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)
.queryParam("id", playlistId)
.build();
byte[] playlistBytes = getter.get(uri.toString()).getContent();
JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID);
if (channelId.isMissingNode()) {
return null;
}
return fetchForChannel(googleAuthKey, channelId.asText());
}
}

78
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedEntryKeyword.java
View File

@@ -1,39 +1,39 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
/**
* A keyword used in a search query
*/
@Getter
@RequiredArgsConstructor
public class FeedEntryKeyword {
public enum Mode {
INCLUDE, EXCLUDE
}
private final String keyword;
private final Mode mode;
public static List<FeedEntryKeyword> fromQueryString(String keywords) {
List<FeedEntryKeyword> list = new ArrayList<>();
if (keywords != null) {
for (String keyword : StringUtils.split(keywords)) {
boolean not = false;
if (keyword.startsWith("-") || keyword.startsWith("!")) {
not = true;
keyword = keyword.substring(1);
}
list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
}
}
return list;
}
}
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
/**
* A keyword used in a search query
*/
@Getter
@RequiredArgsConstructor
public class FeedEntryKeyword {
public enum Mode {
INCLUDE, EXCLUDE
}
private final String keyword;
private final Mode mode;
public static List<FeedEntryKeyword> fromQueryString(String keywords) {
List<FeedEntryKeyword> list = new ArrayList<>();
if (keywords != null) {
for (String keyword : StringUtils.split(keywords)) {
boolean not = false;
if (keyword.startsWith("-") || keyword.startsWith("!")) {
not = true;
keyword = keyword.substring(1);
}
list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
}
}
return list;
}
}

246
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
View File

@@ -1,123 +1,123 @@
package com.commafeed.backend.feed;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HostNotAllowedException;
import com.commafeed.backend.HttpGetter.HttpRequest;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.SchemeNotAllowedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.parser.FeedParser;
import com.commafeed.backend.feed.parser.FeedParser.FeedParsingException;
import com.commafeed.backend.feed.parser.FeedParserResult;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import io.quarkus.arc.All;
import lombok.extern.slf4j.Slf4j;
/**
* Fetches a feed then parses it
*/
@Slf4j
@Singleton
public class FeedFetcher {
private final FeedParser parser;
private final HttpGetter getter;
private final List<FeedURLProvider> urlProviders;
public FeedFetcher(FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
this.parser = parser;
this.getter = getter;
this.urlProviders = urlProviders;
}
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
Instant lastPublishedDate, String lastContentHash) throws FeedParsingException, IOException, NotModifiedException,
TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException, NoFeedFoundException {
log.debug("Fetching feed {}", feedUrl);
HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
byte[] content = result.getContent();
FeedParserResult parserResult;
try {
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
} catch (FeedParsingException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, new String(result.getContent(), StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
result = getter.get(HttpRequest.builder(extractedUrl).lastModified(lastModified).eTag(eTag).build());
content = result.getContent();
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
} else {
throw new NoFeedFoundException(e);
}
} else {
throw e;
}
}
if (content == null) {
throw new IOException("Feed content is empty.");
}
boolean lastModifiedHeaderValueChanged = !StringUtils.equals(lastModified, result.getLastModifiedSince());
boolean etagHeaderValueChanged = !StringUtils.equals(eTag, result.getETag());
String hash = Digests.sha1Hex(content);
if (lastContentHash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException("content hash not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
}
if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
}
return new FeedFetcherResult(parserResult, result.getUrlAfterRedirect(), result.getLastModifiedSince(), result.getETag(), hash,
result.getValidFor());
}
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) {
for (FeedURLProvider urlProvider : urlProviders) {
String feedUrl = urlProvider.get(url, urlContent);
if (feedUrl != null) {
return feedUrl;
}
}
return null;
}
public record FeedFetcherResult(FeedParserResult feed, String urlAfterRedirect, String lastModifiedHeader, String lastETagHeader,
String contentHash, Duration validFor) {
}
public static class NoFeedFoundException extends Exception {
private static final long serialVersionUID = 1L;
public NoFeedFoundException(Throwable cause) {
super("This URL does not point to an RSS feed or a website with an RSS feed.", cause);
}
}
}
package com.commafeed.backend.feed;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HostNotAllowedException;
import com.commafeed.backend.HttpGetter.HttpRequest;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.SchemeNotAllowedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.parser.FeedParser;
import com.commafeed.backend.feed.parser.FeedParser.FeedParsingException;
import com.commafeed.backend.feed.parser.FeedParserResult;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import io.quarkus.arc.All;
import lombok.extern.slf4j.Slf4j;
/**
* Fetches a feed then parses it
*/
@Slf4j
@Singleton
public class FeedFetcher {
private final FeedParser parser;
private final HttpGetter getter;
private final List<FeedURLProvider> urlProviders;
public FeedFetcher(FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
this.parser = parser;
this.getter = getter;
this.urlProviders = urlProviders;
}
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
Instant lastPublishedDate, String lastContentHash) throws FeedParsingException, IOException, NotModifiedException,
TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException, NoFeedFoundException {
log.debug("Fetching feed {}", feedUrl);
HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
byte[] content = result.getContent();
FeedParserResult parserResult;
try {
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
} catch (FeedParsingException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, new String(result.getContent(), StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
result = getter.get(HttpRequest.builder(extractedUrl).lastModified(lastModified).eTag(eTag).build());
content = result.getContent();
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
} else {
throw new NoFeedFoundException(e);
}
} else {
throw e;
}
}
if (content == null) {
throw new IOException("Feed content is empty.");
}
boolean lastModifiedHeaderValueChanged = !StringUtils.equals(lastModified, result.getLastModifiedSince());
boolean etagHeaderValueChanged = !StringUtils.equals(eTag, result.getETag());
String hash = Digests.sha1Hex(content);
if (lastContentHash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException("content hash not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
}
if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
}
return new FeedFetcherResult(parserResult, result.getUrlAfterRedirect(), result.getLastModifiedSince(), result.getETag(), hash,
result.getValidFor());
}
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) {
for (FeedURLProvider urlProvider : urlProviders) {
String feedUrl = urlProvider.get(url, urlContent);
if (feedUrl != null) {
return feedUrl;
}
}
return null;
}
public record FeedFetcherResult(FeedParserResult feed, String urlAfterRedirect, String lastModifiedHeader, String lastETagHeader,
String contentHash, Duration validFor) {
}
public static class NoFeedFoundException extends Exception {
private static final long serialVersionUID = 1L;
public NoFeedFoundException(Throwable cause) {
super("This URL does not point to an RSS feed or a website with an RSS feed.", cause);
}
}
}

428
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshEngine.java
View File

@@ -1,214 +1,214 @@
package com.commafeed.backend.feed;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedRefreshEngine {
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedRefreshWorker worker;
private final FeedRefreshUpdater updater;
private final CommaFeedConfiguration config;
private final Meter refill;
private final BlockingDeque<Feed> queue;
private final ExecutorService feedProcessingLoopExecutor;
private final ExecutorService refillLoopExecutor;
private final ExecutorService refillExecutor;
private final ThreadPoolExecutor workerExecutor;
private final ThreadPoolExecutor databaseUpdaterExecutor;
public FeedRefreshEngine(UnitOfWork unitOfWork, FeedDAO feedDAO, FeedRefreshWorker worker, FeedRefreshUpdater updater,
CommaFeedConfiguration config, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.worker = worker;
this.updater = updater;
this.config = config;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
this.queue = new LinkedBlockingDeque<>();
this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor();
this.refillLoopExecutor = Executors.newSingleThreadExecutor();
this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.feedRefresh().databaseThreads());
metrics.register(MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(MetricRegistry.name(getClass(), "worker", "active"), (Gauge<Integer>) workerExecutor::getActiveCount);
metrics.register(MetricRegistry.name(getClass(), "updater", "active"), (Gauge<Integer>) databaseUpdaterExecutor::getActiveCount);
}
public void start() {
startFeedProcessingLoop();
startRefillLoop();
}
private void startFeedProcessingLoop() {
// take a feed from the queue, process it, rince, repeat
feedProcessingLoopExecutor.submit(() -> {
while (!feedProcessingLoopExecutor.isShutdown()) {
try {
// take() is blocking until a feed is available from the queue
Feed feed = queue.take();
// send the feed to be processed
log.debug("got feed {} from the queue, send it for processing", feed.getId());
processFeedAsync(feed);
// we removed a feed from the queue, try to refill it as it may now be empty
if (queue.isEmpty()) {
log.debug("took the last feed from the queue, try to refill");
refillQueueAsync();
}
} catch (InterruptedException e) {
log.debug("interrupted while waiting for a feed in the queue");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
private void startRefillLoop() {
// refill the queue at regular intervals if it's empty
refillLoopExecutor.submit(() -> {
while (!refillLoopExecutor.isShutdown()) {
try {
if (queue.isEmpty()) {
log.debug("refilling queue");
refillQueueAsync();
}
log.debug("sleeping for 15s");
TimeUnit.SECONDS.sleep(15);
} catch (InterruptedException e) {
log.debug("interrupted while sleeping");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
public void refreshImmediately(Feed feed) {
log.debug("add feed {} at the start of the queue", feed.getId());
// remove the feed from the queue if it was already queued to avoid refreshing it twice
queue.removeIf(f -> f.getId().equals(feed.getId()));
queue.addFirst(feed);
}
private void refillQueueAsync() {
CompletableFuture.runAsync(() -> {
if (!queue.isEmpty()) {
return;
}
refill.mark();
List<Feed> nextUpdatableFeeds = getNextUpdatableFeeds(getBatchSize());
log.debug("found {} feeds that are up for refresh", nextUpdatableFeeds.size());
for (Feed feed : nextUpdatableFeeds) {
// add the feed only if it was not already queued
if (queue.stream().noneMatch(f -> f.getId().equals(feed.getId()))) {
queue.addLast(feed);
}
}
}, refillExecutor).whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while refilling the queue", ex);
}
});
}
private void processFeedAsync(Feed feed) {
CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
.thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
.whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while processing feed {}", feed.getUrl(), ex);
}
});
}
private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(() -> {
Instant lastLoginThreshold = config.feedRefresh().userInactivityPeriod().isZero() ? null
: Instant.now().minus(config.feedRefresh().userInactivityPeriod());
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
// update disabledUntil to prevent feeds from being returned again by feedDAO.findNextUpdatable()
Instant nextUpdateDate = Instant.now().plus(config.feedRefresh().interval());
feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
return feeds;
});
}
private int getBatchSize() {
return Math.min(100, 3 * config.feedRefresh().httpThreads());
}
public void stop() {
this.feedProcessingLoopExecutor.shutdownNow();
this.refillLoopExecutor.shutdownNow();
this.refillExecutor.shutdownNow();
this.workerExecutor.shutdownNow();
this.databaseUpdaterExecutor.shutdownNow();
}
/**
* returns an ExecutorService with a single thread that discards tasks if a task is already running
*/
private ThreadPoolExecutor newDiscardingSingleThreadExecutorService() {
ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/**
* returns an ExecutorService that blocks submissions until a thread is available
*/
private ThreadPoolExecutor newBlockingExecutorService(int threads) {
ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler((r, e) -> {
if (e.isShutdown()) {
return;
}
try {
e.getQueue().put(r);
} catch (InterruptedException ex) {
log.debug("interrupted while waiting for a slot in the queue.", ex);
Thread.currentThread().interrupt();
}
});
return pool;
}
}
package com.commafeed.backend.feed;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedRefreshEngine {
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedRefreshWorker worker;
private final FeedRefreshUpdater updater;
private final CommaFeedConfiguration config;
private final Meter refill;
private final BlockingDeque<Feed> queue;
private final ExecutorService feedProcessingLoopExecutor;
private final ExecutorService refillLoopExecutor;
private final ExecutorService refillExecutor;
private final ThreadPoolExecutor workerExecutor;
private final ThreadPoolExecutor databaseUpdaterExecutor;
public FeedRefreshEngine(UnitOfWork unitOfWork, FeedDAO feedDAO, FeedRefreshWorker worker, FeedRefreshUpdater updater,
CommaFeedConfiguration config, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.worker = worker;
this.updater = updater;
this.config = config;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
this.queue = new LinkedBlockingDeque<>();
this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor();
this.refillLoopExecutor = Executors.newSingleThreadExecutor();
this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.feedRefresh().databaseThreads());
metrics.register(MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(MetricRegistry.name(getClass(), "worker", "active"), (Gauge<Integer>) workerExecutor::getActiveCount);
metrics.register(MetricRegistry.name(getClass(), "updater", "active"), (Gauge<Integer>) databaseUpdaterExecutor::getActiveCount);
}
public void start() {
startFeedProcessingLoop();
startRefillLoop();
}
private void startFeedProcessingLoop() {
// take a feed from the queue, process it, rince, repeat
feedProcessingLoopExecutor.submit(() -> {
while (!feedProcessingLoopExecutor.isShutdown()) {
try {
// take() is blocking until a feed is available from the queue
Feed feed = queue.take();
// send the feed to be processed
log.debug("got feed {} from the queue, send it for processing", feed.getId());
processFeedAsync(feed);
// we removed a feed from the queue, try to refill it as it may now be empty
if (queue.isEmpty()) {
log.debug("took the last feed from the queue, try to refill");
refillQueueAsync();
}
} catch (InterruptedException e) {
log.debug("interrupted while waiting for a feed in the queue");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
private void startRefillLoop() {
// refill the queue at regular intervals if it's empty
refillLoopExecutor.submit(() -> {
while (!refillLoopExecutor.isShutdown()) {
try {
if (queue.isEmpty()) {
log.debug("refilling queue");
refillQueueAsync();
}
log.debug("sleeping for 15s");
TimeUnit.SECONDS.sleep(15);
} catch (InterruptedException e) {
log.debug("interrupted while sleeping");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
public void refreshImmediately(Feed feed) {
log.debug("add feed {} at the start of the queue", feed.getId());
// remove the feed from the queue if it was already queued to avoid refreshing it twice
queue.removeIf(f -> f.getId().equals(feed.getId()));
queue.addFirst(feed);
}
private void refillQueueAsync() {
CompletableFuture.runAsync(() -> {
if (!queue.isEmpty()) {
return;
}
refill.mark();
List<Feed> nextUpdatableFeeds = getNextUpdatableFeeds(getBatchSize());
log.debug("found {} feeds that are up for refresh", nextUpdatableFeeds.size());
for (Feed feed : nextUpdatableFeeds) {
// add the feed only if it was not already queued
if (queue.stream().noneMatch(f -> f.getId().equals(feed.getId()))) {
queue.addLast(feed);
}
}
}, refillExecutor).whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while refilling the queue", ex);
}
});
}
private void processFeedAsync(Feed feed) {
CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
.thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
.whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while processing feed {}", feed.getUrl(), ex);
}
});
}
private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(() -> {
Instant lastLoginThreshold = config.feedRefresh().userInactivityPeriod().isZero() ? null
: Instant.now().minus(config.feedRefresh().userInactivityPeriod());
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
// update disabledUntil to prevent feeds from being returned again by feedDAO.findNextUpdatable()
Instant nextUpdateDate = Instant.now().plus(config.feedRefresh().interval());
feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
return feeds;
});
}
private int getBatchSize() {
return Math.min(100, 3 * config.feedRefresh().httpThreads());
}
public void stop() {
this.feedProcessingLoopExecutor.shutdownNow();
this.refillLoopExecutor.shutdownNow();
this.refillExecutor.shutdownNow();
this.workerExecutor.shutdownNow();
this.databaseUpdaterExecutor.shutdownNow();
}
/**
* returns an ExecutorService with a single thread that discards tasks if a task is already running
*/
private ThreadPoolExecutor newDiscardingSingleThreadExecutorService() {
ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/**
* returns an ExecutorService that blocks submissions until a thread is available
*/
private ThreadPoolExecutor newBlockingExecutorService(int threads) {
ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler((r, e) -> {
if (e.isShutdown()) {
return;
}
try {
e.getQueue().put(r);
} catch (InterruptedException ex) {
log.debug("interrupted while waiting for a slot in the queue.", ex);
Thread.currentThread().interrupt();
}
});
return pool;
}
}

168
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshIntervalCalculator.java
View File

@@ -1,84 +1,84 @@
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.FeedRefreshErrorHandling;
import com.google.common.primitives.Longs;
@Singleton
public class FeedRefreshIntervalCalculator {
private final Duration interval;
private final Duration maxInterval;
private final boolean empirical;
private final FeedRefreshErrorHandling errorHandling;
private final InstantSource instantSource;
public FeedRefreshIntervalCalculator(CommaFeedConfiguration config, InstantSource instantSource) {
this.interval = config.feedRefresh().interval();
this.maxInterval = config.feedRefresh().maxInterval();
this.empirical = config.feedRefresh().intervalEmpirical();
this.errorHandling = config.feedRefresh().errors();
this.instantSource = instantSource;
}
public Instant onFetchSuccess(Instant publishedDate, Long averageEntryInterval, Duration validFor) {
Instant instant = empirical ? computeEmpiricalRefreshInterval(publishedDate, averageEntryInterval)
: instantSource.instant().plus(interval);
return constrainToBounds(ObjectUtils.max(instant, instantSource.instant().plus(validFor)));
}
public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) {
return onFetchSuccess(publishedDate, averageEntryInterval, Duration.ZERO);
}
public Instant onTooManyRequests(Instant retryAfter, int errorCount) {
return constrainToBounds(ObjectUtils.max(retryAfter, onFetchError(errorCount)));
}
public Instant onFetchError(int errorCount) {
if (errorCount < errorHandling.retriesBeforeBackoff()) {
return constrainToBounds(instantSource.instant().plus(interval));
}
Duration retryInterval = errorHandling.backoffInterval().multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L);
return constrainToBounds(instantSource.instant().plus(retryInterval));
}
private Instant computeEmpiricalRefreshInterval(Instant publishedDate, Long averageEntryInterval) {
Instant now = instantSource.instant();
if (publishedDate == null) {
return now.plus(maxInterval);
}
long daysSinceLastPublication = ChronoUnit.DAYS.between(publishedDate, now);
if (daysSinceLastPublication >= 30) {
return now.plus(maxInterval);
} else if (daysSinceLastPublication >= 14) {
return now.plus(maxInterval.dividedBy(2));
} else if (daysSinceLastPublication >= 7) {
return now.plus(maxInterval.dividedBy(4));
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor
int factor = 2;
long millis = Longs.constrainToRange(averageEntryInterval / factor, interval.toMillis(), maxInterval.dividedBy(4).toMillis());
return now.plusMillis(millis);
} else {
// unknown case
return now.plus(maxInterval);
}
}
private Instant constrainToBounds(Instant instant) {
return ObjectUtils.max(ObjectUtils.min(instant, instantSource.instant().plus(maxInterval)), instantSource.instant().plus(interval));
}
}
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.FeedRefreshErrorHandling;
import com.google.common.primitives.Longs;
@Singleton
public class FeedRefreshIntervalCalculator {
private final Duration interval;
private final Duration maxInterval;
private final boolean empirical;
private final FeedRefreshErrorHandling errorHandling;
private final InstantSource instantSource;
public FeedRefreshIntervalCalculator(CommaFeedConfiguration config, InstantSource instantSource) {
this.interval = config.feedRefresh().interval();
this.maxInterval = config.feedRefresh().maxInterval();
this.empirical = config.feedRefresh().intervalEmpirical();
this.errorHandling = config.feedRefresh().errors();
this.instantSource = instantSource;
}
public Instant onFetchSuccess(Instant publishedDate, Long averageEntryInterval, Duration validFor) {
Instant instant = empirical ? computeEmpiricalRefreshInterval(publishedDate, averageEntryInterval)
: instantSource.instant().plus(interval);
return constrainToBounds(ObjectUtils.max(instant, instantSource.instant().plus(validFor)));
}
public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) {
return onFetchSuccess(publishedDate, averageEntryInterval, Duration.ZERO);
}
public Instant onTooManyRequests(Instant retryAfter, int errorCount) {
return constrainToBounds(ObjectUtils.max(retryAfter, onFetchError(errorCount)));
}
public Instant onFetchError(int errorCount) {
if (errorCount < errorHandling.retriesBeforeBackoff()) {
return constrainToBounds(instantSource.instant().plus(interval));
}
Duration retryInterval = errorHandling.backoffInterval().multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L);
return constrainToBounds(instantSource.instant().plus(retryInterval));
}
private Instant computeEmpiricalRefreshInterval(Instant publishedDate, Long averageEntryInterval) {
Instant now = instantSource.instant();
if (publishedDate == null) {
return now.plus(maxInterval);
}
long daysSinceLastPublication = ChronoUnit.DAYS.between(publishedDate, now);
if (daysSinceLastPublication >= 30) {
return now.plus(maxInterval);
} else if (daysSinceLastPublication >= 14) {
return now.plus(maxInterval.dividedBy(2));
} else if (daysSinceLastPublication >= 7) {
return now.plus(maxInterval.dividedBy(4));
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor
int factor = 2;
long millis = Longs.constrainToRange(averageEntryInterval / factor, interval.toMillis(), maxInterval.dividedBy(4).toMillis());
return now.plusMillis(millis);
} else {
// unknown case
return now.plus(maxInterval);
}
}
private Instant constrainToBounds(Instant instant) {
return ObjectUtils.max(ObjectUtils.min(instant, instantSource.instant().plus(maxInterval)), instantSource.instant().plus(interval));
}
}

360
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
View File

@@ -1,180 +1,180 @@
package com.commafeed.backend.feed;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions;
import com.google.common.util.concurrent.Striped;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Updates the feed in the database and inserts new entries
*/
@Slf4j
@Singleton
public class FeedRefreshUpdater {
private final UnitOfWork unitOfWork;
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final WebSocketSessions webSocketSessions;
private final Striped<Lock> locks;
private final Meter feedUpdated;
private final Meter entryInserted;
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO, WebSocketSessions webSocketSessions) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.webSocketSessions = webSocketSessions;
locks = Striped.lazyWeakLock(100000);
feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
private AddEntryResult addEntry(final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
boolean processed = false;
boolean inserted = false;
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread = new HashSet<>();
// lock on feed, make sure we are not updating the same feed twice at
// the same time
String key1 = StringUtils.trimToEmpty(String.valueOf(feed.getId()));
// lock on content, make sure we are not updating the same entry
// twice at the same time
Content content = entry.content();
String key2 = Digests.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title()));
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
Lock lock2 = iterator.next();
boolean locked1 = false;
boolean locked2 = false;
try {
// try to lock, give up after 1 minute
locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
processed = true;
inserted = unitOfWork.call(() -> {
boolean newEntry = false;
FeedEntry feedEntry = feedEntryService.find(feed, entry);
if (feedEntry == null) {
feedEntry = feedEntryService.create(feed, entry);
newEntry = true;
}
if (newEntry) {
entryInserted.mark();
for (FeedSubscription sub : subscriptions) {
boolean unread = feedEntryService.applyFilter(sub, feedEntry);
if (unread) {
subscriptionsForWhichEntryIsUnread.add(sub);
}
}
}
return newEntry;
});
} else {
log.error("lock timeout for {} - {}", feed.getUrl(), key1);
}
} catch (InterruptedException e) {
log.error("interrupted while waiting for lock for {} : {}", feed.getUrl(), e.getMessage(), e);
} finally {
if (locked1) {
lock1.unlock();
}
if (locked2) {
lock2.unlock();
}
}
return new AddEntryResult(processed, inserted, subscriptionsForWhichEntryIsUnread);
}
public boolean update(Feed feed, List<Entry> entries) {
boolean processed = true;
long inserted = 0;
Map<FeedSubscription, Long> unreadCountBySubscription = new HashMap<>();
if (!entries.isEmpty()) {
List<FeedSubscription> subscriptions = null;
for (Entry entry : entries) {
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
processed &= addEntryResult.processed;
inserted += addEntryResult.inserted ? 1 : 0;
addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(sub -> unreadCountBySubscription.merge(sub, 1L, Long::sum));
}
if (inserted == 0) {
feed.setMessage("No new entries found");
} else if (inserted > 0) {
feed.setMessage("Found %s new entries".formatted(inserted));
}
}
if (!processed) {
// requeue asap
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
}
if (inserted > 0) {
feedUpdated.mark();
}
unitOfWork.run(() -> feedService.update(feed));
notifyOverWebsocket(unreadCountBySubscription);
return processed;
}
private void notifyOverWebsocket(Map<FeedSubscription, Long> unreadCountBySubscription) {
unreadCountBySubscription.forEach((sub, unreadCount) -> webSocketSessions.sendMessage(sub.getUser(),
WebSocketMessageBuilder.newFeedEntries(sub, unreadCount)));
}
@AllArgsConstructor
private static class AddEntryResult {
private final boolean processed;
private final boolean inserted;
private final Set<FeedSubscription> subscriptionsForWhichEntryIsUnread;
}
}
package com.commafeed.backend.feed;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions;
import com.google.common.util.concurrent.Striped;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Updates the feed in the database and inserts new entries
*/
@Slf4j
@Singleton
public class FeedRefreshUpdater {
private final UnitOfWork unitOfWork;
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final WebSocketSessions webSocketSessions;
private final Striped<Lock> locks;
private final Meter feedUpdated;
private final Meter entryInserted;
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO, WebSocketSessions webSocketSessions) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.webSocketSessions = webSocketSessions;
locks = Striped.lazyWeakLock(100000);
feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
private AddEntryResult addEntry(final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
boolean processed = false;
boolean inserted = false;
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread = new HashSet<>();
// lock on feed, make sure we are not updating the same feed twice at
// the same time
String key1 = StringUtils.trimToEmpty(String.valueOf(feed.getId()));
// lock on content, make sure we are not updating the same entry
// twice at the same time
Content content = entry.content();
String key2 = Digests.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title()));
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
Lock lock2 = iterator.next();
boolean locked1 = false;
boolean locked2 = false;
try {
// try to lock, give up after 1 minute
locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
processed = true;
inserted = unitOfWork.call(() -> {
boolean newEntry = false;
FeedEntry feedEntry = feedEntryService.find(feed, entry);
if (feedEntry == null) {
feedEntry = feedEntryService.create(feed, entry);
newEntry = true;
}
if (newEntry) {
entryInserted.mark();
for (FeedSubscription sub : subscriptions) {
boolean unread = feedEntryService.applyFilter(sub, feedEntry);
if (unread) {
subscriptionsForWhichEntryIsUnread.add(sub);
}
}
}
return newEntry;
});
} else {
log.error("lock timeout for {} - {}", feed.getUrl(), key1);
}
} catch (InterruptedException e) {
log.error("interrupted while waiting for lock for {} : {}", feed.getUrl(), e.getMessage(), e);
} finally {
if (locked1) {
lock1.unlock();
}
if (locked2) {
lock2.unlock();
}
}
return new AddEntryResult(processed, inserted, subscriptionsForWhichEntryIsUnread);
}
public boolean update(Feed feed, List<Entry> entries) {
boolean processed = true;
long inserted = 0;
Map<FeedSubscription, Long> unreadCountBySubscription = new HashMap<>();
if (!entries.isEmpty()) {
List<FeedSubscription> subscriptions = null;
for (Entry entry : entries) {
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
processed &= addEntryResult.processed;
inserted += addEntryResult.inserted ? 1 : 0;
addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(sub -> unreadCountBySubscription.merge(sub, 1L, Long::sum));
}
if (inserted == 0) {
feed.setMessage("No new entries found");
} else if (inserted > 0) {
feed.setMessage("Found %s new entries".formatted(inserted));
}
}
if (!processed) {
// requeue asap
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
}
if (inserted > 0) {
feedUpdated.mark();
}
unitOfWork.run(() -> feedService.update(feed));
notifyOverWebsocket(unreadCountBySubscription);
return processed;
}
private void notifyOverWebsocket(Map<FeedSubscription, Long> unreadCountBySubscription) {
unreadCountBySubscription.forEach((sub, unreadCount) -> webSocketSessions.sendMessage(sub.getUser(),
WebSocketMessageBuilder.newFeedEntries(sub, unreadCount)));
}
@AllArgsConstructor
private static class AddEntryResult {
private final boolean processed;
private final boolean inserted;
private final Set<FeedSubscription> subscriptionsForWhichEntryIsUnread;
}
}

250
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
View File

@@ -1,125 +1,125 @@
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
/**
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link FeedRefreshUpdater} does that)
*/
@Slf4j
@Singleton
public class FeedRefreshWorker {
private final FeedRefreshIntervalCalculator refreshIntervalCalculator;
private final FeedFetcher fetcher;
private final CommaFeedConfiguration config;
private final Meter feedFetched;
public FeedRefreshWorker(FeedRefreshIntervalCalculator refreshIntervalCalculator, FeedFetcher fetcher, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.refreshIntervalCalculator = refreshIntervalCalculator;
this.fetcher = fetcher;
this.config = config;
this.feedFetched = metrics.meter(MetricRegistry.name(getClass(), "feedFetched"));
}
public FeedRefreshWorkerResult update(Feed feed) {
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FeedFetcherResult result = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
List<Entry> entries = result.feed().entries();
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
entries = entries.stream().filter(entry -> entry.published().isAfter(threshold)).toList();
}
String urlAfterRedirect = result.urlAfterRedirect();
if (StringUtils.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;
}
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(result.feed().link());
feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(result.contentHash());
feed.setLastPublishedDate(result.feed().lastPublishedDate());
feed.setAverageEntryInterval(result.feed().averageEntryInterval());
feed.setLastEntryDate(result.feed().lastEntryDate());
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(),
result.feed().averageEntryInterval(), result.validFor()));
return new FeedRefreshWorkerResult(feed, entries);
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
feed.setErrorCount(0);
feed.setMessage(e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFeedNotModified(feed.getLastPublishedDate(), feed.getAverageEntryInterval()));
if (e.getNewLastModifiedHeader() != null) {
feed.setLastModifiedHeader(e.getNewLastModifiedHeader());
}
if (e.getNewEtagHeader() != null) {
feed.setEtagHeader(e.getNewEtagHeader());
}
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (TooManyRequestsException e) {
log.debug("Too many requests : {}", feed.getUrl());
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Server indicated that we are sending too many requests");
feed.setDisabledUntil(refreshIntervalCalculator.onTooManyRequests(e.getRetryAfter(), feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (Exception e) {
log.debug("unable to refresh feed {}", feed.getUrl(), e);
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Unable to refresh feed : " + e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} finally {
feedFetched.mark();
}
}
public record FeedRefreshWorkerResult(Feed feed, List<Entry> entries) {
}
}
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
/**
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link FeedRefreshUpdater} does that)
*/
@Slf4j
@Singleton
public class FeedRefreshWorker {
private final FeedRefreshIntervalCalculator refreshIntervalCalculator;
private final FeedFetcher fetcher;
private final CommaFeedConfiguration config;
private final Meter feedFetched;
public FeedRefreshWorker(FeedRefreshIntervalCalculator refreshIntervalCalculator, FeedFetcher fetcher, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.refreshIntervalCalculator = refreshIntervalCalculator;
this.fetcher = fetcher;
this.config = config;
this.feedFetched = metrics.meter(MetricRegistry.name(getClass(), "feedFetched"));
}
public FeedRefreshWorkerResult update(Feed feed) {
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FeedFetcherResult result = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
List<Entry> entries = result.feed().entries();
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
entries = entries.stream().filter(entry -> entry.published().isAfter(threshold)).toList();
}
String urlAfterRedirect = result.urlAfterRedirect();
if (StringUtils.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;
}
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(result.feed().link());
feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(result.contentHash());
feed.setLastPublishedDate(result.feed().lastPublishedDate());
feed.setAverageEntryInterval(result.feed().averageEntryInterval());
feed.setLastEntryDate(result.feed().lastEntryDate());
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(),
result.feed().averageEntryInterval(), result.validFor()));
return new FeedRefreshWorkerResult(feed, entries);
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
feed.setErrorCount(0);
feed.setMessage(e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFeedNotModified(feed.getLastPublishedDate(), feed.getAverageEntryInterval()));
if (e.getNewLastModifiedHeader() != null) {
feed.setLastModifiedHeader(e.getNewLastModifiedHeader());
}
if (e.getNewEtagHeader() != null) {
feed.setEtagHeader(e.getNewEtagHeader());
}
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (TooManyRequestsException e) {
log.debug("Too many requests : {}", feed.getUrl());
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Server indicated that we are sending too many requests");
feed.setDisabledUntil(refreshIntervalCalculator.onTooManyRequests(e.getRetryAfter(), feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (Exception e) {
log.debug("unable to refresh feed {}", feed.getUrl(), e);
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Unable to refresh feed : " + e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} finally {
feedFetched.mark();
}
}
public record FeedRefreshWorkerResult(Feed feed, List<Entry> entries) {
}
}

436
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java
View File

@@ -1,218 +1,218 @@
package com.commafeed.backend.feed;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.utils.Base64;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.netpreserve.urlcanon.Canonicalizer;
import org.netpreserve.urlcanon.ParsedUrl;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.feed.parser.TextDirectionDetector;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry;
import lombok.extern.slf4j.Slf4j;
/**
* Utility methods related to feed handling
*
*/
@Slf4j
public class FeedUtils {
private static final String ESCAPED_QUESTION_MARK = Pattern.quote("?");
public static String truncate(String string, int length) {
if (string != null) {
string = string.substring(0, Math.min(length, string.length()));
}
return string;
}
public static boolean isHttp(String url) {
return url.startsWith("http://");
}
public static boolean isHttps(String url) {
return url.startsWith("https://");
}
public static boolean isAbsoluteUrl(String url) {
return isHttp(url) || isHttps(url);
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates
*/
public static String normalizeURL(String url) {
if (url == null) {
return null;
}
ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
String normalized = parsedUrl.toString();
if (normalized == null) {
normalized = url;
}
// convert to lower case, the url probably won't work in some cases
// after that but we don't care we just want to compare urls to avoid
// duplicates
normalized = normalized.toLowerCase();
// store all urls as http
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
// remove the www. part
normalized = normalized.replace("//www.", "//");
// feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedburner feeds have a special treatment
if (normalized.split(ESCAPED_QUESTION_MARK)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = normalized.split(ESCAPED_QUESTION_MARK)[0];
normalized = StringUtils.removeEnd(normalized, "/");
}
return normalized;
}
public static boolean isRTL(String title, String content) {
String text = StringUtils.isNotBlank(content) ? content : title;
if (StringUtils.isBlank(text)) {
return false;
}
String stripped = Jsoup.parse(text).text();
if (StringUtils.isBlank(stripped)) {
return false;
}
return TextDirectionDetector.detect(stripped) == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
public static String removeTrailingSlash(String url) {
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}
return url;
}
/**
*
* @param relativeUrl
* the url of the entry
* @param feedLink
* the url of the feed as described in the feed
* @param feedUrl
* the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry
*/
public static String toAbsoluteUrl(String relativeUrl, String feedLink, String feedUrl) {
String baseUrl = (feedLink != null && isAbsoluteUrl(feedLink)) ? feedLink : feedUrl;
if (baseUrl == null) {
return null;
}
try {
return new URL(new URL(baseUrl), relativeUrl).toString();
} catch (MalformedURLException e) {
log.debug("could not parse url : {}", e.getMessage(), e);
return null;
}
}
public static String getFaviconUrl(FeedSubscription subscription) {
return "rest/feed/favicon/" + subscription.getId();
}
public static String proxyImages(String content) {
if (StringUtils.isBlank(content)) {
return content;
}
Document doc = Jsoup.parse(content);
Elements elements = doc.select("img");
for (Element element : elements) {
String href = element.attr("src");
if (StringUtils.isNotBlank(href)) {
String proxy = proxyImage(href);
element.attr("src", proxy);
}
}
return doc.body().html();
}
public static String proxyImage(String url) {
if (StringUtils.isBlank(url)) {
return url;
}
return "rest/server/proxy?u=" + imageProxyEncoder(url);
}
public static String rot13(String msg) {
StringBuilder message = new StringBuilder();
for (char c : msg.toCharArray()) {
if (c >= 'a' && c <= 'm') {
c += 13;
} else if (c >= 'n' && c <= 'z') {
c -= 13;
} else if (c >= 'A' && c <= 'M') {
c += 13;
} else if (c >= 'N' && c <= 'Z') {
c -= 13;
}
message.append(c);
}
return message.toString();
}
public static String imageProxyEncoder(String url) {
return Base64.encodeBase64String(rot13(url).getBytes());
}
public static String imageProxyDecoder(String code) {
return rot13(new String(Base64.decodeBase64(code)));
}
public static void removeUnwantedFromSearch(List<Entry> entries, List<FeedEntryKeyword> keywords) {
Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
Entry entry = it.next();
boolean keep = true;
for (FeedEntryKeyword keyword : keywords) {
String title = entry.getTitle() == null ? null : Jsoup.parse(entry.getTitle()).text();
String content = entry.getContent() == null ? null : Jsoup.parse(entry.getContent()).text();
boolean condition = !StringUtils.containsIgnoreCase(content, keyword.getKeyword())
&& !StringUtils.containsIgnoreCase(title, keyword.getKeyword());
if (keyword.getMode() == Mode.EXCLUDE) {
condition = !condition;
}
if (condition) {
keep = false;
break;
}
}
if (!keep) {
it.remove();
}
}
}
}
package com.commafeed.backend.feed;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.utils.Base64;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.netpreserve.urlcanon.Canonicalizer;
import org.netpreserve.urlcanon.ParsedUrl;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.feed.parser.TextDirectionDetector;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry;
import lombok.extern.slf4j.Slf4j;
/**
* Utility methods related to feed handling
*
*/
@Slf4j
public class FeedUtils {
private static final String ESCAPED_QUESTION_MARK = Pattern.quote("?");
public static String truncate(String string, int length) {
if (string != null) {
string = string.substring(0, Math.min(length, string.length()));
}
return string;
}
public static boolean isHttp(String url) {
return url.startsWith("http://");
}
public static boolean isHttps(String url) {
return url.startsWith("https://");
}
public static boolean isAbsoluteUrl(String url) {
return isHttp(url) || isHttps(url);
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates
*/
public static String normalizeURL(String url) {
if (url == null) {
return null;
}
ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
String normalized = parsedUrl.toString();
if (normalized == null) {
normalized = url;
}
// convert to lower case, the url probably won't work in some cases
// after that but we don't care we just want to compare urls to avoid
// duplicates
normalized = normalized.toLowerCase();
// store all urls as http
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
// remove the www. part
normalized = normalized.replace("//www.", "//");
// feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedburner feeds have a special treatment
if (normalized.split(ESCAPED_QUESTION_MARK)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = normalized.split(ESCAPED_QUESTION_MARK)[0];
normalized = StringUtils.removeEnd(normalized, "/");
}
return normalized;
}
public static boolean isRTL(String title, String content) {
String text = StringUtils.isNotBlank(content) ? content : title;
if (StringUtils.isBlank(text)) {
return false;
}
String stripped = Jsoup.parse(text).text();
if (StringUtils.isBlank(stripped)) {
return false;
}
return TextDirectionDetector.detect(stripped) == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
public static String removeTrailingSlash(String url) {
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}
return url;
}
/**
*
* @param relativeUrl
* the url of the entry
* @param feedLink
* the url of the feed as described in the feed
* @param feedUrl
* the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry
*/
public static String toAbsoluteUrl(String relativeUrl, String feedLink, String feedUrl) {
String baseUrl = (feedLink != null && isAbsoluteUrl(feedLink)) ? feedLink : feedUrl;
if (baseUrl == null) {
return null;
}
try {
return new URL(new URL(baseUrl), relativeUrl).toString();
} catch (MalformedURLException e) {
log.debug("could not parse url : {}", e.getMessage(), e);
return null;
}
}
public static String getFaviconUrl(FeedSubscription subscription) {
return "rest/feed/favicon/" + subscription.getId();
}
public static String proxyImages(String content) {
if (StringUtils.isBlank(content)) {
return content;
}
Document doc = Jsoup.parse(content);
Elements elements = doc.select("img");
for (Element element : elements) {
String href = element.attr("src");
if (StringUtils.isNotBlank(href)) {
String proxy = proxyImage(href);
element.attr("src", proxy);
}
}
return doc.body().html();
}
public static String proxyImage(String url) {
if (StringUtils.isBlank(url)) {
return url;
}
return "rest/server/proxy?u=" + imageProxyEncoder(url);
}
public static String rot13(String msg) {
StringBuilder message = new StringBuilder();
for (char c : msg.toCharArray()) {
if (c >= 'a' && c <= 'm') {
c += 13;
} else if (c >= 'n' && c <= 'z') {
c -= 13;
} else if (c >= 'A' && c <= 'M') {
c += 13;
} else if (c >= 'N' && c <= 'Z') {
c -= 13;
}
message.append(c);
}
return message.toString();
}
public static String imageProxyEncoder(String url) {
return Base64.encodeBase64String(rot13(url).getBytes());
}
public static String imageProxyDecoder(String code) {
return rot13(new String(Base64.decodeBase64(code)));
}
public static void removeUnwantedFromSearch(List<Entry> entries, List<FeedEntryKeyword> keywords) {
Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
Entry entry = it.next();
boolean keep = true;
for (FeedEntryKeyword keyword : keywords) {
String title = entry.getTitle() == null ? null : Jsoup.parse(entry.getTitle()).text();
String content = entry.getContent() == null ? null : Jsoup.parse(entry.getContent()).text();
boolean condition = !StringUtils.containsIgnoreCase(content, keyword.getKeyword())
&& !StringUtils.containsIgnoreCase(title, keyword.getKeyword());
if (keyword.getMode() == Mode.EXCLUDE) {
condition = !condition;
}
if (condition) {
keep = false;
break;
}
}
if (!keep) {
it.remove();
}
}
}
}

140
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/EncodingDetector.java
View File

@@ -1,70 +1,70 @@
package com.commafeed.backend.feed.parser;
import java.nio.charset.Charset;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
@Singleton
class EncodingDetector {
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public Charset getEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (StringUtils.startsWithIgnoreCase(extracted, "iso-8859-")) {
if (!StringUtils.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (StringUtils.startsWithIgnoreCase(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Extract the declared encoding from the xml
*/
public String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = StringUtils.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
/**
* Detect encoding by analyzing characters in the array
*/
private Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
}
package com.commafeed.backend.feed.parser;
import java.nio.charset.Charset;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
@Singleton
class EncodingDetector {
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public Charset getEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (StringUtils.startsWithIgnoreCase(extracted, "iso-8859-")) {
if (!StringUtils.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (StringUtils.startsWithIgnoreCase(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Extract the declared encoding from the xml
*/
public String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = StringUtils.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
/**
* Detect encoding by analyzing characters in the array
*/
private Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
}

140
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedCleaner.java
View File

@@ -1,70 +1,70 @@
package com.commafeed.backend.feed.parser;
import java.util.Collection;
import java.util.regex.Pattern;
import jakarta.inject.Singleton;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.apache.commons.lang3.StringUtils;
@Singleton
class FeedCleaner {
private static final Pattern DOCTYPE_PATTERN = Pattern.compile("<!DOCTYPE[^>]*>", Pattern.CASE_INSENSITIVE);
public String trimInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
StringBuilder sb = new StringBuilder();
boolean firstTagFound = false;
for (int i = 0; i < xml.length(); i++) {
char c = xml.charAt(i);
if (!firstTagFound) {
if (c == '<') {
firstTagFound = true;
} else {
continue;
}
}
if (c >= 32 || c == 9 || c == 10 || c == 13) {
if (!Character.isHighSurrogate(c) && !Character.isLowSurrogate(c)) {
sb.append(c);
}
}
}
return sb.toString();
}
// https://stackoverflow.com/a/40836618
public String replaceHtmlEntitiesWithNumericEntities(String source) {
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
Collection<Emit> emits = Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build().parseText(source);
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
return sb.toString();
}
public String removeDoctypeDeclarations(String xml) {
return DOCTYPE_PATTERN.matcher(xml).replaceAll("");
}
}
package com.commafeed.backend.feed.parser;
import java.util.Collection;
import java.util.regex.Pattern;
import jakarta.inject.Singleton;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.apache.commons.lang3.StringUtils;
@Singleton
class FeedCleaner {
private static final Pattern DOCTYPE_PATTERN = Pattern.compile("<!DOCTYPE[^>]*>", Pattern.CASE_INSENSITIVE);
public String trimInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
StringBuilder sb = new StringBuilder();
boolean firstTagFound = false;
for (int i = 0; i < xml.length(); i++) {
char c = xml.charAt(i);
if (!firstTagFound) {
if (c == '<') {
firstTagFound = true;
} else {
continue;
}
}
if (c >= 32 || c == 9 || c == 10 || c == 13) {
if (!Character.isHighSurrogate(c) && !Character.isLowSurrogate(c)) {
sb.append(c);
}
}
}
return sb.toString();
}
// https://stackoverflow.com/a/40836618
public String replaceHtmlEntitiesWithNumericEntities(String source) {
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
Collection<Emit> emits = Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build().parseText(source);
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
return sb.toString();
}
public String removeDoctypeDeclarations(String xml) {
return DOCTYPE_PATTERN.matcher(xml).replaceAll("");
}
}

570
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParser.java
View File

@@ -1,285 +1,285 @@
package com.commafeed.backend.feed.parser;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.google.common.collect.Iterables;
import com.rometools.modules.mediarss.MediaEntryModule;
import com.rometools.modules.mediarss.MediaModule;
import com.rometools.modules.mediarss.types.MediaGroup;
import com.rometools.modules.mediarss.types.Metadata;
import com.rometools.modules.mediarss.types.Thumbnail;
import com.rometools.rome.feed.synd.SyndCategory;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.SyndFeedInput;
import lombok.RequiredArgsConstructor;
/**
* Parses raw xml into a FeedParserResult object
*/
@RequiredArgsConstructor
@Singleton
public class FeedParser {
private static final Namespace ATOM_10_NS = Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Instant START = Instant.ofEpochMilli(86400000);
private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
private final EncodingDetector encodingDetector;
private final FeedCleaner feedCleaner;
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedParsingException {
try {
Charset encoding = encodingDetector.getEncoding(xml);
String xmlString = feedCleaner.trimInvalidXmlCharacters(new String(xml, encoding));
if (xmlString == null) {
throw new FeedParsingException("Input string is null for url " + feedUrl);
}
xmlString = feedCleaner.replaceHtmlEntitiesWithNumericEntities(xmlString);
xmlString = feedCleaner.removeDoctypeDeclarations(xmlString);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed feed = new SyndFeedInput().build(source);
handleForeignMarkup(feed);
String title = feed.getTitle();
String link = feed.getLink();
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
if (lastPublishedDate == null || lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
lastPublishedDate = lastEntryDate;
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
return new FeedParserResult(title, link, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
} catch (FeedParsingException e) {
throw e;
} catch (Exception e) {
throw new FeedParsingException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
/**
* Adds atom links for rss feeds
*/
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
private List<Entry> buildEntries(SyndFeed feed, String feedUrl) {
List<Entry> entries = new ArrayList<>();
for (SyndEntry item : feed.getEntries()) {
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
String url = buildEntryUrl(feed, feedUrl, item);
if (StringUtils.isBlank(url) && FeedUtils.isAbsoluteUrl(guid)) {
// if link is empty but guid is used as url, use guid
url = guid;
}
Instant publishedDate = buildEntryPublishedDate(item);
Content content = buildContent(item);
entries.add(new Entry(guid, url, publishedDate, content));
}
entries.sort(Comparator.comparing(Entry::published).reversed());
return entries;
}
private Content buildContent(SyndEntry item) {
String title = getTitle(item);
String content = getContent(item);
String author = StringUtils.trimToNull(item.getAuthor());
String categories = StringUtils
.trimToNull(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")));
Enclosure enclosure = buildEnclosure(item);
Media media = buildMedia(item);
return new Content(title, content, author, categories, enclosure, media);
}
private Enclosure buildEnclosure(SyndEntry item) {
SyndEnclosure enclosure = Iterables.getFirst(item.getEnclosures(), null);
if (enclosure == null) {
return null;
}
return new Enclosure(enclosure.getUrl(), enclosure.getType());
}
private Instant buildEntryPublishedDate(SyndEntry item) {
Date date = item.getPublishedDate();
if (date == null) {
date = item.getUpdatedDate();
}
return toValidInstant(date, true);
}
private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
if (url == null || FeedUtils.isAbsoluteUrl(url)) {
// url is absolute, nothing to do
return url;
}
// url is relative, trying to resolve it
String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
return FeedUtils.toAbsoluteUrl(url, feedLink, feedUrl);
}
private Instant toValidInstant(Date date, boolean nullToNow) {
Instant now = Instant.now();
if (date == null) {
return nullToNow ? now : null;
}
Instant instant = date.toInstant();
if (instant.isBefore(START) || instant.isAfter(END)) {
return now;
}
if (instant.isAfter(now)) {
return now;
}
return instant;
}
private String getContent(SyndEntry item) {
String content;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private Media buildMedia(SyndEntry item) {
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module == null) {
return null;
}
Media media = buildMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = buildMedia(group.getMetadata());
}
return media;
}
private Media buildMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
String description = metadata.getDescription();
String thumbnailUrl = null;
Integer thumbnailWidth = null;
Integer thumbnailHeight = null;
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
thumbnailWidth = thumbnail.getWidth();
thumbnailHeight = thumbnail.getHeight();
if (thumbnail.getUrl() != null) {
thumbnailUrl = thumbnail.getUrl().toString();
}
}
if (description == null && thumbnailUrl == null) {
return null;
}
return new Media(description, thumbnailUrl, thumbnailWidth, thumbnailHeight);
}
private Long averageTimeBetweenEntries(List<Entry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < entries.size() - 1; i++) {
long diff = Math.abs(entries.get(i).published().toEpochMilli() - entries.get(i + 1).published().toEpochMilli());
stats.addValue(diff);
}
return (long) stats.getMean();
}
public static class FeedParsingException extends Exception {
private static final long serialVersionUID = 1L;
public FeedParsingException(String message) {
super(message);
}
public FeedParsingException(String message, Throwable cause) {
super(message, cause);
}
}
}
package com.commafeed.backend.feed.parser;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.google.common.collect.Iterables;
import com.rometools.modules.mediarss.MediaEntryModule;
import com.rometools.modules.mediarss.MediaModule;
import com.rometools.modules.mediarss.types.MediaGroup;
import com.rometools.modules.mediarss.types.Metadata;
import com.rometools.modules.mediarss.types.Thumbnail;
import com.rometools.rome.feed.synd.SyndCategory;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.SyndFeedInput;
import lombok.RequiredArgsConstructor;
/**
* Parses raw xml into a FeedParserResult object
*/
@RequiredArgsConstructor
@Singleton
public class FeedParser {
private static final Namespace ATOM_10_NS = Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Instant START = Instant.ofEpochMilli(86400000);
private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
private final EncodingDetector encodingDetector;
private final FeedCleaner feedCleaner;
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedParsingException {
try {
Charset encoding = encodingDetector.getEncoding(xml);
String xmlString = feedCleaner.trimInvalidXmlCharacters(new String(xml, encoding));
if (xmlString == null) {
throw new FeedParsingException("Input string is null for url " + feedUrl);
}
xmlString = feedCleaner.replaceHtmlEntitiesWithNumericEntities(xmlString);
xmlString = feedCleaner.removeDoctypeDeclarations(xmlString);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed feed = new SyndFeedInput().build(source);
handleForeignMarkup(feed);
String title = feed.getTitle();
String link = feed.getLink();
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
if (lastPublishedDate == null || lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
lastPublishedDate = lastEntryDate;
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
return new FeedParserResult(title, link, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
} catch (FeedParsingException e) {
throw e;
} catch (Exception e) {
throw new FeedParsingException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
/**
* Adds atom links for rss feeds
*/
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
private List<Entry> buildEntries(SyndFeed feed, String feedUrl) {
List<Entry> entries = new ArrayList<>();
for (SyndEntry item : feed.getEntries()) {
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
String url = buildEntryUrl(feed, feedUrl, item);
if (StringUtils.isBlank(url) && FeedUtils.isAbsoluteUrl(guid)) {
// if link is empty but guid is used as url, use guid
url = guid;
}
Instant publishedDate = buildEntryPublishedDate(item);
Content content = buildContent(item);
entries.add(new Entry(guid, url, publishedDate, content));
}
entries.sort(Comparator.comparing(Entry::published).reversed());
return entries;
}
private Content buildContent(SyndEntry item) {
String title = getTitle(item);
String content = getContent(item);
String author = StringUtils.trimToNull(item.getAuthor());
String categories = StringUtils
.trimToNull(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")));
Enclosure enclosure = buildEnclosure(item);
Media media = buildMedia(item);
return new Content(title, content, author, categories, enclosure, media);
}
private Enclosure buildEnclosure(SyndEntry item) {
SyndEnclosure enclosure = Iterables.getFirst(item.getEnclosures(), null);
if (enclosure == null) {
return null;
}
return new Enclosure(enclosure.getUrl(), enclosure.getType());
}
private Instant buildEntryPublishedDate(SyndEntry item) {
Date date = item.getPublishedDate();
if (date == null) {
date = item.getUpdatedDate();
}
return toValidInstant(date, true);
}
private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
if (url == null || FeedUtils.isAbsoluteUrl(url)) {
// url is absolute, nothing to do
return url;
}
// url is relative, trying to resolve it
String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
return FeedUtils.toAbsoluteUrl(url, feedLink, feedUrl);
}
private Instant toValidInstant(Date date, boolean nullToNow) {
Instant now = Instant.now();
if (date == null) {
return nullToNow ? now : null;
}
Instant instant = date.toInstant();
if (instant.isBefore(START) || instant.isAfter(END)) {
return now;
}
if (instant.isAfter(now)) {
return now;
}
return instant;
}
private String getContent(SyndEntry item) {
String content;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private Media buildMedia(SyndEntry item) {
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module == null) {
return null;
}
Media media = buildMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = buildMedia(group.getMetadata());
}
return media;
}
private Media buildMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
String description = metadata.getDescription();
String thumbnailUrl = null;
Integer thumbnailWidth = null;
Integer thumbnailHeight = null;
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
thumbnailWidth = thumbnail.getWidth();
thumbnailHeight = thumbnail.getHeight();
if (thumbnail.getUrl() != null) {
thumbnailUrl = thumbnail.getUrl().toString();
}
}
if (description == null && thumbnailUrl == null) {
return null;
}
return new Media(description, thumbnailUrl, thumbnailWidth, thumbnailHeight);
}
private Long averageTimeBetweenEntries(List<Entry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < entries.size() - 1; i++) {
long diff = Math.abs(entries.get(i).published().toEpochMilli() - entries.get(i + 1).published().toEpochMilli());
stats.addValue(diff);
}
return (long) stats.getMean();
}
public static class FeedParsingException extends Exception {
private static final long serialVersionUID = 1L;
public FeedParsingException(String message) {
super(message);
}
public FeedParsingException(String message, Throwable cause) {
super(message, cause);
}
}
}

40
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParserResult.java
View File

@@ -1,20 +1,20 @@
package com.commafeed.backend.feed.parser;
import java.time.Instant;
import java.util.List;
public record FeedParserResult(String title, String link, Instant lastPublishedDate, Long averageEntryInterval, Instant lastEntryDate,
List<Entry> entries) {
public record Entry(String guid, String url, Instant published, Content content) {
}
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {
}
public record Enclosure(String url, String type) {
}
public record Media(String description, String thumbnailUrl, Integer thumbnailWidth, Integer thumbnailHeight) {
}
}
package com.commafeed.backend.feed.parser;
import java.time.Instant;
import java.util.List;
public record FeedParserResult(String title, String link, Instant lastPublishedDate, Long averageEntryInterval, Instant lastEntryDate,
List<Entry> entries) {
public record Entry(String guid, String url, Instant published, Content content) {
}
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {
}
public record Enclosure(String url, String type) {
}
public record Media(String description, String thumbnailUrl, Integer thumbnailWidth, Integer thumbnailHeight) {
}
}

544
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/HtmlEntities.java
View File

@@ -1,272 +1,272 @@
package com.commafeed.backend.feed.parser;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import lombok.experimental.UtilityClass;
@UtilityClass
class HtmlEntities {
public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final String[] HTML_ENTITIES;
public static final String[] NUMERIC_ENTITIES;
static {
Map<String, String> map = new LinkedHashMap<>();
map.put("&Aacute;", "&#193;");
map.put("&aacute;", "&#225;");
map.put("&Acirc;", "&#194;");
map.put("&acirc;", "&#226;");
map.put("&acute;", "&#180;");
map.put("&AElig;", "&#198;");
map.put("&aelig;", "&#230;");
map.put("&Agrave;", "&#192;");
map.put("&agrave;", "&#224;");
map.put("&alefsym;", "&#8501;");
map.put("&Alpha;", "&#913;");
map.put("&alpha;", "&#945;");
map.put("&amp;", "&#38;");
map.put("&and;", "&#8743;");
map.put("&ang;", "&#8736;");
map.put("&Aring;", "&#197;");
map.put("&aring;", "&#229;");
map.put("&asymp;", "&#8776;");
map.put("&Atilde;", "&#195;");
map.put("&atilde;", "&#227;");
map.put("&Auml;", "&#196;");
map.put("&auml;", "&#228;");
map.put("&bdquo;", "&#8222;");
map.put("&Beta;", "&#914;");
map.put("&beta;", "&#946;");
map.put("&brvbar;", "&#166;");
map.put("&bull;", "&#8226;");
map.put("&cap;", "&#8745;");
map.put("&Ccedil;", "&#199;");
map.put("&ccedil;", "&#231;");
map.put("&cedil;", "&#184;");
map.put("&cent;", "&#162;");
map.put("&Chi;", "&#935;");
map.put("&chi;", "&#967;");
map.put("&circ;", "&#710;");
map.put("&clubs;", "&#9827;");
map.put("&cong;", "&#8773;");
map.put("&copy;", "&#169;");
map.put("&crarr;", "&#8629;");
map.put("&cup;", "&#8746;");
map.put("&curren;", "&#164;");
map.put("&dagger;", "&#8224;");
map.put("&Dagger;", "&#8225;");
map.put("&darr;", "&#8595;");
map.put("&dArr;", "&#8659;");
map.put("&deg;", "&#176;");
map.put("&Delta;", "&#916;");
map.put("&delta;", "&#948;");
map.put("&diams;", "&#9830;");
map.put("&divide;", "&#247;");
map.put("&Eacute;", "&#201;");
map.put("&eacute;", "&#233;");
map.put("&Ecirc;", "&#202;");
map.put("&ecirc;", "&#234;");
map.put("&Egrave;", "&#200;");
map.put("&egrave;", "&#232;");
map.put("&empty;", "&#8709;");
map.put("&emsp;", "&#8195;");
map.put("&ensp;", "&#8194;");
map.put("&Epsilon;", "&#917;");
map.put("&epsilon;", "&#949;");
map.put("&equiv;", "&#8801;");
map.put("&Eta;", "&#919;");
map.put("&eta;", "&#951;");
map.put("&ETH;", "&#208;");
map.put("&eth;", "&#240;");
map.put("&Euml;", "&#203;");
map.put("&euml;", "&#235;");
map.put("&euro;", "&#8364;");
map.put("&exist;", "&#8707;");
map.put("&fnof;", "&#402;");
map.put("&forall;", "&#8704;");
map.put("&frac12;", "&#189;");
map.put("&frac14;", "&#188;");
map.put("&frac34;", "&#190;");
map.put("&frasl;", "&#8260;");
map.put("&Gamma;", "&#915;");
map.put("&gamma;", "&#947;");
map.put("&ge;", "&#8805;");
map.put("&harr;", "&#8596;");
map.put("&hArr;", "&#8660;");
map.put("&hearts;", "&#9829;");
map.put("&hellip;", "&#8230;");
map.put("&Iacute;", "&#205;");
map.put("&iacute;", "&#237;");
map.put("&Icirc;", "&#206;");
map.put("&icirc;", "&#238;");
map.put("&iexcl;", "&#161;");
map.put("&Igrave;", "&#204;");
map.put("&igrave;", "&#236;");
map.put("&image;", "&#8465;");
map.put("&infin;", "&#8734;");
map.put("&int;", "&#8747;");
map.put("&Iota;", "&#921;");
map.put("&iota;", "&#953;");
map.put("&iquest;", "&#191;");
map.put("&isin;", "&#8712;");
map.put("&Iuml;", "&#207;");
map.put("&iuml;", "&#239;");
map.put("&Kappa;", "&#922;");
map.put("&kappa;", "&#954;");
map.put("&Lambda;", "&#923;");
map.put("&lambda;", "&#955;");
map.put("&lang;", "&#9001;");
map.put("&laquo;", "&#171;");
map.put("&larr;", "&#8592;");
map.put("&lArr;", "&#8656;");
map.put("&lceil;", "&#8968;");
map.put("&ldquo;", "&#8220;");
map.put("&le;", "&#8804;");
map.put("&lfloor;", "&#8970;");
map.put("&lowast;", "&#8727;");
map.put("&loz;", "&#9674;");
map.put("&lrm;", "&#8206;");
map.put("&lsaquo;", "&#8249;");
map.put("&lsquo;", "&#8216;");
map.put("&macr;", "&#175;");
map.put("&mdash;", "&#8212;");
map.put("&micro;", "&#181;");
map.put("&middot;", "&#183;");
map.put("&minus;", "&#8722;");
map.put("&Mu;", "&#924;");
map.put("&mu;", "&#956;");
map.put("&nabla;", "&#8711;");
map.put("&nbsp;", "&#160;");
map.put("&ndash;", "&#8211;");
map.put("&ne;", "&#8800;");
map.put("&ni;", "&#8715;");
map.put("&not;", "&#172;");
map.put("&notin;", "&#8713;");
map.put("&nsub;", "&#8836;");
map.put("&Ntilde;", "&#209;");
map.put("&ntilde;", "&#241;");
map.put("&Nu;", "&#925;");
map.put("&nu;", "&#957;");
map.put("&Oacute;", "&#211;");
map.put("&oacute;", "&#243;");
map.put("&Ocirc;", "&#212;");
map.put("&ocirc;", "&#244;");
map.put("&OElig;", "&#338;");
map.put("&oelig;", "&#339;");
map.put("&Ograve;", "&#210;");
map.put("&ograve;", "&#242;");
map.put("&oline;", "&#8254;");
map.put("&Omega;", "&#937;");
map.put("&omega;", "&#969;");
map.put("&Omicron;", "&#927;");
map.put("&omicron;", "&#959;");
map.put("&oplus;", "&#8853;");
map.put("&or;", "&#8744;");
map.put("&ordf;", "&#170;");
map.put("&ordm;", "&#186;");
map.put("&Oslash;", "&#216;");
map.put("&oslash;", "&#248;");
map.put("&Otilde;", "&#213;");
map.put("&otilde;", "&#245;");
map.put("&otimes;", "&#8855;");
map.put("&Ouml;", "&#214;");
map.put("&ouml;", "&#246;");
map.put("&para;", "&#182;");
map.put("&part;", "&#8706;");
map.put("&permil;", "&#8240;");
map.put("&perp;", "&#8869;");
map.put("&Phi;", "&#934;");
map.put("&phi;", "&#966;");
map.put("&Pi;", "&#928;");
map.put("&pi;", "&#960;");
map.put("&piv;", "&#982;");
map.put("&plusmn;", "&#177;");
map.put("&pound;", "&#163;");
map.put("&prime;", "&#8242;");
map.put("&Prime;", "&#8243;");
map.put("&prod;", "&#8719;");
map.put("&prop;", "&#8733;");
map.put("&Psi;", "&#936;");
map.put("&psi;", "&#968;");
map.put("&quot;", "&#34;");
map.put("&radic;", "&#8730;");
map.put("&rang;", "&#9002;");
map.put("&raquo;", "&#187;");
map.put("&rarr;", "&#8594;");
map.put("&rArr;", "&#8658;");
map.put("&rceil;", "&#8969;");
map.put("&rdquo;", "&#8221;");
map.put("&real;", "&#8476;");
map.put("&reg;", "&#174;");
map.put("&rfloor;", "&#8971;");
map.put("&Rho;", "&#929;");
map.put("&rho;", "&#961;");
map.put("&rlm;", "&#8207;");
map.put("&rsaquo;", "&#8250;");
map.put("&rsquo;", "&#8217;");
map.put("&sbquo;", "&#8218;");
map.put("&Scaron;", "&#352;");
map.put("&scaron;", "&#353;");
map.put("&sdot;", "&#8901;");
map.put("&sect;", "&#167;");
map.put("&shy;", "&#173;");
map.put("&Sigma;", "&#931;");
map.put("&sigma;", "&#963;");
map.put("&sigmaf;", "&#962;");
map.put("&sim;", "&#8764;");
map.put("&spades;", "&#9824;");
map.put("&sub;", "&#8834;");
map.put("&sube;", "&#8838;");
map.put("&sum;", "&#8721;");
map.put("&sup1;", "&#185;");
map.put("&sup2;", "&#178;");
map.put("&sup3;", "&#179;");
map.put("&sup;", "&#8835;");
map.put("&supe;", "&#8839;");
map.put("&szlig;", "&#223;");
map.put("&Tau;", "&#932;");
map.put("&tau;", "&#964;");
map.put("&there4;", "&#8756;");
map.put("&Theta;", "&#920;");
map.put("&theta;", "&#952;");
map.put("&thetasym;", "&#977;");
map.put("&thinsp;", "&#8201;");
map.put("&THORN;", "&#222;");
map.put("&thorn;", "&#254;");
map.put("&tilde;", "&#732;");
map.put("&times;", "&#215;");
map.put("&trade;", "&#8482;");
map.put("&Uacute;", "&#218;");
map.put("&uacute;", "&#250;");
map.put("&uarr;", "&#8593;");
map.put("&uArr;", "&#8657;");
map.put("&Ucirc;", "&#219;");
map.put("&ucirc;", "&#251;");
map.put("&Ugrave;", "&#217;");
map.put("&ugrave;", "&#249;");
map.put("&uml;", "&#168;");
map.put("&upsih;", "&#978;");
map.put("&Upsilon;", "&#933;");
map.put("&upsilon;", "&#965;");
map.put("&Uuml;", "&#220;");
map.put("&uuml;", "&#252;");
map.put("&weierp;", "&#8472;");
map.put("&Xi;", "&#926;");
map.put("&xi;", "&#958;");
map.put("&Yacute;", "&#221;");
map.put("&yacute;", "&#253;");
map.put("&yen;", "&#165;");
map.put("&yuml;", "&#255;");
map.put("&Yuml;", "&#376;");
map.put("&Zeta;", "&#918;");
map.put("&zeta;", "&#950;");
map.put("&zwj;", "&#8205;");
map.put("&zwnj;", "&#8204;");
HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map);
HTML_ENTITIES = map.keySet().toArray(new String[0]);
NUMERIC_ENTITIES = map.values().toArray(new String[0]);
}
}
package com.commafeed.backend.feed.parser;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import lombok.experimental.UtilityClass;
@UtilityClass
class HtmlEntities {
public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final String[] HTML_ENTITIES;
public static final String[] NUMERIC_ENTITIES;
static {
Map<String, String> map = new LinkedHashMap<>();
map.put("&Aacute;", "&#193;");
map.put("&aacute;", "&#225;");
map.put("&Acirc;", "&#194;");
map.put("&acirc;", "&#226;");
map.put("&acute;", "&#180;");
map.put("&AElig;", "&#198;");
map.put("&aelig;", "&#230;");
map.put("&Agrave;", "&#192;");
map.put("&agrave;", "&#224;");
map.put("&alefsym;", "&#8501;");
map.put("&Alpha;", "&#913;");
map.put("&alpha;", "&#945;");
map.put("&amp;", "&#38;");
map.put("&and;", "&#8743;");
map.put("&ang;", "&#8736;");
map.put("&Aring;", "&#197;");
map.put("&aring;", "&#229;");
map.put("&asymp;", "&#8776;");
map.put("&Atilde;", "&#195;");
map.put("&atilde;", "&#227;");
map.put("&Auml;", "&#196;");
map.put("&auml;", "&#228;");
map.put("&bdquo;", "&#8222;");
map.put("&Beta;", "&#914;");
map.put("&beta;", "&#946;");
map.put("&brvbar;", "&#166;");
map.put("&bull;", "&#8226;");
map.put("&cap;", "&#8745;");
map.put("&Ccedil;", "&#199;");
map.put("&ccedil;", "&#231;");
map.put("&cedil;", "&#184;");
map.put("&cent;", "&#162;");
map.put("&Chi;", "&#935;");
map.put("&chi;", "&#967;");
map.put("&circ;", "&#710;");
map.put("&clubs;", "&#9827;");
map.put("&cong;", "&#8773;");
map.put("&copy;", "&#169;");
map.put("&crarr;", "&#8629;");
map.put("&cup;", "&#8746;");
map.put("&curren;", "&#164;");
map.put("&dagger;", "&#8224;");
map.put("&Dagger;", "&#8225;");
map.put("&darr;", "&#8595;");
map.put("&dArr;", "&#8659;");
map.put("&deg;", "&#176;");
map.put("&Delta;", "&#916;");
map.put("&delta;", "&#948;");
map.put("&diams;", "&#9830;");
map.put("&divide;", "&#247;");
map.put("&Eacute;", "&#201;");
map.put("&eacute;", "&#233;");
map.put("&Ecirc;", "&#202;");
map.put("&ecirc;", "&#234;");
map.put("&Egrave;", "&#200;");
map.put("&egrave;", "&#232;");
map.put("&empty;", "&#8709;");
map.put("&emsp;", "&#8195;");
map.put("&ensp;", "&#8194;");
map.put("&Epsilon;", "&#917;");
map.put("&epsilon;", "&#949;");
map.put("&equiv;", "&#8801;");
map.put("&Eta;", "&#919;");
map.put("&eta;", "&#951;");
map.put("&ETH;", "&#208;");
map.put("&eth;", "&#240;");
map.put("&Euml;", "&#203;");
map.put("&euml;", "&#235;");
map.put("&euro;", "&#8364;");
map.put("&exist;", "&#8707;");
map.put("&fnof;", "&#402;");
map.put("&forall;", "&#8704;");
map.put("&frac12;", "&#189;");
map.put("&frac14;", "&#188;");
map.put("&frac34;", "&#190;");
map.put("&frasl;", "&#8260;");
map.put("&Gamma;", "&#915;");
map.put("&gamma;", "&#947;");
map.put("&ge;", "&#8805;");
map.put("&harr;", "&#8596;");
map.put("&hArr;", "&#8660;");
map.put("&hearts;", "&#9829;");
map.put("&hellip;", "&#8230;");
map.put("&Iacute;", "&#205;");
map.put("&iacute;", "&#237;");
map.put("&Icirc;", "&#206;");
map.put("&icirc;", "&#238;");
map.put("&iexcl;", "&#161;");
map.put("&Igrave;", "&#204;");
map.put("&igrave;", "&#236;");
map.put("&image;", "&#8465;");
map.put("&infin;", "&#8734;");
map.put("&int;", "&#8747;");
map.put("&Iota;", "&#921;");
map.put("&iota;", "&#953;");
map.put("&iquest;", "&#191;");
map.put("&isin;", "&#8712;");
map.put("&Iuml;", "&#207;");
map.put("&iuml;", "&#239;");
map.put("&Kappa;", "&#922;");
map.put("&kappa;", "&#954;");
map.put("&Lambda;", "&#923;");
map.put("&lambda;", "&#955;");
map.put("&lang;", "&#9001;");
map.put("&laquo;", "&#171;");
map.put("&larr;", "&#8592;");
map.put("&lArr;", "&#8656;");
map.put("&lceil;", "&#8968;");
map.put("&ldquo;", "&#8220;");
map.put("&le;", "&#8804;");
map.put("&lfloor;", "&#8970;");
map.put("&lowast;", "&#8727;");
map.put("&loz;", "&#9674;");
map.put("&lrm;", "&#8206;");
map.put("&lsaquo;", "&#8249;");
map.put("&lsquo;", "&#8216;");
map.put("&macr;", "&#175;");
map.put("&mdash;", "&#8212;");
map.put("&micro;", "&#181;");
map.put("&middot;", "&#183;");
map.put("&minus;", "&#8722;");
map.put("&Mu;", "&#924;");
map.put("&mu;", "&#956;");
map.put("&nabla;", "&#8711;");
map.put("&nbsp;", "&#160;");
map.put("&ndash;", "&#8211;");
map.put("&ne;", "&#8800;");
map.put("&ni;", "&#8715;");
map.put("&not;", "&#172;");
map.put("&notin;", "&#8713;");
map.put("&nsub;", "&#8836;");
map.put("&Ntilde;", "&#209;");
map.put("&ntilde;", "&#241;");
map.put("&Nu;", "&#925;");
map.put("&nu;", "&#957;");
map.put("&Oacute;", "&#211;");
map.put("&oacute;", "&#243;");
map.put("&Ocirc;", "&#212;");
map.put("&ocirc;", "&#244;");
map.put("&OElig;", "&#338;");
map.put("&oelig;", "&#339;");
map.put("&Ograve;", "&#210;");
map.put("&ograve;", "&#242;");
map.put("&oline;", "&#8254;");
map.put("&Omega;", "&#937;");
map.put("&omega;", "&#969;");
map.put("&Omicron;", "&#927;");
map.put("&omicron;", "&#959;");
map.put("&oplus;", "&#8853;");
map.put("&or;", "&#8744;");
map.put("&ordf;", "&#170;");
map.put("&ordm;", "&#186;");
map.put("&Oslash;", "&#216;");
map.put("&oslash;", "&#248;");
map.put("&Otilde;", "&#213;");
map.put("&otilde;", "&#245;");
map.put("&otimes;", "&#8855;");
map.put("&Ouml;", "&#214;");
map.put("&ouml;", "&#246;");
map.put("&para;", "&#182;");
map.put("&part;", "&#8706;");
map.put("&permil;", "&#8240;");
map.put("&perp;", "&#8869;");
map.put("&Phi;", "&#934;");
map.put("&phi;", "&#966;");
map.put("&Pi;", "&#928;");
map.put("&pi;", "&#960;");
map.put("&piv;", "&#982;");
map.put("&plusmn;", "&#177;");
map.put("&pound;", "&#163;");
map.put("&prime;", "&#8242;");
map.put("&Prime;", "&#8243;");
map.put("&prod;", "&#8719;");
map.put("&prop;", "&#8733;");
map.put("&Psi;", "&#936;");
map.put("&psi;", "&#968;");
map.put("&quot;", "&#34;");
map.put("&radic;", "&#8730;");
map.put("&rang;", "&#9002;");
map.put("&raquo;", "&#187;");
map.put("&rarr;", "&#8594;");
map.put("&rArr;", "&#8658;");
map.put("&rceil;", "&#8969;");
map.put("&rdquo;", "&#8221;");
map.put("&real;", "&#8476;");
map.put("&reg;", "&#174;");
map.put("&rfloor;", "&#8971;");
map.put("&Rho;", "&#929;");
map.put("&rho;", "&#961;");
map.put("&rlm;", "&#8207;");
map.put("&rsaquo;", "&#8250;");
map.put("&rsquo;", "&#8217;");
map.put("&sbquo;", "&#8218;");
map.put("&Scaron;", "&#352;");
map.put("&scaron;", "&#353;");
map.put("&sdot;", "&#8901;");
map.put("&sect;", "&#167;");
map.put("&shy;", "&#173;");
map.put("&Sigma;", "&#931;");
map.put("&sigma;", "&#963;");
map.put("&sigmaf;", "&#962;");
map.put("&sim;", "&#8764;");
map.put("&spades;", "&#9824;");
map.put("&sub;", "&#8834;");
map.put("&sube;", "&#8838;");
map.put("&sum;", "&#8721;");
map.put("&sup1;", "&#185;");
map.put("&sup2;", "&#178;");
map.put("&sup3;", "&#179;");
map.put("&sup;", "&#8835;");
map.put("&supe;", "&#8839;");
map.put("&szlig;", "&#223;");
map.put("&Tau;", "&#932;");
map.put("&tau;", "&#964;");
map.put("&there4;", "&#8756;");
map.put("&Theta;", "&#920;");
map.put("&theta;", "&#952;");
map.put("&thetasym;", "&#977;");
map.put("&thinsp;", "&#8201;");
map.put("&THORN;", "&#222;");
map.put("&thorn;", "&#254;");
map.put("&tilde;", "&#732;");
map.put("&times;", "&#215;");
map.put("&trade;", "&#8482;");
map.put("&Uacute;", "&#218;");
map.put("&uacute;", "&#250;");
map.put("&uarr;", "&#8593;");
map.put("&uArr;", "&#8657;");
map.put("&Ucirc;", "&#219;");
map.put("&ucirc;", "&#251;");
map.put("&Ugrave;", "&#217;");
map.put("&ugrave;", "&#249;");
map.put("&uml;", "&#168;");
map.put("&upsih;", "&#978;");
map.put("&Upsilon;", "&#933;");
map.put("&upsilon;", "&#965;");
map.put("&Uuml;", "&#220;");
map.put("&uuml;", "&#252;");
map.put("&weierp;", "&#8472;");
map.put("&Xi;", "&#926;");
map.put("&xi;", "&#958;");
map.put("&Yacute;", "&#221;");
map.put("&yacute;", "&#253;");
map.put("&yen;", "&#165;");
map.put("&yuml;", "&#255;");
map.put("&Yuml;", "&#376;");
map.put("&Zeta;", "&#918;");
map.put("&zeta;", "&#950;");
map.put("&zwj;", "&#8205;");
map.put("&zwnj;", "&#8204;");
HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map);
HTML_ENTITIES = map.keySet().toArray(new String[0]);
NUMERIC_ENTITIES = map.values().toArray(new String[0]);
}
}

112
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/TextDirectionDetector.java
View File

@@ -1,56 +1,56 @@
package com.commafeed.backend.feed.parser;
import java.text.Bidi;
import java.util.regex.Pattern;
import org.apache.commons.lang3.math.NumberUtils;
public class TextDirectionDetector {
private static final Pattern WORDS_PATTERN = Pattern.compile("\\s+");
private static final Pattern URL_PATTERN = Pattern.compile("^https?://.*");
private static final double RTL_THRESHOLD = 0.4D;
public enum Direction {
LEFT_TO_RIGHT, RIGHT_TO_LEFT
}
public static Direction detect(String input) {
if (input == null || input.isBlank()) {
return Direction.LEFT_TO_RIGHT;
}
long rtl = 0;
long total = 0;
for (String token : WORDS_PATTERN.split(input)) {
// skip urls
if (URL_PATTERN.matcher(token).matches()) {
continue;
}
// skip numbers
if (NumberUtils.isCreatable(token)) {
continue;
}
boolean requiresBidi = Bidi.requiresBidi(token.toCharArray(), 0, token.length());
if (requiresBidi) {
Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT);
if (bidi.getBaseLevel() == 1) {
rtl++;
}
}
total++;
}
if (total == 0) {
return Direction.LEFT_TO_RIGHT;
}
double ratio = (double) rtl / total;
return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
}
}
package com.commafeed.backend.feed.parser;
import java.text.Bidi;
import java.util.regex.Pattern;
import org.apache.commons.lang3.math.NumberUtils;
public class TextDirectionDetector {
private static final Pattern WORDS_PATTERN = Pattern.compile("\\s+");
private static final Pattern URL_PATTERN = Pattern.compile("^https?://.*");
private static final double RTL_THRESHOLD = 0.4D;
public enum Direction {
LEFT_TO_RIGHT, RIGHT_TO_LEFT
}
public static Direction detect(String input) {
if (input == null || input.isBlank()) {
return Direction.LEFT_TO_RIGHT;
}
long rtl = 0;
long total = 0;
for (String token : WORDS_PATTERN.split(input)) {
// skip urls
if (URL_PATTERN.matcher(token).matches()) {
continue;
}
// skip numbers
if (NumberUtils.isCreatable(token)) {
continue;
}
boolean requiresBidi = Bidi.requiresBidi(token.toCharArray(), 0, token.length());
if (requiresBidi) {
Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT);
if (bidi.getBaseLevel() == 1) {
rtl++;
}
}
total++;
}
if (total == 0) {
return Direction.LEFT_TO_RIGHT;
}
double ratio = (double) rtl / total;
return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
}
}

66
commafeed-server/src/main/java/com/commafeed/backend/model/AbstractModel.java
View File

@@ -1,33 +1,33 @@
package com.commafeed.backend.model;
import java.io.Serializable;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
import jakarta.persistence.Id;
import jakarta.persistence.MappedSuperclass;
import jakarta.persistence.TableGenerator;
import lombok.Getter;
import lombok.Setter;
/**
* Abstract model for all entities, defining id and table generator
*
*/
@SuppressWarnings("serial")
@MappedSuperclass
@Getter
@Setter
public abstract class AbstractModel implements Serializable {
@Id
@GeneratedValue(strategy = GenerationType.TABLE, generator = "gen")
@TableGenerator(
name = "gen",
table = "hibernate_sequences",
pkColumnName = "sequence_name",
valueColumnName = "sequence_next_hi_value",
allocationSize = 1000)
private Long id;
}
package com.commafeed.backend.model;
import java.io.Serializable;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
import jakarta.persistence.Id;
import jakarta.persistence.MappedSuperclass;
import jakarta.persistence.TableGenerator;
import lombok.Getter;
import lombok.Setter;
/**
* Abstract model for all entities, defining id and table generator
*
*/
@SuppressWarnings("serial")
@MappedSuperclass
@Getter
@Setter
public abstract class AbstractModel implements Serializable {
@Id
@GeneratedValue(strategy = GenerationType.TABLE, generator = "gen")
@TableGenerator(
name = "gen",
table = "hibernate_sequences",
pkColumnName = "sequence_name",
valueColumnName = "sequence_next_hi_value",
allocationSize = 1000)
private Long id;
}

222
commafeed-server/src/main/java/com/commafeed/backend/model/Feed.java
View File

@@ -1,111 +1,111 @@
package com.commafeed.backend.model;
import java.sql.Types;
import java.time.Instant;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Lob;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDS")
@SuppressWarnings("serial")
@Getter
@Setter
public class Feed extends AbstractModel {
/**
* The url of the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARCHAR)
private String url;
/**
* cache the url after potential http 30x redirects
*/
@Column(name = "url_after_redirect", length = 2048, nullable = false)
private String urlAfterRedirect;
@Column(length = 2048, nullable = false)
private String normalizedUrl;
@Column(length = 40, nullable = false)
private String normalizedUrlHash;
/**
* The url of the website, extracted from the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String link;
/**
* Last time we tried to fetch the feed
*/
@Column
private Instant lastUpdated;
/**
* Last publishedDate value in the feed
*/
@Column
private Instant lastPublishedDate;
/**
* date of the last entry of the feed
*/
@Column
private Instant lastEntryDate;
/**
* error message while retrieving the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String message;
/**
* times we failed to retrieve the feed
*/
private int errorCount;
/**
* feed refresh is disabled until this date
*/
@Column
private Instant disabledUntil;
/**
* http header returned by the feed
*/
@Column(length = 64)
private String lastModifiedHeader;
/**
* http header returned by the feed
*/
@Column(length = 255)
private String etagHeader;
/**
* average time between entries in the feed
*/
private Long averageEntryInterval;
/**
* last hash of the content of the feed xml
*/
@Column(length = 40)
private String lastContentHash;
}
package com.commafeed.backend.model;
import java.sql.Types;
import java.time.Instant;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Lob;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDS")
@SuppressWarnings("serial")
@Getter
@Setter
public class Feed extends AbstractModel {
/**
* The url of the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARCHAR)
private String url;
/**
* cache the url after potential http 30x redirects
*/
@Column(name = "url_after_redirect", length = 2048, nullable = false)
private String urlAfterRedirect;
@Column(length = 2048, nullable = false)
private String normalizedUrl;
@Column(length = 40, nullable = false)
private String normalizedUrlHash;
/**
* The url of the website, extracted from the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String link;
/**
* Last time we tried to fetch the feed
*/
@Column
private Instant lastUpdated;
/**
* Last publishedDate value in the feed
*/
@Column
private Instant lastPublishedDate;
/**
* date of the last entry of the feed
*/
@Column
private Instant lastEntryDate;
/**
* error message while retrieving the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String message;
/**
* times we failed to retrieve the feed
*/
private int errorCount;
/**
* feed refresh is disabled until this date
*/
@Column
private Instant disabledUntil;
/**
* http header returned by the feed
*/
@Column(length = 64)
private String lastModifiedHeader;
/**
* http header returned by the feed
*/
@Column(length = 255)
private String etagHeader;
/**
* average time between entries in the feed
*/
private Long averageEntryInterval;
/**
* last hash of the content of the feed xml
*/
@Column(length = 40)
private String lastContentHash;
}

86
commafeed-server/src/main/java/com/commafeed/backend/model/FeedCategory.java
View File

@@ -1,43 +1,43 @@
package com.commafeed.backend.model;
import java.util.Set;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDCATEGORIES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedCategory extends AbstractModel {
@Column(length = 128, nullable = false)
private String name;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory parent;
@OneToMany(mappedBy = "parent")
private Set<FeedCategory> children;
@OneToMany(mappedBy = "category")
private Set<FeedSubscription> subscriptions;
private boolean collapsed;
private int position;
}
package com.commafeed.backend.model;
import java.util.Set;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDCATEGORIES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedCategory extends AbstractModel {
@Column(length = 128, nullable = false)
private String name;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory parent;
@OneToMany(mappedBy = "parent")
private Set<FeedCategory> children;
@OneToMany(mappedBy = "category")
private Set<FeedSubscription> subscriptions;
private boolean collapsed;
private int position;
}

120
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntry.java
View File

@@ -1,60 +1,60 @@
package com.commafeed.backend.model;
import java.time.Instant;
import java.util.Set;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRIES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntry extends AbstractModel {
@Column(length = 2048, nullable = false)
private String guid;
@Column(length = 40, nullable = false)
private String guidHash;
@ManyToOne(fetch = FetchType.LAZY)
private Feed feed;
@ManyToOne(fetch = FetchType.LAZY, optional = false)
@JoinColumn(nullable = false, updatable = false)
private FeedEntryContent content;
@Column(length = 2048)
private String url;
/**
* the moment the entry was inserted in the database
*/
@Column
private Instant inserted;
/**
* the moment the entry was published in the feed
*
*/
@Column(name = "updated")
private Instant published;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryTag> tags;
}
package com.commafeed.backend.model;
import java.time.Instant;
import java.util.Set;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRIES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntry extends AbstractModel {
@Column(length = 2048, nullable = false)
private String guid;
@Column(length = 40, nullable = false)
private String guidHash;
@ManyToOne(fetch = FetchType.LAZY)
private Feed feed;
@ManyToOne(fetch = FetchType.LAZY, optional = false)
@JoinColumn(nullable = false, updatable = false)
private FeedEntryContent content;
@Column(length = 2048)
private String url;
/**
* the moment the entry was inserted in the database
*/
@Column
private Instant inserted;
/**
* the moment the entry was published in the feed
*
*/
@Column(name = "updated")
private Instant published;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryTag> tags;
}

210
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryContent.java
View File

@@ -1,105 +1,105 @@
package com.commafeed.backend.model;
import java.sql.Types;
import java.util.Set;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.Lob;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.hibernate.annotations.JdbcTypeCode;
import com.commafeed.backend.feed.FeedUtils;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYCONTENTS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryContent extends AbstractModel {
public enum Direction {
ltr, rtl, unknown
}
@Column(length = 2048)
private String title;
@Column(length = 40)
private String titleHash;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String content;
@Column(length = 40)
private String contentHash;
@Column(name = "author", length = 128)
private String author;
@Column(length = 2048)
private String enclosureUrl;
@Column(length = 255)
private String enclosureType;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String mediaDescription;
@Column(length = 2048)
private String mediaThumbnailUrl;
private Integer mediaThumbnailWidth;
private Integer mediaThumbnailHeight;
@Column(length = 4096)
private String categories;
@Column
@Enumerated(EnumType.STRING)
private Direction direction = Direction.unknown;
@OneToMany(mappedBy = "content")
private Set<FeedEntry> entries;
public boolean equivalentTo(FeedEntryContent c) {
if (c == null) {
return false;
}
return new EqualsBuilder().append(title, c.title)
.append(content, c.content)
.append(author, c.author)
.append(categories, c.categories)
.append(enclosureUrl, c.enclosureUrl)
.append(enclosureType, c.enclosureType)
.append(mediaDescription, c.mediaDescription)
.append(mediaThumbnailUrl, c.mediaThumbnailUrl)
.append(mediaThumbnailWidth, c.mediaThumbnailWidth)
.append(mediaThumbnailHeight, c.mediaThumbnailHeight)
.build();
}
public boolean isRTL() {
if (direction == Direction.rtl) {
return true;
} else if (direction == Direction.ltr) {
return false;
} else {
// detect on the fly for content that was inserted before the direction field was added
return FeedUtils.isRTL(title, content);
}
}
}
package com.commafeed.backend.model;
import java.sql.Types;
import java.util.Set;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.Lob;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.hibernate.annotations.JdbcTypeCode;
import com.commafeed.backend.feed.FeedUtils;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYCONTENTS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryContent extends AbstractModel {
public enum Direction {
ltr, rtl, unknown
}
@Column(length = 2048)
private String title;
@Column(length = 40)
private String titleHash;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String content;
@Column(length = 40)
private String contentHash;
@Column(name = "author", length = 128)
private String author;
@Column(length = 2048)
private String enclosureUrl;
@Column(length = 255)
private String enclosureType;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String mediaDescription;
@Column(length = 2048)
private String mediaThumbnailUrl;
private Integer mediaThumbnailWidth;
private Integer mediaThumbnailHeight;
@Column(length = 4096)
private String categories;
@Column
@Enumerated(EnumType.STRING)
private Direction direction = Direction.unknown;
@OneToMany(mappedBy = "content")
private Set<FeedEntry> entries;
public boolean equivalentTo(FeedEntryContent c) {
if (c == null) {
return false;
}
return new EqualsBuilder().append(title, c.title)
.append(content, c.content)
.append(author, c.author)
.append(categories, c.categories)
.append(enclosureUrl, c.enclosureUrl)
.append(enclosureType, c.enclosureType)
.append(mediaDescription, c.mediaDescription)
.append(mediaThumbnailUrl, c.mediaThumbnailUrl)
.append(mediaThumbnailWidth, c.mediaThumbnailWidth)
.append(mediaThumbnailHeight, c.mediaThumbnailHeight)
.build();
}
public boolean isRTL() {
if (direction == Direction.rtl) {
return true;
} else if (direction == Direction.ltr) {
return false;
} else {
// detect on the fly for content that was inserted before the direction field was added
return FeedUtils.isRTL(title, content);
}
}
}

138
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryStatus.java
View File

@@ -1,69 +1,69 @@
package com.commafeed.backend.model;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import jakarta.persistence.Transient;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYSTATUSES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryStatus extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedSubscription subscription;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedEntry entry;
@Column(name = "read_status")
private boolean read;
private boolean starred;
@Transient
private boolean markable;
@Transient
private List<FeedEntryTag> tags = new ArrayList<>();
/**
* Denormalization starts here
*/
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@Column
private Instant entryInserted;
@Column(name = "entryUpdated")
private Instant entryPublished;
public FeedEntryStatus() {
}
public FeedEntryStatus(User user, FeedSubscription subscription, FeedEntry entry) {
this.user = user;
this.subscription = subscription;
this.entry = entry;
this.entryInserted = entry.getInserted();
this.entryPublished = entry.getPublished();
}
}
package com.commafeed.backend.model;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import jakarta.persistence.Transient;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYSTATUSES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryStatus extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedSubscription subscription;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedEntry entry;
@Column(name = "read_status")
private boolean read;
private boolean starred;
@Transient
private boolean markable;
@Transient
private List<FeedEntryTag> tags = new ArrayList<>();
/**
* Denormalization starts here
*/
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@Column
private Instant entryInserted;
@Column(name = "entryUpdated")
private Instant entryPublished;
public FeedEntryStatus() {
}
public FeedEntryStatus(User user, FeedSubscription subscription, FeedEntry entry) {
this.user = user;
this.subscription = subscription;
this.entry = entry;
this.entryInserted = entry.getInserted();
this.entryPublished = entry.getPublished();
}
}

80
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryTag.java
View File

@@ -1,40 +1,40 @@
package com.commafeed.backend.model;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYTAGS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryTag extends AbstractModel {
@JoinColumn(name = "user_id")
@ManyToOne(fetch = FetchType.LAZY)
private User user;
@JoinColumn(name = "entry_id")
@ManyToOne(fetch = FetchType.LAZY)
private FeedEntry entry;
@Column(name = "name", length = 40)
private String name;
public FeedEntryTag() {
}
public FeedEntryTag(User user, FeedEntry entry, String name) {
this.name = name;
this.entry = entry;
this.user = user;
}
}
package com.commafeed.backend.model;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYTAGS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryTag extends AbstractModel {
@JoinColumn(name = "user_id")
@ManyToOne(fetch = FetchType.LAZY)
private User user;
@JoinColumn(name = "entry_id")
@ManyToOne(fetch = FetchType.LAZY)
private FeedEntry entry;
@Column(name = "name", length = 40)
private String name;
public FeedEntryTag() {
}
public FeedEntryTag(User user, FeedEntry entry, String name) {
this.name = name;
this.entry = entry;
this.user = user;
}
}

92
commafeed-server/src/main/java/com/commafeed/backend/model/FeedSubscription.java
View File

@@ -1,46 +1,46 @@
package com.commafeed.backend.model;
import java.util.Set;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDSUBSCRIPTIONS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedSubscription extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private Feed feed;
@Column(length = 128, nullable = false)
private String title;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory category;
@OneToMany(mappedBy = "subscription", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
private int position;
@Column(name = "filtering_expression", length = 4096)
private String filter;
}
package com.commafeed.backend.model;
import java.util.Set;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDSUBSCRIPTIONS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedSubscription extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private Feed feed;
@Column(length = 128, nullable = false)
private String title;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory category;
@OneToMany(mappedBy = "subscription", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
private int position;
@Column(name = "filtering_expression", length = 4096)
private String filter;
}

82
commafeed-server/src/main/java/com/commafeed/backend/model/Models.java
View File

@@ -1,41 +1,41 @@
package com.commafeed.backend.model;
import java.time.Duration;
import java.time.Instant;
import org.hibernate.Hibernate;
import org.hibernate.HibernateException;
import org.hibernate.proxy.HibernateProxy;
import org.hibernate.proxy.LazyInitializer;
import lombok.experimental.UtilityClass;
@UtilityClass
public class Models {
public static final Instant MINIMUM_INSTANT = Instant.EPOCH
// mariadb timestamp range starts at 1970-01-01 00:00:01
.plusSeconds(1)
// make sure the timestamp fits for all timezones
.plus(Duration.ofHours(24));
/**
* initialize a proxy
*/
public static void initialize(Object proxy) throws HibernateException {
Hibernate.initialize(proxy);
}
/**
* extract the id from the proxy without initializing it
*/
public static Long getId(AbstractModel model) {
if (model instanceof HibernateProxy proxy) {
LazyInitializer lazyInitializer = proxy.getHibernateLazyInitializer();
if (lazyInitializer.isUninitialized()) {
return (Long) lazyInitializer.getIdentifier();
}
}
return model.getId();
}
}
package com.commafeed.backend.model;
import java.time.Duration;
import java.time.Instant;
import org.hibernate.Hibernate;
import org.hibernate.HibernateException;
import org.hibernate.proxy.HibernateProxy;
import org.hibernate.proxy.LazyInitializer;
import lombok.experimental.UtilityClass;
@UtilityClass
public class Models {
public static final Instant MINIMUM_INSTANT = Instant.EPOCH
// mariadb timestamp range starts at 1970-01-01 00:00:01
.plusSeconds(1)
// make sure the timestamp fits for all timezones
.plus(Duration.ofHours(24));
/**
* initialize a proxy
*/
public static void initialize(Object proxy) throws HibernateException {
Hibernate.initialize(proxy);
}
/**
* extract the id from the proxy without initializing it
*/
public static Long getId(AbstractModel model) {
if (model instanceof HibernateProxy proxy) {
LazyInitializer lazyInitializer = proxy.getHibernateLazyInitializer();
if (lazyInitializer.isUninitialized()) {
return (Long) lazyInitializer.getIdentifier();
}
}
return model.getId();
}
}

118
commafeed-server/src/main/java/com/commafeed/backend/model/User.java
View File

@@ -1,59 +1,59 @@
package com.commafeed.backend.model;
import java.sql.Types;
import java.time.Instant;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Lob;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERS")
@SuppressWarnings("serial")
@Getter
@Setter
public class User extends AbstractModel {
@Column(length = 32, nullable = false, unique = true)
private String name;
@Column(length = 255, unique = true)
private String email;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] password;
@Column(length = 40, unique = true)
private String apiKey;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] salt;
@Column(nullable = false)
private boolean disabled;
@Column
private Instant lastLogin;
@Column
private Instant created;
@Column(length = 40)
private String recoverPasswordToken;
@Column
private Instant recoverPasswordTokenDate;
@Column
private Instant lastForceRefresh;
}
package com.commafeed.backend.model;
import java.sql.Types;
import java.time.Instant;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Lob;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERS")
@SuppressWarnings("serial")
@Getter
@Setter
public class User extends AbstractModel {
@Column(length = 32, nullable = false, unique = true)
private String name;
@Column(length = 255, unique = true)
private String email;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] password;
@Column(length = 40, unique = true)
private String apiKey;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] salt;
@Column(nullable = false)
private boolean disabled;
@Column
private Instant lastLogin;
@Column
private Instant created;
@Column(length = 40)
private String recoverPasswordToken;
@Column
private Instant recoverPasswordTokenDate;
@Column
private Instant lastForceRefresh;
}

86
commafeed-server/src/main/java/com/commafeed/backend/model/UserRole.java
View File

@@ -1,43 +1,43 @@
package com.commafeed.backend.model;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.OneToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERROLES")
@SuppressWarnings("serial")
@Getter
@Setter
public class UserRole extends AbstractModel {
public enum Role {
USER, ADMIN
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false)
private User user;
@Column(name = "roleName", nullable = false)
@Enumerated(EnumType.STRING)
private Role role;
public UserRole() {
}
public UserRole(User user, Role role) {
this.user = user;
this.role = role;
}
}
package com.commafeed.backend.model;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.OneToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERROLES")
@SuppressWarnings("serial")
@Getter
@Setter
public class UserRole extends AbstractModel {
public enum Role {
USER, ADMIN
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false)
private User user;
@Column(name = "roleName", nullable = false)
@Enumerated(EnumType.STRING)
private Role role;
public UserRole() {
}
public UserRole(User user, Role role) {
this.user = user;
this.role = role;
}
}

214
commafeed-server/src/main/java/com/commafeed/backend/model/UserSettings.java
View File

@@ -1,107 +1,107 @@
package com.commafeed.backend.model;
import java.sql.Types;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.Lob;
import jakarta.persistence.OneToOne;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERSETTINGS")
@SuppressWarnings("serial")
@Getter
@Setter
public class UserSettings extends AbstractModel {
public enum ReadingMode {
all, unread
}
public enum ReadingOrder {
asc, desc
}
public enum ViewMode {
title, cozy, detailed, expanded
}
public enum ScrollMode {
always, never, if_needed
}
public enum IconDisplayMode {
always, never, on_desktop, on_mobile
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false, unique = true)
private User user;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingMode readingMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingOrder readingOrder;
@Column(name = "user_lang", length = 4)
private String language;
private boolean showRead;
private boolean scrollMarks;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customCss;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customJs;
@Column(name = "scroll_speed")
private int scrollSpeed;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ScrollMode scrollMode;
private int entriesToKeepOnTopWhenScrolling;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode starIconDisplayMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode externalLinkIconDisplayMode;
private boolean markAllAsReadConfirmation;
private boolean customContextMenu;
private boolean mobileFooter;
private boolean unreadCountTitle;
private boolean unreadCountFavicon;
private boolean email;
private boolean gmail;
private boolean facebook;
private boolean twitter;
private boolean tumblr;
private boolean pocket;
private boolean instapaper;
private boolean buffer;
}
package com.commafeed.backend.model;
import java.sql.Types;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.Lob;
import jakarta.persistence.OneToOne;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERSETTINGS")
@SuppressWarnings("serial")
@Getter
@Setter
public class UserSettings extends AbstractModel {
public enum ReadingMode {
all, unread
}
public enum ReadingOrder {
asc, desc
}
public enum ViewMode {
title, cozy, detailed, expanded
}
public enum ScrollMode {
always, never, if_needed
}
public enum IconDisplayMode {
always, never, on_desktop, on_mobile
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false, unique = true)
private User user;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingMode readingMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingOrder readingOrder;
@Column(name = "user_lang", length = 4)
private String language;
private boolean showRead;
private boolean scrollMarks;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customCss;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customJs;
@Column(name = "scroll_speed")
private int scrollSpeed;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ScrollMode scrollMode;
private int entriesToKeepOnTopWhenScrolling;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode starIconDisplayMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode externalLinkIconDisplayMode;
private boolean markAllAsReadConfirmation;
private boolean customContextMenu;
private boolean mobileFooter;
private boolean unreadCountTitle;
private boolean unreadCountFavicon;
private boolean email;
private boolean gmail;
private boolean facebook;
private boolean twitter;
private boolean tumblr;
private boolean pocket;
private boolean instapaper;
private boolean buffer;
}

170
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLExporter.java
View File

@@ -1,85 +1,85 @@
package com.commafeed.backend.opml;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.rometools.opml.feed.opml.Attribute;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OPMLExporter {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
public Opml export(User user) {
Opml opml = new Opml();
opml.setFeedType("opml_1.0");
opml.setTitle(String.format("%s subscriptions in CommaFeed", user.getName()));
opml.setCreated(new Date());
List<FeedCategory> categories = feedCategoryDAO.findAll(user);
categories.sort(Comparator.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0)));
List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user);
subscriptions.sort(Comparator.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0)));
// export root categories
for (FeedCategory cat : categories.stream().filter(c -> c.getParent() == null).toList()) {
opml.getOutlines().add(buildCategoryOutline(cat, categories, subscriptions));
}
// export root subscriptions
for (FeedSubscription sub : subscriptions.stream().filter(s -> s.getCategory() == null).toList()) {
opml.getOutlines().add(buildSubscriptionOutline(sub));
}
return opml;
}
private Outline buildCategoryOutline(FeedCategory cat, List<FeedCategory> categories, List<FeedSubscription> subscriptions) {
Outline outline = new Outline();
outline.setText(cat.getName());
outline.setTitle(cat.getName());
for (FeedCategory child : categories.stream()
.filter(c -> c.getParent() != null && c.getParent().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildCategoryOutline(child, categories, subscriptions));
}
for (FeedSubscription sub : subscriptions.stream()
.filter(s -> s.getCategory() != null && s.getCategory().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildSubscriptionOutline(sub));
}
return outline;
}
private Outline buildSubscriptionOutline(FeedSubscription sub) {
Outline outline = new Outline();
outline.setText(sub.getTitle());
outline.setTitle(sub.getTitle());
outline.setType("rss");
outline.getAttributes().add(new Attribute("xmlUrl", sub.getFeed().getUrl()));
if (sub.getFeed().getLink() != null) {
outline.getAttributes().add(new Attribute("htmlUrl", sub.getFeed().getLink()));
}
return outline;
}
}
package com.commafeed.backend.opml;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.rometools.opml.feed.opml.Attribute;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OPMLExporter {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
public Opml export(User user) {
Opml opml = new Opml();
opml.setFeedType("opml_1.0");
opml.setTitle(String.format("%s subscriptions in CommaFeed", user.getName()));
opml.setCreated(new Date());
List<FeedCategory> categories = feedCategoryDAO.findAll(user);
categories.sort(Comparator.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0)));
List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user);
subscriptions.sort(Comparator.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0)));
// export root categories
for (FeedCategory cat : categories.stream().filter(c -> c.getParent() == null).toList()) {
opml.getOutlines().add(buildCategoryOutline(cat, categories, subscriptions));
}
// export root subscriptions
for (FeedSubscription sub : subscriptions.stream().filter(s -> s.getCategory() == null).toList()) {
opml.getOutlines().add(buildSubscriptionOutline(sub));
}
return opml;
}
private Outline buildCategoryOutline(FeedCategory cat, List<FeedCategory> categories, List<FeedSubscription> subscriptions) {
Outline outline = new Outline();
outline.setText(cat.getName());
outline.setTitle(cat.getName());
for (FeedCategory child : categories.stream()
.filter(c -> c.getParent() != null && c.getParent().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildCategoryOutline(child, categories, subscriptions));
}
for (FeedSubscription sub : subscriptions.stream()
.filter(s -> s.getCategory() != null && s.getCategory().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildSubscriptionOutline(sub));
}
return outline;
}
private Outline buildSubscriptionOutline(FeedSubscription sub) {
Outline outline = new Outline();
outline.setText(sub.getTitle());
outline.setTitle(sub.getTitle());
outline.setType("rss");
outline.getAttributes().add(new Attribute("xmlUrl", sub.getFeed().getUrl()));
if (sub.getFeed().getLink() != null) {
outline.getAttributes().add(new Attribute("htmlUrl", sub.getFeed().getLink()));
}
return outline;
}
}

164
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLImporter.java
View File

@@ -1,82 +1,82 @@
package com.commafeed.backend.opml;
import java.io.StringReader;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedSubscriptionService;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.WireFeedInput;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class OPMLImporter {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionService feedSubscriptionService;
public void importOpml(User user, String xml) throws IllegalArgumentException, FeedException {
xml = xml.substring(xml.indexOf('<'));
WireFeedInput input = new WireFeedInput();
Opml feed = (Opml) input.build(new StringReader(xml));
List<Outline> outlines = feed.getOutlines();
for (int i = 0; i < outlines.size(); i++) {
handleOutline(user, outlines.get(i), null, i);
}
}
private void handleOutline(User user, Outline outline, FeedCategory parent, int position) {
List<Outline> children = outline.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
FeedCategory category = feedCategoryDAO.findByName(user, name, parent);
if (category == null) {
if (StringUtils.isBlank(name)) {
name = "Unnamed category";
}
category = new FeedCategory();
category.setName(name);
category.setParent(parent);
category.setUser(user);
category.setPosition(position);
feedCategoryDAO.saveOrUpdate(category);
}
for (int i = 0; i < children.size(); i++) {
handleOutline(user, children.get(i), category, i);
}
} else {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
if (StringUtils.isBlank(name)) {
name = "Unnamed subscription";
}
// make sure we continue with the import process even if a feed failed
try {
feedSubscriptionService.subscribe(user, outline.getXmlUrl(), name, parent, position);
} catch (Exception e) {
log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage());
}
}
}
}
package com.commafeed.backend.opml;
import java.io.StringReader;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedSubscriptionService;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.WireFeedInput;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class OPMLImporter {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionService feedSubscriptionService;
public void importOpml(User user, String xml) throws IllegalArgumentException, FeedException {
xml = xml.substring(xml.indexOf('<'));
WireFeedInput input = new WireFeedInput();
Opml feed = (Opml) input.build(new StringReader(xml));
List<Outline> outlines = feed.getOutlines();
for (int i = 0; i < outlines.size(); i++) {
handleOutline(user, outlines.get(i), null, i);
}
}
private void handleOutline(User user, Outline outline, FeedCategory parent, int position) {
List<Outline> children = outline.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
FeedCategory category = feedCategoryDAO.findByName(user, name, parent);
if (category == null) {
if (StringUtils.isBlank(name)) {
name = "Unnamed category";
}
category = new FeedCategory();
category.setName(name);
category.setParent(parent);
category.setUser(user);
category.setPosition(position);
feedCategoryDAO.saveOrUpdate(category);
}
for (int i = 0; i < children.size(); i++) {
handleOutline(user, children.get(i), category, i);
}
} else {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
if (StringUtils.isBlank(name)) {
name = "Unnamed subscription";
}
// make sure we continue with the import process even if a feed failed
try {
feedSubscriptionService.subscribe(user, outline.getXmlUrl(), name, parent, position);
} catch (Exception e) {
log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage());
}
}
}
}

52
commafeed-server/src/main/java/com/commafeed/backend/rome/OPML11Generator.java
View File

@@ -1,26 +1,26 @@
package com.commafeed.backend.rome;
import org.jdom2.Element;
import com.rometools.opml.feed.opml.Opml;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Add missing title to the generated OPML
*
*/
@RegisterForReflection
public class OPML11Generator extends com.rometools.opml.io.impl.OPML10Generator {
public OPML11Generator() {
super("opml_1.1");
}
@Override
protected Element generateHead(Opml opml) {
Element head = new Element("head");
addNotNullSimpleElement(head, "title", opml.getTitle());
return head;
}
}
package com.commafeed.backend.rome;
import org.jdom2.Element;
import com.rometools.opml.feed.opml.Opml;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Add missing title to the generated OPML
*
*/
@RegisterForReflection
public class OPML11Generator extends com.rometools.opml.io.impl.OPML10Generator {
public OPML11Generator() {
super("opml_1.1");
}
@Override
protected Element generateHead(Opml opml) {
Element head = new Element("head");
addNotNullSimpleElement(head, "title", opml.getTitle());
return head;
}
}

76
commafeed-server/src/main/java/com/commafeed/backend/rome/OPML11Parser.java
View File

@@ -1,38 +1,38 @@
package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Document;
import org.jdom2.Element;
import com.rometools.opml.io.impl.OPML10Parser;
import com.rometools.rome.feed.WireFeed;
import com.rometools.rome.io.FeedException;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support for OPML 1.1 parsing
*
*/
@RegisterForReflection
public class OPML11Parser extends OPML10Parser {
public OPML11Parser() {
super("opml_1.1");
}
@Override
public boolean isMyType(Document document) {
Element e = document.getRootElement();
return e.getName().equals("opml");
}
@Override
public WireFeed parse(Document document, boolean validate, Locale locale) throws IllegalArgumentException, FeedException {
document.getRootElement().getChildren().add(new Element("head"));
return super.parse(document, validate, locale);
}
}
package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Document;
import org.jdom2.Element;
import com.rometools.opml.io.impl.OPML10Parser;
import com.rometools.rome.feed.WireFeed;
import com.rometools.rome.io.FeedException;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support for OPML 1.1 parsing
*
*/
@RegisterForReflection
public class OPML11Parser extends OPML10Parser {
public OPML11Parser() {
super("opml_1.1");
}
@Override
public boolean isMyType(Document document) {
Element e = document.getRootElement();
return e.getName().equals("opml");
}
@Override
public WireFeed parse(Document document, boolean validate, Locale locale) throws IllegalArgumentException, FeedException {
document.getRootElement().getChildren().add(new Element("head"));
return super.parse(document, validate, locale);
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/rome/RSS090DescriptionConverter.java
View File

@@ -1,30 +1,30 @@
package com.commafeed.backend.rome;
import com.rometools.rome.feed.rss.Description;
import com.rometools.rome.feed.rss.Item;
import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.impl.ConverterForRSS090;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support description tag for RSS09
*
*/
@RegisterForReflection
public class RSS090DescriptionConverter extends ConverterForRSS090 {
@Override
protected SyndEntry createSyndEntry(Item item, boolean preserveWireItem) {
SyndEntry entry = super.createSyndEntry(item, preserveWireItem);
Description desc = item.getDescription();
if (desc != null) {
SyndContentImpl syndDesc = new SyndContentImpl();
syndDesc.setValue(desc.getValue());
entry.setDescription(syndDesc);
}
return entry;
}
}
package com.commafeed.backend.rome;
import com.rometools.rome.feed.rss.Description;
import com.rometools.rome.feed.rss.Item;
import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.impl.ConverterForRSS090;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support description tag for RSS09
*
*/
@RegisterForReflection
public class RSS090DescriptionConverter extends ConverterForRSS090 {
@Override
protected SyndEntry createSyndEntry(Item item, boolean preserveWireItem) {
SyndEntry entry = super.createSyndEntry(item, preserveWireItem);
Description desc = item.getDescription();
if (desc != null) {
SyndContentImpl syndDesc = new SyndContentImpl();
syndDesc.setValue(desc.getValue());
entry.setDescription(syndDesc);
}
return entry;
}
}

64
commafeed-server/src/main/java/com/commafeed/backend/rome/RSS090DescriptionParser.java
View File

@@ -1,32 +1,32 @@
package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Element;
import com.rometools.rome.feed.rss.Description;
import com.rometools.rome.feed.rss.Item;
import com.rometools.rome.io.impl.RSS090Parser;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support description tag for RSS09
*
*/
@RegisterForReflection
public class RSS090DescriptionParser extends RSS090Parser {
@Override
protected Item parseItem(Element rssRoot, Element eItem, Locale locale) {
Item item = super.parseItem(rssRoot, eItem, locale);
Element e = eItem.getChild("description", getRSSNamespace());
if (e != null) {
Description desc = new Description();
desc.setValue(e.getText());
item.setDescription(desc);
}
return item;
}
}
package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Element;
import com.rometools.rome.feed.rss.Description;
import com.rometools.rome.feed.rss.Item;
import com.rometools.rome.io.impl.RSS090Parser;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support description tag for RSS09
*
*/
@RegisterForReflection
public class RSS090DescriptionParser extends RSS090Parser {
@Override
protected Item parseItem(Element rssRoot, Element eItem, Locale locale) {
Item item = super.parseItem(rssRoot, eItem, locale);
Element e = eItem.getChild("description", getRSSNamespace());
if (e != null) {
Description desc = new Description();
desc.setValue(e.getText());
item.setDescription(desc);
}
return item;
}
}

96
commafeed-server/src/main/java/com/commafeed/backend/rome/RSSRDF10Parser.java
View File

@@ -1,48 +1,48 @@
package com.commafeed.backend.rome;
import java.util.List;
import org.apache.commons.collections4.CollectionUtils;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.Namespace;
import com.google.common.collect.Lists;
import com.rometools.rome.io.impl.RSS10Parser;
import io.quarkus.runtime.annotations.RegisterForReflection;
@RegisterForReflection
public class RSSRDF10Parser extends RSS10Parser {
private static final String RSS_URI = "http://purl.org/rss/1.0/";
private static final Namespace RSS_NS = Namespace.getNamespace(RSS_URI);
public RSSRDF10Parser() {
super("rss_1.0", RSS_NS);
}
@Override
public boolean isMyType(Document document) {
boolean ok;
Element rssRoot = document.getRootElement();
Namespace defaultNS = rssRoot.getNamespace();
List<Namespace> additionalNSs = Lists.newArrayList(rssRoot.getAdditionalNamespaces());
List<Element> children = rssRoot.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
Element child = children.get(0);
additionalNSs.add(child.getNamespace());
additionalNSs.addAll(child.getAdditionalNamespaces());
}
ok = defaultNS != null && defaultNS.equals(getRDFNamespace());
if (ok) {
ok = false;
for (int i = 0; !ok && i < additionalNSs.size(); i++) {
ok = getRSSNamespace().equals(additionalNSs.get(i));
}
}
return ok;
}
}
package com.commafeed.backend.rome;
import java.util.List;
import org.apache.commons.collections4.CollectionUtils;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.Namespace;
import com.google.common.collect.Lists;
import com.rometools.rome.io.impl.RSS10Parser;
import io.quarkus.runtime.annotations.RegisterForReflection;
@RegisterForReflection
public class RSSRDF10Parser extends RSS10Parser {
private static final String RSS_URI = "http://purl.org/rss/1.0/";
private static final Namespace RSS_NS = Namespace.getNamespace(RSS_URI);
public RSSRDF10Parser() {
super("rss_1.0", RSS_NS);
}
@Override
public boolean isMyType(Document document) {
boolean ok;
Element rssRoot = document.getRootElement();
Namespace defaultNS = rssRoot.getNamespace();
List<Namespace> additionalNSs = Lists.newArrayList(rssRoot.getAdditionalNamespaces());
List<Element> children = rssRoot.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
Element child = children.get(0);
additionalNSs.add(child.getNamespace());
additionalNSs.addAll(child.getAdditionalNamespaces());
}
ok = defaultNS != null && defaultNS.equals(getRDFNamespace());
if (ok) {
ok = false;
for (int i = 0; !ok && i < additionalNSs.size(); i++) {
ok = getRSSNamespace().equals(additionalNSs.get(i));
}
}
return ok;
}
}

350
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentCleaningService.java
View File

@@ -1,175 +1,175 @@
package com.commafeed.backend.service;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Document.OutputSettings;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Safelist;
import org.w3c.css.sac.CSSException;
import org.w3c.css.sac.CSSParseException;
import org.w3c.css.sac.ErrorHandler;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
import com.steadystate.css.parser.CSSOMParser;
import com.steadystate.css.parser.SACParserCSS21;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
@Singleton
public class FeedEntryContentCleaningService {
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
public String clean(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
}
return content;
}
private static Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1",
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
private String escapeIFrameCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private String escapeImgCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser(new SACParserCSS21());
parser.setErrorHandler(new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug("warning while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug("fatal error while parsing css: {}", exception.getMessage(), exception);
}
});
return parser;
}
}
package com.commafeed.backend.service;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Document.OutputSettings;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Safelist;
import org.w3c.css.sac.CSSException;
import org.w3c.css.sac.CSSParseException;
import org.w3c.css.sac.ErrorHandler;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
import com.steadystate.css.parser.CSSOMParser;
import com.steadystate.css.parser.SACParserCSS21;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
@Singleton
public class FeedEntryContentCleaningService {
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
public String clean(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
}
return content;
}
private static Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1",
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
private String escapeIFrameCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private String escapeImgCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser(new SACParserCSS21());
parser.setErrorHandler(new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug("warning while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug("fatal error while parsing css: {}", exception.getMessage(), exception);
}
});
return parser;
}
}

142
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentService.java
View File

@@ -1,71 +1,71 @@
package com.commafeed.backend.service;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.commafeed.backend.model.FeedEntryContent;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryContentService {
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryContentCleaningService cleaningService;
/**
* this is NOT thread-safe
*/
public FeedEntryContent findOrCreate(Content content, String baseUrl) {
FeedEntryContent entryContent = buildContent(content, baseUrl);
Optional<FeedEntryContent> existing = feedEntryContentDAO.findExisting(entryContent.getContentHash(), entryContent.getTitleHash())
.stream()
.filter(entryContent::equivalentTo)
.findFirst();
if (existing.isPresent()) {
return existing.get();
} else {
feedEntryContentDAO.saveOrUpdate(entryContent);
return entryContent;
}
}
private FeedEntryContent buildContent(Content content, String baseUrl) {
FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitleHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.title())));
entryContent.setContentHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.content())));
entryContent.setTitle(FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false));
entryContent.setAuthor(FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
entryContent.setDirection(
FeedUtils.isRTL(content.title(), content.content()) ? FeedEntryContent.Direction.rtl : FeedEntryContent.Direction.ltr);
Enclosure enclosure = content.enclosure();
if (enclosure != null) {
entryContent.setEnclosureUrl(FeedUtils.truncate(enclosure.url(), 2048));
entryContent.setEnclosureType(enclosure.type());
}
Media media = content.media();
if (media != null) {
entryContent.setMediaDescription(cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048));
entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
entryContent.setMediaThumbnailHeight(media.thumbnailHeight());
}
return entryContent;
}
}
package com.commafeed.backend.service;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.commafeed.backend.model.FeedEntryContent;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryContentService {
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryContentCleaningService cleaningService;
/**
* this is NOT thread-safe
*/
public FeedEntryContent findOrCreate(Content content, String baseUrl) {
FeedEntryContent entryContent = buildContent(content, baseUrl);
Optional<FeedEntryContent> existing = feedEntryContentDAO.findExisting(entryContent.getContentHash(), entryContent.getTitleHash())
.stream()
.filter(entryContent::equivalentTo)
.findFirst();
if (existing.isPresent()) {
return existing.get();
} else {
feedEntryContentDAO.saveOrUpdate(entryContent);
return entryContent;
}
}
private FeedEntryContent buildContent(Content content, String baseUrl) {
FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitleHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.title())));
entryContent.setContentHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.content())));
entryContent.setTitle(FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false));
entryContent.setAuthor(FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
entryContent.setDirection(
FeedUtils.isRTL(content.title(), content.content()) ? FeedEntryContent.Direction.rtl : FeedEntryContent.Direction.ltr);
Enclosure enclosure = content.enclosure();
if (enclosure != null) {
entryContent.setEnclosureUrl(FeedUtils.truncate(enclosure.url(), 2048));
entryContent.setEnclosureType(enclosure.type());
}
Media media = content.media();
if (media != null) {
entryContent.setMediaDescription(cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048));
entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
entryContent.setMediaThumbnailHeight(media.thumbnailHeight());
}
return entryContent;
}
}

248
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryFilteringService.java
View File

@@ -1,124 +1,124 @@
package com.commafeed.backend.service;
import java.time.Year;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import jakarta.inject.Singleton;
import org.apache.commons.jexl2.JexlContext;
import org.apache.commons.jexl2.JexlEngine;
import org.apache.commons.jexl2.JexlException;
import org.apache.commons.jexl2.JexlInfo;
import org.apache.commons.jexl2.MapContext;
import org.apache.commons.jexl2.Script;
import org.apache.commons.jexl2.introspection.JexlMethod;
import org.apache.commons.jexl2.introspection.JexlPropertyGet;
import org.apache.commons.jexl2.introspection.Uberspect;
import org.apache.commons.jexl2.introspection.UberspectImpl;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.model.FeedEntry;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryFilteringService {
private static final JexlEngine ENGINE = initEngine();
private final ExecutorService executor = Executors.newCachedThreadPool();
private final CommaFeedConfiguration config;
private static JexlEngine initEngine() {
// classloader that prevents object creation
ClassLoader cl = new ClassLoader() {
@Override
protected Class<?> loadClass(String name, boolean resolve) {
return null;
}
};
// uberspect that prevents access to .class and .getClass()
Uberspect uberspect = new UberspectImpl(LogFactory.getLog(JexlEngine.class)) {
@Override
public JexlPropertyGet getPropertyGet(Object obj, Object identifier, JexlInfo info) {
if ("class".equals(identifier)) {
return null;
}
return super.getPropertyGet(obj, identifier, info);
}
@Override
public JexlMethod getMethod(Object obj, String method, Object[] args, JexlInfo info) {
if ("getClass".equals(method)) {
return null;
}
return super.getMethod(obj, method, args, info);
}
};
JexlEngine engine = new JexlEngine(uberspect, null, null, null);
engine.setStrict(true);
engine.setClassLoader(cl);
return engine;
}
public boolean filterMatchesEntry(String filter, FeedEntry entry) throws FeedEntryFilterException {
if (StringUtils.isBlank(filter)) {
return true;
}
Script script;
try {
script = ENGINE.createScript(filter);
} catch (JexlException e) {
throw new FeedEntryFilterException("Exception while parsing expression " + filter, e);
}
JexlContext context = new MapContext();
context.set("title", entry.getContent().getTitle() == null ? "" : Jsoup.parse(entry.getContent().getTitle()).text().toLowerCase());
context.set("author", entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor().toLowerCase());
context.set("content",
entry.getContent().getContent() == null ? "" : Jsoup.parse(entry.getContent().getContent()).text().toLowerCase());
context.set("url", entry.getUrl() == null ? "" : entry.getUrl().toLowerCase());
context.set("categories", entry.getContent().getCategories() == null ? "" : entry.getContent().getCategories().toLowerCase());
context.set("year", Year.now().getValue());
Callable<Object> callable = script.callable(context);
Future<Object> future = executor.submit(callable);
Object result;
try {
result = future.get(config.feedRefresh().filteringExpressionEvaluationTimeout().toMillis(), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e);
} catch (ExecutionException e) {
throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}
try {
return (boolean) result;
} catch (ClassCastException e) {
throw new FeedEntryFilterException(e.getMessage(), e);
}
}
@SuppressWarnings("serial")
public static class FeedEntryFilterException extends Exception {
public FeedEntryFilterException(String message, Throwable t) {
super(message, t);
}
}
}
package com.commafeed.backend.service;
import java.time.Year;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import jakarta.inject.Singleton;
import org.apache.commons.jexl2.JexlContext;
import org.apache.commons.jexl2.JexlEngine;
import org.apache.commons.jexl2.JexlException;
import org.apache.commons.jexl2.JexlInfo;
import org.apache.commons.jexl2.MapContext;
import org.apache.commons.jexl2.Script;
import org.apache.commons.jexl2.introspection.JexlMethod;
import org.apache.commons.jexl2.introspection.JexlPropertyGet;
import org.apache.commons.jexl2.introspection.Uberspect;
import org.apache.commons.jexl2.introspection.UberspectImpl;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.model.FeedEntry;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryFilteringService {
private static final JexlEngine ENGINE = initEngine();
private final ExecutorService executor = Executors.newCachedThreadPool();
private final CommaFeedConfiguration config;
private static JexlEngine initEngine() {
// classloader that prevents object creation
ClassLoader cl = new ClassLoader() {
@Override
protected Class<?> loadClass(String name, boolean resolve) {
return null;
}
};
// uberspect that prevents access to .class and .getClass()
Uberspect uberspect = new UberspectImpl(LogFactory.getLog(JexlEngine.class)) {
@Override
public JexlPropertyGet getPropertyGet(Object obj, Object identifier, JexlInfo info) {
if ("class".equals(identifier)) {
return null;
}
return super.getPropertyGet(obj, identifier, info);
}
@Override
public JexlMethod getMethod(Object obj, String method, Object[] args, JexlInfo info) {
if ("getClass".equals(method)) {
return null;
}
return super.getMethod(obj, method, args, info);
}
};
JexlEngine engine = new JexlEngine(uberspect, null, null, null);
engine.setStrict(true);
engine.setClassLoader(cl);
return engine;
}
public boolean filterMatchesEntry(String filter, FeedEntry entry) throws FeedEntryFilterException {
if (StringUtils.isBlank(filter)) {
return true;
}
Script script;
try {
script = ENGINE.createScript(filter);
} catch (JexlException e) {
throw new FeedEntryFilterException("Exception while parsing expression " + filter, e);
}
JexlContext context = new MapContext();
context.set("title", entry.getContent().getTitle() == null ? "" : Jsoup.parse(entry.getContent().getTitle()).text().toLowerCase());
context.set("author", entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor().toLowerCase());
context.set("content",
entry.getContent().getContent() == null ? "" : Jsoup.parse(entry.getContent().getContent()).text().toLowerCase());
context.set("url", entry.getUrl() == null ? "" : entry.getUrl().toLowerCase());
context.set("categories", entry.getContent().getCategories() == null ? "" : entry.getContent().getCategories().toLowerCase());
context.set("year", Year.now().getValue());
Callable<Object> callable = script.callable(context);
Future<Object> future = executor.submit(callable);
Object result;
try {
result = future.get(config.feedRefresh().filteringExpressionEvaluationTimeout().toMillis(), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e);
} catch (ExecutionException e) {
throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}
try {
return (boolean) result;
} catch (ClassCastException e) {
throw new FeedEntryFilterException(e.getMessage(), e);
}
}
@SuppressWarnings("serial")
public static class FeedEntryFilterException extends Exception {
public FeedEntryFilterException(String message, Throwable t) {
super(message, t);
}
}
}

262
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java
View File

@@ -1,131 +1,131 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class FeedEntryService {
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
public FeedEntry find(Feed feed, Entry entry) {
String guidHash = Digests.sha1Hex(entry.guid());
return feedEntryDAO.findExisting(guidHash, feed);
}
public FeedEntry create(Feed feed, Entry entry) {
FeedEntry feedEntry = new FeedEntry();
feedEntry.setGuid(FeedUtils.truncate(entry.guid(), 2048));
feedEntry.setGuidHash(Digests.sha1Hex(entry.guid()));
feedEntry.setUrl(FeedUtils.truncate(entry.url(), 2048));
feedEntry.setPublished(entry.published());
feedEntry.setInserted(Instant.now());
feedEntry.setFeed(feed);
feedEntry.setContent(feedEntryContentService.findOrCreate(entry.content(), feed.getLink()));
feedEntryDAO.saveOrUpdate(feedEntry);
return feedEntry;
}
public boolean applyFilter(FeedSubscription sub, FeedEntry entry) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.saveOrUpdate(status);
}
return matches;
}
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, entry.getFeed());
if (sub == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
if (status.isMarkable()) {
status.setRead(read);
feedEntryStatusDAO.saveOrUpdate(status);
}
}
public void starEntry(User user, Long entryId, Long subscriptionId, boolean starred) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subscriptionId);
if (sub == null) {
return;
}
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
status.setStarred(starred);
feedEntryStatusDAO.saveOrUpdate(status);
}
public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Instant olderThan, Instant insertedBefore,
List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null,
false, null, null, null);
markList(statuses, olderThan, insertedBefore);
}
public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, -1, -1, null, false);
markList(statuses, olderThan, insertedBefore);
}
private void markList(List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statusesToMark = statuses.stream().filter(FeedEntryStatus::isMarkable).filter(s -> {
Instant entryDate = s.getEntry().getPublished();
return olderThan == null || entryDate == null || entryDate.isBefore(olderThan);
}).filter(s -> {
Instant insertedDate = s.getEntry().getInserted();
return insertedBefore == null || insertedDate == null || insertedDate.isBefore(insertedBefore);
}).toList();
statusesToMark.forEach(s -> s.setRead(true));
feedEntryStatusDAO.saveOrUpdate(statusesToMark);
}
}
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class FeedEntryService {
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
public FeedEntry find(Feed feed, Entry entry) {
String guidHash = Digests.sha1Hex(entry.guid());
return feedEntryDAO.findExisting(guidHash, feed);
}
public FeedEntry create(Feed feed, Entry entry) {
FeedEntry feedEntry = new FeedEntry();
feedEntry.setGuid(FeedUtils.truncate(entry.guid(), 2048));
feedEntry.setGuidHash(Digests.sha1Hex(entry.guid()));
feedEntry.setUrl(FeedUtils.truncate(entry.url(), 2048));
feedEntry.setPublished(entry.published());
feedEntry.setInserted(Instant.now());
feedEntry.setFeed(feed);
feedEntry.setContent(feedEntryContentService.findOrCreate(entry.content(), feed.getLink()));
feedEntryDAO.saveOrUpdate(feedEntry);
return feedEntry;
}
public boolean applyFilter(FeedSubscription sub, FeedEntry entry) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.saveOrUpdate(status);
}
return matches;
}
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, entry.getFeed());
if (sub == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
if (status.isMarkable()) {
status.setRead(read);
feedEntryStatusDAO.saveOrUpdate(status);
}
}
public void starEntry(User user, Long entryId, Long subscriptionId, boolean starred) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subscriptionId);
if (sub == null) {
return;
}
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
status.setStarred(starred);
feedEntryStatusDAO.saveOrUpdate(status);
}
public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Instant olderThan, Instant insertedBefore,
List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null,
false, null, null, null);
markList(statuses, olderThan, insertedBefore);
}
public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, -1, -1, null, false);
markList(statuses, olderThan, insertedBefore);
}
private void markList(List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statusesToMark = statuses.stream().filter(FeedEntryStatus::isMarkable).filter(s -> {
Instant entryDate = s.getEntry().getPublished();
return olderThan == null || entryDate == null || entryDate.isBefore(olderThan);
}).filter(s -> {
Instant insertedDate = s.getEntry().getInserted();
return insertedBefore == null || insertedDate == null || insertedDate.isBefore(insertedBefore);
}).toList();
statusesToMark.forEach(s -> s.setRead(true));
feedEntryStatusDAO.saveOrUpdate(statusesToMark);
}
}

86
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryTagService.java
View File

@@ -1,43 +1,43 @@
package com.commafeed.backend.service;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryTagDAO;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryTagService {
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO;
public void updateTags(User user, Long entryId, List<String> tagNames) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry);
Set<String> existingTagNames = existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
List<FeedEntryTag> addList = tagNames.stream()
.filter(name -> !existingTagNames.contains(name))
.map(name -> new FeedEntryTag(user, entry, name))
.toList();
List<FeedEntryTag> removeList = existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).toList();
feedEntryTagDAO.saveOrUpdate(addList);
feedEntryTagDAO.delete(removeList);
}
}
package com.commafeed.backend.service;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryTagDAO;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryTagService {
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO;
public void updateTags(User user, Long entryId, List<String> tagNames) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry);
Set<String> existingTagNames = existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
List<FeedEntryTag> addList = tagNames.stream()
.filter(name -> !existingTagNames.contains(name))
.map(name -> new FeedEntryTag(user, entry, name))
.toList();
List<FeedEntryTag> removeList = existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).toList();
feedEntryTagDAO.saveOrUpdate(addList);
feedEntryTagDAO.delete(removeList);
}
}

160
commafeed-server/src/main/java/com/commafeed/backend/service/FeedService.java
View File

@@ -1,80 +1,80 @@
package com.commafeed.backend.service;
import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
import com.commafeed.backend.favicon.Favicon;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.Models;
import com.google.common.io.Resources;
import io.quarkus.arc.All;
@Singleton
public class FeedService {
private final FeedDAO feedDAO;
private final List<AbstractFaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
public FeedService(FeedDAO feedDAO, @All List<AbstractFaviconFetcher> faviconFetchers) {
this.feedDAO = feedDAO;
this.faviconFetchers = faviconFetchers;
try {
defaultFavicon = new Favicon(
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif");
} catch (IOException e) {
throw new RuntimeException("could not load default favicon", e);
}
}
public synchronized Feed findOrCreate(String url) {
String normalizedUrl = FeedUtils.normalizeURL(url);
String normalizedUrlHash = Digests.sha1Hex(normalizedUrl);
Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash);
if (feed == null) {
feed = new Feed();
feed.setUrl(url);
feed.setNormalizedUrl(normalizedUrl);
feed.setNormalizedUrlHash(normalizedUrlHash);
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
feedDAO.persist(feed);
}
return feed;
}
public void update(Feed feed) {
String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(Digests.sha1Hex(normalized));
feed.setLastUpdated(Instant.now());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.merge(feed);
}
public Favicon fetchFavicon(Feed feed) {
Favicon icon = null;
for (AbstractFaviconFetcher faviconFetcher : faviconFetchers) {
icon = faviconFetcher.fetch(feed);
if (icon != null) {
break;
}
}
if (icon == null) {
icon = defaultFavicon;
}
return icon;
}
}
package com.commafeed.backend.service;
import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
import com.commafeed.backend.favicon.Favicon;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.Models;
import com.google.common.io.Resources;
import io.quarkus.arc.All;
@Singleton
public class FeedService {
private final FeedDAO feedDAO;
private final List<AbstractFaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
public FeedService(FeedDAO feedDAO, @All List<AbstractFaviconFetcher> faviconFetchers) {
this.feedDAO = feedDAO;
this.faviconFetchers = faviconFetchers;
try {
defaultFavicon = new Favicon(
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif");
} catch (IOException e) {
throw new RuntimeException("could not load default favicon", e);
}
}
public synchronized Feed findOrCreate(String url) {
String normalizedUrl = FeedUtils.normalizeURL(url);
String normalizedUrlHash = Digests.sha1Hex(normalizedUrl);
Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash);
if (feed == null) {
feed = new Feed();
feed.setUrl(url);
feed.setNormalizedUrl(normalizedUrl);
feed.setNormalizedUrlHash(normalizedUrlHash);
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
feedDAO.persist(feed);
}
return feed;
}
public void update(Feed feed) {
String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(Digests.sha1Hex(normalized));
feed.setLastUpdated(Instant.now());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.merge(feed);
}
public Favicon fetchFavicon(Feed feed) {
Favicon icon = null;
for (AbstractFaviconFetcher faviconFetcher : faviconFetchers) {
icon = faviconFetcher.fetch(feed);
if (icon != null) {
break;
}
}
if (icon == null) {
icon = defaultFavicon;
}
return icon;
}
}

296
commafeed-server/src/main/java/com/commafeed/backend/service/FeedSubscriptionService.java
View File

@@ -1,148 +1,148 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.UnreadCount;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedSubscriptionService {
private final FeedDAO feedDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService;
private final FeedRefreshEngine feedRefreshEngine;
private final CommaFeedConfiguration config;
public FeedSubscriptionService(FeedDAO feedDAO, FeedEntryStatusDAO feedEntryStatusDAO, FeedSubscriptionDAO feedSubscriptionDAO,
FeedService feedService, FeedRefreshEngine feedRefreshEngine, CommaFeedConfiguration config) {
this.feedDAO = feedDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.feedService = feedService;
this.feedRefreshEngine = feedRefreshEngine;
this.config = config;
// automatically refresh new feeds after they are subscribed to
// we need to use this hook because the feed needs to have been persisted before being processed by the feed engine
feedSubscriptionDAO.onPostCommitInsert(sub -> {
Feed feed = sub.getFeed();
if (feed.getDisabledUntil() == null || feed.getDisabledUntil().isBefore(Instant.now())) {
feedRefreshEngine.refreshImmediately(feed);
}
});
}
public long subscribe(User user, String url, String title) {
return subscribe(user, url, title, null, 0);
}
public long subscribe(User user, String url, String title, FeedCategory parent) {
return subscribe(user, url, title, parent, 0);
}
public long subscribe(User user, String url, String title, FeedCategory category, int position) {
Integer maxFeedsPerUser = config.database().cleanup().maxFeedsPerUser();
if (maxFeedsPerUser > 0 && feedSubscriptionDAO.count(user) >= maxFeedsPerUser) {
String message = String.format("You cannot subscribe to more feeds on this CommaFeed instance (max %s feeds per user)",
maxFeedsPerUser);
throw new FeedSubscriptionException(message);
}
Feed feed = feedService.findOrCreate(url);
// upgrade feed to https if it was using http
if (FeedUtils.isHttp(feed.getUrl()) && FeedUtils.isHttps(url)) {
feed.setUrl(url);
feedDAO.saveOrUpdate(feed);
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
if (sub == null) {
sub = new FeedSubscription();
sub.setFeed(feed);
sub.setUser(user);
}
sub.setCategory(category);
sub.setPosition(position);
sub.setTitle(FeedUtils.truncate(title, 128));
feedSubscriptionDAO.saveOrUpdate(sub);
return sub.getId();
}
public boolean unsubscribe(User user, Long subId) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subId);
if (sub != null) {
feedSubscriptionDAO.delete(sub);
return true;
} else {
return false;
}
}
public void refreshAll(User user) throws ForceFeedRefreshTooSoonException {
Instant lastForceRefresh = user.getLastForceRefresh();
if (lastForceRefresh != null && lastForceRefresh.plus(config.feedRefresh().forceRefreshCooldownDuration()).isAfter(Instant.now())) {
throw new ForceFeedRefreshTooSoonException();
}
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
user.setLastForceRefresh(Instant.now());
}
public void refreshAllUpForRefresh(User user) {
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Instant disabledUntil = sub.getFeed().getDisabledUntil();
if (disabledUntil == null || disabledUntil.isBefore(Instant.now())) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
}
}
public Map<Long, UnreadCount> getUnreadCount(User user) {
return feedSubscriptionDAO.findAll(user)
.stream()
.collect(Collectors.toMap(FeedSubscription::getId, feedEntryStatusDAO::getUnreadCount));
}
@SuppressWarnings("serial")
public static class FeedSubscriptionException extends RuntimeException {
private FeedSubscriptionException(String msg) {
super(msg);
}
}
@SuppressWarnings("serial")
public static class ForceFeedRefreshTooSoonException extends Exception {
private ForceFeedRefreshTooSoonException() {
super();
}
}
}
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.UnreadCount;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedSubscriptionService {
private final FeedDAO feedDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService;
private final FeedRefreshEngine feedRefreshEngine;
private final CommaFeedConfiguration config;
public FeedSubscriptionService(FeedDAO feedDAO, FeedEntryStatusDAO feedEntryStatusDAO, FeedSubscriptionDAO feedSubscriptionDAO,
FeedService feedService, FeedRefreshEngine feedRefreshEngine, CommaFeedConfiguration config) {
this.feedDAO = feedDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.feedService = feedService;
this.feedRefreshEngine = feedRefreshEngine;
this.config = config;
// automatically refresh new feeds after they are subscribed to
// we need to use this hook because the feed needs to have been persisted before being processed by the feed engine
feedSubscriptionDAO.onPostCommitInsert(sub -> {
Feed feed = sub.getFeed();
if (feed.getDisabledUntil() == null || feed.getDisabledUntil().isBefore(Instant.now())) {
feedRefreshEngine.refreshImmediately(feed);
}
});
}
public long subscribe(User user, String url, String title) {
return subscribe(user, url, title, null, 0);
}
public long subscribe(User user, String url, String title, FeedCategory parent) {
return subscribe(user, url, title, parent, 0);
}
public long subscribe(User user, String url, String title, FeedCategory category, int position) {
Integer maxFeedsPerUser = config.database().cleanup().maxFeedsPerUser();
if (maxFeedsPerUser > 0 && feedSubscriptionDAO.count(user) >= maxFeedsPerUser) {
String message = String.format("You cannot subscribe to more feeds on this CommaFeed instance (max %s feeds per user)",
maxFeedsPerUser);
throw new FeedSubscriptionException(message);
}
Feed feed = feedService.findOrCreate(url);
// upgrade feed to https if it was using http
if (FeedUtils.isHttp(feed.getUrl()) && FeedUtils.isHttps(url)) {
feed.setUrl(url);
feedDAO.saveOrUpdate(feed);
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
if (sub == null) {
sub = new FeedSubscription();
sub.setFeed(feed);
sub.setUser(user);
}
sub.setCategory(category);
sub.setPosition(position);
sub.setTitle(FeedUtils.truncate(title, 128));
feedSubscriptionDAO.saveOrUpdate(sub);
return sub.getId();
}
public boolean unsubscribe(User user, Long subId) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subId);
if (sub != null) {
feedSubscriptionDAO.delete(sub);
return true;
} else {
return false;
}
}
public void refreshAll(User user) throws ForceFeedRefreshTooSoonException {
Instant lastForceRefresh = user.getLastForceRefresh();
if (lastForceRefresh != null && lastForceRefresh.plus(config.feedRefresh().forceRefreshCooldownDuration()).isAfter(Instant.now())) {
throw new ForceFeedRefreshTooSoonException();
}
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
user.setLastForceRefresh(Instant.now());
}
public void refreshAllUpForRefresh(User user) {
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Instant disabledUntil = sub.getFeed().getDisabledUntil();
if (disabledUntil == null || disabledUntil.isBefore(Instant.now())) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
}
}
public Map<Long, UnreadCount> getUnreadCount(User user) {
return feedSubscriptionDAO.findAll(user)
.stream()
.collect(Collectors.toMap(FeedSubscription::getId, feedEntryStatusDAO::getUnreadCount));
}
@SuppressWarnings("serial")
public static class FeedSubscriptionException extends RuntimeException {
private FeedSubscriptionException(String msg) {
super(msg);
}
}
@SuppressWarnings("serial")
public static class ForceFeedRefreshTooSoonException extends Exception {
private ForceFeedRefreshTooSoonException() {
super();
}
}
}

42
commafeed-server/src/main/java/com/commafeed/backend/service/MailService.java
View File

@@ -1,21 +1,21 @@
package com.commafeed.backend.service;
import jakarta.inject.Singleton;
import com.commafeed.backend.model.User;
import io.quarkus.mailer.Mail;
import io.quarkus.mailer.Mailer;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class MailService {
private final Mailer mailer;
public void sendMail(User user, String subject, String content) {
Mail mail = Mail.withHtml(user.getEmail(), "CommaFeed - " + subject, content);
mailer.send(mail);
}
}
package com.commafeed.backend.service;
import jakarta.inject.Singleton;
import com.commafeed.backend.model.User;
import io.quarkus.mailer.Mail;
import io.quarkus.mailer.Mailer;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class MailService {
private final Mailer mailer;
public void sendMail(User user, String subject, String content) {
Mail mail = Mail.withHtml(user.getEmail(), "CommaFeed - " + subject, content);
mailer.send(mail);
}
}

190
commafeed-server/src/main/java/com/commafeed/backend/service/PasswordEncryptionService.java
View File

@@ -1,95 +1,95 @@
package com.commafeed.backend.service;
import java.io.Serializable;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.spec.KeySpec;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
// taken from http://www.javacodegeeks.com/2012/05/secure-password-storage-donts-dos-and.html
@SuppressWarnings("serial")
@Slf4j
@RequiredArgsConstructor
@Singleton
public class PasswordEncryptionService implements Serializable {
public boolean authenticate(String attemptedPassword, byte[] encryptedPassword, byte[] salt) {
if (StringUtils.isBlank(attemptedPassword)) {
return false;
}
// Encrypt the clear-text password using the same salt that was used to
// encrypt the original password
byte[] encryptedAttemptedPassword = null;
try {
encryptedAttemptedPassword = getEncryptedPassword(attemptedPassword, salt);
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
if (encryptedAttemptedPassword == null) {
return false;
}
// Authentication succeeds if encrypted password that the user entered
// is equal to the stored hash
return MessageDigest.isEqual(encryptedPassword, encryptedAttemptedPassword);
}
public byte[] getEncryptedPassword(String password, byte[] salt) {
// PBKDF2 with SHA-1 as the hashing algorithm. Note that the NIST
// specifically names SHA-1 as an acceptable hashing algorithm for
// PBKDF2
String algorithm = "PBKDF2WithHmacSHA1";
// SHA-1 generates 160 bit hashes, so that's what makes sense here
int derivedKeyLength = 160;
// Pick an iteration count that works for you. The NIST recommends at
// least 1,000 iterations:
// http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf
// iOS 4.x reportedly uses 10,000:
// http://blog.crackpassword.com/2010/09/smartphone-forensics-cracking-blackberry-backup-passwords/
int iterations = 20000;
KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, derivedKeyLength);
byte[] bytes = null;
try {
SecretKeyFactory f = SecretKeyFactory.getInstance(algorithm);
SecretKey key = f.generateSecret(spec);
bytes = key.getEncoded();
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
return bytes;
}
public byte[] generateSalt() {
// VERY important to use SecureRandom instead of just Random
byte[] salt = null;
try {
SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
// Generate a 8 byte (64 bit) salt as recommended by RSA PKCS5
salt = new byte[8];
random.nextBytes(salt);
} catch (NoSuchAlgorithmException e) {
// should never happen
log.error(e.getMessage(), e);
}
return salt;
}
}
package com.commafeed.backend.service;
import java.io.Serializable;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.spec.KeySpec;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
// taken from http://www.javacodegeeks.com/2012/05/secure-password-storage-donts-dos-and.html
@SuppressWarnings("serial")
@Slf4j
@RequiredArgsConstructor
@Singleton
public class PasswordEncryptionService implements Serializable {
public boolean authenticate(String attemptedPassword, byte[] encryptedPassword, byte[] salt) {
if (StringUtils.isBlank(attemptedPassword)) {
return false;
}
// Encrypt the clear-text password using the same salt that was used to
// encrypt the original password
byte[] encryptedAttemptedPassword = null;
try {
encryptedAttemptedPassword = getEncryptedPassword(attemptedPassword, salt);
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
if (encryptedAttemptedPassword == null) {
return false;
}
// Authentication succeeds if encrypted password that the user entered
// is equal to the stored hash
return MessageDigest.isEqual(encryptedPassword, encryptedAttemptedPassword);
}
public byte[] getEncryptedPassword(String password, byte[] salt) {
// PBKDF2 with SHA-1 as the hashing algorithm. Note that the NIST
// specifically names SHA-1 as an acceptable hashing algorithm for
// PBKDF2
String algorithm = "PBKDF2WithHmacSHA1";
// SHA-1 generates 160 bit hashes, so that's what makes sense here
int derivedKeyLength = 160;
// Pick an iteration count that works for you. The NIST recommends at
// least 1,000 iterations:
// http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf
// iOS 4.x reportedly uses 10,000:
// http://blog.crackpassword.com/2010/09/smartphone-forensics-cracking-blackberry-backup-passwords/
int iterations = 20000;
KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, derivedKeyLength);
byte[] bytes = null;
try {
SecretKeyFactory f = SecretKeyFactory.getInstance(algorithm);
SecretKey key = f.generateSecret(spec);
bytes = key.getEncoded();
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
return bytes;
}
public byte[] generateSalt() {
// VERY important to use SecureRandom instead of just Random
byte[] salt = null;
try {
SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
// Generate a 8 byte (64 bit) salt as recommended by RSA PKCS5
salt = new byte[8];
random.nextBytes(salt);
} catch (NoSuchAlgorithmException e) {
// should never happen
log.error(e.getMessage(), e);
}
return salt;
}
}

332
commafeed-server/src/main/java/com/commafeed/backend/service/UserService.java
View File

@@ -1,166 +1,166 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.dao.UserRoleDAO;
import com.commafeed.backend.dao.UserSettingsDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.internal.PostLoginActivities;
import com.google.common.base.Preconditions;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class UserService {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UserDAO userDAO;
private final UserRoleDAO userRoleDAO;
private final UserSettingsDAO userSettingsDAO;
private final PasswordEncryptionService encryptionService;
private final CommaFeedConfiguration config;
private final PostLoginActivities postLoginActivities;
/**
* try to log in with given credentials
*/
public Optional<User> login(String nameOrEmail, String password) {
if (nameOrEmail == null || password == null) {
return Optional.empty();
}
User user = userDAO.findByName(nameOrEmail);
if (user == null) {
user = userDAO.findByEmail(nameOrEmail);
}
if (user != null && !user.isDisabled()) {
boolean authenticated = encryptionService.authenticate(password, user.getPassword(), user.getSalt());
if (authenticated) {
performPostLoginActivities(user);
return Optional.of(user);
}
}
return Optional.empty();
}
/**
* try to log in with given api key
*/
public Optional<User> login(String apiKey) {
if (apiKey == null) {
return Optional.empty();
}
User user = userDAO.findByApiKey(apiKey);
if (user != null && !user.isDisabled()) {
performPostLoginActivities(user);
return Optional.of(user);
}
return Optional.empty();
}
/**
* try to log in with given fever api key
*/
public Optional<User> login(long userId, String feverApiKey) {
if (feverApiKey == null) {
return Optional.empty();
}
User user = userDAO.findById(userId);
if (user == null || user.isDisabled() || user.getApiKey() == null) {
return Optional.empty();
}
String computedFeverApiKey = Digests.md5Hex(user.getName() + ":" + user.getApiKey());
if (!computedFeverApiKey.equals(feverApiKey)) {
return Optional.empty();
}
performPostLoginActivities(user);
return Optional.of(user);
}
/**
* should triggers after successful login
*/
public void performPostLoginActivities(User user) {
postLoginActivities.executeFor(user);
}
public User register(String name, String password, String email, Collection<Role> roles) {
return register(name, password, email, roles, false);
}
public User register(String name, String password, String email, Collection<Role> roles, boolean forceRegistration) {
if (!forceRegistration) {
Preconditions.checkState(config.users().allowRegistrations(), "Registrations are closed on this CommaFeed instance");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");
if (StringUtils.isNotBlank(email)) {
Preconditions.checkArgument(userDAO.findByEmail(email) == null, "Email already taken");
}
User user = new User();
byte[] salt = encryptionService.generateSalt();
user.setName(name);
user.setEmail(email);
user.setCreated(Instant.now());
user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.saveOrUpdate(user);
for (Role role : roles) {
userRoleDAO.saveOrUpdate(new UserRole(user, role));
}
return user;
}
public void createAdminUser() {
register(CommaFeedApplication.USERNAME_ADMIN, "admin", "admin@commafeed.com", Arrays.asList(Role.ADMIN, Role.USER), true);
}
public void createDemoUser() {
register(CommaFeedApplication.USERNAME_DEMO, "demo", "demo@commafeed.com", Collections.singletonList(Role.USER), true);
}
public void unregister(User user) {
userSettingsDAO.delete(userSettingsDAO.findByUser(user));
userRoleDAO.delete(userRoleDAO.findAll(user));
feedSubscriptionDAO.delete(feedSubscriptionDAO.findAll(user));
feedCategoryDAO.delete(feedCategoryDAO.findAll(user));
userDAO.delete(user);
}
public String generateApiKey(User user) {
byte[] key = encryptionService.getEncryptedPassword(UUID.randomUUID().toString(), user.getSalt());
return Digests.sha1Hex(key);
}
public Set<Role> getRoles(User user) {
return userRoleDAO.findRoles(user);
}
}
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.dao.UserRoleDAO;
import com.commafeed.backend.dao.UserSettingsDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.internal.PostLoginActivities;
import com.google.common.base.Preconditions;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class UserService {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UserDAO userDAO;
private final UserRoleDAO userRoleDAO;
private final UserSettingsDAO userSettingsDAO;
private final PasswordEncryptionService encryptionService;
private final CommaFeedConfiguration config;
private final PostLoginActivities postLoginActivities;
/**
* try to log in with given credentials
*/
public Optional<User> login(String nameOrEmail, String password) {
if (nameOrEmail == null || password == null) {
return Optional.empty();
}
User user = userDAO.findByName(nameOrEmail);
if (user == null) {
user = userDAO.findByEmail(nameOrEmail);
}
if (user != null && !user.isDisabled()) {
boolean authenticated = encryptionService.authenticate(password, user.getPassword(), user.getSalt());
if (authenticated) {
performPostLoginActivities(user);
return Optional.of(user);
}
}
return Optional.empty();
}
/**
* try to log in with given api key
*/
public Optional<User> login(String apiKey) {
if (apiKey == null) {
return Optional.empty();
}
User user = userDAO.findByApiKey(apiKey);
if (user != null && !user.isDisabled()) {
performPostLoginActivities(user);
return Optional.of(user);
}
return Optional.empty();
}
/**
* try to log in with given fever api key
*/
public Optional<User> login(long userId, String feverApiKey) {
if (feverApiKey == null) {
return Optional.empty();
}
User user = userDAO.findById(userId);
if (user == null || user.isDisabled() || user.getApiKey() == null) {
return Optional.empty();
}
String computedFeverApiKey = Digests.md5Hex(user.getName() + ":" + user.getApiKey());
if (!computedFeverApiKey.equals(feverApiKey)) {
return Optional.empty();
}
performPostLoginActivities(user);
return Optional.of(user);
}
/**
* should triggers after successful login
*/
public void performPostLoginActivities(User user) {
postLoginActivities.executeFor(user);
}
public User register(String name, String password, String email, Collection<Role> roles) {
return register(name, password, email, roles, false);
}
public User register(String name, String password, String email, Collection<Role> roles, boolean forceRegistration) {
if (!forceRegistration) {
Preconditions.checkState(config.users().allowRegistrations(), "Registrations are closed on this CommaFeed instance");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");
if (StringUtils.isNotBlank(email)) {
Preconditions.checkArgument(userDAO.findByEmail(email) == null, "Email already taken");
}
User user = new User();
byte[] salt = encryptionService.generateSalt();
user.setName(name);
user.setEmail(email);
user.setCreated(Instant.now());
user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.saveOrUpdate(user);
for (Role role : roles) {
userRoleDAO.saveOrUpdate(new UserRole(user, role));
}
return user;
}
public void createAdminUser() {
register(CommaFeedApplication.USERNAME_ADMIN, "admin", "admin@commafeed.com", Arrays.asList(Role.ADMIN, Role.USER), true);
}
public void createDemoUser() {
register(CommaFeedApplication.USERNAME_DEMO, "demo", "demo@commafeed.com", Collections.singletonList(Role.USER), true);
}
public void unregister(User user) {
userSettingsDAO.delete(userSettingsDAO.findByUser(user));
userRoleDAO.delete(userRoleDAO.findAll(user));
feedSubscriptionDAO.delete(feedSubscriptionDAO.findAll(user));
feedCategoryDAO.delete(feedCategoryDAO.findAll(user));
userDAO.delete(user);
}
public String generateApiKey(User user) {
byte[] key = encryptionService.getEncryptedPassword(UUID.randomUUID().toString(), user.getSalt());
return Digests.sha1Hex(key);
}
public Set<Role> getRoles(User user) {
return userRoleDAO.findRoles(user);
}
}

266
commafeed-server/src/main/java/com/commafeed/backend/service/db/DatabaseCleaningService.java
View File

@@ -1,133 +1,133 @@
package com.commafeed.backend.service.db;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryDAO.FeedCapacity;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
/**
* Contains utility methods for cleaning the database
*
*/
@Slf4j
@Singleton
public class DatabaseCleaningService {
private final int batchSize;
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final Meter entriesDeletedMeter;
public DatabaseCleaningService(CommaFeedConfiguration config, UnitOfWork unitOfWork, FeedDAO feedDAO, FeedEntryDAO feedEntryDAO,
FeedEntryContentDAO feedEntryContentDAO, FeedEntryStatusDAO feedEntryStatusDAO, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.feedEntryDAO = feedEntryDAO;
this.feedEntryContentDAO = feedEntryContentDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.batchSize = config.database().cleanup().batchSize();
this.entriesDeletedMeter = metrics.meter(MetricRegistry.name(getClass(), "entriesDeleted"));
}
public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
int deleted;
long entriesTotal = 0;
do {
List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) {
long entriesDeleted;
do {
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesDeletedMeter.mark(entriesDeleted);
entriesTotal += entriesDeleted;
log.debug("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
}
deleted = unitOfWork.call(() -> feedDAO.delete(feedDAO.findByIds(feeds.stream().map(AbstractModel::getId).toList())));
total += deleted;
log.debug("removed {} feeds without subscriptions", total);
} while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total);
}
public void cleanContentsWithoutEntries() {
log.info("cleaning contents without entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize));
total += deleted;
log.debug("removed {} contents without entries", total);
} while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total);
}
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
log.info("cleaning entries exceeding feed capacity");
long total = 0;
while (true) {
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize));
if (feeds.isEmpty()) {
break;
}
for (final FeedCapacity feed : feeds) {
long remaining = feed.getCapacity() - maxFeedCapacity;
do {
final long rem = remaining;
int deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(batchSize, rem)));
entriesDeletedMeter.mark(deleted);
total += deleted;
remaining -= deleted;
log.debug("removed {} entries for feeds exceeding capacity", total);
} while (remaining > 0);
}
}
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
}
public void cleanEntriesOlderThan(final Instant olderThan) {
log.info("cleaning old entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize));
entriesDeletedMeter.mark(deleted);
total += deleted;
log.debug("removed {} old entries", total);
} while (deleted != 0);
log.info("cleanup done: {} old entries deleted", total);
}
public void cleanStatusesOlderThan(final Instant olderThan) {
log.info("cleaning old read statuses");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
total += deleted;
log.debug("removed {} old read statuses", total);
} while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
}
}
package com.commafeed.backend.service.db;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryDAO.FeedCapacity;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
/**
* Contains utility methods for cleaning the database
*
*/
@Slf4j
@Singleton
public class DatabaseCleaningService {
private final int batchSize;
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final Meter entriesDeletedMeter;
public DatabaseCleaningService(CommaFeedConfiguration config, UnitOfWork unitOfWork, FeedDAO feedDAO, FeedEntryDAO feedEntryDAO,
FeedEntryContentDAO feedEntryContentDAO, FeedEntryStatusDAO feedEntryStatusDAO, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.feedEntryDAO = feedEntryDAO;
this.feedEntryContentDAO = feedEntryContentDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.batchSize = config.database().cleanup().batchSize();
this.entriesDeletedMeter = metrics.meter(MetricRegistry.name(getClass(), "entriesDeleted"));
}
public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
int deleted;
long entriesTotal = 0;
do {
List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) {
long entriesDeleted;
do {
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesDeletedMeter.mark(entriesDeleted);
entriesTotal += entriesDeleted;
log.debug("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
}
deleted = unitOfWork.call(() -> feedDAO.delete(feedDAO.findByIds(feeds.stream().map(AbstractModel::getId).toList())));
total += deleted;
log.debug("removed {} feeds without subscriptions", total);
} while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total);
}
public void cleanContentsWithoutEntries() {
log.info("cleaning contents without entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize));
total += deleted;
log.debug("removed {} contents without entries", total);
} while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total);
}
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
log.info("cleaning entries exceeding feed capacity");
long total = 0;
while (true) {
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize));
if (feeds.isEmpty()) {
break;
}
for (final FeedCapacity feed : feeds) {
long remaining = feed.getCapacity() - maxFeedCapacity;
do {
final long rem = remaining;
int deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(batchSize, rem)));
entriesDeletedMeter.mark(deleted);
total += deleted;
remaining -= deleted;
log.debug("removed {} entries for feeds exceeding capacity", total);
} while (remaining > 0);
}
}
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
}
public void cleanEntriesOlderThan(final Instant olderThan) {
log.info("cleaning old entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize));
entriesDeletedMeter.mark(deleted);
total += deleted;
log.debug("removed {} old entries", total);
} while (deleted != 0);
log.info("cleanup done: {} old entries deleted", total);
}
public void cleanStatusesOlderThan(final Instant olderThan) {
log.info("cleaning old read statuses");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
total += deleted;
log.debug("removed {} old read statuses", total);
} while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
}
}

126
commafeed-server/src/main/java/com/commafeed/backend/service/db/DatabaseStartupService.java
View File

@@ -1,63 +1,63 @@
package com.commafeed.backend.service.db;
import jakarta.inject.Singleton;
import org.kohsuke.MetaInfServices;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.service.UserService;
import liquibase.database.Database;
import liquibase.database.core.PostgresDatabase;
import liquibase.structure.DatabaseObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class DatabaseStartupService {
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserService userService;
private final CommaFeedConfiguration config;
public void populateInitialData() {
long count = unitOfWork.call(userDAO::count);
if (count == 0) {
unitOfWork.run(this::initialData);
}
}
private void initialData() {
log.info("populating database with default values");
try {
userService.createAdminUser();
if (config.users().createDemoAccount()) {
userService.createDemoUser();
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
/**
* Register a postgresql database in liquibase that doesn't escape columns, so that we can use lower case columns
*/
@MetaInfServices(Database.class)
public static class LowerCaseColumnsPostgresDatabase extends PostgresDatabase {
@Override
public String escapeObjectName(String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}
@Override
public int getPriority() {
return super.getPriority() + 1;
}
}
}
package com.commafeed.backend.service.db;
import jakarta.inject.Singleton;
import org.kohsuke.MetaInfServices;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.service.UserService;
import liquibase.database.Database;
import liquibase.database.core.PostgresDatabase;
import liquibase.structure.DatabaseObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class DatabaseStartupService {
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserService userService;
private final CommaFeedConfiguration config;
public void populateInitialData() {
long count = unitOfWork.call(userDAO::count);
if (count == 0) {
unitOfWork.run(this::initialData);
}
}
private void initialData() {
log.info("populating database with default values");
try {
userService.createAdminUser();
if (config.users().createDemoAccount()) {
userService.createDemoUser();
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
/**
* Register a postgresql database in liquibase that doesn't escape columns, so that we can use lower case columns
*/
@MetaInfServices(Database.class)
public static class LowerCaseColumnsPostgresDatabase extends PostgresDatabase {
@Override
public String escapeObjectName(String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}
@Override
public int getPriority() {
return super.getPriority() + 1;
}
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/service/internal/PostLoginActivities.java
View File

@@ -1,30 +1,30 @@
package com.commafeed.backend.service.internal;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class PostLoginActivities {
private final UserDAO userDAO;
private final UnitOfWork unitOfWork;
public void executeFor(User user) {
// only update lastLogin every once in a while in order to avoid invalidating the cache every time someone logs in
Instant now = Instant.now();
Instant lastLogin = user.getLastLogin();
if (lastLogin == null || ChronoUnit.MINUTES.between(lastLogin, now) >= 30) {
user.setLastLogin(now);
unitOfWork.run(() -> userDAO.saveOrUpdate(user));
}
}
}
package com.commafeed.backend.service.internal;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class PostLoginActivities {
private final UserDAO userDAO;
private final UnitOfWork unitOfWork;
public void executeFor(User user) {
// only update lastLogin every once in a while in order to avoid invalidating the cache every time someone logs in
Instant now = Instant.now();
Instant lastLogin = user.getLastLogin();
if (lastLogin == null || ChronoUnit.MINUTES.between(lastLogin, now) >= 30) {
user.setLastLogin(now);
unitOfWork.run(() -> userDAO.saveOrUpdate(user));
}
}
}

122
commafeed-server/src/main/java/com/commafeed/backend/task/DemoAccountCleanupTask.java
View File

@@ -1,61 +1,61 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.UserService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Singleton
@Slf4j
public class DemoAccountCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserService userService;
@Override
protected void run() {
if (!config.users().createDemoAccount()) {
return;
}
log.info("recreating demo user account");
unitOfWork.run(() -> {
User demoUser = userDAO.findByName(CommaFeedApplication.USERNAME_DEMO);
if (demoUser == null) {
return;
}
userService.unregister(demoUser);
userService.createDemoUser();
});
}
@Override
protected long getInitialDelay() {
return 1;
}
@Override
protected long getPeriod() {
return getTimeUnit().convert(24, TimeUnit.HOURS);
}
@Override
protected TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.UserService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Singleton
@Slf4j
public class DemoAccountCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserService userService;
@Override
protected void run() {
if (!config.users().createDemoAccount()) {
return;
}
log.info("recreating demo user account");
unitOfWork.run(() -> {
User demoUser = userDAO.findByName(CommaFeedApplication.USERNAME_DEMO);
if (demoUser == null) {
return;
}
userService.unregister(demoUser);
userService.createDemoUser();
});
}
@Override
protected long getInitialDelay() {
return 1;
}
@Override
protected long getPeriod() {
return getTimeUnit().convert(24, TimeUnit.HOURS);
}
@Override
protected TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

84
commafeed-server/src/main/java/com/commafeed/backend/task/EntriesExceedingFeedCapacityCleanupTask.java
View File

@@ -1,42 +1,42 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
}
}
@Override
public long getInitialDelay() {
return 10;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
}
}
@Override
public long getInitialDelay() {
return 10;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

90
commafeed-server/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java
View File

@@ -1,45 +1,45 @@
package com.commafeed.backend.task;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OldEntriesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
cleaner.cleanEntriesOlderThan(threshold);
}
}
@Override
public long getInitialDelay() {
return 5;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}
package com.commafeed.backend.task;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OldEntriesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
cleaner.cleanEntriesOlderThan(threshold);
}
}
@Override
public long getInitialDelay() {
return 5;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

86
commafeed-server/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java
View File

@@ -1,43 +1,43 @@
package com.commafeed.backend.task;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OldStatusesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
Instant threshold = config.database().cleanup().statusesInstantThreshold();
if (threshold != null) {
cleaner.cleanStatusesOlderThan(threshold);
}
}
@Override
public long getInitialDelay() {
return 15;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}
package com.commafeed.backend.task;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OldStatusesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
Instant threshold = config.database().cleanup().statusesInstantThreshold();
if (threshold != null) {
cleaner.cleanStatusesOlderThan(threshold);
}
}
@Override
public long getInitialDelay() {
return 15;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

74
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedContentsCleanupTask.java
View File

@@ -1,37 +1,37 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OrphanedContentsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanContentsWithoutEntries();
}
@Override
public long getInitialDelay() {
return 25;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OrphanedContentsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanContentsWithoutEntries();
}
@Override
public long getInitialDelay() {
return 25;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

74
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedFeedsCleanupTask.java
View File

@@ -1,37 +1,37 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OrphanedFeedsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanFeedsWithoutSubscriptions();
}
@Override
public long getInitialDelay() {
return 20;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OrphanedFeedsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanFeedsWithoutSubscriptions();
}
@Override
public long getInitialDelay() {
return 20;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/task/ScheduledTask.java
View File

@@ -1,30 +1,30 @@
package com.commafeed.backend.task;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class ScheduledTask {
protected abstract void run();
protected abstract long getInitialDelay();
protected abstract long getPeriod();
protected abstract TimeUnit getTimeUnit();
public void register(ScheduledExecutorService executor) {
Runnable runnable = () -> {
try {
ScheduledTask.this.run();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
};
log.info("registering task {} for execution every {} {}, starting in {} {}", getClass().getSimpleName(), getPeriod(), getTimeUnit(),
getInitialDelay(), getTimeUnit());
executor.scheduleWithFixedDelay(runnable, getInitialDelay(), getPeriod(), getTimeUnit());
}
}
package com.commafeed.backend.task;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class ScheduledTask {
protected abstract void run();
protected abstract long getInitialDelay();
protected abstract long getPeriod();
protected abstract TimeUnit getTimeUnit();
public void register(ScheduledExecutorService executor) {
Runnable runnable = () -> {
try {
ScheduledTask.this.run();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
};
log.info("registering task {} for execution every {} {}, starting in {} {}", getClass().getSimpleName(), getPeriod(), getTimeUnit(),
getInitialDelay(), getTimeUnit());
executor.scheduleWithFixedDelay(runnable, getInitialDelay(), getPeriod(), getTimeUnit());
}
}

58
commafeed-server/src/main/java/com/commafeed/backend/task/TaskScheduler.java
View File

@@ -1,29 +1,29 @@
package com.commafeed.backend.task;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import jakarta.inject.Singleton;
import io.quarkus.arc.All;
@Singleton
public class TaskScheduler {
private final List<ScheduledTask> tasks;
private final ScheduledExecutorService executor;
public TaskScheduler(@All List<ScheduledTask> tasks) {
this.tasks = tasks;
this.executor = Executors.newScheduledThreadPool(tasks.size());
}
public void start() {
tasks.forEach(task -> task.register(executor));
}
public void stop() {
executor.shutdownNow();
}
}
package com.commafeed.backend.task;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import jakarta.inject.Singleton;
import io.quarkus.arc.All;
@Singleton
public class TaskScheduler {
private final List<ScheduledTask> tasks;
private final ScheduledExecutorService executor;
public TaskScheduler(@All List<ScheduledTask> tasks) {
this.tasks = tasks;
this.executor = Executors.newScheduledThreadPool(tasks.size());
}
public void start() {
tasks.forEach(task -> task.register(executor));
}
public void stop() {
executor.shutdownNow();
}
}

20
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/FeedURLProvider.java
View File

@@ -1,10 +1,10 @@
package com.commafeed.backend.urlprovider;
/**
* Tries to find a feed url given the url and page content
*/
public interface FeedURLProvider {
String get(String url, String urlContent);
}
package com.commafeed.backend.urlprovider;
/**
* Tries to find a feed url given the url and page content
*/
public interface FeedURLProvider {
String get(String url, String urlContent);
}

62
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/InPageReferenceFeedURLProvider.java
View File

@@ -1,31 +1,31 @@
package com.commafeed.backend.urlprovider;
import jakarta.inject.Singleton;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
@Singleton
public class InPageReferenceFeedURLProvider implements FeedURLProvider {
@Override
public String get(String url, String urlContent) {
String foundUrl = null;
Document doc = Jsoup.parse(urlContent, url);
String root = doc.children().get(0).tagName();
if ("html".equals(root)) {
Elements atom = doc.select("link[type=application/atom+xml]");
Elements rss = doc.select("link[type=application/rss+xml]");
if (!atom.isEmpty()) {
foundUrl = atom.get(0).attr("abs:href");
} else if (!rss.isEmpty()) {
foundUrl = rss.get(0).attr("abs:href");
}
}
return foundUrl;
}
}
package com.commafeed.backend.urlprovider;
import jakarta.inject.Singleton;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
@Singleton
public class InPageReferenceFeedURLProvider implements FeedURLProvider {
@Override
public String get(String url, String urlContent) {
String foundUrl = null;
Document doc = Jsoup.parse(urlContent, url);
String root = doc.children().get(0).tagName();
if ("html".equals(root)) {
Elements atom = doc.select("link[type=application/atom+xml]");
Elements rss = doc.select("link[type=application/rss+xml]");
if (!atom.isEmpty()) {
foundUrl = atom.get(0).attr("abs:href");
} else if (!rss.isEmpty()) {
foundUrl = rss.get(0).attr("abs:href");
}
}
return foundUrl;
}
}

56
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/YoutubeFeedURLProvider.java
View File

@@ -1,28 +1,28 @@
package com.commafeed.backend.urlprovider;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
/**
* Workaround for Youtube channels
*
* converts the channel URL https://www.youtube.com/channel/CHANNEL_ID to the valid feed URL
* https://www.youtube.com/feeds/videos.xml?channel_id=CHANNEL_ID
*/
@Singleton
public class YoutubeFeedURLProvider implements FeedURLProvider {
private static final String PREFIX = "https://www.youtube.com/channel/";
private static final String REPLACEMENT_PREFIX = "https://www.youtube.com/feeds/videos.xml?channel_id=";
@Override
public String get(String url, String urlContent) {
if (!StringUtils.startsWithIgnoreCase(url, PREFIX)) {
return null;
}
return REPLACEMENT_PREFIX + url.substring(PREFIX.length());
}
}
package com.commafeed.backend.urlprovider;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
/**
* Workaround for Youtube channels
*
* converts the channel URL https://www.youtube.com/channel/CHANNEL_ID to the valid feed URL
* https://www.youtube.com/feeds/videos.xml?channel_id=CHANNEL_ID
*/
@Singleton
public class YoutubeFeedURLProvider implements FeedURLProvider {
private static final String PREFIX = "https://www.youtube.com/channel/";
private static final String REPLACEMENT_PREFIX = "https://www.youtube.com/feeds/videos.xml?channel_id=";
@Override
public String get(String url, String urlContent) {
if (!StringUtils.startsWithIgnoreCase(url, PREFIX)) {
return null;
}
return REPLACEMENT_PREFIX + url.substring(PREFIX.length());
}
}

80
commafeed-server/src/main/java/com/commafeed/frontend/model/Category.java
View File

@@ -1,41 +1,41 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Entry details")
@Data
@RegisterForReflection
public class Category implements Serializable {
@Schema(description = "category id", requiredMode = RequiredMode.REQUIRED)
private String id;
@Schema(description = "parent category id")
private String parentId;
@Schema(description = "parent category name")
private String parentName;
@Schema(description = "category id", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "category children categories", requiredMode = RequiredMode.REQUIRED)
private List<Category> children = new ArrayList<>();
@Schema(description = "category feeds", requiredMode = RequiredMode.REQUIRED)
private List<Subscription> feeds = new ArrayList<>();
@Schema(description = "whether the category is expanded or collapsed", requiredMode = RequiredMode.REQUIRED)
private boolean expanded;
@Schema(description = "position of the category in the list", requiredMode = RequiredMode.REQUIRED)
private int position;
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Entry details")
@Data
@RegisterForReflection
public class Category implements Serializable {
@Schema(description = "category id", requiredMode = RequiredMode.REQUIRED)
private String id;
@Schema(description = "parent category id")
private String parentId;
@Schema(description = "parent category name")
private String parentName;
@Schema(description = "category id", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "category children categories", requiredMode = RequiredMode.REQUIRED)
private List<Category> children = new ArrayList<>();
@Schema(description = "category feeds", requiredMode = RequiredMode.REQUIRED)
private List<Subscription> feeds = new ArrayList<>();
@Schema(description = "whether the category is expanded or collapsed", requiredMode = RequiredMode.REQUIRED)
private boolean expanded;
@Schema(description = "position of the category in the list", requiredMode = RequiredMode.REQUIRED)
private int position;
}

100
commafeed-server/src/main/java/com/commafeed/frontend/model/Entries.java
View File

@@ -1,50 +1,50 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "List of entries with some metadata")
@Data
@RegisterForReflection
public class Entries implements Serializable {
@Schema(description = "name of the feed or the category requested", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "error or warning message")
private String message;
@Schema(description = "times the server tried to refresh the feed and failed", requiredMode = RequiredMode.REQUIRED)
private int errorCount;
@Schema(description = "URL of the website, extracted from the feed, only filled if querying for feed entries, not category entries")
private String feedLink;
@Schema(description = "list generation timestamp", requiredMode = RequiredMode.REQUIRED)
private long timestamp;
@Schema(description = "if the query has more elements", requiredMode = RequiredMode.REQUIRED)
private boolean hasMore;
@Schema(description = "the requested offset")
private int offset;
@Schema(description = "the requested limit")
private int limit;
@Schema(description = "list of entries", requiredMode = RequiredMode.REQUIRED)
private List<Entry> entries = new ArrayList<>();
@Schema(
description = "if true, the unread flag was ignored in the request, all entries are returned regardless of their read status",
requiredMode = RequiredMode.REQUIRED)
private boolean ignoredReadStatus;
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "List of entries with some metadata")
@Data
@RegisterForReflection
public class Entries implements Serializable {
@Schema(description = "name of the feed or the category requested", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "error or warning message")
private String message;
@Schema(description = "times the server tried to refresh the feed and failed", requiredMode = RequiredMode.REQUIRED)
private int errorCount;
@Schema(description = "URL of the website, extracted from the feed, only filled if querying for feed entries, not category entries")
private String feedLink;
@Schema(description = "list generation timestamp", requiredMode = RequiredMode.REQUIRED)
private long timestamp;
@Schema(description = "if the query has more elements", requiredMode = RequiredMode.REQUIRED)
private boolean hasMore;
@Schema(description = "the requested offset")
private int offset;
@Schema(description = "the requested limit")
private int limit;
@Schema(description = "list of entries", requiredMode = RequiredMode.REQUIRED)
private List<Entry> entries = new ArrayList<>();
@Schema(
description = "if true, the unread flag was ignored in the request, all entries are returned regardless of their read status",
requiredMode = RequiredMode.REQUIRED)
private boolean ignoredReadStatus;
}

348
commafeed-server/src/main/java/com/commafeed/frontend/model/Entry.java
View File

@@ -1,174 +1,174 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEnclosureImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndEntryImpl;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Entry details")
@Data
@RegisterForReflection
public class Entry implements Serializable {
@Schema(description = "entry id", requiredMode = RequiredMode.REQUIRED)
private String id;
@Schema(description = "entry guid", requiredMode = RequiredMode.REQUIRED)
private String guid;
@Schema(description = "entry title", requiredMode = RequiredMode.REQUIRED)
private String title;
@Schema(description = "entry content", requiredMode = RequiredMode.REQUIRED)
private String content;
@Schema(description = "comma-separated list of categories")
private String categories;
@Schema(description = "whether entry content and title are rtl", requiredMode = RequiredMode.REQUIRED)
private boolean rtl;
@Schema(description = "entry author")
private String author;
@Schema(description = "entry enclosure url, if any")
private String enclosureUrl;
@Schema(description = "entry enclosure mime type, if any")
private String enclosureType;
@Schema(description = "entry media description, if any")
private String mediaDescription;
@Schema(description = "entry media thumbnail url, if any")
private String mediaThumbnailUrl;
@Schema(description = "entry media thumbnail width, if any")
private Integer mediaThumbnailWidth;
@Schema(description = "entry media thumbnail height, if any")
private Integer mediaThumbnailHeight;
@Schema(description = "entry publication date", type = "number", requiredMode = RequiredMode.REQUIRED)
private Instant date;
@Schema(description = "entry insertion date in the database", type = "number", requiredMode = RequiredMode.REQUIRED)
private Instant insertedDate;
@Schema(description = "feed id", requiredMode = RequiredMode.REQUIRED)
private String feedId;
@Schema(description = "feed name", requiredMode = RequiredMode.REQUIRED)
private String feedName;
@Schema(description = "this entry's feed url", requiredMode = RequiredMode.REQUIRED)
private String feedUrl;
@Schema(description = "this entry's website url", requiredMode = RequiredMode.REQUIRED)
private String feedLink;
@Schema(description = "The favicon url to use for this feed", requiredMode = RequiredMode.REQUIRED)
private String iconUrl;
@Schema(description = "entry url", requiredMode = RequiredMode.REQUIRED)
private String url;
@Schema(description = "read status", requiredMode = RequiredMode.REQUIRED)
private boolean read;
@Schema(description = "starred status", requiredMode = RequiredMode.REQUIRED)
private boolean starred;
@Schema(description = "whether the entry is still markable (old entry statuses are discarded)", requiredMode = RequiredMode.REQUIRED)
private boolean markable;
@Schema(description = "tags", requiredMode = RequiredMode.REQUIRED)
private List<String> tags;
public static Entry build(FeedEntryStatus status, boolean proxyImages) {
Entry entry = new Entry();
FeedEntry feedEntry = status.getEntry();
FeedSubscription sub = status.getSubscription();
FeedEntryContent content = feedEntry.getContent();
entry.setId(String.valueOf(feedEntry.getId()));
entry.setGuid(feedEntry.getGuid());
entry.setRead(status.isRead());
entry.setStarred(status.isStarred());
entry.setMarkable(status.isMarkable());
entry.setDate(feedEntry.getPublished());
entry.setInsertedDate(feedEntry.getInserted());
entry.setUrl(feedEntry.getUrl());
entry.setFeedName(sub.getTitle());
entry.setFeedId(String.valueOf(sub.getId()));
entry.setFeedUrl(sub.getFeed().getUrl());
entry.setFeedLink(sub.getFeed().getLink());
entry.setIconUrl(FeedUtils.getFaviconUrl(sub));
entry.setTags(status.getTags().stream().map(FeedEntryTag::getName).toList());
if (content != null) {
entry.setRtl(content.isRTL());
entry.setTitle(content.getTitle());
entry.setContent(proxyImages ? FeedUtils.proxyImages(content.getContent()) : content.getContent());
entry.setAuthor(content.getAuthor());
entry.setEnclosureType(content.getEnclosureType());
entry.setEnclosureUrl(proxyImages && StringUtils.contains(content.getEnclosureType(), "image")
? FeedUtils.proxyImage(content.getEnclosureUrl())
: content.getEnclosureUrl());
entry.setMediaDescription(content.getMediaDescription());
entry.setMediaThumbnailUrl(proxyImages ? FeedUtils.proxyImage(content.getMediaThumbnailUrl()) : content.getMediaThumbnailUrl());
entry.setMediaThumbnailWidth(content.getMediaThumbnailWidth());
entry.setMediaThumbnailHeight(content.getMediaThumbnailHeight());
entry.setCategories(content.getCategories());
}
return entry;
}
public SyndEntry asRss() {
SyndEntry entry = new SyndEntryImpl();
entry.setUri(getGuid());
entry.setTitle(getTitle());
entry.setAuthor(getAuthor());
SyndContentImpl content = new SyndContentImpl();
content.setValue(getContent());
entry.setContents(Collections.singletonList(content));
if (getEnclosureUrl() != null) {
SyndEnclosureImpl enclosure = new SyndEnclosureImpl();
enclosure.setType(getEnclosureType());
enclosure.setUrl(getEnclosureUrl());
entry.setEnclosures(Collections.singletonList(enclosure));
}
entry.setLink(getUrl());
entry.setPublishedDate(getDate() == null ? null : Date.from(getDate()));
return entry;
}
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEnclosureImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndEntryImpl;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Entry details")
@Data
@RegisterForReflection
public class Entry implements Serializable {
@Schema(description = "entry id", requiredMode = RequiredMode.REQUIRED)
private String id;
@Schema(description = "entry guid", requiredMode = RequiredMode.REQUIRED)
private String guid;
@Schema(description = "entry title", requiredMode = RequiredMode.REQUIRED)
private String title;
@Schema(description = "entry content", requiredMode = RequiredMode.REQUIRED)
private String content;
@Schema(description = "comma-separated list of categories")
private String categories;
@Schema(description = "whether entry content and title are rtl", requiredMode = RequiredMode.REQUIRED)
private boolean rtl;
@Schema(description = "entry author")
private String author;
@Schema(description = "entry enclosure url, if any")
private String enclosureUrl;
@Schema(description = "entry enclosure mime type, if any")
private String enclosureType;
@Schema(description = "entry media description, if any")
private String mediaDescription;
@Schema(description = "entry media thumbnail url, if any")
private String mediaThumbnailUrl;
@Schema(description = "entry media thumbnail width, if any")
private Integer mediaThumbnailWidth;
@Schema(description = "entry media thumbnail height, if any")
private Integer mediaThumbnailHeight;
@Schema(description = "entry publication date", type = "number", requiredMode = RequiredMode.REQUIRED)
private Instant date;
@Schema(description = "entry insertion date in the database", type = "number", requiredMode = RequiredMode.REQUIRED)
private Instant insertedDate;
@Schema(description = "feed id", requiredMode = RequiredMode.REQUIRED)
private String feedId;
@Schema(description = "feed name", requiredMode = RequiredMode.REQUIRED)
private String feedName;
@Schema(description = "this entry's feed url", requiredMode = RequiredMode.REQUIRED)
private String feedUrl;
@Schema(description = "this entry's website url", requiredMode = RequiredMode.REQUIRED)
private String feedLink;
@Schema(description = "The favicon url to use for this feed", requiredMode = RequiredMode.REQUIRED)
private String iconUrl;
@Schema(description = "entry url", requiredMode = RequiredMode.REQUIRED)
private String url;
@Schema(description = "read status", requiredMode = RequiredMode.REQUIRED)
private boolean read;
@Schema(description = "starred status", requiredMode = RequiredMode.REQUIRED)
private boolean starred;
@Schema(description = "whether the entry is still markable (old entry statuses are discarded)", requiredMode = RequiredMode.REQUIRED)
private boolean markable;
@Schema(description = "tags", requiredMode = RequiredMode.REQUIRED)
private List<String> tags;
public static Entry build(FeedEntryStatus status, boolean proxyImages) {
Entry entry = new Entry();
FeedEntry feedEntry = status.getEntry();
FeedSubscription sub = status.getSubscription();
FeedEntryContent content = feedEntry.getContent();
entry.setId(String.valueOf(feedEntry.getId()));
entry.setGuid(feedEntry.getGuid());
entry.setRead(status.isRead());
entry.setStarred(status.isStarred());
entry.setMarkable(status.isMarkable());
entry.setDate(feedEntry.getPublished());
entry.setInsertedDate(feedEntry.getInserted());
entry.setUrl(feedEntry.getUrl());
entry.setFeedName(sub.getTitle());
entry.setFeedId(String.valueOf(sub.getId()));
entry.setFeedUrl(sub.getFeed().getUrl());
entry.setFeedLink(sub.getFeed().getLink());
entry.setIconUrl(FeedUtils.getFaviconUrl(sub));
entry.setTags(status.getTags().stream().map(FeedEntryTag::getName).toList());
if (content != null) {
entry.setRtl(content.isRTL());
entry.setTitle(content.getTitle());
entry.setContent(proxyImages ? FeedUtils.proxyImages(content.getContent()) : content.getContent());
entry.setAuthor(content.getAuthor());
entry.setEnclosureType(content.getEnclosureType());
entry.setEnclosureUrl(proxyImages && StringUtils.contains(content.getEnclosureType(), "image")
? FeedUtils.proxyImage(content.getEnclosureUrl())
: content.getEnclosureUrl());
entry.setMediaDescription(content.getMediaDescription());
entry.setMediaThumbnailUrl(proxyImages ? FeedUtils.proxyImage(content.getMediaThumbnailUrl()) : content.getMediaThumbnailUrl());
entry.setMediaThumbnailWidth(content.getMediaThumbnailWidth());
entry.setMediaThumbnailHeight(content.getMediaThumbnailHeight());
entry.setCategories(content.getCategories());
}
return entry;
}
public SyndEntry asRss() {
SyndEntry entry = new SyndEntryImpl();
entry.setUri(getGuid());
entry.setTitle(getTitle());
entry.setAuthor(getAuthor());
SyndContentImpl content = new SyndContentImpl();
content.setValue(getContent());
entry.setContents(Collections.singletonList(content));
if (getEnclosureUrl() != null) {
SyndEnclosureImpl enclosure = new SyndEnclosureImpl();
enclosure.setType(getEnclosureType());
enclosure.setUrl(getEnclosureUrl());
entry.setEnclosures(Collections.singletonList(enclosure));
}
entry.setLink(getUrl());
entry.setPublishedDate(getDate() == null ? null : Date.from(getDate()));
return entry;
}
}

44
commafeed-server/src/main/java/com/commafeed/frontend/model/FeedInfo.java
View File

@@ -1,22 +1,22 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Feed details")
@Data
@RegisterForReflection
public class FeedInfo implements Serializable {
@Schema(description = "url", requiredMode = RequiredMode.REQUIRED)
private String url;
@Schema(description = "title", requiredMode = RequiredMode.REQUIRED)
private String title;
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Feed details")
@Data
@RegisterForReflection
public class FeedInfo implements Serializable {
@Schema(description = "url", requiredMode = RequiredMode.REQUIRED)
private String url;
@Schema(description = "title", requiredMode = RequiredMode.REQUIRED)
private String title;
}

98
commafeed-server/src/main/java/com/commafeed/frontend/model/ServerInfo.java
View File

@@ -1,49 +1,49 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Server infos")
@Data
@RegisterForReflection
public class ServerInfo implements Serializable {
@Schema
private String announcement;
@Schema(requiredMode = RequiredMode.REQUIRED)
private String version;
@Schema(requiredMode = RequiredMode.REQUIRED)
private String gitCommit;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean allowRegistrations;
@Schema
private String googleAnalyticsCode;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean smtpEnabled;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean demoAccountEnabled;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean websocketEnabled;
@Schema(requiredMode = RequiredMode.REQUIRED)
private long websocketPingInterval;
@Schema(requiredMode = RequiredMode.REQUIRED)
private long treeReloadInterval;
@Schema(requiredMode = RequiredMode.REQUIRED)
private long forceRefreshCooldownDuration;
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Server infos")
@Data
@RegisterForReflection
public class ServerInfo implements Serializable {
@Schema
private String announcement;
@Schema(requiredMode = RequiredMode.REQUIRED)
private String version;
@Schema(requiredMode = RequiredMode.REQUIRED)
private String gitCommit;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean allowRegistrations;
@Schema
private String googleAnalyticsCode;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean smtpEnabled;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean demoAccountEnabled;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean websocketEnabled;
@Schema(requiredMode = RequiredMode.REQUIRED)
private long websocketPingInterval;
@Schema(requiredMode = RequiredMode.REQUIRED)
private long treeReloadInterval;
@Schema(requiredMode = RequiredMode.REQUIRED)
private long forceRefreshCooldownDuration;
}

224
commafeed-server/src/main/java/com/commafeed/frontend/model/Settings.java
View File

@@ -1,112 +1,112 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "User settings")
@Data
@RegisterForReflection
public class Settings implements Serializable {
@Schema(description = "user's preferred language, english if none", requiredMode = RequiredMode.REQUIRED)
private String language;
@Schema(
description = "user reads all entries or unread entries only",
allowableValues = "all,unread",
requiredMode = RequiredMode.REQUIRED)
private String readingMode;
@Schema(
description = "user reads entries in ascending or descending order",
allowableValues = "asc,desc",
requiredMode = RequiredMode.REQUIRED)
private String readingOrder;
@Schema(description = "user wants category and feeds with no unread entries shown", requiredMode = RequiredMode.REQUIRED)
private boolean showRead;
@Schema(description = "In expanded view, scroll through entries mark them as read", requiredMode = RequiredMode.REQUIRED)
private boolean scrollMarks;
@Schema(description = "user's custom css for the website")
private String customCss;
@Schema(description = "user's custom js for the website")
private String customJs;
@Schema(description = "user's preferred scroll speed when navigating between entries", requiredMode = RequiredMode.REQUIRED)
private int scrollSpeed;
@Schema(
description = "whether to scroll to the selected entry",
allowableValues = "always,never,if_needed",
requiredMode = RequiredMode.REQUIRED)
private String scrollMode;
@Schema(description = "number of entries to keep above the selected entry when scrolling", requiredMode = RequiredMode.REQUIRED)
private int entriesToKeepOnTopWhenScrolling;
@Schema(
description = "whether to show the star icon in the header of entries",
allowableValues = "always,never,on_desktop,on_mobile",
requiredMode = RequiredMode.REQUIRED)
private String starIconDisplayMode;
@Schema(
description = "whether to show the external link icon in the header of entries",
allowableValues = "always,never,on_desktop,on_mobile",
requiredMode = RequiredMode.REQUIRED)
private String externalLinkIconDisplayMode;
@Schema(description = "ask for confirmation when marking all entries as read", requiredMode = RequiredMode.REQUIRED)
private boolean markAllAsReadConfirmation;
@Schema(description = "show commafeed's own context menu on right click", requiredMode = RequiredMode.REQUIRED)
private boolean customContextMenu;
@Schema(description = "on mobile, show action buttons at the bottom of the screen", requiredMode = RequiredMode.REQUIRED)
private boolean mobileFooter;
@Schema(description = "show unread count in the title", requiredMode = RequiredMode.REQUIRED)
private boolean unreadCountTitle;
@Schema(description = "show unread count in the favicon", requiredMode = RequiredMode.REQUIRED)
private boolean unreadCountFavicon;
@Schema(description = "sharing settings", requiredMode = RequiredMode.REQUIRED)
private SharingSettings sharingSettings = new SharingSettings();
@Schema(description = "User sharing settings")
@Data
public static class SharingSettings implements Serializable {
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean email;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean gmail;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean facebook;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean twitter;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean tumblr;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean pocket;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean instapaper;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean buffer;
}
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "User settings")
@Data
@RegisterForReflection
public class Settings implements Serializable {
@Schema(description = "user's preferred language, english if none", requiredMode = RequiredMode.REQUIRED)
private String language;
@Schema(
description = "user reads all entries or unread entries only",
allowableValues = "all,unread",
requiredMode = RequiredMode.REQUIRED)
private String readingMode;
@Schema(
description = "user reads entries in ascending or descending order",
allowableValues = "asc,desc",
requiredMode = RequiredMode.REQUIRED)
private String readingOrder;
@Schema(description = "user wants category and feeds with no unread entries shown", requiredMode = RequiredMode.REQUIRED)
private boolean showRead;
@Schema(description = "In expanded view, scroll through entries mark them as read", requiredMode = RequiredMode.REQUIRED)
private boolean scrollMarks;
@Schema(description = "user's custom css for the website")
private String customCss;
@Schema(description = "user's custom js for the website")
private String customJs;
@Schema(description = "user's preferred scroll speed when navigating between entries", requiredMode = RequiredMode.REQUIRED)
private int scrollSpeed;
@Schema(
description = "whether to scroll to the selected entry",
allowableValues = "always,never,if_needed",
requiredMode = RequiredMode.REQUIRED)
private String scrollMode;
@Schema(description = "number of entries to keep above the selected entry when scrolling", requiredMode = RequiredMode.REQUIRED)
private int entriesToKeepOnTopWhenScrolling;
@Schema(
description = "whether to show the star icon in the header of entries",
allowableValues = "always,never,on_desktop,on_mobile",
requiredMode = RequiredMode.REQUIRED)
private String starIconDisplayMode;
@Schema(
description = "whether to show the external link icon in the header of entries",
allowableValues = "always,never,on_desktop,on_mobile",
requiredMode = RequiredMode.REQUIRED)
private String externalLinkIconDisplayMode;
@Schema(description = "ask for confirmation when marking all entries as read", requiredMode = RequiredMode.REQUIRED)
private boolean markAllAsReadConfirmation;
@Schema(description = "show commafeed's own context menu on right click", requiredMode = RequiredMode.REQUIRED)
private boolean customContextMenu;
@Schema(description = "on mobile, show action buttons at the bottom of the screen", requiredMode = RequiredMode.REQUIRED)
private boolean mobileFooter;
@Schema(description = "show unread count in the title", requiredMode = RequiredMode.REQUIRED)
private boolean unreadCountTitle;
@Schema(description = "show unread count in the favicon", requiredMode = RequiredMode.REQUIRED)
private boolean unreadCountFavicon;
@Schema(description = "sharing settings", requiredMode = RequiredMode.REQUIRED)
private SharingSettings sharingSettings = new SharingSettings();
@Schema(description = "User sharing settings")
@Data
public static class SharingSettings implements Serializable {
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean email;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean gmail;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean facebook;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean twitter;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean tumblr;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean pocket;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean instapaper;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean buffer;
}
}

170
commafeed-server/src/main/java/com/commafeed/frontend/model/Subscription.java
View File

@@ -1,86 +1,86 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "User information")
@Data
@RegisterForReflection
public class Subscription implements Serializable {
@Schema(description = "subscription id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "subscription name", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "error message while fetching the feed")
private String message;
@Schema(description = "error count", requiredMode = RequiredMode.REQUIRED)
private int errorCount;
@Schema(description = "last time the feed was refreshed", type = "number")
private Instant lastRefresh;
@Schema(description = "next time the feed refresh is planned, null if refresh is already queued", type = "number")
private Instant nextRefresh;
@Schema(description = "this subscription's feed url", requiredMode = RequiredMode.REQUIRED)
private String feedUrl;
@Schema(description = "this subscription's website url", requiredMode = RequiredMode.REQUIRED)
private String feedLink;
@Schema(description = "The favicon url to use for this feed", requiredMode = RequiredMode.REQUIRED)
private String iconUrl;
@Schema(description = "unread count", requiredMode = RequiredMode.REQUIRED)
private long unread;
@Schema(description = "category id")
private String categoryId;
@Schema(description = "position of the subscription's in the list")
private int position;
@Schema(description = "date of the newest item", type = "number")
private Instant newestItemTime;
@Schema(description = "JEXL string evaluated on new entries to mark them as read if they do not match")
private String filter;
public static Subscription build(FeedSubscription subscription, UnreadCount unreadCount) {
FeedCategory category = subscription.getCategory();
Feed feed = subscription.getFeed();
Subscription sub = new Subscription();
sub.setId(subscription.getId());
sub.setName(subscription.getTitle());
sub.setPosition(subscription.getPosition());
sub.setMessage(feed.getMessage());
sub.setErrorCount(feed.getErrorCount());
sub.setFeedUrl(feed.getUrl());
sub.setFeedLink(feed.getLink());
sub.setIconUrl(FeedUtils.getFaviconUrl(subscription));
sub.setLastRefresh(feed.getLastUpdated());
sub.setNextRefresh(
(feed.getDisabledUntil() != null && feed.getDisabledUntil().isBefore(Instant.now())) ? null : feed.getDisabledUntil());
sub.setUnread(unreadCount.getUnreadCount());
sub.setNewestItemTime(unreadCount.getNewestItemTime());
sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));
sub.setFilter(subscription.getFilter());
return sub;
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "User information")
@Data
@RegisterForReflection
public class Subscription implements Serializable {
@Schema(description = "subscription id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "subscription name", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "error message while fetching the feed")
private String message;
@Schema(description = "error count", requiredMode = RequiredMode.REQUIRED)
private int errorCount;
@Schema(description = "last time the feed was refreshed", type = "number")
private Instant lastRefresh;
@Schema(description = "next time the feed refresh is planned, null if refresh is already queued", type = "number")
private Instant nextRefresh;
@Schema(description = "this subscription's feed url", requiredMode = RequiredMode.REQUIRED)
private String feedUrl;
@Schema(description = "this subscription's website url", requiredMode = RequiredMode.REQUIRED)
private String feedLink;
@Schema(description = "The favicon url to use for this feed", requiredMode = RequiredMode.REQUIRED)
private String iconUrl;
@Schema(description = "unread count", requiredMode = RequiredMode.REQUIRED)
private long unread;
@Schema(description = "category id")
private String categoryId;
@Schema(description = "position of the subscription's in the list")
private int position;
@Schema(description = "date of the newest item", type = "number")
private Instant newestItemTime;
@Schema(description = "JEXL string evaluated on new entries to mark them as read if they do not match")
private String filter;
public static Subscription build(FeedSubscription subscription, UnreadCount unreadCount) {
FeedCategory category = subscription.getCategory();
Feed feed = subscription.getFeed();
Subscription sub = new Subscription();
sub.setId(subscription.getId());
sub.setName(subscription.getTitle());
sub.setPosition(subscription.getPosition());
sub.setMessage(feed.getMessage());
sub.setErrorCount(feed.getErrorCount());
sub.setFeedUrl(feed.getUrl());
sub.setFeedLink(feed.getLink());
sub.setIconUrl(FeedUtils.getFaviconUrl(subscription));
sub.setLastRefresh(feed.getLastUpdated());
sub.setNextRefresh(
(feed.getDisabledUntil() != null && feed.getDisabledUntil().isBefore(Instant.now())) ? null : feed.getDisabledUntil());
sub.setUnread(unreadCount.getUnreadCount());
sub.setNewestItemTime(unreadCount.getNewestItemTime());
sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));
sub.setFilter(subscription.getFilter());
return sub;
}
}

68
commafeed-server/src/main/java/com/commafeed/frontend/model/UnreadCount.java
View File

@@ -1,34 +1,34 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Unread count")
@Data
@RegisterForReflection
public class UnreadCount implements Serializable {
@Schema
private long feedId;
@Schema
private long unreadCount;
@Schema(type = "number")
private Instant newestItemTime;
public UnreadCount() {
}
public UnreadCount(long feedId, long unreadCount, Instant newestItemTime) {
this.feedId = feedId;
this.unreadCount = unreadCount;
this.newestItemTime = newestItemTime;
}
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Unread count")
@Data
@RegisterForReflection
public class UnreadCount implements Serializable {
@Schema
private long feedId;
@Schema
private long unreadCount;
@Schema(type = "number")
private Instant newestItemTime;
public UnreadCount() {
}
public UnreadCount(long feedId, long unreadCount, Instant newestItemTime) {
this.feedId = feedId;
this.unreadCount = unreadCount;
this.newestItemTime = newestItemTime;
}
}

94
commafeed-server/src/main/java/com/commafeed/frontend/model/UserModel.java
View File

@@ -1,47 +1,47 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "User information")
@Data
@RegisterForReflection
public class UserModel implements Serializable {
@Schema(description = "user id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "user name", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "user email, if any")
private String email;
@Schema(description = "api key")
private String apiKey;
@Schema(description = "user password, never returned by the api")
private String password;
@Schema(description = "account status", requiredMode = RequiredMode.REQUIRED)
private boolean enabled;
@Schema(description = "account creation date", type = "number")
private Instant created;
@Schema(description = "last login date", type = "number")
private Instant lastLogin;
@Schema(description = "user is admin", requiredMode = RequiredMode.REQUIRED)
private boolean admin;
@Schema(description = "user last force refresh", type = "number")
private Instant lastForceRefresh;
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "User information")
@Data
@RegisterForReflection
public class UserModel implements Serializable {
@Schema(description = "user id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "user name", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "user email, if any")
private String email;
@Schema(description = "api key")
private String apiKey;
@Schema(description = "user password, never returned by the api")
private String password;
@Schema(description = "account status", requiredMode = RequiredMode.REQUIRED)
private boolean enabled;
@Schema(description = "account creation date", type = "number")
private Instant created;
@Schema(description = "last login date", type = "number")
private Instant lastLogin;
@Schema(description = "user is admin", requiredMode = RequiredMode.REQUIRED)
private boolean admin;
@Schema(description = "user last force refresh", type = "number")
private Instant lastForceRefresh;
}

52
commafeed-server/src/main/java/com/commafeed/frontend/model/request/AddCategoryRequest.java
View File

@@ -1,26 +1,26 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Add Category Request")
@Data
public class AddCategoryRequest implements Serializable {
@Schema(description = "name", requiredMode = RequiredMode.REQUIRED)
@NotEmpty
@Size(max = 128)
private String name;
@Schema(description = "parent category id, if any")
@Size(max = 128)
private String parentId;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Add Category Request")
@Data
public class AddCategoryRequest implements Serializable {
@Schema(description = "name", requiredMode = RequiredMode.REQUIRED)
@NotEmpty
@Size(max = 128)
private String name;
@Schema(description = "parent category id, if any")
@Size(max = 128)
private String parentId;
}

62
commafeed-server/src/main/java/com/commafeed/frontend/model/request/AdminSaveUserRequest.java
View File

@@ -1,31 +1,31 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Save User information")
@Data
public class AdminSaveUserRequest implements Serializable {
@Schema(description = "user id")
private Long id;
@Schema(description = "user name", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "user email, if any")
private String email;
@Schema(description = "user password")
private String password;
@Schema(description = "account status", requiredMode = RequiredMode.REQUIRED)
private boolean enabled;
@Schema(description = "user is admin", requiredMode = RequiredMode.REQUIRED)
private boolean admin;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Save User information")
@Data
public class AdminSaveUserRequest implements Serializable {
@Schema(description = "user id")
private Long id;
@Schema(description = "user name", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "user email, if any")
private String email;
@Schema(description = "user password")
private String password;
@Schema(description = "account status", requiredMode = RequiredMode.REQUIRED)
private boolean enabled;
@Schema(description = "user is admin", requiredMode = RequiredMode.REQUIRED)
private boolean admin;
}

60
commafeed-server/src/main/java/com/commafeed/frontend/model/request/CategoryModificationRequest.java
View File

@@ -1,30 +1,30 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Category modification request")
@Data
public class CategoryModificationRequest implements Serializable {
@Schema(description = "id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "new name, null if not changed")
@Size(max = 128)
private String name;
@Schema(description = "new parent category id")
@Size(max = 128)
private String parentId;
@Schema(description = "new display position, null if not changed")
private Integer position;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Category modification request")
@Data
public class CategoryModificationRequest implements Serializable {
@Schema(description = "id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "new name, null if not changed")
@Size(max = 128)
private String name;
@Schema(description = "new parent category id")
@Size(max = 128)
private String parentId;
@Schema(description = "new display position, null if not changed")
private Integer position;
}

40
commafeed-server/src/main/java/com/commafeed/frontend/model/request/CollapseRequest.java
View File

@@ -1,20 +1,20 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Mark Request")
@Data
public class CollapseRequest implements Serializable {
@Schema(description = "category id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "collapse", requiredMode = RequiredMode.REQUIRED)
private boolean collapse;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Mark Request")
@Data
public class CollapseRequest implements Serializable {
@Schema(description = "category id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "collapse", requiredMode = RequiredMode.REQUIRED)
private boolean collapse;
}

44
commafeed-server/src/main/java/com/commafeed/frontend/model/request/FeedInfoRequest.java
View File

@@ -1,22 +1,22 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Feed information request")
@Data
public class FeedInfoRequest implements Serializable {
@Schema(description = "feed url", requiredMode = RequiredMode.REQUIRED)
@NotEmpty
@Size(max = 4096)
private String url;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Feed information request")
@Data
public class FeedInfoRequest implements Serializable {
@Schema(description = "feed url", requiredMode = RequiredMode.REQUIRED)
@NotEmpty
@Size(max = 4096)
private String url;
}

68
commafeed-server/src/main/java/com/commafeed/frontend/model/request/FeedModificationRequest.java
View File

@@ -1,34 +1,34 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Feed modification request")
@Data
public class FeedModificationRequest implements Serializable {
@Schema(description = "id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "new name, null if not changed")
@Size(max = 128)
private String name;
@Schema(description = "new parent category id")
@Size(max = 128)
private String categoryId;
@Schema(description = "new display position, null if not changed")
private Integer position;
@Schema(description = "JEXL string evaluated on new entries to mark them as read if they do not match")
@Size(max = 4096)
private String filter;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Feed modification request")
@Data
public class FeedModificationRequest implements Serializable {
@Schema(description = "id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "new name, null if not changed")
@Size(max = 128)
private String name;
@Schema(description = "new parent category id")
@Size(max = 128)
private String categoryId;
@Schema(description = "new display position, null if not changed")
private Integer position;
@Schema(description = "JEXL string evaluated on new entries to mark them as read if they do not match")
@Size(max = 4096)
private String filter;
}

34
commafeed-server/src/main/java/com/commafeed/frontend/model/request/IDRequest.java
View File

@@ -1,17 +1,17 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema
@Data
public class IDRequest implements Serializable {
@Schema(requiredMode = RequiredMode.REQUIRED)
private Long id;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema
@Data
public class IDRequest implements Serializable {
@Schema(requiredMode = RequiredMode.REQUIRED)
private Long id;
}

90
commafeed-server/src/main/java/com/commafeed/frontend/model/request/MarkRequest.java
View File

@@ -1,45 +1,45 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import java.util.List;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Mark Request")
@Data
public class MarkRequest implements Serializable {
@Schema(description = "entry id, category id, 'all' or 'starred'", requiredMode = RequiredMode.REQUIRED)
@NotEmpty
@Size(max = 128)
private String id;
@Schema(description = "mark as read or unread", requiredMode = RequiredMode.REQUIRED)
private boolean read;
@Schema(description = "mark only entries older than this", requiredMode = RequiredMode.NOT_REQUIRED)
private Long olderThan;
@Schema(
description = "pass the timestamp you got from the entry list to avoid marking entries that may have been fetched in the mean time and never displayed",
requiredMode = RequiredMode.NOT_REQUIRED)
private Long insertedBefore;
@Schema(
description = "only mark read if a feed has these keywords in the title or rss content",
requiredMode = RequiredMode.NOT_REQUIRED)
@Size(max = 128)
private String keywords;
@Schema(
description = "if marking a category or 'all', exclude those subscriptions from the marking",
requiredMode = RequiredMode.NOT_REQUIRED)
private List<Long> excludedSubscriptions;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import java.util.List;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Mark Request")
@Data
public class MarkRequest implements Serializable {
@Schema(description = "entry id, category id, 'all' or 'starred'", requiredMode = RequiredMode.REQUIRED)
@NotEmpty
@Size(max = 128)
private String id;
@Schema(description = "mark as read or unread", requiredMode = RequiredMode.REQUIRED)
private boolean read;
@Schema(description = "mark only entries older than this", requiredMode = RequiredMode.NOT_REQUIRED)
private Long olderThan;
@Schema(
description = "pass the timestamp you got from the entry list to avoid marking entries that may have been fetched in the mean time and never displayed",
requiredMode = RequiredMode.NOT_REQUIRED)
private Long insertedBefore;
@Schema(
description = "only mark read if a feed has these keywords in the title or rss content",
requiredMode = RequiredMode.NOT_REQUIRED)
@Size(max = 128)
private String keywords;
@Schema(
description = "if marking a category or 'all', exclude those subscriptions from the marking",
requiredMode = RequiredMode.NOT_REQUIRED)
private List<Long> excludedSubscriptions;
}

40
commafeed-server/src/main/java/com/commafeed/frontend/model/request/MultipleMarkRequest.java
View File

@@ -1,20 +1,20 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import java.util.List;
import jakarta.validation.Valid;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Multiple Mark Request")
@Data
public class MultipleMarkRequest implements Serializable {
@Schema(description = "list of mark requests", requiredMode = RequiredMode.REQUIRED)
private List<@Valid MarkRequest> requests;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import java.util.List;
import jakarta.validation.Valid;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Multiple Mark Request")
@Data
public class MultipleMarkRequest implements Serializable {
@Schema(description = "list of mark requests", requiredMode = RequiredMode.REQUIRED)
private List<@Valid MarkRequest> requests;
}

Some files were not shown because too many files have changed in this diff Show More