Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-23 04:25:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

use google java formatter to avoid downloading eclipse during build

Browse Source
This commit is contained in:
Athou
2026-07-05 10:18:30 +02:00
parent f4dc279ec9
commit 1bd9b71215
200 changed files with 13923 additions and 12754 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
commafeed-server/src/main/java/com/commafeed/CommaFeedApplication.java
View File

@@ -1,15 +1,13 @@
package com.commafeed;
import jakarta.enterprise.event.Observes;
import jakarta.inject.Singleton;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.feed.ImageProxyUrl;
import com.commafeed.backend.task.TaskScheduler;
import com.commafeed.security.password.PasswordConstraintValidator;
import io.quarkus.runtime.ShutdownEvent;
import io.quarkus.runtime.StartupEvent;
import jakarta.enterprise.event.Observes;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -18,28 +16,28 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
public class CommaFeedApplication {
private final FeedRefreshEngine feedRefreshEngine;
private final TaskScheduler taskScheduler;
private final CommaFeedConfiguration config;
private final FeedRefreshEngine feedRefreshEngine;
private final TaskScheduler taskScheduler;
private final CommaFeedConfiguration config;
public void start(@Observes StartupEvent ev) {
log.info("starting up...");
public void start(@Observes StartupEvent ev) {
log.info("starting up...");
PasswordConstraintValidator.setMinimumPasswordLength(config.users().minimumPasswordLength());
PasswordConstraintValidator.setMinimumPasswordLength(
config.users().minimumPasswordLength());
if (config.imageProxyEnabled()) {
ImageProxyUrl.generateKey();
}
if (config.imageProxyEnabled()) {
ImageProxyUrl.generateKey();
}
feedRefreshEngine.start();
taskScheduler.start();
}
feedRefreshEngine.start();
taskScheduler.start();
}
public void stop(@Observes ShutdownEvent ev) {
log.info("shutting down...");
feedRefreshEngine.stop();
taskScheduler.stop();
}
public void stop(@Observes ShutdownEvent ev) {
log.info("shutting down...");
feedRefreshEngine.stop();
taskScheduler.stop();
}
}

622
commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java
View File

@@ -1,406 +1,360 @@
package com.commafeed;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.Positive;
import com.commafeed.backend.feed.FeedRefreshIntervalCalculator;
import io.quarkus.runtime.annotations.ConfigDocSection;
import io.quarkus.runtime.annotations.ConfigPhase;
import io.quarkus.runtime.annotations.ConfigRoot;
import io.quarkus.runtime.configuration.MemorySize;
import io.smallrye.config.ConfigMapping;
import io.smallrye.config.WithDefault;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.Positive;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
/**
* CommaFeed configuration
*
* Default values are for production, they can be overridden in application.properties for other profiles
* <p>Default values are for production, they can be overridden in application.properties for other
* profiles
*/
@ConfigMapping(prefix = "commafeed")
@ConfigRoot(phase = ConfigPhase.RUN_TIME)
public interface CommaFeedConfiguration {
/**
* Whether to expose a robots.txt file that disallows web crawlers and search engine indexers.
*/
@WithDefault("true")
boolean hideFromWebCrawlers();
/**
* Whether to expose a robots.txt file that disallows web crawlers and search engine indexers.
*/
@WithDefault("true")
boolean hideFromWebCrawlers();
/**
* If enabled, images in feed entries will be proxied through the server instead of accessed directly by the browser.
*
* This is useful if commafeed is accessed through a restricting proxy that blocks some feeds that are followed.
*/
@WithDefault("false")
boolean imageProxyEnabled();
/**
* If enabled, images in feed entries will be proxied through the server instead of accessed
* directly by the browser.
*
* <p>This is useful if commafeed is accessed through a restricting proxy that blocks some feeds
* that are followed.
*/
@WithDefault("false")
boolean imageProxyEnabled();
/**
* Enable password recovery via email.
*
* Quarkus mailer will need to be configured.
*/
@WithDefault("false")
boolean passwordRecoveryEnabled();
/**
* Enable password recovery via email.
*
* <p>Quarkus mailer will need to be configured.
*/
@WithDefault("false")
boolean passwordRecoveryEnabled();
/**
* Message displayed in a notification at the bottom of the page.
*/
Optional<String> announcement();
/** Message displayed in a notification at the bottom of the page. */
Optional<String> announcement();
/**
* Google Auth key for fetching Youtube channel favicons.
*/
Optional<String> googleAuthKey();
/** Google Auth key for fetching Youtube channel favicons. */
Optional<String> googleAuthKey();
/**
* HTTP client configuration
*/
@ConfigDocSection
HttpClient httpClient();
/** HTTP client configuration */
@ConfigDocSection
HttpClient httpClient();
/**
* Feed refresh engine settings.
*/
@ConfigDocSection
FeedRefresh feedRefresh();
/** Feed refresh engine settings. */
@ConfigDocSection
FeedRefresh feedRefresh();
/**
* Push notification settings.
*/
@ConfigDocSection
PushNotifications pushNotifications();
/** Push notification settings. */
@ConfigDocSection
PushNotifications pushNotifications();
/**
* Database settings.
*/
@ConfigDocSection
Database database();
/** Database settings. */
@ConfigDocSection
Database database();
/**
* Users settings.
*/
@ConfigDocSection
Users users();
/** Users settings. */
@ConfigDocSection
Users users();
/**
* Websocket settings.
*/
@ConfigDocSection
Websocket websocket();
/** Websocket settings. */
@ConfigDocSection
Websocket websocket();
/**
* Duration to wait for the feed refresh engine and the task scheduler to stop when the application is shutting down.
*/
@WithDefault("2s")
Duration shutdownTimeout();
/**
* Duration to wait for the feed refresh engine and the task scheduler to stop when the
* application is shutting down.
*/
@WithDefault("2s")
Duration shutdownTimeout();
interface HttpClient {
/**
* User-Agent string that will be used by the http client, leave empty for the default one.
*/
Optional<String> userAgent();
interface HttpClient {
/**
* User-Agent string that will be used by the http client, leave empty for the default one.
*/
Optional<String> userAgent();
/**
* Time to wait for a connection to be established.
*/
@WithDefault("5s")
Duration connectTimeout();
/** Time to wait for a connection to be established. */
@WithDefault("5s")
Duration connectTimeout();
/**
* Time to wait for SSL handshake to complete.
*/
@WithDefault("5s")
Duration sslHandshakeTimeout();
/** Time to wait for SSL handshake to complete. */
@WithDefault("5s")
Duration sslHandshakeTimeout();
/**
* Time to wait between two packets before timeout.
*/
@WithDefault("10s")
Duration socketTimeout();
/** Time to wait between two packets before timeout. */
@WithDefault("10s")
Duration socketTimeout();
/**
* Time to wait for the full response to be received.
*/
@WithDefault("10s")
Duration responseTimeout();
/** Time to wait for the full response to be received. */
@WithDefault("10s")
Duration responseTimeout();
/**
* Time to live for a connection in the pool.
*/
@WithDefault("30s")
Duration connectionTimeToLive();
/** Time to live for a connection in the pool. */
@WithDefault("30s")
Duration connectionTimeToLive();
/**
* Time between eviction runs for idle connections.
*/
@WithDefault("1m")
Duration idleConnectionsEvictionInterval();
/** Time between eviction runs for idle connections. */
@WithDefault("1m")
Duration idleConnectionsEvictionInterval();
/**
* If a feed is larger than this, it will be discarded to prevent memory issues while parsing the feed.
*/
@WithDefault("5M")
MemorySize maxResponseSize();
/**
* If a feed is larger than this, it will be discarded to prevent memory issues while
* parsing the feed.
*/
@WithDefault("5M")
MemorySize maxResponseSize();
/**
* Prevent access to local addresses to mitigate server-side request forgery (SSRF) attacks, which could potentially expose internal
* resources.
*
* You may want to enable this if you host a public instance of CommaFeed with registrations open.
*/
@WithDefault("false")
boolean blockLocalAddresses();
/**
* Prevent access to local addresses to mitigate server-side request forgery (SSRF) attacks,
* which could potentially expose internal resources.
*
* <p>You may want to enable this if you host a public instance of CommaFeed with
* registrations open.
*/
@WithDefault("false")
boolean blockLocalAddresses();
/**
* HTTP client cache configuration
*/
@ConfigDocSection
HttpClientCache cache();
}
/** HTTP client cache configuration */
@ConfigDocSection
HttpClientCache cache();
}
interface HttpClientCache {
/**
* Whether to enable the cache. This cache is used to avoid spamming feeds in short bursts (e.g. when subscribing to a feed for the
* first time or when clicking "fetch all my feeds now").
*/
@WithDefault("true")
boolean enabled();
interface HttpClientCache {
/**
* Whether to enable the cache. This cache is used to avoid spamming feeds in short bursts
* (e.g. when subscribing to a feed for the first time or when clicking "fetch all my feeds
* now").
*/
@WithDefault("true")
boolean enabled();
/**
* Maximum amount of memory the cache can use.
*/
@WithDefault("10M")
MemorySize maximumMemorySize();
/** Maximum amount of memory the cache can use. */
@WithDefault("10M")
MemorySize maximumMemorySize();
/**
* Duration after which an entry is removed from the cache.
*/
@WithDefault("1m")
Duration expiration();
}
/** Duration after which an entry is removed from the cache. */
@WithDefault("1m")
Duration expiration();
}
interface FeedRefresh {
/**
* Default amount of time CommaFeed will wait before refreshing a feed.
*/
@WithDefault("5m")
Duration interval();
interface FeedRefresh {
/** Default amount of time CommaFeed will wait before refreshing a feed. */
@WithDefault("5m")
Duration interval();
/**
* Maximum amount of time CommaFeed will wait before refreshing a feed. This is used as an upper bound when:
*
* <ul>
* <li>an error occurs while refreshing a feed and we're backing off exponentially</li>
* <li>we receive a Cache-Control header from the feed</li>
* <li>we receive a Retry-After header from the feed</li>
* </ul>
*/
@WithDefault("4h")
Duration maxInterval();
/**
* Maximum amount of time CommaFeed will wait before refreshing a feed. This is used as an
* upper bound when:
*
* <ul>
* <li>an error occurs while refreshing a feed and we're backing off exponentially
* <li>we receive a Cache-Control header from the feed
* <li>we receive a Retry-After header from the feed
* </ul>
*/
@WithDefault("4h")
Duration maxInterval();
/**
* If enabled, CommaFeed will calculate the next refresh time based on the feed's average time between entries and the time since
* the last entry was published. The interval will be sometimes between the default refresh interval
* (`commafeed.feed-refresh.interval`) and the maximum refresh interval (`commafeed.feed-refresh.max-interval`).
*
* See {@link FeedRefreshIntervalCalculator} for details.
*/
@WithDefault("true")
boolean intervalEmpirical();
/**
* If enabled, CommaFeed will calculate the next refresh time based on the feed's average
* time between entries and the time since the last entry was published. The interval will
* be sometimes between the default refresh interval (`commafeed.feed-refresh.interval`) and
* the maximum refresh interval (`commafeed.feed-refresh.max-interval`).
*
* <p>See {@link FeedRefreshIntervalCalculator} for details.
*/
@WithDefault("true")
boolean intervalEmpirical();
/**
* Feed refresh engine error handling settings.
*/
@ConfigDocSection
FeedRefreshErrorHandling errors();
/** Feed refresh engine error handling settings. */
@ConfigDocSection
FeedRefreshErrorHandling errors();
/**
* Amount of http threads used to fetch feeds.
*/
@Min(1)
@WithDefault("3")
int httpThreads();
/** Amount of http threads used to fetch feeds. */
@Min(1)
@WithDefault("3")
int httpThreads();
/**
* Amount of threads used to insert new entries in the database.
*/
@Min(1)
@WithDefault("1")
int databaseThreads();
/** Amount of threads used to insert new entries in the database. */
@Min(1)
@WithDefault("1")
int databaseThreads();
/**
* Duration after which a user is considered inactive. Feeds for inactive users are not refreshed until they log in again.
*
* 0 to disable.
*/
@WithDefault("0")
Duration userInactivityPeriod();
/**
* Duration after which a user is considered inactive. Feeds for inactive users are not
* refreshed until they log in again.
*
* <p>0 to disable.
*/
@WithDefault("0")
Duration userInactivityPeriod();
/**
* Duration after which the evaluation of a filtering expresion to mark an entry as read is considered to have timed out.
*/
@WithDefault("500ms")
Duration filteringExpressionEvaluationTimeout();
/**
* Duration after which the evaluation of a filtering expresion to mark an entry as read is
* considered to have timed out.
*/
@WithDefault("500ms")
Duration filteringExpressionEvaluationTimeout();
/**
* Duration after which the "Fetch all my feeds now" action is available again after use to avoid spamming feeds.
*/
@WithDefault("0")
Duration forceRefreshCooldownDuration();
}
/**
* Duration after which the "Fetch all my feeds now" action is available again after use to
* avoid spamming feeds.
*/
@WithDefault("0")
Duration forceRefreshCooldownDuration();
}
interface PushNotifications {
/**
* Whether to enable push notifications to notify users of new entries in their feeds.
*/
@WithDefault("true")
boolean enabled();
interface PushNotifications {
/** Whether to enable push notifications to notify users of new entries in their feeds. */
@WithDefault("true")
boolean enabled();
/**
* Amount of threads used to send external notifications about new entries.
*/
@Min(1)
@WithDefault("5")
int threads();
/** Amount of threads used to send external notifications about new entries. */
@Min(1)
@WithDefault("5")
int threads();
/**
* Maximum amount of notifications that can be queued before new notifications are discarded.
*/
@Min(1)
@WithDefault("100")
int queueCapacity();
}
/**
* Maximum amount of notifications that can be queued before new notifications are
* discarded.
*/
@Min(1)
@WithDefault("100")
int queueCapacity();
}
interface FeedRefreshErrorHandling {
/**
* Number of retries before backoff is applied.
*/
@Min(0)
@WithDefault("3")
int retriesBeforeBackoff();
interface FeedRefreshErrorHandling {
/** Number of retries before backoff is applied. */
@Min(0)
@WithDefault("3")
int retriesBeforeBackoff();
/**
* Duration to wait before retrying after an error. Will be multiplied by the number of errors since the last successful fetch.
*/
@WithDefault("1h")
Duration backoffInterval();
}
/**
* Duration to wait before retrying after an error. Will be multiplied by the number of
* errors since the last successful fetch.
*/
@WithDefault("1h")
Duration backoffInterval();
}
interface Database {
/**
* Timeout applied to all database queries.
*
* 0 to disable.
*/
@WithDefault("0")
Duration queryTimeout();
interface Database {
/**
* Timeout applied to all database queries.
*
* <p>0 to disable.
*/
@WithDefault("0")
Duration queryTimeout();
/**
* Database cleanup settings.
*/
@ConfigDocSection
Cleanup cleanup();
/** Database cleanup settings. */
@ConfigDocSection
Cleanup cleanup();
interface Cleanup {
/**
* Maximum age of feed entries in the database. Older entries will be deleted.
*
* 0 to disable.
*/
@WithDefault("365d")
Duration entriesMaxAge();
interface Cleanup {
/**
* Maximum age of feed entries in the database. Older entries will be deleted.
*
* <p>0 to disable.
*/
@WithDefault("365d")
Duration entriesMaxAge();
/**
* Maximum age of feed entry statuses (read/unread) in the database. Older statuses will be deleted.
*
* 0 to disable.
*/
@WithDefault("0")
Duration statusesMaxAge();
/**
* Maximum age of feed entry statuses (read/unread) in the database. Older statuses will
* be deleted.
*
* <p>0 to disable.
*/
@WithDefault("0")
Duration statusesMaxAge();
/**
* Maximum number of entries per feed to keep in the database.
*
* 0 to disable.
*/
@WithDefault("500")
int maxFeedCapacity();
/**
* Maximum number of entries per feed to keep in the database.
*
* <p>0 to disable.
*/
@WithDefault("500")
int maxFeedCapacity();
/**
* Limit the number of feeds a user can subscribe to.
*
* 0 to disable.
*/
@WithDefault("0")
int maxFeedsPerUser();
/**
* Limit the number of feeds a user can subscribe to.
*
* <p>0 to disable.
*/
@WithDefault("0")
int maxFeedsPerUser();
/**
* Rows to delete per query while cleaning up old entries.
*/
@Positive
@WithDefault("100")
int batchSize();
/** Rows to delete per query while cleaning up old entries. */
@Positive
@WithDefault("100")
int batchSize();
/**
* Whether to keep starred entries when cleaning up old entries.
*/
@WithDefault("true")
boolean keepStarredEntries();
/** Whether to keep starred entries when cleaning up old entries. */
@WithDefault("true")
boolean keepStarredEntries();
default Instant statusesInstantThreshold() {
return statusesMaxAge().toMillis() > 0 ? Instant.now().minus(statusesMaxAge()) : null;
}
}
}
default Instant statusesInstantThreshold() {
return statusesMaxAge().toMillis() > 0
? Instant.now().minus(statusesMaxAge())
: null;
}
}
}
interface Users {
/**
* Whether to let users create accounts for themselves.
*/
@WithDefault("false")
boolean allowRegistrations();
interface Users {
/** Whether to let users create accounts for themselves. */
@WithDefault("false")
boolean allowRegistrations();
/**
* Minimum password length for user accounts.
*/
@WithDefault("4")
int minimumPasswordLength();
/** Minimum password length for user accounts. */
@WithDefault("4")
int minimumPasswordLength();
/**
* Whether an email address is required when creating a user account.
*/
@WithDefault("false")
boolean emailAddressRequired();
/** Whether an email address is required when creating a user account. */
@WithDefault("false")
boolean emailAddressRequired();
/**
* Whether to create a demo account the first time the app starts.
*/
@WithDefault("false")
boolean createDemoAccount();
}
/** Whether to create a demo account the first time the app starts. */
@WithDefault("false")
boolean createDemoAccount();
}
interface Websocket {
/**
* Enable websocket connection so the server can notify web clients that there are new entries for feeds.
*/
@WithDefault("true")
boolean enabled();
interface Websocket {
/**
* Enable websocket connection so the server can notify web clients that there are new
* entries for feeds.
*/
@WithDefault("true")
boolean enabled();
/**
* Interval at which the client will send a ping message on the websocket to keep the connection alive.
*/
@WithDefault("15m")
Duration pingInterval();
/**
* If the websocket connection is disabled or the connection is lost, the client will reload the feed tree at this interval.
*/
@WithDefault("30s")
Duration treeReloadInterval();
}
/**
* Interval at which the client will send a ping message on the websocket to keep the
* connection alive.
*/
@WithDefault("15m")
Duration pingInterval();
/**
* If the websocket connection is disabled or the connection is lost, the client will reload
* the feed tree at this interval.
*/
@WithDefault("30s")
Duration treeReloadInterval();
}
}

2
commafeed-server/src/main/java/com/commafeed/CommaFeedConstants.java
View File

@@ -4,5 +4,5 @@ import lombok.experimental.UtilityClass;
@UtilityClass
public class CommaFeedConstants {
public static final String USERNAME_DEMO = "demo";
public static final String USERNAME_DEMO = "demo";
}

27
commafeed-server/src/main/java/com/commafeed/CommaFeedProducers.java
View File

@@ -1,25 +1,22 @@
package com.commafeed;
import java.time.InstantSource;
import com.codahale.metrics.MetricRegistry;
import jakarta.enterprise.inject.Produces;
import jakarta.inject.Singleton;
import com.codahale.metrics.MetricRegistry;
import java.time.InstantSource;
@Singleton
public class CommaFeedProducers {
@Produces
@Singleton
public InstantSource instantSource() {
return InstantSource.system();
}
@Produces
@Singleton
public MetricRegistry metricRegistry() {
return new MetricRegistry();
}
@Produces
@Singleton
public InstantSource instantSource() {
return InstantSource.system();
}
@Produces
@Singleton
public MetricRegistry metricRegistry() {
return new MetricRegistry();
}
}

29
commafeed-server/src/main/java/com/commafeed/CommaFeedVersion.java
View File

@@ -1,30 +1,27 @@
package com.commafeed;
import jakarta.inject.Singleton;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import jakarta.inject.Singleton;
import lombok.Getter;
@Singleton
@Getter
public class CommaFeedVersion {
private final String version;
private final String gitCommit;
private final String version;
private final String gitCommit;
public CommaFeedVersion() throws IOException {
Properties properties = new Properties();
try (InputStream stream = getClass().getResourceAsStream("/git.properties")) {
if (stream != null) {
properties.load(stream);
}
}
this.version = properties.getProperty("git.build.version", "unknown");
this.gitCommit = properties.getProperty("git.commit.id.abbrev", "unknown");
}
public CommaFeedVersion() throws IOException {
Properties properties = new Properties();
try (InputStream stream = getClass().getResourceAsStream("/git.properties")) {
if (stream != null) {
properties.load(stream);
}
}
this.version = properties.getProperty("git.build.version", "unknown");
this.gitCommit = properties.getProperty("git.commit.id.abbrev", "unknown");
}
}

60
commafeed-server/src/main/java/com/commafeed/ExceptionMappers.java
View File

@@ -1,52 +1,54 @@
package com.commafeed;
import com.commafeed.security.CookieService;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.quarkus.security.AuthenticationFailedException;
import io.quarkus.security.UnauthorizedException;
import jakarta.annotation.Priority;
import jakarta.validation.ValidationException;
import jakarta.ws.rs.core.NewCookie;
import jakarta.ws.rs.ext.Provider;
import lombok.RequiredArgsConstructor;
import org.jboss.resteasy.reactive.RestResponse;
import org.jboss.resteasy.reactive.RestResponse.ResponseBuilder;
import org.jboss.resteasy.reactive.RestResponse.Status;
import org.jboss.resteasy.reactive.server.ServerExceptionMapper;
import com.commafeed.security.CookieService;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.quarkus.security.AuthenticationFailedException;
import io.quarkus.security.UnauthorizedException;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Provider
@Priority(1)
public class ExceptionMappers {
private final CookieService cookieService;
private final CommaFeedConfiguration config;
private final CookieService cookieService;
private final CommaFeedConfiguration config;
@ServerExceptionMapper(UnauthorizedException.class)
public RestResponse<UnauthorizedResponse> unauthorized(UnauthorizedException e) {
return RestResponse.status(Status.UNAUTHORIZED, new UnauthorizedResponse(e.getMessage(), config.users().allowRegistrations()));
}
@ServerExceptionMapper(UnauthorizedException.class)
public RestResponse<UnauthorizedResponse> unauthorized(UnauthorizedException e) {
return RestResponse.status(
Status.UNAUTHORIZED,
new UnauthorizedResponse(e.getMessage(), config.users().allowRegistrations()));
}
@ServerExceptionMapper(AuthenticationFailedException.class)
public RestResponse<AuthenticationFailed> authenticationFailed(AuthenticationFailedException e) {
NewCookie logoutCookie = cookieService.buildLogoutCookie();
return ResponseBuilder.create(Status.UNAUTHORIZED, new AuthenticationFailed(e.getMessage())).cookie(logoutCookie).build();
}
@ServerExceptionMapper(AuthenticationFailedException.class)
public RestResponse<AuthenticationFailed> authenticationFailed(
AuthenticationFailedException e) {
NewCookie logoutCookie = cookieService.buildLogoutCookie();
return ResponseBuilder.create(Status.UNAUTHORIZED, new AuthenticationFailed(e.getMessage()))
.cookie(logoutCookie)
.build();
}
@ServerExceptionMapper(ValidationException.class)
public RestResponse<ValidationFailed> validationFailed(ValidationException e) {
return RestResponse.status(Status.BAD_REQUEST, new ValidationFailed(e.getMessage()));
}
@ServerExceptionMapper(ValidationException.class)
public RestResponse<ValidationFailed> validationFailed(ValidationException e) {
return RestResponse.status(Status.BAD_REQUEST, new ValidationFailed(e.getMessage()));
}
@RegisterForReflection
public record UnauthorizedResponse(String message, boolean allowRegistrations) {}
@RegisterForReflection
public record UnauthorizedResponse(String message, boolean allowRegistrations) {}
@RegisterForReflection
public record AuthenticationFailed(String message) {}
@RegisterForReflection
public record AuthenticationFailed(String message) {}
@RegisterForReflection
public record ValidationFailed(String message) {}
@RegisterForReflection
public record ValidationFailed(String message) {}
}

28
commafeed-server/src/main/java/com/commafeed/JacksonCustomizer.java
View File

@@ -1,29 +1,27 @@
package com.commafeed;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.json.MetricsModule;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.quarkus.jackson.ObjectMapperCustomizer;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
@Singleton
public class JacksonCustomizer implements ObjectMapperCustomizer {
@Override
public void customize(ObjectMapper objectMapper) {
objectMapper.registerModule(new JavaTimeModule());
@Override
public void customize(ObjectMapper objectMapper) {
objectMapper.registerModule(new JavaTimeModule());
// read and write instants as milliseconds instead of nanoseconds
objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true)
.configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false)
.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false);
// read and write instants as milliseconds instead of nanoseconds
objectMapper
.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true)
.configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false)
.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false);
// add support for serializing metrics
objectMapper.registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
}
// add support for serializing metrics
objectMapper.registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
}
}

544
commafeed-server/src/main/java/com/commafeed/NativeImageClasses.java
View File

@@ -6,221 +6,343 @@ import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import io.quarkus.runtime.annotations.RegisterForReflection;
@RegisterForReflection(
targets = {
// metrics
MetricRegistry.class, Meter.class, Gauge.class, Counter.class, Timer.class, Histogram.class,
targets = {
// metrics
MetricRegistry.class,
Meter.class,
Gauge.class,
Counter.class,
Timer.class,
Histogram.class,
// rome
java.util.Date.class, com.rometools.opml.feed.synd.impl.TreeCategoryImpl.class,
com.rometools.rome.feed.synd.SyndFeedImpl.class, com.rometools.rome.feed.module.DCSubjectImpl.class,
com.rometools.rome.feed.synd.SyndEntryImpl.class, com.rometools.modules.psc.types.SimpleChapter.class,
com.rometools.rome.feed.synd.SyndCategoryImpl.class, com.rometools.rome.feed.synd.SyndImageImpl.class,
com.rometools.rome.feed.synd.SyndContentImpl.class, com.rometools.rome.feed.synd.SyndEnclosureImpl.class,
// rome
java.util.Date.class,
com.rometools.opml.feed.synd.impl.TreeCategoryImpl.class,
com.rometools.rome.feed.synd.SyndFeedImpl.class,
com.rometools.rome.feed.module.DCSubjectImpl.class,
com.rometools.rome.feed.synd.SyndEntryImpl.class,
com.rometools.modules.psc.types.SimpleChapter.class,
com.rometools.rome.feed.synd.SyndCategoryImpl.class,
com.rometools.rome.feed.synd.SyndImageImpl.class,
com.rometools.rome.feed.synd.SyndContentImpl.class,
com.rometools.rome.feed.synd.SyndEnclosureImpl.class,
// rome cloneable
com.rometools.modules.activitystreams.types.Article.class, com.rometools.modules.activitystreams.types.Audio.class,
com.rometools.modules.activitystreams.types.Bookmark.class, com.rometools.modules.activitystreams.types.Comment.class,
com.rometools.modules.activitystreams.types.Event.class, com.rometools.modules.activitystreams.types.File.class,
com.rometools.modules.activitystreams.types.Folder.class, com.rometools.modules.activitystreams.types.List.class,
com.rometools.modules.activitystreams.types.Note.class, com.rometools.modules.activitystreams.types.Person.class,
com.rometools.modules.activitystreams.types.Photo.class, com.rometools.modules.activitystreams.types.PhotoAlbum.class,
com.rometools.modules.activitystreams.types.Place.class, com.rometools.modules.activitystreams.types.Playlist.class,
com.rometools.modules.activitystreams.types.Product.class, com.rometools.modules.activitystreams.types.Review.class,
com.rometools.modules.activitystreams.types.Service.class, com.rometools.modules.activitystreams.types.Song.class,
com.rometools.modules.activitystreams.types.Status.class, com.rometools.modules.base.types.DateTimeRange.class,
com.rometools.modules.base.types.FloatUnit.class, com.rometools.modules.base.types.GenderEnumeration.class,
com.rometools.modules.base.types.IntUnit.class, com.rometools.modules.base.types.PriceTypeEnumeration.class,
com.rometools.modules.base.types.ShippingType.class, com.rometools.modules.base.types.ShortDate.class,
com.rometools.modules.base.types.Size.class, com.rometools.modules.base.types.YearType.class,
com.rometools.modules.content.ContentItem.class, com.rometools.modules.georss.GeoRSSPoint.class,
com.rometools.modules.georss.geometries.Envelope.class, com.rometools.modules.georss.geometries.LineString.class,
com.rometools.modules.georss.geometries.LinearRing.class, com.rometools.modules.georss.geometries.Point.class,
com.rometools.modules.georss.geometries.Polygon.class, com.rometools.modules.georss.geometries.Position.class,
com.rometools.modules.georss.geometries.PositionList.class, com.rometools.modules.mediarss.types.MediaGroup.class,
com.rometools.modules.mediarss.types.Metadata.class, com.rometools.modules.mediarss.types.Thumbnail.class,
com.rometools.modules.opensearch.entity.OSQuery.class, com.rometools.modules.photocast.types.PhotoDate.class,
com.rometools.modules.sle.types.DateValue.class, com.rometools.modules.sle.types.Group.class,
com.rometools.modules.sle.types.NumberValue.class, com.rometools.modules.sle.types.Sort.class,
com.rometools.modules.sle.types.StringValue.class, com.rometools.modules.yahooweather.types.Astronomy.class,
com.rometools.modules.yahooweather.types.Atmosphere.class, com.rometools.modules.yahooweather.types.Condition.class,
com.rometools.modules.yahooweather.types.Forecast.class, com.rometools.modules.yahooweather.types.Location.class,
com.rometools.modules.yahooweather.types.Units.class, com.rometools.modules.yahooweather.types.Wind.class,
com.rometools.opml.feed.opml.Attribute.class, com.rometools.opml.feed.opml.Opml.class,
com.rometools.opml.feed.opml.Outline.class, com.rometools.rome.feed.atom.Category.class,
com.rometools.rome.feed.atom.Content.class, com.rometools.rome.feed.atom.Entry.class,
com.rometools.rome.feed.atom.Feed.class, com.rometools.rome.feed.atom.Generator.class,
com.rometools.rome.feed.atom.Link.class, com.rometools.rome.feed.atom.Person.class,
com.rometools.rome.feed.rss.Category.class, com.rometools.rome.feed.rss.Channel.class,
com.rometools.rome.feed.rss.Cloud.class, com.rometools.rome.feed.rss.Content.class,
com.rometools.rome.feed.rss.Description.class, com.rometools.rome.feed.rss.Enclosure.class,
com.rometools.rome.feed.rss.Guid.class, com.rometools.rome.feed.rss.Image.class, com.rometools.rome.feed.rss.Item.class,
com.rometools.rome.feed.rss.Source.class, com.rometools.rome.feed.rss.TextInput.class,
com.rometools.rome.feed.synd.SyndLinkImpl.class, com.rometools.rome.feed.synd.SyndPersonImpl.class,
java.util.ArrayList.class,
// rome cloneable
com.rometools.modules.activitystreams.types.Article.class,
com.rometools.modules.activitystreams.types.Audio.class,
com.rometools.modules.activitystreams.types.Bookmark.class,
com.rometools.modules.activitystreams.types.Comment.class,
com.rometools.modules.activitystreams.types.Event.class,
com.rometools.modules.activitystreams.types.File.class,
com.rometools.modules.activitystreams.types.Folder.class,
com.rometools.modules.activitystreams.types.List.class,
com.rometools.modules.activitystreams.types.Note.class,
com.rometools.modules.activitystreams.types.Person.class,
com.rometools.modules.activitystreams.types.Photo.class,
com.rometools.modules.activitystreams.types.PhotoAlbum.class,
com.rometools.modules.activitystreams.types.Place.class,
com.rometools.modules.activitystreams.types.Playlist.class,
com.rometools.modules.activitystreams.types.Product.class,
com.rometools.modules.activitystreams.types.Review.class,
com.rometools.modules.activitystreams.types.Service.class,
com.rometools.modules.activitystreams.types.Song.class,
com.rometools.modules.activitystreams.types.Status.class,
com.rometools.modules.base.types.DateTimeRange.class,
com.rometools.modules.base.types.FloatUnit.class,
com.rometools.modules.base.types.GenderEnumeration.class,
com.rometools.modules.base.types.IntUnit.class,
com.rometools.modules.base.types.PriceTypeEnumeration.class,
com.rometools.modules.base.types.ShippingType.class,
com.rometools.modules.base.types.ShortDate.class,
com.rometools.modules.base.types.Size.class,
com.rometools.modules.base.types.YearType.class,
com.rometools.modules.content.ContentItem.class,
com.rometools.modules.georss.GeoRSSPoint.class,
com.rometools.modules.georss.geometries.Envelope.class,
com.rometools.modules.georss.geometries.LineString.class,
com.rometools.modules.georss.geometries.LinearRing.class,
com.rometools.modules.georss.geometries.Point.class,
com.rometools.modules.georss.geometries.Polygon.class,
com.rometools.modules.georss.geometries.Position.class,
com.rometools.modules.georss.geometries.PositionList.class,
com.rometools.modules.mediarss.types.MediaGroup.class,
com.rometools.modules.mediarss.types.Metadata.class,
com.rometools.modules.mediarss.types.Thumbnail.class,
com.rometools.modules.opensearch.entity.OSQuery.class,
com.rometools.modules.photocast.types.PhotoDate.class,
com.rometools.modules.sle.types.DateValue.class,
com.rometools.modules.sle.types.Group.class,
com.rometools.modules.sle.types.NumberValue.class,
com.rometools.modules.sle.types.Sort.class,
com.rometools.modules.sle.types.StringValue.class,
com.rometools.modules.yahooweather.types.Astronomy.class,
com.rometools.modules.yahooweather.types.Atmosphere.class,
com.rometools.modules.yahooweather.types.Condition.class,
com.rometools.modules.yahooweather.types.Forecast.class,
com.rometools.modules.yahooweather.types.Location.class,
com.rometools.modules.yahooweather.types.Units.class,
com.rometools.modules.yahooweather.types.Wind.class,
com.rometools.opml.feed.opml.Attribute.class,
com.rometools.opml.feed.opml.Opml.class,
com.rometools.opml.feed.opml.Outline.class,
com.rometools.rome.feed.atom.Category.class,
com.rometools.rome.feed.atom.Content.class,
com.rometools.rome.feed.atom.Entry.class,
com.rometools.rome.feed.atom.Feed.class,
com.rometools.rome.feed.atom.Generator.class,
com.rometools.rome.feed.atom.Link.class,
com.rometools.rome.feed.atom.Person.class,
com.rometools.rome.feed.rss.Category.class,
com.rometools.rome.feed.rss.Channel.class,
com.rometools.rome.feed.rss.Cloud.class,
com.rometools.rome.feed.rss.Content.class,
com.rometools.rome.feed.rss.Description.class,
com.rometools.rome.feed.rss.Enclosure.class,
com.rometools.rome.feed.rss.Guid.class,
com.rometools.rome.feed.rss.Image.class,
com.rometools.rome.feed.rss.Item.class,
com.rometools.rome.feed.rss.Source.class,
com.rometools.rome.feed.rss.TextInput.class,
com.rometools.rome.feed.synd.SyndLinkImpl.class,
com.rometools.rome.feed.synd.SyndPersonImpl.class,
java.util.ArrayList.class,
// rome modules
com.rometools.modules.sse.modules.Conflict.class, com.rometools.modules.sse.modules.Conflicts.class,
com.rometools.modules.cc.CreativeCommonsImpl.class, com.rometools.modules.feedpress.modules.FeedpressModuleImpl.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleImpl.class, com.rometools.modules.sse.modules.Sharing.class,
com.rometools.modules.georss.SimpleModuleImpl.class, com.rometools.modules.atom.modules.AtomLinkModuleImpl.class,
com.rometools.modules.itunes.EntryInformationImpl.class, com.rometools.modules.sse.modules.Update.class,
com.rometools.modules.photocast.PhotocastModuleImpl.class, com.rometools.modules.itunes.FeedInformationImpl.class,
com.rometools.modules.yahooweather.YWeatherModuleImpl.class, com.rometools.modules.feedburner.FeedBurnerImpl.class,
com.rometools.modules.sse.modules.Related.class, com.rometools.modules.fyyd.modules.FyydModuleImpl.class,
com.rometools.modules.psc.modules.PodloveSimpleChapterModuleImpl.class, com.rometools.modules.thr.ThreadingModuleImpl.class,
com.rometools.modules.sse.modules.Sync.class, com.rometools.modules.sle.SimpleListExtensionImpl.class,
com.rometools.modules.slash.SlashImpl.class, com.rometools.modules.sse.modules.History.class,
com.rometools.modules.georss.GMLModuleImpl.class, com.rometools.modules.base.CustomTagsImpl.class,
com.rometools.modules.base.GoogleBaseImpl.class, com.rometools.modules.sle.SleEntryImpl.class,
com.rometools.modules.mediarss.MediaEntryModuleImpl.class, com.rometools.modules.content.ContentModuleImpl.class,
com.rometools.modules.georss.W3CGeoModuleImpl.class, com.rometools.rome.feed.module.DCModuleImpl.class,
com.rometools.modules.mediarss.MediaModuleImpl.class, com.rometools.rome.feed.module.SyModuleImpl.class,
// rome modules
com.rometools.modules.sse.modules.Conflict.class,
com.rometools.modules.sse.modules.Conflicts.class,
com.rometools.modules.cc.CreativeCommonsImpl.class,
com.rometools.modules.feedpress.modules.FeedpressModuleImpl.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleImpl.class,
com.rometools.modules.sse.modules.Sharing.class,
com.rometools.modules.georss.SimpleModuleImpl.class,
com.rometools.modules.atom.modules.AtomLinkModuleImpl.class,
com.rometools.modules.itunes.EntryInformationImpl.class,
com.rometools.modules.sse.modules.Update.class,
com.rometools.modules.photocast.PhotocastModuleImpl.class,
com.rometools.modules.itunes.FeedInformationImpl.class,
com.rometools.modules.yahooweather.YWeatherModuleImpl.class,
com.rometools.modules.feedburner.FeedBurnerImpl.class,
com.rometools.modules.sse.modules.Related.class,
com.rometools.modules.fyyd.modules.FyydModuleImpl.class,
com.rometools.modules.psc.modules.PodloveSimpleChapterModuleImpl.class,
com.rometools.modules.thr.ThreadingModuleImpl.class,
com.rometools.modules.sse.modules.Sync.class,
com.rometools.modules.sle.SimpleListExtensionImpl.class,
com.rometools.modules.slash.SlashImpl.class,
com.rometools.modules.sse.modules.History.class,
com.rometools.modules.georss.GMLModuleImpl.class,
com.rometools.modules.base.CustomTagsImpl.class,
com.rometools.modules.base.GoogleBaseImpl.class,
com.rometools.modules.sle.SleEntryImpl.class,
com.rometools.modules.mediarss.MediaEntryModuleImpl.class,
com.rometools.modules.content.ContentModuleImpl.class,
com.rometools.modules.georss.W3CGeoModuleImpl.class,
com.rometools.rome.feed.module.DCModuleImpl.class,
com.rometools.modules.mediarss.MediaModuleImpl.class,
com.rometools.rome.feed.module.SyModuleImpl.class,
// extracted from all 3 rome.properties files of rome library
com.rometools.rome.io.impl.RSS090Parser.class, com.rometools.rome.io.impl.RSS091NetscapeParser.class,
com.rometools.rome.io.impl.RSS091UserlandParser.class, com.rometools.rome.io.impl.RSS092Parser.class,
com.rometools.rome.io.impl.RSS093Parser.class, com.rometools.rome.io.impl.RSS094Parser.class,
com.rometools.rome.io.impl.RSS10Parser.class, com.rometools.rome.io.impl.RSS20wNSParser.class,
com.rometools.rome.io.impl.RSS20Parser.class, com.rometools.rome.io.impl.Atom10Parser.class,
com.rometools.rome.io.impl.Atom03Parser.class,
com.rometools.rome.io.impl.SyModuleParser.class, com.rometools.rome.io.impl.DCModuleParser.class,
com.rometools.rome.io.impl.RSS090Generator.class, com.rometools.rome.io.impl.RSS091NetscapeGenerator.class,
com.rometools.rome.io.impl.RSS091UserlandGenerator.class, com.rometools.rome.io.impl.RSS092Generator.class,
com.rometools.rome.io.impl.RSS093Generator.class, com.rometools.rome.io.impl.RSS094Generator.class,
com.rometools.rome.io.impl.RSS10Generator.class, com.rometools.rome.io.impl.RSS20Generator.class,
com.rometools.rome.io.impl.Atom10Generator.class, com.rometools.rome.io.impl.Atom03Generator.class,
com.rometools.rome.feed.synd.impl.ConverterForAtom10.class, com.rometools.rome.feed.synd.impl.ConverterForAtom03.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS090.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Netscape.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Userland.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS092.class, com.rometools.rome.feed.synd.impl.ConverterForRSS093.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS094.class, com.rometools.rome.feed.synd.impl.ConverterForRSS10.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS20.class,
com.rometools.modules.mediarss.io.RSS20YahooParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.feedpress.io.FeedpressParser.class, com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.content.io.ContentModuleParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.atom.io.AtomModuleParser.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ItemParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.thr.io.ThreadingModuleParser.class, com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.sle.io.ModuleGenerator.class, com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class, com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class, com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class, com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.thr.io.ThreadingModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.opml.io.impl.OPML10Generator.class, com.rometools.opml.io.impl.OPML20Generator.class,
com.rometools.opml.io.impl.OPML10Parser.class, com.rometools.opml.io.impl.OPML20Parser.class,
com.rometools.opml.feed.synd.impl.ConverterForOPML10.class, com.rometools.opml.feed.synd.impl.ConverterForOPML20.class, })
public class NativeImageClasses {
}
// extracted from all 3 rome.properties files of rome library
com.rometools.rome.io.impl.RSS090Parser.class,
com.rometools.rome.io.impl.RSS091NetscapeParser.class,
com.rometools.rome.io.impl.RSS091UserlandParser.class,
com.rometools.rome.io.impl.RSS092Parser.class,
com.rometools.rome.io.impl.RSS093Parser.class,
com.rometools.rome.io.impl.RSS094Parser.class,
com.rometools.rome.io.impl.RSS10Parser.class,
com.rometools.rome.io.impl.RSS20wNSParser.class,
com.rometools.rome.io.impl.RSS20Parser.class,
com.rometools.rome.io.impl.Atom10Parser.class,
com.rometools.rome.io.impl.Atom03Parser.class,
com.rometools.rome.io.impl.SyModuleParser.class,
com.rometools.rome.io.impl.DCModuleParser.class,
com.rometools.rome.io.impl.RSS090Generator.class,
com.rometools.rome.io.impl.RSS091NetscapeGenerator.class,
com.rometools.rome.io.impl.RSS091UserlandGenerator.class,
com.rometools.rome.io.impl.RSS092Generator.class,
com.rometools.rome.io.impl.RSS093Generator.class,
com.rometools.rome.io.impl.RSS094Generator.class,
com.rometools.rome.io.impl.RSS10Generator.class,
com.rometools.rome.io.impl.RSS20Generator.class,
com.rometools.rome.io.impl.Atom10Generator.class,
com.rometools.rome.io.impl.Atom03Generator.class,
com.rometools.rome.feed.synd.impl.ConverterForAtom10.class,
com.rometools.rome.feed.synd.impl.ConverterForAtom03.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS090.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Netscape.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Userland.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS092.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS093.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS094.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS10.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS20.class,
com.rometools.modules.mediarss.io.RSS20YahooParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class,
com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.sle.io.ItemParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class,
com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class,
com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class,
com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.thr.io.ThreadingModuleParser.class,
com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.sle.io.ModuleGenerator.class,
com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class,
com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class,
com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.thr.io.ThreadingModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.opml.io.impl.OPML10Generator.class,
com.rometools.opml.io.impl.OPML20Generator.class,
com.rometools.opml.io.impl.OPML10Parser.class,
com.rometools.opml.io.impl.OPML20Parser.class,
com.rometools.opml.feed.synd.impl.ConverterForOPML10.class,
com.rometools.opml.feed.synd.impl.ConverterForOPML20.class,
})
public class NativeImageClasses {}

28
commafeed-server/src/main/java/com/commafeed/backend/Digests.java
View File

@@ -1,29 +1,27 @@
package com.commafeed.backend;
import java.nio.charset.StandardCharsets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import java.nio.charset.StandardCharsets;
import lombok.experimental.UtilityClass;
@UtilityClass
@SuppressWarnings("deprecation")
public class Digests {
public static String sha1Hex(byte[] input) {
return hashBytesToHex(Hashing.sha1(), input);
}
public static String sha1Hex(byte[] input) {
return hashBytesToHex(Hashing.sha1(), input);
}
public static String sha1Hex(String input) {
return hashBytesToHex(Hashing.sha1(), input.getBytes(StandardCharsets.UTF_8));
}
public static String sha1Hex(String input) {
return hashBytesToHex(Hashing.sha1(), input.getBytes(StandardCharsets.UTF_8));
}
public static String md5Hex(String input) {
return hashBytesToHex(Hashing.md5(), input.getBytes(StandardCharsets.UTF_8));
}
public static String md5Hex(String input) {
return hashBytesToHex(Hashing.md5(), input.getBytes(StandardCharsets.UTF_8));
}
private static String hashBytesToHex(HashFunction function, byte[] input) {
return function.hashBytes(input).toString();
}
private static String hashBytesToHex(HashFunction function, byte[] input) {
return function.hashBytes(input).toString();
}
}

246
commafeed-server/src/main/java/com/commafeed/backend/HttpClientFactory.java
View File

@@ -1,5 +1,12 @@
package com.commafeed.backend;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedVersion;
import com.google.common.net.HttpHeaders;
import inet.ipaddr.IPAddress;
import inet.ipaddr.IPAddressNetwork;
import inet.ipaddr.IPAddressString;
import jakarta.inject.Singleton;
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
@@ -8,9 +15,9 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.SequencedMap;
import java.util.zip.GZIPInputStream;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.apache5.util.Apache5SslUtils;
import org.apache.hc.client5.http.DnsResolver;
import org.apache.hc.client5.http.SystemDefaultDnsResolver;
import org.apache.hc.client5.http.config.ConnectionConfig;
@@ -35,137 +42,148 @@ import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
import org.brotli.dec.BrotliInputStream;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedVersion;
import com.google.common.net.HttpHeaders;
import inet.ipaddr.IPAddress;
import inet.ipaddr.IPAddressNetwork;
import inet.ipaddr.IPAddressString;
import lombok.RequiredArgsConstructor;
import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.apache5.util.Apache5SslUtils;
@Singleton
@RequiredArgsConstructor
public class HttpClientFactory {
private static final DnsResolver DNS_RESOLVER = SystemDefaultDnsResolver.INSTANCE;
private static final IPAddress CGNAT_RANGE = new IPAddressString("100.64.0.0/10").getAddress();
private static final DnsResolver DNS_RESOLVER = SystemDefaultDnsResolver.INSTANCE;
private static final IPAddress CGNAT_RANGE = new IPAddressString("100.64.0.0/10").getAddress();
private final CommaFeedConfiguration config;
private final CommaFeedVersion version;
private final CommaFeedConfiguration config;
private final CommaFeedVersion version;
public CloseableHttpClient newClient(int poolSize) {
PoolingHttpClientConnectionManager connectionManager = newConnectionManager(config, poolSize);
String userAgent = config.httpClient()
.userAgent()
.orElseGet(() -> String.format("CommaFeed/%s (https://github.com/Athou/commafeed)", version.getVersion()));
return newClient(config, connectionManager, userAgent);
}
public CloseableHttpClient newClient(int poolSize) {
PoolingHttpClientConnectionManager connectionManager =
newConnectionManager(config, poolSize);
String userAgent =
config.httpClient()
.userAgent()
.orElseGet(
() ->
String.format(
"CommaFeed/%s (https://github.com/Athou/commafeed)",
version.getVersion()));
return newClient(config, connectionManager, userAgent);
}
private CloseableHttpClient newClient(CommaFeedConfiguration config, HttpClientConnectionManager connectionManager, String userAgent) {
List<Header> headers = new ArrayList<>();
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
private CloseableHttpClient newClient(
CommaFeedConfiguration config,
HttpClientConnectionManager connectionManager,
String userAgent) {
List<Header> headers = new ArrayList<>();
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
SequencedMap<String, InputStreamFactory> contentDecoderMap = new LinkedHashMap<>();
contentDecoderMap.put(ContentCoding.GZIP.token(), GZIPInputStream::new);
contentDecoderMap.put(ContentCoding.DEFLATE.token(), DeflateInputStream::new);
contentDecoderMap.put(ContentCoding.BROTLI.token(), BrotliInputStream::new);
SequencedMap<String, InputStreamFactory> contentDecoderMap = new LinkedHashMap<>();
contentDecoderMap.put(ContentCoding.GZIP.token(), GZIPInputStream::new);
contentDecoderMap.put(ContentCoding.DEFLATE.token(), DeflateInputStream::new);
contentDecoderMap.put(ContentCoding.BROTLI.token(), BrotliInputStream::new);
RedirectStrategy redirectStrategy = config.httpClient().blockLocalAddresses()
? new BlockLocalAddressesRedirectStrategy(DNS_RESOLVER)
: new DefaultRedirectStrategy();
RedirectStrategy redirectStrategy =
config.httpClient().blockLocalAddresses()
? new BlockLocalAddressesRedirectStrategy(DNS_RESOLVER)
: new DefaultRedirectStrategy();
return HttpClientBuilder.create()
.disableConnectionState()
.useSystemProperties()
.disableAutomaticRetries()
.disableCookieManagement()
.setUserAgent(userAgent)
.setDefaultHeaders(headers)
.setConnectionManager(connectionManager)
.evictExpiredConnections()
.evictIdleConnections(TimeValue.of(config.httpClient().idleConnectionsEvictionInterval()))
.setContentDecoderRegistry(new LinkedHashMap<>(contentDecoderMap))
.setRedirectStrategy(redirectStrategy)
.build();
}
return HttpClientBuilder.create()
.disableConnectionState()
.useSystemProperties()
.disableAutomaticRetries()
.disableCookieManagement()
.setUserAgent(userAgent)
.setDefaultHeaders(headers)
.setConnectionManager(connectionManager)
.evictExpiredConnections()
.evictIdleConnections(
TimeValue.of(config.httpClient().idleConnectionsEvictionInterval()))
.setContentDecoderRegistry(new LinkedHashMap<>(contentDecoderMap))
.setRedirectStrategy(redirectStrategy)
.build();
}
private PoolingHttpClientConnectionManager newConnectionManager(CommaFeedConfiguration config, int poolSize) {
SSLFactory sslFactory = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
DnsResolver dnsResolver = config.httpClient().blockLocalAddresses() ? new BlockLocalAddressesDnsResolver(DNS_RESOLVER)
: DNS_RESOLVER;
private PoolingHttpClientConnectionManager newConnectionManager(
CommaFeedConfiguration config, int poolSize) {
SSLFactory sslFactory =
SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
DnsResolver dnsResolver =
config.httpClient().blockLocalAddresses()
? new BlockLocalAddressesDnsResolver(DNS_RESOLVER)
: DNS_RESOLVER;
return PoolingHttpClientConnectionManagerBuilder.create()
.setTlsSocketStrategy(Apache5SslUtils.toTlsSocketStrategy(sslFactory))
.setDefaultConnectionConfig(ConnectionConfig.custom()
.setConnectTimeout(Timeout.of(config.httpClient().connectTimeout()))
.setSocketTimeout(Timeout.of(config.httpClient().socketTimeout()))
.setTimeToLive(Timeout.of(config.httpClient().connectionTimeToLive()))
.build())
.setDefaultTlsConfig(TlsConfig.custom().setHandshakeTimeout(Timeout.of(config.httpClient().sslHandshakeTimeout())).build())
.setMaxConnPerRoute(poolSize)
.setMaxConnTotal(poolSize)
.setDnsResolver(dnsResolver)
.build();
return PoolingHttpClientConnectionManagerBuilder.create()
.setTlsSocketStrategy(Apache5SslUtils.toTlsSocketStrategy(sslFactory))
.setDefaultConnectionConfig(
ConnectionConfig.custom()
.setConnectTimeout(Timeout.of(config.httpClient().connectTimeout()))
.setSocketTimeout(Timeout.of(config.httpClient().socketTimeout()))
.setTimeToLive(
Timeout.of(config.httpClient().connectionTimeToLive()))
.build())
.setDefaultTlsConfig(
TlsConfig.custom()
.setHandshakeTimeout(
Timeout.of(config.httpClient().sslHandshakeTimeout()))
.build())
.setMaxConnPerRoute(poolSize)
.setMaxConnTotal(poolSize)
.setDnsResolver(dnsResolver)
.build();
}
}
private static boolean isLocalAddress(InetAddress address) {
IPAddress ip = new IPAddressNetwork.IPAddressGenerator().from(address);
return ip.isLocal() || ip.isLoopback() || ip.isMulticast() || CGNAT_RANGE.contains(ip);
}
private static boolean isLocalAddress(InetAddress address) {
IPAddress ip = new IPAddressNetwork.IPAddressGenerator().from(address);
return ip.isLocal() || ip.isLoopback() || ip.isMulticast() || CGNAT_RANGE.contains(ip);
}
private record BlockLocalAddressesDnsResolver(DnsResolver delegate) implements DnsResolver {
@Override
public InetAddress[] resolve(String host) throws UnknownHostException {
InetAddress[] addresses = delegate.resolve(host);
for (InetAddress addr : addresses) {
if (isLocalAddress(addr)) {
throw new UnknownHostException(
"Access to local address blocked: " + addr.getHostAddress());
}
}
return addresses;
}
private record BlockLocalAddressesDnsResolver(DnsResolver delegate) implements DnsResolver {
@Override
public InetAddress[] resolve(String host) throws UnknownHostException {
InetAddress[] addresses = delegate.resolve(host);
for (InetAddress addr : addresses) {
if (isLocalAddress(addr)) {
throw new UnknownHostException("Access to local address blocked: " + addr.getHostAddress());
}
}
return addresses;
}
@Override
public String resolveCanonicalHostname(String host) throws UnknownHostException {
return delegate.resolveCanonicalHostname(host);
}
}
@Override
public String resolveCanonicalHostname(String host) throws UnknownHostException {
return delegate.resolveCanonicalHostname(host);
}
}
@RequiredArgsConstructor
private static class BlockLocalAddressesRedirectStrategy extends DefaultRedirectStrategy {
@RequiredArgsConstructor
private static class BlockLocalAddressesRedirectStrategy extends DefaultRedirectStrategy {
private final DnsResolver delegate;
private final DnsResolver delegate;
@Override
public URI getLocationURI(HttpRequest request, HttpResponse response, HttpContext context)
throws HttpException {
URI redirectUri = super.getLocationURI(request, response, context);
@Override
public URI getLocationURI(HttpRequest request, HttpResponse response, HttpContext context) throws HttpException {
URI redirectUri = super.getLocationURI(request, response, context);
String host = redirectUri.getHost();
if (host == null) {
throw new HttpException("Redirect URI does not have a host: " + redirectUri);
}
String host = redirectUri.getHost();
if (host == null) {
throw new HttpException("Redirect URI does not have a host: " + redirectUri);
}
InetAddress[] addresses;
try {
addresses = delegate.resolve(host);
} catch (UnknownHostException e) {
throw new HttpException("Unknown host: " + host);
}
InetAddress[] addresses;
try {
addresses = delegate.resolve(host);
} catch (UnknownHostException e) {
throw new HttpException("Unknown host: " + host);
}
for (InetAddress addr : addresses) {
if (isLocalAddress(addr)) {
throw new HttpException("Access to local address blocked: " + addr.getHostAddress());
}
}
return redirectUri;
}
}
for (InetAddress addr : addresses) {
if (isLocalAddress(addr)) {
throw new HttpException(
"Access to local address blocked: " + addr.getHostAddress());
}
}
return redirectUri;
}
}
}

524
commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java
View File

@@ -1,5 +1,15 @@
package com.commafeed.backend;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.HttpClientCache;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Iterables;
import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.CacheControl;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
@@ -8,10 +18,12 @@ import java.time.Instant;
import java.time.InstantSource;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.CacheControl;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Lombok;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.config.RequestConfig;
@@ -27,282 +39,326 @@ import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.util.Timeout;
import org.jboss.resteasy.reactive.common.headers.CacheControlDelegate;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.HttpClientCache;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Iterables;
import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Lombok;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Smart HTTP getter: handles gzip, ssl, last modified and etag headers
*/
/** Smart HTTP getter: handles gzip, ssl, last modified and etag headers */
@Singleton
@Slf4j
public class HttpGetter {
private final CommaFeedConfiguration config;
private final InstantSource instantSource;
private final CloseableHttpClient client;
private final Cache<HttpRequest, HttpResponse> cache;
private final CommaFeedConfiguration config;
private final InstantSource instantSource;
private final CloseableHttpClient client;
private final Cache<HttpRequest, HttpResponse> cache;
public HttpGetter(CommaFeedConfiguration config, InstantSource instantSource, HttpClientFactory httpClientFactory,
MetricRegistry metrics) {
this.config = config;
this.instantSource = instantSource;
this.client = httpClientFactory.newClient(config.feedRefresh().httpThreads());
this.cache = newCache(config);
public HttpGetter(
CommaFeedConfiguration config,
InstantSource instantSource,
HttpClientFactory httpClientFactory,
MetricRegistry metrics) {
this.config = config;
this.instantSource = instantSource;
this.client = httpClientFactory.newClient(config.feedRefresh().httpThreads());
this.cache = newCache(config);
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "size"), () -> cache == null ? 0 : cache.size());
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "memoryUsage"),
() -> cache == null ? 0 : cache.asMap().values().stream().mapToInt(e -> ArrayUtils.getLength(e.content)).sum());
}
metrics.registerGauge(
MetricRegistry.name(getClass(), "cache", "size"),
() -> cache == null ? 0 : cache.size());
metrics.registerGauge(
MetricRegistry.name(getClass(), "cache", "memoryUsage"),
() ->
cache == null
? 0
: cache.asMap().values().stream()
.mapToInt(e -> ArrayUtils.getLength(e.content))
.sum());
}
public HttpResult get(String url) throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
return get(HttpRequest.builder(url).build());
}
public HttpResult get(String url)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
return get(HttpRequest.builder(url).build());
}
public HttpResult get(HttpRequest request)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
URI uri = URI.create(request.getUrl());
ensureHttpScheme(uri.getScheme());
public HttpResult get(HttpRequest request)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
URI uri = URI.create(request.getUrl());
ensureHttpScheme(uri.getScheme());
final HttpResponse response;
if (cache == null) {
response = invoke(request);
} else {
try {
response = cache.get(request, () -> invoke(request));
} catch (ExecutionException e) {
if (e.getCause() instanceof IOException ioe) {
throw ioe;
} else {
throw Lombok.sneakyThrow(e);
}
}
}
final HttpResponse response;
if (cache == null) {
response = invoke(request);
} else {
try {
response = cache.get(request, () -> invoke(request));
} catch (ExecutionException e) {
if (e.getCause() instanceof IOException ioe) {
throw ioe;
} else {
throw Lombok.sneakyThrow(e);
}
}
}
int code = response.code();
if (code == HttpStatus.SC_TOO_MANY_REQUESTS || code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.retryAfter() != null) {
throw new TooManyRequestsException(response.retryAfter());
}
int code = response.code();
if (code == HttpStatus.SC_TOO_MANY_REQUESTS
|| code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.retryAfter() != null) {
throw new TooManyRequestsException(response.retryAfter());
}
if (code == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received");
}
if (code == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received");
}
if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);
}
if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);
}
String lastModifiedHeader = response.lastModifiedHeader();
String eTagHeader = response.eTagHeader();
String lastModifiedHeader = response.lastModifiedHeader();
String eTagHeader = response.eTagHeader();
Duration validFor = Optional.ofNullable(response.cacheControl())
.filter(cc -> cc.getMaxAge() >= 0)
.map(cc -> Duration.ofSeconds(cc.getMaxAge()))
.orElse(Duration.ZERO);
Duration validFor =
Optional.ofNullable(response.cacheControl())
.filter(cc -> cc.getMaxAge() >= 0)
.map(cc -> Duration.ofSeconds(cc.getMaxAge()))
.orElse(Duration.ZERO);
return new HttpResult(response.content(), response.contentType(), lastModifiedHeader, eTagHeader, response.urlAfterRedirect(),
validFor);
}
return new HttpResult(
response.content(),
response.contentType(),
lastModifiedHeader,
eTagHeader,
response.urlAfterRedirect(),
validFor);
}
private void ensureHttpScheme(String scheme) throws SchemeNotAllowedException {
if (!"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme)) {
throw new SchemeNotAllowedException(scheme);
}
}
private void ensureHttpScheme(String scheme) throws SchemeNotAllowedException {
if (!"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme)) {
throw new SchemeNotAllowedException(scheme);
}
}
private HttpResponse invoke(HttpRequest request) throws IOException {
log.debug("fetching {}", request.getUrl());
private HttpResponse invoke(HttpRequest request) throws IOException {
log.debug("fetching {}", request.getUrl());
HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
// causes issues with some feeds
// see https://github.com/Athou/commafeed/issues/1572
// and https://issues.apache.org/jira/browse/HTTPCLIENT-2344
.setProtocolUpgradeEnabled(false)
.build());
HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(
RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
// causes issues with some feeds
// see https://github.com/Athou/commafeed/issues/1572
// and https://issues.apache.org/jira/browse/HTTPCLIENT-2344
.setProtocolUpgradeEnabled(false)
.build());
return client.execute(request.toClassicHttpRequest(), context, resp -> {
byte[] content = resp.getEntity() == null ? null
: toByteArray(resp.getEntity(), config.httpClient().maxResponseSize().asLongValue());
int code = resp.getCode();
String lastModifiedHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.LAST_MODIFIED))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
String eTagHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.ETAG))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
return client.execute(
request.toClassicHttpRequest(),
context,
resp -> {
byte[] content =
resp.getEntity() == null
? null
: toByteArray(
resp.getEntity(),
config.httpClient().maxResponseSize().asLongValue());
int code = resp.getCode();
String lastModifiedHeader =
Optional.ofNullable(resp.getFirstHeader(HttpHeaders.LAST_MODIFIED))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
String eTagHeader =
Optional.ofNullable(resp.getFirstHeader(HttpHeaders.ETAG))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
CacheControl cacheControl = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.CACHE_CONTROL))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(HttpGetter::toCacheControl)
.orElse(null);
CacheControl cacheControl =
Optional.ofNullable(resp.getFirstHeader(HttpHeaders.CACHE_CONTROL))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(HttpGetter::toCacheControl)
.orElse(null);
Instant retryAfter = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(this::toInstant)
.orElse(null);
Instant retryAfter =
Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(this::toInstant)
.orElse(null);
String contentType = Optional.ofNullable(resp.getEntity()).map(HttpEntity::getContentType).orElse(null);
String urlAfterRedirect = Optional.ofNullable(context.getRedirectLocations())
.map(RedirectLocations::getAll)
.map(l -> Iterables.getLast(l, null))
.map(URI::toString)
.orElse(request.getUrl());
String contentType =
Optional.ofNullable(resp.getEntity())
.map(HttpEntity::getContentType)
.orElse(null);
String urlAfterRedirect =
Optional.ofNullable(context.getRedirectLocations())
.map(RedirectLocations::getAll)
.map(l -> Iterables.getLast(l, null))
.map(URI::toString)
.orElse(request.getUrl());
return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, retryAfter, content, contentType, urlAfterRedirect);
});
}
return new HttpResponse(
code,
lastModifiedHeader,
eTagHeader,
cacheControl,
retryAfter,
content,
contentType,
urlAfterRedirect);
});
}
private static CacheControl toCacheControl(String headerValue) {
try {
return CacheControlDelegate.INSTANCE.fromString(headerValue);
} catch (Exception e) {
log.debug("Invalid Cache-Control header: {}", headerValue);
return null;
}
}
private static CacheControl toCacheControl(String headerValue) {
try {
return CacheControlDelegate.INSTANCE.fromString(headerValue);
} catch (Exception e) {
log.debug("Invalid Cache-Control header: {}", headerValue);
return null;
}
}
private Instant toInstant(String headerValue) {
if (headerValue == null) {
return null;
}
private Instant toInstant(String headerValue) {
if (headerValue == null) {
return null;
}
if (StringUtils.isNumeric(headerValue)) {
return instantSource.instant().plusSeconds(Long.parseLong(headerValue));
}
if (StringUtils.isNumeric(headerValue)) {
return instantSource.instant().plusSeconds(Long.parseLong(headerValue));
}
return DateUtils.parseStandardDate(headerValue);
}
return DateUtils.parseStandardDate(headerValue);
}
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
if (entity.getContentLength() > maxBytes) {
throw new IOException(
"Response size (%s bytes) exceeds the maximum allowed size (%s bytes)".formatted(entity.getContentLength(), maxBytes));
}
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
if (entity.getContentLength() > maxBytes) {
throw new IOException(
"Response size (%s bytes) exceeds the maximum allowed size (%s bytes)"
.formatted(entity.getContentLength(), maxBytes));
}
try (InputStream input = entity.getContent()) {
if (input == null) {
return null;
}
try (InputStream input = entity.getContent()) {
if (input == null) {
return null;
}
byte[] bytes = ByteStreams.limit(input, maxBytes + 1).readAllBytes();
if (bytes.length > maxBytes) {
throw new IOException("Response size exceeds the maximum allowed size (%s bytes)".formatted(maxBytes));
}
return bytes;
}
}
byte[] bytes = ByteStreams.limit(input, maxBytes + 1).readAllBytes();
if (bytes.length > maxBytes) {
throw new IOException(
"Response size exceeds the maximum allowed size (%s bytes)"
.formatted(maxBytes));
}
return bytes;
}
}
private static Cache<HttpRequest, HttpResponse> newCache(CommaFeedConfiguration config) {
HttpClientCache cacheConfig = config.httpClient().cache();
if (!cacheConfig.enabled()) {
return null;
}
private static Cache<HttpRequest, HttpResponse> newCache(CommaFeedConfiguration config) {
HttpClientCache cacheConfig = config.httpClient().cache();
if (!cacheConfig.enabled()) {
return null;
}
return CacheBuilder.newBuilder()
.weigher((HttpRequest key, HttpResponse value) -> value.content() != null ? value.content().length : 0)
.maximumWeight(cacheConfig.maximumMemorySize().asLongValue())
.expireAfterWrite(cacheConfig.expiration())
.build();
}
return CacheBuilder.newBuilder()
.weigher(
(HttpRequest key, HttpResponse value) ->
value.content() != null ? value.content().length : 0)
.maximumWeight(cacheConfig.maximumMemorySize().asLongValue())
.expireAfterWrite(cacheConfig.expiration())
.build();
}
public static class SchemeNotAllowedException extends Exception {
private static final long serialVersionUID = 1L;
public static class SchemeNotAllowedException extends Exception {
private static final long serialVersionUID = 1L;
public SchemeNotAllowedException(String scheme) {
super("Scheme not allowed: " + scheme);
}
}
public SchemeNotAllowedException(String scheme) {
super("Scheme not allowed: " + scheme);
}
}
@Getter
public static class NotModifiedException extends Exception {
private static final long serialVersionUID = 1L;
@Getter
public static class NotModifiedException extends Exception {
private static final long serialVersionUID = 1L;
/**
* if the value of this header changed, this is its new value
*/
private final String newLastModifiedHeader;
/** if the value of this header changed, this is its new value */
private final String newLastModifiedHeader;
/**
* if the value of this header changed, this is its new value
*/
private final String newEtagHeader;
/** if the value of this header changed, this is its new value */
private final String newEtagHeader;
public NotModifiedException(String message) {
this(message, null, null);
}
public NotModifiedException(String message) {
this(message, null, null);
}
public NotModifiedException(String message, String newLastModifiedHeader, String newEtagHeader) {
super(message);
this.newLastModifiedHeader = newLastModifiedHeader;
this.newEtagHeader = newEtagHeader;
}
}
public NotModifiedException(
String message, String newLastModifiedHeader, String newEtagHeader) {
super(message);
this.newLastModifiedHeader = newLastModifiedHeader;
this.newEtagHeader = newEtagHeader;
}
}
@RequiredArgsConstructor
@Getter
public static class TooManyRequestsException extends Exception {
private static final long serialVersionUID = 1L;
@RequiredArgsConstructor
@Getter
public static class TooManyRequestsException extends Exception {
private static final long serialVersionUID = 1L;
private final Instant retryAfter;
}
private final Instant retryAfter;
}
@Getter
public static class HttpResponseException extends IOException {
private static final long serialVersionUID = 1L;
@Getter
public static class HttpResponseException extends IOException {
private static final long serialVersionUID = 1L;
private final int code;
private final int code;
public HttpResponseException(int code, String message) {
super(message);
this.code = code;
}
}
public HttpResponseException(int code, String message) {
super(message);
this.code = code;
}
}
@Builder(builderMethodName = "")
@EqualsAndHashCode
@Getter
public static class HttpRequest {
private String url;
private String lastModified;
private String eTag;
@Builder(builderMethodName = "")
@EqualsAndHashCode
@Getter
public static class HttpRequest {
private String url;
private String lastModified;
private String eTag;
public static HttpRequestBuilder builder(String url) {
return new HttpRequestBuilder().url(url);
}
public static HttpRequestBuilder builder(String url) {
return new HttpRequestBuilder().url(url);
}
public ClassicHttpRequest toClassicHttpRequest() {
ClassicHttpRequest req = ClassicRequestBuilder.get(url).build();
if (lastModified != null) {
req.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
}
if (eTag != null) {
req.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
}
return req;
}
}
public ClassicHttpRequest toClassicHttpRequest() {
ClassicHttpRequest req = ClassicRequestBuilder.get(url).build();
if (lastModified != null) {
req.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
}
if (eTag != null) {
req.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
}
return req;
}
}
private record HttpResponse(int code, String lastModifiedHeader, String eTagHeader, CacheControl cacheControl, Instant retryAfter,
byte[] content, String contentType, String urlAfterRedirect) {}
public record HttpResult(byte[] content, String contentType, String lastModifiedSince, String eTag, String urlAfterRedirect,
Duration validFor) {}
private record HttpResponse(
int code,
String lastModifiedHeader,
String eTagHeader,
CacheControl cacheControl,
Instant retryAfter,
byte[] content,
String contentType,
String urlAfterRedirect) {}
public record HttpResult(
byte[] content,
String contentType,
String lastModifiedSince,
String eTag,
String urlAfterRedirect,
Duration validFor) {}
}

155
commafeed-server/src/main/java/com/commafeed/backend/Urls.java
View File

@@ -2,105 +2,104 @@ package com.commafeed.backend;
import java.net.URI;
import java.util.regex.Pattern;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.Strings;
import org.netpreserve.urlcanon.Canonicalizer;
import org.netpreserve.urlcanon.ParsedUrl;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
@UtilityClass
@Slf4j
public class Urls {
private static final Pattern QUESTION_MARK = Pattern.compile(Pattern.quote("?"));
private static final Pattern QUESTION_MARK = Pattern.compile(Pattern.quote("?"));
public static boolean isHttp(String url) {
return url.startsWith("http://");
}
public static boolean isHttp(String url) {
return url.startsWith("http://");
}
public static boolean isHttps(String url) {
return url.startsWith("https://");
}
public static boolean isHttps(String url) {
return url.startsWith("https://");
}
public static boolean isAbsolute(String url) {
return isHttp(url) || isHttps(url);
}
public static boolean isAbsolute(String url) {
return isHttp(url) || isHttps(url);
}
/**
*
* @param relativeUrl
* the url of the entry
* @param feedLink
* the url of the feed as described in the feed
* @param feedUrl
* the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry
*/
public static String toAbsolute(String relativeUrl, String feedLink, String feedUrl) {
String baseUrl = (feedLink != null && isAbsolute(feedLink)) ? feedLink : feedUrl;
if (baseUrl == null) {
return null;
}
/**
* @param relativeUrl the url of the entry
* @param feedLink the url of the feed as described in the feed
* @param feedUrl the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry
*/
public static String toAbsolute(String relativeUrl, String feedLink, String feedUrl) {
String baseUrl = (feedLink != null && isAbsolute(feedLink)) ? feedLink : feedUrl;
if (baseUrl == null) {
return null;
}
try {
return URI.create(baseUrl).resolve(relativeUrl).toString();
} catch (IllegalArgumentException e) {
log.debug("Unable to create absolute url from relative url: {} base: {}", relativeUrl, baseUrl, e);
return null;
}
}
try {
return URI.create(baseUrl).resolve(relativeUrl).toString();
} catch (IllegalArgumentException e) {
log.debug(
"Unable to create absolute url from relative url: {} base: {}",
relativeUrl,
baseUrl,
e);
return null;
}
}
public static String removeTrailingSlash(String url) {
if (url == null) {
return null;
}
public static String removeTrailingSlash(String url) {
if (url == null) {
return null;
}
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}
return url;
}
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}
return url;
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates
*/
public static String normalize(String url) {
if (url == null) {
return null;
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to
* identify a feed and avoid duplicates
*/
public static String normalize(String url) {
if (url == null) {
return null;
}
ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
String normalized = parsedUrl.toString();
if (normalized == null) {
normalized = url;
}
ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
String normalized = parsedUrl.toString();
if (normalized == null) {
normalized = url;
}
// convert to lower case, the url probably won't work in some cases
// after that but we don't care we just want to compare urls to avoid
// duplicates
normalized = normalized.toLowerCase();
// convert to lower case, the url probably won't work in some cases
// after that but we don't care we just want to compare urls to avoid
// duplicates
normalized = normalized.toLowerCase();
// store all urls as http
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
// store all urls as http
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
// remove the www. part
normalized = normalized.replace("//www.", "//");
// remove the www. part
normalized = normalized.replace("//www.", "//");
// feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedburner feeds have a special treatment
if (QUESTION_MARK.split(normalized)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = QUESTION_MARK.split(normalized)[0];
normalized = Strings.CS.removeEnd(normalized, "/");
}
// feedburner feeds have a special treatment
if (QUESTION_MARK.split(normalized)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = QUESTION_MARK.split(normalized)[0];
normalized = Strings.CS.removeEnd(normalized, "/");
}
return normalized;
}
return normalized;
}
}

109
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java
View File

@@ -1,71 +1,76 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.QFeedCategory;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
import com.querydsl.core.types.Predicate;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
import java.util.Objects;
@Singleton
public class FeedCategoryDAO extends GenericDAO<FeedCategory> {
private static final QFeedCategory CATEGORY = QFeedCategory.feedCategory;
private static final QFeedCategory CATEGORY = QFeedCategory.feedCategory;
public FeedCategoryDAO(EntityManager entityManager) {
super(entityManager, FeedCategory.class);
}
public FeedCategoryDAO(EntityManager entityManager) {
super(entityManager, FeedCategory.class);
}
public List<FeedCategory> findAll(User user) {
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user)).join(CATEGORY.user, QUser.user).fetchJoin().fetch();
}
public List<FeedCategory> findAll(User user) {
return query().selectFrom(CATEGORY)
.where(CATEGORY.user.eq(user))
.join(CATEGORY.user, QUser.user)
.fetchJoin()
.fetch();
}
public FeedCategory findById(User user, Long id) {
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.id.eq(id)).fetchOne();
}
public FeedCategory findById(User user, Long id) {
return query().selectFrom(CATEGORY)
.where(CATEGORY.user.eq(user), CATEGORY.id.eq(id))
.fetchOne();
}
public FeedCategory findByName(User user, String name, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.name.eq(name), parentPredicate).fetchOne();
}
public FeedCategory findByName(User user, String name, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY)
.where(CATEGORY.user.eq(user), CATEGORY.name.eq(name), parentPredicate)
.fetchOne();
}
public List<FeedCategory> findByParent(User user, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), parentPredicate).fetch();
}
public List<FeedCategory> findByParent(User user, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), parentPredicate).fetch();
}
public List<FeedCategory> findAllChildrenCategories(User user, FeedCategory parent) {
return findAll(user).stream().filter(c -> isChild(c, parent)).toList();
}
private boolean isChild(FeedCategory child, FeedCategory parent) {
if (parent == null) {
return true;
}
boolean isChild = false;
while (child != null) {
if (Objects.equals(child.getId(), parent.getId())) {
isChild = true;
break;
}
child = child.getParent();
}
return isChild;
}
public List<FeedCategory> findAllChildrenCategories(User user, FeedCategory parent) {
return findAll(user).stream().filter(c -> isChild(c, parent)).toList();
}
private boolean isChild(FeedCategory child, FeedCategory parent) {
if (parent == null) {
return true;
}
boolean isChild = false;
while (child != null) {
if (Objects.equals(child.getId(), parent.getId())) {
isChild = true;
break;
}
child = child.getParent();
}
return isChild;
}
}

93
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedDAO.java
View File

@@ -1,64 +1,69 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.apache.commons.lang3.Strings;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.QFeed;
import com.commafeed.backend.model.QFeedSubscription;
import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.impl.JPAQuery;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.time.Instant;
import java.util.List;
import org.apache.commons.lang3.Strings;
@Singleton
public class FeedDAO extends GenericDAO<Feed> {
private static final QFeed FEED = QFeed.feed;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private static final QFeed FEED = QFeed.feed;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
public FeedDAO(EntityManager entityManager) {
super(entityManager, Feed.class);
}
public FeedDAO(EntityManager entityManager) {
super(entityManager, Feed.class);
}
public List<Feed> findByIds(List<Long> id) {
return query().selectFrom(FEED).where(FEED.id.in(id)).fetch();
}
public List<Feed> findByIds(List<Long> id) {
return query().selectFrom(FEED).where(FEED.id.in(id)).fetch();
}
public List<Feed> findNextUpdatable(int count, Instant lastLoginThreshold) {
JPAQuery<Feed> query = query().selectFrom(FEED)
.distinct()
// join on subscriptions to only refresh feeds that have subscribers
.join(SUBSCRIPTION)
.on(SUBSCRIPTION.feed.eq(FEED))
.where(FEED.disabledUntil.isNull().or(FEED.disabledUntil.lt(Instant.now())));
public List<Feed> findNextUpdatable(int count, Instant lastLoginThreshold) {
JPAQuery<Feed> query =
query().selectFrom(FEED)
.distinct()
// join on subscriptions to only refresh feeds that have subscribers
.join(SUBSCRIPTION)
.on(SUBSCRIPTION.feed.eq(FEED))
.where(
FEED.disabledUntil
.isNull()
.or(FEED.disabledUntil.lt(Instant.now())));
if (lastLoginThreshold != null) {
query.join(SUBSCRIPTION.user).where(SUBSCRIPTION.user.lastLogin.gt(lastLoginThreshold));
}
if (lastLoginThreshold != null) {
query.join(SUBSCRIPTION.user).where(SUBSCRIPTION.user.lastLogin.gt(lastLoginThreshold));
}
return query.orderBy(FEED.disabledUntil.asc()).limit(count).fetch();
}
return query.orderBy(FEED.disabledUntil.asc()).limit(count).fetch();
}
public void setDisabledUntil(List<Long> feedIds, Instant date) {
updateQuery(FEED).set(FEED.disabledUntil, date).where(FEED.id.in(feedIds)).execute();
}
public void setDisabledUntil(List<Long> feedIds, Instant date) {
updateQuery(FEED).set(FEED.disabledUntil, date).where(FEED.id.in(feedIds)).execute();
}
public Feed findByUrl(String normalizedUrl, String normalizedUrlHash) {
return query().selectFrom(FEED)
.where(FEED.normalizedUrlHash.eq(normalizedUrlHash))
.fetch()
.stream()
.filter(f -> Strings.CS.equals(normalizedUrl, f.getNormalizedUrl()))
.findFirst()
.orElse(null);
}
public Feed findByUrl(String normalizedUrl, String normalizedUrlHash) {
return query()
.selectFrom(FEED)
.where(FEED.normalizedUrlHash.eq(normalizedUrlHash))
.fetch()
.stream()
.filter(f -> Strings.CS.equals(normalizedUrl, f.getNormalizedUrl()))
.findFirst()
.orElse(null);
}
public List<Feed> findWithoutSubscriptions(int max) {
QFeedSubscription sub = QFeedSubscription.feedSubscription;
return query().selectFrom(FEED).where(JPAExpressions.selectOne().from(sub).where(sub.feed.eq(FEED)).notExists()).limit(max).fetch();
}
public List<Feed> findWithoutSubscriptions(int max) {
QFeedSubscription sub = QFeedSubscription.feedSubscription;
return query().selectFrom(FEED)
.where(JPAExpressions.selectOne().from(sub).where(sub.feed.eq(FEED)).notExists())
.limit(max)
.fetch();
}
}

48
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java
View File

@@ -1,36 +1,38 @@
package com.commafeed.backend.dao;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
@Singleton
public class FeedEntryContentDAO extends GenericDAO<FeedEntryContent> {
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
public FeedEntryContentDAO(EntityManager entityManager) {
super(entityManager, FeedEntryContent.class);
}
public FeedEntryContentDAO(EntityManager entityManager) {
super(entityManager, FeedEntryContent.class);
}
public List<FeedEntryContent> findExisting(String contentHash, String titleHash) {
return query().select(CONTENT).from(CONTENT).where(CONTENT.contentHash.eq(contentHash), CONTENT.titleHash.eq(titleHash)).fetch();
}
public List<FeedEntryContent> findExisting(String contentHash, String titleHash) {
return query().select(CONTENT)
.from(CONTENT)
.where(CONTENT.contentHash.eq(contentHash), CONTENT.titleHash.eq(titleHash))
.fetch();
}
public long deleteWithoutEntries(int max) {
List<Long> ids = query().select(CONTENT.id)
.from(CONTENT)
.leftJoin(ENTRY)
.on(ENTRY.content.id.eq(CONTENT.id))
.where(ENTRY.id.isNull())
.limit(max)
.fetch();
return deleteQuery(CONTENT).where(CONTENT.id.in(ids)).execute();
}
public long deleteWithoutEntries(int max) {
List<Long> ids =
query().select(CONTENT.id)
.from(CONTENT)
.leftJoin(ENTRY)
.on(ENTRY.content.id.eq(CONTENT.id))
.where(ENTRY.id.isNull())
.limit(max)
.fetch();
return deleteQuery(CONTENT).where(CONTENT.id.in(ids)).execute();
}
}

144
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
View File

@@ -1,14 +1,5 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.QFeedEntry;
@@ -16,83 +7,98 @@ import com.google.common.collect.Lists;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.NumberExpression;
import com.querydsl.jpa.impl.JPAQuery;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@Singleton
public class FeedEntryDAO extends GenericDAO<FeedEntry> {
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final int IN_CLAUSE_BATCH_SIZE = 1000;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final int IN_CLAUSE_BATCH_SIZE = 1000;
public FeedEntryDAO(EntityManager entityManager) {
super(entityManager, FeedEntry.class);
}
public FeedEntryDAO(EntityManager entityManager) {
super(entityManager, FeedEntry.class);
}
public FeedEntry findExisting(String guidHash, Feed feed) {
return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne();
}
public FeedEntry findExisting(String guidHash, Feed feed) {
return query().select(ENTRY)
.from(ENTRY)
.where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed))
.limit(1)
.fetchOne();
}
public Set<String> findExistingGuidHashes(Set<String> guidHashes, Feed feed) {
if (guidHashes.isEmpty()) {
return Set.of();
}
public Set<String> findExistingGuidHashes(Set<String> guidHashes, Feed feed) {
if (guidHashes.isEmpty()) {
return Set.of();
}
Set<String> result = new HashSet<>();
for (List<String> batch : Lists.partition(new ArrayList<>(guidHashes), IN_CLAUSE_BATCH_SIZE)) {
result.addAll(query().select(ENTRY.guidHash).from(ENTRY).where(ENTRY.feed.eq(feed), ENTRY.guidHash.in(batch)).fetch());
}
return result;
}
Set<String> result = new HashSet<>();
for (List<String> batch :
Lists.partition(new ArrayList<>(guidHashes), IN_CLAUSE_BATCH_SIZE)) {
result.addAll(
query().select(ENTRY.guidHash)
.from(ENTRY)
.where(ENTRY.feed.eq(feed), ENTRY.guidHash.in(batch))
.fetch());
}
return result;
}
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max, boolean keepStarredEntries) {
NumberExpression<Long> count = ENTRY.id.count();
JPAQuery<Tuple> query = query().select(ENTRY.feed.id, count).from(ENTRY);
public List<FeedCapacity> findFeedsExceedingCapacity(
long maxCapacity, long max, boolean keepStarredEntries) {
NumberExpression<Long> count = ENTRY.id.count();
JPAQuery<Tuple> query = query().select(ENTRY.feed.id, count).from(ENTRY);
if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY));
}
if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY));
}
return query.groupBy(ENTRY.feed)
.having(count.gt(maxCapacity))
.limit(max)
.fetch()
.stream()
.map(t -> new FeedCapacity(t.get(ENTRY.feed.id), t.get(count)))
.toList();
}
return query.groupBy(ENTRY.feed).having(count.gt(maxCapacity)).limit(max).fetch().stream()
.map(t -> new FeedCapacity(t.get(ENTRY.feed.id), t.get(count)))
.toList();
}
public int delete(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).limit(max).fetch();
return delete(list);
}
public int delete(Long feedId, long max) {
List<FeedEntry> list =
query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).limit(max).fetch();
return delete(list);
}
/**
* Delete entries older than a certain date
*/
public int deleteEntriesOlderThan(Instant olderThan, long max, boolean keepStarredEntries) {
JPAQuery<FeedEntry> query = query().selectFrom(ENTRY)
.where(ENTRY.published.lt(olderThan))
.orderBy(ENTRY.published.asc())
.limit(max);
/** Delete entries older than a certain date */
public int deleteEntriesOlderThan(Instant olderThan, long max, boolean keepStarredEntries) {
JPAQuery<FeedEntry> query =
query().selectFrom(ENTRY)
.where(ENTRY.published.lt(olderThan))
.orderBy(ENTRY.published.asc())
.limit(max);
if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY));
}
if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY));
}
return delete(query.fetch());
}
return delete(query.fetch());
}
/**
* Delete the oldest entries of a feed
*/
public int deleteOldEntries(Long feedId, long max, boolean keepStarredEntries) {
JPAQuery<FeedEntry> query = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).orderBy(ENTRY.published.asc()).limit(max);
/** Delete the oldest entries of a feed */
public int deleteOldEntries(Long feedId, long max, boolean keepStarredEntries) {
JPAQuery<FeedEntry> query =
query().selectFrom(ENTRY)
.where(ENTRY.feed.id.eq(feedId))
.orderBy(ENTRY.published.asc())
.limit(max);
if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY));
}
if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY));
}
return delete(query.fetch());
}
return delete(query.fetch());
}
public record FeedCapacity(Long id, Long capacity) {}
public record FeedCapacity(Long id, Long capacity) {}
}

492
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java
View File

@@ -1,16 +1,5 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.apache.commons.collections4.CollectionUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
@@ -32,273 +21,316 @@ import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.Expressions;
import com.querydsl.core.types.dsl.NumberExpression;
import com.querydsl.jpa.impl.JPAQuery;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.collections4.CollectionUtils;
@Singleton
public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final QFeed FEED = QFeed.feed;
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final QFeed FEED = QFeed.feed;
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private final FeedEntryTagDAO feedEntryTagDAO;
private final CommaFeedConfiguration config;
private final FeedEntryTagDAO feedEntryTagDAO;
private final CommaFeedConfiguration config;
public FeedEntryStatusDAO(EntityManager entityManager, FeedEntryTagDAO feedEntryTagDAO, CommaFeedConfiguration config) {
super(entityManager, FeedEntryStatus.class);
this.feedEntryTagDAO = feedEntryTagDAO;
this.config = config;
}
public FeedEntryStatusDAO(
EntityManager entityManager,
FeedEntryTagDAO feedEntryTagDAO,
CommaFeedConfiguration config) {
super(entityManager, FeedEntryStatus.class);
this.feedEntryTagDAO = feedEntryTagDAO;
this.config = config;
}
public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) {
List<FeedEntryStatus> statuses = query().selectFrom(STATUS).where(STATUS.entry.eq(entry), STATUS.subscription.eq(sub)).fetch();
FeedEntryStatus status = statuses.stream().findFirst().orElse(null);
return handleStatus(user, status, sub, entry);
}
public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) {
List<FeedEntryStatus> statuses =
query().selectFrom(STATUS)
.where(STATUS.entry.eq(entry), STATUS.subscription.eq(sub))
.fetch();
FeedEntryStatus status = statuses.stream().findFirst().orElse(null);
return handleStatus(user, status, sub, entry);
}
/**
* creates an artificial "unread" status if status is null
*/
private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
if (status == null) {
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
boolean read = statusesInstantThreshold != null && entry.getPublished().isBefore(statusesInstantThreshold);
status = new FeedEntryStatus(user, sub, entry);
status.setRead(read);
status.setMarkable(!read);
} else {
status.setMarkable(true);
}
return status;
}
/** creates an artificial "unread" status if status is null */
private FeedEntryStatus handleStatus(
User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
if (status == null) {
Instant statusesInstantThreshold =
config.database().cleanup().statusesInstantThreshold();
boolean read =
statusesInstantThreshold != null
&& entry.getPublished().isBefore(statusesInstantThreshold);
status = new FeedEntryStatus(user, sub, entry);
status.setRead(read);
status.setMarkable(!read);
} else {
status.setMarkable(true);
}
return status;
}
private void fetchTags(User user, List<FeedEntryStatus> statuses) {
Map<Long, List<FeedEntryTag>> tagsByEntryIds = feedEntryTagDAO.findByEntries(user,
statuses.stream().map(FeedEntryStatus::getEntry).toList());
for (FeedEntryStatus status : statuses) {
List<FeedEntryTag> tags = tagsByEntryIds.get(status.getEntry().getId());
status.setTags(tags == null ? List.of() : tags);
}
}
private void fetchTags(User user, List<FeedEntryStatus> statuses) {
Map<Long, List<FeedEntryTag>> tagsByEntryIds =
feedEntryTagDAO.findByEntries(
user, statuses.stream().map(FeedEntryStatus::getEntry).toList());
for (FeedEntryStatus status : statuses) {
List<FeedEntryTag> tags = tagsByEntryIds.get(status.getEntry().getId());
status.setTags(tags == null ? List.of() : tags);
}
}
public List<FeedEntryStatus> findStarred(User user, List<FeedEntryKeyword> keywords, Instant newerThan, int offset, int limit,
ReadingOrder order, boolean includeContent) {
JPAQuery<FeedEntryStatus> query = query().selectFrom(STATUS).where(STATUS.user.eq(user), STATUS.starred.isTrue());
if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
query.join(STATUS.entry).fetchJoin();
query.join(STATUS.entry.content, CONTENT).fetchJoin();
}
public List<FeedEntryStatus> findStarred(
User user,
List<FeedEntryKeyword> keywords,
Instant newerThan,
int offset,
int limit,
ReadingOrder order,
boolean includeContent) {
JPAQuery<FeedEntryStatus> query =
query().selectFrom(STATUS).where(STATUS.user.eq(user), STATUS.starred.isTrue());
if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
query.join(STATUS.entry).fetchJoin();
query.join(STATUS.entry.content, CONTENT).fetchJoin();
}
if (CollectionUtils.isNotEmpty(keywords)) {
applyKeywordsFilter(query, keywords);
}
if (CollectionUtils.isNotEmpty(keywords)) {
applyKeywordsFilter(query, keywords);
}
if (newerThan != null) {
query.where(STATUS.entryInserted.gt(newerThan));
}
if (newerThan != null) {
query.where(STATUS.entryInserted.gt(newerThan));
}
if (order == ReadingOrder.ASC) {
query.orderBy(STATUS.entryPublished.asc(), STATUS.id.asc());
} else {
query.orderBy(STATUS.entryPublished.desc(), STATUS.id.desc());
}
if (order == ReadingOrder.ASC) {
query.orderBy(STATUS.entryPublished.asc(), STATUS.id.asc());
} else {
query.orderBy(STATUS.entryPublished.desc(), STATUS.id.desc());
}
if (offset > -1) {
query.offset(offset);
}
if (offset > -1) {
query.offset(offset);
}
if (limit > -1) {
query.limit(limit);
}
if (limit > -1) {
query.limit(limit);
}
setTimeout(query, config.database().queryTimeout());
setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = query.fetch();
statuses.forEach(s -> s.setMarkable(true));
if (includeContent) {
fetchTags(user, statuses);
}
List<FeedEntryStatus> statuses = query.fetch();
statuses.forEach(s -> s.setMarkable(true));
if (includeContent) {
fetchTags(user, statuses);
}
return statuses;
}
return statuses;
}
public List<FeedEntryStatus> findBySubscriptions(User user, List<FeedSubscription> subs, boolean unreadOnly,
List<FeedEntryKeyword> keywords, Instant newerThan, int offset, int limit, ReadingOrder order, boolean includeContent,
String tag, Long minEntryId, Long maxEntryId) {
Map<Long, List<FeedSubscription>> subsByFeedId = subs.stream().collect(Collectors.groupingBy(s -> s.getFeed().getId()));
public List<FeedEntryStatus> findBySubscriptions(
User user,
List<FeedSubscription> subs,
boolean unreadOnly,
List<FeedEntryKeyword> keywords,
Instant newerThan,
int offset,
int limit,
ReadingOrder order,
boolean includeContent,
String tag,
Long minEntryId,
Long maxEntryId) {
Map<Long, List<FeedSubscription>> subsByFeedId =
subs.stream().collect(Collectors.groupingBy(s -> s.getFeed().getId()));
JPAQuery<Tuple> query = query().select(ENTRY, STATUS).from(ENTRY);
query.leftJoin(ENTRY.statuses, STATUS).on(STATUS.subscription.in(subs));
query.where(ENTRY.feed.id.in(subsByFeedId.keySet()));
JPAQuery<Tuple> query = query().select(ENTRY, STATUS).from(ENTRY);
query.leftJoin(ENTRY.statuses, STATUS).on(STATUS.subscription.in(subs));
query.where(ENTRY.feed.id.in(subsByFeedId.keySet()));
if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
query.join(ENTRY.content, CONTENT).fetchJoin();
}
if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
query.join(ENTRY.content, CONTENT).fetchJoin();
}
if (CollectionUtils.isNotEmpty(keywords)) {
applyKeywordsFilter(query, keywords);
}
if (CollectionUtils.isNotEmpty(keywords)) {
applyKeywordsFilter(query, keywords);
}
if (unreadOnly && tag == null) {
query.where(buildUnreadPredicate());
}
if (unreadOnly && tag == null) {
query.where(buildUnreadPredicate());
}
if (tag != null) {
BooleanBuilder and = new BooleanBuilder();
and.and(TAG.user.id.eq(user.getId()));
and.and(TAG.name.eq(tag));
query.join(ENTRY.tags, TAG).on(and);
}
if (tag != null) {
BooleanBuilder and = new BooleanBuilder();
and.and(TAG.user.id.eq(user.getId()));
and.and(TAG.name.eq(tag));
query.join(ENTRY.tags, TAG).on(and);
}
if (newerThan != null) {
query.where(ENTRY.inserted.goe(newerThan));
}
if (newerThan != null) {
query.where(ENTRY.inserted.goe(newerThan));
}
if (minEntryId != null) {
query.where(ENTRY.id.gt(minEntryId));
}
if (minEntryId != null) {
query.where(ENTRY.id.gt(minEntryId));
}
if (maxEntryId != null) {
query.where(ENTRY.id.lt(maxEntryId));
}
if (maxEntryId != null) {
query.where(ENTRY.id.lt(maxEntryId));
}
if (order != null) {
if (order == ReadingOrder.ASC) {
query.orderBy(ENTRY.published.asc(), ENTRY.id.asc());
} else {
query.orderBy(ENTRY.published.desc(), ENTRY.id.desc());
}
}
if (order != null) {
if (order == ReadingOrder.ASC) {
query.orderBy(ENTRY.published.asc(), ENTRY.id.asc());
} else {
query.orderBy(ENTRY.published.desc(), ENTRY.id.desc());
}
}
if (offset > -1) {
query.offset(offset);
}
if (offset > -1) {
query.offset(offset);
}
if (limit > -1) {
query.limit(limit);
}
if (limit > -1) {
query.limit(limit);
}
setTimeout(query, config.database().queryTimeout());
setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = new ArrayList<>();
List<Tuple> tuples = query.fetch();
for (Tuple tuple : tuples) {
FeedEntry e = tuple.get(ENTRY);
FeedEntryStatus s = tuple.get(STATUS);
for (FeedSubscription sub : subsByFeedId.get(e.getFeed().getId())) {
statuses.add(handleStatus(user, s, sub, e));
}
}
List<FeedEntryStatus> statuses = new ArrayList<>();
List<Tuple> tuples = query.fetch();
for (Tuple tuple : tuples) {
FeedEntry e = tuple.get(ENTRY);
FeedEntryStatus s = tuple.get(STATUS);
for (FeedSubscription sub : subsByFeedId.get(e.getFeed().getId())) {
statuses.add(handleStatus(user, s, sub, e));
}
}
if (includeContent) {
fetchTags(user, statuses);
}
if (includeContent) {
fetchTags(user, statuses);
}
return statuses;
}
return statuses;
}
private void applyKeywordsFilter(JPAQuery<?> query, List<FeedEntryKeyword> keywords) {
for (FeedEntryKeyword keyword : keywords) {
BooleanBuilder or = new BooleanBuilder();
or.or(CONTENT.content.containsIgnoreCase(keyword.keyword()));
or.or(CONTENT.title.containsIgnoreCase(keyword.keyword()));
if (keyword.mode() == Mode.EXCLUDE) {
or.not();
}
query.where(or);
}
}
private void applyKeywordsFilter(JPAQuery<?> query, List<FeedEntryKeyword> keywords) {
for (FeedEntryKeyword keyword : keywords) {
BooleanBuilder or = new BooleanBuilder();
or.or(CONTENT.content.containsIgnoreCase(keyword.keyword()));
or.or(CONTENT.title.containsIgnoreCase(keyword.keyword()));
if (keyword.mode() == Mode.EXCLUDE) {
or.not();
}
query.where(or);
}
}
public UnreadCount getUnreadCount(FeedSubscription sub) {
JPAQuery<Tuple> query = query().select(ENTRY.count(), ENTRY.published.max())
.from(ENTRY)
.leftJoin(ENTRY.statuses, STATUS)
.on(STATUS.subscription.eq(sub))
.where(ENTRY.feed.eq(sub.getFeed()))
.where(buildUnreadPredicate());
public UnreadCount getUnreadCount(FeedSubscription sub) {
JPAQuery<Tuple> query =
query().select(ENTRY.count(), ENTRY.published.max())
.from(ENTRY)
.leftJoin(ENTRY.statuses, STATUS)
.on(STATUS.subscription.eq(sub))
.where(ENTRY.feed.eq(sub.getFeed()))
.where(buildUnreadPredicate());
Tuple tuple = query.fetchOne();
Long count = tuple.get(ENTRY.count());
Instant published = tuple.get(ENTRY.published.max());
return new UnreadCount(sub.getId(), count == null ? 0 : count, published);
}
Tuple tuple = query.fetchOne();
Long count = tuple.get(ENTRY.count());
Instant published = tuple.get(ENTRY.published.max());
return new UnreadCount(sub.getId(), count == null ? 0 : count, published);
}
private BooleanBuilder buildUnreadPredicate() {
BooleanBuilder or = new BooleanBuilder();
or.or(STATUS.read.isNull());
or.or(STATUS.read.isFalse());
private BooleanBuilder buildUnreadPredicate() {
BooleanBuilder or = new BooleanBuilder();
or.or(STATUS.read.isNull());
or.or(STATUS.read.isFalse());
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
if (statusesInstantThreshold != null) {
return or.and(ENTRY.published.goe(statusesInstantThreshold));
} else {
return or;
}
}
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
if (statusesInstantThreshold != null) {
return or.and(ENTRY.published.goe(statusesInstantThreshold));
} else {
return or;
}
}
public long deleteOldStatuses(Instant olderThan, int limit) {
List<Long> ids = query().select(STATUS.id)
.from(STATUS)
.where(STATUS.entryInserted.lt(olderThan), STATUS.starred.isFalse())
.limit(limit)
.fetch();
return deleteQuery(STATUS).where(STATUS.id.in(ids)).execute();
}
public long deleteOldStatuses(Instant olderThan, int limit) {
List<Long> ids =
query().select(STATUS.id)
.from(STATUS)
.where(STATUS.entryInserted.lt(olderThan), STATUS.starred.isFalse())
.limit(limit)
.fetch();
return deleteQuery(STATUS).where(STATUS.id.in(ids)).execute();
}
public long autoMarkAsRead(int limit) {
Instant now = Instant.now();
public long autoMarkAsRead(int limit) {
Instant now = Instant.now();
BooleanBuilder where = new BooleanBuilder();
where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.isNotNull());
where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.gt(0));
BooleanBuilder where = new BooleanBuilder();
where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.isNotNull());
where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.gt(0));
NumberExpression<Integer> daysDiff = Expressions.numberTemplate(Integer.class, "TIMESTAMPDIFF(DAY, {0}, {1})", ENTRY.published,
now);
where.and(daysDiff.goe(SUBSCRIPTION.autoMarkAsReadAfterDays));
NumberExpression<Integer> daysDiff =
Expressions.numberTemplate(
Integer.class, "TIMESTAMPDIFF(DAY, {0}, {1})", ENTRY.published, now);
where.and(daysDiff.goe(SUBSCRIPTION.autoMarkAsReadAfterDays));
where.and(buildUnreadPredicate());
where.and(buildUnreadPredicate());
List<Tuple> tuples = query().select(ENTRY, STATUS, SUBSCRIPTION)
.from(ENTRY)
.join(ENTRY.feed, FEED)
.join(SUBSCRIPTION)
.on(SUBSCRIPTION.feed.eq(FEED))
.leftJoin(ENTRY.statuses, STATUS)
.on(STATUS.subscription.eq(SUBSCRIPTION))
.where(where)
.limit(limit)
.fetch();
List<Tuple> tuples =
query().select(ENTRY, STATUS, SUBSCRIPTION)
.from(ENTRY)
.join(ENTRY.feed, FEED)
.join(SUBSCRIPTION)
.on(SUBSCRIPTION.feed.eq(FEED))
.leftJoin(ENTRY.statuses, STATUS)
.on(STATUS.subscription.eq(SUBSCRIPTION))
.where(where)
.limit(limit)
.fetch();
long updated = 0;
long updated = 0;
// Update existing statuses
List<Long> statusIdsToUpdate = tuples.stream()
.map(t -> t.get(STATUS))
.filter(s -> s != null && s.getId() != null)
.map(FeedEntryStatus::getId)
.distinct()
.toList();
// Update existing statuses
List<Long> statusIdsToUpdate =
tuples.stream()
.map(t -> t.get(STATUS))
.filter(s -> s != null && s.getId() != null)
.map(FeedEntryStatus::getId)
.distinct()
.toList();
if (!statusIdsToUpdate.isEmpty()) {
updated += updateQuery(STATUS).where(STATUS.id.in(statusIdsToUpdate)).set(STATUS.read, true).execute();
}
if (!statusIdsToUpdate.isEmpty()) {
updated +=
updateQuery(STATUS)
.where(STATUS.id.in(statusIdsToUpdate))
.set(STATUS.read, true)
.execute();
}
// Insert new statuses for entries without existing status
for (Tuple tuple : tuples) {
FeedEntryStatus status = tuple.get(STATUS);
if (status == null || status.getId() == null) {
FeedEntry entry = tuple.get(ENTRY);
FeedSubscription sub = tuple.get(SUBSCRIPTION);
FeedEntryStatus newStatus = new FeedEntryStatus(sub.getUser(), sub, entry);
newStatus.setRead(true);
persist(newStatus);
updated++;
}
}
return updated;
}
// Insert new statuses for entries without existing status
for (Tuple tuple : tuples) {
FeedEntryStatus status = tuple.get(STATUS);
if (status == null || status.getId() == null) {
FeedEntry entry = tuple.get(ENTRY);
FeedSubscription sub = tuple.get(SUBSCRIPTION);
FeedEntryStatus newStatus = new FeedEntryStatus(sub.getUser(), sub, entry);
newStatus.setRead(true);
persist(newStatus);
updated++;
}
}
return updated;
}
}

47
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java
View File

@@ -1,39 +1,38 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Singleton
public class FeedEntryTagDAO extends GenericDAO<FeedEntryTag> {
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
public FeedEntryTagDAO(EntityManager entityManager) {
super(entityManager, FeedEntryTag.class);
}
public FeedEntryTagDAO(EntityManager entityManager) {
super(entityManager, FeedEntryTag.class);
}
public List<String> findByUser(User user) {
return query().selectDistinct(TAG.name).from(TAG).where(TAG.user.eq(user)).fetch();
}
public List<String> findByUser(User user) {
return query().selectDistinct(TAG.name).from(TAG).where(TAG.user.eq(user)).fetch();
}
public List<FeedEntryTag> findByEntry(User user, FeedEntry entry) {
return query().selectFrom(TAG).where(TAG.user.eq(user), TAG.entry.eq(entry)).fetch();
}
public List<FeedEntryTag> findByEntry(User user, FeedEntry entry) {
return query().selectFrom(TAG).where(TAG.user.eq(user), TAG.entry.eq(entry)).fetch();
}
public Map<Long, List<FeedEntryTag>> findByEntries(User user, List<FeedEntry> entries) {
return query().selectFrom(TAG)
.where(TAG.user.eq(user), TAG.entry.in(entries))
.fetch()
.stream()
.collect(Collectors.groupingBy(t -> t.getEntry().getId()));
}
public Map<Long, List<FeedEntryTag>> findByEntries(User user, List<FeedEntry> entries) {
return query()
.selectFrom(TAG)
.where(TAG.user.eq(user), TAG.entry.in(entries))
.fetch()
.stream()
.collect(Collectors.groupingBy(t -> t.getEntry().getId()));
}
}

216
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java
View File

@@ -1,20 +1,5 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.event.service.spi.EventListenerRegistry;
import org.hibernate.event.spi.EventType;
import org.hibernate.event.spi.PostCommitInsertEventListener;
import org.hibernate.event.spi.PostInsertEvent;
import org.hibernate.persister.entity.EntityPersister;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
@@ -23,109 +8,136 @@ import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.QFeedSubscription;
import com.commafeed.backend.model.User;
import com.querydsl.jpa.JPQLQuery;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.event.service.spi.EventListenerRegistry;
import org.hibernate.event.spi.EventType;
import org.hibernate.event.spi.PostCommitInsertEventListener;
import org.hibernate.event.spi.PostInsertEvent;
import org.hibernate.persister.entity.EntityPersister;
@Singleton
public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> {
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private final EntityManager entityManager;
private final EntityManager entityManager;
public FeedSubscriptionDAO(EntityManager entityManager) {
super(entityManager, FeedSubscription.class);
this.entityManager = entityManager;
}
public FeedSubscriptionDAO(EntityManager entityManager) {
super(entityManager, FeedSubscription.class);
this.entityManager = entityManager;
}
public void onPostCommitInsert(Consumer<FeedSubscription> consumer) {
entityManager.unwrap(SharedSessionContractImplementor.class)
.getFactory()
.getServiceRegistry()
.getService(EventListenerRegistry.class)
.getEventListenerGroup(EventType.POST_COMMIT_INSERT)
.appendListener(new PostCommitInsertEventListener() {
@Override
public void onPostInsert(PostInsertEvent event) {
if (event.getEntity() instanceof FeedSubscription s) {
consumer.accept(s);
}
}
public void onPostCommitInsert(Consumer<FeedSubscription> consumer) {
entityManager
.unwrap(SharedSessionContractImplementor.class)
.getFactory()
.getServiceRegistry()
.getService(EventListenerRegistry.class)
.getEventListenerGroup(EventType.POST_COMMIT_INSERT)
.appendListener(
new PostCommitInsertEventListener() {
@Override
public void onPostInsert(PostInsertEvent event) {
if (event.getEntity() instanceof FeedSubscription s) {
consumer.accept(s);
}
}
@Override
public boolean requiresPostCommitHandling(EntityPersister persister) {
return true;
}
@Override
public boolean requiresPostCommitHandling(EntityPersister persister) {
return true;
}
@Override
public void onPostInsertCommitFailed(PostInsertEvent event) {
// do nothing
}
});
}
@Override
public void onPostInsertCommitFailed(PostInsertEvent event) {
// do nothing
}
});
}
public FeedSubscription findById(User user, Long id) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.id.eq(id))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
FeedSubscription sub = subs.stream().findFirst().orElse(null);
return initRelations(sub);
}
public FeedSubscription findById(User user, Long id) {
List<FeedSubscription> subs =
query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.id.eq(id))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
FeedSubscription sub = subs.stream().findFirst().orElse(null);
return initRelations(sub);
}
public List<FeedSubscription> findByFeed(Feed feed) {
return query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.feed.eq(feed)).fetch();
}
public List<FeedSubscription> findByFeed(Feed feed) {
return query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.feed.eq(feed)).fetch();
}
public FeedSubscription findByFeed(User user, Feed feed) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.feed.eq(feed))
.fetch();
FeedSubscription sub = subs.stream().findFirst().orElse(null);
return initRelations(sub);
}
public FeedSubscription findByFeed(User user, Feed feed) {
List<FeedSubscription> subs =
query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.feed.eq(feed))
.fetch();
FeedSubscription sub = subs.stream().findFirst().orElse(null);
return initRelations(sub);
}
public List<FeedSubscription> findAll(User user) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
return initRelations(subs);
}
public List<FeedSubscription> findAll(User user) {
List<FeedSubscription> subs =
query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
return initRelations(subs);
}
public Long count(User user) {
return query().select(SUBSCRIPTION.count()).from(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user)).fetchOne();
}
public Long count(User user) {
return query().select(SUBSCRIPTION.count())
.from(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user))
.fetchOne();
}
public List<FeedSubscription> findByCategory(User user, FeedCategory category) {
JPQLQuery<FeedSubscription> query = query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user));
if (category == null) {
query.where(SUBSCRIPTION.category.isNull());
} else {
query.where(SUBSCRIPTION.category.eq(category));
}
return initRelations(query.fetch());
}
public List<FeedSubscription> findByCategory(User user, FeedCategory category) {
JPQLQuery<FeedSubscription> query =
query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user));
if (category == null) {
query.where(SUBSCRIPTION.category.isNull());
} else {
query.where(SUBSCRIPTION.category.eq(category));
}
return initRelations(query.fetch());
}
public List<FeedSubscription> findByCategories(User user, List<FeedCategory> categories) {
Set<Long> categoryIds = categories.stream().map(AbstractModel::getId).collect(Collectors.toSet());
return findAll(user).stream().filter(s -> s.getCategory() != null && categoryIds.contains(s.getCategory().getId())).toList();
}
public List<FeedSubscription> findByCategories(User user, List<FeedCategory> categories) {
Set<Long> categoryIds =
categories.stream().map(AbstractModel::getId).collect(Collectors.toSet());
return findAll(user).stream()
.filter(
s ->
s.getCategory() != null
&& categoryIds.contains(s.getCategory().getId()))
.toList();
}
private List<FeedSubscription> initRelations(List<FeedSubscription> list) {
list.forEach(this::initRelations);
return list;
}
private List<FeedSubscription> initRelations(List<FeedSubscription> list) {
list.forEach(this::initRelations);
return list;
}
private FeedSubscription initRelations(FeedSubscription sub) {
if (sub != null) {
Models.initialize(sub.getFeed());
Models.initialize(sub.getCategory());
}
return sub;
}
private FeedSubscription initRelations(FeedSubscription sub) {
if (sub != null) {
Models.initialize(sub.getFeed());
Models.initialize(sub.getCategory());
}
return sub;
}
}

81
commafeed-server/src/main/java/com/commafeed/backend/dao/GenericDAO.java
View File

@@ -1,66 +1,61 @@
package com.commafeed.backend.dao;
import java.time.Duration;
import java.util.Collection;
import jakarta.persistence.EntityManager;
import org.hibernate.jpa.SpecHints;
import com.commafeed.backend.model.AbstractModel;
import com.querydsl.core.types.EntityPath;
import com.querydsl.jpa.impl.JPADeleteClause;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
import com.querydsl.jpa.impl.JPAUpdateClause;
import jakarta.persistence.EntityManager;
import java.time.Duration;
import java.util.Collection;
import lombok.RequiredArgsConstructor;
import org.hibernate.jpa.SpecHints;
@RequiredArgsConstructor
public abstract class GenericDAO<T extends AbstractModel> {
private final EntityManager entityManager;
private final Class<T> entityClass;
private final EntityManager entityManager;
private final Class<T> entityClass;
protected JPAQueryFactory query() {
return new JPAQueryFactory(entityManager);
}
protected JPAQueryFactory query() {
return new JPAQueryFactory(entityManager);
}
protected JPAUpdateClause updateQuery(EntityPath<T> entityPath) {
return new JPAUpdateClause(entityManager, entityPath);
}
protected JPAUpdateClause updateQuery(EntityPath<T> entityPath) {
return new JPAUpdateClause(entityManager, entityPath);
}
protected JPADeleteClause deleteQuery(EntityPath<T> entityPath) {
return new JPADeleteClause(entityManager, entityPath);
}
protected JPADeleteClause deleteQuery(EntityPath<T> entityPath) {
return new JPADeleteClause(entityManager, entityPath);
}
public void persist(T model) {
entityManager.persist(model);
}
public void persist(T model) {
entityManager.persist(model);
}
public T merge(T model) {
return entityManager.merge(model);
}
public T merge(T model) {
return entityManager.merge(model);
}
public T findById(Long id) {
return entityManager.find(entityClass, id);
}
public T findById(Long id) {
return entityManager.find(entityClass, id);
}
public void delete(T object) {
if (object != null) {
entityManager.remove(object);
}
}
public void delete(T object) {
if (object != null) {
entityManager.remove(object);
}
}
public int delete(Collection<T> objects) {
objects.forEach(this::delete);
return objects.size();
}
protected void setTimeout(JPAQuery<?> query, Duration timeout) {
if (!timeout.isZero()) {
query.setHint(SpecHints.HINT_SPEC_QUERY_TIMEOUT, Math.toIntExact(timeout.toMillis()));
}
}
public int delete(Collection<T> objects) {
objects.forEach(this::delete);
return objects.size();
}
protected void setTimeout(JPAQuery<?> query, Duration timeout) {
if (!timeout.isZero()) {
query.setHint(SpecHints.HINT_SPEC_QUERY_TIMEOUT, Math.toIntExact(timeout.toMillis()));
}
}
}

12
commafeed-server/src/main/java/com/commafeed/backend/dao/Predicates.java
View File

@@ -4,15 +4,17 @@ import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryStatus;
import com.querydsl.core.types.dsl.BooleanExpression;
import com.querydsl.jpa.JPAExpressions;
import lombok.experimental.UtilityClass;
@UtilityClass
public class Predicates {
private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
public static BooleanExpression isNotStarred(QFeedEntry entry) {
return JPAExpressions.selectOne().from(STATUS).where(STATUS.entry.eq(entry).and(STATUS.starred.isTrue())).notExists();
}
public static BooleanExpression isNotStarred(QFeedEntry entry) {
return JPAExpressions.selectOne()
.from(STATUS)
.where(STATUS.entry.eq(entry).and(STATUS.starred.isTrue()))
.notExists();
}
}

18
commafeed-server/src/main/java/com/commafeed/backend/dao/UnitOfWork.java
View File

@@ -1,19 +1,17 @@
package com.commafeed.backend.dao;
import java.util.concurrent.Callable;
import jakarta.inject.Singleton;
import io.quarkus.narayana.jta.QuarkusTransaction;
import jakarta.inject.Singleton;
import java.util.concurrent.Callable;
@Singleton
public class UnitOfWork {
public void run(Runnable runnable) {
QuarkusTransaction.joiningExisting().run(runnable);
}
public void run(Runnable runnable) {
QuarkusTransaction.joiningExisting().run(runnable);
}
public <T> T call(Callable<T> callable) {
return QuarkusTransaction.joiningExisting().call(callable);
}
public <T> T call(Callable<T> callable) {
return QuarkusTransaction.joiningExisting().call(callable);
}
}

37
commafeed-server/src/main/java/com/commafeed/backend/dao/UserDAO.java
View File

@@ -1,33 +1,32 @@
package com.commafeed.backend.dao;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton
public class UserDAO extends GenericDAO<User> {
private static final QUser USER = QUser.user;
private static final QUser USER = QUser.user;
public UserDAO(EntityManager entityManager) {
super(entityManager, User.class);
}
public UserDAO(EntityManager entityManager) {
super(entityManager, User.class);
}
public User findByName(String name) {
return query().selectFrom(USER).where(USER.name.equalsIgnoreCase(name)).fetchOne();
}
public User findByName(String name) {
return query().selectFrom(USER).where(USER.name.equalsIgnoreCase(name)).fetchOne();
}
public User findByApiKey(String key) {
return query().selectFrom(USER).where(USER.apiKey.equalsIgnoreCase(key)).fetchOne();
}
public User findByApiKey(String key) {
return query().selectFrom(USER).where(USER.apiKey.equalsIgnoreCase(key)).fetchOne();
}
public User findByEmail(String email) {
return query().selectFrom(USER).where(USER.email.equalsIgnoreCase(email)).fetchOne();
}
public User findByEmail(String email) {
return query().selectFrom(USER).where(USER.email.equalsIgnoreCase(email)).fetchOne();
}
public long count() {
return query().select(USER.count()).from(USER).fetchOne();
}
public long count() {
return query().select(USER.count()).from(USER).fetchOne();
}
}

44
commafeed-server/src/main/java/com/commafeed/backend/dao/UserRoleDAO.java
View File

@@ -1,39 +1,37 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUserRole;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
@Singleton
public class UserRoleDAO extends GenericDAO<UserRole> {
private static final QUserRole ROLE = QUserRole.userRole;
private static final QUserRole ROLE = QUserRole.userRole;
public UserRoleDAO(EntityManager entityManager) {
super(entityManager, UserRole.class);
}
public UserRoleDAO(EntityManager entityManager) {
super(entityManager, UserRole.class);
}
public List<UserRole> findAll() {
return query().selectFrom(ROLE).leftJoin(ROLE.user).fetchJoin().distinct().fetch();
}
public List<UserRole> findAll() {
return query().selectFrom(ROLE).leftJoin(ROLE.user).fetchJoin().distinct().fetch();
}
public List<UserRole> findAll(User user) {
return query().selectFrom(ROLE).where(ROLE.user.eq(user)).distinct().fetch();
}
public List<UserRole> findAll(User user) {
return query().selectFrom(ROLE).where(ROLE.user.eq(user)).distinct().fetch();
}
public Set<Role> findRoles(User user) {
return findAll(user).stream().map(UserRole::getRole).collect(Collectors.toSet());
}
public Set<Role> findRoles(User user) {
return findAll(user).stream().map(UserRole::getRole).collect(Collectors.toSet());
}
public long countAdmins() {
return query().select(ROLE.count()).from(ROLE).where(ROLE.role.eq(Role.ADMIN)).fetchOne();
}
public long countAdmins() {
return query().select(ROLE.count()).from(ROLE).where(ROLE.role.eq(Role.ADMIN)).fetchOne();
}
}

19
commafeed-server/src/main/java/com/commafeed/backend/dao/UserSettingsDAO.java
View File

@@ -1,22 +1,21 @@
package com.commafeed.backend.dao;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUserSettings;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
@Singleton
public class UserSettingsDAO extends GenericDAO<UserSettings> {
private static final QUserSettings SETTINGS = QUserSettings.userSettings;
private static final QUserSettings SETTINGS = QUserSettings.userSettings;
public UserSettingsDAO(EntityManager entityManager) {
super(entityManager, UserSettings.class);
}
public UserSettingsDAO(EntityManager entityManager) {
super(entityManager, UserSettings.class);
}
public UserSettings findByUser(User user) {
return query().selectFrom(SETTINGS).where(SETTINGS.user.eq(user)).fetchFirst();
}
public UserSettings findByUser(User user) {
return query().selectFrom(SETTINGS).where(SETTINGS.user.eq(user)).fetchFirst();
}
}

89
commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
View File

@@ -1,21 +1,17 @@
package com.commafeed.backend.favicon;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
@Slf4j
@RequiredArgsConstructor
@@ -23,44 +19,49 @@ import lombok.extern.slf4j.Slf4j;
@Priority(3)
public class FacebookFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("www.facebook.com")) {
return null;
}
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("www.facebook.com")) {
return null;
}
String userName = extractUserName(url);
if (userName == null) {
return null;
}
String userName = extractUserName(url);
if (userName == null) {
return null;
}
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
String iconUrl =
String.format(
"https://graph.facebook.com/%s/picture?type=square&height=16", userName);
try {
log.debug("Getting Facebook user's icon, {}", url);
try {
log.debug("Getting Facebook user's icon, {}", url);
HttpResult iconResult = getter.get(iconUrl);
return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e);
return null;
}
}
HttpResult iconResult = getter.get(iconUrl);
return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e);
return null;
}
}
private String extractUserName(String url) {
URI uri;
try {
uri = new URI(url);
} catch (URISyntaxException e) {
log.debug("could not parse url", e);
return null;
}
List<NameValuePair> params = new URIBuilder(uri).getQueryParams();
return params.stream().filter(p -> "id".equals(p.getName())).map(NameValuePair::getValue).findFirst().orElse(null);
}
private String extractUserName(String url) {
URI uri;
try {
uri = new URI(url);
} catch (URISyntaxException e) {
log.debug("could not parse url", e);
return null;
}
List<NameValuePair> params = new URIBuilder(uri).getQueryParams();
return params.stream()
.filter(p -> "id".equals(p.getName()))
.map(NameValuePair::getValue)
.findFirst()
.orElse(null);
}
}

28
commafeed-server/src/main/java/com/commafeed/backend/favicon/Favicon.java
View File

@@ -1,24 +1,24 @@
package com.commafeed.backend.favicon;
import jakarta.ws.rs.core.MediaType;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public record Favicon(byte[] icon, MediaType mediaType) {
private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.valueOf("image/x-icon");
private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.valueOf("image/x-icon");
public Favicon(byte[] icon, String contentType) {
this(icon, parseMediaType(contentType));
}
public Favicon(byte[] icon, String contentType) {
this(icon, parseMediaType(contentType));
}
private static MediaType parseMediaType(String contentType) {
try {
return MediaType.valueOf(contentType);
} catch (Exception e) {
log.debug("invalid content type '{}' received, returning default value", contentType, e);
return DEFAULT_MEDIA_TYPE;
}
}
}
private static MediaType parseMediaType(String contentType) {
try {
return MediaType.valueOf(contentType);
} catch (Exception e) {
log.debug(
"invalid content type '{}' received, returning default value", contentType, e);
return DEFAULT_MEDIA_TYPE;
}
}
}

3
commafeed-server/src/main/java/com/commafeed/backend/favicon/FaviconFetcher.java
View File

@@ -4,6 +4,5 @@ import com.commafeed.backend.model.Feed;
public interface FaviconFetcher {
Favicon fetch(Feed feed);
Favicon fetch(Feed feed);
}

40
commafeed-server/src/main/java/com/commafeed/backend/favicon/FeedFaviconFetcher.java
View File

@@ -1,39 +1,35 @@
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Fetch favicon from the url declared in the feed.
*/
/** Fetch favicon from the url declared in the feed. */
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(2)
public class FeedFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getIconUrl();
if (url == null) {
return null;
}
@Override
public Favicon fetch(Feed feed) {
String url = feed.getIconUrl();
if (url == null) {
return null;
}
try {
HttpResult result = getter.get(url);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve icon declared in the feed {}", url, e);
return null;
}
}
try {
HttpResult result = getter.get(url);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve icon declared in the feed {}", url, e);
return null;
}
}
}

83
commafeed-server/src/main/java/com/commafeed/backend/favicon/HtmlFaviconFetcher.java
View File

@@ -1,65 +1,60 @@
package com.commafeed.backend.favicon;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Extracts favicon url from html page.
*/
/** Extracts favicon url from html page. */
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(1)
public class HtmlFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getLink();
if (url == null) {
return null;
}
@Override
public Favicon fetch(Feed feed) {
String url = feed.getLink();
if (url == null) {
return null;
}
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.content()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon", e);
return null;
}
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.content()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon", e);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
String href = icons.getFirst().attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
String href = icons.getFirst().attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
try {
HttpResult result = getter.get(href);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href, e);
return null;
}
}
try {
HttpResult result = getter.get(href);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href, e);
return null;
}
}
}

49
commafeed-server/src/main/java/com/commafeed/backend/favicon/RootFaviconFetcher.java
View File

@@ -1,45 +1,40 @@
package com.commafeed.backend.favicon;
import java.net.URI;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import java.net.URI;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Fetches favicon from root of the domain (e.g. https://example.com/favicon.ico)
*/
/** Fetches favicon from root of the domain (e.g. https://example.com/favicon.ico) */
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(0)
public class RootFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getLink();
if (url == null) {
url = feed.getUrl();
}
@Override
public Favicon fetch(Feed feed) {
String url = feed.getLink();
if (url == null) {
url = feed.getUrl();
}
try {
URI uri = URI.create(url.trim());
String faviconUrl = "%s://%s/favicon.ico".formatted(uri.getScheme(), uri.getHost());
try {
URI uri = URI.create(url.trim());
String faviconUrl = "%s://%s/favicon.ico".formatted(uri.getScheme(), uri.getHost());
log.debug("getting root icon at {}", faviconUrl);
HttpResult result = getter.get(faviconUrl);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
return null;
}
}
log.debug("getting root icon at {}", faviconUrl);
HttpResult result = getter.get(faviconUrl);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
return null;
}
}
}

210
commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
View File

@@ -1,18 +1,5 @@
package com.commafeed.backend.favicon;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Optional;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.UriBuilder;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
@@ -23,9 +10,18 @@ import com.commafeed.backend.model.Feed;
import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.UriBuilder;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Optional;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
@Slf4j
@RequiredArgsConstructor
@@ -33,99 +29,121 @@ import lombok.extern.slf4j.Slf4j;
@Priority(3)
public class YoutubeFaviconFetcher implements FaviconFetcher {
private static final String PART_SNIPPET = "snippet";
private static final String PART_SNIPPET = "snippet";
private static final JsonPointer CHANNEL_THUMBNAIL_URL = JsonPointer.compile("/items/0/snippet/thumbnails/default/url");
private static final JsonPointer PLAYLIST_CHANNEL_ID = JsonPointer.compile("/items/0/snippet/channelId");
private static final JsonPointer CHANNEL_THUMBNAIL_URL =
JsonPointer.compile("/items/0/snippet/thumbnails/default/url");
private static final JsonPointer PLAYLIST_CHANNEL_ID =
JsonPointer.compile("/items/0/snippet/channelId");
private final HttpGetter getter;
private final CommaFeedConfiguration config;
private final ObjectMapper objectMapper;
private final HttpGetter getter;
private final CommaFeedConfiguration config;
private final ObjectMapper objectMapper;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
return null;
}
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
return null;
}
Optional<String> googleAuthKey = config.googleAuthKey();
if (googleAuthKey.isEmpty()) {
log.debug("no google auth key configured");
return null;
}
Optional<String> googleAuthKey = config.googleAuthKey();
if (googleAuthKey.isEmpty()) {
log.debug("no google auth key configured");
return null;
}
try {
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
Optional<NameValuePair> channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id")).findFirst();
Optional<NameValuePair> playlistId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id")).findFirst();
try {
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
Optional<NameValuePair> userId =
params.stream()
.filter(nvp -> nvp.getName().equalsIgnoreCase("user"))
.findFirst();
Optional<NameValuePair> channelId =
params.stream()
.filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id"))
.findFirst();
Optional<NameValuePair> playlistId =
params.stream()
.filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id"))
.findFirst();
byte[] response = null;
if (userId.isPresent()) {
log.debug("contacting youtube api for user {}", userId.get().getValue());
response = fetchForUser(googleAuthKey.get(), userId.get().getValue());
} else if (channelId.isPresent()) {
log.debug("contacting youtube api for channel {}", channelId.get().getValue());
response = fetchForChannel(googleAuthKey.get(), channelId.get().getValue());
} else if (playlistId.isPresent()) {
log.debug("contacting youtube api for playlist {}", playlistId.get().getValue());
response = fetchForPlaylist(googleAuthKey.get(), playlistId.get().getValue());
}
if (ArrayUtils.isEmpty(response)) {
log.debug("youtube api returned empty response");
return null;
}
byte[] response = null;
if (userId.isPresent()) {
log.debug("contacting youtube api for user {}", userId.get().getValue());
response = fetchForUser(googleAuthKey.get(), userId.get().getValue());
} else if (channelId.isPresent()) {
log.debug("contacting youtube api for channel {}", channelId.get().getValue());
response = fetchForChannel(googleAuthKey.get(), channelId.get().getValue());
} else if (playlistId.isPresent()) {
log.debug("contacting youtube api for playlist {}", playlistId.get().getValue());
response = fetchForPlaylist(googleAuthKey.get(), playlistId.get().getValue());
}
if (ArrayUtils.isEmpty(response)) {
log.debug("youtube api returned empty response");
return null;
}
JsonNode thumbnailUrl = objectMapper.readTree(response).at(CHANNEL_THUMBNAIL_URL);
if (thumbnailUrl.isMissingNode()) {
log.debug("youtube api returned invalid response");
return null;
}
JsonNode thumbnailUrl = objectMapper.readTree(response).at(CHANNEL_THUMBNAIL_URL);
if (thumbnailUrl.isMissingNode()) {
log.debug("youtube api returned invalid response");
return null;
}
HttpResult iconResult = getter.get(thumbnailUrl.asText());
return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve YouTube icon", e);
return null;
}
}
HttpResult iconResult = getter.get(thumbnailUrl.asText());
return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve YouTube icon", e);
return null;
}
}
private byte[] fetchForUser(String googleAuthKey, String userId)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", PART_SNIPPET)
.queryParam("key", googleAuthKey)
.queryParam("forUsername", userId)
.build();
return getter.get(uri.toString()).content();
}
private byte[] fetchForUser(String googleAuthKey, String userId)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
URI uri =
UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", PART_SNIPPET)
.queryParam("key", googleAuthKey)
.queryParam("forUsername", userId)
.build();
return getter.get(uri.toString()).content();
}
private byte[] fetchForChannel(String googleAuthKey, String channelId)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", PART_SNIPPET)
.queryParam("key", googleAuthKey)
.queryParam("id", channelId)
.build();
return getter.get(uri.toString()).content();
}
private byte[] fetchForChannel(String googleAuthKey, String channelId)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
URI uri =
UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", PART_SNIPPET)
.queryParam("key", googleAuthKey)
.queryParam("id", channelId)
.build();
return getter.get(uri.toString()).content();
}
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
.queryParam("part", PART_SNIPPET)
.queryParam("key", googleAuthKey)
.queryParam("id", playlistId)
.build();
byte[] playlistBytes = getter.get(uri.toString()).content();
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
URI uri =
UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
.queryParam("part", PART_SNIPPET)
.queryParam("key", googleAuthKey)
.queryParam("id", playlistId)
.build();
byte[] playlistBytes = getter.get(uri.toString()).content();
JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID);
if (channelId.isMissingNode()) {
return new byte[0];
}
return fetchForChannel(googleAuthKey, channelId.asText());
}
JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID);
if (channelId.isMissingNode()) {
return new byte[0];
}
return fetchForChannel(googleAuthKey, channelId.asText());
}
}

40
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedEntryKeyword.java
View File

@@ -2,30 +2,28 @@ package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
/**
* A keyword used in a search query
*/
/** A keyword used in a search query */
public record FeedEntryKeyword(String keyword, Mode mode) {
public enum Mode {
INCLUDE, EXCLUDE
}
public enum Mode {
INCLUDE,
EXCLUDE
}
public static List<FeedEntryKeyword> fromQueryString(String keywords) {
List<FeedEntryKeyword> list = new ArrayList<>();
if (keywords != null) {
for (String keyword : StringUtils.split(keywords)) {
boolean not = false;
if (keyword.startsWith("-") || keyword.startsWith("!")) {
not = true;
keyword = keyword.substring(1);
}
list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
}
}
return list;
}
public static List<FeedEntryKeyword> fromQueryString(String keywords) {
List<FeedEntryKeyword> list = new ArrayList<>();
if (keywords != null) {
for (String keyword : StringUtils.split(keywords)) {
boolean not = false;
if (keyword.startsWith("-") || keyword.startsWith("!")) {
not = true;
keyword = keyword.substring(1);
}
list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
}
}
return list;
}
}

196
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
View File

@@ -1,16 +1,5 @@
package com.commafeed.backend.feed;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Strings;
import com.commafeed.backend.Digests;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpRequest;
@@ -22,96 +11,139 @@ import com.commafeed.backend.feed.parser.FeedParser;
import com.commafeed.backend.feed.parser.FeedParser.FeedParsingException;
import com.commafeed.backend.feed.parser.FeedParserResult;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import io.quarkus.arc.All;
import jakarta.inject.Singleton;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.Strings;
/**
* Fetches a feed then parses it
*/
/** Fetches a feed then parses it */
@Slf4j
@Singleton
public class FeedFetcher {
private final FeedParser parser;
private final HttpGetter getter;
private final List<FeedURLProvider> urlProviders;
private final FeedParser parser;
private final HttpGetter getter;
private final List<FeedURLProvider> urlProviders;
public FeedFetcher(FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
this.parser = parser;
this.getter = getter;
this.urlProviders = urlProviders;
}
public FeedFetcher(
FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
this.parser = parser;
this.getter = getter;
this.urlProviders = urlProviders;
}
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
Instant lastPublishedDate, String lastContentHash) throws FeedParsingException, IOException, NotModifiedException,
TooManyRequestsException, SchemeNotAllowedException, NoFeedFoundException {
log.debug("Fetching feed {}", feedUrl);
public FeedFetcherResult fetch(
String feedUrl,
boolean extractFeedUrlFromHtml,
String lastModified,
String eTag,
Instant lastPublishedDate,
String lastContentHash)
throws FeedParsingException,
IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException,
NoFeedFoundException {
log.debug("Fetching feed {}", feedUrl);
HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
byte[] content = result.content();
HttpResult result =
getter.get(
HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
byte[] content = result.content();
FeedParserResult parserResult;
try {
parserResult = parser.parse(result.urlAfterRedirect(), content);
} catch (FeedParsingException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, new String(result.content(), StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
FeedParserResult parserResult;
try {
parserResult = parser.parse(result.urlAfterRedirect(), content);
} catch (FeedParsingException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl =
extractFeedUrl(
urlProviders,
feedUrl,
new String(result.content(), StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
result = getter.get(HttpRequest.builder(extractedUrl).lastModified(lastModified).eTag(eTag).build());
content = result.content();
parserResult = parser.parse(result.urlAfterRedirect(), content);
} else {
throw new NoFeedFoundException(e);
}
} else {
throw e;
}
}
result =
getter.get(
HttpRequest.builder(extractedUrl)
.lastModified(lastModified)
.eTag(eTag)
.build());
content = result.content();
parserResult = parser.parse(result.urlAfterRedirect(), content);
} else {
throw new NoFeedFoundException(e);
}
} else {
throw e;
}
}
if (content == null) {
throw new IOException("Feed content is empty.");
}
if (content == null) {
throw new IOException("Feed content is empty.");
}
boolean lastModifiedHeaderValueChanged = !Strings.CS.equals(lastModified, result.lastModifiedSince());
boolean etagHeaderValueChanged = !Strings.CS.equals(eTag, result.eTag());
boolean lastModifiedHeaderValueChanged =
!Strings.CS.equals(lastModified, result.lastModifiedSince());
boolean etagHeaderValueChanged = !Strings.CS.equals(eTag, result.eTag());
String hash = Digests.sha1Hex(content);
if (lastContentHash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException("content hash not modified", lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
etagHeaderValueChanged ? result.eTag() : null);
}
String hash = Digests.sha1Hex(content);
if (lastContentHash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException(
"content hash not modified",
lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
etagHeaderValueChanged ? result.eTag() : null);
}
if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified", lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
etagHeaderValueChanged ? result.eTag() : null);
}
if (lastPublishedDate != null
&& lastPublishedDate.equals(parserResult.lastPublishedDate())) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException(
"publishedDate not modified",
lastModifiedHeaderValueChanged ? result.lastModifiedSince() : null,
etagHeaderValueChanged ? result.eTag() : null);
}
return new FeedFetcherResult(parserResult, result.urlAfterRedirect(), result.lastModifiedSince(), result.eTag(), hash,
result.validFor());
}
return new FeedFetcherResult(
parserResult,
result.urlAfterRedirect(),
result.lastModifiedSince(),
result.eTag(),
hash,
result.validFor());
}
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) {
return urlProviders.stream()
.flatMap(provider -> provider.get(url, urlContent).stream())
.filter(StringUtils::isNotBlank)
.findFirst()
.orElse(null);
}
private static String extractFeedUrl(
List<FeedURLProvider> urlProviders, String url, String urlContent) {
return urlProviders.stream()
.flatMap(provider -> provider.get(url, urlContent).stream())
.filter(StringUtils::isNotBlank)
.findFirst()
.orElse(null);
}
public record FeedFetcherResult(FeedParserResult feed, String urlAfterRedirect, String lastModifiedHeader, String lastETagHeader,
String contentHash, Duration validFor) {}
public record FeedFetcherResult(
FeedParserResult feed,
String urlAfterRedirect,
String lastModifiedHeader,
String lastETagHeader,
String contentHash,
Duration validFor) {}
public static class NoFeedFoundException extends Exception {
private static final long serialVersionUID = 1L;
public NoFeedFoundException(Throwable cause) {
super("This URL does not point to an RSS feed or a website with an RSS feed.", cause);
}
}
public static class NoFeedFoundException extends Exception {
private static final long serialVersionUID = 1L;
public NoFeedFoundException(Throwable cause) {
super("This URL does not point to an RSS feed or a website with an RSS feed.", cause);
}
}
}

468
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshEngine.java
View File

@@ -1,19 +1,5 @@
package com.commafeed.backend.feed;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
@@ -25,229 +11,297 @@ import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.google.common.util.concurrent.MoreExecutors;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedRefreshEngine {
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedRefreshWorker worker;
private final FeedRefreshUpdater updater;
private final FeedUpdateNotifier notifier;
private final CommaFeedConfiguration config;
private final Meter refill;
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedRefreshWorker worker;
private final FeedRefreshUpdater updater;
private final FeedUpdateNotifier notifier;
private final CommaFeedConfiguration config;
private final Meter refill;
private final BlockingDeque<Feed> queue;
private final BlockingDeque<Feed> queue;
private ExecutorService feedProcessingLoopExecutor;
private ExecutorService refillLoopExecutor;
private ThreadPoolExecutor refillExecutor;
private ThreadPoolExecutor workerExecutor;
private ThreadPoolExecutor databaseUpdaterExecutor;
private ThreadPoolExecutor notifierExecutor;
private ExecutorService feedProcessingLoopExecutor;
private ExecutorService refillLoopExecutor;
private ThreadPoolExecutor refillExecutor;
private ThreadPoolExecutor workerExecutor;
private ThreadPoolExecutor databaseUpdaterExecutor;
private ThreadPoolExecutor notifierExecutor;
public FeedRefreshEngine(UnitOfWork unitOfWork, FeedDAO feedDAO, FeedRefreshWorker worker, FeedRefreshUpdater updater,
FeedUpdateNotifier notifier, CommaFeedConfiguration config, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.worker = worker;
this.updater = updater;
this.notifier = notifier;
this.config = config;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
public FeedRefreshEngine(
UnitOfWork unitOfWork,
FeedDAO feedDAO,
FeedRefreshWorker worker,
FeedRefreshUpdater updater,
FeedUpdateNotifier notifier,
CommaFeedConfiguration config,
MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.worker = worker;
this.updater = updater;
this.notifier = notifier;
this.config = config;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
this.queue = new LinkedBlockingDeque<>();
this.queue = new LinkedBlockingDeque<>();
metrics.register(MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(MetricRegistry.name(getClass(), "worker", "active"), (Gauge<Integer>) () -> workerExecutor.getActiveCount());
metrics.register(MetricRegistry.name(getClass(), "updater", "active"),
(Gauge<Integer>) () -> databaseUpdaterExecutor.getActiveCount());
metrics.register(MetricRegistry.name(getClass(), "notifier", "active"), (Gauge<Integer>) () -> notifierExecutor.getActiveCount());
metrics.register(MetricRegistry.name(getClass(), "notifier", "queue"), (Gauge<Integer>) () -> notifierExecutor.getQueue().size());
}
metrics.register(
MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(
MetricRegistry.name(getClass(), "worker", "active"),
(Gauge<Integer>) () -> workerExecutor.getActiveCount());
metrics.register(
MetricRegistry.name(getClass(), "updater", "active"),
(Gauge<Integer>) () -> databaseUpdaterExecutor.getActiveCount());
metrics.register(
MetricRegistry.name(getClass(), "notifier", "active"),
(Gauge<Integer>) () -> notifierExecutor.getActiveCount());
metrics.register(
MetricRegistry.name(getClass(), "notifier", "queue"),
(Gauge<Integer>) () -> notifierExecutor.getQueue().size());
}
private void createExecutors() {
this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor();
this.refillLoopExecutor = Executors.newSingleThreadExecutor();
this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.feedRefresh().databaseThreads());
this.notifierExecutor = newDiscardingExecutorService(config.pushNotifications().threads(),
config.pushNotifications().queueCapacity());
}
private void createExecutors() {
this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor();
this.refillLoopExecutor = Executors.newSingleThreadExecutor();
this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor =
newBlockingExecutorService(config.feedRefresh().databaseThreads());
this.notifierExecutor =
newDiscardingExecutorService(
config.pushNotifications().threads(),
config.pushNotifications().queueCapacity());
}
public void start() {
createExecutors();
startFeedProcessingLoop();
startRefillLoop();
}
public void start() {
createExecutors();
startFeedProcessingLoop();
startRefillLoop();
}
private void startFeedProcessingLoop() {
// take a feed from the queue, process it, rince, repeat
feedProcessingLoopExecutor.submit(() -> {
while (!feedProcessingLoopExecutor.isShutdown()) {
try {
// take() is blocking until a feed is available from the queue
Feed feed = queue.take();
private void startFeedProcessingLoop() {
// take a feed from the queue, process it, rince, repeat
feedProcessingLoopExecutor.submit(
() -> {
while (!feedProcessingLoopExecutor.isShutdown()) {
try {
// take() is blocking until a feed is available from the queue
Feed feed = queue.take();
// send the feed to be processed
log.debug("got feed {} from the queue, send it for processing", feed.getId());
processFeedAsync(feed);
// send the feed to be processed
log.debug(
"got feed {} from the queue, send it for processing",
feed.getId());
processFeedAsync(feed);
// we removed a feed from the queue, try to refill it as it may now be empty
if (queue.isEmpty()) {
log.debug("took the last feed from the queue, try to refill");
refillQueueAsync();
}
} catch (InterruptedException e) {
log.debug("interrupted while waiting for a feed in the queue");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
// we removed a feed from the queue, try to refill it as it may now be
// empty
if (queue.isEmpty()) {
log.debug("took the last feed from the queue, try to refill");
refillQueueAsync();
}
} catch (InterruptedException e) {
log.debug("interrupted while waiting for a feed in the queue");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
private void startRefillLoop() {
// refill the queue at regular intervals if it's empty
refillLoopExecutor.submit(() -> {
while (!refillLoopExecutor.isShutdown()) {
try {
if (queue.isEmpty()) {
log.debug("refilling queue");
refillQueueAsync();
}
private void startRefillLoop() {
// refill the queue at regular intervals if it's empty
refillLoopExecutor.submit(
() -> {
while (!refillLoopExecutor.isShutdown()) {
try {
if (queue.isEmpty()) {
log.debug("refilling queue");
refillQueueAsync();
}
log.debug("sleeping for 15s");
TimeUnit.SECONDS.sleep(15);
} catch (InterruptedException e) {
log.debug("interrupted while sleeping");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
log.debug("sleeping for 15s");
TimeUnit.SECONDS.sleep(15);
} catch (InterruptedException e) {
log.debug("interrupted while sleeping");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
public void refreshImmediately(Feed feed) {
log.debug("add feed {} at the start of the queue", feed.getId());
// remove the feed from the queue if it was already queued to avoid refreshing it twice
queue.removeIf(f -> f.getId().equals(feed.getId()));
queue.addFirst(feed);
}
public void refreshImmediately(Feed feed) {
log.debug("add feed {} at the start of the queue", feed.getId());
// remove the feed from the queue if it was already queued to avoid refreshing it twice
queue.removeIf(f -> f.getId().equals(feed.getId()));
queue.addFirst(feed);
}
private void refillQueueAsync() {
CompletableFuture.runAsync(() -> {
if (!queue.isEmpty()) {
return;
}
private void refillQueueAsync() {
CompletableFuture.runAsync(
() -> {
if (!queue.isEmpty()) {
return;
}
refill.mark();
refill.mark();
List<Feed> nextUpdatableFeeds = getNextUpdatableFeeds(getBatchSize());
log.debug("found {} feeds that are up for refresh", nextUpdatableFeeds.size());
for (Feed feed : nextUpdatableFeeds) {
// add the feed only if it was not already queued
if (queue.stream().noneMatch(f -> f.getId().equals(feed.getId()))) {
queue.addLast(feed);
}
}
}, refillExecutor).whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while refilling the queue", ex);
}
});
}
List<Feed> nextUpdatableFeeds = getNextUpdatableFeeds(getBatchSize());
log.debug(
"found {} feeds that are up for refresh",
nextUpdatableFeeds.size());
for (Feed feed : nextUpdatableFeeds) {
// add the feed only if it was not already queued
if (queue.stream().noneMatch(f -> f.getId().equals(feed.getId()))) {
queue.addLast(feed);
}
}
},
refillExecutor)
.whenComplete(
(data, ex) -> {
if (ex != null) {
log.error("error while refilling the queue", ex);
}
});
}
private void processFeedAsync(Feed feed) {
CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
.thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
.thenCompose(r -> {
List<CompletableFuture<Void>> futures = r.insertedUnreadEntriesBySubscription().entrySet().stream().map(e -> {
FeedSubscription sub = e.getKey();
List<FeedEntry> entries = e.getValue();
private void processFeedAsync(Feed feed) {
CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
.thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
.thenCompose(
r -> {
List<CompletableFuture<Void>> futures =
r.insertedUnreadEntriesBySubscription().entrySet().stream()
.map(
e -> {
FeedSubscription sub = e.getKey();
List<FeedEntry> entries = e.getValue();
notifier.notifyOverWebsocket(sub, entries);
return CompletableFuture.runAsync(() -> notifier.sendPushNotifications(sub, entries), notifierExecutor);
}).toList();
return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new));
})
.exceptionally(ex -> {
log.error("error while processing feed {}", feed.getUrl(), ex);
return null;
});
}
notifier.notifyOverWebsocket(sub, entries);
return CompletableFuture.runAsync(
() ->
notifier
.sendPushNotifications(
sub,
entries),
notifierExecutor);
})
.toList();
return CompletableFuture.allOf(
futures.toArray(CompletableFuture[]::new));
})
.exceptionally(
ex -> {
log.error("error while processing feed {}", feed.getUrl(), ex);
return null;
});
}
private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(() -> {
Instant lastLoginThreshold = config.feedRefresh().userInactivityPeriod().isZero() ? null
: Instant.now().minus(config.feedRefresh().userInactivityPeriod());
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
if (!feeds.isEmpty()) {
// update disabledUntil to prevent feeds from being returned again by feedDAO.findNextUpdatable()
Instant nextUpdateDate = Instant.now().plus(config.feedRefresh().interval());
feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
}
return feeds;
});
}
private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(
() -> {
Instant lastLoginThreshold =
config.feedRefresh().userInactivityPeriod().isZero()
? null
: Instant.now()
.minus(config.feedRefresh().userInactivityPeriod());
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
if (!feeds.isEmpty()) {
// update disabledUntil to prevent feeds from being returned again by
// feedDAO.findNextUpdatable()
Instant nextUpdateDate =
Instant.now().plus(config.feedRefresh().interval());
feedDAO.setDisabledUntil(
feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
}
return feeds;
});
}
private int getBatchSize() {
return Math.min(100, 3 * config.feedRefresh().httpThreads());
}
private int getBatchSize() {
return Math.min(100, 3 * config.feedRefresh().httpThreads());
}
public void stop() {
MoreExecutors.shutdownAndAwaitTermination(this.feedProcessingLoopExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.refillLoopExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.refillExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.workerExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.databaseUpdaterExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.notifierExecutor, config.shutdownTimeout());
public void stop() {
MoreExecutors.shutdownAndAwaitTermination(
this.feedProcessingLoopExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(
this.refillLoopExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.refillExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.workerExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(
this.databaseUpdaterExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.notifierExecutor, config.shutdownTimeout());
queue.clear();
}
queue.clear();
}
/**
* returns an ExecutorService with a single thread that discards tasks if a task is already running
*/
private ThreadPoolExecutor newDiscardingSingleThreadExecutorService() {
ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/**
* returns an ExecutorService with a single thread that discards tasks if a task is already
* running
*/
private ThreadPoolExecutor newDiscardingSingleThreadExecutorService() {
ThreadPoolExecutor pool =
new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/**
* returns an ExecutorService that discards tasks if the queue is full
*/
private ThreadPoolExecutor newDiscardingExecutorService(int threads, int queueCapacity) {
ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(queueCapacity));
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/** returns an ExecutorService that discards tasks if the queue is full */
private ThreadPoolExecutor newDiscardingExecutorService(int threads, int queueCapacity) {
ThreadPoolExecutor pool =
new ThreadPoolExecutor(
threads,
threads,
0L,
TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(queueCapacity));
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/**
* returns an ExecutorService that blocks submissions until a thread is available
*/
private ThreadPoolExecutor newBlockingExecutorService(int threads) {
ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler((r, e) -> {
if (e.isShutdown()) {
return;
}
/** returns an ExecutorService that blocks submissions until a thread is available */
private ThreadPoolExecutor newBlockingExecutorService(int threads) {
ThreadPoolExecutor pool =
new ThreadPoolExecutor(
threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler(
(r, e) -> {
if (e.isShutdown()) {
return;
}
try {
e.getQueue().put(r);
} catch (InterruptedException ex) {
log.debug("interrupted while waiting for a slot in the queue.", ex);
Thread.currentThread().interrupt();
}
});
return pool;
}
try {
e.getQueue().put(r);
} catch (InterruptedException ex) {
log.debug("interrupted while waiting for a slot in the queue.", ex);
Thread.currentThread().interrupt();
}
});
return pool;
}
}

139
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshIntervalCalculator.java
View File

@@ -1,84 +1,95 @@
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.FeedRefreshErrorHandling;
import com.google.common.primitives.Longs;
import jakarta.inject.Singleton;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.time.temporal.ChronoUnit;
import org.apache.commons.lang3.ObjectUtils;
@Singleton
public class FeedRefreshIntervalCalculator {
private final Duration interval;
private final Duration maxInterval;
private final boolean empirical;
private final FeedRefreshErrorHandling errorHandling;
private final InstantSource instantSource;
private final Duration interval;
private final Duration maxInterval;
private final boolean empirical;
private final FeedRefreshErrorHandling errorHandling;
private final InstantSource instantSource;
public FeedRefreshIntervalCalculator(CommaFeedConfiguration config, InstantSource instantSource) {
this.interval = config.feedRefresh().interval();
this.maxInterval = config.feedRefresh().maxInterval();
this.empirical = config.feedRefresh().intervalEmpirical();
this.errorHandling = config.feedRefresh().errors();
this.instantSource = instantSource;
}
public FeedRefreshIntervalCalculator(
CommaFeedConfiguration config, InstantSource instantSource) {
this.interval = config.feedRefresh().interval();
this.maxInterval = config.feedRefresh().maxInterval();
this.empirical = config.feedRefresh().intervalEmpirical();
this.errorHandling = config.feedRefresh().errors();
this.instantSource = instantSource;
}
public Instant onFetchSuccess(Instant publishedDate, Long averageEntryInterval, Duration validFor) {
Instant instant = empirical ? computeEmpiricalRefreshInterval(publishedDate, averageEntryInterval)
: instantSource.instant().plus(interval);
return constrainToBounds(ObjectUtils.max(instant, instantSource.instant().plus(validFor)));
}
public Instant onFetchSuccess(
Instant publishedDate, Long averageEntryInterval, Duration validFor) {
Instant instant =
empirical
? computeEmpiricalRefreshInterval(publishedDate, averageEntryInterval)
: instantSource.instant().plus(interval);
return constrainToBounds(ObjectUtils.max(instant, instantSource.instant().plus(validFor)));
}
public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) {
return onFetchSuccess(publishedDate, averageEntryInterval, Duration.ZERO);
}
public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) {
return onFetchSuccess(publishedDate, averageEntryInterval, Duration.ZERO);
}
public Instant onTooManyRequests(Instant retryAfter, int errorCount) {
return constrainToBounds(ObjectUtils.max(retryAfter, onFetchError(errorCount)));
}
public Instant onTooManyRequests(Instant retryAfter, int errorCount) {
return constrainToBounds(ObjectUtils.max(retryAfter, onFetchError(errorCount)));
}
public Instant onFetchError(int errorCount) {
if (errorCount < errorHandling.retriesBeforeBackoff()) {
return constrainToBounds(instantSource.instant().plus(interval));
}
public Instant onFetchError(int errorCount) {
if (errorCount < errorHandling.retriesBeforeBackoff()) {
return constrainToBounds(instantSource.instant().plus(interval));
}
Duration retryInterval = errorHandling.backoffInterval().multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L);
return constrainToBounds(instantSource.instant().plus(retryInterval));
}
Duration retryInterval =
errorHandling
.backoffInterval()
.multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L);
return constrainToBounds(instantSource.instant().plus(retryInterval));
}
private Instant computeEmpiricalRefreshInterval(Instant publishedDate, Long averageEntryInterval) {
Instant now = instantSource.instant();
private Instant computeEmpiricalRefreshInterval(
Instant publishedDate, Long averageEntryInterval) {
Instant now = instantSource.instant();
if (publishedDate == null) {
return now.plus(maxInterval);
}
if (publishedDate == null) {
return now.plus(maxInterval);
}
long daysSinceLastPublication = ChronoUnit.DAYS.between(publishedDate, now);
if (daysSinceLastPublication >= 30) {
return now.plus(maxInterval);
} else if (daysSinceLastPublication >= 14) {
return now.plus(maxInterval.dividedBy(2));
} else if (daysSinceLastPublication >= 7) {
return now.plus(maxInterval.dividedBy(4));
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor
int factor = 2;
long millis = Longs.constrainToRange(averageEntryInterval / factor, interval.toMillis(), maxInterval.dividedBy(4).toMillis());
return now.plusMillis(millis);
} else {
// unknown case
return now.plus(maxInterval);
}
}
long daysSinceLastPublication = ChronoUnit.DAYS.between(publishedDate, now);
if (daysSinceLastPublication >= 30) {
return now.plus(maxInterval);
} else if (daysSinceLastPublication >= 14) {
return now.plus(maxInterval.dividedBy(2));
} else if (daysSinceLastPublication >= 7) {
return now.plus(maxInterval.dividedBy(4));
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor
int factor = 2;
long millis =
Longs.constrainToRange(
averageEntryInterval / factor,
interval.toMillis(),
maxInterval.dividedBy(4).toMillis());
return now.plusMillis(millis);
} else {
// unknown case
return now.plus(maxInterval);
}
}
private Instant constrainToBounds(Instant instant) {
return ObjectUtils.max(ObjectUtils.min(instant, instantSource.instant().plus(maxInterval)), instantSource.instant().plus(interval));
}
private Instant constrainToBounds(Instant instant) {
return ObjectUtils.max(
ObjectUtils.min(instant, instantSource.instant().plus(maxInterval)),
instantSource.instant().plus(interval));
}
}

270
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
View File

@@ -1,20 +1,5 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.Digests;
@@ -29,142 +14,171 @@ import com.commafeed.backend.model.Models;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.google.common.util.concurrent.Striped;
import jakarta.inject.Singleton;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
/**
* Updates the feed in the database and inserts new entries
*/
/** Updates the feed in the database and inserts new entries */
@Slf4j
@Singleton
public class FeedRefreshUpdater {
private final UnitOfWork unitOfWork;
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UnitOfWork unitOfWork;
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final Striped<Lock> locks;
private final Striped<Lock> locks;
private final Meter feedUpdated;
private final Meter entryInserted;
private final Meter feedUpdated;
private final Meter entryInserted;
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedSubscriptionDAO = feedSubscriptionDAO;
public FeedRefreshUpdater(
UnitOfWork unitOfWork,
FeedService feedService,
FeedEntryService feedEntryService,
MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedSubscriptionDAO = feedSubscriptionDAO;
locks = Striped.lazyWeakLock(100000);
locks = Striped.lazyWeakLock(100000);
feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
private AddEntryResult addEntry(final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
boolean processed = false;
FeedEntry insertedEntry = null;
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread = new HashSet<>();
private AddEntryResult addEntry(
final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
boolean processed = false;
FeedEntry insertedEntry = null;
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread = new HashSet<>();
// lock on feed, make sure we are not updating the same feed twice at
// the same time
String key1 = StringUtils.trimToEmpty(String.valueOf(feed.getId()));
// lock on feed, make sure we are not updating the same feed twice at
// the same time
String key1 = StringUtils.trimToEmpty(String.valueOf(feed.getId()));
// lock on content, make sure we are not updating the same entry
// twice at the same time
Content content = entry.content();
String key2 = Digests.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title()));
// lock on content, make sure we are not updating the same entry
// twice at the same time
Content content = entry.content();
String key2 = Digests.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title()));
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
Lock lock2 = iterator.next();
boolean locked1 = false;
boolean locked2 = false;
try {
// try to lock, give up after 1 minute
locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
processed = true;
insertedEntry = unitOfWork.call(() -> {
if (feedEntryService.find(feed, entry) != null) {
// entry already exists, nothing to do
return null;
}
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
Lock lock2 = iterator.next();
boolean locked1 = false;
boolean locked2 = false;
try {
// try to lock, give up after 1 minute
locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
processed = true;
insertedEntry =
unitOfWork.call(
() -> {
if (feedEntryService.find(feed, entry) != null) {
// entry already exists, nothing to do
return null;
}
FeedEntry feedEntry = feedEntryService.create(feed, entry);
entryInserted.mark();
for (FeedSubscription sub : subscriptions) {
boolean unread = feedEntryService.applyFilter(sub, feedEntry);
if (unread) {
subscriptionsForWhichEntryIsUnread.add(sub);
}
}
return feedEntry;
});
} else {
log.error("lock timeout for {} - {}", feed.getUrl(), key1);
}
} catch (InterruptedException e) {
log.error("interrupted while waiting for lock for {} : {}", feed.getUrl(), e.getMessage(), e);
Thread.currentThread().interrupt();
} finally {
if (locked1) {
lock1.unlock();
}
if (locked2) {
lock2.unlock();
}
}
return new AddEntryResult(processed, insertedEntry, subscriptionsForWhichEntryIsUnread);
}
FeedEntry feedEntry = feedEntryService.create(feed, entry);
entryInserted.mark();
for (FeedSubscription sub : subscriptions) {
boolean unread =
feedEntryService.applyFilter(sub, feedEntry);
if (unread) {
subscriptionsForWhichEntryIsUnread.add(sub);
}
}
return feedEntry;
});
} else {
log.error("lock timeout for {} - {}", feed.getUrl(), key1);
}
} catch (InterruptedException e) {
log.error(
"interrupted while waiting for lock for {} : {}",
feed.getUrl(),
e.getMessage(),
e);
Thread.currentThread().interrupt();
} finally {
if (locked1) {
lock1.unlock();
}
if (locked2) {
lock2.unlock();
}
}
return new AddEntryResult(processed, insertedEntry, subscriptionsForWhichEntryIsUnread);
}
public FeedRefreshUpdaterResult update(Feed feed, List<Entry> entries) {
boolean processed = true;
long inserted = 0;
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription = new HashMap<>();
public FeedRefreshUpdaterResult update(Feed feed, List<Entry> entries) {
boolean processed = true;
long inserted = 0;
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription =
new HashMap<>();
if (!entries.isEmpty()) {
List<FeedSubscription> subscriptions = null;
List<Entry> newEntries = unitOfWork.call(() -> feedEntryService.removeExistingEntries(feed, entries));
for (Entry entry : newEntries) {
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
processed &= addEntryResult.processed;
inserted += addEntryResult.insertedEntry != null ? 1 : 0;
addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(sub -> {
if (addEntryResult.insertedEntry != null) {
insertedUnreadEntriesBySubscription.computeIfAbsent(sub, k -> new ArrayList<>()).add(addEntryResult.insertedEntry);
}
});
}
if (!entries.isEmpty()) {
List<FeedSubscription> subscriptions = null;
List<Entry> newEntries =
unitOfWork.call(() -> feedEntryService.removeExistingEntries(feed, entries));
for (Entry entry : newEntries) {
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
processed &= addEntryResult.processed;
inserted += addEntryResult.insertedEntry != null ? 1 : 0;
addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(
sub -> {
if (addEntryResult.insertedEntry != null) {
insertedUnreadEntriesBySubscription
.computeIfAbsent(sub, k -> new ArrayList<>())
.add(addEntryResult.insertedEntry);
}
});
}
if (inserted == 0) {
feed.setMessage("No new entries found");
} else if (inserted > 0) {
feed.setMessage("Found %s new entries".formatted(inserted));
}
}
if (inserted == 0) {
feed.setMessage("No new entries found");
} else if (inserted > 0) {
feed.setMessage("Found %s new entries".formatted(inserted));
}
}
if (!processed) {
// requeue asap
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
}
if (!processed) {
// requeue asap
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
}
if (inserted > 0) {
feedUpdated.mark();
}
if (inserted > 0) {
feedUpdated.mark();
}
unitOfWork.run(() -> feedService.update(feed));
unitOfWork.run(() -> feedService.update(feed));
return new FeedRefreshUpdaterResult(insertedUnreadEntriesBySubscription);
}
return new FeedRefreshUpdaterResult(insertedUnreadEntriesBySubscription);
}
private record AddEntryResult(boolean processed, FeedEntry insertedEntry, Set<FeedSubscription> subscriptionsForWhichEntryIsUnread) {}
public record FeedRefreshUpdaterResult(Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription) {}
private record AddEntryResult(
boolean processed,
FeedEntry insertedEntry,
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread) {}
public record FeedRefreshUpdaterResult(
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription) {}
}

190
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
View File

@@ -1,15 +1,5 @@
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.Strings;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
@@ -18,108 +8,132 @@ import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import jakarta.inject.Singleton;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.Strings;
/**
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link FeedRefreshUpdater} does that)
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link
* FeedRefreshUpdater} does that)
*/
@Slf4j
@Singleton
public class FeedRefreshWorker {
private final FeedRefreshIntervalCalculator refreshIntervalCalculator;
private final FeedFetcher fetcher;
private final CommaFeedConfiguration config;
private final Meter feedFetched;
private final FeedRefreshIntervalCalculator refreshIntervalCalculator;
private final FeedFetcher fetcher;
private final CommaFeedConfiguration config;
private final Meter feedFetched;
public FeedRefreshWorker(FeedRefreshIntervalCalculator refreshIntervalCalculator, FeedFetcher fetcher, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.refreshIntervalCalculator = refreshIntervalCalculator;
this.fetcher = fetcher;
this.config = config;
this.feedFetched = metrics.meter(MetricRegistry.name(getClass(), "feedFetched"));
public FeedRefreshWorker(
FeedRefreshIntervalCalculator refreshIntervalCalculator,
FeedFetcher fetcher,
CommaFeedConfiguration config,
MetricRegistry metrics) {
this.refreshIntervalCalculator = refreshIntervalCalculator;
this.fetcher = fetcher;
this.config = config;
this.feedFetched = metrics.meter(MetricRegistry.name(getClass(), "feedFetched"));
}
}
public FeedRefreshWorkerResult update(Feed feed) {
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FeedFetcherResult result =
fetcher.fetch(
url,
false,
feed.getLastModifiedHeader(),
feed.getEtagHeader(),
feed.getLastPublishedDate(),
feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
public FeedRefreshWorkerResult update(Feed feed) {
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FeedFetcherResult result = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
List<Entry> entries = result.feed().entries();
List<Entry> entries = result.feed().entries();
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
entries =
entries.stream()
.filter(entry -> entry.published().isAfter(threshold))
.toList();
}
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
entries = entries.stream().filter(entry -> entry.published().isAfter(threshold)).toList();
}
String urlAfterRedirect = result.urlAfterRedirect();
if (Strings.CS.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;
}
String urlAfterRedirect = result.urlAfterRedirect();
if (Strings.CS.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;
}
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(result.feed().link());
feed.setIconUrl(result.feed().iconUrl());
feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(result.contentHash());
feed.setLastPublishedDate(result.feed().lastPublishedDate());
feed.setAverageEntryInterval(result.feed().averageEntryInterval());
feed.setLastEntryDate(result.feed().lastEntryDate());
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(result.feed().link());
feed.setIconUrl(result.feed().iconUrl());
feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(result.contentHash());
feed.setLastPublishedDate(result.feed().lastPublishedDate());
feed.setAverageEntryInterval(result.feed().averageEntryInterval());
feed.setLastEntryDate(result.feed().lastEntryDate());
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(
refreshIntervalCalculator.onFetchSuccess(
result.feed().lastPublishedDate(),
result.feed().averageEntryInterval(),
result.validFor()));
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(),
result.feed().averageEntryInterval(), result.validFor()));
return new FeedRefreshWorkerResult(feed, entries);
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
return new FeedRefreshWorkerResult(feed, entries);
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
feed.setErrorCount(0);
feed.setMessage(e.getMessage());
feed.setDisabledUntil(
refreshIntervalCalculator.onFeedNotModified(
feed.getLastPublishedDate(), feed.getAverageEntryInterval()));
feed.setErrorCount(0);
feed.setMessage(e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFeedNotModified(feed.getLastPublishedDate(), feed.getAverageEntryInterval()));
if (e.getNewLastModifiedHeader() != null) {
feed.setLastModifiedHeader(e.getNewLastModifiedHeader());
}
if (e.getNewLastModifiedHeader() != null) {
feed.setLastModifiedHeader(e.getNewLastModifiedHeader());
}
if (e.getNewEtagHeader() != null) {
feed.setEtagHeader(e.getNewEtagHeader());
}
if (e.getNewEtagHeader() != null) {
feed.setEtagHeader(e.getNewEtagHeader());
}
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (TooManyRequestsException e) {
log.debug("Too many requests : {}", feed.getUrl());
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (TooManyRequestsException e) {
log.debug("Too many requests : {}", feed.getUrl());
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Server indicated that we are sending too many requests");
feed.setDisabledUntil(
refreshIntervalCalculator.onTooManyRequests(
e.getRetryAfter(), feed.getErrorCount()));
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Server indicated that we are sending too many requests");
feed.setDisabledUntil(refreshIntervalCalculator.onTooManyRequests(e.getRetryAfter(), feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (Exception e) {
log.debug("unable to refresh feed {}", feed.getUrl(), e);
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (Exception e) {
log.debug("unable to refresh feed {}", feed.getUrl(), e);
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Unable to refresh feed : " + e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount()));
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Unable to refresh feed : " + e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} finally {
feedFetched.mark();
}
}
public record FeedRefreshWorkerResult(Feed feed, List<Entry> entries) {}
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} finally {
feedFetched.mark();
}
}
public record FeedRefreshWorkerResult(Feed feed, List<Entry> entries) {}
}

55
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUpdateNotifier.java
View File

@@ -1,9 +1,5 @@
package com.commafeed.backend.feed;
import java.util.List;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserSettingsDAO;
@@ -13,7 +9,8 @@ import com.commafeed.backend.model.UserSettings;
import com.commafeed.backend.service.PushNotificationService;
import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions;
import jakarta.inject.Singleton;
import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -22,29 +19,33 @@ import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
public class FeedUpdateNotifier {
private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork;
private final UserSettingsDAO userSettingsDAO;
private final WebSocketSessions webSocketSessions;
private final PushNotificationService pushNotificationService;
private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork;
private final UserSettingsDAO userSettingsDAO;
private final WebSocketSessions webSocketSessions;
private final PushNotificationService pushNotificationService;
public void notifyOverWebsocket(FeedSubscription sub, List<FeedEntry> entries) {
if (!entries.isEmpty()) {
webSocketSessions.sendMessage(sub.getUser(), WebSocketMessageBuilder.newFeedEntries(sub, entries.size()));
}
}
public void notifyOverWebsocket(FeedSubscription sub, List<FeedEntry> entries) {
if (!entries.isEmpty()) {
webSocketSessions.sendMessage(
sub.getUser(), WebSocketMessageBuilder.newFeedEntries(sub, entries.size()));
}
}
public void sendPushNotifications(FeedSubscription sub, List<FeedEntry> entries) {
if (!config.pushNotifications().enabled() || !sub.isPushNotificationsEnabled() || entries.isEmpty()) {
return;
}
UserSettings settings = unitOfWork.call(() -> userSettingsDAO.findByUser(sub.getUser()));
if (settings != null && settings.getPushNotifications() != null && settings.getPushNotifications().getType() != null) {
for (FeedEntry entry : entries) {
pushNotificationService.notify(settings.getPushNotifications(), sub, entry);
}
}
}
public void sendPushNotifications(FeedSubscription sub, List<FeedEntry> entries) {
if (!config.pushNotifications().enabled()
|| !sub.isPushNotificationsEnabled()
|| entries.isEmpty()) {
return;
}
UserSettings settings = unitOfWork.call(() -> userSettingsDAO.findByUser(sub.getUser()));
if (settings != null
&& settings.getPushNotifications() != null
&& settings.getPushNotifications().getType() != null) {
for (FeedEntry entry : entries) {
pushNotificationService.notify(settings.getPushNotifications(), sub, entry);
}
}
}
}

136
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java
View File

@@ -1,14 +1,5 @@
package com.commafeed.backend.feed;
import java.util.Collections;
import java.util.Date;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.commafeed.backend.feed.parser.TextDirectionDetector;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry;
@@ -16,87 +7,90 @@ import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEnclosureImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndEntryImpl;
import java.util.Collections;
import java.util.Date;
import lombok.experimental.UtilityClass;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/**
* Utility methods related to feed handling
*
*/
/** Utility methods related to feed handling */
@UtilityClass
@Slf4j
public class FeedUtils {
public static String truncate(String string, int length) {
return StringUtils.truncate(string, length);
}
public static String truncate(String string, int length) {
return StringUtils.truncate(string, length);
}
public static boolean isRTL(String title, String content) {
String text = StringUtils.isNotBlank(content) ? content : title;
if (StringUtils.isBlank(text)) {
return false;
}
public static boolean isRTL(String title, String content) {
String text = StringUtils.isNotBlank(content) ? content : title;
if (StringUtils.isBlank(text)) {
return false;
}
String stripped = Jsoup.parse(text).text();
if (StringUtils.isBlank(stripped)) {
return false;
}
String stripped = Jsoup.parse(text).text();
if (StringUtils.isBlank(stripped)) {
return false;
}
return TextDirectionDetector.detect(stripped) == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
return TextDirectionDetector.detect(stripped)
== TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
public static String getFaviconUrl(FeedSubscription subscription) {
return "rest/feed/favicon/" + subscription.getId();
}
public static String getFaviconUrl(FeedSubscription subscription) {
return "rest/feed/favicon/" + subscription.getId();
}
public static String proxyImages(String content) {
if (StringUtils.isBlank(content)) {
return content;
}
public static String proxyImages(String content) {
if (StringUtils.isBlank(content)) {
return content;
}
Document doc = Jsoup.parse(content);
Elements elements = doc.select("img");
for (Element element : elements) {
String href = element.attr("src");
if (StringUtils.isNotBlank(href)) {
String proxy = proxyImage(href);
element.attr("src", proxy);
}
}
Document doc = Jsoup.parse(content);
Elements elements = doc.select("img");
for (Element element : elements) {
String href = element.attr("src");
if (StringUtils.isNotBlank(href)) {
String proxy = proxyImage(href);
element.attr("src", proxy);
}
}
return doc.body().html();
}
return doc.body().html();
}
public static String proxyImage(String url) {
if (StringUtils.isBlank(url)) {
return url;
}
public static String proxyImage(String url) {
if (StringUtils.isBlank(url)) {
return url;
}
return "rest/server/proxy?u=" + ImageProxyUrl.encode(url);
}
return "rest/server/proxy?u=" + ImageProxyUrl.encode(url);
}
public static SyndEntry asRss(Entry entry) {
SyndEntry e = new SyndEntryImpl();
public static SyndEntry asRss(Entry entry) {
SyndEntry e = new SyndEntryImpl();
e.setUri(entry.getGuid());
e.setTitle(entry.getTitle());
e.setAuthor(entry.getAuthor());
e.setUri(entry.getGuid());
e.setTitle(entry.getTitle());
e.setAuthor(entry.getAuthor());
SyndContentImpl c = new SyndContentImpl();
c.setValue(entry.getContent());
e.setContents(Collections.singletonList(c));
SyndContentImpl c = new SyndContentImpl();
c.setValue(entry.getContent());
e.setContents(Collections.singletonList(c));
if (entry.getEnclosureUrl() != null) {
SyndEnclosureImpl enclosure = new SyndEnclosureImpl();
enclosure.setType(entry.getEnclosureType());
enclosure.setUrl(entry.getEnclosureUrl());
e.setEnclosures(Collections.singletonList(enclosure));
}
e.setLink(entry.getUrl());
e.setPublishedDate(entry.getDate() == null ? null : Date.from(entry.getDate()));
return e;
}
if (entry.getEnclosureUrl() != null) {
SyndEnclosureImpl enclosure = new SyndEnclosureImpl();
enclosure.setType(entry.getEnclosureType());
enclosure.setUrl(entry.getEnclosureUrl());
e.setEnclosures(Collections.singletonList(enclosure));
}
e.setLink(entry.getUrl());
e.setPublishedDate(entry.getDate() == null ? null : Date.from(entry.getDate()));
return e;
}
}

83
commafeed-server/src/main/java/com/commafeed/backend/feed/ImageProxyUrl.java
View File

@@ -1,70 +1,65 @@
package com.commafeed.backend.feed;
import com.google.common.primitives.Bytes;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Base64;
import javax.crypto.Cipher;
import javax.crypto.SecretKey;
import javax.crypto.spec.GCMParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import org.apache.commons.lang3.RandomUtils;
import com.google.common.primitives.Bytes;
import lombok.experimental.UtilityClass;
import org.apache.commons.lang3.RandomUtils;
@UtilityClass
public class ImageProxyUrl {
private static final int GCM_IV_LENGTH = 12;
private static final int GCM_TAG_LENGTH = 128;
private static final int GCM_IV_LENGTH = 12;
private static final int GCM_TAG_LENGTH = 128;
private static SecretKey key;
private static SecretKey key;
public static void generateKey() {
key = new SecretKeySpec(RandomUtils.secure().randomBytes(32), "AES");
}
public static void generateKey() {
key = new SecretKeySpec(RandomUtils.secure().randomBytes(32), "AES");
}
public static String encode(String url) {
if (key == null) {
throw new IllegalStateException("Key not initialized");
}
public static String encode(String url) {
if (key == null) {
throw new IllegalStateException("Key not initialized");
}
try {
byte[] iv = RandomUtils.secure().randomBytes(GCM_IV_LENGTH);
try {
byte[] iv = RandomUtils.secure().randomBytes(GCM_IV_LENGTH);
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
cipher.init(Cipher.ENCRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_LENGTH, iv));
byte[] encrypted = cipher.doFinal(url.getBytes(StandardCharsets.UTF_8));
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
cipher.init(Cipher.ENCRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_LENGTH, iv));
byte[] encrypted = cipher.doFinal(url.getBytes(StandardCharsets.UTF_8));
byte[] combined = Bytes.concat(iv, encrypted);
return Base64.getUrlEncoder().withoutPadding().encodeToString(combined);
} catch (Exception e) {
throw new IllegalStateException("Failed to encode URL", e);
}
}
byte[] combined = Bytes.concat(iv, encrypted);
return Base64.getUrlEncoder().withoutPadding().encodeToString(combined);
} catch (Exception e) {
throw new IllegalStateException("Failed to encode URL", e);
}
}
public static String decode(String code) {
if (key == null) {
throw new IllegalStateException("Key not initialized");
}
public static String decode(String code) {
if (key == null) {
throw new IllegalStateException("Key not initialized");
}
try {
byte[] combined = Base64.getUrlDecoder().decode(code);
try {
byte[] combined = Base64.getUrlDecoder().decode(code);
byte[] iv = Arrays.copyOfRange(combined, 0, GCM_IV_LENGTH);
byte[] encrypted = Arrays.copyOfRange(combined, GCM_IV_LENGTH, combined.length);
byte[] iv = Arrays.copyOfRange(combined, 0, GCM_IV_LENGTH);
byte[] encrypted = Arrays.copyOfRange(combined, GCM_IV_LENGTH, combined.length);
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_LENGTH, iv));
byte[] decrypted = cipher.doFinal(encrypted);
return new String(decrypted, StandardCharsets.UTF_8);
} catch (Exception e) {
throw new IllegalStateException("Failed to decode URL", e);
}
}
Cipher cipher = Cipher.getInstance("AES/GCM/NoPadding");
cipher.init(Cipher.DECRYPT_MODE, key, new GCMParameterSpec(GCM_TAG_LENGTH, iv));
byte[] decrypted = cipher.doFinal(encrypted);
return new String(decrypted, StandardCharsets.UTF_8);
} catch (Exception e) {
throw new IllegalStateException("Failed to decode URL", e);
}
}
}

105
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/EncodingDetector.java
View File

@@ -1,70 +1,61 @@
package com.commafeed.backend.feed.parser;
import java.nio.charset.Charset;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.Strings;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
import jakarta.inject.Singleton;
import java.nio.charset.Charset;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.Strings;
@Singleton
public class EncodingDetector {
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public Charset getEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (Strings.CI.startsWith(extracted, "iso-8859-")) {
if (!Strings.CS.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (Strings.CI.startsWith(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by
* detecting the characters used in the feed
*/
public Charset getEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (Strings.CI.startsWith(extracted, "iso-8859-")) {
if (!Strings.CS.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (Strings.CI.startsWith(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Extract the declared encoding from the xml
*/
public String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
/** Extract the declared encoding from the xml */
public String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = Strings.CS.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = Strings.CS.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
/**
* Detect encoding by analyzing characters in the array
*/
private Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
/** Detect encoding by analyzing characters in the array */
private Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
}

468
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParser.java
View File

@@ -1,25 +1,5 @@
package com.commafeed.backend.feed.parser;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemProperties;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.Urls;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
@@ -38,258 +18,290 @@ import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.SyndFeedInput;
import jakarta.inject.Singleton;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemProperties;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
/**
* Parses raw xml into a FeedParserResult object
*/
/** Parses raw xml into a FeedParserResult object */
@Singleton
public class FeedParser {
private static final Namespace ATOM_10_NS = Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Namespace ATOM_10_NS =
Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Instant START = Instant.ofEpochMilli(86400000);
private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
private static final Instant START = Instant.ofEpochMilli(86400000);
private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
private static final Comparator<Entry> ENTRY_COMPARATOR = Comparator.comparing(Entry::published).reversed();
private static final Comparator<Entry> ENTRY_COMPARATOR =
Comparator.comparing(Entry::published).reversed();
private final EncodingDetector encodingDetector;
private final XMLCleaner xmlCleaner;
private final EncodingDetector encodingDetector;
private final XMLCleaner xmlCleaner;
public FeedParser(EncodingDetector encodingDetector, XMLCleaner xmlCleaner) {
this.encodingDetector = encodingDetector;
this.xmlCleaner = xmlCleaner;
public FeedParser(EncodingDetector encodingDetector, XMLCleaner xmlCleaner) {
this.encodingDetector = encodingDetector;
this.xmlCleaner = xmlCleaner;
// disable entity expansion limits added in JDK24+ (#1961)
// we already strip doctype declarations in XMLCleaner to prevent xxe attacks
// we also already limit the size of feeds we download in HttpGetter
System.setProperty(SystemProperties.JDK_XML_MAX_GENERAL_ENTITY_SIZE_LIMIT, "0");
System.setProperty(SystemProperties.JDK_XML_TOTAL_ENTITY_SIZE_LIMIT, "0");
}
// disable entity expansion limits added in JDK24+ (#1961)
// we already strip doctype declarations in XMLCleaner to prevent xxe attacks
// we also already limit the size of feeds we download in HttpGetter
System.setProperty(SystemProperties.JDK_XML_MAX_GENERAL_ENTITY_SIZE_LIMIT, "0");
System.setProperty(SystemProperties.JDK_XML_TOTAL_ENTITY_SIZE_LIMIT, "0");
}
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedParsingException {
try {
Charset encoding = encodingDetector.getEncoding(xml);
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedParsingException {
try {
Charset encoding = encodingDetector.getEncoding(xml);
String xmlString = xmlCleaner.clean(new String(xml, encoding));
if (xmlString == null) {
throw new FeedParsingException("Input string is empty for url " + feedUrl);
}
String xmlString = xmlCleaner.clean(new String(xml, encoding));
if (xmlString == null) {
throw new FeedParsingException("Input string is empty for url " + feedUrl);
}
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed feed = new SyndFeedInput().build(source);
handleForeignMarkup(feed);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed feed = new SyndFeedInput().build(source);
handleForeignMarkup(feed);
String title = feed.getTitle();
String link = feed.getLink();
String iconUrl = feed.getIcon() != null ? feed.getIcon().getUrl() : null;
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
if (lastPublishedDate == null || lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
lastPublishedDate = lastEntryDate;
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
String title = feed.getTitle();
String link = feed.getLink();
String iconUrl = feed.getIcon() != null ? feed.getIcon().getUrl() : null;
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
if (lastPublishedDate == null
|| lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
lastPublishedDate = lastEntryDate;
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
return new FeedParserResult(title, link, iconUrl, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
} catch (FeedParsingException e) {
throw e;
} catch (Exception e) {
throw new FeedParsingException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
return new FeedParserResult(
title,
link,
iconUrl,
lastPublishedDate,
averageEntryInterval,
lastEntryDate,
entries);
} catch (FeedParsingException e) {
throw e;
} catch (Exception e) {
throw new FeedParsingException(
String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
/**
* Adds atom links for rss feeds
*/
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
/** Adds atom links for rss feeds */
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
private List<Entry> buildEntries(SyndFeed feed, String feedUrl) {
List<Entry> entries = new ArrayList<>();
private List<Entry> buildEntries(SyndFeed feed, String feedUrl) {
List<Entry> entries = new ArrayList<>();
for (SyndEntry item : feed.getEntries()) {
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
for (SyndEntry item : feed.getEntries()) {
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
String url = buildEntryUrl(feed, feedUrl, item);
if (StringUtils.isBlank(url) && Urls.isAbsolute(guid)) {
// if link is empty but guid is used as url, use guid
url = guid;
}
String url = buildEntryUrl(feed, feedUrl, item);
if (StringUtils.isBlank(url) && Urls.isAbsolute(guid)) {
// if link is empty but guid is used as url, use guid
url = guid;
}
Instant publishedDate = buildEntryPublishedDate(item);
Content content = buildContent(item);
Instant publishedDate = buildEntryPublishedDate(item);
Content content = buildContent(item);
entries.add(new Entry(guid, url, publishedDate, content));
}
entries.add(new Entry(guid, url, publishedDate, content));
}
entries.sort(ENTRY_COMPARATOR);
return entries;
}
entries.sort(ENTRY_COMPARATOR);
return entries;
}
private Content buildContent(SyndEntry item) {
String title = getTitle(item);
String content = getContent(item);
String author = StringUtils.trimToNull(item.getAuthor());
String categories = StringUtils
.trimToNull(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")));
private Content buildContent(SyndEntry item) {
String title = getTitle(item);
String content = getContent(item);
String author = StringUtils.trimToNull(item.getAuthor());
String categories =
StringUtils.trimToNull(
item.getCategories().stream()
.map(SyndCategory::getName)
.collect(Collectors.joining(", ")));
Enclosure enclosure = buildEnclosure(item);
Media media = buildMedia(item);
return new Content(title, content, author, categories, enclosure, media);
}
Enclosure enclosure = buildEnclosure(item);
Media media = buildMedia(item);
return new Content(title, content, author, categories, enclosure, media);
}
private Enclosure buildEnclosure(SyndEntry item) {
SyndEnclosure enclosure = item.getEnclosures().stream().findFirst().orElse(null);
if (enclosure == null) {
return null;
}
private Enclosure buildEnclosure(SyndEntry item) {
SyndEnclosure enclosure = item.getEnclosures().stream().findFirst().orElse(null);
if (enclosure == null) {
return null;
}
return new Enclosure(enclosure.getUrl(), enclosure.getType());
}
return new Enclosure(enclosure.getUrl(), enclosure.getType());
}
private Instant buildEntryPublishedDate(SyndEntry item) {
Date date = item.getPublishedDate();
if (date == null) {
date = item.getUpdatedDate();
}
return toValidInstant(date, true);
}
private Instant buildEntryPublishedDate(SyndEntry item) {
Date date = item.getPublishedDate();
if (date == null) {
date = item.getUpdatedDate();
}
return toValidInstant(date, true);
}
private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
if (url == null || Urls.isAbsolute(url)) {
// url is absolute, nothing to do
return url;
}
private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
if (url == null || Urls.isAbsolute(url)) {
// url is absolute, nothing to do
return url;
}
// url is relative, trying to resolve it
String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
return Urls.toAbsolute(url, feedLink, feedUrl);
}
// url is relative, trying to resolve it
String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
return Urls.toAbsolute(url, feedLink, feedUrl);
}
private Instant toValidInstant(Date date, boolean nullToNow) {
Instant now = Instant.now();
if (date == null) {
return nullToNow ? now : null;
}
private Instant toValidInstant(Date date, boolean nullToNow) {
Instant now = Instant.now();
if (date == null) {
return nullToNow ? now : null;
}
Instant instant = date.toInstant();
if (instant.isBefore(START) || instant.isAfter(END)) {
return now;
}
Instant instant = date.toInstant();
if (instant.isBefore(START) || instant.isAfter(END)) {
return now;
}
if (instant.isAfter(now)) {
return now;
}
return instant;
}
if (instant.isAfter(now)) {
return now;
}
return instant;
}
private String getContent(SyndEntry item) {
String content;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getContent(SyndEntry item) {
String content;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content =
item.getContents().stream()
.map(SyndContent::getValue)
.collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private Media buildMedia(SyndEntry item) {
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module == null) {
return null;
}
private Media buildMedia(SyndEntry item) {
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module == null) {
return null;
}
Media media = buildMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = buildMedia(group.getMetadata());
}
Media media = buildMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = buildMedia(group.getMetadata());
}
return media;
}
return media;
}
private Media buildMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
private Media buildMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
String description = metadata.getDescription();
String description = metadata.getDescription();
String thumbnailUrl = null;
Integer thumbnailWidth = null;
Integer thumbnailHeight = null;
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
thumbnailWidth = thumbnail.getWidth();
thumbnailHeight = thumbnail.getHeight();
if (thumbnail.getUrl() != null) {
thumbnailUrl = thumbnail.getUrl().toString();
}
}
String thumbnailUrl = null;
Integer thumbnailWidth = null;
Integer thumbnailHeight = null;
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
thumbnailWidth = thumbnail.getWidth();
thumbnailHeight = thumbnail.getHeight();
if (thumbnail.getUrl() != null) {
thumbnailUrl = thumbnail.getUrl().toString();
}
}
if (description == null && thumbnailUrl == null) {
return null;
}
if (description == null && thumbnailUrl == null) {
return null;
}
return new Media(description, thumbnailUrl, thumbnailWidth, thumbnailHeight);
}
return new Media(description, thumbnailUrl, thumbnailWidth, thumbnailHeight);
}
private Long averageTimeBetweenEntries(List<Entry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
private Long averageTimeBetweenEntries(List<Entry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < entries.size() - 1; i++) {
long diff = Math.abs(entries.get(i).published().toEpochMilli() - entries.get(i + 1).published().toEpochMilli());
stats.addValue(diff);
}
return (long) stats.getMean();
}
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < entries.size() - 1; i++) {
long diff =
Math.abs(
entries.get(i).published().toEpochMilli()
- entries.get(i + 1).published().toEpochMilli());
stats.addValue(diff);
}
return (long) stats.getMean();
}
public static class FeedParsingException extends Exception {
private static final long serialVersionUID = 1L;
public static class FeedParsingException extends Exception {
private static final long serialVersionUID = 1L;
public FeedParsingException(String message) {
super(message);
}
public FeedParsingException(String message, Throwable cause) {
super(message, cause);
}
}
public FeedParsingException(String message) {
super(message);
}
public FeedParsingException(String message, Throwable cause) {
super(message, cause);
}
}
}

28
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParserResult.java
View File

@@ -3,13 +3,29 @@ package com.commafeed.backend.feed.parser;
import java.time.Instant;
import java.util.List;
public record FeedParserResult(String title, String link, String iconUrl, Instant lastPublishedDate, Long averageEntryInterval,
Instant lastEntryDate, List<Entry> entries) {
public record Entry(String guid, String url, Instant published, Content content) {}
public record FeedParserResult(
String title,
String link,
String iconUrl,
Instant lastPublishedDate,
Long averageEntryInterval,
Instant lastEntryDate,
List<Entry> entries) {
public record Entry(String guid, String url, Instant published, Content content) {}
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {}
public record Content(
String title,
String content,
String author,
String categories,
Enclosure enclosure,
Media media) {}
public record Enclosure(String url, String type) {}
public record Enclosure(String url, String type) {}
public record Media(String description, String thumbnailUrl, Integer thumbnailWidth, Integer thumbnailHeight) {}
public record Media(
String description,
String thumbnailUrl,
Integer thumbnailWidth,
Integer thumbnailHeight) {}
}

515
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/HtmlEntities.java
View File

@@ -4,268 +4,267 @@ import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import lombok.experimental.UtilityClass;
@UtilityClass
class HtmlEntities {
public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final List<String> HTML_ENTITIES;
public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final List<String> HTML_ENTITIES;
static {
Map<String, String> map = new LinkedHashMap<>();
map.put("&Aacute;", "&#193;");
map.put("&aacute;", "&#225;");
map.put("&Acirc;", "&#194;");
map.put("&acirc;", "&#226;");
map.put("&acute;", "&#180;");
map.put("&AElig;", "&#198;");
map.put("&aelig;", "&#230;");
map.put("&Agrave;", "&#192;");
map.put("&agrave;", "&#224;");
map.put("&alefsym;", "&#8501;");
map.put("&Alpha;", "&#913;");
map.put("&alpha;", "&#945;");
map.put("&amp;", "&#38;");
map.put("&and;", "&#8743;");
map.put("&ang;", "&#8736;");
map.put("&Aring;", "&#197;");
map.put("&aring;", "&#229;");
map.put("&asymp;", "&#8776;");
map.put("&Atilde;", "&#195;");
map.put("&atilde;", "&#227;");
map.put("&Auml;", "&#196;");
map.put("&auml;", "&#228;");
map.put("&bdquo;", "&#8222;");
map.put("&Beta;", "&#914;");
map.put("&beta;", "&#946;");
map.put("&brvbar;", "&#166;");
map.put("&bull;", "&#8226;");
map.put("&cap;", "&#8745;");
map.put("&Ccedil;", "&#199;");
map.put("&ccedil;", "&#231;");
map.put("&cedil;", "&#184;");
map.put("&cent;", "&#162;");
map.put("&Chi;", "&#935;");
map.put("&chi;", "&#967;");
map.put("&circ;", "&#710;");
map.put("&clubs;", "&#9827;");
map.put("&cong;", "&#8773;");
map.put("&copy;", "&#169;");
map.put("&crarr;", "&#8629;");
map.put("&cup;", "&#8746;");
map.put("&curren;", "&#164;");
map.put("&dagger;", "&#8224;");
map.put("&Dagger;", "&#8225;");
map.put("&darr;", "&#8595;");
map.put("&dArr;", "&#8659;");
map.put("&deg;", "&#176;");
map.put("&Delta;", "&#916;");
map.put("&delta;", "&#948;");
map.put("&diams;", "&#9830;");
map.put("&divide;", "&#247;");
map.put("&Eacute;", "&#201;");
map.put("&eacute;", "&#233;");
map.put("&Ecirc;", "&#202;");
map.put("&ecirc;", "&#234;");
map.put("&Egrave;", "&#200;");
map.put("&egrave;", "&#232;");
map.put("&empty;", "&#8709;");
map.put("&emsp;", "&#8195;");
map.put("&ensp;", "&#8194;");
map.put("&Epsilon;", "&#917;");
map.put("&epsilon;", "&#949;");
map.put("&equiv;", "&#8801;");
map.put("&Eta;", "&#919;");
map.put("&eta;", "&#951;");
map.put("&ETH;", "&#208;");
map.put("&eth;", "&#240;");
map.put("&Euml;", "&#203;");
map.put("&euml;", "&#235;");
map.put("&euro;", "&#8364;");
map.put("&exist;", "&#8707;");
map.put("&fnof;", "&#402;");
map.put("&forall;", "&#8704;");
map.put("&frac12;", "&#189;");
map.put("&frac14;", "&#188;");
map.put("&frac34;", "&#190;");
map.put("&frasl;", "&#8260;");
map.put("&Gamma;", "&#915;");
map.put("&gamma;", "&#947;");
map.put("&ge;", "&#8805;");
map.put("&harr;", "&#8596;");
map.put("&hArr;", "&#8660;");
map.put("&hearts;", "&#9829;");
map.put("&hellip;", "&#8230;");
map.put("&Iacute;", "&#205;");
map.put("&iacute;", "&#237;");
map.put("&Icirc;", "&#206;");
map.put("&icirc;", "&#238;");
map.put("&iexcl;", "&#161;");
map.put("&Igrave;", "&#204;");
map.put("&igrave;", "&#236;");
map.put("&image;", "&#8465;");
map.put("&infin;", "&#8734;");
map.put("&int;", "&#8747;");
map.put("&Iota;", "&#921;");
map.put("&iota;", "&#953;");
map.put("&iquest;", "&#191;");
map.put("&isin;", "&#8712;");
map.put("&Iuml;", "&#207;");
map.put("&iuml;", "&#239;");
map.put("&Kappa;", "&#922;");
map.put("&kappa;", "&#954;");
map.put("&Lambda;", "&#923;");
map.put("&lambda;", "&#955;");
map.put("&lang;", "&#9001;");
map.put("&laquo;", "&#171;");
map.put("&larr;", "&#8592;");
map.put("&lArr;", "&#8656;");
map.put("&lceil;", "&#8968;");
map.put("&ldquo;", "&#8220;");
map.put("&le;", "&#8804;");
map.put("&lfloor;", "&#8970;");
map.put("&lowast;", "&#8727;");
map.put("&loz;", "&#9674;");
map.put("&lrm;", "&#8206;");
map.put("&lsaquo;", "&#8249;");
map.put("&lsquo;", "&#8216;");
map.put("&macr;", "&#175;");
map.put("&mdash;", "&#8212;");
map.put("&micro;", "&#181;");
map.put("&middot;", "&#183;");
map.put("&minus;", "&#8722;");
map.put("&Mu;", "&#924;");
map.put("&mu;", "&#956;");
map.put("&nabla;", "&#8711;");
map.put("&nbsp;", "&#160;");
map.put("&ndash;", "&#8211;");
map.put("&ne;", "&#8800;");
map.put("&ni;", "&#8715;");
map.put("&not;", "&#172;");
map.put("&notin;", "&#8713;");
map.put("&nsub;", "&#8836;");
map.put("&Ntilde;", "&#209;");
map.put("&ntilde;", "&#241;");
map.put("&Nu;", "&#925;");
map.put("&nu;", "&#957;");
map.put("&Oacute;", "&#211;");
map.put("&oacute;", "&#243;");
map.put("&Ocirc;", "&#212;");
map.put("&ocirc;", "&#244;");
map.put("&OElig;", "&#338;");
map.put("&oelig;", "&#339;");
map.put("&Ograve;", "&#210;");
map.put("&ograve;", "&#242;");
map.put("&oline;", "&#8254;");
map.put("&Omega;", "&#937;");
map.put("&omega;", "&#969;");
map.put("&Omicron;", "&#927;");
map.put("&omicron;", "&#959;");
map.put("&oplus;", "&#8853;");
map.put("&or;", "&#8744;");
map.put("&ordf;", "&#170;");
map.put("&ordm;", "&#186;");
map.put("&Oslash;", "&#216;");
map.put("&oslash;", "&#248;");
map.put("&Otilde;", "&#213;");
map.put("&otilde;", "&#245;");
map.put("&otimes;", "&#8855;");
map.put("&Ouml;", "&#214;");
map.put("&ouml;", "&#246;");
map.put("&para;", "&#182;");
map.put("&part;", "&#8706;");
map.put("&permil;", "&#8240;");
map.put("&perp;", "&#8869;");
map.put("&Phi;", "&#934;");
map.put("&phi;", "&#966;");
map.put("&Pi;", "&#928;");
map.put("&pi;", "&#960;");
map.put("&piv;", "&#982;");
map.put("&plusmn;", "&#177;");
map.put("&pound;", "&#163;");
map.put("&prime;", "&#8242;");
map.put("&Prime;", "&#8243;");
map.put("&prod;", "&#8719;");
map.put("&prop;", "&#8733;");
map.put("&Psi;", "&#936;");
map.put("&psi;", "&#968;");
map.put("&quot;", "&#34;");
map.put("&radic;", "&#8730;");
map.put("&rang;", "&#9002;");
map.put("&raquo;", "&#187;");
map.put("&rarr;", "&#8594;");
map.put("&rArr;", "&#8658;");
map.put("&rceil;", "&#8969;");
map.put("&rdquo;", "&#8221;");
map.put("&real;", "&#8476;");
map.put("&reg;", "&#174;");
map.put("&rfloor;", "&#8971;");
map.put("&Rho;", "&#929;");
map.put("&rho;", "&#961;");
map.put("&rlm;", "&#8207;");
map.put("&rsaquo;", "&#8250;");
map.put("&rsquo;", "&#8217;");
map.put("&sbquo;", "&#8218;");
map.put("&Scaron;", "&#352;");
map.put("&scaron;", "&#353;");
map.put("&sdot;", "&#8901;");
map.put("&sect;", "&#167;");
map.put("&shy;", "&#173;");
map.put("&Sigma;", "&#931;");
map.put("&sigma;", "&#963;");
map.put("&sigmaf;", "&#962;");
map.put("&sim;", "&#8764;");
map.put("&spades;", "&#9824;");
map.put("&sub;", "&#8834;");
map.put("&sube;", "&#8838;");
map.put("&sum;", "&#8721;");
map.put("&sup1;", "&#185;");
map.put("&sup2;", "&#178;");
map.put("&sup3;", "&#179;");
map.put("&sup;", "&#8835;");
map.put("&supe;", "&#8839;");
map.put("&szlig;", "&#223;");
map.put("&Tau;", "&#932;");
map.put("&tau;", "&#964;");
map.put("&there4;", "&#8756;");
map.put("&Theta;", "&#920;");
map.put("&theta;", "&#952;");
map.put("&thetasym;", "&#977;");
map.put("&thinsp;", "&#8201;");
map.put("&THORN;", "&#222;");
map.put("&thorn;", "&#254;");
map.put("&tilde;", "&#732;");
map.put("&times;", "&#215;");
map.put("&trade;", "&#8482;");
map.put("&Uacute;", "&#218;");
map.put("&uacute;", "&#250;");
map.put("&uarr;", "&#8593;");
map.put("&uArr;", "&#8657;");
map.put("&Ucirc;", "&#219;");
map.put("&ucirc;", "&#251;");
map.put("&Ugrave;", "&#217;");
map.put("&ugrave;", "&#249;");
map.put("&uml;", "&#168;");
map.put("&upsih;", "&#978;");
map.put("&Upsilon;", "&#933;");
map.put("&upsilon;", "&#965;");
map.put("&Uuml;", "&#220;");
map.put("&uuml;", "&#252;");
map.put("&weierp;", "&#8472;");
map.put("&Xi;", "&#926;");
map.put("&xi;", "&#958;");
map.put("&Yacute;", "&#221;");
map.put("&yacute;", "&#253;");
map.put("&yen;", "&#165;");
map.put("&yuml;", "&#255;");
map.put("&Yuml;", "&#376;");
map.put("&Zeta;", "&#918;");
map.put("&zeta;", "&#950;");
map.put("&zwj;", "&#8205;");
map.put("&zwnj;", "&#8204;");
static {
Map<String, String> map = new LinkedHashMap<>();
map.put("&Aacute;", "&#193;");
map.put("&aacute;", "&#225;");
map.put("&Acirc;", "&#194;");
map.put("&acirc;", "&#226;");
map.put("&acute;", "&#180;");
map.put("&AElig;", "&#198;");
map.put("&aelig;", "&#230;");
map.put("&Agrave;", "&#192;");
map.put("&agrave;", "&#224;");
map.put("&alefsym;", "&#8501;");
map.put("&Alpha;", "&#913;");
map.put("&alpha;", "&#945;");
map.put("&amp;", "&#38;");
map.put("&and;", "&#8743;");
map.put("&ang;", "&#8736;");
map.put("&Aring;", "&#197;");
map.put("&aring;", "&#229;");
map.put("&asymp;", "&#8776;");
map.put("&Atilde;", "&#195;");
map.put("&atilde;", "&#227;");
map.put("&Auml;", "&#196;");
map.put("&auml;", "&#228;");
map.put("&bdquo;", "&#8222;");
map.put("&Beta;", "&#914;");
map.put("&beta;", "&#946;");
map.put("&brvbar;", "&#166;");
map.put("&bull;", "&#8226;");
map.put("&cap;", "&#8745;");
map.put("&Ccedil;", "&#199;");
map.put("&ccedil;", "&#231;");
map.put("&cedil;", "&#184;");
map.put("&cent;", "&#162;");
map.put("&Chi;", "&#935;");
map.put("&chi;", "&#967;");
map.put("&circ;", "&#710;");
map.put("&clubs;", "&#9827;");
map.put("&cong;", "&#8773;");
map.put("&copy;", "&#169;");
map.put("&crarr;", "&#8629;");
map.put("&cup;", "&#8746;");
map.put("&curren;", "&#164;");
map.put("&dagger;", "&#8224;");
map.put("&Dagger;", "&#8225;");
map.put("&darr;", "&#8595;");
map.put("&dArr;", "&#8659;");
map.put("&deg;", "&#176;");
map.put("&Delta;", "&#916;");
map.put("&delta;", "&#948;");
map.put("&diams;", "&#9830;");
map.put("&divide;", "&#247;");
map.put("&Eacute;", "&#201;");
map.put("&eacute;", "&#233;");
map.put("&Ecirc;", "&#202;");
map.put("&ecirc;", "&#234;");
map.put("&Egrave;", "&#200;");
map.put("&egrave;", "&#232;");
map.put("&empty;", "&#8709;");
map.put("&emsp;", "&#8195;");
map.put("&ensp;", "&#8194;");
map.put("&Epsilon;", "&#917;");
map.put("&epsilon;", "&#949;");
map.put("&equiv;", "&#8801;");
map.put("&Eta;", "&#919;");
map.put("&eta;", "&#951;");
map.put("&ETH;", "&#208;");
map.put("&eth;", "&#240;");
map.put("&Euml;", "&#203;");
map.put("&euml;", "&#235;");
map.put("&euro;", "&#8364;");
map.put("&exist;", "&#8707;");
map.put("&fnof;", "&#402;");
map.put("&forall;", "&#8704;");
map.put("&frac12;", "&#189;");
map.put("&frac14;", "&#188;");
map.put("&frac34;", "&#190;");
map.put("&frasl;", "&#8260;");
map.put("&Gamma;", "&#915;");
map.put("&gamma;", "&#947;");
map.put("&ge;", "&#8805;");
map.put("&harr;", "&#8596;");
map.put("&hArr;", "&#8660;");
map.put("&hearts;", "&#9829;");
map.put("&hellip;", "&#8230;");
map.put("&Iacute;", "&#205;");
map.put("&iacute;", "&#237;");
map.put("&Icirc;", "&#206;");
map.put("&icirc;", "&#238;");
map.put("&iexcl;", "&#161;");
map.put("&Igrave;", "&#204;");
map.put("&igrave;", "&#236;");
map.put("&image;", "&#8465;");
map.put("&infin;", "&#8734;");
map.put("&int;", "&#8747;");
map.put("&Iota;", "&#921;");
map.put("&iota;", "&#953;");
map.put("&iquest;", "&#191;");
map.put("&isin;", "&#8712;");
map.put("&Iuml;", "&#207;");
map.put("&iuml;", "&#239;");
map.put("&Kappa;", "&#922;");
map.put("&kappa;", "&#954;");
map.put("&Lambda;", "&#923;");
map.put("&lambda;", "&#955;");
map.put("&lang;", "&#9001;");
map.put("&laquo;", "&#171;");
map.put("&larr;", "&#8592;");
map.put("&lArr;", "&#8656;");
map.put("&lceil;", "&#8968;");
map.put("&ldquo;", "&#8220;");
map.put("&le;", "&#8804;");
map.put("&lfloor;", "&#8970;");
map.put("&lowast;", "&#8727;");
map.put("&loz;", "&#9674;");
map.put("&lrm;", "&#8206;");
map.put("&lsaquo;", "&#8249;");
map.put("&lsquo;", "&#8216;");
map.put("&macr;", "&#175;");
map.put("&mdash;", "&#8212;");
map.put("&micro;", "&#181;");
map.put("&middot;", "&#183;");
map.put("&minus;", "&#8722;");
map.put("&Mu;", "&#924;");
map.put("&mu;", "&#956;");
map.put("&nabla;", "&#8711;");
map.put("&nbsp;", "&#160;");
map.put("&ndash;", "&#8211;");
map.put("&ne;", "&#8800;");
map.put("&ni;", "&#8715;");
map.put("&not;", "&#172;");
map.put("&notin;", "&#8713;");
map.put("&nsub;", "&#8836;");
map.put("&Ntilde;", "&#209;");
map.put("&ntilde;", "&#241;");
map.put("&Nu;", "&#925;");
map.put("&nu;", "&#957;");
map.put("&Oacute;", "&#211;");
map.put("&oacute;", "&#243;");
map.put("&Ocirc;", "&#212;");
map.put("&ocirc;", "&#244;");
map.put("&OElig;", "&#338;");
map.put("&oelig;", "&#339;");
map.put("&Ograve;", "&#210;");
map.put("&ograve;", "&#242;");
map.put("&oline;", "&#8254;");
map.put("&Omega;", "&#937;");
map.put("&omega;", "&#969;");
map.put("&Omicron;", "&#927;");
map.put("&omicron;", "&#959;");
map.put("&oplus;", "&#8853;");
map.put("&or;", "&#8744;");
map.put("&ordf;", "&#170;");
map.put("&ordm;", "&#186;");
map.put("&Oslash;", "&#216;");
map.put("&oslash;", "&#248;");
map.put("&Otilde;", "&#213;");
map.put("&otilde;", "&#245;");
map.put("&otimes;", "&#8855;");
map.put("&Ouml;", "&#214;");
map.put("&ouml;", "&#246;");
map.put("&para;", "&#182;");
map.put("&part;", "&#8706;");
map.put("&permil;", "&#8240;");
map.put("&perp;", "&#8869;");
map.put("&Phi;", "&#934;");
map.put("&phi;", "&#966;");
map.put("&Pi;", "&#928;");
map.put("&pi;", "&#960;");
map.put("&piv;", "&#982;");
map.put("&plusmn;", "&#177;");
map.put("&pound;", "&#163;");
map.put("&prime;", "&#8242;");
map.put("&Prime;", "&#8243;");
map.put("&prod;", "&#8719;");
map.put("&prop;", "&#8733;");
map.put("&Psi;", "&#936;");
map.put("&psi;", "&#968;");
map.put("&quot;", "&#34;");
map.put("&radic;", "&#8730;");
map.put("&rang;", "&#9002;");
map.put("&raquo;", "&#187;");
map.put("&rarr;", "&#8594;");
map.put("&rArr;", "&#8658;");
map.put("&rceil;", "&#8969;");
map.put("&rdquo;", "&#8221;");
map.put("&real;", "&#8476;");
map.put("&reg;", "&#174;");
map.put("&rfloor;", "&#8971;");
map.put("&Rho;", "&#929;");
map.put("&rho;", "&#961;");
map.put("&rlm;", "&#8207;");
map.put("&rsaquo;", "&#8250;");
map.put("&rsquo;", "&#8217;");
map.put("&sbquo;", "&#8218;");
map.put("&Scaron;", "&#352;");
map.put("&scaron;", "&#353;");
map.put("&sdot;", "&#8901;");
map.put("&sect;", "&#167;");
map.put("&shy;", "&#173;");
map.put("&Sigma;", "&#931;");
map.put("&sigma;", "&#963;");
map.put("&sigmaf;", "&#962;");
map.put("&sim;", "&#8764;");
map.put("&spades;", "&#9824;");
map.put("&sub;", "&#8834;");
map.put("&sube;", "&#8838;");
map.put("&sum;", "&#8721;");
map.put("&sup1;", "&#185;");
map.put("&sup2;", "&#178;");
map.put("&sup3;", "&#179;");
map.put("&sup;", "&#8835;");
map.put("&supe;", "&#8839;");
map.put("&szlig;", "&#223;");
map.put("&Tau;", "&#932;");
map.put("&tau;", "&#964;");
map.put("&there4;", "&#8756;");
map.put("&Theta;", "&#920;");
map.put("&theta;", "&#952;");
map.put("&thetasym;", "&#977;");
map.put("&thinsp;", "&#8201;");
map.put("&THORN;", "&#222;");
map.put("&thorn;", "&#254;");
map.put("&tilde;", "&#732;");
map.put("&times;", "&#215;");
map.put("&trade;", "&#8482;");
map.put("&Uacute;", "&#218;");
map.put("&uacute;", "&#250;");
map.put("&uarr;", "&#8593;");
map.put("&uArr;", "&#8657;");
map.put("&Ucirc;", "&#219;");
map.put("&ucirc;", "&#251;");
map.put("&Ugrave;", "&#217;");
map.put("&ugrave;", "&#249;");
map.put("&uml;", "&#168;");
map.put("&upsih;", "&#978;");
map.put("&Upsilon;", "&#933;");
map.put("&upsilon;", "&#965;");
map.put("&Uuml;", "&#220;");
map.put("&uuml;", "&#252;");
map.put("&weierp;", "&#8472;");
map.put("&Xi;", "&#926;");
map.put("&xi;", "&#958;");
map.put("&Yacute;", "&#221;");
map.put("&yacute;", "&#253;");
map.put("&yen;", "&#165;");
map.put("&yuml;", "&#255;");
map.put("&Yuml;", "&#376;");
map.put("&Zeta;", "&#918;");
map.put("&zeta;", "&#950;");
map.put("&zwj;", "&#8205;");
map.put("&zwnj;", "&#8204;");
HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map);
HTML_ENTITIES = List.copyOf(map.keySet());
}
HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map);
HTML_ENTITIES = List.copyOf(map.keySet());
}
}

75
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/TextDirectionDetector.java
View File

@@ -2,55 +2,54 @@ package com.commafeed.backend.feed.parser;
import java.text.Bidi;
import java.util.regex.Pattern;
import org.apache.commons.lang3.math.NumberUtils;
public class TextDirectionDetector {
private static final Pattern WORDS_PATTERN = Pattern.compile("\\s+");
private static final Pattern URL_PATTERN = Pattern.compile("^https?://.*");
private static final Pattern WORDS_PATTERN = Pattern.compile("\\s+");
private static final Pattern URL_PATTERN = Pattern.compile("^https?://.*");
private static final double RTL_THRESHOLD = 0.4D;
private static final double RTL_THRESHOLD = 0.4D;
public enum Direction {
LEFT_TO_RIGHT, RIGHT_TO_LEFT
}
public enum Direction {
LEFT_TO_RIGHT,
RIGHT_TO_LEFT
}
public static Direction detect(String input) {
if (input == null || input.isBlank()) {
return Direction.LEFT_TO_RIGHT;
}
public static Direction detect(String input) {
if (input == null || input.isBlank()) {
return Direction.LEFT_TO_RIGHT;
}
long rtl = 0;
long total = 0;
for (String token : WORDS_PATTERN.split(input)) {
// skip urls
if (URL_PATTERN.matcher(token).matches()) {
continue;
}
long rtl = 0;
long total = 0;
for (String token : WORDS_PATTERN.split(input)) {
// skip urls
if (URL_PATTERN.matcher(token).matches()) {
continue;
}
// skip numbers
if (NumberUtils.isCreatable(token)) {
continue;
}
// skip numbers
if (NumberUtils.isCreatable(token)) {
continue;
}
boolean requiresBidi = Bidi.requiresBidi(token.toCharArray(), 0, token.length());
if (requiresBidi) {
Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT);
if (bidi.getBaseLevel() == 1) {
rtl++;
}
}
boolean requiresBidi = Bidi.requiresBidi(token.toCharArray(), 0, token.length());
if (requiresBidi) {
Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT);
if (bidi.getBaseLevel() == 1) {
rtl++;
}
}
total++;
}
total++;
}
if (total == 0) {
return Direction.LEFT_TO_RIGHT;
}
double ratio = (double) rtl / total;
return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
}
if (total == 0) {
return Direction.LEFT_TO_RIGHT;
}
double ratio = (double) rtl / total;
return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
}
}

107
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/XMLCleaner.java
View File

@@ -1,10 +1,8 @@
package com.commafeed.backend.feed.parser;
import jakarta.inject.Singleton;
import java.util.Collection;
import java.util.regex.Pattern;
import jakarta.inject.Singleton;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.apache.commons.lang3.StringUtils;
@@ -13,70 +11,71 @@ import org.jdom2.Verifier;
@Singleton
public class XMLCleaner {
private static final Pattern DOCTYPE_PATTERN = Pattern.compile("<!DOCTYPE[^>]*>", Pattern.CASE_INSENSITIVE);
private static final Pattern DOCTYPE_PATTERN =
Pattern.compile("<!DOCTYPE[^>]*>", Pattern.CASE_INSENSITIVE);
private final Trie trie = Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build();
private final Trie trie =
Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build();
public String clean(String xml) {
xml = removeCharactersBeforeFirstXmlTag(xml);
xml = removeInvalidXmlCharacters(xml);
xml = replaceHtmlEntitiesWithNumericEntities(xml);
xml = removeDoctypeDeclarations(xml);
return xml;
}
public String clean(String xml) {
xml = removeCharactersBeforeFirstXmlTag(xml);
xml = removeInvalidXmlCharacters(xml);
xml = replaceHtmlEntitiesWithNumericEntities(xml);
xml = removeDoctypeDeclarations(xml);
return xml;
}
String removeCharactersBeforeFirstXmlTag(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
String removeCharactersBeforeFirstXmlTag(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
int pos = xml.indexOf('<');
return pos < 0 ? null : xml.substring(pos);
}
int pos = xml.indexOf('<');
return pos < 0 ? null : xml.substring(pos);
}
String removeInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
String removeInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
return xml.codePoints()
.filter(Verifier::isXMLCharacter)
.collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append)
.toString();
}
return xml.codePoints()
.filter(Verifier::isXMLCharacter)
.collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append)
.toString();
}
// https://stackoverflow.com/a/40836618
String replaceHtmlEntitiesWithNumericEntities(String source) {
if (StringUtils.isBlank(source)) {
return null;
}
// https://stackoverflow.com/a/40836618
String replaceHtmlEntitiesWithNumericEntities(String source) {
if (StringUtils.isBlank(source)) {
return null;
}
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
Collection<Emit> emits = trie.parseText(source);
Collection<Emit> emits = trie.parseText(source);
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
return sb.toString();
}
return sb.toString();
}
String removeDoctypeDeclarations(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
return DOCTYPE_PATTERN.matcher(xml).replaceAll("");
}
String removeDoctypeDeclarations(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
return DOCTYPE_PATTERN.matcher(xml).replaceAll("");
}
}

27
commafeed-server/src/main/java/com/commafeed/backend/model/AbstractModel.java
View File

@@ -1,33 +1,28 @@
package com.commafeed.backend.model;
import java.io.Serializable;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
import jakarta.persistence.Id;
import jakarta.persistence.MappedSuperclass;
import jakarta.persistence.TableGenerator;
import java.io.Serializable;
import lombok.Getter;
import lombok.Setter;
/**
* Abstract model for all entities, defining id and table generator
*
*/
/** Abstract model for all entities, defining id and table generator */
@SuppressWarnings("serial")
@MappedSuperclass
@Getter
@Setter
public abstract class AbstractModel implements Serializable {
@Id
@GeneratedValue(strategy = GenerationType.TABLE, generator = "gen")
@TableGenerator(
name = "gen",
table = "hibernate_sequences",
pkColumnName = "sequence_name",
valueColumnName = "sequence_next_hi_value",
allocationSize = 1000)
private Long id;
@Id
@GeneratedValue(strategy = GenerationType.TABLE, generator = "gen")
@TableGenerator(
name = "gen",
table = "hibernate_sequences",
pkColumnName = "sequence_name",
valueColumnName = "sequence_next_hi_value",
allocationSize = 1000)
private Long id;
}

134
commafeed-server/src/main/java/com/commafeed/backend/model/Feed.java
View File

@@ -1,17 +1,14 @@
package com.commafeed.backend.model;
import java.sql.Types;
import java.time.Instant;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Lob;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import java.sql.Types;
import java.time.Instant;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.annotations.JdbcTypeCode;
@Entity
@Table(name = "FEEDS")
@@ -20,97 +17,66 @@ import lombok.Setter;
@Setter
public class Feed extends AbstractModel {
/**
* The url of the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARCHAR)
private String url;
/** The url of the feed */
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARCHAR)
private String url;
/**
* cache the url after potential http 30x redirects
*/
@Column(name = "url_after_redirect", length = 2048, nullable = false)
private String urlAfterRedirect;
/** cache the url after potential http 30x redirects */
@Column(name = "url_after_redirect", length = 2048, nullable = false)
private String urlAfterRedirect;
@Column(length = 2048, nullable = false)
private String normalizedUrl;
@Column(length = 2048, nullable = false)
private String normalizedUrl;
@Column(length = 40, nullable = false)
private String normalizedUrlHash;
@Column(length = 40, nullable = false)
private String normalizedUrlHash;
/**
* The url of the website, extracted from the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String link;
/** The url of the website, extracted from the feed */
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String link;
@Lob
@Column(name = "icon_url", length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String iconUrl;
@Lob
@Column(name = "icon_url", length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String iconUrl;
/**
* Last time we tried to fetch the feed
*/
@Column
private Instant lastUpdated;
/** Last time we tried to fetch the feed */
@Column private Instant lastUpdated;
/**
* Last publishedDate value in the feed
*/
@Column
private Instant lastPublishedDate;
/** Last publishedDate value in the feed */
@Column private Instant lastPublishedDate;
/**
* date of the last entry of the feed
*/
@Column
private Instant lastEntryDate;
/** date of the last entry of the feed */
@Column private Instant lastEntryDate;
/**
* error message while retrieving the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String message;
/** error message while retrieving the feed */
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String message;
/**
* times we failed to retrieve the feed
*/
private int errorCount;
/** times we failed to retrieve the feed */
private int errorCount;
/**
* feed refresh is disabled until this date
*/
@Column
private Instant disabledUntil;
/** feed refresh is disabled until this date */
@Column private Instant disabledUntil;
/**
* http header returned by the feed
*/
@Column(length = 64)
private String lastModifiedHeader;
/** http header returned by the feed */
@Column(length = 64)
private String lastModifiedHeader;
/**
* http header returned by the feed
*/
@Column(length = 255)
private String etagHeader;
/** http header returned by the feed */
@Column(length = 255)
private String etagHeader;
/**
* average time between entries in the feed
*/
private Long averageEntryInterval;
/**
* last hash of the content of the feed xml
*/
@Column(length = 40)
private String lastContentHash;
/** average time between entries in the feed */
private Long averageEntryInterval;
/** last hash of the content of the feed xml */
@Column(length = 40)
private String lastContentHash;
}

20
commafeed-server/src/main/java/com/commafeed/backend/model/FeedCategory.java
View File

@@ -6,7 +6,6 @@ import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@@ -17,18 +16,17 @@ import lombok.Setter;
@Setter
public class FeedCategory extends AbstractModel {
@Column(length = 128, nullable = false)
private String name;
@Column(length = 128, nullable = false)
private String name;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory parent;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory parent;
private boolean collapsed;
private int position;
private boolean collapsed;
private int position;
}

53
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntry.java
View File

@@ -1,8 +1,5 @@
package com.commafeed.backend.model;
import java.time.Instant;
import java.util.Set;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
@@ -11,7 +8,8 @@ import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import java.time.Instant;
import java.util.Set;
import lombok.Getter;
import lombok.Setter;
@@ -22,39 +20,32 @@ import lombok.Setter;
@Setter
public class FeedEntry extends AbstractModel {
@Column(length = 2048, nullable = false)
private String guid;
@Column(length = 2048, nullable = false)
private String guid;
@Column(length = 40, nullable = false)
private String guidHash;
@Column(length = 40, nullable = false)
private String guidHash;
@ManyToOne(fetch = FetchType.LAZY)
private Feed feed;
@ManyToOne(fetch = FetchType.LAZY)
private Feed feed;
@ManyToOne(fetch = FetchType.LAZY, optional = false)
@JoinColumn(nullable = false, updatable = false)
private FeedEntryContent content;
@ManyToOne(fetch = FetchType.LAZY, optional = false)
@JoinColumn(nullable = false, updatable = false)
private FeedEntryContent content;
@Column(length = 2048)
private String url;
@Column(length = 2048)
private String url;
/**
* the moment the entry was inserted in the database
*/
@Column
private Instant inserted;
/** the moment the entry was inserted in the database */
@Column private Instant inserted;
/**
* the moment the entry was published in the feed
*
*/
@Column(name = "updated")
private Instant published;
/** the moment the entry was published in the feed */
@Column(name = "updated")
private Instant published;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryTag> tags;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryTag> tags;
}

119
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryContent.java
View File

@@ -1,21 +1,17 @@
package com.commafeed.backend.model;
import java.sql.Types;
import com.fasterxml.jackson.annotation.JsonProperty;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.Lob;
import jakarta.persistence.Table;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.hibernate.annotations.JdbcTypeCode;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.sql.Types;
import lombok.Getter;
import lombok.Setter;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.hibernate.annotations.JdbcTypeCode;
@Entity
@Table(name = "FEEDENTRYCONTENTS")
@@ -24,73 +20,74 @@ import lombok.Setter;
@Setter
public class FeedEntryContent extends AbstractModel {
public enum Direction {
@JsonProperty("ltr")
LTR,
public enum Direction {
@JsonProperty("ltr")
LTR,
@JsonProperty("rtl")
RTL,
@JsonProperty("rtl")
RTL,
@JsonProperty("unknown")
UNKNOWN
}
@JsonProperty("unknown")
UNKNOWN
}
@Column(length = 2048)
private String title;
@Column(length = 2048)
private String title;
@Column(length = 40)
private String titleHash;
@Column(length = 40)
private String titleHash;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String content;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String content;
@Column(length = 40)
private String contentHash;
@Column(length = 40)
private String contentHash;
@Column(name = "author", length = 128)
private String author;
@Column(name = "author", length = 128)
private String author;
@Column(length = 2048)
private String enclosureUrl;
@Column(length = 2048)
private String enclosureUrl;
@Column(length = 255)
private String enclosureType;
@Column(length = 255)
private String enclosureType;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String mediaDescription;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String mediaDescription;
@Column(length = 2048)
private String mediaThumbnailUrl;
@Column(length = 2048)
private String mediaThumbnailUrl;
private Integer mediaThumbnailWidth;
private Integer mediaThumbnailHeight;
private Integer mediaThumbnailWidth;
private Integer mediaThumbnailHeight;
@Column(length = 4096)
private String categories;
@Column(length = 4096)
private String categories;
@Column
@Enumerated(EnumType.STRING)
private Direction direction = Direction.UNKNOWN;
@Column
@Enumerated(EnumType.STRING)
private Direction direction = Direction.UNKNOWN;
public boolean equivalentTo(FeedEntryContent c) {
if (c == null) {
return false;
}
public boolean equivalentTo(FeedEntryContent c) {
if (c == null) {
return false;
}
return new EqualsBuilder().append(title, c.title)
.append(content, c.content)
.append(author, c.author)
.append(categories, c.categories)
.append(enclosureUrl, c.enclosureUrl)
.append(enclosureType, c.enclosureType)
.append(mediaDescription, c.mediaDescription)
.append(mediaThumbnailUrl, c.mediaThumbnailUrl)
.append(mediaThumbnailWidth, c.mediaThumbnailWidth)
.append(mediaThumbnailHeight, c.mediaThumbnailHeight)
.build();
}
return new EqualsBuilder()
.append(title, c.title)
.append(content, c.content)
.append(author, c.author)
.append(categories, c.categories)
.append(enclosureUrl, c.enclosureUrl)
.append(enclosureType, c.enclosureType)
.append(mediaDescription, c.mediaDescription)
.append(mediaThumbnailUrl, c.mediaThumbnailUrl)
.append(mediaThumbnailWidth, c.mediaThumbnailWidth)
.append(mediaThumbnailHeight, c.mediaThumbnailHeight)
.build();
}
}

68
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryStatus.java
View File

@@ -1,9 +1,5 @@
package com.commafeed.backend.model;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
@@ -11,7 +7,9 @@ import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import jakarta.persistence.Transient;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import lombok.Getter;
import lombok.Setter;
@@ -22,48 +20,40 @@ import lombok.Setter;
@Setter
public class FeedEntryStatus extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedSubscription subscription;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedSubscription subscription;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedEntry entry;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedEntry entry;
@Column(name = "read_status")
private boolean read;
private boolean starred;
@Column(name = "read_status")
private boolean read;
@Transient
private boolean markable;
private boolean starred;
@Transient
private List<FeedEntryTag> tags = new ArrayList<>();
@Transient private boolean markable;
/**
* Denormalization starts here
*/
@Transient private List<FeedEntryTag> tags = new ArrayList<>();
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
/** Denormalization starts here */
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@Column
private Instant entryInserted;
@Column private Instant entryInserted;
@Column(name = "entryUpdated")
private Instant entryPublished;
@Column(name = "entryUpdated")
private Instant entryPublished;
public FeedEntryStatus() {
}
public FeedEntryStatus(User user, FeedSubscription subscription, FeedEntry entry) {
this.user = user;
this.subscription = subscription;
this.entry = entry;
this.entryInserted = entry.getInserted();
this.entryPublished = entry.getPublished();
}
public FeedEntryStatus() {}
public FeedEntryStatus(User user, FeedSubscription subscription, FeedEntry entry) {
this.user = user;
this.subscription = subscription;
this.entry = entry;
this.entryInserted = entry.getInserted();
this.entryPublished = entry.getPublished();
}
}

31
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryTag.java
View File

@@ -6,7 +6,6 @@ import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@@ -17,24 +16,22 @@ import lombok.Setter;
@Setter
public class FeedEntryTag extends AbstractModel {
@JoinColumn(name = "user_id")
@ManyToOne(fetch = FetchType.LAZY)
private User user;
@JoinColumn(name = "user_id")
@ManyToOne(fetch = FetchType.LAZY)
private User user;
@JoinColumn(name = "entry_id")
@ManyToOne(fetch = FetchType.LAZY)
private FeedEntry entry;
@JoinColumn(name = "entry_id")
@ManyToOne(fetch = FetchType.LAZY)
private FeedEntry entry;
@Column(name = "name", length = 40)
private String name;
@Column(name = "name", length = 40)
private String name;
public FeedEntryTag() {
}
public FeedEntryTag(User user, FeedEntry entry, String name) {
this.name = name;
this.entry = entry;
this.user = user;
}
public FeedEntryTag() {}
public FeedEntryTag(User user, FeedEntry entry, String name) {
this.name = name;
this.entry = entry;
this.user = user;
}
}

47
commafeed-server/src/main/java/com/commafeed/backend/model/FeedSubscription.java
View File

@@ -1,7 +1,5 @@
package com.commafeed.backend.model;
import java.util.Set;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
@@ -10,7 +8,7 @@ import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import java.util.Set;
import lombok.Getter;
import lombok.Setter;
@@ -21,35 +19,34 @@ import lombok.Setter;
@Setter
public class FeedSubscription extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private Feed feed;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private Feed feed;
@Column(length = 128, nullable = false)
private String title;
@Column(length = 128, nullable = false)
private String title;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory category;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory category;
@OneToMany(mappedBy = "subscription", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
@OneToMany(mappedBy = "subscription", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
private int position;
private int position;
@Column(name = "filtering_expression", length = 4096)
private String filter;
@Column(name = "filtering_expression", length = 4096)
private String filter;
@Column(name = "filtering_expression_legacy", length = 4096)
private String filterLegacy;
@Column(name = "filtering_expression_legacy", length = 4096)
private String filterLegacy;
@Column(name = "push_notifications_enabled")
private boolean pushNotificationsEnabled;
@Column(name = "auto_mark_as_read_after_days")
private Integer autoMarkAsReadAfterDays;
@Column(name = "push_notifications_enabled")
private boolean pushNotificationsEnabled;
@Column(name = "auto_mark_as_read_after_days")
private Integer autoMarkAsReadAfterDays;
}

47
commafeed-server/src/main/java/com/commafeed/backend/model/Models.java
View File

@@ -2,40 +2,35 @@ package com.commafeed.backend.model;
import java.time.Duration;
import java.time.Instant;
import lombok.experimental.UtilityClass;
import org.hibernate.Hibernate;
import org.hibernate.HibernateException;
import org.hibernate.proxy.HibernateProxy;
import org.hibernate.proxy.LazyInitializer;
import lombok.experimental.UtilityClass;
@UtilityClass
public class Models {
public static final Instant MINIMUM_INSTANT = Instant.EPOCH
// mariadb timestamp range starts at 1970-01-01 00:00:01
.plusSeconds(1)
// make sure the timestamp fits for all timezones
.plus(Duration.ofHours(24));
public static final Instant MINIMUM_INSTANT =
Instant.EPOCH
// mariadb timestamp range starts at 1970-01-01 00:00:01
.plusSeconds(1)
// make sure the timestamp fits for all timezones
.plus(Duration.ofHours(24));
/**
* initialize a proxy
*/
public static void initialize(Object proxy) throws HibernateException {
Hibernate.initialize(proxy);
}
/** initialize a proxy */
public static void initialize(Object proxy) throws HibernateException {
Hibernate.initialize(proxy);
}
/**
* extract the id from the proxy without initializing it
*/
public static Long getId(AbstractModel model) {
if (model instanceof HibernateProxy proxy) {
LazyInitializer lazyInitializer = proxy.getHibernateLazyInitializer();
if (lazyInitializer.isUninitialized()) {
return (Long) lazyInitializer.getIdentifier();
}
}
return model.getId();
}
/** extract the id from the proxy without initializing it */
public static Long getId(AbstractModel model) {
if (model instanceof HibernateProxy proxy) {
LazyInitializer lazyInitializer = proxy.getHibernateLazyInitializer();
if (lazyInitializer.isUninitialized()) {
return (Long) lazyInitializer.getIdentifier();
}
}
return model.getId();
}
}

57
commafeed-server/src/main/java/com/commafeed/backend/model/User.java
View File

@@ -1,17 +1,14 @@
package com.commafeed.backend.model;
import java.sql.Types;
import java.time.Instant;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Lob;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import java.sql.Types;
import java.time.Instant;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.annotations.JdbcTypeCode;
@Entity
@Table(name = "USERS")
@@ -20,40 +17,36 @@ import lombok.Setter;
@Setter
public class User extends AbstractModel {
@Column(length = 32, nullable = false, unique = true)
private String name;
@Column(length = 32, nullable = false, unique = true)
private String name;
@Column(length = 255, unique = true)
private String email;
@Column(length = 255, unique = true)
private String email;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] password;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] password;
@Column(length = 40, unique = true)
private String apiKey;
@Column(length = 40, unique = true)
private String apiKey;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] salt;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] salt;
@Column(nullable = false)
private boolean disabled;
@Column(nullable = false)
private boolean disabled;
@Column
private Instant lastLogin;
@Column private Instant lastLogin;
@Column
private Instant created;
@Column private Instant created;
@Column(length = 40)
private String recoverPasswordToken;
@Column(length = 40)
private String recoverPasswordToken;
@Column
private Instant recoverPasswordTokenDate;
@Column private Instant recoverPasswordTokenDate;
@Column
private Instant lastForceRefresh;
@Column private Instant lastForceRefresh;
}

33
commafeed-server/src/main/java/com/commafeed/backend/model/UserRole.java
View File

@@ -8,7 +8,6 @@ import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.OneToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@@ -19,25 +18,23 @@ import lombok.Setter;
@Setter
public class UserRole extends AbstractModel {
public enum Role {
USER, ADMIN
}
public enum Role {
USER,
ADMIN
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false)
private User user;
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false)
private User user;
@Column(name = "roleName", nullable = false)
@Enumerated(EnumType.STRING)
private Role role;
@Column(name = "roleName", nullable = false)
@Enumerated(EnumType.STRING)
private Role role;
public UserRole() {
}
public UserRole(User user, Role role) {
this.user = user;
this.role = role;
}
public UserRole() {}
public UserRole(User user, Role role) {
this.user = user;
this.role = role;
}
}

241
commafeed-server/src/main/java/com/commafeed/backend/model/UserSettings.java
View File

@@ -1,8 +1,6 @@
package com.commafeed.backend.model;
import java.io.Serializable;
import java.sql.Types;
import com.fasterxml.jackson.annotation.JsonProperty;
import jakarta.persistence.Column;
import jakarta.persistence.Embeddable;
import jakarta.persistence.Embedded;
@@ -14,13 +12,11 @@ import jakarta.persistence.JoinColumn;
import jakarta.persistence.Lob;
import jakarta.persistence.OneToOne;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.io.Serializable;
import java.sql.Types;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.annotations.JdbcTypeCode;
@Entity
@Table(name = "USERSETTINGS")
@@ -29,155 +25,154 @@ import lombok.Setter;
@Setter
public class UserSettings extends AbstractModel {
public enum ReadingMode {
@JsonProperty("all")
ALL,
public enum ReadingMode {
@JsonProperty("all")
ALL,
@JsonProperty("unread")
UNREAD;
@JsonProperty("unread")
UNREAD;
// method called for query parameters
public static ReadingMode fromString(final String s) {
return ReadingMode.valueOf(s.toUpperCase());
}
}
// method called for query parameters
public static ReadingMode fromString(final String s) {
return ReadingMode.valueOf(s.toUpperCase());
}
}
public enum ReadingOrder {
@JsonProperty("asc")
ASC,
public enum ReadingOrder {
@JsonProperty("asc")
ASC,
@JsonProperty("desc")
DESC;
@JsonProperty("desc")
DESC;
// method called for query parameters
public static ReadingOrder fromString(final String s) {
return ReadingOrder.valueOf(s.toUpperCase());
}
}
// method called for query parameters
public static ReadingOrder fromString(final String s) {
return ReadingOrder.valueOf(s.toUpperCase());
}
}
public enum ScrollMode {
@JsonProperty("always")
ALWAYS,
public enum ScrollMode {
@JsonProperty("always")
ALWAYS,
@JsonProperty("never")
NEVER,
@JsonProperty("never")
NEVER,
@JsonProperty("if_needed")
IF_NEEDED
}
@JsonProperty("if_needed")
IF_NEEDED
}
public enum IconDisplayMode {
@JsonProperty("always")
ALWAYS,
public enum IconDisplayMode {
@JsonProperty("always")
ALWAYS,
@JsonProperty("never")
NEVER,
@JsonProperty("never")
NEVER,
@JsonProperty("on_desktop")
ON_DESKTOP,
@JsonProperty("on_desktop")
ON_DESKTOP,
@JsonProperty("on_mobile")
ON_MOBILE
}
@JsonProperty("on_mobile")
ON_MOBILE
}
public enum PushNotificationType {
@JsonProperty("ntfy")
NTFY,
public enum PushNotificationType {
@JsonProperty("ntfy")
NTFY,
@JsonProperty("gotify")
GOTIFY,
@JsonProperty("gotify")
GOTIFY,
@JsonProperty("pushover")
PUSHOVER
}
@JsonProperty("pushover")
PUSHOVER
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false, unique = true)
private User user;
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false, unique = true)
private User user;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingMode readingMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingMode readingMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingOrder readingOrder;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingOrder readingOrder;
@Column(name = "user_lang", length = 4)
private String language;
@Column(name = "user_lang", length = 4)
private String language;
private boolean showRead;
private boolean scrollMarks;
private boolean showRead;
private boolean scrollMarks;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customCss;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customCss;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customJs;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customJs;
@Column(name = "scroll_speed")
private int scrollSpeed;
@Column(name = "scroll_speed")
private int scrollSpeed;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ScrollMode scrollMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ScrollMode scrollMode;
private int entriesToKeepOnTopWhenScrolling;
private int entriesToKeepOnTopWhenScrolling;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode starIconDisplayMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode starIconDisplayMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode externalLinkIconDisplayMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode externalLinkIconDisplayMode;
@Column(name = "primary_color", length = 32)
private String primaryColor;
@Column(name = "primary_color", length = 32)
private String primaryColor;
private boolean markAllAsReadConfirmation;
private boolean markAllAsReadNavigateToNextUnread;
private boolean customContextMenu;
private boolean mobileFooter;
private boolean unreadCountTitle;
private boolean unreadCountFavicon;
private boolean disablePullToRefresh;
private boolean markAllAsReadConfirmation;
private boolean markAllAsReadNavigateToNextUnread;
private boolean customContextMenu;
private boolean mobileFooter;
private boolean unreadCountTitle;
private boolean unreadCountFavicon;
private boolean disablePullToRefresh;
private boolean email;
private boolean gmail;
private boolean facebook;
private boolean twitter;
private boolean tumblr;
private boolean instapaper;
private boolean buffer;
private boolean email;
private boolean gmail;
private boolean facebook;
private boolean twitter;
private boolean tumblr;
private boolean instapaper;
private boolean buffer;
@Embedded
private PushNotificationUserSettings pushNotifications = new PushNotificationUserSettings();
@Embedded
private PushNotificationUserSettings pushNotifications = new PushNotificationUserSettings();
@Embeddable
@SuppressWarnings("serial")
@Getter
@Setter
public static class PushNotificationUserSettings implements Serializable {
@Enumerated(EnumType.STRING)
@Column(name = "push_notification_type", length = 16)
private PushNotificationType type;
@Embeddable
@SuppressWarnings("serial")
@Getter
@Setter
public static class PushNotificationUserSettings implements Serializable {
@Enumerated(EnumType.STRING)
@Column(name = "push_notification_type", length = 16)
private PushNotificationType type;
@Column(name = "push_notification_server_url", length = 1024)
private String serverUrl;
@Column(name = "push_notification_server_url", length = 1024)
private String serverUrl;
@Column(name = "push_notification_user_id", length = 512)
private String userId;
@Column(name = "push_notification_user_id", length = 512)
private String userId;
@Column(name = "push_notification_user_secret", length = 512)
private String userSecret;
@Column(name = "push_notification_topic", length = 256)
private String topic;
}
@Column(name = "push_notification_user_secret", length = 512)
private String userSecret;
@Column(name = "push_notification_topic", length = 256)
private String topic;
}
}

129
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLExporter.java
View File

@@ -1,13 +1,5 @@
package com.commafeed.backend.opml;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.model.FeedCategory;
@@ -16,75 +8,88 @@ import com.commafeed.backend.model.User;
import com.rometools.opml.feed.opml.Attribute;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import jakarta.inject.Singleton;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.ObjectUtils;
@RequiredArgsConstructor
@Singleton
public class OPMLExporter {
private static final Comparator<FeedCategory> CATEGORY_COMPARATOR = Comparator
.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0));
private static final Comparator<FeedSubscription> SUBSCRIPTION_COMPARATOR = Comparator
.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0));
private static final Comparator<FeedCategory> CATEGORY_COMPARATOR =
Comparator.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0));
private static final Comparator<FeedSubscription> SUBSCRIPTION_COMPARATOR =
Comparator.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0));
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
public Opml export(User user) {
Opml opml = new Opml();
opml.setFeedType("opml_1.0");
opml.setTitle(String.format("%s subscriptions in CommaFeed", user.getName()));
opml.setCreated(new Date());
public Opml export(User user) {
Opml opml = new Opml();
opml.setFeedType("opml_1.0");
opml.setTitle(String.format("%s subscriptions in CommaFeed", user.getName()));
opml.setCreated(new Date());
List<FeedCategory> categories = feedCategoryDAO.findAll(user);
categories.sort(CATEGORY_COMPARATOR);
List<FeedCategory> categories = feedCategoryDAO.findAll(user);
categories.sort(CATEGORY_COMPARATOR);
List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user);
subscriptions.sort(SUBSCRIPTION_COMPARATOR);
List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user);
subscriptions.sort(SUBSCRIPTION_COMPARATOR);
// export root categories
for (FeedCategory cat : categories.stream().filter(c -> c.getParent() == null).toList()) {
opml.getOutlines().add(buildCategoryOutline(cat, categories, subscriptions));
}
// export root categories
for (FeedCategory cat : categories.stream().filter(c -> c.getParent() == null).toList()) {
opml.getOutlines().add(buildCategoryOutline(cat, categories, subscriptions));
}
// export root subscriptions
for (FeedSubscription sub : subscriptions.stream().filter(s -> s.getCategory() == null).toList()) {
opml.getOutlines().add(buildSubscriptionOutline(sub));
}
// export root subscriptions
for (FeedSubscription sub :
subscriptions.stream().filter(s -> s.getCategory() == null).toList()) {
opml.getOutlines().add(buildSubscriptionOutline(sub));
}
return opml;
return opml;
}
}
private Outline buildCategoryOutline(
FeedCategory cat, List<FeedCategory> categories, List<FeedSubscription> subscriptions) {
Outline outline = new Outline();
outline.setText(cat.getName());
outline.setTitle(cat.getName());
private Outline buildCategoryOutline(FeedCategory cat, List<FeedCategory> categories, List<FeedSubscription> subscriptions) {
Outline outline = new Outline();
outline.setText(cat.getName());
outline.setTitle(cat.getName());
for (FeedCategory child :
categories.stream()
.filter(
c ->
c.getParent() != null
&& c.getParent().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildCategoryOutline(child, categories, subscriptions));
}
for (FeedCategory child : categories.stream()
.filter(c -> c.getParent() != null && c.getParent().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildCategoryOutline(child, categories, subscriptions));
}
for (FeedSubscription sub :
subscriptions.stream()
.filter(
s ->
s.getCategory() != null
&& s.getCategory().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildSubscriptionOutline(sub));
}
return outline;
}
for (FeedSubscription sub : subscriptions.stream()
.filter(s -> s.getCategory() != null && s.getCategory().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildSubscriptionOutline(sub));
}
return outline;
}
private Outline buildSubscriptionOutline(FeedSubscription sub) {
Outline outline = new Outline();
outline.setText(sub.getTitle());
outline.setTitle(sub.getTitle());
outline.setType("rss");
outline.getAttributes().add(new Attribute("xmlUrl", sub.getFeed().getUrl()));
if (sub.getFeed().getLink() != null) {
outline.getAttributes().add(new Attribute("htmlUrl", sub.getFeed().getLink()));
}
return outline;
}
private Outline buildSubscriptionOutline(FeedSubscription sub) {
Outline outline = new Outline();
outline.setText(sub.getTitle());
outline.setTitle(sub.getTitle());
outline.setType("rss");
outline.getAttributes().add(new Attribute("xmlUrl", sub.getFeed().getUrl()));
if (sub.getFeed().getLink() != null) {
outline.getAttributes().add(new Attribute("htmlUrl", sub.getFeed().getLink()));
}
return outline;
}
}

121
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLImporter.java
View File

@@ -1,13 +1,5 @@
package com.commafeed.backend.opml;
import java.io.StringReader;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.XMLCleaner;
@@ -18,71 +10,76 @@ import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.WireFeedInput;
import jakarta.inject.Singleton;
import java.io.StringReader;
import java.util.List;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class OPMLImporter {
private final XMLCleaner xmlCleaner;
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionService feedSubscriptionService;
private final XMLCleaner xmlCleaner;
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionService feedSubscriptionService;
public void importOpml(User user, String xml) throws IllegalArgumentException, FeedException {
xml = xmlCleaner.clean(xml);
if (xml == null) {
throw new IllegalArgumentException("Invalid OPML");
}
public void importOpml(User user, String xml) throws IllegalArgumentException, FeedException {
xml = xmlCleaner.clean(xml);
if (xml == null) {
throw new IllegalArgumentException("Invalid OPML");
}
WireFeedInput input = new WireFeedInput();
Opml feed = (Opml) input.build(new StringReader(xml));
List<Outline> outlines = feed.getOutlines();
for (int i = 0; i < outlines.size(); i++) {
handleOutline(user, outlines.get(i), null, i);
}
}
WireFeedInput input = new WireFeedInput();
Opml feed = (Opml) input.build(new StringReader(xml));
List<Outline> outlines = feed.getOutlines();
for (int i = 0; i < outlines.size(); i++) {
handleOutline(user, outlines.get(i), null, i);
}
}
private void handleOutline(User user, Outline outline, FeedCategory parent, int position) {
List<Outline> children = outline.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
FeedCategory category = feedCategoryDAO.findByName(user, name, parent);
if (category == null) {
if (StringUtils.isBlank(name)) {
name = "Unnamed category";
}
private void handleOutline(User user, Outline outline, FeedCategory parent, int position) {
List<Outline> children = outline.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
FeedCategory category = feedCategoryDAO.findByName(user, name, parent);
if (category == null) {
if (StringUtils.isBlank(name)) {
name = "Unnamed category";
}
category = new FeedCategory();
category.setName(name);
category.setParent(parent);
category.setUser(user);
category.setPosition(position);
feedCategoryDAO.persist(category);
}
category = new FeedCategory();
category.setName(name);
category.setParent(parent);
category.setUser(user);
category.setPosition(position);
feedCategoryDAO.persist(category);
}
for (int i = 0; i < children.size(); i++) {
handleOutline(user, children.get(i), category, i);
}
} else {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
if (StringUtils.isBlank(name)) {
name = "Unnamed subscription";
}
// make sure we continue with the import process even if a feed failed
try {
feedSubscriptionService.subscribe(user, outline.getXmlUrl(), name, parent, position);
} catch (Exception e) {
log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage());
}
}
}
for (int i = 0; i < children.size(); i++) {
handleOutline(user, children.get(i), category, i);
}
} else {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
if (StringUtils.isBlank(name)) {
name = "Unnamed subscription";
}
// make sure we continue with the import process even if a feed failed
try {
feedSubscriptionService.subscribe(
user, outline.getXmlUrl(), name, parent, position);
} catch (Exception e) {
log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage());
}
}
}
}

42
commafeed-server/src/main/java/com/commafeed/backend/rome/OPML11Parser.java
View File

@@ -1,38 +1,32 @@
package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Document;
import org.jdom2.Element;
import com.rometools.opml.io.impl.OPML10Parser;
import com.rometools.rome.feed.WireFeed;
import com.rometools.rome.io.FeedException;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.util.Locale;
import org.jdom2.Document;
import org.jdom2.Element;
/**
* Support for OPML 1.1 parsing
*
*/
/** Support for OPML 1.1 parsing */
@RegisterForReflection
public class OPML11Parser extends OPML10Parser {
public OPML11Parser() {
super("opml_1.1");
}
public OPML11Parser() {
super("opml_1.1");
}
@Override
public boolean isMyType(Document document) {
Element e = document.getRootElement();
@Override
public boolean isMyType(Document document) {
Element e = document.getRootElement();
return e.getName().equals("opml");
return e.getName().equals("opml");
}
}
@Override
public WireFeed parse(Document document, boolean validate, Locale locale) throws IllegalArgumentException, FeedException {
document.getRootElement().getChildren().add(new Element("head"));
return super.parse(document, validate, locale);
}
@Override
public WireFeed parse(Document document, boolean validate, Locale locale)
throws IllegalArgumentException, FeedException {
document.getRootElement().getChildren().add(new Element("head"));
return super.parse(document, validate, locale);
}
}

317
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentCleaningService.java
View File

@@ -1,12 +1,14 @@
package com.commafeed.backend.service;
import com.steadystate.css.parser.CSSOMParser;
import com.steadystate.css.parser.SACParserCSS21;
import jakarta.inject.Singleton;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
@@ -21,155 +23,212 @@ import org.w3c.css.sac.ErrorHandler;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
import com.steadystate.css.parser.CSSOMParser;
import com.steadystate.css.parser.SACParserCSS21;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
@Singleton
public class FeedEntryContentCleaningService {
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES =
Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES =
Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] {'(', ')'};
public String clean(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
public String clean(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
}
return content;
}
clean.outputSettings(
new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
}
return content;
}
private static Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1",
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
private static Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags(
"a",
"b",
"blockquote",
"br",
"caption",
"cite",
"code",
"col",
"colgroup",
"dd",
"div",
"dl",
"dt",
"em",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"i",
"iframe",
"img",
"li",
"ol",
"p",
"pre",
"q",
"small",
"strike",
"strong",
"sub",
"sup",
"table",
"tbody",
"td",
"tfoot",
"th",
"thead",
"tr",
"u",
"ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes(
"iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes(
"td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes(
"th",
"border",
"bordercolor",
"abbr",
"axis",
"colspan",
"rowspan",
"scope",
"width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
private String escapeIFrameCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
private String escapeIFrameCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl =
buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property)
&& StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private String escapeImgCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
private String escapeImgCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl =
buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
if (ALLOWED_IMG_CSS_RULES.contains(property)
&& StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser(new SACParserCSS21());
private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser(new SACParserCSS21());
parser.setErrorHandler(new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug("warning while parsing css: {}", exception.getMessage(), exception);
}
parser.setErrorHandler(
new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug(
"warning while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug("fatal error while parsing css: {}", exception.getMessage(), exception);
}
});
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug(
"fatal error while parsing css: {}",
exception.getMessage(),
exception);
}
});
return parser;
}
return parser;
}
}

100
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentService.java
View File

@@ -1,11 +1,5 @@
package com.commafeed.backend.service;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils;
@@ -13,59 +7,65 @@ import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.commafeed.backend.model.FeedEntryContent;
import jakarta.inject.Singleton;
import java.util.Optional;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
@RequiredArgsConstructor
@Singleton
public class FeedEntryContentService {
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryContentCleaningService cleaningService;
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryContentCleaningService cleaningService;
/**
* this is NOT thread-safe
*/
public FeedEntryContent findOrCreate(Content content, String baseUrl) {
FeedEntryContent entryContent = buildContent(content, baseUrl);
Optional<FeedEntryContent> existing = feedEntryContentDAO.findExisting(entryContent.getContentHash(), entryContent.getTitleHash())
.stream()
.filter(entryContent::equivalentTo)
.findFirst();
if (existing.isPresent()) {
return existing.get();
} else {
feedEntryContentDAO.persist(entryContent);
return entryContent;
}
}
/** this is NOT thread-safe */
public FeedEntryContent findOrCreate(Content content, String baseUrl) {
FeedEntryContent entryContent = buildContent(content, baseUrl);
Optional<FeedEntryContent> existing =
feedEntryContentDAO
.findExisting(entryContent.getContentHash(), entryContent.getTitleHash())
.stream()
.filter(entryContent::equivalentTo)
.findFirst();
if (existing.isPresent()) {
return existing.get();
} else {
feedEntryContentDAO.persist(entryContent);
return entryContent;
}
}
private FeedEntryContent buildContent(Content content, String baseUrl) {
FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitleHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.title())));
entryContent.setContentHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.content())));
entryContent.setTitle(FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false));
entryContent.setAuthor(FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
entryContent.setDirection(
FeedUtils.isRTL(content.title(), content.content()) ? FeedEntryContent.Direction.RTL : FeedEntryContent.Direction.LTR);
private FeedEntryContent buildContent(Content content, String baseUrl) {
FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitleHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.title())));
entryContent.setContentHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.content())));
entryContent.setTitle(
FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false));
entryContent.setAuthor(
FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
entryContent.setDirection(
FeedUtils.isRTL(content.title(), content.content())
? FeedEntryContent.Direction.RTL
: FeedEntryContent.Direction.LTR);
Enclosure enclosure = content.enclosure();
if (enclosure != null) {
entryContent.setEnclosureUrl(FeedUtils.truncate(enclosure.url(), 2048));
entryContent.setEnclosureType(enclosure.type());
}
Enclosure enclosure = content.enclosure();
if (enclosure != null) {
entryContent.setEnclosureUrl(FeedUtils.truncate(enclosure.url(), 2048));
entryContent.setEnclosureType(enclosure.type());
}
Media media = content.media();
if (media != null) {
entryContent.setMediaDescription(cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048));
entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
entryContent.setMediaThumbnailHeight(media.thumbnailHeight());
}
return entryContent;
}
Media media = content.media();
if (media != null) {
entryContent.setMediaDescription(
cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048));
entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
entryContent.setMediaThumbnailHeight(media.thumbnailHeight());
}
return entryContent;
}
}

166
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryFilteringService.java
View File

@@ -1,22 +1,7 @@
package com.commafeed.backend.service;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.model.FeedEntry;
import dev.cel.common.CelAbstractSyntaxTree;
import dev.cel.common.CelValidationException;
import dev.cel.common.types.SimpleType;
@@ -25,83 +10,112 @@ import dev.cel.compiler.CelCompilerFactory;
import dev.cel.runtime.CelEvaluationException;
import dev.cel.runtime.CelRuntime;
import dev.cel.runtime.CelRuntimeFactory;
import jakarta.inject.Singleton;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
@RequiredArgsConstructor
@Singleton
public class FeedEntryFilteringService {
private static final CelCompiler CEL_COMPILER = CelCompilerFactory.standardCelCompilerBuilder()
.addVar("title", SimpleType.STRING)
.addVar("titleLower", SimpleType.STRING)
.addVar("author", SimpleType.STRING)
.addVar("authorLower", SimpleType.STRING)
.addVar("content", SimpleType.STRING)
.addVar("contentLower", SimpleType.STRING)
.addVar("url", SimpleType.STRING)
.addVar("urlLower", SimpleType.STRING)
.addVar("categories", SimpleType.STRING)
.addVar("categoriesLower", SimpleType.STRING)
.build();
private static final CelRuntime CEL_RUNTIME = CelRuntimeFactory.standardCelRuntimeBuilder().build();
private static final CelCompiler CEL_COMPILER =
CelCompilerFactory.standardCelCompilerBuilder()
.addVar("title", SimpleType.STRING)
.addVar("titleLower", SimpleType.STRING)
.addVar("author", SimpleType.STRING)
.addVar("authorLower", SimpleType.STRING)
.addVar("content", SimpleType.STRING)
.addVar("contentLower", SimpleType.STRING)
.addVar("url", SimpleType.STRING)
.addVar("urlLower", SimpleType.STRING)
.addVar("categories", SimpleType.STRING)
.addVar("categoriesLower", SimpleType.STRING)
.build();
private static final CelRuntime CEL_RUNTIME =
CelRuntimeFactory.standardCelRuntimeBuilder().build();
private final ExecutorService executor = Executors.newCachedThreadPool();
private final CommaFeedConfiguration config;
private final ExecutorService executor = Executors.newCachedThreadPool();
private final CommaFeedConfiguration config;
public boolean filterMatchesEntry(String filter, FeedEntry entry) throws FeedEntryFilterException {
if (StringUtils.isBlank(filter)) {
return true;
}
public boolean filterMatchesEntry(String filter, FeedEntry entry)
throws FeedEntryFilterException {
if (StringUtils.isBlank(filter)) {
return true;
}
String title = entry.getContent().getTitle() == null ? "" : Jsoup.parse(entry.getContent().getTitle()).text();
String author = entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor();
String content = entry.getContent().getContent() == null ? "" : Jsoup.parse(entry.getContent().getContent()).text();
String url = entry.getUrl() == null ? "" : entry.getUrl();
String categories = entry.getContent().getCategories() == null ? "" : entry.getContent().getCategories();
String title =
entry.getContent().getTitle() == null
? ""
: Jsoup.parse(entry.getContent().getTitle()).text();
String author =
entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor();
String content =
entry.getContent().getContent() == null
? ""
: Jsoup.parse(entry.getContent().getContent()).text();
String url = entry.getUrl() == null ? "" : entry.getUrl();
String categories =
entry.getContent().getCategories() == null
? ""
: entry.getContent().getCategories();
Map<String, Object> data = new HashMap<>();
data.put("title", title);
data.put("titleLower", title.toLowerCase());
Map<String, Object> data = new HashMap<>();
data.put("title", title);
data.put("titleLower", title.toLowerCase());
data.put("author", author);
data.put("authorLower", author.toLowerCase());
data.put("author", author);
data.put("authorLower", author.toLowerCase());
data.put("content", content);
data.put("contentLower", content.toLowerCase());
data.put("content", content);
data.put("contentLower", content.toLowerCase());
data.put("url", url);
data.put("urlLower", url.toLowerCase());
data.put("url", url);
data.put("urlLower", url.toLowerCase());
data.put("categories", categories);
data.put("categoriesLower", categories.toLowerCase());
data.put("categories", categories);
data.put("categoriesLower", categories.toLowerCase());
Future<Object> future = executor.submit(() -> evaluateCelExpression(filter, data));
Object result;
try {
result = future.get(config.feedRefresh().filteringExpressionEvaluationTimeout().toMillis(), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e);
} catch (ExecutionException e) {
throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}
Future<Object> future = executor.submit(() -> evaluateCelExpression(filter, data));
Object result;
try {
result =
future.get(
config.feedRefresh().filteringExpressionEvaluationTimeout().toMillis(),
TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FeedEntryFilterException(
"interrupted while evaluating expression " + filter, e);
} catch (ExecutionException e) {
throw new FeedEntryFilterException(
"Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}
return Boolean.TRUE.equals(result);
}
return Boolean.TRUE.equals(result);
}
private Object evaluateCelExpression(String expression, Map<String, Object> data)
throws CelValidationException, CelEvaluationException {
CelAbstractSyntaxTree ast = CEL_COMPILER.compile(expression).getAst();
CelRuntime.Program program = CEL_RUNTIME.createProgram(ast);
return program.eval(data);
}
private Object evaluateCelExpression(String expression, Map<String, Object> data)
throws CelValidationException, CelEvaluationException {
CelAbstractSyntaxTree ast = CEL_COMPILER.compile(expression).getAst();
CelRuntime.Program program = CEL_RUNTIME.createProgram(ast);
return program.eval(data);
}
@SuppressWarnings("serial")
public static class FeedEntryFilterException extends Exception {
public FeedEntryFilterException(String message, Throwable t) {
super(message, t);
}
}
@SuppressWarnings("serial")
public static class FeedEntryFilterException extends Exception {
public FeedEntryFilterException(String message, Throwable t) {
super(message, t);
}
}
}

228
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java
View File

@@ -1,12 +1,5 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
@@ -20,7 +13,11 @@ import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -29,113 +26,146 @@ import lombok.extern.slf4j.Slf4j;
@Singleton
public class FeedEntryService {
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
public FeedEntry find(Feed feed, Entry entry) {
String guidHash = Digests.sha1Hex(entry.guid());
return feedEntryDAO.findExisting(guidHash, feed);
}
public FeedEntry find(Feed feed, Entry entry) {
String guidHash = Digests.sha1Hex(entry.guid());
return feedEntryDAO.findExisting(guidHash, feed);
}
public FeedEntry create(Feed feed, Entry entry) {
FeedEntry feedEntry = new FeedEntry();
feedEntry.setGuid(FeedUtils.truncate(entry.guid(), 2048));
feedEntry.setGuidHash(Digests.sha1Hex(entry.guid()));
feedEntry.setUrl(FeedUtils.truncate(entry.url(), 2048));
feedEntry.setPublished(entry.published());
feedEntry.setInserted(Instant.now());
feedEntry.setFeed(feed);
feedEntry.setContent(feedEntryContentService.findOrCreate(entry.content(), feed.getLink()));
public FeedEntry create(Feed feed, Entry entry) {
FeedEntry feedEntry = new FeedEntry();
feedEntry.setGuid(FeedUtils.truncate(entry.guid(), 2048));
feedEntry.setGuidHash(Digests.sha1Hex(entry.guid()));
feedEntry.setUrl(FeedUtils.truncate(entry.url(), 2048));
feedEntry.setPublished(entry.published());
feedEntry.setInserted(Instant.now());
feedEntry.setFeed(feed);
feedEntry.setContent(feedEntryContentService.findOrCreate(entry.content(), feed.getLink()));
feedEntryDAO.persist(feedEntry);
return feedEntry;
}
feedEntryDAO.persist(feedEntry);
return feedEntry;
}
public List<Entry> removeExistingEntries(Feed feed, List<Entry> entries) {
Set<String> guidHashes = entries.stream().map(e -> Digests.sha1Hex(e.guid())).collect(Collectors.toSet());
Set<String> existingGuidHashes = feedEntryDAO.findExistingGuidHashes(guidHashes, feed);
return entries.stream().filter(e -> !existingGuidHashes.contains(Digests.sha1Hex(e.guid()))).toList();
}
public List<Entry> removeExistingEntries(Feed feed, List<Entry> entries) {
Set<String> guidHashes =
entries.stream().map(e -> Digests.sha1Hex(e.guid())).collect(Collectors.toSet());
Set<String> existingGuidHashes = feedEntryDAO.findExistingGuidHashes(guidHashes, feed);
return entries.stream()
.filter(e -> !existingGuidHashes.contains(Digests.sha1Hex(e.guid())))
.toList();
}
public boolean applyFilter(FeedSubscription sub, FeedEntry entry) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
public boolean applyFilter(FeedSubscription sub, FeedEntry entry) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.persist(status);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.persist(status);
}
return matches;
}
return matches;
}
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, entry.getFeed());
if (sub == null) {
return;
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, entry.getFeed());
if (sub == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
if (status.isMarkable()) {
status.setRead(read);
feedEntryStatusDAO.merge(status);
}
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
if (status.isMarkable()) {
status.setRead(read);
feedEntryStatusDAO.merge(status);
}
}
public void starEntry(User user, Long entryId, Long subscriptionId, boolean starred) {
public void starEntry(User user, Long entryId, Long subscriptionId, boolean starred) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subscriptionId);
if (sub == null) {
return;
}
FeedSubscription sub = feedSubscriptionDAO.findById(user, subscriptionId);
if (sub == null) {
return;
}
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
status.setStarred(starred);
feedEntryStatusDAO.merge(status);
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
status.setStarred(starred);
feedEntryStatusDAO.merge(status);
}
public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Instant olderThan, Instant insertedBefore,
List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null,
false, null, null, null);
markList(statuses, olderThan, insertedBefore);
}
public void markSubscriptionEntries(
User user,
List<FeedSubscription> subscriptions,
Instant olderThan,
Instant insertedBefore,
List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses =
feedEntryStatusDAO.findBySubscriptions(
user,
subscriptions,
true,
keywords,
null,
-1,
-1,
null,
false,
null,
null,
null);
markList(statuses, olderThan, insertedBefore);
}
public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, null, -1, -1, null, false);
markList(statuses, olderThan, insertedBefore);
}
public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statuses =
feedEntryStatusDAO.findStarred(user, null, null, -1, -1, null, false);
markList(statuses, olderThan, insertedBefore);
}
private void markList(List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statusesToMark = statuses.stream().filter(FeedEntryStatus::isMarkable).filter(s -> {
Instant entryDate = s.getEntry().getPublished();
return olderThan == null || entryDate == null || entryDate.isBefore(olderThan);
}).filter(s -> {
Instant insertedDate = s.getEntry().getInserted();
return insertedBefore == null || insertedDate == null || insertedDate.isBefore(insertedBefore);
}).toList();
private void markList(
List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statusesToMark =
statuses.stream()
.filter(FeedEntryStatus::isMarkable)
.filter(
s -> {
Instant entryDate = s.getEntry().getPublished();
return olderThan == null
|| entryDate == null
|| entryDate.isBefore(olderThan);
})
.filter(
s -> {
Instant insertedDate = s.getEntry().getInserted();
return insertedBefore == null
|| insertedDate == null
|| insertedDate.isBefore(insertedBefore);
})
.toList();
statusesToMark.forEach(s -> {
s.setRead(true);
feedEntryStatusDAO.merge(s);
});
}
statusesToMark.forEach(
s -> {
s.setRead(true);
feedEntryStatusDAO.merge(s);
});
}
}

49
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryTagService.java
View File

@@ -1,43 +1,42 @@
package com.commafeed.backend.service;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryTagDAO;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.User;
import jakarta.inject.Singleton;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryTagService {
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO;
public void updateTags(User user, Long entryId, List<String> tagNames) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
public void updateTags(User user, Long entryId, List<String> tagNames) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry);
Set<String> existingTagNames = existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry);
Set<String> existingTagNames =
existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
List<FeedEntryTag> addList = tagNames.stream()
.filter(name -> !existingTagNames.contains(name))
.map(name -> new FeedEntryTag(user, entry, name))
.toList();
List<FeedEntryTag> removeList = existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).toList();
addList.forEach(feedEntryTagDAO::persist);
feedEntryTagDAO.delete(removeList);
}
List<FeedEntryTag> addList =
tagNames.stream()
.filter(name -> !existingTagNames.contains(name))
.map(name -> new FeedEntryTag(user, entry, name))
.toList();
List<FeedEntryTag> removeList =
existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).toList();
addList.forEach(feedEntryTagDAO::persist);
feedEntryTagDAO.delete(removeList);
}
}

99
commafeed-server/src/main/java/com/commafeed/backend/service/FeedFaviconService.java
View File

@@ -1,71 +1,72 @@
package com.commafeed.backend.service;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.MediaType;
import org.apache.commons.lang3.ArrayUtils;
import com.commafeed.backend.favicon.Favicon;
import com.commafeed.backend.favicon.FaviconFetcher;
import com.commafeed.backend.model.Feed;
import com.google.common.io.Resources;
import io.quarkus.arc.All;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.MediaType;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ArrayUtils;
@Singleton
@Slf4j
public class FeedFaviconService {
private static final Set<MediaType> ICON_MIMETYPE_BLACKLIST = Set.of(MediaType.APPLICATION_XML_TYPE, MediaType.TEXT_HTML_TYPE);
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
private static final Set<MediaType> ICON_MIMETYPE_BLACKLIST =
Set.of(MediaType.APPLICATION_XML_TYPE, MediaType.TEXT_HTML_TYPE);
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
private final List<FaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
private final List<FaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
public FeedFaviconService(@All List<FaviconFetcher> faviconFetchers) throws IOException {
this.faviconFetchers = faviconFetchers;
this.defaultFavicon = new Favicon(
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif");
}
public FeedFaviconService(@All List<FaviconFetcher> faviconFetchers) throws IOException {
this.faviconFetchers = faviconFetchers;
this.defaultFavicon =
new Favicon(
Resources.toByteArray(
Objects.requireNonNull(
getClass().getResource("/images/default_favicon.gif"))),
"image/gif");
}
public Favicon fetchFavicon(Feed feed) {
for (FaviconFetcher faviconFetcher : faviconFetchers) {
Favicon icon = faviconFetcher.fetch(feed);
if (isFaviconValid(icon)) {
return icon;
}
}
return defaultFavicon;
}
public Favicon fetchFavicon(Feed feed) {
for (FaviconFetcher faviconFetcher : faviconFetchers) {
Favicon icon = faviconFetcher.fetch(feed);
if (isFaviconValid(icon)) {
return icon;
}
}
return defaultFavicon;
}
private static boolean isFaviconValid(Favicon favicon) {
if (favicon == null || ArrayUtils.isEmpty(favicon.icon())) {
return false;
}
private static boolean isFaviconValid(Favicon favicon) {
if (favicon == null || ArrayUtils.isEmpty(favicon.icon())) {
return false;
}
long length = favicon.icon().length;
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
long length = favicon.icon().length;
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
if (ICON_MIMETYPE_BLACKLIST.stream().anyMatch(bl -> bl.isCompatible(favicon.mediaType()))) {
log.debug("Content-Type {} is blacklisted", favicon.mediaType());
return false;
}
if (ICON_MIMETYPE_BLACKLIST.stream().anyMatch(bl -> bl.isCompatible(favicon.mediaType()))) {
log.debug("Content-Type {} is blacklisted", favicon.mediaType());
return false;
}
return true;
}
return true;
}
}

54
commafeed-server/src/main/java/com/commafeed/backend/service/FeedService.java
View File

@@ -1,46 +1,42 @@
package com.commafeed.backend.service;
import java.time.Instant;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.Urls;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.Models;
import jakarta.inject.Singleton;
import java.time.Instant;
import lombok.RequiredArgsConstructor;
@Singleton
@RequiredArgsConstructor
public class FeedService {
private final FeedDAO feedDAO;
private final FeedDAO feedDAO;
public synchronized Feed findOrCreate(String url) {
String normalizedUrl = Urls.normalize(url);
String normalizedUrlHash = Digests.sha1Hex(normalizedUrl);
Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash);
if (feed == null) {
feed = new Feed();
feed.setUrl(url);
feed.setNormalizedUrl(normalizedUrl);
feed.setNormalizedUrlHash(normalizedUrlHash);
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
feedDAO.persist(feed);
}
return feed;
}
public void update(Feed feed) {
String normalized = Urls.normalize(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(Digests.sha1Hex(normalized));
feed.setLastUpdated(Instant.now());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.merge(feed);
}
public synchronized Feed findOrCreate(String url) {
String normalizedUrl = Urls.normalize(url);
String normalizedUrlHash = Digests.sha1Hex(normalizedUrl);
Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash);
if (feed == null) {
feed = new Feed();
feed.setUrl(url);
feed.setNormalizedUrl(normalizedUrl);
feed.setNormalizedUrlHash(normalizedUrlHash);
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
feedDAO.persist(feed);
}
return feed;
}
public void update(Feed feed) {
String normalized = Urls.normalize(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(Digests.sha1Hex(normalized));
feed.setLastUpdated(Instant.now());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.merge(feed);
}
}

193
commafeed-server/src/main/java/com/commafeed/backend/service/FeedSubscriptionService.java
View File

@@ -1,12 +1,5 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.Urls;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
@@ -18,107 +11,125 @@ import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.UnreadCount;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedSubscriptionService {
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService;
private final FeedRefreshEngine feedRefreshEngine;
private final CommaFeedConfiguration config;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService;
private final FeedRefreshEngine feedRefreshEngine;
private final CommaFeedConfiguration config;
public FeedSubscriptionService(FeedEntryStatusDAO feedEntryStatusDAO, FeedSubscriptionDAO feedSubscriptionDAO, FeedService feedService,
FeedRefreshEngine feedRefreshEngine, CommaFeedConfiguration config) {
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.feedService = feedService;
this.feedRefreshEngine = feedRefreshEngine;
this.config = config;
public FeedSubscriptionService(
FeedEntryStatusDAO feedEntryStatusDAO,
FeedSubscriptionDAO feedSubscriptionDAO,
FeedService feedService,
FeedRefreshEngine feedRefreshEngine,
CommaFeedConfiguration config) {
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.feedService = feedService;
this.feedRefreshEngine = feedRefreshEngine;
this.config = config;
// automatically refresh new feeds after they are subscribed to
// we need to use this hook because the feed needs to have been persisted before being processed by the feed engine
feedSubscriptionDAO.onPostCommitInsert(sub -> {
Feed feed = sub.getFeed();
if (feed.getDisabledUntil() == null || feed.getDisabledUntil().isBefore(Instant.now())) {
feedRefreshEngine.refreshImmediately(feed);
}
});
}
// automatically refresh new feeds after they are subscribed to
// we need to use this hook because the feed needs to have been persisted before being
// processed
// by the feed engine
feedSubscriptionDAO.onPostCommitInsert(
sub -> {
Feed feed = sub.getFeed();
if (feed.getDisabledUntil() == null
|| feed.getDisabledUntil().isBefore(Instant.now())) {
feedRefreshEngine.refreshImmediately(feed);
}
});
}
public long subscribe(User user, String url, String title, FeedCategory category, int position) {
Integer maxFeedsPerUser = config.database().cleanup().maxFeedsPerUser();
if (maxFeedsPerUser > 0 && feedSubscriptionDAO.count(user) >= maxFeedsPerUser) {
String message = String.format("You cannot subscribe to more feeds on this CommaFeed instance (max %s feeds per user)",
maxFeedsPerUser);
throw new FeedSubscriptionException(message);
}
public long subscribe(
User user, String url, String title, FeedCategory category, int position) {
Integer maxFeedsPerUser = config.database().cleanup().maxFeedsPerUser();
if (maxFeedsPerUser > 0 && feedSubscriptionDAO.count(user) >= maxFeedsPerUser) {
String message =
String.format(
"You cannot subscribe to more feeds on this CommaFeed instance (max %s feeds per user)",
maxFeedsPerUser);
throw new FeedSubscriptionException(message);
}
Feed feed = feedService.findOrCreate(url);
Feed feed = feedService.findOrCreate(url);
// upgrade feed to https if it was using http
if (Urls.isHttp(feed.getUrl()) && Urls.isHttps(url)) {
feed.setUrl(url);
}
// upgrade feed to https if it was using http
if (Urls.isHttp(feed.getUrl()) && Urls.isHttps(url)) {
feed.setUrl(url);
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
if (sub == null) {
sub = new FeedSubscription();
sub.setFeed(feed);
sub.setUser(user);
}
sub.setCategory(category);
sub.setPosition(position);
sub.setTitle(FeedUtils.truncate(title, 128));
return feedSubscriptionDAO.merge(sub).getId();
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
if (sub == null) {
sub = new FeedSubscription();
sub.setFeed(feed);
sub.setUser(user);
}
sub.setCategory(category);
sub.setPosition(position);
sub.setTitle(FeedUtils.truncate(title, 128));
return feedSubscriptionDAO.merge(sub).getId();
}
public boolean unsubscribe(User user, Long subId) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subId);
if (sub != null) {
feedSubscriptionDAO.delete(sub);
return true;
} else {
return false;
}
}
public boolean unsubscribe(User user, Long subId) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subId);
if (sub != null) {
feedSubscriptionDAO.delete(sub);
return true;
} else {
return false;
}
}
public void refreshAll(User user) throws ForceFeedRefreshTooSoonException {
Instant lastForceRefresh = user.getLastForceRefresh();
if (lastForceRefresh != null && lastForceRefresh.plus(config.feedRefresh().forceRefreshCooldownDuration()).isAfter(Instant.now())) {
throw new ForceFeedRefreshTooSoonException();
}
public void refreshAll(User user) throws ForceFeedRefreshTooSoonException {
Instant lastForceRefresh = user.getLastForceRefresh();
if (lastForceRefresh != null
&& lastForceRefresh
.plus(config.feedRefresh().forceRefreshCooldownDuration())
.isAfter(Instant.now())) {
throw new ForceFeedRefreshTooSoonException();
}
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
user.setLastForceRefresh(Instant.now());
}
user.setLastForceRefresh(Instant.now());
}
public Map<Long, UnreadCount> getUnreadCount(User user) {
return feedSubscriptionDAO.findAll(user)
.stream()
.collect(Collectors.toMap(FeedSubscription::getId, feedEntryStatusDAO::getUnreadCount));
}
public Map<Long, UnreadCount> getUnreadCount(User user) {
return feedSubscriptionDAO.findAll(user).stream()
.collect(
Collectors.toMap(
FeedSubscription::getId, feedEntryStatusDAO::getUnreadCount));
}
@SuppressWarnings("serial")
public static class FeedSubscriptionException extends RuntimeException {
private FeedSubscriptionException(String msg) {
super(msg);
}
}
@SuppressWarnings("serial")
public static class ForceFeedRefreshTooSoonException extends Exception {
private ForceFeedRefreshTooSoonException() {
super();
}
}
@SuppressWarnings("serial")
public static class FeedSubscriptionException extends RuntimeException {
private FeedSubscriptionException(String msg) {
super(msg);
}
}
@SuppressWarnings("serial")
public static class ForceFeedRefreshTooSoonException extends Exception {
private ForceFeedRefreshTooSoonException() {
super();
}
}
}

14
commafeed-server/src/main/java/com/commafeed/backend/service/MailService.java
View File

@@ -1,21 +1,19 @@
package com.commafeed.backend.service;
import jakarta.inject.Singleton;
import com.commafeed.backend.model.User;
import io.quarkus.mailer.Mail;
import io.quarkus.mailer.Mailer;
import jakarta.inject.Singleton;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class MailService {
private final Mailer mailer;
private final Mailer mailer;
public void sendMail(User user, String subject, String content) {
Mail mail = Mail.withHtml(user.getEmail(), "CommaFeed - " + subject, content);
mailer.send(mail);
}
public void sendMail(User user, String subject, String content) {
Mail mail = Mail.withHtml(user.getEmail(), "CommaFeed - " + subject, content);
mailer.send(mail);
}
}

127
commafeed-server/src/main/java/com/commafeed/backend/service/PasswordEncryptionService.java
View File

@@ -1,20 +1,16 @@
package com.commafeed.backend.service;
import jakarta.inject.Singleton;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.spec.KeySpec;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
// taken from http://www.javacodegeeks.com/2012/05/secure-password-storage-donts-dos-and.html
@Slf4j
@@ -22,72 +18,71 @@ import lombok.extern.slf4j.Slf4j;
@Singleton
public class PasswordEncryptionService {
public boolean authenticate(String attemptedPassword, byte[] encryptedPassword, byte[] salt) {
if (StringUtils.isBlank(attemptedPassword)) {
return false;
}
// Encrypt the clear-text password using the same salt that was used to
// encrypt the original password
byte[] encryptedAttemptedPassword = null;
try {
encryptedAttemptedPassword = getEncryptedPassword(attemptedPassword, salt);
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
public boolean authenticate(String attemptedPassword, byte[] encryptedPassword, byte[] salt) {
if (StringUtils.isBlank(attemptedPassword)) {
return false;
}
// Encrypt the clear-text password using the same salt that was used to
// encrypt the original password
byte[] encryptedAttemptedPassword = null;
try {
encryptedAttemptedPassword = getEncryptedPassword(attemptedPassword, salt);
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
if (encryptedAttemptedPassword == null) {
return false;
}
if (encryptedAttemptedPassword == null) {
return false;
}
// Authentication succeeds if encrypted password that the user entered
// is equal to the stored hash
return MessageDigest.isEqual(encryptedPassword, encryptedAttemptedPassword);
}
// Authentication succeeds if encrypted password that the user entered
// is equal to the stored hash
return MessageDigest.isEqual(encryptedPassword, encryptedAttemptedPassword);
}
public byte[] getEncryptedPassword(String password, byte[] salt) {
// PBKDF2 with SHA-1 as the hashing algorithm. Note that the NIST
// specifically names SHA-1 as an acceptable hashing algorithm for
// PBKDF2
String algorithm = "PBKDF2WithHmacSHA1";
// SHA-1 generates 160 bit hashes, so that's what makes sense here
int derivedKeyLength = 160;
// Pick an iteration count that works for you. The NIST recommends at
// least 1,000 iterations:
// http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf
// iOS 4.x reportedly uses 10,000:
// http://blog.crackpassword.com/2010/09/smartphone-forensics-cracking-blackberry-backup-passwords/
int iterations = 20000;
public byte[] getEncryptedPassword(String password, byte[] salt) {
// PBKDF2 with SHA-1 as the hashing algorithm. Note that the NIST
// specifically names SHA-1 as an acceptable hashing algorithm for
// PBKDF2
String algorithm = "PBKDF2WithHmacSHA1";
// SHA-1 generates 160 bit hashes, so that's what makes sense here
int derivedKeyLength = 160;
// Pick an iteration count that works for you. The NIST recommends at
// least 1,000 iterations:
// http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf
// iOS 4.x reportedly uses 10,000:
// http://blog.crackpassword.com/2010/09/smartphone-forensics-cracking-blackberry-backup-passwords/
int iterations = 20000;
KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, derivedKeyLength);
KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, derivedKeyLength);
byte[] bytes = null;
try {
SecretKeyFactory f = SecretKeyFactory.getInstance(algorithm);
SecretKey key = f.generateSecret(spec);
bytes = key.getEncoded();
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
return bytes;
}
byte[] bytes = null;
try {
SecretKeyFactory f = SecretKeyFactory.getInstance(algorithm);
SecretKey key = f.generateSecret(spec);
bytes = key.getEncoded();
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
return bytes;
}
public byte[] generateSalt() {
// VERY important to use SecureRandom instead of just Random
public byte[] generateSalt() {
// VERY important to use SecureRandom instead of just Random
byte[] salt = null;
try {
SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
// Generate a 8 byte (64 bit) salt as recommended by RSA PKCS5
salt = new byte[8];
random.nextBytes(salt);
} catch (NoSuchAlgorithmException e) {
// should never happen
log.error(e.getMessage(), e);
}
return salt;
}
byte[] salt = null;
try {
SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
// Generate a 8 byte (64 bit) salt as recommended by RSA PKCS5
salt = new byte[8];
random.nextBytes(salt);
} catch (NoSuchAlgorithmException e) {
// should never happen
log.error(e.getMessage(), e);
}
return salt;
}
}

307
commafeed-server/src/main/java/com/commafeed/backend/service/PushNotificationService.java
View File

@@ -1,12 +1,20 @@
package com.commafeed.backend.service;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpClientFactory;
import com.commafeed.backend.Urls;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.UserSettings.PushNotificationUserSettings;
import io.vertx.core.json.JsonObject;
import jakarta.inject.Singleton;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import jakarta.inject.Singleton;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.config.RequestConfig;
@@ -18,163 +26,196 @@ import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.hc.core5.util.Timeout;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpClientFactory;
import com.commafeed.backend.Urls;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.UserSettings.PushNotificationUserSettings;
import io.vertx.core.json.JsonObject;
import lombok.extern.slf4j.Slf4j;
@Singleton
@Slf4j
public class PushNotificationService {
private final CloseableHttpClient httpClient;
private final Meter meter;
private final CommaFeedConfiguration config;
private final CloseableHttpClient httpClient;
private final Meter meter;
private final CommaFeedConfiguration config;
public PushNotificationService(HttpClientFactory httpClientFactory, MetricRegistry metrics, CommaFeedConfiguration config) {
this.httpClient = httpClientFactory.newClient(config.pushNotifications().threads());
this.meter = metrics.meter(MetricRegistry.name(getClass(), "notify"));
this.config = config;
}
public PushNotificationService(
HttpClientFactory httpClientFactory,
MetricRegistry metrics,
CommaFeedConfiguration config) {
this.httpClient = httpClientFactory.newClient(config.pushNotifications().threads());
this.meter = metrics.meter(MetricRegistry.name(getClass(), "notify"));
this.config = config;
}
public void notify(PushNotificationUserSettings settings, FeedSubscription subscription, FeedEntry entry) {
if (!config.pushNotifications().enabled() || settings.getType() == null) {
return;
}
public void notify(
PushNotificationUserSettings settings, FeedSubscription subscription, FeedEntry entry) {
if (!config.pushNotifications().enabled() || settings.getType() == null) {
return;
}
log.debug("sending {} push notification for entry {} in feed {}", settings.getType(), entry.getId(),
subscription.getFeed().getId());
String entryTitle = entry.getContent() != null ? entry.getContent().getTitle() : null;
String entryUrl = entry.getUrl();
String feedTitle = subscription.getTitle();
log.debug(
"sending {} push notification for entry {} in feed {}",
settings.getType(),
entry.getId(),
subscription.getFeed().getId());
String entryTitle = entry.getContent() != null ? entry.getContent().getTitle() : null;
String entryUrl = entry.getUrl();
String feedTitle = subscription.getTitle();
if (StringUtils.isBlank(entryTitle)) {
entryTitle = "New entry";
}
if (StringUtils.isBlank(entryTitle)) {
entryTitle = "New entry";
}
try {
switch (settings.getType()) {
case NTFY -> sendNtfy(settings, feedTitle, entryTitle, entryUrl);
case GOTIFY -> sendGotify(settings, feedTitle, entryTitle, entryUrl);
case PUSHOVER -> sendPushover(settings, feedTitle, entryTitle, entryUrl);
default -> throw new IllegalStateException("unsupported notification type: " + settings.getType());
}
} catch (IOException e) {
throw new PushNotificationException("Failed to send external notification", e);
}
try {
switch (settings.getType()) {
case NTFY -> sendNtfy(settings, feedTitle, entryTitle, entryUrl);
case GOTIFY -> sendGotify(settings, feedTitle, entryTitle, entryUrl);
case PUSHOVER -> sendPushover(settings, feedTitle, entryTitle, entryUrl);
default ->
throw new IllegalStateException(
"unsupported notification type: " + settings.getType());
}
} catch (IOException e) {
throw new PushNotificationException("Failed to send external notification", e);
}
meter.mark();
}
meter.mark();
}
private void sendNtfy(PushNotificationUserSettings settings, String feedTitle, String entryTitle, String entryUrl) throws IOException {
String serverUrl = Urls.removeTrailingSlash(settings.getServerUrl());
String topic = settings.getTopic();
private void sendNtfy(
PushNotificationUserSettings settings,
String feedTitle,
String entryTitle,
String entryUrl)
throws IOException {
String serverUrl = Urls.removeTrailingSlash(settings.getServerUrl());
String topic = settings.getTopic();
if (StringUtils.isBlank(serverUrl) || StringUtils.isBlank(topic)) {
log.warn("ntfy notification skipped: missing server URL or topic");
return;
}
if (StringUtils.isBlank(serverUrl) || StringUtils.isBlank(topic)) {
log.warn("ntfy notification skipped: missing server URL or topic");
return;
}
HttpPost request = new HttpPost(serverUrl + "/" + topic);
request.setConfig(RequestConfig.custom().setResponseTimeout(Timeout.of(config.httpClient().responseTimeout())).build());
request.addHeader("Title", feedTitle);
request.setEntity(new StringEntity(entryTitle, StandardCharsets.UTF_8));
HttpPost request = new HttpPost(serverUrl + "/" + topic);
request.setConfig(
RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
.build());
request.addHeader("Title", feedTitle);
request.setEntity(new StringEntity(entryTitle, StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(entryUrl)) {
request.addHeader("Click", entryUrl);
}
if (StringUtils.isNotBlank(entryUrl)) {
request.addHeader("Click", entryUrl);
}
if (StringUtils.isNotBlank(settings.getUserSecret())) {
request.addHeader("Authorization", "Bearer " + settings.getUserSecret());
}
if (StringUtils.isNotBlank(settings.getUserSecret())) {
request.addHeader("Authorization", "Bearer " + settings.getUserSecret());
}
httpClient.execute(request, response -> {
if (response.getCode() >= 400) {
throw new PushNotificationException("ntfy notification failed with status " + response.getCode());
}
return null;
});
}
httpClient.execute(
request,
response -> {
if (response.getCode() >= 400) {
throw new PushNotificationException(
"ntfy notification failed with status " + response.getCode());
}
return null;
});
}
private void sendGotify(PushNotificationUserSettings settings, String feedTitle, String entryTitle, String entryUrl)
throws IOException {
String serverUrl = Urls.removeTrailingSlash(settings.getServerUrl());
String token = settings.getUserSecret();
private void sendGotify(
PushNotificationUserSettings settings,
String feedTitle,
String entryTitle,
String entryUrl)
throws IOException {
String serverUrl = Urls.removeTrailingSlash(settings.getServerUrl());
String token = settings.getUserSecret();
if (StringUtils.isBlank(serverUrl) || StringUtils.isBlank(token)) {
log.warn("gotify notification skipped: missing server URL or token");
return;
}
if (StringUtils.isBlank(serverUrl) || StringUtils.isBlank(token)) {
log.warn("gotify notification skipped: missing server URL or token");
return;
}
JsonObject json = new JsonObject();
json.put("title", feedTitle);
json.put("message", entryTitle);
json.put("priority", 5);
if (StringUtils.isNotBlank(entryUrl)) {
json.put("extras",
new JsonObject().put("client::notification", new JsonObject().put("click", new JsonObject().put("url", entryUrl))));
}
JsonObject json = new JsonObject();
json.put("title", feedTitle);
json.put("message", entryTitle);
json.put("priority", 5);
if (StringUtils.isNotBlank(entryUrl)) {
json.put(
"extras",
new JsonObject()
.put(
"client::notification",
new JsonObject()
.put("click", new JsonObject().put("url", entryUrl))));
}
HttpPost request = new HttpPost(serverUrl + "/message");
request.setConfig(RequestConfig.custom().setResponseTimeout(Timeout.of(config.httpClient().responseTimeout())).build());
request.addHeader("X-Gotify-Key", token);
request.setEntity(new StringEntity(json.toString(), ContentType.APPLICATION_JSON));
HttpPost request = new HttpPost(serverUrl + "/message");
request.setConfig(
RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
.build());
request.addHeader("X-Gotify-Key", token);
request.setEntity(new StringEntity(json.toString(), ContentType.APPLICATION_JSON));
httpClient.execute(request, response -> {
if (response.getCode() >= 400) {
throw new PushNotificationException("gotify notification failed with status " + response.getCode());
}
return null;
});
}
httpClient.execute(
request,
response -> {
if (response.getCode() >= 400) {
throw new PushNotificationException(
"gotify notification failed with status " + response.getCode());
}
return null;
});
}
private void sendPushover(PushNotificationUserSettings settings, String feedTitle, String entryTitle, String entryUrl)
throws IOException {
String token = settings.getUserSecret();
String userKey = settings.getUserId();
private void sendPushover(
PushNotificationUserSettings settings,
String feedTitle,
String entryTitle,
String entryUrl)
throws IOException {
String token = settings.getUserSecret();
String userKey = settings.getUserId();
if (StringUtils.isBlank(token) || StringUtils.isBlank(userKey)) {
log.warn("pushover notification skipped: missing token or user key");
return;
}
if (StringUtils.isBlank(token) || StringUtils.isBlank(userKey)) {
log.warn("pushover notification skipped: missing token or user key");
return;
}
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("token", token));
params.add(new BasicNameValuePair("user", userKey));
params.add(new BasicNameValuePair("title", feedTitle));
params.add(new BasicNameValuePair("message", entryTitle));
if (StringUtils.isNotBlank(entryUrl)) {
params.add(new BasicNameValuePair("url", entryUrl));
}
List<NameValuePair> params = new ArrayList<>();
params.add(new BasicNameValuePair("token", token));
params.add(new BasicNameValuePair("user", userKey));
params.add(new BasicNameValuePair("title", feedTitle));
params.add(new BasicNameValuePair("message", entryTitle));
if (StringUtils.isNotBlank(entryUrl)) {
params.add(new BasicNameValuePair("url", entryUrl));
}
HttpPost request = new HttpPost("https://api.pushover.net/1/messages.json");
request.setConfig(RequestConfig.custom().setResponseTimeout(Timeout.of(config.httpClient().responseTimeout())).build());
request.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8));
HttpPost request = new HttpPost("https://api.pushover.net/1/messages.json");
request.setConfig(
RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
.build());
request.setEntity(new UrlEncodedFormEntity(params, StandardCharsets.UTF_8));
httpClient.execute(request, response -> {
if (response.getCode() >= 400) {
throw new PushNotificationException("pushover notification failed with status " + response.getCode());
}
return null;
});
}
httpClient.execute(
request,
response -> {
if (response.getCode() >= 400) {
throw new PushNotificationException(
"pushover notification failed with status " + response.getCode());
}
return null;
});
}
public static class PushNotificationException extends RuntimeException {
private static final long serialVersionUID = -3392881821584833819L;
public static class PushNotificationException extends RuntimeException {
private static final long serialVersionUID = -3392881821584833819L;
public PushNotificationException(String message) {
super(message);
}
public PushNotificationException(String message) {
super(message);
}
public PushNotificationException(String message, Throwable cause) {
super(message, cause);
}
}
public PushNotificationException(String message, Throwable cause) {
super(message, cause);
}
}
}

241
commafeed-server/src/main/java/com/commafeed/backend/service/UserService.java
View File

@@ -1,16 +1,5 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConstants;
import com.commafeed.backend.Digests;
@@ -24,138 +13,152 @@ import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.internal.PostLoginActivities;
import com.google.common.base.Preconditions;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
@RequiredArgsConstructor
@Singleton
public class UserService {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UserDAO userDAO;
private final UserRoleDAO userRoleDAO;
private final UserSettingsDAO userSettingsDAO;
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UserDAO userDAO;
private final UserRoleDAO userRoleDAO;
private final UserSettingsDAO userSettingsDAO;
private final PasswordEncryptionService encryptionService;
private final CommaFeedConfiguration config;
private final PasswordEncryptionService encryptionService;
private final CommaFeedConfiguration config;
private final PostLoginActivities postLoginActivities;
private final PostLoginActivities postLoginActivities;
/**
* try to log in with given credentials
*/
public Optional<User> login(String nameOrEmail, String password) {
if (nameOrEmail == null || password == null) {
return Optional.empty();
}
/** try to log in with given credentials */
public Optional<User> login(String nameOrEmail, String password) {
if (nameOrEmail == null || password == null) {
return Optional.empty();
}
User user = userDAO.findByName(nameOrEmail);
if (user == null) {
user = userDAO.findByEmail(nameOrEmail);
}
if (user != null && !user.isDisabled()) {
boolean authenticated = encryptionService.authenticate(password, user.getPassword(), user.getSalt());
if (authenticated) {
performPostLoginActivities(user);
return Optional.of(user);
}
}
return Optional.empty();
}
User user = userDAO.findByName(nameOrEmail);
if (user == null) {
user = userDAO.findByEmail(nameOrEmail);
}
if (user != null && !user.isDisabled()) {
boolean authenticated =
encryptionService.authenticate(password, user.getPassword(), user.getSalt());
if (authenticated) {
performPostLoginActivities(user);
return Optional.of(user);
}
}
return Optional.empty();
}
/**
* try to log in with given api key
*/
public Optional<User> login(String apiKey) {
if (apiKey == null) {
return Optional.empty();
}
/** try to log in with given api key */
public Optional<User> login(String apiKey) {
if (apiKey == null) {
return Optional.empty();
}
User user = userDAO.findByApiKey(apiKey);
if (user != null && !user.isDisabled()) {
performPostLoginActivities(user);
return Optional.of(user);
}
return Optional.empty();
}
User user = userDAO.findByApiKey(apiKey);
if (user != null && !user.isDisabled()) {
performPostLoginActivities(user);
return Optional.of(user);
}
return Optional.empty();
}
/**
* try to log in with given fever api key
*/
public Optional<User> login(long userId, String feverApiKey) {
if (feverApiKey == null) {
return Optional.empty();
}
/** try to log in with given fever api key */
public Optional<User> login(long userId, String feverApiKey) {
if (feverApiKey == null) {
return Optional.empty();
}
User user = userDAO.findById(userId);
if (user == null || user.isDisabled() || user.getApiKey() == null) {
return Optional.empty();
}
User user = userDAO.findById(userId);
if (user == null || user.isDisabled() || user.getApiKey() == null) {
return Optional.empty();
}
String computedFeverApiKey = Digests.md5Hex(user.getName() + ":" + user.getApiKey());
if (!computedFeverApiKey.equalsIgnoreCase(feverApiKey)) {
return Optional.empty();
}
String computedFeverApiKey = Digests.md5Hex(user.getName() + ":" + user.getApiKey());
if (!computedFeverApiKey.equalsIgnoreCase(feverApiKey)) {
return Optional.empty();
}
performPostLoginActivities(user);
return Optional.of(user);
}
performPostLoginActivities(user);
return Optional.of(user);
}
/**
* should triggers after successful login
*/
public void performPostLoginActivities(User user) {
postLoginActivities.executeFor(user);
}
/** should triggers after successful login */
public void performPostLoginActivities(User user) {
postLoginActivities.executeFor(user);
}
public User register(String name, String password, String email, Collection<Role> roles) {
return register(name, password, email, roles, false);
}
public User register(String name, String password, String email, Collection<Role> roles) {
return register(name, password, email, roles, false);
}
public User register(String name, String password, String email, Collection<Role> roles, boolean forceRegistration) {
public User register(
String name,
String password,
String email,
Collection<Role> roles,
boolean forceRegistration) {
if (!forceRegistration) {
Preconditions.checkState(config.users().allowRegistrations(), "Registrations are closed on this CommaFeed instance");
}
if (!forceRegistration) {
Preconditions.checkState(
config.users().allowRegistrations(),
"Registrations are closed on this CommaFeed instance");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");
if (StringUtils.isNotBlank(email)) {
Preconditions.checkArgument(userDAO.findByEmail(email) == null, "Email already taken");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");
if (StringUtils.isNotBlank(email)) {
Preconditions.checkArgument(userDAO.findByEmail(email) == null, "Email already taken");
}
User user = new User();
byte[] salt = encryptionService.generateSalt();
user.setName(name);
user.setEmail(email);
user.setCreated(Instant.now());
user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.persist(user);
for (Role role : roles) {
userRoleDAO.persist(new UserRole(user, role));
}
return user;
}
User user = new User();
byte[] salt = encryptionService.generateSalt();
user.setName(name);
user.setEmail(email);
user.setCreated(Instant.now());
user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.persist(user);
for (Role role : roles) {
userRoleDAO.persist(new UserRole(user, role));
}
return user;
}
public void createDemoUser() {
register(CommaFeedConstants.USERNAME_DEMO, "demo", "demo@commafeed.com", Collections.singletonList(Role.USER), true);
}
public void createDemoUser() {
register(
CommaFeedConstants.USERNAME_DEMO,
"demo",
"demo@commafeed.com",
Collections.singletonList(Role.USER),
true);
}
public void unregister(User user) {
userSettingsDAO.delete(userSettingsDAO.findByUser(user));
userRoleDAO.delete(userRoleDAO.findAll(user));
feedSubscriptionDAO.delete(feedSubscriptionDAO.findAll(user));
feedCategoryDAO.delete(feedCategoryDAO.findAll(user));
userDAO.delete(user);
}
public void unregister(User user) {
userSettingsDAO.delete(userSettingsDAO.findByUser(user));
userRoleDAO.delete(userRoleDAO.findAll(user));
feedSubscriptionDAO.delete(feedSubscriptionDAO.findAll(user));
feedCategoryDAO.delete(feedCategoryDAO.findAll(user));
userDAO.delete(user);
}
public String generateApiKey(User user) {
byte[] key = encryptionService.getEncryptedPassword(UUID.randomUUID().toString(), user.getSalt());
return Digests.sha1Hex(key);
}
public String generateApiKey(User user) {
byte[] key =
encryptionService.getEncryptedPassword(
UUID.randomUUID().toString(), user.getSalt());
return Digests.sha1Hex(key);
}
public Set<Role> getRoles(User user) {
return userRoleDAO.findRoles(user);
}
public Set<Role> getRoles(User user) {
return userRoleDAO.findRoles(user);
}
}

328
commafeed-server/src/main/java/com/commafeed/backend/service/db/DatabaseCleaningService.java
View File

@@ -1,10 +1,5 @@
package com.commafeed.backend.service.db;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
@@ -16,173 +11,202 @@ import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
/**
* Contains utility methods for cleaning the database
*
*/
/** Contains utility methods for cleaning the database */
@Slf4j
@Singleton
public class DatabaseCleaningService {
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final int batchSize;
private final boolean keepStarredEntries;
private final Meter entriesDeletedMeter;
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final int batchSize;
private final boolean keepStarredEntries;
private final Meter entriesDeletedMeter;
public DatabaseCleaningService(CommaFeedConfiguration config, UnitOfWork unitOfWork, FeedDAO feedDAO, FeedEntryDAO feedEntryDAO,
FeedEntryContentDAO feedEntryContentDAO, FeedEntryStatusDAO feedEntryStatusDAO, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.feedEntryDAO = feedEntryDAO;
this.feedEntryContentDAO = feedEntryContentDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.batchSize = config.database().cleanup().batchSize();
this.keepStarredEntries = config.database().cleanup().keepStarredEntries();
this.entriesDeletedMeter = metrics.meter(MetricRegistry.name(getClass(), "entriesDeleted"));
}
public DatabaseCleaningService(
CommaFeedConfiguration config,
UnitOfWork unitOfWork,
FeedDAO feedDAO,
FeedEntryDAO feedEntryDAO,
FeedEntryContentDAO feedEntryContentDAO,
FeedEntryStatusDAO feedEntryStatusDAO,
MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.feedEntryDAO = feedEntryDAO;
this.feedEntryContentDAO = feedEntryContentDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.batchSize = config.database().cleanup().batchSize();
this.keepStarredEntries = config.database().cleanup().keepStarredEntries();
this.entriesDeletedMeter = metrics.meter(MetricRegistry.name(getClass(), "entriesDeleted"));
}
public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
int deleted;
long entriesTotal = 0;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of feeds without subscriptions");
return;
}
public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
int deleted;
long entriesTotal = 0;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of feeds without subscriptions");
return;
}
List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) {
long entriesDeleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of feeds without subscriptions");
return;
}
List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) {
long entriesDeleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of feeds without subscriptions");
return;
}
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesDeletedMeter.mark(entriesDeleted);
entriesTotal += entriesDeleted;
log.debug("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
}
deleted = unitOfWork.call(() -> feedDAO.delete(feedDAO.findByIds(feeds.stream().map(AbstractModel::getId).toList())));
total += deleted;
log.debug("removed {} feeds without subscriptions", total);
} while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total);
}
entriesDeleted =
unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesDeletedMeter.mark(entriesDeleted);
entriesTotal += entriesDeleted;
log.debug("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
}
deleted =
unitOfWork.call(
() ->
feedDAO.delete(
feedDAO.findByIds(
feeds.stream()
.map(AbstractModel::getId)
.toList())));
total += deleted;
log.debug("removed {} feeds without subscriptions", total);
} while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total);
}
public void cleanContentsWithoutEntries() {
log.info("cleaning contents without entries");
long total = 0;
long deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of contents without entries");
return;
}
public void cleanContentsWithoutEntries() {
log.info("cleaning contents without entries");
long total = 0;
long deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of contents without entries");
return;
}
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize));
total += deleted;
log.debug("removed {} contents without entries", total);
} while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total);
}
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize));
total += deleted;
log.debug("removed {} contents without entries", total);
} while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total);
}
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
log.info("cleaning entries exceeding feed capacity");
long total = 0;
while (true) {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of entries exceeding feed capacity");
return;
}
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
log.info("cleaning entries exceeding feed capacity");
long total = 0;
while (true) {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of entries exceeding feed capacity");
return;
}
List<FeedCapacity> feeds = unitOfWork
.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize, keepStarredEntries));
if (feeds.isEmpty()) {
break;
}
List<FeedCapacity> feeds =
unitOfWork.call(
() ->
feedEntryDAO.findFeedsExceedingCapacity(
maxFeedCapacity, batchSize, keepStarredEntries));
if (feeds.isEmpty()) {
break;
}
for (final FeedCapacity feed : feeds) {
long remaining = feed.capacity() - maxFeedCapacity;
int deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of entries exceeding feed capacity");
return;
}
for (final FeedCapacity feed : feeds) {
long remaining = feed.capacity() - maxFeedCapacity;
int deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info(
"interrupted, stopping cleanup of entries exceeding feed capacity");
return;
}
final long rem = remaining;
deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.id(), Math.min(batchSize, rem), keepStarredEntries));
entriesDeletedMeter.mark(deleted);
total += deleted;
remaining -= deleted;
log.debug("removed {} entries for feeds exceeding capacity", total);
} while (deleted > 0 && remaining > 0);
}
}
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
}
final long rem = remaining;
deleted =
unitOfWork.call(
() ->
feedEntryDAO.deleteOldEntries(
feed.id(),
Math.min(batchSize, rem),
keepStarredEntries));
entriesDeletedMeter.mark(deleted);
total += deleted;
remaining -= deleted;
log.debug("removed {} entries for feeds exceeding capacity", total);
} while (deleted > 0 && remaining > 0);
}
}
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
}
public void cleanEntriesOlderThan(final Instant olderThan) {
log.info("cleaning old entries");
long total = 0;
long deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of old entries");
return;
}
public void cleanEntriesOlderThan(final Instant olderThan) {
log.info("cleaning old entries");
long total = 0;
long deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of old entries");
return;
}
deleted = unitOfWork.call(() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize, keepStarredEntries));
entriesDeletedMeter.mark(deleted);
total += deleted;
log.debug("removed {} old entries", total);
} while (deleted != 0);
log.info("cleanup done: {} old entries deleted", total);
}
deleted =
unitOfWork.call(
() ->
feedEntryDAO.deleteEntriesOlderThan(
olderThan, batchSize, keepStarredEntries));
entriesDeletedMeter.mark(deleted);
total += deleted;
log.debug("removed {} old entries", total);
} while (deleted != 0);
log.info("cleanup done: {} old entries deleted", total);
}
public void cleanStatusesOlderThan(final Instant olderThan) {
log.info("cleaning old read statuses");
long total = 0;
long deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of old read statuses");
return;
}
public void cleanStatusesOlderThan(final Instant olderThan) {
log.info("cleaning old read statuses");
long total = 0;
long deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping cleanup of old read statuses");
return;
}
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
total += deleted;
log.debug("removed {} old read statuses", total);
} while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
}
deleted =
unitOfWork.call(
() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
total += deleted;
log.debug("removed {} old read statuses", total);
} while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
}
public void autoMarkAsRead() {
log.info("marking entries as read based on autoMarkAsReadAfterDays");
long total = 0;
long marked;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping marking entries as read");
return;
}
public void autoMarkAsRead() {
log.info("marking entries as read based on autoMarkAsReadAfterDays");
long total = 0;
long marked;
do {
if (Thread.currentThread().isInterrupted()) {
log.info("interrupted, stopping marking entries as read");
return;
}
marked = unitOfWork.call(() -> feedEntryStatusDAO.autoMarkAsRead(batchSize));
total += marked;
log.debug("marked {} entries as read", total);
} while (marked != 0);
log.info("cleanup done: marked {} entries as read", total);
}
marked = unitOfWork.call(() -> feedEntryStatusDAO.autoMarkAsRead(batchSize));
total += marked;
log.debug("marked {} entries as read", total);
} while (marked != 0);
log.info("cleanup done: marked {} entries as read", total);
}
}

48
commafeed-server/src/main/java/com/commafeed/backend/service/db/DatabaseStartupService.java
View File

@@ -1,44 +1,42 @@
package com.commafeed.backend.service.db;
import jakarta.inject.Singleton;
import org.kohsuke.MetaInfServices;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import jakarta.inject.Singleton;
import liquibase.database.Database;
import liquibase.database.core.PostgresDatabase;
import liquibase.structure.DatabaseObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.kohsuke.MetaInfServices;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class DatabaseStartupService {
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
public boolean isInitialSetupRequired() {
return unitOfWork.call(userDAO::count) == 0;
}
public boolean isInitialSetupRequired() {
return unitOfWork.call(userDAO::count) == 0;
}
/**
* Register a postgresql database in liquibase that doesn't escape columns, so that we can use lower case columns
*/
@MetaInfServices(Database.class)
public static class LowerCaseColumnsPostgresDatabase extends PostgresDatabase {
@Override
public String escapeObjectName(String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}
@Override
public int getPriority() {
return super.getPriority() + 1;
}
}
/**
* Register a postgresql database in liquibase that doesn't escape columns, so that we can use
* lower case columns
*/
@MetaInfServices(Database.class)
public static class LowerCaseColumnsPostgresDatabase extends PostgresDatabase {
@Override
public String escapeObjectName(
String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}
@Override
public int getPriority() {
return super.getPriority() + 1;
}
}
}

33
commafeed-server/src/main/java/com/commafeed/backend/service/internal/PostLoginActivities.java
View File

@@ -1,30 +1,29 @@
package com.commafeed.backend.service.internal;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class PostLoginActivities {
private final UserDAO userDAO;
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UnitOfWork unitOfWork;
public void executeFor(User user) {
// only update lastLogin every once in a while in order to avoid invalidating the cache every time someone logs in
Instant now = Instant.now();
Instant lastLogin = user.getLastLogin();
if (lastLogin == null || ChronoUnit.MINUTES.between(lastLogin, now) >= 30) {
user.setLastLogin(now);
unitOfWork.run(() -> userDAO.merge(user));
}
}
public void executeFor(User user) {
// only update lastLogin every once in a while in order to avoid invalidating the cache
// every
// time someone logs in
Instant now = Instant.now();
Instant lastLogin = user.getLastLogin();
if (lastLogin == null || ChronoUnit.MINUTES.between(lastLogin, now) >= 30) {
user.setLastLogin(now);
unitOfWork.run(() -> userDAO.merge(user));
}
}
}

42
commafeed-server/src/main/java/com/commafeed/backend/task/AutoMarkAsReadTask.java
View File

@@ -1,37 +1,33 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class AutoMarkAsReadTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.autoMarkAsRead();
}
@Override
public void run() {
cleaner.autoMarkAsRead();
}
@Override
public long getInitialDelay() {
return 30;
}
@Override
public long getInitialDelay() {
return 30;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

72
commafeed-server/src/main/java/com/commafeed/backend/task/DemoAccountCleanupTask.java
View File

@@ -1,16 +1,13 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConstants;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.UserService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -19,43 +16,42 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class DemoAccountCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserService userService;
private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserService userService;
@Override
protected void run() {
if (!config.users().createDemoAccount()) {
return;
}
@Override
protected void run() {
if (!config.users().createDemoAccount()) {
return;
}
log.info("recreating demo user account");
unitOfWork.run(() -> {
User demoUser = userDAO.findByName(CommaFeedConstants.USERNAME_DEMO);
if (demoUser == null) {
return;
}
log.info("recreating demo user account");
unitOfWork.run(
() -> {
User demoUser = userDAO.findByName(CommaFeedConstants.USERNAME_DEMO);
if (demoUser == null) {
return;
}
userService.unregister(demoUser);
userService.createDemoUser();
});
userService.unregister(demoUser);
userService.createDemoUser();
});
}
}
@Override
protected long getInitialDelay() {
return 1;
}
@Override
protected long getInitialDelay() {
return 1;
}
@Override
protected long getPeriod() {
return getTimeUnit().convert(24, TimeUnit.HOURS);
}
@Override
protected TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
protected long getPeriod() {
return getTimeUnit().convert(24, TimeUnit.HOURS);
}
@Override
protected TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

50
commafeed-server/src/main/java/com/commafeed/backend/task/EntriesExceedingFeedCapacityCleanupTask.java
View File

@@ -1,42 +1,38 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
}
}
@Override
public void run() {
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
}
}
@Override
public long getInitialDelay() {
return 10;
}
@Override
public long getInitialDelay() {
return 10;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

56
commafeed-server/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java
View File

@@ -1,45 +1,41 @@
package com.commafeed.backend.task;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OldEntriesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
cleaner.cleanEntriesOlderThan(threshold);
}
}
@Override
public void run() {
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
cleaner.cleanEntriesOlderThan(threshold);
}
}
@Override
public long getInitialDelay() {
return 5;
}
@Override
public long getInitialDelay() {
return 5;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

52
commafeed-server/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java
View File

@@ -1,43 +1,39 @@
package com.commafeed.backend.task;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OldStatusesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
Instant threshold = config.database().cleanup().statusesInstantThreshold();
if (threshold != null) {
cleaner.cleanStatusesOlderThan(threshold);
}
}
@Override
public void run() {
Instant threshold = config.database().cleanup().statusesInstantThreshold();
if (threshold != null) {
cleaner.cleanStatusesOlderThan(threshold);
}
}
@Override
public long getInitialDelay() {
return 15;
}
@Override
public long getInitialDelay() {
return 15;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

42
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedContentsCleanupTask.java
View File

@@ -1,37 +1,33 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OrphanedContentsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanContentsWithoutEntries();
}
@Override
public void run() {
cleaner.cleanContentsWithoutEntries();
}
@Override
public long getInitialDelay() {
return 25;
}
@Override
public long getInitialDelay() {
return 25;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

42
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedFeedsCleanupTask.java
View File

@@ -1,37 +1,33 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import jakarta.inject.Singleton;
import java.util.concurrent.TimeUnit;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OrphanedFeedsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanFeedsWithoutSubscriptions();
}
@Override
public void run() {
cleaner.cleanFeedsWithoutSubscriptions();
}
@Override
public long getInitialDelay() {
return 20;
}
@Override
public long getInitialDelay() {
return 20;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

39
commafeed-server/src/main/java/com/commafeed/backend/task/ScheduledTask.java
View File

@@ -2,29 +2,34 @@ package com.commafeed.backend.task;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class ScheduledTask {
protected abstract void run();
protected abstract void run();
protected abstract long getInitialDelay();
protected abstract long getInitialDelay();
protected abstract long getPeriod();
protected abstract long getPeriod();
protected abstract TimeUnit getTimeUnit();
protected abstract TimeUnit getTimeUnit();
public void register(ScheduledExecutorService executor) {
Runnable runnable = () -> {
try {
ScheduledTask.this.run();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
};
log.debug("registering task {} for execution every {} {}, starting in {} {}", getClass().getSimpleName(), getPeriod(),
getTimeUnit(), getInitialDelay(), getTimeUnit());
executor.scheduleWithFixedDelay(runnable, getInitialDelay(), getPeriod(), getTimeUnit());
}
public void register(ScheduledExecutorService executor) {
Runnable runnable =
() -> {
try {
ScheduledTask.this.run();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
};
log.debug(
"registering task {} for execution every {} {}, starting in {} {}",
getClass().getSimpleName(),
getPeriod(),
getTimeUnit(),
getInitialDelay(),
getTimeUnit());
executor.scheduleWithFixedDelay(runnable, getInitialDelay(), getPeriod(), getTimeUnit());
}
}

39
commafeed-server/src/main/java/com/commafeed/backend/task/TaskScheduler.java
View File

@@ -1,37 +1,34 @@
package com.commafeed.backend.task;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.google.common.util.concurrent.MoreExecutors;
import io.quarkus.arc.All;
import jakarta.inject.Singleton;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class TaskScheduler {
private final List<ScheduledTask> tasks;
private final CommaFeedConfiguration config;
private final List<ScheduledTask> tasks;
private final CommaFeedConfiguration config;
private ScheduledExecutorService executor;
private ScheduledExecutorService executor;
public TaskScheduler(@All List<ScheduledTask> tasks, CommaFeedConfiguration config) {
this.tasks = tasks;
this.config = config;
}
public TaskScheduler(@All List<ScheduledTask> tasks, CommaFeedConfiguration config) {
this.tasks = tasks;
this.config = config;
}
public void start() {
this.executor = Executors.newScheduledThreadPool(tasks.size());
this.tasks.forEach(task -> task.register(executor));
}
public void start() {
this.executor = Executors.newScheduledThreadPool(tasks.size());
this.tasks.forEach(task -> task.register(executor));
}
public void stop() {
MoreExecutors.shutdownAndAwaitTermination(executor, config.shutdownTimeout());
}
public void stop() {
MoreExecutors.shutdownAndAwaitTermination(executor, config.shutdownTimeout());
}
}

7
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/FeedURLProvider.java
View File

@@ -2,11 +2,8 @@ package com.commafeed.backend.urlprovider;
import java.util.List;
/**
* Tries to find a feed url given the url and page content
*/
/** Tries to find a feed url given the url and page content */
public interface FeedURLProvider {
List<String> get(String url, String urlContent);
List<String> get(String url, String urlContent);
}

27
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/InPageReferenceFeedURLProvider.java
View File

@@ -1,25 +1,24 @@
package com.commafeed.backend.urlprovider;
import jakarta.inject.Singleton;
import java.util.List;
import java.util.stream.Stream;
import jakarta.inject.Singleton;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
@Singleton
public class InPageReferenceFeedURLProvider implements FeedURLProvider {
@Override
public List<String> get(String url, String urlContent) {
Document doc = Jsoup.parse(urlContent, url);
if (!"html".equals(doc.children().getFirst().tagName())) {
return List.of();
}
return Stream.concat(doc.select("link[type=application/atom+xml]").stream(), doc.select("link[type=application/rss+xml]").stream())
.map(node -> node.attr("abs:href"))
.toList();
}
@Override
public List<String> get(String url, String urlContent) {
Document doc = Jsoup.parse(urlContent, url);
if (!"html".equals(doc.children().getFirst().tagName())) {
return List.of();
}
return Stream.concat(
doc.select("link[type=application/atom+xml]").stream(),
doc.select("link[type=application/rss+xml]").stream())
.map(node -> node.attr("abs:href"))
.toList();
}
}

28
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/YoutubeFeedURLProvider.java
View File

@@ -1,30 +1,28 @@
package com.commafeed.backend.urlprovider;
import java.util.List;
import jakarta.inject.Singleton;
import java.util.List;
import org.apache.commons.lang3.Strings;
/**
* Workaround for Youtube channels
*
* converts the channel URL https://www.youtube.com/channel/CHANNEL_ID to the valid feed URL
*
* <p>converts the channel URL https://www.youtube.com/channel/CHANNEL_ID to the valid feed URL
* https://www.youtube.com/feeds/videos.xml?channel_id=CHANNEL_ID
*/
@Singleton
public class YoutubeFeedURLProvider implements FeedURLProvider {
private static final String PREFIX = "https://www.youtube.com/channel/";
private static final String REPLACEMENT_PREFIX = "https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String PREFIX = "https://www.youtube.com/channel/";
private static final String REPLACEMENT_PREFIX =
"https://www.youtube.com/feeds/videos.xml?channel_id=";
@Override
public List<String> get(String url, String urlContent) {
if (!Strings.CI.startsWith(url, PREFIX)) {
return List.of();
}
return List.of(REPLACEMENT_PREFIX + url.substring(PREFIX.length()));
}
@Override
public List<String> get(String url, String urlContent) {
if (!Strings.CI.startsWith(url, PREFIX)) {
return List.of();
}
return List.of(REPLACEMENT_PREFIX + url.substring(PREFIX.length()));
}
}

40
commafeed-server/src/main/java/com/commafeed/frontend/model/Category.java
View File

@@ -1,13 +1,11 @@
package com.commafeed.frontend.model;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import io.quarkus.runtime.annotations.RegisterForReflection;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "Entry details")
@@ -15,27 +13,27 @@ import lombok.Data;
@RegisterForReflection
public class Category implements Serializable {
@Schema(description = "category id", required = true)
private String id;
@Schema(description = "category id", required = true)
private String id;
@Schema(description = "parent category id")
private String parentId;
@Schema(description = "parent category id")
private String parentId;
@Schema(description = "parent category name")
private String parentName;
@Schema(description = "parent category name")
private String parentName;
@Schema(description = "category id", required = true)
private String name;
@Schema(description = "category id", required = true)
private String name;
@Schema(description = "category children categories", required = true)
private List<Category> children = new ArrayList<>();
@Schema(description = "category children categories", required = true)
private List<Category> children = new ArrayList<>();
@Schema(description = "category feeds", required = true)
private List<Subscription> feeds = new ArrayList<>();
@Schema(description = "category feeds", required = true)
private List<Subscription> feeds = new ArrayList<>();
@Schema(description = "whether the category is expanded or collapsed", required = true)
private boolean expanded;
@Schema(description = "whether the category is expanded or collapsed", required = true)
private boolean expanded;
@Schema(description = "position of the category in the list", required = true)
private int position;
}
@Schema(description = "position of the category in the list", required = true)
private int position;
}

54
commafeed-server/src/main/java/com/commafeed/frontend/model/Entries.java
View File

@@ -1,13 +1,11 @@
package com.commafeed.frontend.model;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import io.quarkus.runtime.annotations.RegisterForReflection;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "List of entries with some metadata")
@@ -15,36 +13,38 @@ import lombok.Data;
@RegisterForReflection
public class Entries implements Serializable {
@Schema(description = "name of the feed or the category requested", required = true)
private String name;
@Schema(description = "name of the feed or the category requested", required = true)
private String name;
@Schema(description = "error or warning message")
private String message;
@Schema(description = "error or warning message")
private String message;
@Schema(description = "times the server tried to refresh the feed and failed", required = true)
private int errorCount;
@Schema(description = "times the server tried to refresh the feed and failed", required = true)
private int errorCount;
@Schema(description = "URL of the website, extracted from the feed, only filled if querying for feed entries, not category entries")
private String feedLink;
@Schema(
description =
"URL of the website, extracted from the feed, only filled if querying for feed entries, not category entries")
private String feedLink;
@Schema(description = "list generation timestamp", required = true)
private long timestamp;
@Schema(description = "list generation timestamp", required = true)
private long timestamp;
@Schema(description = "if the query has more elements", required = true)
private boolean hasMore;
@Schema(description = "if the query has more elements", required = true)
private boolean hasMore;
@Schema(description = "the requested offset")
private int offset;
@Schema(description = "the requested offset")
private int offset;
@Schema(description = "the requested limit")
private int limit;
@Schema(description = "the requested limit")
private int limit;
@Schema(description = "list of entries", required = true)
private List<Entry> entries = new ArrayList<>();
@Schema(
description = "if true, the unread flag was ignored in the request, all entries are returned regardless of their read status",
required = true)
private boolean ignoredReadStatus;
@Schema(description = "list of entries", required = true)
private List<Entry> entries = new ArrayList<>();
@Schema(
description =
"if true, the unread flag was ignored in the request, all entries are returned regardless of their read status",
required = true)
private boolean ignoredReadStatus;
}

200
commafeed-server/src/main/java/com/commafeed/frontend/model/Entry.java
View File

@@ -1,22 +1,19 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import java.util.List;
import org.apache.commons.lang3.Strings;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import java.time.Instant;
import java.util.List;
import lombok.Data;
import org.apache.commons.lang3.Strings;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "Entry details")
@@ -24,123 +21,134 @@ import lombok.Data;
@RegisterForReflection
public class Entry implements Serializable {
@Schema(description = "entry id", required = true)
private String id;
@Schema(description = "entry id", required = true)
private String id;
@Schema(description = "entry guid", required = true)
private String guid;
@Schema(description = "entry guid", required = true)
private String guid;
@Schema(description = "entry title", required = true)
private String title;
@Schema(description = "entry title", required = true)
private String title;
@Schema(description = "entry content", required = true)
private String content;
@Schema(description = "entry content", required = true)
private String content;
@Schema(description = "comma-separated list of categories")
private String categories;
@Schema(description = "comma-separated list of categories")
private String categories;
@Schema(description = "whether entry content and title are rtl", required = true)
private boolean rtl;
@Schema(description = "whether entry content and title are rtl", required = true)
private boolean rtl;
@Schema(description = "entry author")
private String author;
@Schema(description = "entry author")
private String author;
@Schema(description = "entry enclosure url, if any")
private String enclosureUrl;
@Schema(description = "entry enclosure url, if any")
private String enclosureUrl;
@Schema(description = "entry enclosure mime type, if any")
private String enclosureType;
@Schema(description = "entry enclosure mime type, if any")
private String enclosureType;
@Schema(description = "entry media description, if any")
private String mediaDescription;
@Schema(description = "entry media description, if any")
private String mediaDescription;
@Schema(description = "entry media thumbnail url, if any")
private String mediaThumbnailUrl;
@Schema(description = "entry media thumbnail url, if any")
private String mediaThumbnailUrl;
@Schema(description = "entry media thumbnail width, if any")
private Integer mediaThumbnailWidth;
@Schema(description = "entry media thumbnail width, if any")
private Integer mediaThumbnailWidth;
@Schema(description = "entry media thumbnail height, if any")
private Integer mediaThumbnailHeight;
@Schema(description = "entry media thumbnail height, if any")
private Integer mediaThumbnailHeight;
@Schema(description = "entry publication date", type = SchemaType.INTEGER, required = true)
private Instant date;
@Schema(description = "entry publication date", type = SchemaType.INTEGER, required = true)
private Instant date;
@Schema(description = "entry insertion date in the database", type = SchemaType.INTEGER, required = true)
private Instant insertedDate;
@Schema(
description = "entry insertion date in the database",
type = SchemaType.INTEGER,
required = true)
private Instant insertedDate;
@Schema(description = "feed id", required = true)
private String feedId;
@Schema(description = "feed id", required = true)
private String feedId;
@Schema(description = "feed name", required = true)
private String feedName;
@Schema(description = "feed name", required = true)
private String feedName;
@Schema(description = "this entry's feed url", required = true)
private String feedUrl;
@Schema(description = "this entry's feed url", required = true)
private String feedUrl;
@Schema(description = "this entry's website url", required = true)
private String feedLink;
@Schema(description = "this entry's website url", required = true)
private String feedLink;
@Schema(description = "The favicon url to use for this feed", required = true)
private String iconUrl;
@Schema(description = "The favicon url to use for this feed", required = true)
private String iconUrl;
@Schema(description = "entry url", required = true)
private String url;
@Schema(description = "entry url", required = true)
private String url;
@Schema(description = "read status", required = true)
private boolean read;
@Schema(description = "read status", required = true)
private boolean read;
@Schema(description = "starred status", required = true)
private boolean starred;
@Schema(description = "starred status", required = true)
private boolean starred;
@Schema(description = "whether the entry is still markable (old entry statuses are discarded)", required = true)
private boolean markable;
@Schema(
description = "whether the entry is still markable (old entry statuses are discarded)",
required = true)
private boolean markable;
@Schema(description = "tags", required = true)
private List<String> tags;
@Schema(description = "tags", required = true)
private List<String> tags;
public static Entry build(FeedEntryStatus status, boolean proxyImages) {
Entry entry = new Entry();
public static Entry build(FeedEntryStatus status, boolean proxyImages) {
Entry entry = new Entry();
FeedEntry feedEntry = status.getEntry();
FeedSubscription sub = status.getSubscription();
FeedEntryContent content = feedEntry.getContent();
FeedEntry feedEntry = status.getEntry();
FeedSubscription sub = status.getSubscription();
FeedEntryContent content = feedEntry.getContent();
entry.setId(String.valueOf(feedEntry.getId()));
entry.setGuid(feedEntry.getGuid());
entry.setRead(status.isRead());
entry.setStarred(status.isStarred());
entry.setMarkable(status.isMarkable());
entry.setDate(feedEntry.getPublished());
entry.setInsertedDate(feedEntry.getInserted());
entry.setUrl(feedEntry.getUrl());
entry.setFeedName(sub.getTitle());
entry.setFeedId(String.valueOf(sub.getId()));
entry.setFeedUrl(sub.getFeed().getUrl());
entry.setFeedLink(sub.getFeed().getLink());
entry.setIconUrl(FeedUtils.getFaviconUrl(sub));
entry.setTags(status.getTags().stream().map(FeedEntryTag::getName).toList());
entry.setId(String.valueOf(feedEntry.getId()));
entry.setGuid(feedEntry.getGuid());
entry.setRead(status.isRead());
entry.setStarred(status.isStarred());
entry.setMarkable(status.isMarkable());
entry.setDate(feedEntry.getPublished());
entry.setInsertedDate(feedEntry.getInserted());
entry.setUrl(feedEntry.getUrl());
entry.setFeedName(sub.getTitle());
entry.setFeedId(String.valueOf(sub.getId()));
entry.setFeedUrl(sub.getFeed().getUrl());
entry.setFeedLink(sub.getFeed().getLink());
entry.setIconUrl(FeedUtils.getFaviconUrl(sub));
entry.setTags(status.getTags().stream().map(FeedEntryTag::getName).toList());
if (content != null) {
entry.setRtl(content.getDirection() == FeedEntryContent.Direction.RTL);
entry.setTitle(content.getTitle());
entry.setContent(proxyImages ? FeedUtils.proxyImages(content.getContent()) : content.getContent());
entry.setAuthor(content.getAuthor());
if (content != null) {
entry.setRtl(content.getDirection() == FeedEntryContent.Direction.RTL);
entry.setTitle(content.getTitle());
entry.setContent(
proxyImages
? FeedUtils.proxyImages(content.getContent())
: content.getContent());
entry.setAuthor(content.getAuthor());
entry.setEnclosureType(content.getEnclosureType());
entry.setEnclosureUrl(proxyImages && Strings.CS.contains(content.getEnclosureType(), "image")
? FeedUtils.proxyImage(content.getEnclosureUrl())
: content.getEnclosureUrl());
entry.setEnclosureType(content.getEnclosureType());
entry.setEnclosureUrl(
proxyImages && Strings.CS.contains(content.getEnclosureType(), "image")
? FeedUtils.proxyImage(content.getEnclosureUrl())
: content.getEnclosureUrl());
entry.setMediaDescription(content.getMediaDescription());
entry.setMediaThumbnailUrl(proxyImages ? FeedUtils.proxyImage(content.getMediaThumbnailUrl()) : content.getMediaThumbnailUrl());
entry.setMediaThumbnailWidth(content.getMediaThumbnailWidth());
entry.setMediaThumbnailHeight(content.getMediaThumbnailHeight());
entry.setMediaDescription(content.getMediaDescription());
entry.setMediaThumbnailUrl(
proxyImages
? FeedUtils.proxyImage(content.getMediaThumbnailUrl())
: content.getMediaThumbnailUrl());
entry.setMediaThumbnailWidth(content.getMediaThumbnailWidth());
entry.setMediaThumbnailHeight(content.getMediaThumbnailHeight());
entry.setCategories(content.getCategories());
}
return entry;
}
entry.setCategories(content.getCategories());
}
return entry;
}
}

15
commafeed-server/src/main/java/com/commafeed/frontend/model/FeedInfo.java
View File

@@ -1,11 +1,9 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "Feed details")
@@ -13,10 +11,9 @@ import lombok.Data;
@RegisterForReflection
public class FeedInfo implements Serializable {
@Schema(description = "url", required = true)
private String url;
@Schema(description = "title", required = true)
private String title;
@Schema(description = "url", required = true)
private String url;
@Schema(description = "title", required = true)
private String title;
}

62
commafeed-server/src/main/java/com/commafeed/frontend/model/ServerInfo.java
View File

@@ -1,11 +1,9 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "Server infos")
@@ -13,46 +11,44 @@ import lombok.Data;
@RegisterForReflection
public class ServerInfo implements Serializable {
@Schema
private String announcement;
@Schema private String announcement;
@Schema(required = true)
private String version;
@Schema(required = true)
private String version;
@Schema(required = true)
private String gitCommit;
@Schema(required = true)
private String gitCommit;
@Schema(required = true)
private boolean allowRegistrations;
@Schema(required = true)
private boolean allowRegistrations;
@Schema(required = true)
private boolean emailAddressRequired;
@Schema(required = true)
private boolean emailAddressRequired;
@Schema(required = true)
private boolean smtpEnabled;
@Schema(required = true)
private boolean smtpEnabled;
@Schema(required = true)
private boolean demoAccountEnabled;
@Schema(required = true)
private boolean demoAccountEnabled;
@Schema(required = true)
private boolean websocketEnabled;
@Schema(required = true)
private boolean websocketEnabled;
@Schema(required = true)
private long websocketPingInterval;
@Schema(required = true)
private long websocketPingInterval;
@Schema(required = true)
private long treeReloadInterval;
@Schema(required = true)
private long treeReloadInterval;
@Schema(required = true)
private long forceRefreshCooldownDuration;
@Schema(required = true)
private long forceRefreshCooldownDuration;
@Schema(required = true)
private boolean initialSetupRequired;
@Schema(required = true)
private boolean initialSetupRequired;
@Schema(required = true)
private int minimumPasswordLength;
@Schema(required = true)
private boolean pushNotificationsEnabled;
@Schema(required = true)
private int minimumPasswordLength;
@Schema(required = true)
private boolean pushNotificationsEnabled;
}

174
commafeed-server/src/main/java/com/commafeed/frontend/model/Settings.java
View File

@@ -1,17 +1,14 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import com.commafeed.backend.model.UserSettings.IconDisplayMode;
import com.commafeed.backend.model.UserSettings.PushNotificationType;
import com.commafeed.backend.model.UserSettings.ReadingMode;
import com.commafeed.backend.model.UserSettings.ReadingOrder;
import com.commafeed.backend.model.UserSettings.ScrollMode;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "User settings")
@@ -19,113 +16,128 @@ import lombok.Data;
@RegisterForReflection
public class Settings implements Serializable {
@Schema(description = "user's preferred language, english if none")
private String language;
@Schema(description = "user's preferred language, english if none")
private String language;
@Schema(description = "user reads all entries or unread entries only", required = true)
private ReadingMode readingMode;
@Schema(description = "user reads all entries or unread entries only", required = true)
private ReadingMode readingMode;
@Schema(description = "user reads entries in ascending or descending order", required = true)
private ReadingOrder readingOrder;
@Schema(description = "user reads entries in ascending or descending order", required = true)
private ReadingOrder readingOrder;
@Schema(description = "user wants category and feeds with no unread entries shown", required = true)
private boolean showRead;
@Schema(
description = "user wants category and feeds with no unread entries shown",
required = true)
private boolean showRead;
@Schema(description = "In expanded view, scroll through entries mark them as read", required = true)
private boolean scrollMarks;
@Schema(
description = "In expanded view, scroll through entries mark them as read",
required = true)
private boolean scrollMarks;
@Schema(description = "user's custom css for the website")
private String customCss;
@Schema(description = "user's custom css for the website")
private String customCss;
@Schema(description = "user's custom js for the website")
private String customJs;
@Schema(description = "user's custom js for the website")
private String customJs;
@Schema(description = "user's preferred scroll speed when navigating between entries", required = true)
private int scrollSpeed;
@Schema(
description = "user's preferred scroll speed when navigating between entries",
required = true)
private int scrollSpeed;
@Schema(description = "whether to scroll to the selected entry", required = true)
private ScrollMode scrollMode;
@Schema(description = "whether to scroll to the selected entry", required = true)
private ScrollMode scrollMode;
@Schema(description = "number of entries to keep above the selected entry when scrolling", required = true)
private int entriesToKeepOnTopWhenScrolling;
@Schema(
description = "number of entries to keep above the selected entry when scrolling",
required = true)
private int entriesToKeepOnTopWhenScrolling;
@Schema(description = "whether to show the star icon in the header of entries", required = true)
private IconDisplayMode starIconDisplayMode;
@Schema(description = "whether to show the star icon in the header of entries", required = true)
private IconDisplayMode starIconDisplayMode;
@Schema(description = "whether to show the external link icon in the header of entries", required = true)
private IconDisplayMode externalLinkIconDisplayMode;
@Schema(
description = "whether to show the external link icon in the header of entries",
required = true)
private IconDisplayMode externalLinkIconDisplayMode;
@Schema(description = "ask for confirmation when marking all entries as read", required = true)
private boolean markAllAsReadConfirmation;
@Schema(description = "ask for confirmation when marking all entries as read", required = true)
private boolean markAllAsReadConfirmation;
@Schema(description = "navigate to the next unread category or feed after marking all entries as read", required = true)
private boolean markAllAsReadNavigateToNextUnread;
@Schema(
description =
"navigate to the next unread category or feed after marking all entries as read",
required = true)
private boolean markAllAsReadNavigateToNextUnread;
@Schema(description = "show commafeed's own context menu on right click", required = true)
private boolean customContextMenu;
@Schema(description = "show commafeed's own context menu on right click", required = true)
private boolean customContextMenu;
@Schema(description = "on mobile, show action buttons at the bottom of the screen", required = true)
private boolean mobileFooter;
@Schema(
description = "on mobile, show action buttons at the bottom of the screen",
required = true)
private boolean mobileFooter;
@Schema(description = "show unread count in the title", required = true)
private boolean unreadCountTitle;
@Schema(description = "show unread count in the title", required = true)
private boolean unreadCountTitle;
@Schema(description = "show unread count in the favicon", required = true)
private boolean unreadCountFavicon;
@Schema(description = "show unread count in the favicon", required = true)
private boolean unreadCountFavicon;
@Schema(description = "disable pull to refresh", required = true)
private boolean disablePullToRefresh;
@Schema(description = "disable pull to refresh", required = true)
private boolean disablePullToRefresh;
@Schema(description = "primary theme color to use in the UI")
private String primaryColor;
@Schema(description = "primary theme color to use in the UI")
private String primaryColor;
@Schema(description = "sharing settings", required = true)
private SharingSettings sharingSettings = new SharingSettings();
@Schema(description = "sharing settings", required = true)
private SharingSettings sharingSettings = new SharingSettings();
@Schema(description = "push notification settings", required = true)
private PushNotificationSettings pushNotificationSettings = new PushNotificationSettings();
@Schema(description = "push notification settings", required = true)
private PushNotificationSettings pushNotificationSettings = new PushNotificationSettings();
@Schema(description = "User notification settings")
@Data
public static class PushNotificationSettings implements Serializable {
@Schema(description = "notification provider type")
private PushNotificationType type;
@Schema(description = "User notification settings")
@Data
public static class PushNotificationSettings implements Serializable {
@Schema(description = "notification provider type")
private PushNotificationType type;
@Schema(description = "server URL for ntfy or gotify")
private String serverUrl;
@Schema(description = "server URL for ntfy or gotify")
private String serverUrl;
@Schema(description = "user Id")
private String userId;
@Schema(description = "user Id")
private String userId;
@Schema(description = "user secret for authentication with the service")
private String userSecret;
@Schema(description = "user secret for authentication with the service")
private String userSecret;
@Schema(description = "topic")
private String topic;
}
@Schema(description = "topic")
private String topic;
}
@Schema(description = "User sharing settings")
@Data
public static class SharingSettings implements Serializable {
@Schema(required = true)
private boolean email;
@Schema(description = "User sharing settings")
@Data
public static class SharingSettings implements Serializable {
@Schema(required = true)
private boolean email;
@Schema(required = true)
private boolean gmail;
@Schema(required = true)
private boolean gmail;
@Schema(required = true)
private boolean facebook;
@Schema(required = true)
private boolean facebook;
@Schema(required = true)
private boolean twitter;
@Schema(required = true)
private boolean twitter;
@Schema(required = true)
private boolean tumblr;
@Schema(required = true)
private boolean tumblr;
@Schema(required = true)
private boolean instapaper;
@Schema(required = true)
private boolean instapaper;
@Schema(required = true)
private boolean buffer;
}
@Schema(required = true)
private boolean buffer;
}
}

141
commafeed-server/src/main/java/com/commafeed/frontend/model/Subscription.java
View File

@@ -1,18 +1,15 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import java.time.Instant;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "User information")
@@ -20,80 +17,90 @@ import lombok.Data;
@RegisterForReflection
public class Subscription implements Serializable {
@Schema(description = "subscription id", required = true)
private Long id;
@Schema(description = "subscription id", required = true)
private Long id;
@Schema(description = "subscription name", required = true)
private String name;
@Schema(description = "subscription name", required = true)
private String name;
@Schema(description = "error message while fetching the feed")
private String message;
@Schema(description = "error message while fetching the feed")
private String message;
@Schema(description = "error count", required = true)
private int errorCount;
@Schema(description = "error count", required = true)
private int errorCount;
@Schema(description = "last time the feed was refreshed", type = SchemaType.INTEGER)
private Instant lastRefresh;
@Schema(description = "last time the feed was refreshed", type = SchemaType.INTEGER)
private Instant lastRefresh;
@Schema(description = "next time the feed refresh is planned, null if refresh is already queued", type = SchemaType.INTEGER)
private Instant nextRefresh;
@Schema(
description =
"next time the feed refresh is planned, null if refresh is already queued",
type = SchemaType.INTEGER)
private Instant nextRefresh;
@Schema(description = "this subscription's feed url", required = true)
private String feedUrl;
@Schema(description = "this subscription's feed url", required = true)
private String feedUrl;
@Schema(description = "this subscription's website url", required = true)
private String feedLink;
@Schema(description = "this subscription's website url", required = true)
private String feedLink;
@Schema(description = "The favicon url to use for this feed", required = true)
private String iconUrl;
@Schema(description = "The favicon url to use for this feed", required = true)
private String iconUrl;
@Schema(description = "unread count", required = true)
private long unread;
@Schema(description = "unread count", required = true)
private long unread;
@Schema(description = "category id")
private String categoryId;
@Schema(description = "category id")
private String categoryId;
@Schema(description = "position of the subscription's in the list")
private int position;
@Schema(description = "position of the subscription's in the list")
private int position;
@Schema(description = "date of the newest item", type = SchemaType.INTEGER)
private Instant newestItemTime;
@Schema(description = "date of the newest item", type = SchemaType.INTEGER)
private Instant newestItemTime;
@Schema(description = "CEL string evaluated on new entries to mark them as read if they do not match")
private String filter;
@Schema(
description =
"CEL string evaluated on new entries to mark them as read if they do not match")
private String filter;
@Schema(description = "JEXL legacy filter")
private String filterLegacy;
@Schema(description = "JEXL legacy filter")
private String filterLegacy;
@Schema(description = "whether to send push notifications for new entries of this feed", required = true)
private boolean pushNotificationsEnabled;
@Schema(
description = "whether to send push notifications for new entries of this feed",
required = true)
private boolean pushNotificationsEnabled;
@Schema(description = "automatically mark entries as read after this many days (null to disable)")
private Integer autoMarkAsReadAfterDays;
@Schema(
description =
"automatically mark entries as read after this many days (null to disable)")
private Integer autoMarkAsReadAfterDays;
public static Subscription build(FeedSubscription subscription, UnreadCount unreadCount) {
FeedCategory category = subscription.getCategory();
Feed feed = subscription.getFeed();
Subscription sub = new Subscription();
sub.setId(subscription.getId());
sub.setName(subscription.getTitle());
sub.setPosition(subscription.getPosition());
sub.setMessage(feed.getMessage());
sub.setErrorCount(feed.getErrorCount());
sub.setFeedUrl(feed.getUrl());
sub.setFeedLink(feed.getLink());
sub.setIconUrl(FeedUtils.getFaviconUrl(subscription));
sub.setLastRefresh(feed.getLastUpdated());
sub.setNextRefresh(
(feed.getDisabledUntil() != null && feed.getDisabledUntil().isBefore(Instant.now())) ? null : feed.getDisabledUntil());
sub.setUnread(unreadCount.getUnreadCount());
sub.setNewestItemTime(unreadCount.getNewestItemTime());
sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));
sub.setFilter(subscription.getFilter());
sub.setFilterLegacy(subscription.getFilterLegacy());
sub.setPushNotificationsEnabled(subscription.isPushNotificationsEnabled());
sub.setAutoMarkAsReadAfterDays(subscription.getAutoMarkAsReadAfterDays());
return sub;
}
}
public static Subscription build(FeedSubscription subscription, UnreadCount unreadCount) {
FeedCategory category = subscription.getCategory();
Feed feed = subscription.getFeed();
Subscription sub = new Subscription();
sub.setId(subscription.getId());
sub.setName(subscription.getTitle());
sub.setPosition(subscription.getPosition());
sub.setMessage(feed.getMessage());
sub.setErrorCount(feed.getErrorCount());
sub.setFeedUrl(feed.getUrl());
sub.setFeedLink(feed.getLink());
sub.setIconUrl(FeedUtils.getFaviconUrl(subscription));
sub.setLastRefresh(feed.getLastUpdated());
sub.setNextRefresh(
(feed.getDisabledUntil() != null && feed.getDisabledUntil().isBefore(Instant.now()))
? null
: feed.getDisabledUntil());
sub.setUnread(unreadCount.getUnreadCount());
sub.setNewestItemTime(unreadCount.getNewestItemTime());
sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));
sub.setFilter(subscription.getFilter());
sub.setFilterLegacy(subscription.getFilterLegacy());
sub.setPushNotificationsEnabled(subscription.isPushNotificationsEnabled());
sub.setAutoMarkAsReadAfterDays(subscription.getAutoMarkAsReadAfterDays());
return sub;
}
}

30
commafeed-server/src/main/java/com/commafeed/frontend/model/UnreadCount.java
View File

@@ -1,36 +1,30 @@
package com.commafeed.frontend.model;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import java.time.Instant;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import io.quarkus.runtime.annotations.RegisterForReflection;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Unread count")
@Data
@RegisterForReflection
public class UnreadCount implements Serializable {
@Schema
private long feedId;
@Schema private long feedId;
@Schema
private long unreadCount;
@Schema private long unreadCount;
@Schema(type = SchemaType.INTEGER)
private Instant newestItemTime;
@Schema(type = SchemaType.INTEGER)
private Instant newestItemTime;
public UnreadCount() {
}
public UnreadCount(long feedId, long unreadCount, Instant newestItemTime) {
this.feedId = feedId;
this.unreadCount = unreadCount;
this.newestItemTime = newestItemTime;
}
public UnreadCount() {}
public UnreadCount(long feedId, long unreadCount, Instant newestItemTime) {
this.feedId = feedId;
this.unreadCount = unreadCount;
this.newestItemTime = newestItemTime;
}
}

47
commafeed-server/src/main/java/com/commafeed/frontend/model/UserModel.java
View File

@@ -1,48 +1,45 @@
package com.commafeed.frontend.model;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.io.Serializable;
import java.time.Instant;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.enums.SchemaType;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import io.quarkus.runtime.annotations.RegisterForReflection;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "User information")
@Data
@RegisterForReflection
public class UserModel implements Serializable {
@Schema(description = "user id", required = true)
private Long id;
@Schema(description = "user id", required = true)
private Long id;
@Schema(description = "user name", required = true)
private String name;
@Schema(description = "user name", required = true)
private String name;
@Schema(description = "user email, if any")
private String email;
@Schema(description = "user email, if any")
private String email;
@Schema(description = "api key")
private String apiKey;
@Schema(description = "api key")
private String apiKey;
@Schema(description = "user password, never returned by the api")
private String password;
@Schema(description = "user password, never returned by the api")
private String password;
@Schema(description = "account status", required = true)
private boolean enabled;
@Schema(description = "account status", required = true)
private boolean enabled;
@Schema(description = "account creation date", type = SchemaType.INTEGER)
private Instant created;
@Schema(description = "account creation date", type = SchemaType.INTEGER)
private Instant created;
@Schema(description = "last login date", type = SchemaType.INTEGER)
private Instant lastLogin;
@Schema(description = "last login date", type = SchemaType.INTEGER)
private Instant lastLogin;
@Schema(description = "user is admin", required = true)
private boolean admin;
@Schema(description = "user last force refresh", type = SchemaType.INTEGER)
private Instant lastForceRefresh;
@Schema(description = "user is admin", required = true)
private boolean admin;
@Schema(description = "user last force refresh", type = SchemaType.INTEGER)
private Instant lastForceRefresh;
}

22
commafeed-server/src/main/java/com/commafeed/frontend/model/request/AddCategoryRequest.java
View File

@@ -1,26 +1,22 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.Size;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import java.io.Serializable;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "Add Category Request")
@Data
public class AddCategoryRequest implements Serializable {
@Schema(description = "name", required = true)
@NotEmpty
@Size(max = 128)
private String name;
@Schema(description = "parent category id, if any")
@Size(max = 128)
private String parentId;
@Schema(description = "name", required = true)
@NotEmpty
@Size(max = 128)
private String name;
@Schema(description = "parent category id, if any")
@Size(max = 128)
private String parentId;
}

33
commafeed-server/src/main/java/com/commafeed/frontend/model/request/AdminSaveUserRequest.java
View File

@@ -1,34 +1,31 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
import com.commafeed.security.password.ValidPassword;
import java.io.Serializable;
import lombok.Data;
import org.eclipse.microprofile.openapi.annotations.media.Schema;
@SuppressWarnings("serial")
@Schema(description = "Save User information")
@Data
public class AdminSaveUserRequest implements Serializable {
@Schema(description = "user id")
private Long id;
@Schema(description = "user id")
private Long id;
@Schema(description = "user name", required = true)
private String name;
@Schema(description = "user name", required = true)
private String name;
@Schema(description = "user email, if any")
private String email;
@Schema(description = "user email, if any")
private String email;
@Schema(description = "user password")
@ValidPassword
private String password;
@Schema(description = "user password")
@ValidPassword
private String password;
@Schema(description = "account status", required = true)
private boolean enabled;
@Schema(description = "account status", required = true)
private boolean enabled;
@Schema(description = "user is admin", required = true)
private boolean admin;
@Schema(description = "user is admin", required = true)
private boolean admin;
}

Some files were not shown because too many files have changed in this diff Show More