garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

normalize line endings

Browse Source
This commit is contained in:
Athou
2025-03-10 08:38:21 +01:00
parent ec4554c76e
commit fb7f041454
223 changed files with 18091 additions and 18093 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
commafeed-server/src/main/assembly/zip-quarkus-app.xml
View File

@@ -1,21 +1,21 @@
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.2.0 https://maven.apache.org/xsd/assembly-2.2.0.xsd">
<id>zip-quarkus-app</id>
<includeBaseDirectory>true</includeBaseDirectory>
<baseDirectory>commafeed-${project.version}-${build.database}</baseDirectory>
<formats>
<format>zip</format>
</formats>
<fileSets>
<fileSet>
<directory>${project.build.directory}/quarkus-app</directory>
<outputDirectory>/</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>
</fileSets>
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.2.0 https://maven.apache.org/xsd/assembly-2.2.0.xsd">
<id>zip-quarkus-app</id>
<includeBaseDirectory>true</includeBaseDirectory>
<baseDirectory>commafeed-${project.version}-${build.database}</baseDirectory>
<formats>
<format>zip</format>
</formats>
<fileSets>
<fileSet>
<directory>${project.build.directory}/quarkus-app</directory>
<outputDirectory>/</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>
</fileSets>
</assembly>

190
commafeed-server/src/main/docker/README.md
View File

@@ -1,95 +1,95 @@
# CommaFeed
Official docker images for https://github.com/Athou/commafeed/
## Quickstart
Start CommaFeed with a H2 embedded database. Then login as `admin/admin` on http://localhost:8082/
### docker
`docker run --name commafeed --detach --publish 8082:8082 --restart unless-stopped --volume /path/to/commafeed/db:/commafeed/data --memory 256M athou/commafeed:latest-h2`
### docker-compose
```
services:
commafeed:
image: athou/commafeed:latest-h2
restart: unless-stopped
volumes:
- /path/to/commafeed/db:/commafeed/data
deploy:
resources:
limits:
memory: 256M
ports:
- 8082:8082
```
## Advanced
While using the H2 embedded database is perfectly fine for small instances, you may want to have more control over the
database. Here's an example that uses PostgreSQL (note the image tag change from `latest-h2` to `latest-postgresql`):
```
services:
commafeed:
image: athou/commafeed:latest-postgresql
restart: unless-stopped
environment:
- QUARKUS_DATASOURCE_JDBC_URL=jdbc:postgresql://postgresql:5432/commafeed
- QUARKUS_DATASOURCE_USERNAME=commafeed
- QUARKUS_DATASOURCE_PASSWORD=commafeed
deploy:
resources:
limits:
memory: 256M
ports:
- 8082:8082
postgresql:
image: postgres:latest
restart: unless-stopped
environment:
POSTGRES_USER: commafeed
POSTGRES_PASSWORD: commafeed
POSTGRES_DB: commafeed
volumes:
- /path/to/commafeed/db:/var/lib/postgresql/data
```
CommaFeed also supports:
- MySQL:
`QUARKUS_DATASOURCE_JDBC_URL=jdbc:mysql://localhost/commafeed?autoReconnect=true&failOverReadOnly=false&maxReconnects=20&rewriteBatchedStatements=true&timezone=UTC`
- MariaDB:
`QUARKUS_DATASOURCE_JDBC_URL=jdbc:mariadb://localhost/commafeed?autoReconnect=true&failOverReadOnly=false&maxReconnects=20&rewriteBatchedStatements=true&timezone=UTC`
## Configuration
All [CommaFeed settings](https://athou.github.io/commafeed/documentation) are
optional and have sensible default values.
Settings are overrideable with environment variables. For instance, `commafeed.feed-refresh.interval-empirical` can be
set with the `COMMAFEED_FEED_REFRESH_INTERVAL_EMPIRICAL` variable.
When logging in, credentials are stored in an encrypted cookie. The encryption key is randomly generated at startup,
meaning that you will have to log back in after each restart of the application. To prevent this, you can set the
`QUARKUS_HTTP_AUTH_SESSION_ENCRYPTION_KEY` variable to a fixed value (min. 16 characters).
All other Quarkus settings can be found [here](https://quarkus.io/guides/all-config).
### Updates
When CommaFeed is up and running, you can subscribe to [this feed](https://github.com/Athou/commafeed/releases.atom) to be notified of new releases.
## Docker tags
Tags are of the form `<version>-<database>[-jvm]` where:
- `<version>` is either:
- a specific CommaFeed version (e.g. `5.0.0`)
- `latest` (always points to the latest version)
- `master` (always points to the latest git commit)
- `<database>` is the database to use (`h2`, `postgresql`, `mysql` or `mariadb`)
- `-jvm` is optional and indicates that CommaFeed is running on a JVM, and not compiled natively.
# CommaFeed
Official docker images for https://github.com/Athou/commafeed/
## Quickstart
Start CommaFeed with a H2 embedded database. Then login as `admin/admin` on http://localhost:8082/
### docker
`docker run --name commafeed --detach --publish 8082:8082 --restart unless-stopped --volume /path/to/commafeed/db:/commafeed/data --memory 256M athou/commafeed:latest-h2`
### docker-compose
```
services:
commafeed:
image: athou/commafeed:latest-h2
restart: unless-stopped
volumes:
- /path/to/commafeed/db:/commafeed/data
deploy:
resources:
limits:
memory: 256M
ports:
- 8082:8082
```
## Advanced
While using the H2 embedded database is perfectly fine for small instances, you may want to have more control over the
database. Here's an example that uses PostgreSQL (note the image tag change from `latest-h2` to `latest-postgresql`):
```
services:
commafeed:
image: athou/commafeed:latest-postgresql
restart: unless-stopped
environment:
- QUARKUS_DATASOURCE_JDBC_URL=jdbc:postgresql://postgresql:5432/commafeed
- QUARKUS_DATASOURCE_USERNAME=commafeed
- QUARKUS_DATASOURCE_PASSWORD=commafeed
deploy:
resources:
limits:
memory: 256M
ports:
- 8082:8082
postgresql:
image: postgres:latest
restart: unless-stopped
environment:
POSTGRES_USER: commafeed
POSTGRES_PASSWORD: commafeed
POSTGRES_DB: commafeed
volumes:
- /path/to/commafeed/db:/var/lib/postgresql/data
```
CommaFeed also supports:
- MySQL:
`QUARKUS_DATASOURCE_JDBC_URL=jdbc:mysql://localhost/commafeed?autoReconnect=true&failOverReadOnly=false&maxReconnects=20&rewriteBatchedStatements=true&timezone=UTC`
- MariaDB:
`QUARKUS_DATASOURCE_JDBC_URL=jdbc:mariadb://localhost/commafeed?autoReconnect=true&failOverReadOnly=false&maxReconnects=20&rewriteBatchedStatements=true&timezone=UTC`
## Configuration
All [CommaFeed settings](https://athou.github.io/commafeed/documentation) are
optional and have sensible default values.
Settings are overrideable with environment variables. For instance, `commafeed.feed-refresh.interval-empirical` can be
set with the `COMMAFEED_FEED_REFRESH_INTERVAL_EMPIRICAL` variable.
When logging in, credentials are stored in an encrypted cookie. The encryption key is randomly generated at startup,
meaning that you will have to log back in after each restart of the application. To prevent this, you can set the
`QUARKUS_HTTP_AUTH_SESSION_ENCRYPTION_KEY` variable to a fixed value (min. 16 characters).
All other Quarkus settings can be found [here](https://quarkus.io/guides/all-config).
### Updates
When CommaFeed is up and running, you can subscribe to [this feed](https://github.com/Athou/commafeed/releases.atom) to be notified of new releases.
## Docker tags
Tags are of the form `<version>-<database>[-jvm]` where:
- `<version>` is either:
- a specific CommaFeed version (e.g. `5.0.0`)
- `latest` (always points to the latest version)
- `master` (always points to the latest git commit)
- `<database>` is the database to use (`h2`, `postgresql`, `mysql` or `mariadb`)
- `-jvm` is optional and indicates that CommaFeed is running on a JVM, and not compiled natively.

82
commafeed-server/src/main/java/com/commafeed/CommaFeedApplication.java
View File

@@ -1,41 +1,41 @@
package com.commafeed;
import jakarta.enterprise.event.Observes;
import jakarta.inject.Singleton;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.service.db.DatabaseStartupService;
import com.commafeed.backend.task.TaskScheduler;
import com.commafeed.security.password.PasswordConstraintValidator;
import io.quarkus.runtime.ShutdownEvent;
import io.quarkus.runtime.StartupEvent;
import lombok.RequiredArgsConstructor;
@Singleton
@RequiredArgsConstructor
public class CommaFeedApplication {
public static final String USERNAME_ADMIN = "admin";
public static final String USERNAME_DEMO = "demo";
private final DatabaseStartupService databaseStartupService;
private final FeedRefreshEngine feedRefreshEngine;
private final TaskScheduler taskScheduler;
private final CommaFeedConfiguration config;
public void start(@Observes StartupEvent ev) {
PasswordConstraintValidator.setStrict(config.users().strictPasswordPolicy());
databaseStartupService.populateInitialData();
feedRefreshEngine.start();
taskScheduler.start();
}
public void stop(@Observes ShutdownEvent ev) {
feedRefreshEngine.stop();
taskScheduler.stop();
}
}
package com.commafeed;
import jakarta.enterprise.event.Observes;
import jakarta.inject.Singleton;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.service.db.DatabaseStartupService;
import com.commafeed.backend.task.TaskScheduler;
import com.commafeed.security.password.PasswordConstraintValidator;
import io.quarkus.runtime.ShutdownEvent;
import io.quarkus.runtime.StartupEvent;
import lombok.RequiredArgsConstructor;
@Singleton
@RequiredArgsConstructor
public class CommaFeedApplication {
public static final String USERNAME_ADMIN = "admin";
public static final String USERNAME_DEMO = "demo";
private final DatabaseStartupService databaseStartupService;
private final FeedRefreshEngine feedRefreshEngine;
private final TaskScheduler taskScheduler;
private final CommaFeedConfiguration config;
public void start(@Observes StartupEvent ev) {
PasswordConstraintValidator.setStrict(config.users().strictPasswordPolicy());
databaseStartupService.populateInitialData();
feedRefreshEngine.start();
taskScheduler.start();
}
public void stop(@Observes ShutdownEvent ev) {
feedRefreshEngine.stop();
taskScheduler.stop();
}
}

732
commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java
View File

@@ -1,366 +1,366 @@
package com.commafeed;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.Positive;
import com.commafeed.backend.feed.FeedRefreshIntervalCalculator;
import io.quarkus.runtime.annotations.ConfigDocSection;
import io.quarkus.runtime.annotations.ConfigPhase;
import io.quarkus.runtime.annotations.ConfigRoot;
import io.quarkus.runtime.configuration.MemorySize;
import io.smallrye.config.ConfigMapping;
import io.smallrye.config.WithDefault;
/**
* CommaFeed configuration
*
* Default values are for production, they can be overridden in application.properties for other profiles
*/
@ConfigMapping(prefix = "commafeed")
@ConfigRoot(phase = ConfigPhase.RUN_TIME)
public interface CommaFeedConfiguration {
/**
* Whether to expose a robots.txt file that disallows web crawlers and search engine indexers.
*/
@WithDefault("true")
boolean hideFromWebCrawlers();
/**
* If enabled, images in feed entries will be proxied through the server instead of accessed directly by the browser.
*
* This is useful if commafeed is accessed through a restricting proxy that blocks some feeds that are followed.
*/
@WithDefault("false")
boolean imageProxyEnabled();
/**
* Enable password recovery via email.
*
* Quarkus mailer will need to be configured.
*/
@WithDefault("false")
boolean passwordRecoveryEnabled();
/**
* Message displayed in a notification at the bottom of the page.
*/
Optional<String> announcement();
/**
* Google Analytics tracking code.
*/
Optional<String> googleAnalyticsTrackingCode();
/**
* Google Auth key for fetching Youtube channel favicons.
*/
Optional<String> googleAuthKey();
/**
* HTTP client configuration
*/
@ConfigDocSection
HttpClient httpClient();
/**
* Feed refresh engine settings.
*/
@ConfigDocSection
FeedRefresh feedRefresh();
/**
* Database settings.
*/
@ConfigDocSection
Database database();
/**
* Users settings.
*/
@ConfigDocSection
Users users();
/**
* Websocket settings.
*/
@ConfigDocSection
Websocket websocket();
interface HttpClient {
/**
* User-Agent string that will be used by the http client, leave empty for the default one.
*/
Optional<String> userAgent();
/**
* Time to wait for a connection to be established.
*/
@WithDefault("5s")
Duration connectTimeout();
/**
* Time to wait for SSL handshake to complete.
*/
@WithDefault("5s")
Duration sslHandshakeTimeout();
/**
* Time to wait between two packets before timeout.
*/
@WithDefault("10s")
Duration socketTimeout();
/**
* Time to wait for the full response to be received.
*/
@WithDefault("10s")
Duration responseTimeout();
/**
* Time to live for a connection in the pool.
*/
@WithDefault("30s")
Duration connectionTimeToLive();
/**
* Time between eviction runs for idle connections.
*/
@WithDefault("1m")
Duration idleConnectionsEvictionInterval();
/**
* If a feed is larger than this, it will be discarded to prevent memory issues while parsing the feed.
*/
@WithDefault("5M")
MemorySize maxResponseSize();
/**
* Prevent access to local addresses to mitigate server-side request forgery (SSRF) attacks, which could potentially expose internal
* resources.
*
* You may want to disable this if you subscribe to feeds that are only available on your local network and you trust all users of
* your CommaFeed instance.
*/
@WithDefault("true")
boolean blockLocalAddresses();
/**
* HTTP client cache configuration
*/
@ConfigDocSection
HttpClientCache cache();
}
interface HttpClientCache {
/**
* Whether to enable the cache. This cache is used to avoid spamming feeds in short bursts (e.g. when subscribing to a feed for the
* first time or when clicking "fetch all my feeds now").
*/
@WithDefault("true")
boolean enabled();
/**
* Maximum amount of memory the cache can use.
*/
@WithDefault("10M")
MemorySize maximumMemorySize();
/**
* Duration after which an entry is removed from the cache.
*/
@WithDefault("1m")
Duration expiration();
}
interface FeedRefresh {
/**
* Default amount of time CommaFeed will wait before refreshing a feed.
*/
@WithDefault("5m")
Duration interval();
/**
* Maximum amount of time CommaFeed will wait before refreshing a feed. This is used as an upper bound when:
*
* <ul>
* <li>an error occurs while refreshing a feed and we're backing off exponentially</li>
* <li>we receive a Cache-Control header from the feed</li>
* <li>we receive a Retry-After header from the feed</li>
* </ul>
*/
@WithDefault("4h")
Duration maxInterval();
/**
* If enabled, CommaFeed will calculate the next refresh time based on the feed's average time between entries and the time since
* the last entry was published. The interval will be sometimes between the default refresh interval
* (`commafeed.feed-refresh.interval`) and the maximum refresh interval (`commafeed.feed-refresh.max-interval`).
*
* See {@link FeedRefreshIntervalCalculator} for details.
*/
@WithDefault("true")
boolean intervalEmpirical();
/**
* Feed refresh engine error handling settings.
*/
@ConfigDocSection
FeedRefreshErrorHandling errors();
/**
* Amount of http threads used to fetch feeds.
*/
@Min(1)
@WithDefault("3")
int httpThreads();
/**
* Amount of threads used to insert new entries in the database.
*/
@Min(1)
@WithDefault("1")
int databaseThreads();
/**
* Duration after which a user is considered inactive. Feeds for inactive users are not refreshed until they log in again.
*
* 0 to disable.
*/
@WithDefault("0")
Duration userInactivityPeriod();
/**
* Duration after which the evaluation of a filtering expresion to mark an entry as read is considered to have timed out.
*/
@WithDefault("500ms")
Duration filteringExpressionEvaluationTimeout();
/**
* Duration after which the "Fetch all my feeds now" action is available again after use to avoid spamming feeds.
*/
@WithDefault("0")
Duration forceRefreshCooldownDuration();
}
interface FeedRefreshErrorHandling {
/**
* Number of retries before backoff is applied.
*/
@Min(0)
@WithDefault("3")
int retriesBeforeBackoff();
/**
* Duration to wait before retrying after an error. Will be multiplied by the number of errors since the last successful fetch.
*/
@WithDefault("1h")
Duration backoffInterval();
}
interface Database {
/**
* Timeout applied to all database queries.
*
* 0 to disable.
*/
@WithDefault("0")
Duration queryTimeout();
/**
* Database cleanup settings.
*/
@ConfigDocSection
Cleanup cleanup();
interface Cleanup {
/**
* Maximum age of feed entries in the database. Older entries will be deleted.
*
* 0 to disable.
*/
@WithDefault("365d")
Duration entriesMaxAge();
/**
* Maximum age of feed entry statuses (read/unread) in the database. Older statuses will be deleted.
*
* 0 to disable.
*/
@WithDefault("0")
Duration statusesMaxAge();
/**
* Maximum number of entries per feed to keep in the database.
*
* 0 to disable.
*/
@WithDefault("500")
int maxFeedCapacity();
/**
* Limit the number of feeds a user can subscribe to.
*
* 0 to disable.
*/
@WithDefault("0")
int maxFeedsPerUser();
/**
* Rows to delete per query while cleaning up old entries.
*/
@Positive
@WithDefault("100")
int batchSize();
default Instant statusesInstantThreshold() {
return statusesMaxAge().toMillis() > 0 ? Instant.now().minus(statusesMaxAge()) : null;
}
}
}
interface Users {
/**
* Whether to let users create accounts for themselves.
*/
@WithDefault("false")
boolean allowRegistrations();
/**
* Whether to enable strict password validation (1 uppercase char, 1 lowercase char, 1 digit, 1 special char).
*/
@WithDefault("true")
boolean strictPasswordPolicy();
/**
* Whether to create a demo account the first time the app starts.
*/
@WithDefault("false")
boolean createDemoAccount();
}
interface Websocket {
/**
* Enable websocket connection so the server can notify web clients that there are new entries for feeds.
*/
@WithDefault("true")
boolean enabled();
/**
* Interval at which the client will send a ping message on the websocket to keep the connection alive.
*/
@WithDefault("15m")
Duration pingInterval();
/**
* If the websocket connection is disabled or the connection is lost, the client will reload the feed tree at this interval.
*/
@WithDefault("30s")
Duration treeReloadInterval();
}
}
package com.commafeed;
import java.time.Duration;
import java.time.Instant;
import java.util.Optional;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.Positive;
import com.commafeed.backend.feed.FeedRefreshIntervalCalculator;
import io.quarkus.runtime.annotations.ConfigDocSection;
import io.quarkus.runtime.annotations.ConfigPhase;
import io.quarkus.runtime.annotations.ConfigRoot;
import io.quarkus.runtime.configuration.MemorySize;
import io.smallrye.config.ConfigMapping;
import io.smallrye.config.WithDefault;
/**
* CommaFeed configuration
*
* Default values are for production, they can be overridden in application.properties for other profiles
*/
@ConfigMapping(prefix = "commafeed")
@ConfigRoot(phase = ConfigPhase.RUN_TIME)
public interface CommaFeedConfiguration {
/**
* Whether to expose a robots.txt file that disallows web crawlers and search engine indexers.
*/
@WithDefault("true")
boolean hideFromWebCrawlers();
/**
* If enabled, images in feed entries will be proxied through the server instead of accessed directly by the browser.
*
* This is useful if commafeed is accessed through a restricting proxy that blocks some feeds that are followed.
*/
@WithDefault("false")
boolean imageProxyEnabled();
/**
* Enable password recovery via email.
*
* Quarkus mailer will need to be configured.
*/
@WithDefault("false")
boolean passwordRecoveryEnabled();
/**
* Message displayed in a notification at the bottom of the page.
*/
Optional<String> announcement();
/**
* Google Analytics tracking code.
*/
Optional<String> googleAnalyticsTrackingCode();
/**
* Google Auth key for fetching Youtube channel favicons.
*/
Optional<String> googleAuthKey();
/**
* HTTP client configuration
*/
@ConfigDocSection
HttpClient httpClient();
/**
* Feed refresh engine settings.
*/
@ConfigDocSection
FeedRefresh feedRefresh();
/**
* Database settings.
*/
@ConfigDocSection
Database database();
/**
* Users settings.
*/
@ConfigDocSection
Users users();
/**
* Websocket settings.
*/
@ConfigDocSection
Websocket websocket();
interface HttpClient {
/**
* User-Agent string that will be used by the http client, leave empty for the default one.
*/
Optional<String> userAgent();
/**
* Time to wait for a connection to be established.
*/
@WithDefault("5s")
Duration connectTimeout();
/**
* Time to wait for SSL handshake to complete.
*/
@WithDefault("5s")
Duration sslHandshakeTimeout();
/**
* Time to wait between two packets before timeout.
*/
@WithDefault("10s")
Duration socketTimeout();
/**
* Time to wait for the full response to be received.
*/
@WithDefault("10s")
Duration responseTimeout();
/**
* Time to live for a connection in the pool.
*/
@WithDefault("30s")
Duration connectionTimeToLive();
/**
* Time between eviction runs for idle connections.
*/
@WithDefault("1m")
Duration idleConnectionsEvictionInterval();
/**
* If a feed is larger than this, it will be discarded to prevent memory issues while parsing the feed.
*/
@WithDefault("5M")
MemorySize maxResponseSize();
/**
* Prevent access to local addresses to mitigate server-side request forgery (SSRF) attacks, which could potentially expose internal
* resources.
*
* You may want to disable this if you subscribe to feeds that are only available on your local network and you trust all users of
* your CommaFeed instance.
*/
@WithDefault("true")
boolean blockLocalAddresses();
/**
* HTTP client cache configuration
*/
@ConfigDocSection
HttpClientCache cache();
}
interface HttpClientCache {
/**
* Whether to enable the cache. This cache is used to avoid spamming feeds in short bursts (e.g. when subscribing to a feed for the
* first time or when clicking "fetch all my feeds now").
*/
@WithDefault("true")
boolean enabled();
/**
* Maximum amount of memory the cache can use.
*/
@WithDefault("10M")
MemorySize maximumMemorySize();
/**
* Duration after which an entry is removed from the cache.
*/
@WithDefault("1m")
Duration expiration();
}
interface FeedRefresh {
/**
* Default amount of time CommaFeed will wait before refreshing a feed.
*/
@WithDefault("5m")
Duration interval();
/**
* Maximum amount of time CommaFeed will wait before refreshing a feed. This is used as an upper bound when:
*
* <ul>
* <li>an error occurs while refreshing a feed and we're backing off exponentially</li>
* <li>we receive a Cache-Control header from the feed</li>
* <li>we receive a Retry-After header from the feed</li>
* </ul>
*/
@WithDefault("4h")
Duration maxInterval();
/**
* If enabled, CommaFeed will calculate the next refresh time based on the feed's average time between entries and the time since
* the last entry was published. The interval will be sometimes between the default refresh interval
* (`commafeed.feed-refresh.interval`) and the maximum refresh interval (`commafeed.feed-refresh.max-interval`).
*
* See {@link FeedRefreshIntervalCalculator} for details.
*/
@WithDefault("true")
boolean intervalEmpirical();
/**
* Feed refresh engine error handling settings.
*/
@ConfigDocSection
FeedRefreshErrorHandling errors();
/**
* Amount of http threads used to fetch feeds.
*/
@Min(1)
@WithDefault("3")
int httpThreads();
/**
* Amount of threads used to insert new entries in the database.
*/
@Min(1)
@WithDefault("1")
int databaseThreads();
/**
* Duration after which a user is considered inactive. Feeds for inactive users are not refreshed until they log in again.
*
* 0 to disable.
*/
@WithDefault("0")
Duration userInactivityPeriod();
/**
* Duration after which the evaluation of a filtering expresion to mark an entry as read is considered to have timed out.
*/
@WithDefault("500ms")
Duration filteringExpressionEvaluationTimeout();
/**
* Duration after which the "Fetch all my feeds now" action is available again after use to avoid spamming feeds.
*/
@WithDefault("0")
Duration forceRefreshCooldownDuration();
}
interface FeedRefreshErrorHandling {
/**
* Number of retries before backoff is applied.
*/
@Min(0)
@WithDefault("3")
int retriesBeforeBackoff();
/**
* Duration to wait before retrying after an error. Will be multiplied by the number of errors since the last successful fetch.
*/
@WithDefault("1h")
Duration backoffInterval();
}
interface Database {
/**
* Timeout applied to all database queries.
*
* 0 to disable.
*/
@WithDefault("0")
Duration queryTimeout();
/**
* Database cleanup settings.
*/
@ConfigDocSection
Cleanup cleanup();
interface Cleanup {
/**
* Maximum age of feed entries in the database. Older entries will be deleted.
*
* 0 to disable.
*/
@WithDefault("365d")
Duration entriesMaxAge();
/**
* Maximum age of feed entry statuses (read/unread) in the database. Older statuses will be deleted.
*
* 0 to disable.
*/
@WithDefault("0")
Duration statusesMaxAge();
/**
* Maximum number of entries per feed to keep in the database.
*
* 0 to disable.
*/
@WithDefault("500")
int maxFeedCapacity();
/**
* Limit the number of feeds a user can subscribe to.
*
* 0 to disable.
*/
@WithDefault("0")
int maxFeedsPerUser();
/**
* Rows to delete per query while cleaning up old entries.
*/
@Positive
@WithDefault("100")
int batchSize();
default Instant statusesInstantThreshold() {
return statusesMaxAge().toMillis() > 0 ? Instant.now().minus(statusesMaxAge()) : null;
}
}
}
interface Users {
/**
* Whether to let users create accounts for themselves.
*/
@WithDefault("false")
boolean allowRegistrations();
/**
* Whether to enable strict password validation (1 uppercase char, 1 lowercase char, 1 digit, 1 special char).
*/
@WithDefault("true")
boolean strictPasswordPolicy();
/**
* Whether to create a demo account the first time the app starts.
*/
@WithDefault("false")
boolean createDemoAccount();
}
interface Websocket {
/**
* Enable websocket connection so the server can notify web clients that there are new entries for feeds.
*/
@WithDefault("true")
boolean enabled();
/**
* Interval at which the client will send a ping message on the websocket to keep the connection alive.
*/
@WithDefault("15m")
Duration pingInterval();
/**
* If the websocket connection is disabled or the connection is lost, the client will reload the feed tree at this interval.
*/
@WithDefault("30s")
Duration treeReloadInterval();
}
}

50
commafeed-server/src/main/java/com/commafeed/CommaFeedProducers.java
View File

@@ -1,25 +1,25 @@
package com.commafeed;
import java.time.InstantSource;
import jakarta.enterprise.inject.Produces;
import jakarta.inject.Singleton;
import com.codahale.metrics.MetricRegistry;
@Singleton
public class CommaFeedProducers {
@Produces
@Singleton
public InstantSource instantSource() {
return InstantSource.system();
}
@Produces
@Singleton
public MetricRegistry metricRegistry() {
return new MetricRegistry();
}
}
package com.commafeed;
import java.time.InstantSource;
import jakarta.enterprise.inject.Produces;
import jakarta.inject.Singleton;
import com.codahale.metrics.MetricRegistry;
@Singleton
public class CommaFeedProducers {
@Produces
@Singleton
public InstantSource instantSource() {
return InstantSource.system();
}
@Produces
@Singleton
public MetricRegistry metricRegistry() {
return new MetricRegistry();
}
}

64
commafeed-server/src/main/java/com/commafeed/CommaFeedVersion.java
View File

@@ -1,32 +1,32 @@
package com.commafeed;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import jakarta.inject.Singleton;
import lombok.Getter;
@Singleton
@Getter
public class CommaFeedVersion {
private final String version;
private final String gitCommit;
public CommaFeedVersion() {
Properties properties = new Properties();
try (InputStream stream = getClass().getResourceAsStream("/git.properties")) {
if (stream != null) {
properties.load(stream);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
this.version = properties.getProperty("git.build.version", "unknown");
this.gitCommit = properties.getProperty("git.commit.id.abbrev", "unknown");
}
}
package com.commafeed;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import jakarta.inject.Singleton;
import lombok.Getter;
@Singleton
@Getter
public class CommaFeedVersion {
private final String version;
private final String gitCommit;
public CommaFeedVersion() {
Properties properties = new Properties();
try (InputStream stream = getClass().getResourceAsStream("/git.properties")) {
if (stream != null) {
properties.load(stream);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
this.version = properties.getProperty("git.build.version", "unknown");
this.gitCommit = properties.getProperty("git.commit.id.abbrev", "unknown");
}
}

100
commafeed-server/src/main/java/com/commafeed/ExceptionMappers.java
View File

@@ -1,50 +1,50 @@
package com.commafeed;
import jakarta.annotation.Priority;
import jakarta.validation.ValidationException;
import jakarta.ws.rs.ext.Provider;
import org.jboss.resteasy.reactive.RestResponse;
import org.jboss.resteasy.reactive.RestResponse.Status;
import org.jboss.resteasy.reactive.server.ServerExceptionMapper;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.quarkus.security.AuthenticationFailedException;
import io.quarkus.security.UnauthorizedException;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Provider
@Priority(1)
public class ExceptionMappers {
private final CommaFeedConfiguration config;
@ServerExceptionMapper(UnauthorizedException.class)
public RestResponse<UnauthorizedResponse> unauthorized(UnauthorizedException e) {
return RestResponse.status(RestResponse.Status.UNAUTHORIZED,
new UnauthorizedResponse(e.getMessage(), config.users().allowRegistrations()));
}
@ServerExceptionMapper(AuthenticationFailedException.class)
public RestResponse<AuthenticationFailed> authenticationFailed(AuthenticationFailedException e) {
return RestResponse.status(RestResponse.Status.UNAUTHORIZED, new AuthenticationFailed(e.getMessage()));
}
@ServerExceptionMapper(ValidationException.class)
public RestResponse<ValidationFailed> validationFailed(ValidationException e) {
return RestResponse.status(Status.BAD_REQUEST, new ValidationFailed(e.getMessage()));
}
@RegisterForReflection
public record UnauthorizedResponse(String message, boolean allowRegistrations) {
}
@RegisterForReflection
public record AuthenticationFailed(String message) {
}
@RegisterForReflection
public record ValidationFailed(String message) {
}
}
package com.commafeed;
import jakarta.annotation.Priority;
import jakarta.validation.ValidationException;
import jakarta.ws.rs.ext.Provider;
import org.jboss.resteasy.reactive.RestResponse;
import org.jboss.resteasy.reactive.RestResponse.Status;
import org.jboss.resteasy.reactive.server.ServerExceptionMapper;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.quarkus.security.AuthenticationFailedException;
import io.quarkus.security.UnauthorizedException;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Provider
@Priority(1)
public class ExceptionMappers {
private final CommaFeedConfiguration config;
@ServerExceptionMapper(UnauthorizedException.class)
public RestResponse<UnauthorizedResponse> unauthorized(UnauthorizedException e) {
return RestResponse.status(RestResponse.Status.UNAUTHORIZED,
new UnauthorizedResponse(e.getMessage(), config.users().allowRegistrations()));
}
@ServerExceptionMapper(AuthenticationFailedException.class)
public RestResponse<AuthenticationFailed> authenticationFailed(AuthenticationFailedException e) {
return RestResponse.status(RestResponse.Status.UNAUTHORIZED, new AuthenticationFailed(e.getMessage()));
}
@ServerExceptionMapper(ValidationException.class)
public RestResponse<ValidationFailed> validationFailed(ValidationException e) {
return RestResponse.status(Status.BAD_REQUEST, new ValidationFailed(e.getMessage()));
}
@RegisterForReflection
public record UnauthorizedResponse(String message, boolean allowRegistrations) {
}
@RegisterForReflection
public record AuthenticationFailed(String message) {
}
@RegisterForReflection
public record ValidationFailed(String message) {
}
}

58
commafeed-server/src/main/java/com/commafeed/JacksonCustomizer.java
View File

@@ -1,29 +1,29 @@
package com.commafeed;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.json.MetricsModule;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.quarkus.jackson.ObjectMapperCustomizer;
@Singleton
public class JacksonCustomizer implements ObjectMapperCustomizer {
@Override
public void customize(ObjectMapper objectMapper) {
objectMapper.registerModule(new JavaTimeModule());
// read and write instants as milliseconds instead of nanoseconds
objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true)
.configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false)
.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false);
// add support for serializing metrics
objectMapper.registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
}
}
package com.commafeed;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.json.MetricsModule;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import io.quarkus.jackson.ObjectMapperCustomizer;
@Singleton
public class JacksonCustomizer implements ObjectMapperCustomizer {
@Override
public void customize(ObjectMapper objectMapper) {
objectMapper.registerModule(new JavaTimeModule());
// read and write instants as milliseconds instead of nanoseconds
objectMapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, true)
.configure(SerializationFeature.WRITE_DATE_TIMESTAMPS_AS_NANOSECONDS, false)
.configure(DeserializationFeature.READ_DATE_TIMESTAMPS_AS_NANOSECONDS, false);
// add support for serializing metrics
objectMapper.registerModule(new MetricsModule(TimeUnit.SECONDS, TimeUnit.SECONDS, false));
}
}

452
commafeed-server/src/main/java/com/commafeed/NativeImageClasses.java
View File

@@ -1,226 +1,226 @@
package com.commafeed;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import io.quarkus.runtime.annotations.RegisterForReflection;
@RegisterForReflection(
targets = {
// metrics
MetricRegistry.class, Meter.class, Gauge.class, Counter.class, Timer.class, Histogram.class,
// rome
java.util.Date.class, com.rometools.opml.feed.synd.impl.TreeCategoryImpl.class,
com.rometools.rome.feed.synd.SyndFeedImpl.class, com.rometools.rome.feed.module.DCSubjectImpl.class,
com.rometools.rome.feed.synd.SyndEntryImpl.class, com.rometools.modules.psc.types.SimpleChapter.class,
com.rometools.rome.feed.synd.SyndCategoryImpl.class, com.rometools.rome.feed.synd.SyndImageImpl.class,
com.rometools.rome.feed.synd.SyndContentImpl.class, com.rometools.rome.feed.synd.SyndEnclosureImpl.class,
// rome cloneable
com.rometools.modules.activitystreams.types.Article.class, com.rometools.modules.activitystreams.types.Audio.class,
com.rometools.modules.activitystreams.types.Bookmark.class, com.rometools.modules.activitystreams.types.Comment.class,
com.rometools.modules.activitystreams.types.Event.class, com.rometools.modules.activitystreams.types.File.class,
com.rometools.modules.activitystreams.types.Folder.class, com.rometools.modules.activitystreams.types.List.class,
com.rometools.modules.activitystreams.types.Note.class, com.rometools.modules.activitystreams.types.Person.class,
com.rometools.modules.activitystreams.types.Photo.class, com.rometools.modules.activitystreams.types.PhotoAlbum.class,
com.rometools.modules.activitystreams.types.Place.class, com.rometools.modules.activitystreams.types.Playlist.class,
com.rometools.modules.activitystreams.types.Product.class, com.rometools.modules.activitystreams.types.Review.class,
com.rometools.modules.activitystreams.types.Service.class, com.rometools.modules.activitystreams.types.Song.class,
com.rometools.modules.activitystreams.types.Status.class, com.rometools.modules.base.types.DateTimeRange.class,
com.rometools.modules.base.types.FloatUnit.class, com.rometools.modules.base.types.GenderEnumeration.class,
com.rometools.modules.base.types.IntUnit.class, com.rometools.modules.base.types.PriceTypeEnumeration.class,
com.rometools.modules.base.types.ShippingType.class, com.rometools.modules.base.types.ShortDate.class,
com.rometools.modules.base.types.Size.class, com.rometools.modules.base.types.YearType.class,
com.rometools.modules.content.ContentItem.class, com.rometools.modules.georss.GeoRSSPoint.class,
com.rometools.modules.georss.geometries.Envelope.class, com.rometools.modules.georss.geometries.LineString.class,
com.rometools.modules.georss.geometries.LinearRing.class, com.rometools.modules.georss.geometries.Point.class,
com.rometools.modules.georss.geometries.Polygon.class, com.rometools.modules.georss.geometries.Position.class,
com.rometools.modules.georss.geometries.PositionList.class, com.rometools.modules.mediarss.types.MediaGroup.class,
com.rometools.modules.mediarss.types.Metadata.class, com.rometools.modules.mediarss.types.Thumbnail.class,
com.rometools.modules.opensearch.entity.OSQuery.class, com.rometools.modules.photocast.types.PhotoDate.class,
com.rometools.modules.sle.types.DateValue.class, com.rometools.modules.sle.types.Group.class,
com.rometools.modules.sle.types.NumberValue.class, com.rometools.modules.sle.types.Sort.class,
com.rometools.modules.sle.types.StringValue.class, com.rometools.modules.yahooweather.types.Astronomy.class,
com.rometools.modules.yahooweather.types.Atmosphere.class, com.rometools.modules.yahooweather.types.Condition.class,
com.rometools.modules.yahooweather.types.Forecast.class, com.rometools.modules.yahooweather.types.Location.class,
com.rometools.modules.yahooweather.types.Units.class, com.rometools.modules.yahooweather.types.Wind.class,
com.rometools.opml.feed.opml.Attribute.class, com.rometools.opml.feed.opml.Opml.class,
com.rometools.opml.feed.opml.Outline.class, com.rometools.rome.feed.atom.Category.class,
com.rometools.rome.feed.atom.Content.class, com.rometools.rome.feed.atom.Entry.class,
com.rometools.rome.feed.atom.Feed.class, com.rometools.rome.feed.atom.Generator.class,
com.rometools.rome.feed.atom.Link.class, com.rometools.rome.feed.atom.Person.class,
com.rometools.rome.feed.rss.Category.class, com.rometools.rome.feed.rss.Channel.class,
com.rometools.rome.feed.rss.Cloud.class, com.rometools.rome.feed.rss.Content.class,
com.rometools.rome.feed.rss.Description.class, com.rometools.rome.feed.rss.Enclosure.class,
com.rometools.rome.feed.rss.Guid.class, com.rometools.rome.feed.rss.Image.class, com.rometools.rome.feed.rss.Item.class,
com.rometools.rome.feed.rss.Source.class, com.rometools.rome.feed.rss.TextInput.class,
com.rometools.rome.feed.synd.SyndLinkImpl.class, com.rometools.rome.feed.synd.SyndPersonImpl.class,
java.util.ArrayList.class,
// rome modules
com.rometools.modules.sse.modules.Conflict.class, com.rometools.modules.sse.modules.Conflicts.class,
com.rometools.modules.cc.CreativeCommonsImpl.class, com.rometools.modules.feedpress.modules.FeedpressModuleImpl.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleImpl.class, com.rometools.modules.sse.modules.Sharing.class,
com.rometools.modules.georss.SimpleModuleImpl.class, com.rometools.modules.atom.modules.AtomLinkModuleImpl.class,
com.rometools.modules.itunes.EntryInformationImpl.class, com.rometools.modules.sse.modules.Update.class,
com.rometools.modules.photocast.PhotocastModuleImpl.class, com.rometools.modules.itunes.FeedInformationImpl.class,
com.rometools.modules.yahooweather.YWeatherModuleImpl.class, com.rometools.modules.feedburner.FeedBurnerImpl.class,
com.rometools.modules.sse.modules.Related.class, com.rometools.modules.fyyd.modules.FyydModuleImpl.class,
com.rometools.modules.psc.modules.PodloveSimpleChapterModuleImpl.class, com.rometools.modules.thr.ThreadingModuleImpl.class,
com.rometools.modules.sse.modules.Sync.class, com.rometools.modules.sle.SimpleListExtensionImpl.class,
com.rometools.modules.slash.SlashImpl.class, com.rometools.modules.sse.modules.History.class,
com.rometools.modules.georss.GMLModuleImpl.class, com.rometools.modules.base.CustomTagsImpl.class,
com.rometools.modules.base.GoogleBaseImpl.class, com.rometools.modules.sle.SleEntryImpl.class,
com.rometools.modules.mediarss.MediaEntryModuleImpl.class, com.rometools.modules.content.ContentModuleImpl.class,
com.rometools.modules.georss.W3CGeoModuleImpl.class, com.rometools.rome.feed.module.DCModuleImpl.class,
com.rometools.modules.mediarss.MediaModuleImpl.class, com.rometools.rome.feed.module.SyModuleImpl.class,
// extracted from all 3 rome.properties files of rome library
com.rometools.rome.io.impl.RSS090Parser.class, com.rometools.rome.io.impl.RSS091NetscapeParser.class,
com.rometools.rome.io.impl.RSS091UserlandParser.class, com.rometools.rome.io.impl.RSS092Parser.class,
com.rometools.rome.io.impl.RSS093Parser.class, com.rometools.rome.io.impl.RSS094Parser.class,
com.rometools.rome.io.impl.RSS10Parser.class, com.rometools.rome.io.impl.RSS20wNSParser.class,
com.rometools.rome.io.impl.RSS20Parser.class, com.rometools.rome.io.impl.Atom10Parser.class,
com.rometools.rome.io.impl.Atom03Parser.class,
com.rometools.rome.io.impl.SyModuleParser.class, com.rometools.rome.io.impl.DCModuleParser.class,
com.rometools.rome.io.impl.RSS090Generator.class, com.rometools.rome.io.impl.RSS091NetscapeGenerator.class,
com.rometools.rome.io.impl.RSS091UserlandGenerator.class, com.rometools.rome.io.impl.RSS092Generator.class,
com.rometools.rome.io.impl.RSS093Generator.class, com.rometools.rome.io.impl.RSS094Generator.class,
com.rometools.rome.io.impl.RSS10Generator.class, com.rometools.rome.io.impl.RSS20Generator.class,
com.rometools.rome.io.impl.Atom10Generator.class, com.rometools.rome.io.impl.Atom03Generator.class,
com.rometools.rome.feed.synd.impl.ConverterForAtom10.class, com.rometools.rome.feed.synd.impl.ConverterForAtom03.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS090.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Netscape.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Userland.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS092.class, com.rometools.rome.feed.synd.impl.ConverterForRSS093.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS094.class, com.rometools.rome.feed.synd.impl.ConverterForRSS10.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS20.class,
com.rometools.modules.mediarss.io.RSS20YahooParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.feedpress.io.FeedpressParser.class, com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.content.io.ContentModuleParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.atom.io.AtomModuleParser.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ItemParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.thr.io.ThreadingModuleParser.class, com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.sle.io.ModuleGenerator.class, com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class, com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class, com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class, com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.thr.io.ThreadingModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.opml.io.impl.OPML10Generator.class, com.rometools.opml.io.impl.OPML20Generator.class,
com.rometools.opml.io.impl.OPML10Parser.class, com.rometools.opml.io.impl.OPML20Parser.class,
com.rometools.opml.feed.synd.impl.ConverterForOPML10.class, com.rometools.opml.feed.synd.impl.ConverterForOPML20.class, })
public class NativeImageClasses {
}
package com.commafeed;
import com.codahale.metrics.Counter;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Histogram;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import io.quarkus.runtime.annotations.RegisterForReflection;
@RegisterForReflection(
targets = {
// metrics
MetricRegistry.class, Meter.class, Gauge.class, Counter.class, Timer.class, Histogram.class,
// rome
java.util.Date.class, com.rometools.opml.feed.synd.impl.TreeCategoryImpl.class,
com.rometools.rome.feed.synd.SyndFeedImpl.class, com.rometools.rome.feed.module.DCSubjectImpl.class,
com.rometools.rome.feed.synd.SyndEntryImpl.class, com.rometools.modules.psc.types.SimpleChapter.class,
com.rometools.rome.feed.synd.SyndCategoryImpl.class, com.rometools.rome.feed.synd.SyndImageImpl.class,
com.rometools.rome.feed.synd.SyndContentImpl.class, com.rometools.rome.feed.synd.SyndEnclosureImpl.class,
// rome cloneable
com.rometools.modules.activitystreams.types.Article.class, com.rometools.modules.activitystreams.types.Audio.class,
com.rometools.modules.activitystreams.types.Bookmark.class, com.rometools.modules.activitystreams.types.Comment.class,
com.rometools.modules.activitystreams.types.Event.class, com.rometools.modules.activitystreams.types.File.class,
com.rometools.modules.activitystreams.types.Folder.class, com.rometools.modules.activitystreams.types.List.class,
com.rometools.modules.activitystreams.types.Note.class, com.rometools.modules.activitystreams.types.Person.class,
com.rometools.modules.activitystreams.types.Photo.class, com.rometools.modules.activitystreams.types.PhotoAlbum.class,
com.rometools.modules.activitystreams.types.Place.class, com.rometools.modules.activitystreams.types.Playlist.class,
com.rometools.modules.activitystreams.types.Product.class, com.rometools.modules.activitystreams.types.Review.class,
com.rometools.modules.activitystreams.types.Service.class, com.rometools.modules.activitystreams.types.Song.class,
com.rometools.modules.activitystreams.types.Status.class, com.rometools.modules.base.types.DateTimeRange.class,
com.rometools.modules.base.types.FloatUnit.class, com.rometools.modules.base.types.GenderEnumeration.class,
com.rometools.modules.base.types.IntUnit.class, com.rometools.modules.base.types.PriceTypeEnumeration.class,
com.rometools.modules.base.types.ShippingType.class, com.rometools.modules.base.types.ShortDate.class,
com.rometools.modules.base.types.Size.class, com.rometools.modules.base.types.YearType.class,
com.rometools.modules.content.ContentItem.class, com.rometools.modules.georss.GeoRSSPoint.class,
com.rometools.modules.georss.geometries.Envelope.class, com.rometools.modules.georss.geometries.LineString.class,
com.rometools.modules.georss.geometries.LinearRing.class, com.rometools.modules.georss.geometries.Point.class,
com.rometools.modules.georss.geometries.Polygon.class, com.rometools.modules.georss.geometries.Position.class,
com.rometools.modules.georss.geometries.PositionList.class, com.rometools.modules.mediarss.types.MediaGroup.class,
com.rometools.modules.mediarss.types.Metadata.class, com.rometools.modules.mediarss.types.Thumbnail.class,
com.rometools.modules.opensearch.entity.OSQuery.class, com.rometools.modules.photocast.types.PhotoDate.class,
com.rometools.modules.sle.types.DateValue.class, com.rometools.modules.sle.types.Group.class,
com.rometools.modules.sle.types.NumberValue.class, com.rometools.modules.sle.types.Sort.class,
com.rometools.modules.sle.types.StringValue.class, com.rometools.modules.yahooweather.types.Astronomy.class,
com.rometools.modules.yahooweather.types.Atmosphere.class, com.rometools.modules.yahooweather.types.Condition.class,
com.rometools.modules.yahooweather.types.Forecast.class, com.rometools.modules.yahooweather.types.Location.class,
com.rometools.modules.yahooweather.types.Units.class, com.rometools.modules.yahooweather.types.Wind.class,
com.rometools.opml.feed.opml.Attribute.class, com.rometools.opml.feed.opml.Opml.class,
com.rometools.opml.feed.opml.Outline.class, com.rometools.rome.feed.atom.Category.class,
com.rometools.rome.feed.atom.Content.class, com.rometools.rome.feed.atom.Entry.class,
com.rometools.rome.feed.atom.Feed.class, com.rometools.rome.feed.atom.Generator.class,
com.rometools.rome.feed.atom.Link.class, com.rometools.rome.feed.atom.Person.class,
com.rometools.rome.feed.rss.Category.class, com.rometools.rome.feed.rss.Channel.class,
com.rometools.rome.feed.rss.Cloud.class, com.rometools.rome.feed.rss.Content.class,
com.rometools.rome.feed.rss.Description.class, com.rometools.rome.feed.rss.Enclosure.class,
com.rometools.rome.feed.rss.Guid.class, com.rometools.rome.feed.rss.Image.class, com.rometools.rome.feed.rss.Item.class,
com.rometools.rome.feed.rss.Source.class, com.rometools.rome.feed.rss.TextInput.class,
com.rometools.rome.feed.synd.SyndLinkImpl.class, com.rometools.rome.feed.synd.SyndPersonImpl.class,
java.util.ArrayList.class,
// rome modules
com.rometools.modules.sse.modules.Conflict.class, com.rometools.modules.sse.modules.Conflicts.class,
com.rometools.modules.cc.CreativeCommonsImpl.class, com.rometools.modules.feedpress.modules.FeedpressModuleImpl.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleImpl.class, com.rometools.modules.sse.modules.Sharing.class,
com.rometools.modules.georss.SimpleModuleImpl.class, com.rometools.modules.atom.modules.AtomLinkModuleImpl.class,
com.rometools.modules.itunes.EntryInformationImpl.class, com.rometools.modules.sse.modules.Update.class,
com.rometools.modules.photocast.PhotocastModuleImpl.class, com.rometools.modules.itunes.FeedInformationImpl.class,
com.rometools.modules.yahooweather.YWeatherModuleImpl.class, com.rometools.modules.feedburner.FeedBurnerImpl.class,
com.rometools.modules.sse.modules.Related.class, com.rometools.modules.fyyd.modules.FyydModuleImpl.class,
com.rometools.modules.psc.modules.PodloveSimpleChapterModuleImpl.class, com.rometools.modules.thr.ThreadingModuleImpl.class,
com.rometools.modules.sse.modules.Sync.class, com.rometools.modules.sle.SimpleListExtensionImpl.class,
com.rometools.modules.slash.SlashImpl.class, com.rometools.modules.sse.modules.History.class,
com.rometools.modules.georss.GMLModuleImpl.class, com.rometools.modules.base.CustomTagsImpl.class,
com.rometools.modules.base.GoogleBaseImpl.class, com.rometools.modules.sle.SleEntryImpl.class,
com.rometools.modules.mediarss.MediaEntryModuleImpl.class, com.rometools.modules.content.ContentModuleImpl.class,
com.rometools.modules.georss.W3CGeoModuleImpl.class, com.rometools.rome.feed.module.DCModuleImpl.class,
com.rometools.modules.mediarss.MediaModuleImpl.class, com.rometools.rome.feed.module.SyModuleImpl.class,
// extracted from all 3 rome.properties files of rome library
com.rometools.rome.io.impl.RSS090Parser.class, com.rometools.rome.io.impl.RSS091NetscapeParser.class,
com.rometools.rome.io.impl.RSS091UserlandParser.class, com.rometools.rome.io.impl.RSS092Parser.class,
com.rometools.rome.io.impl.RSS093Parser.class, com.rometools.rome.io.impl.RSS094Parser.class,
com.rometools.rome.io.impl.RSS10Parser.class, com.rometools.rome.io.impl.RSS20wNSParser.class,
com.rometools.rome.io.impl.RSS20Parser.class, com.rometools.rome.io.impl.Atom10Parser.class,
com.rometools.rome.io.impl.Atom03Parser.class,
com.rometools.rome.io.impl.SyModuleParser.class, com.rometools.rome.io.impl.DCModuleParser.class,
com.rometools.rome.io.impl.RSS090Generator.class, com.rometools.rome.io.impl.RSS091NetscapeGenerator.class,
com.rometools.rome.io.impl.RSS091UserlandGenerator.class, com.rometools.rome.io.impl.RSS092Generator.class,
com.rometools.rome.io.impl.RSS093Generator.class, com.rometools.rome.io.impl.RSS094Generator.class,
com.rometools.rome.io.impl.RSS10Generator.class, com.rometools.rome.io.impl.RSS20Generator.class,
com.rometools.rome.io.impl.Atom10Generator.class, com.rometools.rome.io.impl.Atom03Generator.class,
com.rometools.rome.feed.synd.impl.ConverterForAtom10.class, com.rometools.rome.feed.synd.impl.ConverterForAtom03.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS090.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Netscape.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS091Userland.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS092.class, com.rometools.rome.feed.synd.impl.ConverterForRSS093.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS094.class, com.rometools.rome.feed.synd.impl.ConverterForRSS10.class,
com.rometools.rome.feed.synd.impl.ConverterForRSS20.class,
com.rometools.modules.mediarss.io.RSS20YahooParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class, com.rometools.modules.atom.io.AtomModuleParser.class,
com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ModuleParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class, com.rometools.modules.feedpress.io.FeedpressParser.class,
com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.feedpress.io.FeedpressParser.class, com.rometools.modules.fyyd.io.FyydParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.content.io.ContentModuleParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.itunes.io.ITunesParser.class, com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.atom.io.AtomModuleParser.class, com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class,
com.rometools.modules.georss.SimpleParser.class, com.rometools.modules.georss.W3CGeoParser.class,
com.rometools.modules.photocast.io.Parser.class, com.rometools.modules.itunes.io.ITunesParserOldNamespace.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class, com.rometools.modules.sle.io.ItemParser.class,
com.rometools.modules.yahooweather.io.WeatherModuleParser.class,
com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.ModuleParserRSS1.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.content.io.ContentModuleParser.class,
com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.cc.io.ModuleParserRSS2.class, com.rometools.modules.base.io.GoogleBaseParser.class,
com.rometools.modules.base.io.CustomTagParser.class, com.rometools.modules.slash.io.SlashModuleParser.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleParser.class, com.rometools.modules.georss.SimpleParser.class,
com.rometools.modules.georss.W3CGeoParser.class, com.rometools.modules.photocast.io.Parser.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.AlternateMediaModuleParser.class,
com.rometools.modules.thr.io.ThreadingModuleParser.class, com.rometools.modules.psc.io.PodloveSimpleChapterParser.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.sle.io.ModuleGenerator.class, com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class, com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class,
com.rometools.modules.georss.SimpleGenerator.class, com.rometools.modules.georss.W3CGeoGenerator.class,
com.rometools.modules.photocast.io.Generator.class, com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.feedpress.io.FeedpressGenerator.class, com.rometools.modules.fyyd.io.FyydGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class, com.rometools.modules.itunes.io.ITunesGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.atom.io.AtomModuleGenerator.class,
com.rometools.modules.yahooweather.io.WeatherModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.base.io.GoogleBaseGenerator.class, com.rometools.modules.content.io.ContentModuleGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.GoogleBaseGenerator.class,
com.rometools.modules.base.io.CustomTagGenerator.class, com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.modules.cc.io.CCModuleGenerator.class, com.rometools.modules.base.io.CustomTagGenerator.class,
com.rometools.modules.slash.io.SlashModuleGenerator.class,
com.rometools.modules.opensearch.impl.OpenSearchModuleGenerator.class, com.rometools.modules.georss.SimpleGenerator.class,
com.rometools.modules.georss.W3CGeoGenerator.class, com.rometools.modules.photocast.io.Generator.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class, com.rometools.modules.thr.io.ThreadingModuleGenerator.class,
com.rometools.modules.psc.io.PodloveSimpleChapterGenerator.class,
com.rometools.modules.mediarss.io.MediaModuleParser.class,
com.rometools.modules.mediarss.io.MediaModuleGenerator.class,
com.rometools.opml.io.impl.OPML10Generator.class, com.rometools.opml.io.impl.OPML20Generator.class,
com.rometools.opml.io.impl.OPML10Parser.class, com.rometools.opml.io.impl.OPML20Parser.class,
com.rometools.opml.feed.synd.impl.ConverterForOPML10.class, com.rometools.opml.feed.synd.impl.ConverterForOPML20.class, })
public class NativeImageClasses {
}

58
commafeed-server/src/main/java/com/commafeed/backend/Digests.java
View File

@@ -1,29 +1,29 @@
package com.commafeed.backend;
import java.nio.charset.StandardCharsets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import lombok.experimental.UtilityClass;
@UtilityClass
@SuppressWarnings("deprecation")
public class Digests {
public static String sha1Hex(byte[] input) {
return hashBytesToHex(Hashing.sha1(), input);
}
public static String sha1Hex(String input) {
return hashBytesToHex(Hashing.sha1(), input.getBytes(StandardCharsets.UTF_8));
}
public static String md5Hex(String input) {
return hashBytesToHex(Hashing.md5(), input.getBytes(StandardCharsets.UTF_8));
}
private static String hashBytesToHex(HashFunction function, byte[] input) {
return function.hashBytes(input).toString();
}
}
package com.commafeed.backend;
import java.nio.charset.StandardCharsets;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import lombok.experimental.UtilityClass;
@UtilityClass
@SuppressWarnings("deprecation")
public class Digests {
public static String sha1Hex(byte[] input) {
return hashBytesToHex(Hashing.sha1(), input);
}
public static String sha1Hex(String input) {
return hashBytesToHex(Hashing.sha1(), input.getBytes(StandardCharsets.UTF_8));
}
public static String md5Hex(String input) {
return hashBytesToHex(Hashing.md5(), input.getBytes(StandardCharsets.UTF_8));
}
private static String hashBytesToHex(HashFunction function, byte[] input) {
return function.hashBytes(input).toString();
}
}

856
commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java
View File

@@ -1,428 +1,428 @@
package com.commafeed.backend;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.stream.Stream;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.CacheControl;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.DnsResolver;
import org.apache.hc.client5.http.SystemDefaultDnsResolver;
import org.apache.hc.client5.http.config.ConnectionConfig;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.config.TlsConfig;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.io.HttpClientConnectionManager;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.client5.http.protocol.RedirectLocations;
import org.apache.hc.client5.http.utils.DateUtils;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.http.message.BasicHeader;
import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
import org.jboss.resteasy.reactive.common.headers.CacheControlDelegate;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.HttpClientCache;
import com.commafeed.CommaFeedVersion;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Iterables;
import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.apache5.util.Apache5SslUtils;
/**
* Smart HTTP getter: handles gzip, ssl, last modified and etag headers
*/
@Singleton
@Slf4j
public class HttpGetter {
private final CommaFeedConfiguration config;
private final InstantSource instantSource;
private final CloseableHttpClient client;
private final Cache<HttpRequest, HttpResponse> cache;
private final DnsResolver dnsResolver = SystemDefaultDnsResolver.INSTANCE;
public HttpGetter(CommaFeedConfiguration config, InstantSource instantSource, CommaFeedVersion version, MetricRegistry metrics) {
this.config = config;
this.instantSource = instantSource;
PoolingHttpClientConnectionManager connectionManager = newConnectionManager(config);
String userAgent = config.httpClient()
.userAgent()
.orElseGet(() -> String.format("CommaFeed/%s (https://github.com/Athou/commafeed)", version.getVersion()));
this.client = newClient(connectionManager, userAgent, config.httpClient().idleConnectionsEvictionInterval());
this.cache = newCache(config);
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "max"), () -> connectionManager.getTotalStats().getMax());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "size"),
() -> connectionManager.getTotalStats().getAvailable() + connectionManager.getTotalStats().getLeased());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "leased"), () -> connectionManager.getTotalStats().getLeased());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "pending"), () -> connectionManager.getTotalStats().getPending());
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "size"), () -> cache == null ? 0 : cache.size());
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "memoryUsage"),
() -> cache == null ? 0 : cache.asMap().values().stream().mapToInt(e -> e.content != null ? e.content.length : 0).sum());
}
public HttpResult get(String url)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException {
return get(HttpRequest.builder(url).build());
}
public HttpResult get(HttpRequest request)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException {
URI uri = URI.create(request.getUrl());
ensureHttpScheme(uri.getScheme());
if (config.httpClient().blockLocalAddresses()) {
ensurePublicAddress(uri.getHost());
}
final HttpResponse response;
if (cache == null) {
response = invoke(request);
} else {
try {
response = cache.get(request, () -> invoke(request));
} catch (ExecutionException e) {
if (e.getCause() instanceof IOException ioe) {
throw ioe;
} else {
throw new RuntimeException(e);
}
}
}
int code = response.getCode();
if (code == HttpStatus.SC_TOO_MANY_REQUESTS || code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.getRetryAfter() != null) {
throw new TooManyRequestsException(response.getRetryAfter());
}
if (code == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received");
}
if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);
}
String lastModifiedHeader = response.getLastModifiedHeader();
if (lastModifiedHeader != null && lastModifiedHeader.equals(request.getLastModified())) {
throw new NotModifiedException("lastModifiedHeader is the same");
}
String eTagHeader = response.getETagHeader();
if (eTagHeader != null && eTagHeader.equals(request.getETag())) {
throw new NotModifiedException("eTagHeader is the same");
}
Duration validFor = Optional.ofNullable(response.getCacheControl())
.filter(cc -> cc.getMaxAge() >= 0)
.map(cc -> Duration.ofSeconds(cc.getMaxAge()))
.orElse(Duration.ZERO);
return new HttpResult(response.getContent(), response.getContentType(), lastModifiedHeader, eTagHeader,
response.getUrlAfterRedirect(), validFor);
}
private void ensureHttpScheme(String scheme) throws SchemeNotAllowedException {
if (!"http".equals(scheme) && !"https".equals(scheme)) {
throw new SchemeNotAllowedException(scheme);
}
}
private void ensurePublicAddress(String host) throws HostNotAllowedException, UnknownHostException {
if (host == null) {
throw new HostNotAllowedException(null);
}
InetAddress[] addresses = dnsResolver.resolve(host);
if (Stream.of(addresses).anyMatch(this::isPrivateAddress)) {
throw new HostNotAllowedException(host);
}
}
private boolean isPrivateAddress(InetAddress address) {
return address.isSiteLocalAddress() || address.isAnyLocalAddress() || address.isLinkLocalAddress() || address.isLoopbackAddress()
|| address.isMulticastAddress();
}
private HttpResponse invoke(HttpRequest request) throws IOException {
log.debug("fetching {}", request.getUrl());
HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
// causes issues with some feeds
// see https://github.com/Athou/commafeed/issues/1572
// and https://issues.apache.org/jira/browse/HTTPCLIENT-2344
.setProtocolUpgradeEnabled(false)
.build());
return client.execute(request.toClassicHttpRequest(), context, resp -> {
byte[] content = resp.getEntity() == null ? null
: toByteArray(resp.getEntity(), config.httpClient().maxResponseSize().asLongValue());
int code = resp.getCode();
String lastModifiedHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.LAST_MODIFIED))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
String eTagHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.ETAG))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
CacheControl cacheControl = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.CACHE_CONTROL))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(HttpGetter::toCacheControl)
.orElse(null);
Instant retryAfter = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(this::toInstant)
.orElse(null);
String contentType = Optional.ofNullable(resp.getEntity()).map(HttpEntity::getContentType).orElse(null);
String urlAfterRedirect = Optional.ofNullable(context.getRedirectLocations())
.map(RedirectLocations::getAll)
.map(l -> Iterables.getLast(l, null))
.map(URI::toString)
.orElse(request.getUrl());
return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, retryAfter, content, contentType, urlAfterRedirect);
});
}
private static CacheControl toCacheControl(String headerValue) {
try {
return CacheControlDelegate.INSTANCE.fromString(headerValue);
} catch (Exception e) {
log.debug("Invalid Cache-Control header: {}", headerValue);
return null;
}
}
private Instant toInstant(String headerValue) {
if (headerValue == null) {
return null;
}
if (StringUtils.isNumeric(headerValue)) {
return instantSource.instant().plusSeconds(Long.parseLong(headerValue));
}
return DateUtils.parseStandardDate(headerValue);
}
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
if (entity.getContentLength() > maxBytes) {
throw new IOException(
"Response size (%s bytes) exceeds the maximum allowed size (%s bytes)".formatted(entity.getContentLength(), maxBytes));
}
try (InputStream input = entity.getContent()) {
if (input == null) {
return null;
}
byte[] bytes = ByteStreams.limit(input, maxBytes).readAllBytes();
if (bytes.length == maxBytes) {
throw new IOException("Response size exceeds the maximum allowed size (%s bytes)".formatted(maxBytes));
}
return bytes;
}
}
private PoolingHttpClientConnectionManager newConnectionManager(CommaFeedConfiguration config) {
SSLFactory sslFactory = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
int poolSize = config.feedRefresh().httpThreads();
return PoolingHttpClientConnectionManagerBuilder.create()
.setTlsSocketStrategy(Apache5SslUtils.toTlsSocketStrategy(sslFactory))
.setDefaultConnectionConfig(ConnectionConfig.custom()
.setConnectTimeout(Timeout.of(config.httpClient().connectTimeout()))
.setSocketTimeout(Timeout.of(config.httpClient().socketTimeout()))
.setTimeToLive(Timeout.of(config.httpClient().connectionTimeToLive()))
.build())
.setDefaultTlsConfig(TlsConfig.custom().setHandshakeTimeout(Timeout.of(config.httpClient().sslHandshakeTimeout())).build())
.setMaxConnPerRoute(poolSize)
.setMaxConnTotal(poolSize)
.setDnsResolver(dnsResolver)
.build();
}
private static CloseableHttpClient newClient(HttpClientConnectionManager connectionManager, String userAgent,
Duration idleConnectionsEvictionInterval) {
List<Header> headers = new ArrayList<>();
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
return HttpClientBuilder.create()
.useSystemProperties()
.disableAutomaticRetries()
.disableCookieManagement()
.setUserAgent(userAgent)
.setDefaultHeaders(headers)
.setConnectionManager(connectionManager)
.evictExpiredConnections()
.evictIdleConnections(TimeValue.of(idleConnectionsEvictionInterval))
.build();
}
private static Cache<HttpRequest, HttpResponse> newCache(CommaFeedConfiguration config) {
HttpClientCache cacheConfig = config.httpClient().cache();
if (!cacheConfig.enabled()) {
return null;
}
return CacheBuilder.newBuilder()
.weigher((HttpRequest key, HttpResponse value) -> value.getContent() != null ? value.getContent().length : 0)
.maximumWeight(cacheConfig.maximumMemorySize().asLongValue())
.expireAfterWrite(cacheConfig.expiration())
.build();
}
public static class SchemeNotAllowedException extends Exception {
private static final long serialVersionUID = 1L;
public SchemeNotAllowedException(String scheme) {
super("Scheme not allowed: " + scheme);
}
}
public static class HostNotAllowedException extends Exception {
private static final long serialVersionUID = 1L;
public HostNotAllowedException(String host) {
super("Host not allowed: " + host);
}
}
@Getter
public static class NotModifiedException extends Exception {
private static final long serialVersionUID = 1L;
/**
* if the value of this header changed, this is its new value
*/
private final String newLastModifiedHeader;
/**
* if the value of this header changed, this is its new value
*/
private final String newEtagHeader;
public NotModifiedException(String message) {
this(message, null, null);
}
public NotModifiedException(String message, String newLastModifiedHeader, String newEtagHeader) {
super(message);
this.newLastModifiedHeader = newLastModifiedHeader;
this.newEtagHeader = newEtagHeader;
}
}
@RequiredArgsConstructor
@Getter
public static class TooManyRequestsException extends Exception {
private static final long serialVersionUID = 1L;
private final Instant retryAfter;
}
@Getter
public static class HttpResponseException extends IOException {
private static final long serialVersionUID = 1L;
private final int code;
public HttpResponseException(int code, String message) {
super(message);
this.code = code;
}
}
@Builder(builderMethodName = "")
@EqualsAndHashCode
@Getter
public static class HttpRequest {
private String url;
private String lastModified;
private String eTag;
public static HttpRequestBuilder builder(String url) {
return new HttpRequestBuilder().url(url);
}
public ClassicHttpRequest toClassicHttpRequest() {
ClassicHttpRequest req = ClassicRequestBuilder.get(url).build();
if (lastModified != null) {
req.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
}
if (eTag != null) {
req.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
}
return req;
}
}
@Value
private static class HttpResponse {
int code;
String lastModifiedHeader;
String eTagHeader;
CacheControl cacheControl;
Instant retryAfter;
byte[] content;
String contentType;
String urlAfterRedirect;
}
@Value
public static class HttpResult {
byte[] content;
String contentType;
String lastModifiedSince;
String eTag;
String urlAfterRedirect;
Duration validFor;
}
}
package com.commafeed.backend;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.URI;
import java.net.UnknownHostException;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.stream.Stream;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.CacheControl;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.DnsResolver;
import org.apache.hc.client5.http.SystemDefaultDnsResolver;
import org.apache.hc.client5.http.config.ConnectionConfig;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.config.TlsConfig;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder;
import org.apache.hc.client5.http.io.HttpClientConnectionManager;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.client5.http.protocol.RedirectLocations;
import org.apache.hc.client5.http.utils.DateUtils;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
import org.apache.hc.core5.http.message.BasicHeader;
import org.apache.hc.core5.util.TimeValue;
import org.apache.hc.core5.util.Timeout;
import org.jboss.resteasy.reactive.common.headers.CacheControlDelegate;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.HttpClientCache;
import com.commafeed.CommaFeedVersion;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Iterables;
import com.google.common.io.ByteStreams;
import com.google.common.net.HttpHeaders;
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Value;
import lombok.extern.slf4j.Slf4j;
import nl.altindag.ssl.SSLFactory;
import nl.altindag.ssl.apache5.util.Apache5SslUtils;
/**
* Smart HTTP getter: handles gzip, ssl, last modified and etag headers
*/
@Singleton
@Slf4j
public class HttpGetter {
private final CommaFeedConfiguration config;
private final InstantSource instantSource;
private final CloseableHttpClient client;
private final Cache<HttpRequest, HttpResponse> cache;
private final DnsResolver dnsResolver = SystemDefaultDnsResolver.INSTANCE;
public HttpGetter(CommaFeedConfiguration config, InstantSource instantSource, CommaFeedVersion version, MetricRegistry metrics) {
this.config = config;
this.instantSource = instantSource;
PoolingHttpClientConnectionManager connectionManager = newConnectionManager(config);
String userAgent = config.httpClient()
.userAgent()
.orElseGet(() -> String.format("CommaFeed/%s (https://github.com/Athou/commafeed)", version.getVersion()));
this.client = newClient(connectionManager, userAgent, config.httpClient().idleConnectionsEvictionInterval());
this.cache = newCache(config);
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "max"), () -> connectionManager.getTotalStats().getMax());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "size"),
() -> connectionManager.getTotalStats().getAvailable() + connectionManager.getTotalStats().getLeased());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "leased"), () -> connectionManager.getTotalStats().getLeased());
metrics.registerGauge(MetricRegistry.name(getClass(), "pool", "pending"), () -> connectionManager.getTotalStats().getPending());
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "size"), () -> cache == null ? 0 : cache.size());
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "memoryUsage"),
() -> cache == null ? 0 : cache.asMap().values().stream().mapToInt(e -> e.content != null ? e.content.length : 0).sum());
}
public HttpResult get(String url)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException {
return get(HttpRequest.builder(url).build());
}
public HttpResult get(HttpRequest request)
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException {
URI uri = URI.create(request.getUrl());
ensureHttpScheme(uri.getScheme());
if (config.httpClient().blockLocalAddresses()) {
ensurePublicAddress(uri.getHost());
}
final HttpResponse response;
if (cache == null) {
response = invoke(request);
} else {
try {
response = cache.get(request, () -> invoke(request));
} catch (ExecutionException e) {
if (e.getCause() instanceof IOException ioe) {
throw ioe;
} else {
throw new RuntimeException(e);
}
}
}
int code = response.getCode();
if (code == HttpStatus.SC_TOO_MANY_REQUESTS || code == HttpStatus.SC_SERVICE_UNAVAILABLE && response.getRetryAfter() != null) {
throw new TooManyRequestsException(response.getRetryAfter());
}
if (code == HttpStatus.SC_NOT_MODIFIED) {
throw new NotModifiedException("'304 - not modified' http code received");
}
if (code >= 300) {
throw new HttpResponseException(code, "Server returned HTTP error code " + code);
}
String lastModifiedHeader = response.getLastModifiedHeader();
if (lastModifiedHeader != null && lastModifiedHeader.equals(request.getLastModified())) {
throw new NotModifiedException("lastModifiedHeader is the same");
}
String eTagHeader = response.getETagHeader();
if (eTagHeader != null && eTagHeader.equals(request.getETag())) {
throw new NotModifiedException("eTagHeader is the same");
}
Duration validFor = Optional.ofNullable(response.getCacheControl())
.filter(cc -> cc.getMaxAge() >= 0)
.map(cc -> Duration.ofSeconds(cc.getMaxAge()))
.orElse(Duration.ZERO);
return new HttpResult(response.getContent(), response.getContentType(), lastModifiedHeader, eTagHeader,
response.getUrlAfterRedirect(), validFor);
}
private void ensureHttpScheme(String scheme) throws SchemeNotAllowedException {
if (!"http".equals(scheme) && !"https".equals(scheme)) {
throw new SchemeNotAllowedException(scheme);
}
}
private void ensurePublicAddress(String host) throws HostNotAllowedException, UnknownHostException {
if (host == null) {
throw new HostNotAllowedException(null);
}
InetAddress[] addresses = dnsResolver.resolve(host);
if (Stream.of(addresses).anyMatch(this::isPrivateAddress)) {
throw new HostNotAllowedException(host);
}
}
private boolean isPrivateAddress(InetAddress address) {
return address.isSiteLocalAddress() || address.isAnyLocalAddress() || address.isLinkLocalAddress() || address.isLoopbackAddress()
|| address.isMulticastAddress();
}
private HttpResponse invoke(HttpRequest request) throws IOException {
log.debug("fetching {}", request.getUrl());
HttpClientContext context = HttpClientContext.create();
context.setRequestConfig(RequestConfig.custom()
.setResponseTimeout(Timeout.of(config.httpClient().responseTimeout()))
// causes issues with some feeds
// see https://github.com/Athou/commafeed/issues/1572
// and https://issues.apache.org/jira/browse/HTTPCLIENT-2344
.setProtocolUpgradeEnabled(false)
.build());
return client.execute(request.toClassicHttpRequest(), context, resp -> {
byte[] content = resp.getEntity() == null ? null
: toByteArray(resp.getEntity(), config.httpClient().maxResponseSize().asLongValue());
int code = resp.getCode();
String lastModifiedHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.LAST_MODIFIED))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
String eTagHeader = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.ETAG))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.orElse(null);
CacheControl cacheControl = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.CACHE_CONTROL))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(HttpGetter::toCacheControl)
.orElse(null);
Instant retryAfter = Optional.ofNullable(resp.getFirstHeader(HttpHeaders.RETRY_AFTER))
.map(NameValuePair::getValue)
.map(StringUtils::trimToNull)
.map(this::toInstant)
.orElse(null);
String contentType = Optional.ofNullable(resp.getEntity()).map(HttpEntity::getContentType).orElse(null);
String urlAfterRedirect = Optional.ofNullable(context.getRedirectLocations())
.map(RedirectLocations::getAll)
.map(l -> Iterables.getLast(l, null))
.map(URI::toString)
.orElse(request.getUrl());
return new HttpResponse(code, lastModifiedHeader, eTagHeader, cacheControl, retryAfter, content, contentType, urlAfterRedirect);
});
}
private static CacheControl toCacheControl(String headerValue) {
try {
return CacheControlDelegate.INSTANCE.fromString(headerValue);
} catch (Exception e) {
log.debug("Invalid Cache-Control header: {}", headerValue);
return null;
}
}
private Instant toInstant(String headerValue) {
if (headerValue == null) {
return null;
}
if (StringUtils.isNumeric(headerValue)) {
return instantSource.instant().plusSeconds(Long.parseLong(headerValue));
}
return DateUtils.parseStandardDate(headerValue);
}
private static byte[] toByteArray(HttpEntity entity, long maxBytes) throws IOException {
if (entity.getContentLength() > maxBytes) {
throw new IOException(
"Response size (%s bytes) exceeds the maximum allowed size (%s bytes)".formatted(entity.getContentLength(), maxBytes));
}
try (InputStream input = entity.getContent()) {
if (input == null) {
return null;
}
byte[] bytes = ByteStreams.limit(input, maxBytes).readAllBytes();
if (bytes.length == maxBytes) {
throw new IOException("Response size exceeds the maximum allowed size (%s bytes)".formatted(maxBytes));
}
return bytes;
}
}
private PoolingHttpClientConnectionManager newConnectionManager(CommaFeedConfiguration config) {
SSLFactory sslFactory = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
int poolSize = config.feedRefresh().httpThreads();
return PoolingHttpClientConnectionManagerBuilder.create()
.setTlsSocketStrategy(Apache5SslUtils.toTlsSocketStrategy(sslFactory))
.setDefaultConnectionConfig(ConnectionConfig.custom()
.setConnectTimeout(Timeout.of(config.httpClient().connectTimeout()))
.setSocketTimeout(Timeout.of(config.httpClient().socketTimeout()))
.setTimeToLive(Timeout.of(config.httpClient().connectionTimeToLive()))
.build())
.setDefaultTlsConfig(TlsConfig.custom().setHandshakeTimeout(Timeout.of(config.httpClient().sslHandshakeTimeout())).build())
.setMaxConnPerRoute(poolSize)
.setMaxConnTotal(poolSize)
.setDnsResolver(dnsResolver)
.build();
}
private static CloseableHttpClient newClient(HttpClientConnectionManager connectionManager, String userAgent,
Duration idleConnectionsEvictionInterval) {
List<Header> headers = new ArrayList<>();
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
headers.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "no-cache"));
return HttpClientBuilder.create()
.useSystemProperties()
.disableAutomaticRetries()
.disableCookieManagement()
.setUserAgent(userAgent)
.setDefaultHeaders(headers)
.setConnectionManager(connectionManager)
.evictExpiredConnections()
.evictIdleConnections(TimeValue.of(idleConnectionsEvictionInterval))
.build();
}
private static Cache<HttpRequest, HttpResponse> newCache(CommaFeedConfiguration config) {
HttpClientCache cacheConfig = config.httpClient().cache();
if (!cacheConfig.enabled()) {
return null;
}
return CacheBuilder.newBuilder()
.weigher((HttpRequest key, HttpResponse value) -> value.getContent() != null ? value.getContent().length : 0)
.maximumWeight(cacheConfig.maximumMemorySize().asLongValue())
.expireAfterWrite(cacheConfig.expiration())
.build();
}
public static class SchemeNotAllowedException extends Exception {
private static final long serialVersionUID = 1L;
public SchemeNotAllowedException(String scheme) {
super("Scheme not allowed: " + scheme);
}
}
public static class HostNotAllowedException extends Exception {
private static final long serialVersionUID = 1L;
public HostNotAllowedException(String host) {
super("Host not allowed: " + host);
}
}
@Getter
public static class NotModifiedException extends Exception {
private static final long serialVersionUID = 1L;
/**
* if the value of this header changed, this is its new value
*/
private final String newLastModifiedHeader;
/**
* if the value of this header changed, this is its new value
*/
private final String newEtagHeader;
public NotModifiedException(String message) {
this(message, null, null);
}
public NotModifiedException(String message, String newLastModifiedHeader, String newEtagHeader) {
super(message);
this.newLastModifiedHeader = newLastModifiedHeader;
this.newEtagHeader = newEtagHeader;
}
}
@RequiredArgsConstructor
@Getter
public static class TooManyRequestsException extends Exception {
private static final long serialVersionUID = 1L;
private final Instant retryAfter;
}
@Getter
public static class HttpResponseException extends IOException {
private static final long serialVersionUID = 1L;
private final int code;
public HttpResponseException(int code, String message) {
super(message);
this.code = code;
}
}
@Builder(builderMethodName = "")
@EqualsAndHashCode
@Getter
public static class HttpRequest {
private String url;
private String lastModified;
private String eTag;
public static HttpRequestBuilder builder(String url) {
return new HttpRequestBuilder().url(url);
}
public ClassicHttpRequest toClassicHttpRequest() {
ClassicHttpRequest req = ClassicRequestBuilder.get(url).build();
if (lastModified != null) {
req.addHeader(HttpHeaders.IF_MODIFIED_SINCE, lastModified);
}
if (eTag != null) {
req.addHeader(HttpHeaders.IF_NONE_MATCH, eTag);
}
return req;
}
}
@Value
private static class HttpResponse {
int code;
String lastModifiedHeader;
String eTagHeader;
CacheControl cacheControl;
Instant retryAfter;
byte[] content;
String contentType;
String urlAfterRedirect;
}
@Value
public static class HttpResult {
byte[] content;
String contentType;
String lastModifiedSince;
String eTag;
String urlAfterRedirect;
Duration validFor;
}
}

142
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java
View File

@@ -1,71 +1,71 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.QFeedCategory;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
import com.querydsl.core.types.Predicate;
@Singleton
public class FeedCategoryDAO extends GenericDAO<FeedCategory> {
private static final QFeedCategory CATEGORY = QFeedCategory.feedCategory;
public FeedCategoryDAO(EntityManager entityManager) {
super(entityManager, FeedCategory.class);
}
public List<FeedCategory> findAll(User user) {
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user)).join(CATEGORY.user, QUser.user).fetchJoin().fetch();
}
public FeedCategory findById(User user, Long id) {
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.id.eq(id)).fetchOne();
}
public FeedCategory findByName(User user, String name, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.name.eq(name), parentPredicate).fetchOne();
}
public List<FeedCategory> findByParent(User user, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), parentPredicate).fetch();
}
public List<FeedCategory> findAllChildrenCategories(User user, FeedCategory parent) {
return findAll(user).stream().filter(c -> isChild(c, parent)).toList();
}
private boolean isChild(FeedCategory child, FeedCategory parent) {
if (parent == null) {
return true;
}
boolean isChild = false;
while (child != null) {
if (Objects.equals(child.getId(), parent.getId())) {
isChild = true;
break;
}
child = child.getParent();
}
return isChild;
}
}
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.QFeedCategory;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
import com.querydsl.core.types.Predicate;
@Singleton
public class FeedCategoryDAO extends GenericDAO<FeedCategory> {
private static final QFeedCategory CATEGORY = QFeedCategory.feedCategory;
public FeedCategoryDAO(EntityManager entityManager) {
super(entityManager, FeedCategory.class);
}
public List<FeedCategory> findAll(User user) {
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user)).join(CATEGORY.user, QUser.user).fetchJoin().fetch();
}
public FeedCategory findById(User user, Long id) {
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.id.eq(id)).fetchOne();
}
public FeedCategory findByName(User user, String name, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.name.eq(name), parentPredicate).fetchOne();
}
public List<FeedCategory> findByParent(User user, FeedCategory parent) {
Predicate parentPredicate;
if (parent == null) {
parentPredicate = CATEGORY.parent.isNull();
} else {
parentPredicate = CATEGORY.parent.eq(parent);
}
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), parentPredicate).fetch();
}
public List<FeedCategory> findAllChildrenCategories(User user, FeedCategory parent) {
return findAll(user).stream().filter(c -> isChild(c, parent)).toList();
}
private boolean isChild(FeedCategory child, FeedCategory parent) {
if (parent == null) {
return true;
}
boolean isChild = false;
while (child != null) {
if (Objects.equals(child.getId(), parent.getId())) {
isChild = true;
break;
}
child = child.getParent();
}
return isChild;
}
}

128
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedDAO.java
View File

@@ -1,64 +1,64 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.QFeed;
import com.commafeed.backend.model.QFeedSubscription;
import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.impl.JPAQuery;
@Singleton
public class FeedDAO extends GenericDAO<Feed> {
private static final QFeed FEED = QFeed.feed;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
public FeedDAO(EntityManager entityManager) {
super(entityManager, Feed.class);
}
public List<Feed> findByIds(List<Long> id) {
return query().selectFrom(FEED).where(FEED.id.in(id)).fetch();
}
public List<Feed> findNextUpdatable(int count, Instant lastLoginThreshold) {
JPAQuery<Feed> query = query().selectFrom(FEED)
.distinct()
// join on subscriptions to only refresh feeds that have subscribers
.join(SUBSCRIPTION)
.on(SUBSCRIPTION.feed.eq(FEED))
.where(FEED.disabledUntil.isNull().or(FEED.disabledUntil.lt(Instant.now())));
if (lastLoginThreshold != null) {
query.join(SUBSCRIPTION.user).where(SUBSCRIPTION.user.lastLogin.gt(lastLoginThreshold));
}
return query.orderBy(FEED.disabledUntil.asc()).limit(count).fetch();
}
public void setDisabledUntil(List<Long> feedIds, Instant date) {
updateQuery(FEED).set(FEED.disabledUntil, date).where(FEED.id.in(feedIds)).execute();
}
public Feed findByUrl(String normalizedUrl, String normalizedUrlHash) {
return query().selectFrom(FEED)
.where(FEED.normalizedUrlHash.eq(normalizedUrlHash))
.fetch()
.stream()
.filter(f -> StringUtils.equals(normalizedUrl, f.getNormalizedUrl()))
.findFirst()
.orElse(null);
}
public List<Feed> findWithoutSubscriptions(int max) {
QFeedSubscription sub = QFeedSubscription.feedSubscription;
return query().selectFrom(FEED).where(JPAExpressions.selectOne().from(sub).where(sub.feed.eq(FEED)).notExists()).limit(max).fetch();
}
}
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.QFeed;
import com.commafeed.backend.model.QFeedSubscription;
import com.querydsl.jpa.JPAExpressions;
import com.querydsl.jpa.impl.JPAQuery;
@Singleton
public class FeedDAO extends GenericDAO<Feed> {
private static final QFeed FEED = QFeed.feed;
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
public FeedDAO(EntityManager entityManager) {
super(entityManager, Feed.class);
}
public List<Feed> findByIds(List<Long> id) {
return query().selectFrom(FEED).where(FEED.id.in(id)).fetch();
}
public List<Feed> findNextUpdatable(int count, Instant lastLoginThreshold) {
JPAQuery<Feed> query = query().selectFrom(FEED)
.distinct()
// join on subscriptions to only refresh feeds that have subscribers
.join(SUBSCRIPTION)
.on(SUBSCRIPTION.feed.eq(FEED))
.where(FEED.disabledUntil.isNull().or(FEED.disabledUntil.lt(Instant.now())));
if (lastLoginThreshold != null) {
query.join(SUBSCRIPTION.user).where(SUBSCRIPTION.user.lastLogin.gt(lastLoginThreshold));
}
return query.orderBy(FEED.disabledUntil.asc()).limit(count).fetch();
}
public void setDisabledUntil(List<Long> feedIds, Instant date) {
updateQuery(FEED).set(FEED.disabledUntil, date).where(FEED.id.in(feedIds)).execute();
}
public Feed findByUrl(String normalizedUrl, String normalizedUrlHash) {
return query().selectFrom(FEED)
.where(FEED.normalizedUrlHash.eq(normalizedUrlHash))
.fetch()
.stream()
.filter(f -> StringUtils.equals(normalizedUrl, f.getNormalizedUrl()))
.findFirst()
.orElse(null);
}
public List<Feed> findWithoutSubscriptions(int max) {
QFeedSubscription sub = QFeedSubscription.feedSubscription;
return query().selectFrom(FEED).where(JPAExpressions.selectOne().from(sub).where(sub.feed.eq(FEED)).notExists()).limit(max).fetch();
}
}

72
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryContentDAO.java
View File

@@ -1,36 +1,36 @@
package com.commafeed.backend.dao;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
@Singleton
public class FeedEntryContentDAO extends GenericDAO<FeedEntryContent> {
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
public FeedEntryContentDAO(EntityManager entityManager) {
super(entityManager, FeedEntryContent.class);
}
public List<FeedEntryContent> findExisting(String contentHash, String titleHash) {
return query().select(CONTENT).from(CONTENT).where(CONTENT.contentHash.eq(contentHash), CONTENT.titleHash.eq(titleHash)).fetch();
}
public long deleteWithoutEntries(int max) {
List<Long> ids = query().select(CONTENT.id)
.from(CONTENT)
.leftJoin(ENTRY)
.on(ENTRY.content.id.eq(CONTENT.id))
.where(ENTRY.id.isNull())
.limit(max)
.fetch();
return deleteQuery(CONTENT).where(CONTENT.id.in(ids)).execute();
}
}
package com.commafeed.backend.dao;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
@Singleton
public class FeedEntryContentDAO extends GenericDAO<FeedEntryContent> {
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
public FeedEntryContentDAO(EntityManager entityManager) {
super(entityManager, FeedEntryContent.class);
}
public List<FeedEntryContent> findExisting(String contentHash, String titleHash) {
return query().select(CONTENT).from(CONTENT).where(CONTENT.contentHash.eq(contentHash), CONTENT.titleHash.eq(titleHash)).fetch();
}
public long deleteWithoutEntries(int max) {
List<Long> ids = query().select(CONTENT.id)
.from(CONTENT)
.leftJoin(ENTRY)
.on(ENTRY.content.id.eq(CONTENT.id))
.where(ENTRY.id.isNull())
.limit(max)
.fetch();
return deleteQuery(CONTENT).where(CONTENT.id.in(ids)).execute();
}
}

146
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
View File

@@ -1,73 +1,73 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.QFeedEntry;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.NumberExpression;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Singleton
public class FeedEntryDAO extends GenericDAO<FeedEntry> {
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
public FeedEntryDAO(EntityManager entityManager) {
super(entityManager, FeedEntry.class);
}
public FeedEntry findExisting(String guidHash, Feed feed) {
return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne();
}
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max) {
NumberExpression<Long> count = ENTRY.id.count();
List<Tuple> tuples = query().select(ENTRY.feed.id, count)
.from(ENTRY)
.groupBy(ENTRY.feed)
.having(count.gt(maxCapacity))
.limit(max)
.fetch();
return tuples.stream().map(t -> new FeedCapacity(t.get(ENTRY.feed.id), t.get(count))).toList();
}
public int delete(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).limit(max).fetch();
return delete(list);
}
/**
* Delete entries older than a certain date
*/
public int deleteEntriesOlderThan(Instant olderThan, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY)
.where(ENTRY.published.lt(olderThan))
.orderBy(ENTRY.published.asc())
.limit(max)
.fetch();
return delete(list);
}
/**
* Delete the oldest entries of a feed
*/
public int deleteOldEntries(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).orderBy(ENTRY.published.asc()).limit(max).fetch();
return delete(list);
}
@AllArgsConstructor
@Getter
public static class FeedCapacity {
private Long id;
private Long capacity;
}
}
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.QFeedEntry;
import com.querydsl.core.Tuple;
import com.querydsl.core.types.dsl.NumberExpression;
import lombok.AllArgsConstructor;
import lombok.Getter;
@Singleton
public class FeedEntryDAO extends GenericDAO<FeedEntry> {
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
public FeedEntryDAO(EntityManager entityManager) {
super(entityManager, FeedEntry.class);
}
public FeedEntry findExisting(String guidHash, Feed feed) {
return query().select(ENTRY).from(ENTRY).where(ENTRY.guidHash.eq(guidHash), ENTRY.feed.eq(feed)).limit(1).fetchOne();
}
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max) {
NumberExpression<Long> count = ENTRY.id.count();
List<Tuple> tuples = query().select(ENTRY.feed.id, count)
.from(ENTRY)
.groupBy(ENTRY.feed)
.having(count.gt(maxCapacity))
.limit(max)
.fetch();
return tuples.stream().map(t -> new FeedCapacity(t.get(ENTRY.feed.id), t.get(count))).toList();
}
public int delete(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).limit(max).fetch();
return delete(list);
}
/**
* Delete entries older than a certain date
*/
public int deleteEntriesOlderThan(Instant olderThan, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY)
.where(ENTRY.published.lt(olderThan))
.orderBy(ENTRY.published.asc())
.limit(max)
.fetch();
return delete(list);
}
/**
* Delete the oldest entries of a feed
*/
public int deleteOldEntries(Long feedId, long max) {
List<FeedEntry> list = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).orderBy(ENTRY.published.asc()).limit(max).fetch();
return delete(list);
}
@AllArgsConstructor
@Getter
public static class FeedCapacity {
private Long id;
private Long capacity;
}
}

472
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java
View File

@@ -1,236 +1,236 @@
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.apache.commons.collections4.CollectionUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
import com.commafeed.backend.model.QFeedEntryStatus;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings.ReadingOrder;
import com.commafeed.frontend.model.UnreadCount;
import com.google.common.collect.Iterables;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.Tuple;
import com.querydsl.jpa.impl.JPAQuery;
@Singleton
public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
private final FeedEntryTagDAO feedEntryTagDAO;
private final CommaFeedConfiguration config;
public FeedEntryStatusDAO(EntityManager entityManager, FeedEntryTagDAO feedEntryTagDAO, CommaFeedConfiguration config) {
super(entityManager, FeedEntryStatus.class);
this.feedEntryTagDAO = feedEntryTagDAO;
this.config = config;
}
public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) {
List<FeedEntryStatus> statuses = query().selectFrom(STATUS).where(STATUS.entry.eq(entry), STATUS.subscription.eq(sub)).fetch();
FeedEntryStatus status = Iterables.getFirst(statuses, null);
return handleStatus(user, status, sub, entry);
}
/**
* creates an artificial "unread" status if status is null
*/
private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
if (status == null) {
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
boolean read = statusesInstantThreshold != null && entry.getPublished().isBefore(statusesInstantThreshold);
status = new FeedEntryStatus(user, sub, entry);
status.setRead(read);
status.setMarkable(!read);
} else {
status.setMarkable(true);
}
return status;
}
private void fetchTags(User user, List<FeedEntryStatus> statuses) {
Map<Long, List<FeedEntryTag>> tagsByEntryIds = feedEntryTagDAO.findByEntries(user,
statuses.stream().map(FeedEntryStatus::getEntry).toList());
for (FeedEntryStatus status : statuses) {
List<FeedEntryTag> tags = tagsByEntryIds.get(status.getEntry().getId());
status.setTags(tags == null ? List.of() : tags);
}
}
public List<FeedEntryStatus> findStarred(User user, Instant newerThan, int offset, int limit, ReadingOrder order,
boolean includeContent) {
JPAQuery<FeedEntryStatus> query = query().selectFrom(STATUS).where(STATUS.user.eq(user), STATUS.starred.isTrue());
if (includeContent) {
query.join(STATUS.entry).fetchJoin();
query.join(STATUS.entry.content).fetchJoin();
}
if (newerThan != null) {
query.where(STATUS.entryInserted.gt(newerThan));
}
if (order == ReadingOrder.asc) {
query.orderBy(STATUS.entryPublished.asc(), STATUS.id.asc());
} else {
query.orderBy(STATUS.entryPublished.desc(), STATUS.id.desc());
}
if (offset > -1) {
query.offset(offset);
}
if (limit > -1) {
query.limit(limit);
}
setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = query.fetch();
statuses.forEach(s -> s.setMarkable(true));
if (includeContent) {
fetchTags(user, statuses);
}
return statuses;
}
public List<FeedEntryStatus> findBySubscriptions(User user, List<FeedSubscription> subs, boolean unreadOnly,
List<FeedEntryKeyword> keywords, Instant newerThan, int offset, int limit, ReadingOrder order, boolean includeContent,
String tag, Long minEntryId, Long maxEntryId) {
Map<Long, List<FeedSubscription>> subsByFeedId = subs.stream().collect(Collectors.groupingBy(s -> s.getFeed().getId()));
JPAQuery<Tuple> query = query().select(ENTRY, STATUS).from(ENTRY);
query.leftJoin(ENTRY.statuses, STATUS).on(STATUS.subscription.in(subs));
query.where(ENTRY.feed.id.in(subsByFeedId.keySet()));
if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
query.join(ENTRY.content, CONTENT).fetchJoin();
}
if (CollectionUtils.isNotEmpty(keywords)) {
for (FeedEntryKeyword keyword : keywords) {
BooleanBuilder or = new BooleanBuilder();
or.or(CONTENT.content.containsIgnoreCase(keyword.getKeyword()));
or.or(CONTENT.title.containsIgnoreCase(keyword.getKeyword()));
if (keyword.getMode() == Mode.EXCLUDE) {
or.not();
}
query.where(or);
}
}
if (unreadOnly && tag == null) {
query.where(buildUnreadPredicate());
}
if (tag != null) {
BooleanBuilder and = new BooleanBuilder();
and.and(TAG.user.id.eq(user.getId()));
and.and(TAG.name.eq(tag));
query.join(ENTRY.tags, TAG).on(and);
}
if (newerThan != null) {
query.where(ENTRY.inserted.goe(newerThan));
}
if (minEntryId != null) {
query.where(ENTRY.id.gt(minEntryId));
}
if (maxEntryId != null) {
query.where(ENTRY.id.lt(maxEntryId));
}
if (order != null) {
if (order == ReadingOrder.asc) {
query.orderBy(ENTRY.published.asc(), ENTRY.id.asc());
} else {
query.orderBy(ENTRY.published.desc(), ENTRY.id.desc());
}
}
if (offset > -1) {
query.offset(offset);
}
if (limit > -1) {
query.limit(limit);
}
setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = new ArrayList<>();
List<Tuple> tuples = query.fetch();
for (Tuple tuple : tuples) {
FeedEntry e = tuple.get(ENTRY);
FeedEntryStatus s = tuple.get(STATUS);
for (FeedSubscription sub : subsByFeedId.get(e.getFeed().getId())) {
statuses.add(handleStatus(user, s, sub, e));
}
}
if (includeContent) {
fetchTags(user, statuses);
}
return statuses;
}
public UnreadCount getUnreadCount(FeedSubscription sub) {
JPAQuery<Tuple> query = query().select(ENTRY.count(), ENTRY.published.max())
.from(ENTRY)
.leftJoin(ENTRY.statuses, STATUS)
.on(STATUS.subscription.eq(sub))
.where(ENTRY.feed.eq(sub.getFeed()))
.where(buildUnreadPredicate());
Tuple tuple = query.fetchOne();
Long count = tuple.get(ENTRY.count());
Instant published = tuple.get(ENTRY.published.max());
return new UnreadCount(sub.getId(), count == null ? 0 : count, published);
}
private BooleanBuilder buildUnreadPredicate() {
BooleanBuilder or = new BooleanBuilder();
or.or(STATUS.read.isNull());
or.or(STATUS.read.isFalse());
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
if (statusesInstantThreshold != null) {
return or.and(ENTRY.published.goe(statusesInstantThreshold));
} else {
return or;
}
}
public long deleteOldStatuses(Instant olderThan, int limit) {
List<Long> ids = query().select(STATUS.id)
.from(STATUS)
.where(STATUS.entryInserted.lt(olderThan), STATUS.starred.isFalse())
.limit(limit)
.fetch();
return deleteQuery(STATUS).where(STATUS.id.in(ids)).execute();
}
}
package com.commafeed.backend.dao;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.apache.commons.collections4.CollectionUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.QFeedEntry;
import com.commafeed.backend.model.QFeedEntryContent;
import com.commafeed.backend.model.QFeedEntryStatus;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings.ReadingOrder;
import com.commafeed.frontend.model.UnreadCount;
import com.google.common.collect.Iterables;
import com.querydsl.core.BooleanBuilder;
import com.querydsl.core.Tuple;
import com.querydsl.jpa.impl.JPAQuery;
@Singleton
public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
private static final QFeedEntryStatus STATUS = QFeedEntryStatus.feedEntryStatus;
private static final QFeedEntry ENTRY = QFeedEntry.feedEntry;
private static final QFeedEntryContent CONTENT = QFeedEntryContent.feedEntryContent;
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
private final FeedEntryTagDAO feedEntryTagDAO;
private final CommaFeedConfiguration config;
public FeedEntryStatusDAO(EntityManager entityManager, FeedEntryTagDAO feedEntryTagDAO, CommaFeedConfiguration config) {
super(entityManager, FeedEntryStatus.class);
this.feedEntryTagDAO = feedEntryTagDAO;
this.config = config;
}
public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) {
List<FeedEntryStatus> statuses = query().selectFrom(STATUS).where(STATUS.entry.eq(entry), STATUS.subscription.eq(sub)).fetch();
FeedEntryStatus status = Iterables.getFirst(statuses, null);
return handleStatus(user, status, sub, entry);
}
/**
* creates an artificial "unread" status if status is null
*/
private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
if (status == null) {
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
boolean read = statusesInstantThreshold != null && entry.getPublished().isBefore(statusesInstantThreshold);
status = new FeedEntryStatus(user, sub, entry);
status.setRead(read);
status.setMarkable(!read);
} else {
status.setMarkable(true);
}
return status;
}
private void fetchTags(User user, List<FeedEntryStatus> statuses) {
Map<Long, List<FeedEntryTag>> tagsByEntryIds = feedEntryTagDAO.findByEntries(user,
statuses.stream().map(FeedEntryStatus::getEntry).toList());
for (FeedEntryStatus status : statuses) {
List<FeedEntryTag> tags = tagsByEntryIds.get(status.getEntry().getId());
status.setTags(tags == null ? List.of() : tags);
}
}
public List<FeedEntryStatus> findStarred(User user, Instant newerThan, int offset, int limit, ReadingOrder order,
boolean includeContent) {
JPAQuery<FeedEntryStatus> query = query().selectFrom(STATUS).where(STATUS.user.eq(user), STATUS.starred.isTrue());
if (includeContent) {
query.join(STATUS.entry).fetchJoin();
query.join(STATUS.entry.content).fetchJoin();
}
if (newerThan != null) {
query.where(STATUS.entryInserted.gt(newerThan));
}
if (order == ReadingOrder.asc) {
query.orderBy(STATUS.entryPublished.asc(), STATUS.id.asc());
} else {
query.orderBy(STATUS.entryPublished.desc(), STATUS.id.desc());
}
if (offset > -1) {
query.offset(offset);
}
if (limit > -1) {
query.limit(limit);
}
setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = query.fetch();
statuses.forEach(s -> s.setMarkable(true));
if (includeContent) {
fetchTags(user, statuses);
}
return statuses;
}
public List<FeedEntryStatus> findBySubscriptions(User user, List<FeedSubscription> subs, boolean unreadOnly,
List<FeedEntryKeyword> keywords, Instant newerThan, int offset, int limit, ReadingOrder order, boolean includeContent,
String tag, Long minEntryId, Long maxEntryId) {
Map<Long, List<FeedSubscription>> subsByFeedId = subs.stream().collect(Collectors.groupingBy(s -> s.getFeed().getId()));
JPAQuery<Tuple> query = query().select(ENTRY, STATUS).from(ENTRY);
query.leftJoin(ENTRY.statuses, STATUS).on(STATUS.subscription.in(subs));
query.where(ENTRY.feed.id.in(subsByFeedId.keySet()));
if (includeContent || CollectionUtils.isNotEmpty(keywords)) {
query.join(ENTRY.content, CONTENT).fetchJoin();
}
if (CollectionUtils.isNotEmpty(keywords)) {
for (FeedEntryKeyword keyword : keywords) {
BooleanBuilder or = new BooleanBuilder();
or.or(CONTENT.content.containsIgnoreCase(keyword.getKeyword()));
or.or(CONTENT.title.containsIgnoreCase(keyword.getKeyword()));
if (keyword.getMode() == Mode.EXCLUDE) {
or.not();
}
query.where(or);
}
}
if (unreadOnly && tag == null) {
query.where(buildUnreadPredicate());
}
if (tag != null) {
BooleanBuilder and = new BooleanBuilder();
and.and(TAG.user.id.eq(user.getId()));
and.and(TAG.name.eq(tag));
query.join(ENTRY.tags, TAG).on(and);
}
if (newerThan != null) {
query.where(ENTRY.inserted.goe(newerThan));
}
if (minEntryId != null) {
query.where(ENTRY.id.gt(minEntryId));
}
if (maxEntryId != null) {
query.where(ENTRY.id.lt(maxEntryId));
}
if (order != null) {
if (order == ReadingOrder.asc) {
query.orderBy(ENTRY.published.asc(), ENTRY.id.asc());
} else {
query.orderBy(ENTRY.published.desc(), ENTRY.id.desc());
}
}
if (offset > -1) {
query.offset(offset);
}
if (limit > -1) {
query.limit(limit);
}
setTimeout(query, config.database().queryTimeout());
List<FeedEntryStatus> statuses = new ArrayList<>();
List<Tuple> tuples = query.fetch();
for (Tuple tuple : tuples) {
FeedEntry e = tuple.get(ENTRY);
FeedEntryStatus s = tuple.get(STATUS);
for (FeedSubscription sub : subsByFeedId.get(e.getFeed().getId())) {
statuses.add(handleStatus(user, s, sub, e));
}
}
if (includeContent) {
fetchTags(user, statuses);
}
return statuses;
}
public UnreadCount getUnreadCount(FeedSubscription sub) {
JPAQuery<Tuple> query = query().select(ENTRY.count(), ENTRY.published.max())
.from(ENTRY)
.leftJoin(ENTRY.statuses, STATUS)
.on(STATUS.subscription.eq(sub))
.where(ENTRY.feed.eq(sub.getFeed()))
.where(buildUnreadPredicate());
Tuple tuple = query.fetchOne();
Long count = tuple.get(ENTRY.count());
Instant published = tuple.get(ENTRY.published.max());
return new UnreadCount(sub.getId(), count == null ? 0 : count, published);
}
private BooleanBuilder buildUnreadPredicate() {
BooleanBuilder or = new BooleanBuilder();
or.or(STATUS.read.isNull());
or.or(STATUS.read.isFalse());
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
if (statusesInstantThreshold != null) {
return or.and(ENTRY.published.goe(statusesInstantThreshold));
} else {
return or;
}
}
public long deleteOldStatuses(Instant olderThan, int limit) {
List<Long> ids = query().select(STATUS.id)
.from(STATUS)
.where(STATUS.entryInserted.lt(olderThan), STATUS.starred.isFalse())
.limit(limit)
.fetch();
return deleteQuery(STATUS).where(STATUS.id.in(ids)).execute();
}
}

78
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java
View File

@@ -1,39 +1,39 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
@Singleton
public class FeedEntryTagDAO extends GenericDAO<FeedEntryTag> {
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
public FeedEntryTagDAO(EntityManager entityManager) {
super(entityManager, FeedEntryTag.class);
}
public List<String> findByUser(User user) {
return query().selectDistinct(TAG.name).from(TAG).where(TAG.user.eq(user)).fetch();
}
public List<FeedEntryTag> findByEntry(User user, FeedEntry entry) {
return query().selectFrom(TAG).where(TAG.user.eq(user), TAG.entry.eq(entry)).fetch();
}
public Map<Long, List<FeedEntryTag>> findByEntries(User user, List<FeedEntry> entries) {
return query().selectFrom(TAG)
.where(TAG.user.eq(user), TAG.entry.in(entries))
.fetch()
.stream()
.collect(Collectors.groupingBy(t -> t.getEntry().getId()));
}
}
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.QFeedEntryTag;
import com.commafeed.backend.model.User;
@Singleton
public class FeedEntryTagDAO extends GenericDAO<FeedEntryTag> {
private static final QFeedEntryTag TAG = QFeedEntryTag.feedEntryTag;
public FeedEntryTagDAO(EntityManager entityManager) {
super(entityManager, FeedEntryTag.class);
}
public List<String> findByUser(User user) {
return query().selectDistinct(TAG.name).from(TAG).where(TAG.user.eq(user)).fetch();
}
public List<FeedEntryTag> findByEntry(User user, FeedEntry entry) {
return query().selectFrom(TAG).where(TAG.user.eq(user), TAG.entry.eq(entry)).fetch();
}
public Map<Long, List<FeedEntryTag>> findByEntries(User user, List<FeedEntry> entries) {
return query().selectFrom(TAG)
.where(TAG.user.eq(user), TAG.entry.in(entries))
.fetch()
.stream()
.collect(Collectors.groupingBy(t -> t.getEntry().getId()));
}
}

260
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java
View File

@@ -1,130 +1,130 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.event.service.spi.EventListenerRegistry;
import org.hibernate.event.spi.EventType;
import org.hibernate.event.spi.PostCommitInsertEventListener;
import org.hibernate.event.spi.PostInsertEvent;
import org.hibernate.persister.entity.EntityPersister;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.QFeedSubscription;
import com.commafeed.backend.model.User;
import com.google.common.collect.Iterables;
import com.querydsl.jpa.JPQLQuery;
@Singleton
public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> {
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private final EntityManager entityManager;
public FeedSubscriptionDAO(EntityManager entityManager) {
super(entityManager, FeedSubscription.class);
this.entityManager = entityManager;
}
public void onPostCommitInsert(Consumer<FeedSubscription> consumer) {
entityManager.unwrap(SharedSessionContractImplementor.class)
.getFactory()
.getServiceRegistry()
.getService(EventListenerRegistry.class)
.getEventListenerGroup(EventType.POST_COMMIT_INSERT)
.appendListener(new PostCommitInsertEventListener() {
@Override
public void onPostInsert(PostInsertEvent event) {
if (event.getEntity() instanceof FeedSubscription s) {
consumer.accept(s);
}
}
@Override
public boolean requiresPostCommitHandling(EntityPersister persister) {
return true;
}
@Override
public void onPostInsertCommitFailed(PostInsertEvent event) {
// do nothing
}
});
}
public FeedSubscription findById(User user, Long id) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.id.eq(id))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
return initRelations(Iterables.getFirst(subs, null));
}
public List<FeedSubscription> findByFeed(Feed feed) {
return query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.feed.eq(feed)).fetch();
}
public FeedSubscription findByFeed(User user, Feed feed) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.feed.eq(feed))
.fetch();
return initRelations(Iterables.getFirst(subs, null));
}
public List<FeedSubscription> findAll(User user) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
return initRelations(subs);
}
public Long count(User user) {
return query().select(SUBSCRIPTION.count()).from(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user)).fetchOne();
}
public List<FeedSubscription> findByCategory(User user, FeedCategory category) {
JPQLQuery<FeedSubscription> query = query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user));
if (category == null) {
query.where(SUBSCRIPTION.category.isNull());
} else {
query.where(SUBSCRIPTION.category.eq(category));
}
return initRelations(query.fetch());
}
public List<FeedSubscription> findByCategories(User user, List<FeedCategory> categories) {
Set<Long> categoryIds = categories.stream().map(AbstractModel::getId).collect(Collectors.toSet());
return findAll(user).stream().filter(s -> s.getCategory() != null && categoryIds.contains(s.getCategory().getId())).toList();
}
private List<FeedSubscription> initRelations(List<FeedSubscription> list) {
list.forEach(this::initRelations);
return list;
}
private FeedSubscription initRelations(FeedSubscription sub) {
if (sub != null) {
Models.initialize(sub.getFeed());
Models.initialize(sub.getCategory());
}
return sub;
}
}
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import org.hibernate.engine.spi.SharedSessionContractImplementor;
import org.hibernate.event.service.spi.EventListenerRegistry;
import org.hibernate.event.spi.EventType;
import org.hibernate.event.spi.PostCommitInsertEventListener;
import org.hibernate.event.spi.PostInsertEvent;
import org.hibernate.persister.entity.EntityPersister;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.QFeedSubscription;
import com.commafeed.backend.model.User;
import com.google.common.collect.Iterables;
import com.querydsl.jpa.JPQLQuery;
@Singleton
public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> {
private static final QFeedSubscription SUBSCRIPTION = QFeedSubscription.feedSubscription;
private final EntityManager entityManager;
public FeedSubscriptionDAO(EntityManager entityManager) {
super(entityManager, FeedSubscription.class);
this.entityManager = entityManager;
}
public void onPostCommitInsert(Consumer<FeedSubscription> consumer) {
entityManager.unwrap(SharedSessionContractImplementor.class)
.getFactory()
.getServiceRegistry()
.getService(EventListenerRegistry.class)
.getEventListenerGroup(EventType.POST_COMMIT_INSERT)
.appendListener(new PostCommitInsertEventListener() {
@Override
public void onPostInsert(PostInsertEvent event) {
if (event.getEntity() instanceof FeedSubscription s) {
consumer.accept(s);
}
}
@Override
public boolean requiresPostCommitHandling(EntityPersister persister) {
return true;
}
@Override
public void onPostInsertCommitFailed(PostInsertEvent event) {
// do nothing
}
});
}
public FeedSubscription findById(User user, Long id) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.id.eq(id))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
return initRelations(Iterables.getFirst(subs, null));
}
public List<FeedSubscription> findByFeed(Feed feed) {
return query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.feed.eq(feed)).fetch();
}
public FeedSubscription findByFeed(User user, Feed feed) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.feed.eq(feed))
.fetch();
return initRelations(Iterables.getFirst(subs, null));
}
public List<FeedSubscription> findAll(User user) {
List<FeedSubscription> subs = query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user))
.leftJoin(SUBSCRIPTION.feed)
.fetchJoin()
.leftJoin(SUBSCRIPTION.category)
.fetchJoin()
.fetch();
return initRelations(subs);
}
public Long count(User user) {
return query().select(SUBSCRIPTION.count()).from(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user)).fetchOne();
}
public List<FeedSubscription> findByCategory(User user, FeedCategory category) {
JPQLQuery<FeedSubscription> query = query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user));
if (category == null) {
query.where(SUBSCRIPTION.category.isNull());
} else {
query.where(SUBSCRIPTION.category.eq(category));
}
return initRelations(query.fetch());
}
public List<FeedSubscription> findByCategories(User user, List<FeedCategory> categories) {
Set<Long> categoryIds = categories.stream().map(AbstractModel::getId).collect(Collectors.toSet());
return findAll(user).stream().filter(s -> s.getCategory() != null && categoryIds.contains(s.getCategory().getId())).toList();
}
private List<FeedSubscription> initRelations(List<FeedSubscription> list) {
list.forEach(this::initRelations);
return list;
}
private FeedSubscription initRelations(FeedSubscription sub) {
if (sub != null) {
Models.initialize(sub.getFeed());
Models.initialize(sub.getCategory());
}
return sub;
}
}

152
commafeed-server/src/main/java/com/commafeed/backend/dao/GenericDAO.java
View File

@@ -1,76 +1,76 @@
package com.commafeed.backend.dao;
import java.time.Duration;
import java.util.Collection;
import jakarta.persistence.EntityManager;
import org.hibernate.Session;
import org.hibernate.jpa.SpecHints;
import com.commafeed.backend.model.AbstractModel;
import com.querydsl.core.types.EntityPath;
import com.querydsl.jpa.impl.JPADeleteClause;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
import com.querydsl.jpa.impl.JPAUpdateClause;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public abstract class GenericDAO<T extends AbstractModel> {
private final EntityManager entityManager;
private final Class<T> entityClass;
protected JPAQueryFactory query() {
return new JPAQueryFactory(entityManager);
}
protected JPAUpdateClause updateQuery(EntityPath<T> entityPath) {
return new JPAUpdateClause(entityManager, entityPath);
}
protected JPADeleteClause deleteQuery(EntityPath<T> entityPath) {
return new JPADeleteClause(entityManager, entityPath);
}
@SuppressWarnings("deprecation")
public void saveOrUpdate(T model) {
entityManager.unwrap(Session.class).saveOrUpdate(model);
}
public void saveOrUpdate(Collection<T> models) {
models.forEach(this::saveOrUpdate);
}
public void persist(T model) {
entityManager.persist(model);
}
public T merge(T model) {
return entityManager.merge(model);
}
public T findById(Long id) {
return entityManager.find(entityClass, id);
}
public void delete(T object) {
if (object != null) {
entityManager.remove(object);
}
}
public int delete(Collection<T> objects) {
objects.forEach(this::delete);
return objects.size();
}
protected void setTimeout(JPAQuery<?> query, Duration timeout) {
if (!timeout.isZero()) {
query.setHint(SpecHints.HINT_SPEC_QUERY_TIMEOUT, Math.toIntExact(timeout.toMillis()));
}
}
}
package com.commafeed.backend.dao;
import java.time.Duration;
import java.util.Collection;
import jakarta.persistence.EntityManager;
import org.hibernate.Session;
import org.hibernate.jpa.SpecHints;
import com.commafeed.backend.model.AbstractModel;
import com.querydsl.core.types.EntityPath;
import com.querydsl.jpa.impl.JPADeleteClause;
import com.querydsl.jpa.impl.JPAQuery;
import com.querydsl.jpa.impl.JPAQueryFactory;
import com.querydsl.jpa.impl.JPAUpdateClause;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public abstract class GenericDAO<T extends AbstractModel> {
private final EntityManager entityManager;
private final Class<T> entityClass;
protected JPAQueryFactory query() {
return new JPAQueryFactory(entityManager);
}
protected JPAUpdateClause updateQuery(EntityPath<T> entityPath) {
return new JPAUpdateClause(entityManager, entityPath);
}
protected JPADeleteClause deleteQuery(EntityPath<T> entityPath) {
return new JPADeleteClause(entityManager, entityPath);
}
@SuppressWarnings("deprecation")
public void saveOrUpdate(T model) {
entityManager.unwrap(Session.class).saveOrUpdate(model);
}
public void saveOrUpdate(Collection<T> models) {
models.forEach(this::saveOrUpdate);
}
public void persist(T model) {
entityManager.persist(model);
}
public T merge(T model) {
return entityManager.merge(model);
}
public T findById(Long id) {
return entityManager.find(entityClass, id);
}
public void delete(T object) {
if (object != null) {
entityManager.remove(object);
}
}
public int delete(Collection<T> objects) {
objects.forEach(this::delete);
return objects.size();
}
protected void setTimeout(JPAQuery<?> query, Duration timeout) {
if (!timeout.isZero()) {
query.setHint(SpecHints.HINT_SPEC_QUERY_TIMEOUT, Math.toIntExact(timeout.toMillis()));
}
}
}

38
commafeed-server/src/main/java/com/commafeed/backend/dao/UnitOfWork.java
View File

@@ -1,19 +1,19 @@
package com.commafeed.backend.dao;
import java.util.concurrent.Callable;
import jakarta.inject.Singleton;
import io.quarkus.narayana.jta.QuarkusTransaction;
@Singleton
public class UnitOfWork {
public void run(Runnable runnable) {
QuarkusTransaction.joiningExisting().run(runnable);
}
public <T> T call(Callable<T> callable) {
return QuarkusTransaction.joiningExisting().call(callable);
}
}
package com.commafeed.backend.dao;
import java.util.concurrent.Callable;
import jakarta.inject.Singleton;
import io.quarkus.narayana.jta.QuarkusTransaction;
@Singleton
public class UnitOfWork {
public void run(Runnable runnable) {
QuarkusTransaction.joiningExisting().run(runnable);
}
public <T> T call(Callable<T> callable) {
return QuarkusTransaction.joiningExisting().call(callable);
}
}

66
commafeed-server/src/main/java/com/commafeed/backend/dao/UserDAO.java
View File

@@ -1,33 +1,33 @@
package com.commafeed.backend.dao;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
@Singleton
public class UserDAO extends GenericDAO<User> {
private static final QUser USER = QUser.user;
public UserDAO(EntityManager entityManager) {
super(entityManager, User.class);
}
public User findByName(String name) {
return query().selectFrom(USER).where(USER.name.equalsIgnoreCase(name)).fetchOne();
}
public User findByApiKey(String key) {
return query().selectFrom(USER).where(USER.apiKey.equalsIgnoreCase(key)).fetchOne();
}
public User findByEmail(String email) {
return query().selectFrom(USER).where(USER.email.equalsIgnoreCase(email)).fetchOne();
}
public long count() {
return query().select(USER.count()).from(USER).fetchOne();
}
}
package com.commafeed.backend.dao;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUser;
import com.commafeed.backend.model.User;
@Singleton
public class UserDAO extends GenericDAO<User> {
private static final QUser USER = QUser.user;
public UserDAO(EntityManager entityManager) {
super(entityManager, User.class);
}
public User findByName(String name) {
return query().selectFrom(USER).where(USER.name.equalsIgnoreCase(name)).fetchOne();
}
public User findByApiKey(String key) {
return query().selectFrom(USER).where(USER.apiKey.equalsIgnoreCase(key)).fetchOne();
}
public User findByEmail(String email) {
return query().selectFrom(USER).where(USER.email.equalsIgnoreCase(email)).fetchOne();
}
public long count() {
return query().select(USER.count()).from(USER).fetchOne();
}
}

70
commafeed-server/src/main/java/com/commafeed/backend/dao/UserRoleDAO.java
View File

@@ -1,35 +1,35 @@
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUserRole;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
@Singleton
public class UserRoleDAO extends GenericDAO<UserRole> {
private static final QUserRole ROLE = QUserRole.userRole;
public UserRoleDAO(EntityManager entityManager) {
super(entityManager, UserRole.class);
}
public List<UserRole> findAll() {
return query().selectFrom(ROLE).leftJoin(ROLE.user).fetchJoin().distinct().fetch();
}
public List<UserRole> findAll(User user) {
return query().selectFrom(ROLE).where(ROLE.user.eq(user)).distinct().fetch();
}
public Set<Role> findRoles(User user) {
return findAll(user).stream().map(UserRole::getRole).collect(Collectors.toSet());
}
}
package com.commafeed.backend.dao;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUserRole;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
@Singleton
public class UserRoleDAO extends GenericDAO<UserRole> {
private static final QUserRole ROLE = QUserRole.userRole;
public UserRoleDAO(EntityManager entityManager) {
super(entityManager, UserRole.class);
}
public List<UserRole> findAll() {
return query().selectFrom(ROLE).leftJoin(ROLE.user).fetchJoin().distinct().fetch();
}
public List<UserRole> findAll(User user) {
return query().selectFrom(ROLE).where(ROLE.user.eq(user)).distinct().fetch();
}
public Set<Role> findRoles(User user) {
return findAll(user).stream().map(UserRole::getRole).collect(Collectors.toSet());
}
}

44
commafeed-server/src/main/java/com/commafeed/backend/dao/UserSettingsDAO.java
View File

@@ -1,22 +1,22 @@
package com.commafeed.backend.dao;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUserSettings;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings;
@Singleton
public class UserSettingsDAO extends GenericDAO<UserSettings> {
private static final QUserSettings SETTINGS = QUserSettings.userSettings;
public UserSettingsDAO(EntityManager entityManager) {
super(entityManager, UserSettings.class);
}
public UserSettings findByUser(User user) {
return query().selectFrom(SETTINGS).where(SETTINGS.user.eq(user)).fetchFirst();
}
}
package com.commafeed.backend.dao;
import jakarta.inject.Singleton;
import jakarta.persistence.EntityManager;
import com.commafeed.backend.model.QUserSettings;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings;
@Singleton
public class UserSettingsDAO extends GenericDAO<UserSettings> {
private static final QUserSettings SETTINGS = QUserSettings.userSettings;
public UserSettingsDAO(EntityManager entityManager) {
super(entityManager, UserSettings.class);
}
public UserSettings findByUser(User user) {
return query().selectFrom(SETTINGS).where(SETTINGS.user.eq(user)).fetchFirst();
}
}

98
commafeed-server/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java
View File

@@ -1,49 +1,49 @@
package com.commafeed.backend.favicon;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class AbstractFaviconFetcher {
private static final List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
public abstract Favicon fetch(Feed feed);
protected boolean isValidIconResponse(byte[] content, String contentType) {
if (content == null) {
return false;
}
long length = content.length;
if (StringUtils.isNotBlank(contentType)) {
contentType = contentType.split(";")[0];
}
if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
log.debug("Content-Type {} is blacklisted", contentType);
return false;
}
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
return true;
}
}
package com.commafeed.backend.favicon;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class AbstractFaviconFetcher {
private static final List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
public abstract Favicon fetch(Feed feed);
protected boolean isValidIconResponse(byte[] content, String contentType) {
if (content == null) {
return false;
}
long length = content.length;
if (StringUtils.isNotBlank(contentType)) {
contentType = contentType.split(";")[0];
}
if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
log.debug("Content-Type {} is blacklisted", contentType);
return false;
}
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
return true;
}
}

266
commafeed-server/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java
View File

@@ -1,133 +1,133 @@
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Inspired/Ported from https://github.com/potatolondon/getfavicon
*
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(Integer.MIN_VALUE)
public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
Favicon icon = fetch(feed.getLink());
if (icon == null) {
icon = fetch(feed.getUrl());
}
return icon;
}
private Favicon fetch(String url) {
if (url == null) {
log.debug("url is null");
return null;
}
int doubleSlash = url.indexOf("//");
if (doubleSlash == -1) {
doubleSlash = 0;
} else {
doubleSlash += 2;
}
int firstSlash = url.indexOf('/', doubleSlash);
if (firstSlash != -1) {
url = url.substring(0, firstSlash);
}
Favicon icon = getIconAtRoot(url);
if (icon == null) {
icon = getIconInPage(url);
}
return icon;
}
private Favicon getIconAtRoot(String url) {
byte[] bytes = null;
String contentType = null;
try {
url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico";
log.debug("getting root icon at {}", url);
HttpResult result = getter.get(url);
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url);
log.trace("Failed to retrieve iconAtRoot for url {}: ", url, e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private Favicon getIconInPage(String url) {
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.getContent()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon");
log.trace("Failed to retrieve page to find icon", e);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
String href = icons.get(0).attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
log.debug("Found unconfirmed iconInPage at {}", href);
byte[] bytes;
String contentType;
try {
HttpResult result = getter.get(href);
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href);
log.trace("Failed to retrieve icon found in page {}", href, e);
return null;
}
if (!isValidIconResponse(bytes, contentType)) {
log.debug("Invalid icon found for {}", href);
return null;
}
return new Favicon(bytes, contentType);
}
}
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Inspired/Ported from https://github.com/potatolondon/getfavicon
*
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(Integer.MIN_VALUE)
public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
Favicon icon = fetch(feed.getLink());
if (icon == null) {
icon = fetch(feed.getUrl());
}
return icon;
}
private Favicon fetch(String url) {
if (url == null) {
log.debug("url is null");
return null;
}
int doubleSlash = url.indexOf("//");
if (doubleSlash == -1) {
doubleSlash = 0;
} else {
doubleSlash += 2;
}
int firstSlash = url.indexOf('/', doubleSlash);
if (firstSlash != -1) {
url = url.substring(0, firstSlash);
}
Favicon icon = getIconAtRoot(url);
if (icon == null) {
icon = getIconInPage(url);
}
return icon;
}
private Favicon getIconAtRoot(String url) {
byte[] bytes = null;
String contentType = null;
try {
url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico";
log.debug("getting root icon at {}", url);
HttpResult result = getter.get(url);
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url);
log.trace("Failed to retrieve iconAtRoot for url {}: ", url, e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private Favicon getIconInPage(String url) {
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.getContent()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon");
log.trace("Failed to retrieve page to find icon", e);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
String href = icons.get(0).attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
log.debug("Found unconfirmed iconInPage at {}", href);
byte[] bytes;
String contentType;
try {
HttpResult result = getter.get(href);
bytes = result.getContent();
contentType = result.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href);
log.trace("Failed to retrieve icon found in page {}", href, e);
return null;
}
if (!isValidIconResponse(bytes, contentType)) {
log.debug("Invalid icon found for {}", href);
return null;
}
return new Favicon(bytes, contentType);
}
}

146
commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
View File

@@ -1,73 +1,73 @@
package com.commafeed.backend.favicon;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("www.facebook.com")) {
return null;
}
String userName = extractUserName(url);
if (userName == null) {
return null;
}
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
byte[] bytes = null;
String contentType = null;
try {
log.debug("Getting Facebook user's icon, {}", url);
HttpResult iconResult = getter.get(iconUrl);
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private String extractUserName(String url) {
URI uri;
try {
uri = new URI(url);
} catch (URISyntaxException e) {
log.debug("could not parse url", e);
return null;
}
List<NameValuePair> params = new URIBuilder(uri).getQueryParams();
return params.stream().filter(p -> "id".equals(p.getName())).map(NameValuePair::getValue).findFirst().orElse(null);
}
}
package com.commafeed.backend.favicon;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("www.facebook.com")) {
return null;
}
String userName = extractUserName(url);
if (userName == null) {
return null;
}
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
byte[] bytes = null;
String contentType = null;
try {
log.debug("Getting Facebook user's icon, {}", url);
HttpResult iconResult = getter.get(iconUrl);
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
} catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private String extractUserName(String url) {
URI uri;
try {
uri = new URI(url);
} catch (URISyntaxException e) {
log.debug("could not parse url", e);
return null;
}
List<NameValuePair> params = new URIBuilder(uri).getQueryParams();
return params.stream().filter(p -> "id".equals(p.getName())).map(NameValuePair::getValue).findFirst().orElse(null);
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/favicon/Favicon.java
View File

@@ -1,31 +1,31 @@
package com.commafeed.backend.favicon;
import jakarta.ws.rs.core.MediaType;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Getter
@Slf4j
public class Favicon {
private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.valueOf("image/x-icon");
private final byte[] icon;
private final MediaType mediaType;
public Favicon(byte[] icon, String contentType) {
this(icon, parseMediaType(contentType));
}
private static MediaType parseMediaType(String contentType) {
try {
return MediaType.valueOf(contentType);
} catch (Exception e) {
log.debug("invalid content type '{}' received, returning default value", contentType);
return DEFAULT_MEDIA_TYPE;
}
}
package com.commafeed.backend.favicon;
import jakarta.ws.rs.core.MediaType;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Getter
@Slf4j
public class Favicon {
private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.valueOf("image/x-icon");
private final byte[] icon;
private final MediaType mediaType;
public Favicon(byte[] icon, String contentType) {
this(icon, parseMediaType(contentType));
}
private static MediaType parseMediaType(String contentType) {
try {
return MediaType.valueOf(contentType);
} catch (Exception e) {
log.debug("invalid content type '{}' received, returning default value", contentType);
return DEFAULT_MEDIA_TYPE;
}
}
}

270
commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
View File

@@ -1,135 +1,135 @@
package com.commafeed.backend.favicon;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Optional;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.UriBuilder;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HostNotAllowedException;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.SchemeNotAllowedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.model.Feed;
import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
private static final JsonPointer CHANNEL_THUMBNAIL_URL = JsonPointer.compile("/items/0/snippet/thumbnails/default/url");
private static final JsonPointer PLAYLIST_CHANNEL_ID = JsonPointer.compile("/items/0/snippet/channelId");
private final HttpGetter getter;
private final CommaFeedConfiguration config;
private final ObjectMapper objectMapper;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
return null;
}
Optional<String> googleAuthKey = config.googleAuthKey();
if (googleAuthKey.isEmpty()) {
log.debug("no google auth key configured");
return null;
}
byte[] bytes = null;
String contentType = null;
try {
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
Optional<NameValuePair> channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id")).findFirst();
Optional<NameValuePair> playlistId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id")).findFirst();
byte[] response = null;
if (userId.isPresent()) {
log.debug("contacting youtube api for user {}", userId.get().getValue());
response = fetchForUser(googleAuthKey.get(), userId.get().getValue());
} else if (channelId.isPresent()) {
log.debug("contacting youtube api for channel {}", channelId.get().getValue());
response = fetchForChannel(googleAuthKey.get(), channelId.get().getValue());
} else if (playlistId.isPresent()) {
log.debug("contacting youtube api for playlist {}", playlistId.get().getValue());
response = fetchForPlaylist(googleAuthKey.get(), playlistId.get().getValue());
}
if (ArrayUtils.isEmpty(response)) {
log.debug("youtube api returned empty response");
return null;
}
JsonNode thumbnailUrl = objectMapper.readTree(response).at(CHANNEL_THUMBNAIL_URL);
if (thumbnailUrl.isMissingNode()) {
log.debug("youtube api returned invalid response");
return null;
}
HttpResult iconResult = getter.get(thumbnailUrl.asText());
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
} catch (Exception e) {
log.error("Failed to retrieve YouTube icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private byte[] fetchForUser(String googleAuthKey, String userId)
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)
.queryParam("forUsername", userId)
.build();
return getter.get(uri.toString()).getContent();
}
private byte[] fetchForChannel(String googleAuthKey, String channelId)
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)
.queryParam("id", channelId)
.build();
return getter.get(uri.toString()).getContent();
}
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)
.queryParam("id", playlistId)
.build();
byte[] playlistBytes = getter.get(uri.toString()).getContent();
JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID);
if (channelId.isMissingNode()) {
return null;
}
return fetchForChannel(googleAuthKey, channelId.asText());
}
}
package com.commafeed.backend.favicon;
import java.io.IOException;
import java.net.URI;
import java.util.List;
import java.util.Optional;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.UriBuilder;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.net.URIBuilder;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HostNotAllowedException;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.SchemeNotAllowedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.model.Feed;
import com.fasterxml.jackson.core.JsonPointer;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
private static final JsonPointer CHANNEL_THUMBNAIL_URL = JsonPointer.compile("/items/0/snippet/thumbnails/default/url");
private static final JsonPointer PLAYLIST_CHANNEL_ID = JsonPointer.compile("/items/0/snippet/channelId");
private final HttpGetter getter;
private final CommaFeedConfiguration config;
private final ObjectMapper objectMapper;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
return null;
}
Optional<String> googleAuthKey = config.googleAuthKey();
if (googleAuthKey.isEmpty()) {
log.debug("no google auth key configured");
return null;
}
byte[] bytes = null;
String contentType = null;
try {
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
Optional<NameValuePair> channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id")).findFirst();
Optional<NameValuePair> playlistId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id")).findFirst();
byte[] response = null;
if (userId.isPresent()) {
log.debug("contacting youtube api for user {}", userId.get().getValue());
response = fetchForUser(googleAuthKey.get(), userId.get().getValue());
} else if (channelId.isPresent()) {
log.debug("contacting youtube api for channel {}", channelId.get().getValue());
response = fetchForChannel(googleAuthKey.get(), channelId.get().getValue());
} else if (playlistId.isPresent()) {
log.debug("contacting youtube api for playlist {}", playlistId.get().getValue());
response = fetchForPlaylist(googleAuthKey.get(), playlistId.get().getValue());
}
if (ArrayUtils.isEmpty(response)) {
log.debug("youtube api returned empty response");
return null;
}
JsonNode thumbnailUrl = objectMapper.readTree(response).at(CHANNEL_THUMBNAIL_URL);
if (thumbnailUrl.isMissingNode()) {
log.debug("youtube api returned invalid response");
return null;
}
HttpResult iconResult = getter.get(thumbnailUrl.asText());
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
} catch (Exception e) {
log.error("Failed to retrieve YouTube icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private byte[] fetchForUser(String googleAuthKey, String userId)
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)
.queryParam("forUsername", userId)
.build();
return getter.get(uri.toString()).getContent();
}
private byte[] fetchForChannel(String googleAuthKey, String channelId)
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)
.queryParam("id", channelId)
.build();
return getter.get(uri.toString()).getContent();
}
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
.queryParam("part", "snippet")
.queryParam("key", googleAuthKey)
.queryParam("id", playlistId)
.build();
byte[] playlistBytes = getter.get(uri.toString()).getContent();
JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID);
if (channelId.isMissingNode()) {
return null;
}
return fetchForChannel(googleAuthKey, channelId.asText());
}
}

78
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedEntryKeyword.java
View File

@@ -1,39 +1,39 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
/**
* A keyword used in a search query
*/
@Getter
@RequiredArgsConstructor
public class FeedEntryKeyword {
public enum Mode {
INCLUDE, EXCLUDE
}
private final String keyword;
private final Mode mode;
public static List<FeedEntryKeyword> fromQueryString(String keywords) {
List<FeedEntryKeyword> list = new ArrayList<>();
if (keywords != null) {
for (String keyword : StringUtils.split(keywords)) {
boolean not = false;
if (keyword.startsWith("-") || keyword.startsWith("!")) {
not = true;
keyword = keyword.substring(1);
}
list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
}
}
return list;
}
}
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
/**
* A keyword used in a search query
*/
@Getter
@RequiredArgsConstructor
public class FeedEntryKeyword {
public enum Mode {
INCLUDE, EXCLUDE
}
private final String keyword;
private final Mode mode;
public static List<FeedEntryKeyword> fromQueryString(String keywords) {
List<FeedEntryKeyword> list = new ArrayList<>();
if (keywords != null) {
for (String keyword : StringUtils.split(keywords)) {
boolean not = false;
if (keyword.startsWith("-") || keyword.startsWith("!")) {
not = true;
keyword = keyword.substring(1);
}
list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
}
}
return list;
}
}

246
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
View File

@@ -1,123 +1,123 @@
package com.commafeed.backend.feed;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HostNotAllowedException;
import com.commafeed.backend.HttpGetter.HttpRequest;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.SchemeNotAllowedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.parser.FeedParser;
import com.commafeed.backend.feed.parser.FeedParser.FeedParsingException;
import com.commafeed.backend.feed.parser.FeedParserResult;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import io.quarkus.arc.All;
import lombok.extern.slf4j.Slf4j;
/**
* Fetches a feed then parses it
*/
@Slf4j
@Singleton
public class FeedFetcher {
private final FeedParser parser;
private final HttpGetter getter;
private final List<FeedURLProvider> urlProviders;
public FeedFetcher(FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
this.parser = parser;
this.getter = getter;
this.urlProviders = urlProviders;
}
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
Instant lastPublishedDate, String lastContentHash) throws FeedParsingException, IOException, NotModifiedException,
TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException, NoFeedFoundException {
log.debug("Fetching feed {}", feedUrl);
HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
byte[] content = result.getContent();
FeedParserResult parserResult;
try {
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
} catch (FeedParsingException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, new String(result.getContent(), StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
result = getter.get(HttpRequest.builder(extractedUrl).lastModified(lastModified).eTag(eTag).build());
content = result.getContent();
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
} else {
throw new NoFeedFoundException(e);
}
} else {
throw e;
}
}
if (content == null) {
throw new IOException("Feed content is empty.");
}
boolean lastModifiedHeaderValueChanged = !StringUtils.equals(lastModified, result.getLastModifiedSince());
boolean etagHeaderValueChanged = !StringUtils.equals(eTag, result.getETag());
String hash = Digests.sha1Hex(content);
if (lastContentHash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException("content hash not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
}
if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
}
return new FeedFetcherResult(parserResult, result.getUrlAfterRedirect(), result.getLastModifiedSince(), result.getETag(), hash,
result.getValidFor());
}
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) {
for (FeedURLProvider urlProvider : urlProviders) {
String feedUrl = urlProvider.get(url, urlContent);
if (feedUrl != null) {
return feedUrl;
}
}
return null;
}
public record FeedFetcherResult(FeedParserResult feed, String urlAfterRedirect, String lastModifiedHeader, String lastETagHeader,
String contentHash, Duration validFor) {
}
public static class NoFeedFoundException extends Exception {
private static final long serialVersionUID = 1L;
public NoFeedFoundException(Throwable cause) {
super("This URL does not point to an RSS feed or a website with an RSS feed.", cause);
}
}
}
package com.commafeed.backend.feed;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HostNotAllowedException;
import com.commafeed.backend.HttpGetter.HttpRequest;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.SchemeNotAllowedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.parser.FeedParser;
import com.commafeed.backend.feed.parser.FeedParser.FeedParsingException;
import com.commafeed.backend.feed.parser.FeedParserResult;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import io.quarkus.arc.All;
import lombok.extern.slf4j.Slf4j;
/**
* Fetches a feed then parses it
*/
@Slf4j
@Singleton
public class FeedFetcher {
private final FeedParser parser;
private final HttpGetter getter;
private final List<FeedURLProvider> urlProviders;
public FeedFetcher(FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
this.parser = parser;
this.getter = getter;
this.urlProviders = urlProviders;
}
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
Instant lastPublishedDate, String lastContentHash) throws FeedParsingException, IOException, NotModifiedException,
TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException, NoFeedFoundException {
log.debug("Fetching feed {}", feedUrl);
HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
byte[] content = result.getContent();
FeedParserResult parserResult;
try {
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
} catch (FeedParsingException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, new String(result.getContent(), StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
result = getter.get(HttpRequest.builder(extractedUrl).lastModified(lastModified).eTag(eTag).build());
content = result.getContent();
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
} else {
throw new NoFeedFoundException(e);
}
} else {
throw e;
}
}
if (content == null) {
throw new IOException("Feed content is empty.");
}
boolean lastModifiedHeaderValueChanged = !StringUtils.equals(lastModified, result.getLastModifiedSince());
boolean etagHeaderValueChanged = !StringUtils.equals(eTag, result.getETag());
String hash = Digests.sha1Hex(content);
if (lastContentHash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException("content hash not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
}
if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
}
return new FeedFetcherResult(parserResult, result.getUrlAfterRedirect(), result.getLastModifiedSince(), result.getETag(), hash,
result.getValidFor());
}
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) {
for (FeedURLProvider urlProvider : urlProviders) {
String feedUrl = urlProvider.get(url, urlContent);
if (feedUrl != null) {
return feedUrl;
}
}
return null;
}
public record FeedFetcherResult(FeedParserResult feed, String urlAfterRedirect, String lastModifiedHeader, String lastETagHeader,
String contentHash, Duration validFor) {
}
public static class NoFeedFoundException extends Exception {
private static final long serialVersionUID = 1L;
public NoFeedFoundException(Throwable cause) {
super("This URL does not point to an RSS feed or a website with an RSS feed.", cause);
}
}
}

428
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshEngine.java
View File

@@ -1,214 +1,214 @@
package com.commafeed.backend.feed;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedRefreshEngine {
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedRefreshWorker worker;
private final FeedRefreshUpdater updater;
private final CommaFeedConfiguration config;
private final Meter refill;
private final BlockingDeque<Feed> queue;
private final ExecutorService feedProcessingLoopExecutor;
private final ExecutorService refillLoopExecutor;
private final ExecutorService refillExecutor;
private final ThreadPoolExecutor workerExecutor;
private final ThreadPoolExecutor databaseUpdaterExecutor;
public FeedRefreshEngine(UnitOfWork unitOfWork, FeedDAO feedDAO, FeedRefreshWorker worker, FeedRefreshUpdater updater,
CommaFeedConfiguration config, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.worker = worker;
this.updater = updater;
this.config = config;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
this.queue = new LinkedBlockingDeque<>();
this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor();
this.refillLoopExecutor = Executors.newSingleThreadExecutor();
this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.feedRefresh().databaseThreads());
metrics.register(MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(MetricRegistry.name(getClass(), "worker", "active"), (Gauge<Integer>) workerExecutor::getActiveCount);
metrics.register(MetricRegistry.name(getClass(), "updater", "active"), (Gauge<Integer>) databaseUpdaterExecutor::getActiveCount);
}
public void start() {
startFeedProcessingLoop();
startRefillLoop();
}
private void startFeedProcessingLoop() {
// take a feed from the queue, process it, rince, repeat
feedProcessingLoopExecutor.submit(() -> {
while (!feedProcessingLoopExecutor.isShutdown()) {
try {
// take() is blocking until a feed is available from the queue
Feed feed = queue.take();
// send the feed to be processed
log.debug("got feed {} from the queue, send it for processing", feed.getId());
processFeedAsync(feed);
// we removed a feed from the queue, try to refill it as it may now be empty
if (queue.isEmpty()) {
log.debug("took the last feed from the queue, try to refill");
refillQueueAsync();
}
} catch (InterruptedException e) {
log.debug("interrupted while waiting for a feed in the queue");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
private void startRefillLoop() {
// refill the queue at regular intervals if it's empty
refillLoopExecutor.submit(() -> {
while (!refillLoopExecutor.isShutdown()) {
try {
if (queue.isEmpty()) {
log.debug("refilling queue");
refillQueueAsync();
}
log.debug("sleeping for 15s");
TimeUnit.SECONDS.sleep(15);
} catch (InterruptedException e) {
log.debug("interrupted while sleeping");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
public void refreshImmediately(Feed feed) {
log.debug("add feed {} at the start of the queue", feed.getId());
// remove the feed from the queue if it was already queued to avoid refreshing it twice
queue.removeIf(f -> f.getId().equals(feed.getId()));
queue.addFirst(feed);
}
private void refillQueueAsync() {
CompletableFuture.runAsync(() -> {
if (!queue.isEmpty()) {
return;
}
refill.mark();
List<Feed> nextUpdatableFeeds = getNextUpdatableFeeds(getBatchSize());
log.debug("found {} feeds that are up for refresh", nextUpdatableFeeds.size());
for (Feed feed : nextUpdatableFeeds) {
// add the feed only if it was not already queued
if (queue.stream().noneMatch(f -> f.getId().equals(feed.getId()))) {
queue.addLast(feed);
}
}
}, refillExecutor).whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while refilling the queue", ex);
}
});
}
private void processFeedAsync(Feed feed) {
CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
.thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
.whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while processing feed {}", feed.getUrl(), ex);
}
});
}
private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(() -> {
Instant lastLoginThreshold = config.feedRefresh().userInactivityPeriod().isZero() ? null
: Instant.now().minus(config.feedRefresh().userInactivityPeriod());
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
// update disabledUntil to prevent feeds from being returned again by feedDAO.findNextUpdatable()
Instant nextUpdateDate = Instant.now().plus(config.feedRefresh().interval());
feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
return feeds;
});
}
private int getBatchSize() {
return Math.min(100, 3 * config.feedRefresh().httpThreads());
}
public void stop() {
this.feedProcessingLoopExecutor.shutdownNow();
this.refillLoopExecutor.shutdownNow();
this.refillExecutor.shutdownNow();
this.workerExecutor.shutdownNow();
this.databaseUpdaterExecutor.shutdownNow();
}
/**
* returns an ExecutorService with a single thread that discards tasks if a task is already running
*/
private ThreadPoolExecutor newDiscardingSingleThreadExecutorService() {
ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/**
* returns an ExecutorService that blocks submissions until a thread is available
*/
private ThreadPoolExecutor newBlockingExecutorService(int threads) {
ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler((r, e) -> {
if (e.isShutdown()) {
return;
}
try {
e.getQueue().put(r);
} catch (InterruptedException ex) {
log.debug("interrupted while waiting for a slot in the queue.", ex);
Thread.currentThread().interrupt();
}
});
return pool;
}
}
package com.commafeed.backend.feed;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedRefreshEngine {
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedRefreshWorker worker;
private final FeedRefreshUpdater updater;
private final CommaFeedConfiguration config;
private final Meter refill;
private final BlockingDeque<Feed> queue;
private final ExecutorService feedProcessingLoopExecutor;
private final ExecutorService refillLoopExecutor;
private final ExecutorService refillExecutor;
private final ThreadPoolExecutor workerExecutor;
private final ThreadPoolExecutor databaseUpdaterExecutor;
public FeedRefreshEngine(UnitOfWork unitOfWork, FeedDAO feedDAO, FeedRefreshWorker worker, FeedRefreshUpdater updater,
CommaFeedConfiguration config, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.worker = worker;
this.updater = updater;
this.config = config;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
this.queue = new LinkedBlockingDeque<>();
this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor();
this.refillLoopExecutor = Executors.newSingleThreadExecutor();
this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.feedRefresh().databaseThreads());
metrics.register(MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(MetricRegistry.name(getClass(), "worker", "active"), (Gauge<Integer>) workerExecutor::getActiveCount);
metrics.register(MetricRegistry.name(getClass(), "updater", "active"), (Gauge<Integer>) databaseUpdaterExecutor::getActiveCount);
}
public void start() {
startFeedProcessingLoop();
startRefillLoop();
}
private void startFeedProcessingLoop() {
// take a feed from the queue, process it, rince, repeat
feedProcessingLoopExecutor.submit(() -> {
while (!feedProcessingLoopExecutor.isShutdown()) {
try {
// take() is blocking until a feed is available from the queue
Feed feed = queue.take();
// send the feed to be processed
log.debug("got feed {} from the queue, send it for processing", feed.getId());
processFeedAsync(feed);
// we removed a feed from the queue, try to refill it as it may now be empty
if (queue.isEmpty()) {
log.debug("took the last feed from the queue, try to refill");
refillQueueAsync();
}
} catch (InterruptedException e) {
log.debug("interrupted while waiting for a feed in the queue");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
private void startRefillLoop() {
// refill the queue at regular intervals if it's empty
refillLoopExecutor.submit(() -> {
while (!refillLoopExecutor.isShutdown()) {
try {
if (queue.isEmpty()) {
log.debug("refilling queue");
refillQueueAsync();
}
log.debug("sleeping for 15s");
TimeUnit.SECONDS.sleep(15);
} catch (InterruptedException e) {
log.debug("interrupted while sleeping");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
public void refreshImmediately(Feed feed) {
log.debug("add feed {} at the start of the queue", feed.getId());
// remove the feed from the queue if it was already queued to avoid refreshing it twice
queue.removeIf(f -> f.getId().equals(feed.getId()));
queue.addFirst(feed);
}
private void refillQueueAsync() {
CompletableFuture.runAsync(() -> {
if (!queue.isEmpty()) {
return;
}
refill.mark();
List<Feed> nextUpdatableFeeds = getNextUpdatableFeeds(getBatchSize());
log.debug("found {} feeds that are up for refresh", nextUpdatableFeeds.size());
for (Feed feed : nextUpdatableFeeds) {
// add the feed only if it was not already queued
if (queue.stream().noneMatch(f -> f.getId().equals(feed.getId()))) {
queue.addLast(feed);
}
}
}, refillExecutor).whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while refilling the queue", ex);
}
});
}
private void processFeedAsync(Feed feed) {
CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
.thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
.whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while processing feed {}", feed.getUrl(), ex);
}
});
}
private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(() -> {
Instant lastLoginThreshold = config.feedRefresh().userInactivityPeriod().isZero() ? null
: Instant.now().minus(config.feedRefresh().userInactivityPeriod());
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
// update disabledUntil to prevent feeds from being returned again by feedDAO.findNextUpdatable()
Instant nextUpdateDate = Instant.now().plus(config.feedRefresh().interval());
feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
return feeds;
});
}
private int getBatchSize() {
return Math.min(100, 3 * config.feedRefresh().httpThreads());
}
public void stop() {
this.feedProcessingLoopExecutor.shutdownNow();
this.refillLoopExecutor.shutdownNow();
this.refillExecutor.shutdownNow();
this.workerExecutor.shutdownNow();
this.databaseUpdaterExecutor.shutdownNow();
}
/**
* returns an ExecutorService with a single thread that discards tasks if a task is already running
*/
private ThreadPoolExecutor newDiscardingSingleThreadExecutorService() {
ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/**
* returns an ExecutorService that blocks submissions until a thread is available
*/
private ThreadPoolExecutor newBlockingExecutorService(int threads) {
ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler((r, e) -> {
if (e.isShutdown()) {
return;
}
try {
e.getQueue().put(r);
} catch (InterruptedException ex) {
log.debug("interrupted while waiting for a slot in the queue.", ex);
Thread.currentThread().interrupt();
}
});
return pool;
}
}

168
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshIntervalCalculator.java
View File

@@ -1,84 +1,84 @@
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.FeedRefreshErrorHandling;
import com.google.common.primitives.Longs;
@Singleton
public class FeedRefreshIntervalCalculator {
private final Duration interval;
private final Duration maxInterval;
private final boolean empirical;
private final FeedRefreshErrorHandling errorHandling;
private final InstantSource instantSource;
public FeedRefreshIntervalCalculator(CommaFeedConfiguration config, InstantSource instantSource) {
this.interval = config.feedRefresh().interval();
this.maxInterval = config.feedRefresh().maxInterval();
this.empirical = config.feedRefresh().intervalEmpirical();
this.errorHandling = config.feedRefresh().errors();
this.instantSource = instantSource;
}
public Instant onFetchSuccess(Instant publishedDate, Long averageEntryInterval, Duration validFor) {
Instant instant = empirical ? computeEmpiricalRefreshInterval(publishedDate, averageEntryInterval)
: instantSource.instant().plus(interval);
return constrainToBounds(ObjectUtils.max(instant, instantSource.instant().plus(validFor)));
}
public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) {
return onFetchSuccess(publishedDate, averageEntryInterval, Duration.ZERO);
}
public Instant onTooManyRequests(Instant retryAfter, int errorCount) {
return constrainToBounds(ObjectUtils.max(retryAfter, onFetchError(errorCount)));
}
public Instant onFetchError(int errorCount) {
if (errorCount < errorHandling.retriesBeforeBackoff()) {
return constrainToBounds(instantSource.instant().plus(interval));
}
Duration retryInterval = errorHandling.backoffInterval().multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L);
return constrainToBounds(instantSource.instant().plus(retryInterval));
}
private Instant computeEmpiricalRefreshInterval(Instant publishedDate, Long averageEntryInterval) {
Instant now = instantSource.instant();
if (publishedDate == null) {
return now.plus(maxInterval);
}
long daysSinceLastPublication = ChronoUnit.DAYS.between(publishedDate, now);
if (daysSinceLastPublication >= 30) {
return now.plus(maxInterval);
} else if (daysSinceLastPublication >= 14) {
return now.plus(maxInterval.dividedBy(2));
} else if (daysSinceLastPublication >= 7) {
return now.plus(maxInterval.dividedBy(4));
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor
int factor = 2;
long millis = Longs.constrainToRange(averageEntryInterval / factor, interval.toMillis(), maxInterval.dividedBy(4).toMillis());
return now.plusMillis(millis);
} else {
// unknown case
return now.plus(maxInterval);
}
}
private Instant constrainToBounds(Instant instant) {
return ObjectUtils.max(ObjectUtils.min(instant, instantSource.instant().plus(maxInterval)), instantSource.instant().plus(interval));
}
}
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.FeedRefreshErrorHandling;
import com.google.common.primitives.Longs;
@Singleton
public class FeedRefreshIntervalCalculator {
private final Duration interval;
private final Duration maxInterval;
private final boolean empirical;
private final FeedRefreshErrorHandling errorHandling;
private final InstantSource instantSource;
public FeedRefreshIntervalCalculator(CommaFeedConfiguration config, InstantSource instantSource) {
this.interval = config.feedRefresh().interval();
this.maxInterval = config.feedRefresh().maxInterval();
this.empirical = config.feedRefresh().intervalEmpirical();
this.errorHandling = config.feedRefresh().errors();
this.instantSource = instantSource;
}
public Instant onFetchSuccess(Instant publishedDate, Long averageEntryInterval, Duration validFor) {
Instant instant = empirical ? computeEmpiricalRefreshInterval(publishedDate, averageEntryInterval)
: instantSource.instant().plus(interval);
return constrainToBounds(ObjectUtils.max(instant, instantSource.instant().plus(validFor)));
}
public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) {
return onFetchSuccess(publishedDate, averageEntryInterval, Duration.ZERO);
}
public Instant onTooManyRequests(Instant retryAfter, int errorCount) {
return constrainToBounds(ObjectUtils.max(retryAfter, onFetchError(errorCount)));
}
public Instant onFetchError(int errorCount) {
if (errorCount < errorHandling.retriesBeforeBackoff()) {
return constrainToBounds(instantSource.instant().plus(interval));
}
Duration retryInterval = errorHandling.backoffInterval().multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L);
return constrainToBounds(instantSource.instant().plus(retryInterval));
}
private Instant computeEmpiricalRefreshInterval(Instant publishedDate, Long averageEntryInterval) {
Instant now = instantSource.instant();
if (publishedDate == null) {
return now.plus(maxInterval);
}
long daysSinceLastPublication = ChronoUnit.DAYS.between(publishedDate, now);
if (daysSinceLastPublication >= 30) {
return now.plus(maxInterval);
} else if (daysSinceLastPublication >= 14) {
return now.plus(maxInterval.dividedBy(2));
} else if (daysSinceLastPublication >= 7) {
return now.plus(maxInterval.dividedBy(4));
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor
int factor = 2;
long millis = Longs.constrainToRange(averageEntryInterval / factor, interval.toMillis(), maxInterval.dividedBy(4).toMillis());
return now.plusMillis(millis);
} else {
// unknown case
return now.plus(maxInterval);
}
}
private Instant constrainToBounds(Instant instant) {
return ObjectUtils.max(ObjectUtils.min(instant, instantSource.instant().plus(maxInterval)), instantSource.instant().plus(interval));
}
}

360
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
View File

@@ -1,180 +1,180 @@
package com.commafeed.backend.feed;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions;
import com.google.common.util.concurrent.Striped;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Updates the feed in the database and inserts new entries
*/
@Slf4j
@Singleton
public class FeedRefreshUpdater {
private final UnitOfWork unitOfWork;
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final WebSocketSessions webSocketSessions;
private final Striped<Lock> locks;
private final Meter feedUpdated;
private final Meter entryInserted;
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO, WebSocketSessions webSocketSessions) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.webSocketSessions = webSocketSessions;
locks = Striped.lazyWeakLock(100000);
feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
private AddEntryResult addEntry(final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
boolean processed = false;
boolean inserted = false;
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread = new HashSet<>();
// lock on feed, make sure we are not updating the same feed twice at
// the same time
String key1 = StringUtils.trimToEmpty(String.valueOf(feed.getId()));
// lock on content, make sure we are not updating the same entry
// twice at the same time
Content content = entry.content();
String key2 = Digests.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title()));
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
Lock lock2 = iterator.next();
boolean locked1 = false;
boolean locked2 = false;
try {
// try to lock, give up after 1 minute
locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
processed = true;
inserted = unitOfWork.call(() -> {
boolean newEntry = false;
FeedEntry feedEntry = feedEntryService.find(feed, entry);
if (feedEntry == null) {
feedEntry = feedEntryService.create(feed, entry);
newEntry = true;
}
if (newEntry) {
entryInserted.mark();
for (FeedSubscription sub : subscriptions) {
boolean unread = feedEntryService.applyFilter(sub, feedEntry);
if (unread) {
subscriptionsForWhichEntryIsUnread.add(sub);
}
}
}
return newEntry;
});
} else {
log.error("lock timeout for {} - {}", feed.getUrl(), key1);
}
} catch (InterruptedException e) {
log.error("interrupted while waiting for lock for {} : {}", feed.getUrl(), e.getMessage(), e);
} finally {
if (locked1) {
lock1.unlock();
}
if (locked2) {
lock2.unlock();
}
}
return new AddEntryResult(processed, inserted, subscriptionsForWhichEntryIsUnread);
}
public boolean update(Feed feed, List<Entry> entries) {
boolean processed = true;
long inserted = 0;
Map<FeedSubscription, Long> unreadCountBySubscription = new HashMap<>();
if (!entries.isEmpty()) {
List<FeedSubscription> subscriptions = null;
for (Entry entry : entries) {
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
processed &= addEntryResult.processed;
inserted += addEntryResult.inserted ? 1 : 0;
addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(sub -> unreadCountBySubscription.merge(sub, 1L, Long::sum));
}
if (inserted == 0) {
feed.setMessage("No new entries found");
} else if (inserted > 0) {
feed.setMessage("Found %s new entries".formatted(inserted));
}
}
if (!processed) {
// requeue asap
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
}
if (inserted > 0) {
feedUpdated.mark();
}
unitOfWork.run(() -> feedService.update(feed));
notifyOverWebsocket(unreadCountBySubscription);
return processed;
}
private void notifyOverWebsocket(Map<FeedSubscription, Long> unreadCountBySubscription) {
unreadCountBySubscription.forEach((sub, unreadCount) -> webSocketSessions.sendMessage(sub.getUser(),
WebSocketMessageBuilder.newFeedEntries(sub, unreadCount)));
}
@AllArgsConstructor
private static class AddEntryResult {
private final boolean processed;
private final boolean inserted;
private final Set<FeedSubscription> subscriptionsForWhichEntryIsUnread;
}
}
package com.commafeed.backend.feed;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions;
import com.google.common.util.concurrent.Striped;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Updates the feed in the database and inserts new entries
*/
@Slf4j
@Singleton
public class FeedRefreshUpdater {
private final UnitOfWork unitOfWork;
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final WebSocketSessions webSocketSessions;
private final Striped<Lock> locks;
private final Meter feedUpdated;
private final Meter entryInserted;
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO, WebSocketSessions webSocketSessions) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.webSocketSessions = webSocketSessions;
locks = Striped.lazyWeakLock(100000);
feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
private AddEntryResult addEntry(final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
boolean processed = false;
boolean inserted = false;
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread = new HashSet<>();
// lock on feed, make sure we are not updating the same feed twice at
// the same time
String key1 = StringUtils.trimToEmpty(String.valueOf(feed.getId()));
// lock on content, make sure we are not updating the same entry
// twice at the same time
Content content = entry.content();
String key2 = Digests.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title()));
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
Lock lock2 = iterator.next();
boolean locked1 = false;
boolean locked2 = false;
try {
// try to lock, give up after 1 minute
locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
processed = true;
inserted = unitOfWork.call(() -> {
boolean newEntry = false;
FeedEntry feedEntry = feedEntryService.find(feed, entry);
if (feedEntry == null) {
feedEntry = feedEntryService.create(feed, entry);
newEntry = true;
}
if (newEntry) {
entryInserted.mark();
for (FeedSubscription sub : subscriptions) {
boolean unread = feedEntryService.applyFilter(sub, feedEntry);
if (unread) {
subscriptionsForWhichEntryIsUnread.add(sub);
}
}
}
return newEntry;
});
} else {
log.error("lock timeout for {} - {}", feed.getUrl(), key1);
}
} catch (InterruptedException e) {
log.error("interrupted while waiting for lock for {} : {}", feed.getUrl(), e.getMessage(), e);
} finally {
if (locked1) {
lock1.unlock();
}
if (locked2) {
lock2.unlock();
}
}
return new AddEntryResult(processed, inserted, subscriptionsForWhichEntryIsUnread);
}
public boolean update(Feed feed, List<Entry> entries) {
boolean processed = true;
long inserted = 0;
Map<FeedSubscription, Long> unreadCountBySubscription = new HashMap<>();
if (!entries.isEmpty()) {
List<FeedSubscription> subscriptions = null;
for (Entry entry : entries) {
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
processed &= addEntryResult.processed;
inserted += addEntryResult.inserted ? 1 : 0;
addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(sub -> unreadCountBySubscription.merge(sub, 1L, Long::sum));
}
if (inserted == 0) {
feed.setMessage("No new entries found");
} else if (inserted > 0) {
feed.setMessage("Found %s new entries".formatted(inserted));
}
}
if (!processed) {
// requeue asap
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
}
if (inserted > 0) {
feedUpdated.mark();
}
unitOfWork.run(() -> feedService.update(feed));
notifyOverWebsocket(unreadCountBySubscription);
return processed;
}
private void notifyOverWebsocket(Map<FeedSubscription, Long> unreadCountBySubscription) {
unreadCountBySubscription.forEach((sub, unreadCount) -> webSocketSessions.sendMessage(sub.getUser(),
WebSocketMessageBuilder.newFeedEntries(sub, unreadCount)));
}
@AllArgsConstructor
private static class AddEntryResult {
private final boolean processed;
private final boolean inserted;
private final Set<FeedSubscription> subscriptionsForWhichEntryIsUnread;
}
}

250
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
View File

@@ -1,125 +1,125 @@
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
/**
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link FeedRefreshUpdater} does that)
*/
@Slf4j
@Singleton
public class FeedRefreshWorker {
private final FeedRefreshIntervalCalculator refreshIntervalCalculator;
private final FeedFetcher fetcher;
private final CommaFeedConfiguration config;
private final Meter feedFetched;
public FeedRefreshWorker(FeedRefreshIntervalCalculator refreshIntervalCalculator, FeedFetcher fetcher, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.refreshIntervalCalculator = refreshIntervalCalculator;
this.fetcher = fetcher;
this.config = config;
this.feedFetched = metrics.meter(MetricRegistry.name(getClass(), "feedFetched"));
}
public FeedRefreshWorkerResult update(Feed feed) {
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FeedFetcherResult result = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
List<Entry> entries = result.feed().entries();
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
entries = entries.stream().filter(entry -> entry.published().isAfter(threshold)).toList();
}
String urlAfterRedirect = result.urlAfterRedirect();
if (StringUtils.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;
}
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(result.feed().link());
feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(result.contentHash());
feed.setLastPublishedDate(result.feed().lastPublishedDate());
feed.setAverageEntryInterval(result.feed().averageEntryInterval());
feed.setLastEntryDate(result.feed().lastEntryDate());
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(),
result.feed().averageEntryInterval(), result.validFor()));
return new FeedRefreshWorkerResult(feed, entries);
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
feed.setErrorCount(0);
feed.setMessage(e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFeedNotModified(feed.getLastPublishedDate(), feed.getAverageEntryInterval()));
if (e.getNewLastModifiedHeader() != null) {
feed.setLastModifiedHeader(e.getNewLastModifiedHeader());
}
if (e.getNewEtagHeader() != null) {
feed.setEtagHeader(e.getNewEtagHeader());
}
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (TooManyRequestsException e) {
log.debug("Too many requests : {}", feed.getUrl());
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Server indicated that we are sending too many requests");
feed.setDisabledUntil(refreshIntervalCalculator.onTooManyRequests(e.getRetryAfter(), feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (Exception e) {
log.debug("unable to refresh feed {}", feed.getUrl(), e);
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Unable to refresh feed : " + e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} finally {
feedFetched.mark();
}
}
public record FeedRefreshWorkerResult(Feed feed, List<Entry> entries) {
}
}
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
/**
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link FeedRefreshUpdater} does that)
*/
@Slf4j
@Singleton
public class FeedRefreshWorker {
private final FeedRefreshIntervalCalculator refreshIntervalCalculator;
private final FeedFetcher fetcher;
private final CommaFeedConfiguration config;
private final Meter feedFetched;
public FeedRefreshWorker(FeedRefreshIntervalCalculator refreshIntervalCalculator, FeedFetcher fetcher, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.refreshIntervalCalculator = refreshIntervalCalculator;
this.fetcher = fetcher;
this.config = config;
this.feedFetched = metrics.meter(MetricRegistry.name(getClass(), "feedFetched"));
}
public FeedRefreshWorkerResult update(Feed feed) {
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FeedFetcherResult result = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
List<Entry> entries = result.feed().entries();
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
entries = entries.stream().filter(entry -> entry.published().isAfter(threshold)).toList();
}
String urlAfterRedirect = result.urlAfterRedirect();
if (StringUtils.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;
}
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(result.feed().link());
feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(result.contentHash());
feed.setLastPublishedDate(result.feed().lastPublishedDate());
feed.setAverageEntryInterval(result.feed().averageEntryInterval());
feed.setLastEntryDate(result.feed().lastEntryDate());
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(),
result.feed().averageEntryInterval(), result.validFor()));
return new FeedRefreshWorkerResult(feed, entries);
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
feed.setErrorCount(0);
feed.setMessage(e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFeedNotModified(feed.getLastPublishedDate(), feed.getAverageEntryInterval()));
if (e.getNewLastModifiedHeader() != null) {
feed.setLastModifiedHeader(e.getNewLastModifiedHeader());
}
if (e.getNewEtagHeader() != null) {
feed.setEtagHeader(e.getNewEtagHeader());
}
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (TooManyRequestsException e) {
log.debug("Too many requests : {}", feed.getUrl());
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Server indicated that we are sending too many requests");
feed.setDisabledUntil(refreshIntervalCalculator.onTooManyRequests(e.getRetryAfter(), feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (Exception e) {
log.debug("unable to refresh feed {}", feed.getUrl(), e);
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Unable to refresh feed : " + e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} finally {
feedFetched.mark();
}
}
public record FeedRefreshWorkerResult(Feed feed, List<Entry> entries) {
}
}

436
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java
View File

@@ -1,218 +1,218 @@
package com.commafeed.backend.feed;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.utils.Base64;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.netpreserve.urlcanon.Canonicalizer;
import org.netpreserve.urlcanon.ParsedUrl;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.feed.parser.TextDirectionDetector;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry;
import lombok.extern.slf4j.Slf4j;
/**
* Utility methods related to feed handling
*
*/
@Slf4j
public class FeedUtils {
private static final String ESCAPED_QUESTION_MARK = Pattern.quote("?");
public static String truncate(String string, int length) {
if (string != null) {
string = string.substring(0, Math.min(length, string.length()));
}
return string;
}
public static boolean isHttp(String url) {
return url.startsWith("http://");
}
public static boolean isHttps(String url) {
return url.startsWith("https://");
}
public static boolean isAbsoluteUrl(String url) {
return isHttp(url) || isHttps(url);
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates
*/
public static String normalizeURL(String url) {
if (url == null) {
return null;
}
ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
String normalized = parsedUrl.toString();
if (normalized == null) {
normalized = url;
}
// convert to lower case, the url probably won't work in some cases
// after that but we don't care we just want to compare urls to avoid
// duplicates
normalized = normalized.toLowerCase();
// store all urls as http
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
// remove the www. part
normalized = normalized.replace("//www.", "//");
// feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedburner feeds have a special treatment
if (normalized.split(ESCAPED_QUESTION_MARK)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = normalized.split(ESCAPED_QUESTION_MARK)[0];
normalized = StringUtils.removeEnd(normalized, "/");
}
return normalized;
}
public static boolean isRTL(String title, String content) {
String text = StringUtils.isNotBlank(content) ? content : title;
if (StringUtils.isBlank(text)) {
return false;
}
String stripped = Jsoup.parse(text).text();
if (StringUtils.isBlank(stripped)) {
return false;
}
return TextDirectionDetector.detect(stripped) == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
public static String removeTrailingSlash(String url) {
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}
return url;
}
/**
*
* @param relativeUrl
* the url of the entry
* @param feedLink
* the url of the feed as described in the feed
* @param feedUrl
* the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry
*/
public static String toAbsoluteUrl(String relativeUrl, String feedLink, String feedUrl) {
String baseUrl = (feedLink != null && isAbsoluteUrl(feedLink)) ? feedLink : feedUrl;
if (baseUrl == null) {
return null;
}
try {
return new URL(new URL(baseUrl), relativeUrl).toString();
} catch (MalformedURLException e) {
log.debug("could not parse url : {}", e.getMessage(), e);
return null;
}
}
public static String getFaviconUrl(FeedSubscription subscription) {
return "rest/feed/favicon/" + subscription.getId();
}
public static String proxyImages(String content) {
if (StringUtils.isBlank(content)) {
return content;
}
Document doc = Jsoup.parse(content);
Elements elements = doc.select("img");
for (Element element : elements) {
String href = element.attr("src");
if (StringUtils.isNotBlank(href)) {
String proxy = proxyImage(href);
element.attr("src", proxy);
}
}
return doc.body().html();
}
public static String proxyImage(String url) {
if (StringUtils.isBlank(url)) {
return url;
}
return "rest/server/proxy?u=" + imageProxyEncoder(url);
}
public static String rot13(String msg) {
StringBuilder message = new StringBuilder();
for (char c : msg.toCharArray()) {
if (c >= 'a' && c <= 'm') {
c += 13;
} else if (c >= 'n' && c <= 'z') {
c -= 13;
} else if (c >= 'A' && c <= 'M') {
c += 13;
} else if (c >= 'N' && c <= 'Z') {
c -= 13;
}
message.append(c);
}
return message.toString();
}
public static String imageProxyEncoder(String url) {
return Base64.encodeBase64String(rot13(url).getBytes());
}
public static String imageProxyDecoder(String code) {
return rot13(new String(Base64.decodeBase64(code)));
}
public static void removeUnwantedFromSearch(List<Entry> entries, List<FeedEntryKeyword> keywords) {
Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
Entry entry = it.next();
boolean keep = true;
for (FeedEntryKeyword keyword : keywords) {
String title = entry.getTitle() == null ? null : Jsoup.parse(entry.getTitle()).text();
String content = entry.getContent() == null ? null : Jsoup.parse(entry.getContent()).text();
boolean condition = !StringUtils.containsIgnoreCase(content, keyword.getKeyword())
&& !StringUtils.containsIgnoreCase(title, keyword.getKeyword());
if (keyword.getMode() == Mode.EXCLUDE) {
condition = !condition;
}
if (condition) {
keep = false;
break;
}
}
if (!keep) {
it.remove();
}
}
}
}
package com.commafeed.backend.feed;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.utils.Base64;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.netpreserve.urlcanon.Canonicalizer;
import org.netpreserve.urlcanon.ParsedUrl;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.feed.parser.TextDirectionDetector;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry;
import lombok.extern.slf4j.Slf4j;
/**
* Utility methods related to feed handling
*
*/
@Slf4j
public class FeedUtils {
private static final String ESCAPED_QUESTION_MARK = Pattern.quote("?");
public static String truncate(String string, int length) {
if (string != null) {
string = string.substring(0, Math.min(length, string.length()));
}
return string;
}
public static boolean isHttp(String url) {
return url.startsWith("http://");
}
public static boolean isHttps(String url) {
return url.startsWith("https://");
}
public static boolean isAbsoluteUrl(String url) {
return isHttp(url) || isHttps(url);
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates
*/
public static String normalizeURL(String url) {
if (url == null) {
return null;
}
ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
String normalized = parsedUrl.toString();
if (normalized == null) {
normalized = url;
}
// convert to lower case, the url probably won't work in some cases
// after that but we don't care we just want to compare urls to avoid
// duplicates
normalized = normalized.toLowerCase();
// store all urls as http
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
// remove the www. part
normalized = normalized.replace("//www.", "//");
// feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedburner feeds have a special treatment
if (normalized.split(ESCAPED_QUESTION_MARK)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = normalized.split(ESCAPED_QUESTION_MARK)[0];
normalized = StringUtils.removeEnd(normalized, "/");
}
return normalized;
}
public static boolean isRTL(String title, String content) {
String text = StringUtils.isNotBlank(content) ? content : title;
if (StringUtils.isBlank(text)) {
return false;
}
String stripped = Jsoup.parse(text).text();
if (StringUtils.isBlank(stripped)) {
return false;
}
return TextDirectionDetector.detect(stripped) == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
public static String removeTrailingSlash(String url) {
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}
return url;
}
/**
*
* @param relativeUrl
* the url of the entry
* @param feedLink
* the url of the feed as described in the feed
* @param feedUrl
* the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry
*/
public static String toAbsoluteUrl(String relativeUrl, String feedLink, String feedUrl) {
String baseUrl = (feedLink != null && isAbsoluteUrl(feedLink)) ? feedLink : feedUrl;
if (baseUrl == null) {
return null;
}
try {
return new URL(new URL(baseUrl), relativeUrl).toString();
} catch (MalformedURLException e) {
log.debug("could not parse url : {}", e.getMessage(), e);
return null;
}
}
public static String getFaviconUrl(FeedSubscription subscription) {
return "rest/feed/favicon/" + subscription.getId();
}
public static String proxyImages(String content) {
if (StringUtils.isBlank(content)) {
return content;
}
Document doc = Jsoup.parse(content);
Elements elements = doc.select("img");
for (Element element : elements) {
String href = element.attr("src");
if (StringUtils.isNotBlank(href)) {
String proxy = proxyImage(href);
element.attr("src", proxy);
}
}
return doc.body().html();
}
public static String proxyImage(String url) {
if (StringUtils.isBlank(url)) {
return url;
}
return "rest/server/proxy?u=" + imageProxyEncoder(url);
}
public static String rot13(String msg) {
StringBuilder message = new StringBuilder();
for (char c : msg.toCharArray()) {
if (c >= 'a' && c <= 'm') {
c += 13;
} else if (c >= 'n' && c <= 'z') {
c -= 13;
} else if (c >= 'A' && c <= 'M') {
c += 13;
} else if (c >= 'N' && c <= 'Z') {
c -= 13;
}
message.append(c);
}
return message.toString();
}
public static String imageProxyEncoder(String url) {
return Base64.encodeBase64String(rot13(url).getBytes());
}
public static String imageProxyDecoder(String code) {
return rot13(new String(Base64.decodeBase64(code)));
}
public static void removeUnwantedFromSearch(List<Entry> entries, List<FeedEntryKeyword> keywords) {
Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
Entry entry = it.next();
boolean keep = true;
for (FeedEntryKeyword keyword : keywords) {
String title = entry.getTitle() == null ? null : Jsoup.parse(entry.getTitle()).text();
String content = entry.getContent() == null ? null : Jsoup.parse(entry.getContent()).text();
boolean condition = !StringUtils.containsIgnoreCase(content, keyword.getKeyword())
&& !StringUtils.containsIgnoreCase(title, keyword.getKeyword());
if (keyword.getMode() == Mode.EXCLUDE) {
condition = !condition;
}
if (condition) {
keep = false;
break;
}
}
if (!keep) {
it.remove();
}
}
}
}

140
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/EncodingDetector.java
View File

@@ -1,70 +1,70 @@
package com.commafeed.backend.feed.parser;
import java.nio.charset.Charset;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
@Singleton
class EncodingDetector {
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public Charset getEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (StringUtils.startsWithIgnoreCase(extracted, "iso-8859-")) {
if (!StringUtils.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (StringUtils.startsWithIgnoreCase(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Extract the declared encoding from the xml
*/
public String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = StringUtils.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
/**
* Detect encoding by analyzing characters in the array
*/
private Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
}
package com.commafeed.backend.feed.parser;
import java.nio.charset.Charset;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
@Singleton
class EncodingDetector {
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public Charset getEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (StringUtils.startsWithIgnoreCase(extracted, "iso-8859-")) {
if (!StringUtils.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (StringUtils.startsWithIgnoreCase(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Extract the declared encoding from the xml
*/
public String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = StringUtils.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
/**
* Detect encoding by analyzing characters in the array
*/
private Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
}

140
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedCleaner.java
View File

@@ -1,70 +1,70 @@
package com.commafeed.backend.feed.parser;
import java.util.Collection;
import java.util.regex.Pattern;
import jakarta.inject.Singleton;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.apache.commons.lang3.StringUtils;
@Singleton
class FeedCleaner {
private static final Pattern DOCTYPE_PATTERN = Pattern.compile("<!DOCTYPE[^>]*>", Pattern.CASE_INSENSITIVE);
public String trimInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
StringBuilder sb = new StringBuilder();
boolean firstTagFound = false;
for (int i = 0; i < xml.length(); i++) {
char c = xml.charAt(i);
if (!firstTagFound) {
if (c == '<') {
firstTagFound = true;
} else {
continue;
}
}
if (c >= 32 || c == 9 || c == 10 || c == 13) {
if (!Character.isHighSurrogate(c) && !Character.isLowSurrogate(c)) {
sb.append(c);
}
}
}
return sb.toString();
}
// https://stackoverflow.com/a/40836618
public String replaceHtmlEntitiesWithNumericEntities(String source) {
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
Collection<Emit> emits = Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build().parseText(source);
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
return sb.toString();
}
public String removeDoctypeDeclarations(String xml) {
return DOCTYPE_PATTERN.matcher(xml).replaceAll("");
}
}
package com.commafeed.backend.feed.parser;
import java.util.Collection;
import java.util.regex.Pattern;
import jakarta.inject.Singleton;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.apache.commons.lang3.StringUtils;
@Singleton
class FeedCleaner {
private static final Pattern DOCTYPE_PATTERN = Pattern.compile("<!DOCTYPE[^>]*>", Pattern.CASE_INSENSITIVE);
public String trimInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
StringBuilder sb = new StringBuilder();
boolean firstTagFound = false;
for (int i = 0; i < xml.length(); i++) {
char c = xml.charAt(i);
if (!firstTagFound) {
if (c == '<') {
firstTagFound = true;
} else {
continue;
}
}
if (c >= 32 || c == 9 || c == 10 || c == 13) {
if (!Character.isHighSurrogate(c) && !Character.isLowSurrogate(c)) {
sb.append(c);
}
}
}
return sb.toString();
}
// https://stackoverflow.com/a/40836618
public String replaceHtmlEntitiesWithNumericEntities(String source) {
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
Collection<Emit> emits = Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build().parseText(source);
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
return sb.toString();
}
public String removeDoctypeDeclarations(String xml) {
return DOCTYPE_PATTERN.matcher(xml).replaceAll("");
}
}

570
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParser.java
View File

@@ -1,285 +1,285 @@
package com.commafeed.backend.feed.parser;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.google.common.collect.Iterables;
import com.rometools.modules.mediarss.MediaEntryModule;
import com.rometools.modules.mediarss.MediaModule;
import com.rometools.modules.mediarss.types.MediaGroup;
import com.rometools.modules.mediarss.types.Metadata;
import com.rometools.modules.mediarss.types.Thumbnail;
import com.rometools.rome.feed.synd.SyndCategory;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.SyndFeedInput;
import lombok.RequiredArgsConstructor;
/**
* Parses raw xml into a FeedParserResult object
*/
@RequiredArgsConstructor
@Singleton
public class FeedParser {
private static final Namespace ATOM_10_NS = Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Instant START = Instant.ofEpochMilli(86400000);
private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
private final EncodingDetector encodingDetector;
private final FeedCleaner feedCleaner;
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedParsingException {
try {
Charset encoding = encodingDetector.getEncoding(xml);
String xmlString = feedCleaner.trimInvalidXmlCharacters(new String(xml, encoding));
if (xmlString == null) {
throw new FeedParsingException("Input string is null for url " + feedUrl);
}
xmlString = feedCleaner.replaceHtmlEntitiesWithNumericEntities(xmlString);
xmlString = feedCleaner.removeDoctypeDeclarations(xmlString);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed feed = new SyndFeedInput().build(source);
handleForeignMarkup(feed);
String title = feed.getTitle();
String link = feed.getLink();
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
if (lastPublishedDate == null || lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
lastPublishedDate = lastEntryDate;
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
return new FeedParserResult(title, link, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
} catch (FeedParsingException e) {
throw e;
} catch (Exception e) {
throw new FeedParsingException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
/**
* Adds atom links for rss feeds
*/
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
private List<Entry> buildEntries(SyndFeed feed, String feedUrl) {
List<Entry> entries = new ArrayList<>();
for (SyndEntry item : feed.getEntries()) {
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
String url = buildEntryUrl(feed, feedUrl, item);
if (StringUtils.isBlank(url) && FeedUtils.isAbsoluteUrl(guid)) {
// if link is empty but guid is used as url, use guid
url = guid;
}
Instant publishedDate = buildEntryPublishedDate(item);
Content content = buildContent(item);
entries.add(new Entry(guid, url, publishedDate, content));
}
entries.sort(Comparator.comparing(Entry::published).reversed());
return entries;
}
private Content buildContent(SyndEntry item) {
String title = getTitle(item);
String content = getContent(item);
String author = StringUtils.trimToNull(item.getAuthor());
String categories = StringUtils
.trimToNull(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")));
Enclosure enclosure = buildEnclosure(item);
Media media = buildMedia(item);
return new Content(title, content, author, categories, enclosure, media);
}
private Enclosure buildEnclosure(SyndEntry item) {
SyndEnclosure enclosure = Iterables.getFirst(item.getEnclosures(), null);
if (enclosure == null) {
return null;
}
return new Enclosure(enclosure.getUrl(), enclosure.getType());
}
private Instant buildEntryPublishedDate(SyndEntry item) {
Date date = item.getPublishedDate();
if (date == null) {
date = item.getUpdatedDate();
}
return toValidInstant(date, true);
}
private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
if (url == null || FeedUtils.isAbsoluteUrl(url)) {
// url is absolute, nothing to do
return url;
}
// url is relative, trying to resolve it
String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
return FeedUtils.toAbsoluteUrl(url, feedLink, feedUrl);
}
private Instant toValidInstant(Date date, boolean nullToNow) {
Instant now = Instant.now();
if (date == null) {
return nullToNow ? now : null;
}
Instant instant = date.toInstant();
if (instant.isBefore(START) || instant.isAfter(END)) {
return now;
}
if (instant.isAfter(now)) {
return now;
}
return instant;
}
private String getContent(SyndEntry item) {
String content;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private Media buildMedia(SyndEntry item) {
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module == null) {
return null;
}
Media media = buildMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = buildMedia(group.getMetadata());
}
return media;
}
private Media buildMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
String description = metadata.getDescription();
String thumbnailUrl = null;
Integer thumbnailWidth = null;
Integer thumbnailHeight = null;
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
thumbnailWidth = thumbnail.getWidth();
thumbnailHeight = thumbnail.getHeight();
if (thumbnail.getUrl() != null) {
thumbnailUrl = thumbnail.getUrl().toString();
}
}
if (description == null && thumbnailUrl == null) {
return null;
}
return new Media(description, thumbnailUrl, thumbnailWidth, thumbnailHeight);
}
private Long averageTimeBetweenEntries(List<Entry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < entries.size() - 1; i++) {
long diff = Math.abs(entries.get(i).published().toEpochMilli() - entries.get(i + 1).published().toEpochMilli());
stats.addValue(diff);
}
return (long) stats.getMean();
}
public static class FeedParsingException extends Exception {
private static final long serialVersionUID = 1L;
public FeedParsingException(String message) {
super(message);
}
public FeedParsingException(String message, Throwable cause) {
super(message, cause);
}
}
}
package com.commafeed.backend.feed.parser;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.google.common.collect.Iterables;
import com.rometools.modules.mediarss.MediaEntryModule;
import com.rometools.modules.mediarss.MediaModule;
import com.rometools.modules.mediarss.types.MediaGroup;
import com.rometools.modules.mediarss.types.Metadata;
import com.rometools.modules.mediarss.types.Thumbnail;
import com.rometools.rome.feed.synd.SyndCategory;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.SyndFeedInput;
import lombok.RequiredArgsConstructor;
/**
* Parses raw xml into a FeedParserResult object
*/
@RequiredArgsConstructor
@Singleton
public class FeedParser {
private static final Namespace ATOM_10_NS = Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Instant START = Instant.ofEpochMilli(86400000);
private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
private final EncodingDetector encodingDetector;
private final FeedCleaner feedCleaner;
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedParsingException {
try {
Charset encoding = encodingDetector.getEncoding(xml);
String xmlString = feedCleaner.trimInvalidXmlCharacters(new String(xml, encoding));
if (xmlString == null) {
throw new FeedParsingException("Input string is null for url " + feedUrl);
}
xmlString = feedCleaner.replaceHtmlEntitiesWithNumericEntities(xmlString);
xmlString = feedCleaner.removeDoctypeDeclarations(xmlString);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed feed = new SyndFeedInput().build(source);
handleForeignMarkup(feed);
String title = feed.getTitle();
String link = feed.getLink();
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
if (lastPublishedDate == null || lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
lastPublishedDate = lastEntryDate;
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
return new FeedParserResult(title, link, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
} catch (FeedParsingException e) {
throw e;
} catch (Exception e) {
throw new FeedParsingException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
/**
* Adds atom links for rss feeds
*/
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
private List<Entry> buildEntries(SyndFeed feed, String feedUrl) {
List<Entry> entries = new ArrayList<>();
for (SyndEntry item : feed.getEntries()) {
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
String url = buildEntryUrl(feed, feedUrl, item);
if (StringUtils.isBlank(url) && FeedUtils.isAbsoluteUrl(guid)) {
// if link is empty but guid is used as url, use guid
url = guid;
}
Instant publishedDate = buildEntryPublishedDate(item);
Content content = buildContent(item);
entries.add(new Entry(guid, url, publishedDate, content));
}
entries.sort(Comparator.comparing(Entry::published).reversed());
return entries;
}
private Content buildContent(SyndEntry item) {
String title = getTitle(item);
String content = getContent(item);
String author = StringUtils.trimToNull(item.getAuthor());
String categories = StringUtils
.trimToNull(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")));
Enclosure enclosure = buildEnclosure(item);
Media media = buildMedia(item);
return new Content(title, content, author, categories, enclosure, media);
}
private Enclosure buildEnclosure(SyndEntry item) {
SyndEnclosure enclosure = Iterables.getFirst(item.getEnclosures(), null);
if (enclosure == null) {
return null;
}
return new Enclosure(enclosure.getUrl(), enclosure.getType());
}
private Instant buildEntryPublishedDate(SyndEntry item) {
Date date = item.getPublishedDate();
if (date == null) {
date = item.getUpdatedDate();
}
return toValidInstant(date, true);
}
private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
if (url == null || FeedUtils.isAbsoluteUrl(url)) {
// url is absolute, nothing to do
return url;
}
// url is relative, trying to resolve it
String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
return FeedUtils.toAbsoluteUrl(url, feedLink, feedUrl);
}
private Instant toValidInstant(Date date, boolean nullToNow) {
Instant now = Instant.now();
if (date == null) {
return nullToNow ? now : null;
}
Instant instant = date.toInstant();
if (instant.isBefore(START) || instant.isAfter(END)) {
return now;
}
if (instant.isAfter(now)) {
return now;
}
return instant;
}
private String getContent(SyndEntry item) {
String content;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private Media buildMedia(SyndEntry item) {
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module == null) {
return null;
}
Media media = buildMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = buildMedia(group.getMetadata());
}
return media;
}
private Media buildMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
String description = metadata.getDescription();
String thumbnailUrl = null;
Integer thumbnailWidth = null;
Integer thumbnailHeight = null;
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
thumbnailWidth = thumbnail.getWidth();
thumbnailHeight = thumbnail.getHeight();
if (thumbnail.getUrl() != null) {
thumbnailUrl = thumbnail.getUrl().toString();
}
}
if (description == null && thumbnailUrl == null) {
return null;
}
return new Media(description, thumbnailUrl, thumbnailWidth, thumbnailHeight);
}
private Long averageTimeBetweenEntries(List<Entry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < entries.size() - 1; i++) {
long diff = Math.abs(entries.get(i).published().toEpochMilli() - entries.get(i + 1).published().toEpochMilli());
stats.addValue(diff);
}
return (long) stats.getMean();
}
public static class FeedParsingException extends Exception {
private static final long serialVersionUID = 1L;
public FeedParsingException(String message) {
super(message);
}
public FeedParsingException(String message, Throwable cause) {
super(message, cause);
}
}
}

40
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParserResult.java
View File

@@ -1,20 +1,20 @@
package com.commafeed.backend.feed.parser;
import java.time.Instant;
import java.util.List;
public record FeedParserResult(String title, String link, Instant lastPublishedDate, Long averageEntryInterval, Instant lastEntryDate,
List<Entry> entries) {
public record Entry(String guid, String url, Instant published, Content content) {
}
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {
}
public record Enclosure(String url, String type) {
}
public record Media(String description, String thumbnailUrl, Integer thumbnailWidth, Integer thumbnailHeight) {
}
}
package com.commafeed.backend.feed.parser;
import java.time.Instant;
import java.util.List;
public record FeedParserResult(String title, String link, Instant lastPublishedDate, Long averageEntryInterval, Instant lastEntryDate,
List<Entry> entries) {
public record Entry(String guid, String url, Instant published, Content content) {
}
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {
}
public record Enclosure(String url, String type) {
}
public record Media(String description, String thumbnailUrl, Integer thumbnailWidth, Integer thumbnailHeight) {
}
}

544
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/HtmlEntities.java
View File

@@ -1,272 +1,272 @@
package com.commafeed.backend.feed.parser;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import lombok.experimental.UtilityClass;
@UtilityClass
class HtmlEntities {
public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final String[] HTML_ENTITIES;
public static final String[] NUMERIC_ENTITIES;
static {
Map<String, String> map = new LinkedHashMap<>();
map.put("&Aacute;", "&#193;");
map.put("&aacute;", "&#225;");
map.put("&Acirc;", "&#194;");
map.put("&acirc;", "&#226;");
map.put("&acute;", "&#180;");
map.put("&AElig;", "&#198;");
map.put("&aelig;", "&#230;");
map.put("&Agrave;", "&#192;");
map.put("&agrave;", "&#224;");
map.put("&alefsym;", "&#8501;");
map.put("&Alpha;", "&#913;");
map.put("&alpha;", "&#945;");
map.put("&amp;", "&#38;");
map.put("&and;", "&#8743;");
map.put("&ang;", "&#8736;");
map.put("&Aring;", "&#197;");
map.put("&aring;", "&#229;");
map.put("&asymp;", "&#8776;");
map.put("&Atilde;", "&#195;");
map.put("&atilde;", "&#227;");
map.put("&Auml;", "&#196;");
map.put("&auml;", "&#228;");
map.put("&bdquo;", "&#8222;");
map.put("&Beta;", "&#914;");
map.put("&beta;", "&#946;");
map.put("&brvbar;", "&#166;");
map.put("&bull;", "&#8226;");
map.put("&cap;", "&#8745;");
map.put("&Ccedil;", "&#199;");
map.put("&ccedil;", "&#231;");
map.put("&cedil;", "&#184;");
map.put("&cent;", "&#162;");
map.put("&Chi;", "&#935;");
map.put("&chi;", "&#967;");
map.put("&circ;", "&#710;");
map.put("&clubs;", "&#9827;");
map.put("&cong;", "&#8773;");
map.put("&copy;", "&#169;");
map.put("&crarr;", "&#8629;");
map.put("&cup;", "&#8746;");
map.put("&curren;", "&#164;");
map.put("&dagger;", "&#8224;");
map.put("&Dagger;", "&#8225;");
map.put("&darr;", "&#8595;");
map.put("&dArr;", "&#8659;");
map.put("&deg;", "&#176;");
map.put("&Delta;", "&#916;");
map.put("&delta;", "&#948;");
map.put("&diams;", "&#9830;");
map.put("&divide;", "&#247;");
map.put("&Eacute;", "&#201;");
map.put("&eacute;", "&#233;");
map.put("&Ecirc;", "&#202;");
map.put("&ecirc;", "&#234;");
map.put("&Egrave;", "&#200;");
map.put("&egrave;", "&#232;");
map.put("&empty;", "&#8709;");
map.put("&emsp;", "&#8195;");
map.put("&ensp;", "&#8194;");
map.put("&Epsilon;", "&#917;");
map.put("&epsilon;", "&#949;");
map.put("&equiv;", "&#8801;");
map.put("&Eta;", "&#919;");
map.put("&eta;", "&#951;");
map.put("&ETH;", "&#208;");
map.put("&eth;", "&#240;");
map.put("&Euml;", "&#203;");
map.put("&euml;", "&#235;");
map.put("&euro;", "&#8364;");
map.put("&exist;", "&#8707;");
map.put("&fnof;", "&#402;");
map.put("&forall;", "&#8704;");
map.put("&frac12;", "&#189;");
map.put("&frac14;", "&#188;");
map.put("&frac34;", "&#190;");
map.put("&frasl;", "&#8260;");
map.put("&Gamma;", "&#915;");
map.put("&gamma;", "&#947;");
map.put("&ge;", "&#8805;");
map.put("&harr;", "&#8596;");
map.put("&hArr;", "&#8660;");
map.put("&hearts;", "&#9829;");
map.put("&hellip;", "&#8230;");
map.put("&Iacute;", "&#205;");
map.put("&iacute;", "&#237;");
map.put("&Icirc;", "&#206;");
map.put("&icirc;", "&#238;");
map.put("&iexcl;", "&#161;");
map.put("&Igrave;", "&#204;");
map.put("&igrave;", "&#236;");
map.put("&image;", "&#8465;");
map.put("&infin;", "&#8734;");
map.put("&int;", "&#8747;");
map.put("&Iota;", "&#921;");
map.put("&iota;", "&#953;");
map.put("&iquest;", "&#191;");
map.put("&isin;", "&#8712;");
map.put("&Iuml;", "&#207;");
map.put("&iuml;", "&#239;");
map.put("&Kappa;", "&#922;");
map.put("&kappa;", "&#954;");
map.put("&Lambda;", "&#923;");
map.put("&lambda;", "&#955;");
map.put("&lang;", "&#9001;");
map.put("&laquo;", "&#171;");
map.put("&larr;", "&#8592;");
map.put("&lArr;", "&#8656;");
map.put("&lceil;", "&#8968;");
map.put("&ldquo;", "&#8220;");
map.put("&le;", "&#8804;");
map.put("&lfloor;", "&#8970;");
map.put("&lowast;", "&#8727;");
map.put("&loz;", "&#9674;");
map.put("&lrm;", "&#8206;");
map.put("&lsaquo;", "&#8249;");
map.put("&lsquo;", "&#8216;");
map.put("&macr;", "&#175;");
map.put("&mdash;", "&#8212;");
map.put("&micro;", "&#181;");
map.put("&middot;", "&#183;");
map.put("&minus;", "&#8722;");
map.put("&Mu;", "&#924;");
map.put("&mu;", "&#956;");
map.put("&nabla;", "&#8711;");
map.put("&nbsp;", "&#160;");
map.put("&ndash;", "&#8211;");
map.put("&ne;", "&#8800;");
map.put("&ni;", "&#8715;");
map.put("&not;", "&#172;");
map.put("&notin;", "&#8713;");
map.put("&nsub;", "&#8836;");
map.put("&Ntilde;", "&#209;");
map.put("&ntilde;", "&#241;");
map.put("&Nu;", "&#925;");
map.put("&nu;", "&#957;");
map.put("&Oacute;", "&#211;");
map.put("&oacute;", "&#243;");
map.put("&Ocirc;", "&#212;");
map.put("&ocirc;", "&#244;");
map.put("&OElig;", "&#338;");
map.put("&oelig;", "&#339;");
map.put("&Ograve;", "&#210;");
map.put("&ograve;", "&#242;");
map.put("&oline;", "&#8254;");
map.put("&Omega;", "&#937;");
map.put("&omega;", "&#969;");
map.put("&Omicron;", "&#927;");
map.put("&omicron;", "&#959;");
map.put("&oplus;", "&#8853;");
map.put("&or;", "&#8744;");
map.put("&ordf;", "&#170;");
map.put("&ordm;", "&#186;");
map.put("&Oslash;", "&#216;");
map.put("&oslash;", "&#248;");
map.put("&Otilde;", "&#213;");
map.put("&otilde;", "&#245;");
map.put("&otimes;", "&#8855;");
map.put("&Ouml;", "&#214;");
map.put("&ouml;", "&#246;");
map.put("&para;", "&#182;");
map.put("&part;", "&#8706;");
map.put("&permil;", "&#8240;");
map.put("&perp;", "&#8869;");
map.put("&Phi;", "&#934;");
map.put("&phi;", "&#966;");
map.put("&Pi;", "&#928;");
map.put("&pi;", "&#960;");
map.put("&piv;", "&#982;");
map.put("&plusmn;", "&#177;");
map.put("&pound;", "&#163;");
map.put("&prime;", "&#8242;");
map.put("&Prime;", "&#8243;");
map.put("&prod;", "&#8719;");
map.put("&prop;", "&#8733;");
map.put("&Psi;", "&#936;");
map.put("&psi;", "&#968;");
map.put("&quot;", "&#34;");
map.put("&radic;", "&#8730;");
map.put("&rang;", "&#9002;");
map.put("&raquo;", "&#187;");
map.put("&rarr;", "&#8594;");
map.put("&rArr;", "&#8658;");
map.put("&rceil;", "&#8969;");
map.put("&rdquo;", "&#8221;");
map.put("&real;", "&#8476;");
map.put("&reg;", "&#174;");
map.put("&rfloor;", "&#8971;");
map.put("&Rho;", "&#929;");
map.put("&rho;", "&#961;");
map.put("&rlm;", "&#8207;");
map.put("&rsaquo;", "&#8250;");
map.put("&rsquo;", "&#8217;");
map.put("&sbquo;", "&#8218;");
map.put("&Scaron;", "&#352;");
map.put("&scaron;", "&#353;");
map.put("&sdot;", "&#8901;");
map.put("&sect;", "&#167;");
map.put("&shy;", "&#173;");
map.put("&Sigma;", "&#931;");
map.put("&sigma;", "&#963;");
map.put("&sigmaf;", "&#962;");
map.put("&sim;", "&#8764;");
map.put("&spades;", "&#9824;");
map.put("&sub;", "&#8834;");
map.put("&sube;", "&#8838;");
map.put("&sum;", "&#8721;");
map.put("&sup1;", "&#185;");
map.put("&sup2;", "&#178;");
map.put("&sup3;", "&#179;");
map.put("&sup;", "&#8835;");
map.put("&supe;", "&#8839;");
map.put("&szlig;", "&#223;");
map.put("&Tau;", "&#932;");
map.put("&tau;", "&#964;");
map.put("&there4;", "&#8756;");
map.put("&Theta;", "&#920;");
map.put("&theta;", "&#952;");
map.put("&thetasym;", "&#977;");
map.put("&thinsp;", "&#8201;");
map.put("&THORN;", "&#222;");
map.put("&thorn;", "&#254;");
map.put("&tilde;", "&#732;");
map.put("&times;", "&#215;");
map.put("&trade;", "&#8482;");
map.put("&Uacute;", "&#218;");
map.put("&uacute;", "&#250;");
map.put("&uarr;", "&#8593;");
map.put("&uArr;", "&#8657;");
map.put("&Ucirc;", "&#219;");
map.put("&ucirc;", "&#251;");
map.put("&Ugrave;", "&#217;");
map.put("&ugrave;", "&#249;");
map.put("&uml;", "&#168;");
map.put("&upsih;", "&#978;");
map.put("&Upsilon;", "&#933;");
map.put("&upsilon;", "&#965;");
map.put("&Uuml;", "&#220;");
map.put("&uuml;", "&#252;");
map.put("&weierp;", "&#8472;");
map.put("&Xi;", "&#926;");
map.put("&xi;", "&#958;");
map.put("&Yacute;", "&#221;");
map.put("&yacute;", "&#253;");
map.put("&yen;", "&#165;");
map.put("&yuml;", "&#255;");
map.put("&Yuml;", "&#376;");
map.put("&Zeta;", "&#918;");
map.put("&zeta;", "&#950;");
map.put("&zwj;", "&#8205;");
map.put("&zwnj;", "&#8204;");
HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map);
HTML_ENTITIES = map.keySet().toArray(new String[0]);
NUMERIC_ENTITIES = map.values().toArray(new String[0]);
}
}
package com.commafeed.backend.feed.parser;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import lombok.experimental.UtilityClass;
@UtilityClass
class HtmlEntities {
public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final String[] HTML_ENTITIES;
public static final String[] NUMERIC_ENTITIES;
static {
Map<String, String> map = new LinkedHashMap<>();
map.put("&Aacute;", "&#193;");
map.put("&aacute;", "&#225;");
map.put("&Acirc;", "&#194;");
map.put("&acirc;", "&#226;");
map.put("&acute;", "&#180;");
map.put("&AElig;", "&#198;");
map.put("&aelig;", "&#230;");
map.put("&Agrave;", "&#192;");
map.put("&agrave;", "&#224;");
map.put("&alefsym;", "&#8501;");
map.put("&Alpha;", "&#913;");
map.put("&alpha;", "&#945;");
map.put("&amp;", "&#38;");
map.put("&and;", "&#8743;");
map.put("&ang;", "&#8736;");
map.put("&Aring;", "&#197;");
map.put("&aring;", "&#229;");
map.put("&asymp;", "&#8776;");
map.put("&Atilde;", "&#195;");
map.put("&atilde;", "&#227;");
map.put("&Auml;", "&#196;");
map.put("&auml;", "&#228;");
map.put("&bdquo;", "&#8222;");
map.put("&Beta;", "&#914;");
map.put("&beta;", "&#946;");
map.put("&brvbar;", "&#166;");
map.put("&bull;", "&#8226;");
map.put("&cap;", "&#8745;");
map.put("&Ccedil;", "&#199;");
map.put("&ccedil;", "&#231;");
map.put("&cedil;", "&#184;");
map.put("&cent;", "&#162;");
map.put("&Chi;", "&#935;");
map.put("&chi;", "&#967;");
map.put("&circ;", "&#710;");
map.put("&clubs;", "&#9827;");
map.put("&cong;", "&#8773;");
map.put("&copy;", "&#169;");
map.put("&crarr;", "&#8629;");
map.put("&cup;", "&#8746;");
map.put("&curren;", "&#164;");
map.put("&dagger;", "&#8224;");
map.put("&Dagger;", "&#8225;");
map.put("&darr;", "&#8595;");
map.put("&dArr;", "&#8659;");
map.put("&deg;", "&#176;");
map.put("&Delta;", "&#916;");
map.put("&delta;", "&#948;");
map.put("&diams;", "&#9830;");
map.put("&divide;", "&#247;");
map.put("&Eacute;", "&#201;");
map.put("&eacute;", "&#233;");
map.put("&Ecirc;", "&#202;");
map.put("&ecirc;", "&#234;");
map.put("&Egrave;", "&#200;");
map.put("&egrave;", "&#232;");
map.put("&empty;", "&#8709;");
map.put("&emsp;", "&#8195;");
map.put("&ensp;", "&#8194;");
map.put("&Epsilon;", "&#917;");
map.put("&epsilon;", "&#949;");
map.put("&equiv;", "&#8801;");
map.put("&Eta;", "&#919;");
map.put("&eta;", "&#951;");
map.put("&ETH;", "&#208;");
map.put("&eth;", "&#240;");
map.put("&Euml;", "&#203;");
map.put("&euml;", "&#235;");
map.put("&euro;", "&#8364;");
map.put("&exist;", "&#8707;");
map.put("&fnof;", "&#402;");
map.put("&forall;", "&#8704;");
map.put("&frac12;", "&#189;");
map.put("&frac14;", "&#188;");
map.put("&frac34;", "&#190;");
map.put("&frasl;", "&#8260;");
map.put("&Gamma;", "&#915;");
map.put("&gamma;", "&#947;");
map.put("&ge;", "&#8805;");
map.put("&harr;", "&#8596;");
map.put("&hArr;", "&#8660;");
map.put("&hearts;", "&#9829;");
map.put("&hellip;", "&#8230;");
map.put("&Iacute;", "&#205;");
map.put("&iacute;", "&#237;");
map.put("&Icirc;", "&#206;");
map.put("&icirc;", "&#238;");
map.put("&iexcl;", "&#161;");
map.put("&Igrave;", "&#204;");
map.put("&igrave;", "&#236;");
map.put("&image;", "&#8465;");
map.put("&infin;", "&#8734;");
map.put("&int;", "&#8747;");
map.put("&Iota;", "&#921;");
map.put("&iota;", "&#953;");
map.put("&iquest;", "&#191;");
map.put("&isin;", "&#8712;");
map.put("&Iuml;", "&#207;");
map.put("&iuml;", "&#239;");
map.put("&Kappa;", "&#922;");
map.put("&kappa;", "&#954;");
map.put("&Lambda;", "&#923;");
map.put("&lambda;", "&#955;");
map.put("&lang;", "&#9001;");
map.put("&laquo;", "&#171;");
map.put("&larr;", "&#8592;");
map.put("&lArr;", "&#8656;");
map.put("&lceil;", "&#8968;");
map.put("&ldquo;", "&#8220;");
map.put("&le;", "&#8804;");
map.put("&lfloor;", "&#8970;");
map.put("&lowast;", "&#8727;");
map.put("&loz;", "&#9674;");
map.put("&lrm;", "&#8206;");
map.put("&lsaquo;", "&#8249;");
map.put("&lsquo;", "&#8216;");
map.put("&macr;", "&#175;");
map.put("&mdash;", "&#8212;");
map.put("&micro;", "&#181;");
map.put("&middot;", "&#183;");
map.put("&minus;", "&#8722;");
map.put("&Mu;", "&#924;");
map.put("&mu;", "&#956;");
map.put("&nabla;", "&#8711;");
map.put("&nbsp;", "&#160;");
map.put("&ndash;", "&#8211;");
map.put("&ne;", "&#8800;");
map.put("&ni;", "&#8715;");
map.put("&not;", "&#172;");
map.put("&notin;", "&#8713;");
map.put("&nsub;", "&#8836;");
map.put("&Ntilde;", "&#209;");
map.put("&ntilde;", "&#241;");
map.put("&Nu;", "&#925;");
map.put("&nu;", "&#957;");
map.put("&Oacute;", "&#211;");
map.put("&oacute;", "&#243;");
map.put("&Ocirc;", "&#212;");
map.put("&ocirc;", "&#244;");
map.put("&OElig;", "&#338;");
map.put("&oelig;", "&#339;");
map.put("&Ograve;", "&#210;");
map.put("&ograve;", "&#242;");
map.put("&oline;", "&#8254;");
map.put("&Omega;", "&#937;");
map.put("&omega;", "&#969;");
map.put("&Omicron;", "&#927;");
map.put("&omicron;", "&#959;");
map.put("&oplus;", "&#8853;");
map.put("&or;", "&#8744;");
map.put("&ordf;", "&#170;");
map.put("&ordm;", "&#186;");
map.put("&Oslash;", "&#216;");
map.put("&oslash;", "&#248;");
map.put("&Otilde;", "&#213;");
map.put("&otilde;", "&#245;");
map.put("&otimes;", "&#8855;");
map.put("&Ouml;", "&#214;");
map.put("&ouml;", "&#246;");
map.put("&para;", "&#182;");
map.put("&part;", "&#8706;");
map.put("&permil;", "&#8240;");
map.put("&perp;", "&#8869;");
map.put("&Phi;", "&#934;");
map.put("&phi;", "&#966;");
map.put("&Pi;", "&#928;");
map.put("&pi;", "&#960;");
map.put("&piv;", "&#982;");
map.put("&plusmn;", "&#177;");
map.put("&pound;", "&#163;");
map.put("&prime;", "&#8242;");
map.put("&Prime;", "&#8243;");
map.put("&prod;", "&#8719;");
map.put("&prop;", "&#8733;");
map.put("&Psi;", "&#936;");
map.put("&psi;", "&#968;");
map.put("&quot;", "&#34;");
map.put("&radic;", "&#8730;");
map.put("&rang;", "&#9002;");
map.put("&raquo;", "&#187;");
map.put("&rarr;", "&#8594;");
map.put("&rArr;", "&#8658;");
map.put("&rceil;", "&#8969;");
map.put("&rdquo;", "&#8221;");
map.put("&real;", "&#8476;");
map.put("&reg;", "&#174;");
map.put("&rfloor;", "&#8971;");
map.put("&Rho;", "&#929;");
map.put("&rho;", "&#961;");
map.put("&rlm;", "&#8207;");
map.put("&rsaquo;", "&#8250;");
map.put("&rsquo;", "&#8217;");
map.put("&sbquo;", "&#8218;");
map.put("&Scaron;", "&#352;");
map.put("&scaron;", "&#353;");
map.put("&sdot;", "&#8901;");
map.put("&sect;", "&#167;");
map.put("&shy;", "&#173;");
map.put("&Sigma;", "&#931;");
map.put("&sigma;", "&#963;");
map.put("&sigmaf;", "&#962;");
map.put("&sim;", "&#8764;");
map.put("&spades;", "&#9824;");
map.put("&sub;", "&#8834;");
map.put("&sube;", "&#8838;");
map.put("&sum;", "&#8721;");
map.put("&sup1;", "&#185;");
map.put("&sup2;", "&#178;");
map.put("&sup3;", "&#179;");
map.put("&sup;", "&#8835;");
map.put("&supe;", "&#8839;");
map.put("&szlig;", "&#223;");
map.put("&Tau;", "&#932;");
map.put("&tau;", "&#964;");
map.put("&there4;", "&#8756;");
map.put("&Theta;", "&#920;");
map.put("&theta;", "&#952;");
map.put("&thetasym;", "&#977;");
map.put("&thinsp;", "&#8201;");
map.put("&THORN;", "&#222;");
map.put("&thorn;", "&#254;");
map.put("&tilde;", "&#732;");
map.put("&times;", "&#215;");
map.put("&trade;", "&#8482;");
map.put("&Uacute;", "&#218;");
map.put("&uacute;", "&#250;");
map.put("&uarr;", "&#8593;");
map.put("&uArr;", "&#8657;");
map.put("&Ucirc;", "&#219;");
map.put("&ucirc;", "&#251;");
map.put("&Ugrave;", "&#217;");
map.put("&ugrave;", "&#249;");
map.put("&uml;", "&#168;");
map.put("&upsih;", "&#978;");
map.put("&Upsilon;", "&#933;");
map.put("&upsilon;", "&#965;");
map.put("&Uuml;", "&#220;");
map.put("&uuml;", "&#252;");
map.put("&weierp;", "&#8472;");
map.put("&Xi;", "&#926;");
map.put("&xi;", "&#958;");
map.put("&Yacute;", "&#221;");
map.put("&yacute;", "&#253;");
map.put("&yen;", "&#165;");
map.put("&yuml;", "&#255;");
map.put("&Yuml;", "&#376;");
map.put("&Zeta;", "&#918;");
map.put("&zeta;", "&#950;");
map.put("&zwj;", "&#8205;");
map.put("&zwnj;", "&#8204;");
HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map);
HTML_ENTITIES = map.keySet().toArray(new String[0]);
NUMERIC_ENTITIES = map.values().toArray(new String[0]);
}
}

112
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/TextDirectionDetector.java
View File

@@ -1,56 +1,56 @@
package com.commafeed.backend.feed.parser;
import java.text.Bidi;
import java.util.regex.Pattern;
import org.apache.commons.lang3.math.NumberUtils;
public class TextDirectionDetector {
private static final Pattern WORDS_PATTERN = Pattern.compile("\\s+");
private static final Pattern URL_PATTERN = Pattern.compile("^https?://.*");
private static final double RTL_THRESHOLD = 0.4D;
public enum Direction {
LEFT_TO_RIGHT, RIGHT_TO_LEFT
}
public static Direction detect(String input) {
if (input == null || input.isBlank()) {
return Direction.LEFT_TO_RIGHT;
}
long rtl = 0;
long total = 0;
for (String token : WORDS_PATTERN.split(input)) {
// skip urls
if (URL_PATTERN.matcher(token).matches()) {
continue;
}
// skip numbers
if (NumberUtils.isCreatable(token)) {
continue;
}
boolean requiresBidi = Bidi.requiresBidi(token.toCharArray(), 0, token.length());
if (requiresBidi) {
Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT);
if (bidi.getBaseLevel() == 1) {
rtl++;
}
}
total++;
}
if (total == 0) {
return Direction.LEFT_TO_RIGHT;
}
double ratio = (double) rtl / total;
return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
}
}
package com.commafeed.backend.feed.parser;
import java.text.Bidi;
import java.util.regex.Pattern;
import org.apache.commons.lang3.math.NumberUtils;
public class TextDirectionDetector {
private static final Pattern WORDS_PATTERN = Pattern.compile("\\s+");
private static final Pattern URL_PATTERN = Pattern.compile("^https?://.*");
private static final double RTL_THRESHOLD = 0.4D;
public enum Direction {
LEFT_TO_RIGHT, RIGHT_TO_LEFT
}
public static Direction detect(String input) {
if (input == null || input.isBlank()) {
return Direction.LEFT_TO_RIGHT;
}
long rtl = 0;
long total = 0;
for (String token : WORDS_PATTERN.split(input)) {
// skip urls
if (URL_PATTERN.matcher(token).matches()) {
continue;
}
// skip numbers
if (NumberUtils.isCreatable(token)) {
continue;
}
boolean requiresBidi = Bidi.requiresBidi(token.toCharArray(), 0, token.length());
if (requiresBidi) {
Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT);
if (bidi.getBaseLevel() == 1) {
rtl++;
}
}
total++;
}
if (total == 0) {
return Direction.LEFT_TO_RIGHT;
}
double ratio = (double) rtl / total;
return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
}
}

66
commafeed-server/src/main/java/com/commafeed/backend/model/AbstractModel.java
View File

@@ -1,33 +1,33 @@
package com.commafeed.backend.model;
import java.io.Serializable;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
import jakarta.persistence.Id;
import jakarta.persistence.MappedSuperclass;
import jakarta.persistence.TableGenerator;
import lombok.Getter;
import lombok.Setter;
/**
* Abstract model for all entities, defining id and table generator
*
*/
@SuppressWarnings("serial")
@MappedSuperclass
@Getter
@Setter
public abstract class AbstractModel implements Serializable {
@Id
@GeneratedValue(strategy = GenerationType.TABLE, generator = "gen")
@TableGenerator(
name = "gen",
table = "hibernate_sequences",
pkColumnName = "sequence_name",
valueColumnName = "sequence_next_hi_value",
allocationSize = 1000)
private Long id;
}
package com.commafeed.backend.model;
import java.io.Serializable;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
import jakarta.persistence.Id;
import jakarta.persistence.MappedSuperclass;
import jakarta.persistence.TableGenerator;
import lombok.Getter;
import lombok.Setter;
/**
* Abstract model for all entities, defining id and table generator
*
*/
@SuppressWarnings("serial")
@MappedSuperclass
@Getter
@Setter
public abstract class AbstractModel implements Serializable {
@Id
@GeneratedValue(strategy = GenerationType.TABLE, generator = "gen")
@TableGenerator(
name = "gen",
table = "hibernate_sequences",
pkColumnName = "sequence_name",
valueColumnName = "sequence_next_hi_value",
allocationSize = 1000)
private Long id;
}

222
commafeed-server/src/main/java/com/commafeed/backend/model/Feed.java
View File

@@ -1,111 +1,111 @@
package com.commafeed.backend.model;
import java.sql.Types;
import java.time.Instant;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Lob;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDS")
@SuppressWarnings("serial")
@Getter
@Setter
public class Feed extends AbstractModel {
/**
* The url of the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARCHAR)
private String url;
/**
* cache the url after potential http 30x redirects
*/
@Column(name = "url_after_redirect", length = 2048, nullable = false)
private String urlAfterRedirect;
@Column(length = 2048, nullable = false)
private String normalizedUrl;
@Column(length = 40, nullable = false)
private String normalizedUrlHash;
/**
* The url of the website, extracted from the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String link;
/**
* Last time we tried to fetch the feed
*/
@Column
private Instant lastUpdated;
/**
* Last publishedDate value in the feed
*/
@Column
private Instant lastPublishedDate;
/**
* date of the last entry of the feed
*/
@Column
private Instant lastEntryDate;
/**
* error message while retrieving the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String message;
/**
* times we failed to retrieve the feed
*/
private int errorCount;
/**
* feed refresh is disabled until this date
*/
@Column
private Instant disabledUntil;
/**
* http header returned by the feed
*/
@Column(length = 64)
private String lastModifiedHeader;
/**
* http header returned by the feed
*/
@Column(length = 255)
private String etagHeader;
/**
* average time between entries in the feed
*/
private Long averageEntryInterval;
/**
* last hash of the content of the feed xml
*/
@Column(length = 40)
private String lastContentHash;
}
package com.commafeed.backend.model;
import java.sql.Types;
import java.time.Instant;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Lob;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDS")
@SuppressWarnings("serial")
@Getter
@Setter
public class Feed extends AbstractModel {
/**
* The url of the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARCHAR)
private String url;
/**
* cache the url after potential http 30x redirects
*/
@Column(name = "url_after_redirect", length = 2048, nullable = false)
private String urlAfterRedirect;
@Column(length = 2048, nullable = false)
private String normalizedUrl;
@Column(length = 40, nullable = false)
private String normalizedUrlHash;
/**
* The url of the website, extracted from the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String link;
/**
* Last time we tried to fetch the feed
*/
@Column
private Instant lastUpdated;
/**
* Last publishedDate value in the feed
*/
@Column
private Instant lastPublishedDate;
/**
* date of the last entry of the feed
*/
@Column
private Instant lastEntryDate;
/**
* error message while retrieving the feed
*/
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String message;
/**
* times we failed to retrieve the feed
*/
private int errorCount;
/**
* feed refresh is disabled until this date
*/
@Column
private Instant disabledUntil;
/**
* http header returned by the feed
*/
@Column(length = 64)
private String lastModifiedHeader;
/**
* http header returned by the feed
*/
@Column(length = 255)
private String etagHeader;
/**
* average time between entries in the feed
*/
private Long averageEntryInterval;
/**
* last hash of the content of the feed xml
*/
@Column(length = 40)
private String lastContentHash;
}

86
commafeed-server/src/main/java/com/commafeed/backend/model/FeedCategory.java
View File

@@ -1,43 +1,43 @@
package com.commafeed.backend.model;
import java.util.Set;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDCATEGORIES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedCategory extends AbstractModel {
@Column(length = 128, nullable = false)
private String name;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory parent;
@OneToMany(mappedBy = "parent")
private Set<FeedCategory> children;
@OneToMany(mappedBy = "category")
private Set<FeedSubscription> subscriptions;
private boolean collapsed;
private int position;
}
package com.commafeed.backend.model;
import java.util.Set;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDCATEGORIES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedCategory extends AbstractModel {
@Column(length = 128, nullable = false)
private String name;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory parent;
@OneToMany(mappedBy = "parent")
private Set<FeedCategory> children;
@OneToMany(mappedBy = "category")
private Set<FeedSubscription> subscriptions;
private boolean collapsed;
private int position;
}

120
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntry.java
View File

@@ -1,60 +1,60 @@
package com.commafeed.backend.model;
import java.time.Instant;
import java.util.Set;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRIES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntry extends AbstractModel {
@Column(length = 2048, nullable = false)
private String guid;
@Column(length = 40, nullable = false)
private String guidHash;
@ManyToOne(fetch = FetchType.LAZY)
private Feed feed;
@ManyToOne(fetch = FetchType.LAZY, optional = false)
@JoinColumn(nullable = false, updatable = false)
private FeedEntryContent content;
@Column(length = 2048)
private String url;
/**
* the moment the entry was inserted in the database
*/
@Column
private Instant inserted;
/**
* the moment the entry was published in the feed
*
*/
@Column(name = "updated")
private Instant published;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryTag> tags;
}
package com.commafeed.backend.model;
import java.time.Instant;
import java.util.Set;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRIES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntry extends AbstractModel {
@Column(length = 2048, nullable = false)
private String guid;
@Column(length = 40, nullable = false)
private String guidHash;
@ManyToOne(fetch = FetchType.LAZY)
private Feed feed;
@ManyToOne(fetch = FetchType.LAZY, optional = false)
@JoinColumn(nullable = false, updatable = false)
private FeedEntryContent content;
@Column(length = 2048)
private String url;
/**
* the moment the entry was inserted in the database
*/
@Column
private Instant inserted;
/**
* the moment the entry was published in the feed
*
*/
@Column(name = "updated")
private Instant published;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
@OneToMany(mappedBy = "entry", cascade = CascadeType.REMOVE)
private Set<FeedEntryTag> tags;
}

210
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryContent.java
View File

@@ -1,105 +1,105 @@
package com.commafeed.backend.model;
import java.sql.Types;
import java.util.Set;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.Lob;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.hibernate.annotations.JdbcTypeCode;
import com.commafeed.backend.feed.FeedUtils;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYCONTENTS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryContent extends AbstractModel {
public enum Direction {
ltr, rtl, unknown
}
@Column(length = 2048)
private String title;
@Column(length = 40)
private String titleHash;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String content;
@Column(length = 40)
private String contentHash;
@Column(name = "author", length = 128)
private String author;
@Column(length = 2048)
private String enclosureUrl;
@Column(length = 255)
private String enclosureType;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String mediaDescription;
@Column(length = 2048)
private String mediaThumbnailUrl;
private Integer mediaThumbnailWidth;
private Integer mediaThumbnailHeight;
@Column(length = 4096)
private String categories;
@Column
@Enumerated(EnumType.STRING)
private Direction direction = Direction.unknown;
@OneToMany(mappedBy = "content")
private Set<FeedEntry> entries;
public boolean equivalentTo(FeedEntryContent c) {
if (c == null) {
return false;
}
return new EqualsBuilder().append(title, c.title)
.append(content, c.content)
.append(author, c.author)
.append(categories, c.categories)
.append(enclosureUrl, c.enclosureUrl)
.append(enclosureType, c.enclosureType)
.append(mediaDescription, c.mediaDescription)
.append(mediaThumbnailUrl, c.mediaThumbnailUrl)
.append(mediaThumbnailWidth, c.mediaThumbnailWidth)
.append(mediaThumbnailHeight, c.mediaThumbnailHeight)
.build();
}
public boolean isRTL() {
if (direction == Direction.rtl) {
return true;
} else if (direction == Direction.ltr) {
return false;
} else {
// detect on the fly for content that was inserted before the direction field was added
return FeedUtils.isRTL(title, content);
}
}
}
package com.commafeed.backend.model;
import java.sql.Types;
import java.util.Set;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.Lob;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.hibernate.annotations.JdbcTypeCode;
import com.commafeed.backend.feed.FeedUtils;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYCONTENTS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryContent extends AbstractModel {
public enum Direction {
ltr, rtl, unknown
}
@Column(length = 2048)
private String title;
@Column(length = 40)
private String titleHash;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String content;
@Column(length = 40)
private String contentHash;
@Column(name = "author", length = 128)
private String author;
@Column(length = 2048)
private String enclosureUrl;
@Column(length = 255)
private String enclosureType;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String mediaDescription;
@Column(length = 2048)
private String mediaThumbnailUrl;
private Integer mediaThumbnailWidth;
private Integer mediaThumbnailHeight;
@Column(length = 4096)
private String categories;
@Column
@Enumerated(EnumType.STRING)
private Direction direction = Direction.unknown;
@OneToMany(mappedBy = "content")
private Set<FeedEntry> entries;
public boolean equivalentTo(FeedEntryContent c) {
if (c == null) {
return false;
}
return new EqualsBuilder().append(title, c.title)
.append(content, c.content)
.append(author, c.author)
.append(categories, c.categories)
.append(enclosureUrl, c.enclosureUrl)
.append(enclosureType, c.enclosureType)
.append(mediaDescription, c.mediaDescription)
.append(mediaThumbnailUrl, c.mediaThumbnailUrl)
.append(mediaThumbnailWidth, c.mediaThumbnailWidth)
.append(mediaThumbnailHeight, c.mediaThumbnailHeight)
.build();
}
public boolean isRTL() {
if (direction == Direction.rtl) {
return true;
} else if (direction == Direction.ltr) {
return false;
} else {
// detect on the fly for content that was inserted before the direction field was added
return FeedUtils.isRTL(title, content);
}
}
}

138
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryStatus.java
View File

@@ -1,69 +1,69 @@
package com.commafeed.backend.model;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import jakarta.persistence.Transient;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYSTATUSES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryStatus extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedSubscription subscription;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedEntry entry;
@Column(name = "read_status")
private boolean read;
private boolean starred;
@Transient
private boolean markable;
@Transient
private List<FeedEntryTag> tags = new ArrayList<>();
/**
* Denormalization starts here
*/
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@Column
private Instant entryInserted;
@Column(name = "entryUpdated")
private Instant entryPublished;
public FeedEntryStatus() {
}
public FeedEntryStatus(User user, FeedSubscription subscription, FeedEntry entry) {
this.user = user;
this.subscription = subscription;
this.entry = entry;
this.entryInserted = entry.getInserted();
this.entryPublished = entry.getPublished();
}
}
package com.commafeed.backend.model;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import jakarta.persistence.Transient;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYSTATUSES")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryStatus extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedSubscription subscription;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private FeedEntry entry;
@Column(name = "read_status")
private boolean read;
private boolean starred;
@Transient
private boolean markable;
@Transient
private List<FeedEntryTag> tags = new ArrayList<>();
/**
* Denormalization starts here
*/
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@Column
private Instant entryInserted;
@Column(name = "entryUpdated")
private Instant entryPublished;
public FeedEntryStatus() {
}
public FeedEntryStatus(User user, FeedSubscription subscription, FeedEntry entry) {
this.user = user;
this.subscription = subscription;
this.entry = entry;
this.entryInserted = entry.getInserted();
this.entryPublished = entry.getPublished();
}
}

80
commafeed-server/src/main/java/com/commafeed/backend/model/FeedEntryTag.java
View File

@@ -1,40 +1,40 @@
package com.commafeed.backend.model;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYTAGS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryTag extends AbstractModel {
@JoinColumn(name = "user_id")
@ManyToOne(fetch = FetchType.LAZY)
private User user;
@JoinColumn(name = "entry_id")
@ManyToOne(fetch = FetchType.LAZY)
private FeedEntry entry;
@Column(name = "name", length = 40)
private String name;
public FeedEntryTag() {
}
public FeedEntryTag(User user, FeedEntry entry, String name) {
this.name = name;
this.entry = entry;
this.user = user;
}
}
package com.commafeed.backend.model;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDENTRYTAGS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedEntryTag extends AbstractModel {
@JoinColumn(name = "user_id")
@ManyToOne(fetch = FetchType.LAZY)
private User user;
@JoinColumn(name = "entry_id")
@ManyToOne(fetch = FetchType.LAZY)
private FeedEntry entry;
@Column(name = "name", length = 40)
private String name;
public FeedEntryTag() {
}
public FeedEntryTag(User user, FeedEntry entry, String name) {
this.name = name;
this.entry = entry;
this.user = user;
}
}

92
commafeed-server/src/main/java/com/commafeed/backend/model/FeedSubscription.java
View File

@@ -1,46 +1,46 @@
package com.commafeed.backend.model;
import java.util.Set;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDSUBSCRIPTIONS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedSubscription extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private Feed feed;
@Column(length = 128, nullable = false)
private String title;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory category;
@OneToMany(mappedBy = "subscription", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
private int position;
@Column(name = "filtering_expression", length = 4096)
private String filter;
}
package com.commafeed.backend.model;
import java.util.Set;
import jakarta.persistence.CascadeType;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.ManyToOne;
import jakarta.persistence.OneToMany;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "FEEDSUBSCRIPTIONS")
@SuppressWarnings("serial")
@Getter
@Setter
public class FeedSubscription extends AbstractModel {
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private User user;
@ManyToOne(fetch = FetchType.LAZY)
@JoinColumn(nullable = false)
private Feed feed;
@Column(length = 128, nullable = false)
private String title;
@ManyToOne(fetch = FetchType.LAZY)
private FeedCategory category;
@OneToMany(mappedBy = "subscription", cascade = CascadeType.REMOVE)
private Set<FeedEntryStatus> statuses;
private int position;
@Column(name = "filtering_expression", length = 4096)
private String filter;
}

82
commafeed-server/src/main/java/com/commafeed/backend/model/Models.java
View File

@@ -1,41 +1,41 @@
package com.commafeed.backend.model;
import java.time.Duration;
import java.time.Instant;
import org.hibernate.Hibernate;
import org.hibernate.HibernateException;
import org.hibernate.proxy.HibernateProxy;
import org.hibernate.proxy.LazyInitializer;
import lombok.experimental.UtilityClass;
@UtilityClass
public class Models {
public static final Instant MINIMUM_INSTANT = Instant.EPOCH
// mariadb timestamp range starts at 1970-01-01 00:00:01
.plusSeconds(1)
// make sure the timestamp fits for all timezones
.plus(Duration.ofHours(24));
/**
* initialize a proxy
*/
public static void initialize(Object proxy) throws HibernateException {
Hibernate.initialize(proxy);
}
/**
* extract the id from the proxy without initializing it
*/
public static Long getId(AbstractModel model) {
if (model instanceof HibernateProxy proxy) {
LazyInitializer lazyInitializer = proxy.getHibernateLazyInitializer();
if (lazyInitializer.isUninitialized()) {
return (Long) lazyInitializer.getIdentifier();
}
}
return model.getId();
}
}
package com.commafeed.backend.model;
import java.time.Duration;
import java.time.Instant;
import org.hibernate.Hibernate;
import org.hibernate.HibernateException;
import org.hibernate.proxy.HibernateProxy;
import org.hibernate.proxy.LazyInitializer;
import lombok.experimental.UtilityClass;
@UtilityClass
public class Models {
public static final Instant MINIMUM_INSTANT = Instant.EPOCH
// mariadb timestamp range starts at 1970-01-01 00:00:01
.plusSeconds(1)
// make sure the timestamp fits for all timezones
.plus(Duration.ofHours(24));
/**
* initialize a proxy
*/
public static void initialize(Object proxy) throws HibernateException {
Hibernate.initialize(proxy);
}
/**
* extract the id from the proxy without initializing it
*/
public static Long getId(AbstractModel model) {
if (model instanceof HibernateProxy proxy) {
LazyInitializer lazyInitializer = proxy.getHibernateLazyInitializer();
if (lazyInitializer.isUninitialized()) {
return (Long) lazyInitializer.getIdentifier();
}
}
return model.getId();
}
}

118
commafeed-server/src/main/java/com/commafeed/backend/model/User.java
View File

@@ -1,59 +1,59 @@
package com.commafeed.backend.model;
import java.sql.Types;
import java.time.Instant;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Lob;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERS")
@SuppressWarnings("serial")
@Getter
@Setter
public class User extends AbstractModel {
@Column(length = 32, nullable = false, unique = true)
private String name;
@Column(length = 255, unique = true)
private String email;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] password;
@Column(length = 40, unique = true)
private String apiKey;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] salt;
@Column(nullable = false)
private boolean disabled;
@Column
private Instant lastLogin;
@Column
private Instant created;
@Column(length = 40)
private String recoverPasswordToken;
@Column
private Instant recoverPasswordTokenDate;
@Column
private Instant lastForceRefresh;
}
package com.commafeed.backend.model;
import java.sql.Types;
import java.time.Instant;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.Lob;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERS")
@SuppressWarnings("serial")
@Getter
@Setter
public class User extends AbstractModel {
@Column(length = 32, nullable = false, unique = true)
private String name;
@Column(length = 255, unique = true)
private String email;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] password;
@Column(length = 40, unique = true)
private String apiKey;
@Lob
@Column(length = Integer.MAX_VALUE, nullable = false)
@JdbcTypeCode(Types.LONGVARBINARY)
private byte[] salt;
@Column(nullable = false)
private boolean disabled;
@Column
private Instant lastLogin;
@Column
private Instant created;
@Column(length = 40)
private String recoverPasswordToken;
@Column
private Instant recoverPasswordTokenDate;
@Column
private Instant lastForceRefresh;
}

86
commafeed-server/src/main/java/com/commafeed/backend/model/UserRole.java
View File

@@ -1,43 +1,43 @@
package com.commafeed.backend.model;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.OneToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERROLES")
@SuppressWarnings("serial")
@Getter
@Setter
public class UserRole extends AbstractModel {
public enum Role {
USER, ADMIN
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false)
private User user;
@Column(name = "roleName", nullable = false)
@Enumerated(EnumType.STRING)
private Role role;
public UserRole() {
}
public UserRole(User user, Role role) {
this.user = user;
this.role = role;
}
}
package com.commafeed.backend.model;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.OneToOne;
import jakarta.persistence.Table;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERROLES")
@SuppressWarnings("serial")
@Getter
@Setter
public class UserRole extends AbstractModel {
public enum Role {
USER, ADMIN
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false)
private User user;
@Column(name = "roleName", nullable = false)
@Enumerated(EnumType.STRING)
private Role role;
public UserRole() {
}
public UserRole(User user, Role role) {
this.user = user;
this.role = role;
}
}

214
commafeed-server/src/main/java/com/commafeed/backend/model/UserSettings.java
View File

@@ -1,107 +1,107 @@
package com.commafeed.backend.model;
import java.sql.Types;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.Lob;
import jakarta.persistence.OneToOne;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERSETTINGS")
@SuppressWarnings("serial")
@Getter
@Setter
public class UserSettings extends AbstractModel {
public enum ReadingMode {
all, unread
}
public enum ReadingOrder {
asc, desc
}
public enum ViewMode {
title, cozy, detailed, expanded
}
public enum ScrollMode {
always, never, if_needed
}
public enum IconDisplayMode {
always, never, on_desktop, on_mobile
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false, unique = true)
private User user;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingMode readingMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingOrder readingOrder;
@Column(name = "user_lang", length = 4)
private String language;
private boolean showRead;
private boolean scrollMarks;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customCss;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customJs;
@Column(name = "scroll_speed")
private int scrollSpeed;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ScrollMode scrollMode;
private int entriesToKeepOnTopWhenScrolling;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode starIconDisplayMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode externalLinkIconDisplayMode;
private boolean markAllAsReadConfirmation;
private boolean customContextMenu;
private boolean mobileFooter;
private boolean unreadCountTitle;
private boolean unreadCountFavicon;
private boolean email;
private boolean gmail;
private boolean facebook;
private boolean twitter;
private boolean tumblr;
private boolean pocket;
private boolean instapaper;
private boolean buffer;
}
package com.commafeed.backend.model;
import java.sql.Types;
import jakarta.persistence.Column;
import jakarta.persistence.Entity;
import jakarta.persistence.EnumType;
import jakarta.persistence.Enumerated;
import jakarta.persistence.FetchType;
import jakarta.persistence.JoinColumn;
import jakarta.persistence.Lob;
import jakarta.persistence.OneToOne;
import jakarta.persistence.Table;
import org.hibernate.annotations.JdbcTypeCode;
import lombok.Getter;
import lombok.Setter;
@Entity
@Table(name = "USERSETTINGS")
@SuppressWarnings("serial")
@Getter
@Setter
public class UserSettings extends AbstractModel {
public enum ReadingMode {
all, unread
}
public enum ReadingOrder {
asc, desc
}
public enum ViewMode {
title, cozy, detailed, expanded
}
public enum ScrollMode {
always, never, if_needed
}
public enum IconDisplayMode {
always, never, on_desktop, on_mobile
}
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "user_id", nullable = false, unique = true)
private User user;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingMode readingMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ReadingOrder readingOrder;
@Column(name = "user_lang", length = 4)
private String language;
private boolean showRead;
private boolean scrollMarks;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customCss;
@Lob
@Column(length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String customJs;
@Column(name = "scroll_speed")
private int scrollSpeed;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private ScrollMode scrollMode;
private int entriesToKeepOnTopWhenScrolling;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode starIconDisplayMode;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private IconDisplayMode externalLinkIconDisplayMode;
private boolean markAllAsReadConfirmation;
private boolean customContextMenu;
private boolean mobileFooter;
private boolean unreadCountTitle;
private boolean unreadCountFavicon;
private boolean email;
private boolean gmail;
private boolean facebook;
private boolean twitter;
private boolean tumblr;
private boolean pocket;
private boolean instapaper;
private boolean buffer;
}

170
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLExporter.java
View File

@@ -1,85 +1,85 @@
package com.commafeed.backend.opml;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.rometools.opml.feed.opml.Attribute;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OPMLExporter {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
public Opml export(User user) {
Opml opml = new Opml();
opml.setFeedType("opml_1.0");
opml.setTitle(String.format("%s subscriptions in CommaFeed", user.getName()));
opml.setCreated(new Date());
List<FeedCategory> categories = feedCategoryDAO.findAll(user);
categories.sort(Comparator.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0)));
List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user);
subscriptions.sort(Comparator.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0)));
// export root categories
for (FeedCategory cat : categories.stream().filter(c -> c.getParent() == null).toList()) {
opml.getOutlines().add(buildCategoryOutline(cat, categories, subscriptions));
}
// export root subscriptions
for (FeedSubscription sub : subscriptions.stream().filter(s -> s.getCategory() == null).toList()) {
opml.getOutlines().add(buildSubscriptionOutline(sub));
}
return opml;
}
private Outline buildCategoryOutline(FeedCategory cat, List<FeedCategory> categories, List<FeedSubscription> subscriptions) {
Outline outline = new Outline();
outline.setText(cat.getName());
outline.setTitle(cat.getName());
for (FeedCategory child : categories.stream()
.filter(c -> c.getParent() != null && c.getParent().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildCategoryOutline(child, categories, subscriptions));
}
for (FeedSubscription sub : subscriptions.stream()
.filter(s -> s.getCategory() != null && s.getCategory().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildSubscriptionOutline(sub));
}
return outline;
}
private Outline buildSubscriptionOutline(FeedSubscription sub) {
Outline outline = new Outline();
outline.setText(sub.getTitle());
outline.setTitle(sub.getTitle());
outline.setType("rss");
outline.getAttributes().add(new Attribute("xmlUrl", sub.getFeed().getUrl()));
if (sub.getFeed().getLink() != null) {
outline.getAttributes().add(new Attribute("htmlUrl", sub.getFeed().getLink()));
}
return outline;
}
}
package com.commafeed.backend.opml;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.rometools.opml.feed.opml.Attribute;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OPMLExporter {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
public Opml export(User user) {
Opml opml = new Opml();
opml.setFeedType("opml_1.0");
opml.setTitle(String.format("%s subscriptions in CommaFeed", user.getName()));
opml.setCreated(new Date());
List<FeedCategory> categories = feedCategoryDAO.findAll(user);
categories.sort(Comparator.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0)));
List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user);
subscriptions.sort(Comparator.comparingInt(e -> ObjectUtils.firstNonNull(e.getPosition(), 0)));
// export root categories
for (FeedCategory cat : categories.stream().filter(c -> c.getParent() == null).toList()) {
opml.getOutlines().add(buildCategoryOutline(cat, categories, subscriptions));
}
// export root subscriptions
for (FeedSubscription sub : subscriptions.stream().filter(s -> s.getCategory() == null).toList()) {
opml.getOutlines().add(buildSubscriptionOutline(sub));
}
return opml;
}
private Outline buildCategoryOutline(FeedCategory cat, List<FeedCategory> categories, List<FeedSubscription> subscriptions) {
Outline outline = new Outline();
outline.setText(cat.getName());
outline.setTitle(cat.getName());
for (FeedCategory child : categories.stream()
.filter(c -> c.getParent() != null && c.getParent().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildCategoryOutline(child, categories, subscriptions));
}
for (FeedSubscription sub : subscriptions.stream()
.filter(s -> s.getCategory() != null && s.getCategory().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildSubscriptionOutline(sub));
}
return outline;
}
private Outline buildSubscriptionOutline(FeedSubscription sub) {
Outline outline = new Outline();
outline.setText(sub.getTitle());
outline.setTitle(sub.getTitle());
outline.setType("rss");
outline.getAttributes().add(new Attribute("xmlUrl", sub.getFeed().getUrl()));
if (sub.getFeed().getLink() != null) {
outline.getAttributes().add(new Attribute("htmlUrl", sub.getFeed().getLink()));
}
return outline;
}
}

164
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLImporter.java
View File

@@ -1,82 +1,82 @@
package com.commafeed.backend.opml;
import java.io.StringReader;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedSubscriptionService;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.WireFeedInput;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class OPMLImporter {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionService feedSubscriptionService;
public void importOpml(User user, String xml) throws IllegalArgumentException, FeedException {
xml = xml.substring(xml.indexOf('<'));
WireFeedInput input = new WireFeedInput();
Opml feed = (Opml) input.build(new StringReader(xml));
List<Outline> outlines = feed.getOutlines();
for (int i = 0; i < outlines.size(); i++) {
handleOutline(user, outlines.get(i), null, i);
}
}
private void handleOutline(User user, Outline outline, FeedCategory parent, int position) {
List<Outline> children = outline.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
FeedCategory category = feedCategoryDAO.findByName(user, name, parent);
if (category == null) {
if (StringUtils.isBlank(name)) {
name = "Unnamed category";
}
category = new FeedCategory();
category.setName(name);
category.setParent(parent);
category.setUser(user);
category.setPosition(position);
feedCategoryDAO.saveOrUpdate(category);
}
for (int i = 0; i < children.size(); i++) {
handleOutline(user, children.get(i), category, i);
}
} else {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
if (StringUtils.isBlank(name)) {
name = "Unnamed subscription";
}
// make sure we continue with the import process even if a feed failed
try {
feedSubscriptionService.subscribe(user, outline.getXmlUrl(), name, parent, position);
} catch (Exception e) {
log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage());
}
}
}
}
package com.commafeed.backend.opml;
import java.io.StringReader;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedSubscriptionService;
import com.rometools.opml.feed.opml.Opml;
import com.rometools.opml.feed.opml.Outline;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.WireFeedInput;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class OPMLImporter {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionService feedSubscriptionService;
public void importOpml(User user, String xml) throws IllegalArgumentException, FeedException {
xml = xml.substring(xml.indexOf('<'));
WireFeedInput input = new WireFeedInput();
Opml feed = (Opml) input.build(new StringReader(xml));
List<Outline> outlines = feed.getOutlines();
for (int i = 0; i < outlines.size(); i++) {
handleOutline(user, outlines.get(i), null, i);
}
}
private void handleOutline(User user, Outline outline, FeedCategory parent, int position) {
List<Outline> children = outline.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
FeedCategory category = feedCategoryDAO.findByName(user, name, parent);
if (category == null) {
if (StringUtils.isBlank(name)) {
name = "Unnamed category";
}
category = new FeedCategory();
category.setName(name);
category.setParent(parent);
category.setUser(user);
category.setPosition(position);
feedCategoryDAO.saveOrUpdate(category);
}
for (int i = 0; i < children.size(); i++) {
handleOutline(user, children.get(i), category, i);
}
} else {
String name = FeedUtils.truncate(outline.getText(), 128);
if (name == null) {
name = FeedUtils.truncate(outline.getTitle(), 128);
}
if (StringUtils.isBlank(name)) {
name = "Unnamed subscription";
}
// make sure we continue with the import process even if a feed failed
try {
feedSubscriptionService.subscribe(user, outline.getXmlUrl(), name, parent, position);
} catch (Exception e) {
log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage());
}
}
}
}

52
commafeed-server/src/main/java/com/commafeed/backend/rome/OPML11Generator.java
View File

@@ -1,26 +1,26 @@
package com.commafeed.backend.rome;
import org.jdom2.Element;
import com.rometools.opml.feed.opml.Opml;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Add missing title to the generated OPML
*
*/
@RegisterForReflection
public class OPML11Generator extends com.rometools.opml.io.impl.OPML10Generator {
public OPML11Generator() {
super("opml_1.1");
}
@Override
protected Element generateHead(Opml opml) {
Element head = new Element("head");
addNotNullSimpleElement(head, "title", opml.getTitle());
return head;
}
}
package com.commafeed.backend.rome;
import org.jdom2.Element;
import com.rometools.opml.feed.opml.Opml;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Add missing title to the generated OPML
*
*/
@RegisterForReflection
public class OPML11Generator extends com.rometools.opml.io.impl.OPML10Generator {
public OPML11Generator() {
super("opml_1.1");
}
@Override
protected Element generateHead(Opml opml) {
Element head = new Element("head");
addNotNullSimpleElement(head, "title", opml.getTitle());
return head;
}
}

76
commafeed-server/src/main/java/com/commafeed/backend/rome/OPML11Parser.java
View File

@@ -1,38 +1,38 @@
package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Document;
import org.jdom2.Element;
import com.rometools.opml.io.impl.OPML10Parser;
import com.rometools.rome.feed.WireFeed;
import com.rometools.rome.io.FeedException;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support for OPML 1.1 parsing
*
*/
@RegisterForReflection
public class OPML11Parser extends OPML10Parser {
public OPML11Parser() {
super("opml_1.1");
}
@Override
public boolean isMyType(Document document) {
Element e = document.getRootElement();
return e.getName().equals("opml");
}
@Override
public WireFeed parse(Document document, boolean validate, Locale locale) throws IllegalArgumentException, FeedException {
document.getRootElement().getChildren().add(new Element("head"));
return super.parse(document, validate, locale);
}
}
package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Document;
import org.jdom2.Element;
import com.rometools.opml.io.impl.OPML10Parser;
import com.rometools.rome.feed.WireFeed;
import com.rometools.rome.io.FeedException;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support for OPML 1.1 parsing
*
*/
@RegisterForReflection
public class OPML11Parser extends OPML10Parser {
public OPML11Parser() {
super("opml_1.1");
}
@Override
public boolean isMyType(Document document) {
Element e = document.getRootElement();
return e.getName().equals("opml");
}
@Override
public WireFeed parse(Document document, boolean validate, Locale locale) throws IllegalArgumentException, FeedException {
document.getRootElement().getChildren().add(new Element("head"));
return super.parse(document, validate, locale);
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/rome/RSS090DescriptionConverter.java
View File

@@ -1,30 +1,30 @@
package com.commafeed.backend.rome;
import com.rometools.rome.feed.rss.Description;
import com.rometools.rome.feed.rss.Item;
import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.impl.ConverterForRSS090;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support description tag for RSS09
*
*/
@RegisterForReflection
public class RSS090DescriptionConverter extends ConverterForRSS090 {
@Override
protected SyndEntry createSyndEntry(Item item, boolean preserveWireItem) {
SyndEntry entry = super.createSyndEntry(item, preserveWireItem);
Description desc = item.getDescription();
if (desc != null) {
SyndContentImpl syndDesc = new SyndContentImpl();
syndDesc.setValue(desc.getValue());
entry.setDescription(syndDesc);
}
return entry;
}
}
package com.commafeed.backend.rome;
import com.rometools.rome.feed.rss.Description;
import com.rometools.rome.feed.rss.Item;
import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.impl.ConverterForRSS090;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support description tag for RSS09
*
*/
@RegisterForReflection
public class RSS090DescriptionConverter extends ConverterForRSS090 {
@Override
protected SyndEntry createSyndEntry(Item item, boolean preserveWireItem) {
SyndEntry entry = super.createSyndEntry(item, preserveWireItem);
Description desc = item.getDescription();
if (desc != null) {
SyndContentImpl syndDesc = new SyndContentImpl();
syndDesc.setValue(desc.getValue());
entry.setDescription(syndDesc);
}
return entry;
}
}

64
commafeed-server/src/main/java/com/commafeed/backend/rome/RSS090DescriptionParser.java
View File

@@ -1,32 +1,32 @@
package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Element;
import com.rometools.rome.feed.rss.Description;
import com.rometools.rome.feed.rss.Item;
import com.rometools.rome.io.impl.RSS090Parser;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support description tag for RSS09
*
*/
@RegisterForReflection
public class RSS090DescriptionParser extends RSS090Parser {
@Override
protected Item parseItem(Element rssRoot, Element eItem, Locale locale) {
Item item = super.parseItem(rssRoot, eItem, locale);
Element e = eItem.getChild("description", getRSSNamespace());
if (e != null) {
Description desc = new Description();
desc.setValue(e.getText());
item.setDescription(desc);
}
return item;
}
}
package com.commafeed.backend.rome;
import java.util.Locale;
import org.jdom2.Element;
import com.rometools.rome.feed.rss.Description;
import com.rometools.rome.feed.rss.Item;
import com.rometools.rome.io.impl.RSS090Parser;
import io.quarkus.runtime.annotations.RegisterForReflection;
/**
* Support description tag for RSS09
*
*/
@RegisterForReflection
public class RSS090DescriptionParser extends RSS090Parser {
@Override
protected Item parseItem(Element rssRoot, Element eItem, Locale locale) {
Item item = super.parseItem(rssRoot, eItem, locale);
Element e = eItem.getChild("description", getRSSNamespace());
if (e != null) {
Description desc = new Description();
desc.setValue(e.getText());
item.setDescription(desc);
}
return item;
}
}

96
commafeed-server/src/main/java/com/commafeed/backend/rome/RSSRDF10Parser.java
View File

@@ -1,48 +1,48 @@
package com.commafeed.backend.rome;
import java.util.List;
import org.apache.commons.collections4.CollectionUtils;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.Namespace;
import com.google.common.collect.Lists;
import com.rometools.rome.io.impl.RSS10Parser;
import io.quarkus.runtime.annotations.RegisterForReflection;
@RegisterForReflection
public class RSSRDF10Parser extends RSS10Parser {
private static final String RSS_URI = "http://purl.org/rss/1.0/";
private static final Namespace RSS_NS = Namespace.getNamespace(RSS_URI);
public RSSRDF10Parser() {
super("rss_1.0", RSS_NS);
}
@Override
public boolean isMyType(Document document) {
boolean ok;
Element rssRoot = document.getRootElement();
Namespace defaultNS = rssRoot.getNamespace();
List<Namespace> additionalNSs = Lists.newArrayList(rssRoot.getAdditionalNamespaces());
List<Element> children = rssRoot.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
Element child = children.get(0);
additionalNSs.add(child.getNamespace());
additionalNSs.addAll(child.getAdditionalNamespaces());
}
ok = defaultNS != null && defaultNS.equals(getRDFNamespace());
if (ok) {
ok = false;
for (int i = 0; !ok && i < additionalNSs.size(); i++) {
ok = getRSSNamespace().equals(additionalNSs.get(i));
}
}
return ok;
}
}
package com.commafeed.backend.rome;
import java.util.List;
import org.apache.commons.collections4.CollectionUtils;
import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.Namespace;
import com.google.common.collect.Lists;
import com.rometools.rome.io.impl.RSS10Parser;
import io.quarkus.runtime.annotations.RegisterForReflection;
@RegisterForReflection
public class RSSRDF10Parser extends RSS10Parser {
private static final String RSS_URI = "http://purl.org/rss/1.0/";
private static final Namespace RSS_NS = Namespace.getNamespace(RSS_URI);
public RSSRDF10Parser() {
super("rss_1.0", RSS_NS);
}
@Override
public boolean isMyType(Document document) {
boolean ok;
Element rssRoot = document.getRootElement();
Namespace defaultNS = rssRoot.getNamespace();
List<Namespace> additionalNSs = Lists.newArrayList(rssRoot.getAdditionalNamespaces());
List<Element> children = rssRoot.getChildren();
if (CollectionUtils.isNotEmpty(children)) {
Element child = children.get(0);
additionalNSs.add(child.getNamespace());
additionalNSs.addAll(child.getAdditionalNamespaces());
}
ok = defaultNS != null && defaultNS.equals(getRDFNamespace());
if (ok) {
ok = false;
for (int i = 0; !ok && i < additionalNSs.size(); i++) {
ok = getRSSNamespace().equals(additionalNSs.get(i));
}
}
return ok;
}
}

350
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentCleaningService.java
View File

@@ -1,175 +1,175 @@
package com.commafeed.backend.service;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Document.OutputSettings;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Safelist;
import org.w3c.css.sac.CSSException;
import org.w3c.css.sac.CSSParseException;
import org.w3c.css.sac.ErrorHandler;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
import com.steadystate.css.parser.CSSOMParser;
import com.steadystate.css.parser.SACParserCSS21;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
@Singleton
public class FeedEntryContentCleaningService {
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
public String clean(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
}
return content;
}
private static Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1",
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
private String escapeIFrameCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private String escapeImgCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser(new SACParserCSS21());
parser.setErrorHandler(new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug("warning while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug("fatal error while parsing css: {}", exception.getMessage(), exception);
}
});
return parser;
}
}
package com.commafeed.backend.service;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Document.OutputSettings;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Safelist;
import org.w3c.css.sac.CSSException;
import org.w3c.css.sac.CSSParseException;
import org.w3c.css.sac.ErrorHandler;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
import com.steadystate.css.parser.CSSOMParser;
import com.steadystate.css.parser.SACParserCSS21;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
@Singleton
public class FeedEntryContentCleaningService {
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
public String clean(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
}
return content;
}
private static Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1",
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
private String escapeIFrameCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private String escapeImgCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser(new SACParserCSS21());
parser.setErrorHandler(new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug("warning while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug("fatal error while parsing css: {}", exception.getMessage(), exception);
}
});
return parser;
}
}

142
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentService.java
View File

@@ -1,71 +1,71 @@
package com.commafeed.backend.service;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.commafeed.backend.model.FeedEntryContent;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryContentService {
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryContentCleaningService cleaningService;
/**
* this is NOT thread-safe
*/
public FeedEntryContent findOrCreate(Content content, String baseUrl) {
FeedEntryContent entryContent = buildContent(content, baseUrl);
Optional<FeedEntryContent> existing = feedEntryContentDAO.findExisting(entryContent.getContentHash(), entryContent.getTitleHash())
.stream()
.filter(entryContent::equivalentTo)
.findFirst();
if (existing.isPresent()) {
return existing.get();
} else {
feedEntryContentDAO.saveOrUpdate(entryContent);
return entryContent;
}
}
private FeedEntryContent buildContent(Content content, String baseUrl) {
FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitleHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.title())));
entryContent.setContentHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.content())));
entryContent.setTitle(FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false));
entryContent.setAuthor(FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
entryContent.setDirection(
FeedUtils.isRTL(content.title(), content.content()) ? FeedEntryContent.Direction.rtl : FeedEntryContent.Direction.ltr);
Enclosure enclosure = content.enclosure();
if (enclosure != null) {
entryContent.setEnclosureUrl(FeedUtils.truncate(enclosure.url(), 2048));
entryContent.setEnclosureType(enclosure.type());
}
Media media = content.media();
if (media != null) {
entryContent.setMediaDescription(cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048));
entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
entryContent.setMediaThumbnailHeight(media.thumbnailHeight());
}
return entryContent;
}
}
package com.commafeed.backend.service;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.commafeed.backend.model.FeedEntryContent;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryContentService {
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryContentCleaningService cleaningService;
/**
* this is NOT thread-safe
*/
public FeedEntryContent findOrCreate(Content content, String baseUrl) {
FeedEntryContent entryContent = buildContent(content, baseUrl);
Optional<FeedEntryContent> existing = feedEntryContentDAO.findExisting(entryContent.getContentHash(), entryContent.getTitleHash())
.stream()
.filter(entryContent::equivalentTo)
.findFirst();
if (existing.isPresent()) {
return existing.get();
} else {
feedEntryContentDAO.saveOrUpdate(entryContent);
return entryContent;
}
}
private FeedEntryContent buildContent(Content content, String baseUrl) {
FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitleHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.title())));
entryContent.setContentHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.content())));
entryContent.setTitle(FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false));
entryContent.setAuthor(FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
entryContent.setDirection(
FeedUtils.isRTL(content.title(), content.content()) ? FeedEntryContent.Direction.rtl : FeedEntryContent.Direction.ltr);
Enclosure enclosure = content.enclosure();
if (enclosure != null) {
entryContent.setEnclosureUrl(FeedUtils.truncate(enclosure.url(), 2048));
entryContent.setEnclosureType(enclosure.type());
}
Media media = content.media();
if (media != null) {
entryContent.setMediaDescription(cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048));
entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
entryContent.setMediaThumbnailHeight(media.thumbnailHeight());
}
return entryContent;
}
}

248
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryFilteringService.java
View File

@@ -1,124 +1,124 @@
package com.commafeed.backend.service;
import java.time.Year;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import jakarta.inject.Singleton;
import org.apache.commons.jexl2.JexlContext;
import org.apache.commons.jexl2.JexlEngine;
import org.apache.commons.jexl2.JexlException;
import org.apache.commons.jexl2.JexlInfo;
import org.apache.commons.jexl2.MapContext;
import org.apache.commons.jexl2.Script;
import org.apache.commons.jexl2.introspection.JexlMethod;
import org.apache.commons.jexl2.introspection.JexlPropertyGet;
import org.apache.commons.jexl2.introspection.Uberspect;
import org.apache.commons.jexl2.introspection.UberspectImpl;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.model.FeedEntry;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryFilteringService {
private static final JexlEngine ENGINE = initEngine();
private final ExecutorService executor = Executors.newCachedThreadPool();
private final CommaFeedConfiguration config;
private static JexlEngine initEngine() {
// classloader that prevents object creation
ClassLoader cl = new ClassLoader() {
@Override
protected Class<?> loadClass(String name, boolean resolve) {
return null;
}
};
// uberspect that prevents access to .class and .getClass()
Uberspect uberspect = new UberspectImpl(LogFactory.getLog(JexlEngine.class)) {
@Override
public JexlPropertyGet getPropertyGet(Object obj, Object identifier, JexlInfo info) {
if ("class".equals(identifier)) {
return null;
}
return super.getPropertyGet(obj, identifier, info);
}
@Override
public JexlMethod getMethod(Object obj, String method, Object[] args, JexlInfo info) {
if ("getClass".equals(method)) {
return null;
}
return super.getMethod(obj, method, args, info);
}
};
JexlEngine engine = new JexlEngine(uberspect, null, null, null);
engine.setStrict(true);
engine.setClassLoader(cl);
return engine;
}
public boolean filterMatchesEntry(String filter, FeedEntry entry) throws FeedEntryFilterException {
if (StringUtils.isBlank(filter)) {
return true;
}
Script script;
try {
script = ENGINE.createScript(filter);
} catch (JexlException e) {
throw new FeedEntryFilterException("Exception while parsing expression " + filter, e);
}
JexlContext context = new MapContext();
context.set("title", entry.getContent().getTitle() == null ? "" : Jsoup.parse(entry.getContent().getTitle()).text().toLowerCase());
context.set("author", entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor().toLowerCase());
context.set("content",
entry.getContent().getContent() == null ? "" : Jsoup.parse(entry.getContent().getContent()).text().toLowerCase());
context.set("url", entry.getUrl() == null ? "" : entry.getUrl().toLowerCase());
context.set("categories", entry.getContent().getCategories() == null ? "" : entry.getContent().getCategories().toLowerCase());
context.set("year", Year.now().getValue());
Callable<Object> callable = script.callable(context);
Future<Object> future = executor.submit(callable);
Object result;
try {
result = future.get(config.feedRefresh().filteringExpressionEvaluationTimeout().toMillis(), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e);
} catch (ExecutionException e) {
throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}
try {
return (boolean) result;
} catch (ClassCastException e) {
throw new FeedEntryFilterException(e.getMessage(), e);
}
}
@SuppressWarnings("serial")
public static class FeedEntryFilterException extends Exception {
public FeedEntryFilterException(String message, Throwable t) {
super(message, t);
}
}
}
package com.commafeed.backend.service;
import java.time.Year;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import jakarta.inject.Singleton;
import org.apache.commons.jexl2.JexlContext;
import org.apache.commons.jexl2.JexlEngine;
import org.apache.commons.jexl2.JexlException;
import org.apache.commons.jexl2.JexlInfo;
import org.apache.commons.jexl2.MapContext;
import org.apache.commons.jexl2.Script;
import org.apache.commons.jexl2.introspection.JexlMethod;
import org.apache.commons.jexl2.introspection.JexlPropertyGet;
import org.apache.commons.jexl2.introspection.Uberspect;
import org.apache.commons.jexl2.introspection.UberspectImpl;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.model.FeedEntry;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryFilteringService {
private static final JexlEngine ENGINE = initEngine();
private final ExecutorService executor = Executors.newCachedThreadPool();
private final CommaFeedConfiguration config;
private static JexlEngine initEngine() {
// classloader that prevents object creation
ClassLoader cl = new ClassLoader() {
@Override
protected Class<?> loadClass(String name, boolean resolve) {
return null;
}
};
// uberspect that prevents access to .class and .getClass()
Uberspect uberspect = new UberspectImpl(LogFactory.getLog(JexlEngine.class)) {
@Override
public JexlPropertyGet getPropertyGet(Object obj, Object identifier, JexlInfo info) {
if ("class".equals(identifier)) {
return null;
}
return super.getPropertyGet(obj, identifier, info);
}
@Override
public JexlMethod getMethod(Object obj, String method, Object[] args, JexlInfo info) {
if ("getClass".equals(method)) {
return null;
}
return super.getMethod(obj, method, args, info);
}
};
JexlEngine engine = new JexlEngine(uberspect, null, null, null);
engine.setStrict(true);
engine.setClassLoader(cl);
return engine;
}
public boolean filterMatchesEntry(String filter, FeedEntry entry) throws FeedEntryFilterException {
if (StringUtils.isBlank(filter)) {
return true;
}
Script script;
try {
script = ENGINE.createScript(filter);
} catch (JexlException e) {
throw new FeedEntryFilterException("Exception while parsing expression " + filter, e);
}
JexlContext context = new MapContext();
context.set("title", entry.getContent().getTitle() == null ? "" : Jsoup.parse(entry.getContent().getTitle()).text().toLowerCase());
context.set("author", entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor().toLowerCase());
context.set("content",
entry.getContent().getContent() == null ? "" : Jsoup.parse(entry.getContent().getContent()).text().toLowerCase());
context.set("url", entry.getUrl() == null ? "" : entry.getUrl().toLowerCase());
context.set("categories", entry.getContent().getCategories() == null ? "" : entry.getContent().getCategories().toLowerCase());
context.set("year", Year.now().getValue());
Callable<Object> callable = script.callable(context);
Future<Object> future = executor.submit(callable);
Object result;
try {
result = future.get(config.feedRefresh().filteringExpressionEvaluationTimeout().toMillis(), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e);
} catch (ExecutionException e) {
throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}
try {
return (boolean) result;
} catch (ClassCastException e) {
throw new FeedEntryFilterException(e.getMessage(), e);
}
}
@SuppressWarnings("serial")
public static class FeedEntryFilterException extends Exception {
public FeedEntryFilterException(String message, Throwable t) {
super(message, t);
}
}
}

262
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java
View File

@@ -1,131 +1,131 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class FeedEntryService {
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
public FeedEntry find(Feed feed, Entry entry) {
String guidHash = Digests.sha1Hex(entry.guid());
return feedEntryDAO.findExisting(guidHash, feed);
}
public FeedEntry create(Feed feed, Entry entry) {
FeedEntry feedEntry = new FeedEntry();
feedEntry.setGuid(FeedUtils.truncate(entry.guid(), 2048));
feedEntry.setGuidHash(Digests.sha1Hex(entry.guid()));
feedEntry.setUrl(FeedUtils.truncate(entry.url(), 2048));
feedEntry.setPublished(entry.published());
feedEntry.setInserted(Instant.now());
feedEntry.setFeed(feed);
feedEntry.setContent(feedEntryContentService.findOrCreate(entry.content(), feed.getLink()));
feedEntryDAO.saveOrUpdate(feedEntry);
return feedEntry;
}
public boolean applyFilter(FeedSubscription sub, FeedEntry entry) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.saveOrUpdate(status);
}
return matches;
}
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, entry.getFeed());
if (sub == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
if (status.isMarkable()) {
status.setRead(read);
feedEntryStatusDAO.saveOrUpdate(status);
}
}
public void starEntry(User user, Long entryId, Long subscriptionId, boolean starred) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subscriptionId);
if (sub == null) {
return;
}
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
status.setStarred(starred);
feedEntryStatusDAO.saveOrUpdate(status);
}
public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Instant olderThan, Instant insertedBefore,
List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null,
false, null, null, null);
markList(statuses, olderThan, insertedBefore);
}
public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, -1, -1, null, false);
markList(statuses, olderThan, insertedBefore);
}
private void markList(List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statusesToMark = statuses.stream().filter(FeedEntryStatus::isMarkable).filter(s -> {
Instant entryDate = s.getEntry().getPublished();
return olderThan == null || entryDate == null || entryDate.isBefore(olderThan);
}).filter(s -> {
Instant insertedDate = s.getEntry().getInserted();
return insertedBefore == null || insertedDate == null || insertedDate.isBefore(insertedBefore);
}).toList();
statusesToMark.forEach(s -> s.setRead(true));
feedEntryStatusDAO.saveOrUpdate(statusesToMark);
}
}
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class FeedEntryService {
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
public FeedEntry find(Feed feed, Entry entry) {
String guidHash = Digests.sha1Hex(entry.guid());
return feedEntryDAO.findExisting(guidHash, feed);
}
public FeedEntry create(Feed feed, Entry entry) {
FeedEntry feedEntry = new FeedEntry();
feedEntry.setGuid(FeedUtils.truncate(entry.guid(), 2048));
feedEntry.setGuidHash(Digests.sha1Hex(entry.guid()));
feedEntry.setUrl(FeedUtils.truncate(entry.url(), 2048));
feedEntry.setPublished(entry.published());
feedEntry.setInserted(Instant.now());
feedEntry.setFeed(feed);
feedEntry.setContent(feedEntryContentService.findOrCreate(entry.content(), feed.getLink()));
feedEntryDAO.saveOrUpdate(feedEntry);
return feedEntry;
}
public boolean applyFilter(FeedSubscription sub, FeedEntry entry) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.saveOrUpdate(status);
}
return matches;
}
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, entry.getFeed());
if (sub == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
if (status.isMarkable()) {
status.setRead(read);
feedEntryStatusDAO.saveOrUpdate(status);
}
}
public void starEntry(User user, Long entryId, Long subscriptionId, boolean starred) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subscriptionId);
if (sub == null) {
return;
}
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
status.setStarred(starred);
feedEntryStatusDAO.saveOrUpdate(status);
}
public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Instant olderThan, Instant insertedBefore,
List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null,
false, null, null, null);
markList(statuses, olderThan, insertedBefore);
}
public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, -1, -1, null, false);
markList(statuses, olderThan, insertedBefore);
}
private void markList(List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statusesToMark = statuses.stream().filter(FeedEntryStatus::isMarkable).filter(s -> {
Instant entryDate = s.getEntry().getPublished();
return olderThan == null || entryDate == null || entryDate.isBefore(olderThan);
}).filter(s -> {
Instant insertedDate = s.getEntry().getInserted();
return insertedBefore == null || insertedDate == null || insertedDate.isBefore(insertedBefore);
}).toList();
statusesToMark.forEach(s -> s.setRead(true));
feedEntryStatusDAO.saveOrUpdate(statusesToMark);
}
}

86
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryTagService.java
View File

@@ -1,43 +1,43 @@
package com.commafeed.backend.service;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryTagDAO;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryTagService {
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO;
public void updateTags(User user, Long entryId, List<String> tagNames) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry);
Set<String> existingTagNames = existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
List<FeedEntryTag> addList = tagNames.stream()
.filter(name -> !existingTagNames.contains(name))
.map(name -> new FeedEntryTag(user, entry, name))
.toList();
List<FeedEntryTag> removeList = existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).toList();
feedEntryTagDAO.saveOrUpdate(addList);
feedEntryTagDAO.delete(removeList);
}
}
package com.commafeed.backend.service;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryTagDAO;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryTagService {
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO;
public void updateTags(User user, Long entryId, List<String> tagNames) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry);
Set<String> existingTagNames = existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
List<FeedEntryTag> addList = tagNames.stream()
.filter(name -> !existingTagNames.contains(name))
.map(name -> new FeedEntryTag(user, entry, name))
.toList();
List<FeedEntryTag> removeList = existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).toList();
feedEntryTagDAO.saveOrUpdate(addList);
feedEntryTagDAO.delete(removeList);
}
}

160
commafeed-server/src/main/java/com/commafeed/backend/service/FeedService.java
View File

@@ -1,80 +1,80 @@
package com.commafeed.backend.service;
import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
import com.commafeed.backend.favicon.Favicon;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.Models;
import com.google.common.io.Resources;
import io.quarkus.arc.All;
@Singleton
public class FeedService {
private final FeedDAO feedDAO;
private final List<AbstractFaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
public FeedService(FeedDAO feedDAO, @All List<AbstractFaviconFetcher> faviconFetchers) {
this.feedDAO = feedDAO;
this.faviconFetchers = faviconFetchers;
try {
defaultFavicon = new Favicon(
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif");
} catch (IOException e) {
throw new RuntimeException("could not load default favicon", e);
}
}
public synchronized Feed findOrCreate(String url) {
String normalizedUrl = FeedUtils.normalizeURL(url);
String normalizedUrlHash = Digests.sha1Hex(normalizedUrl);
Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash);
if (feed == null) {
feed = new Feed();
feed.setUrl(url);
feed.setNormalizedUrl(normalizedUrl);
feed.setNormalizedUrlHash(normalizedUrlHash);
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
feedDAO.persist(feed);
}
return feed;
}
public void update(Feed feed) {
String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(Digests.sha1Hex(normalized));
feed.setLastUpdated(Instant.now());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.merge(feed);
}
public Favicon fetchFavicon(Feed feed) {
Favicon icon = null;
for (AbstractFaviconFetcher faviconFetcher : faviconFetchers) {
icon = faviconFetcher.fetch(feed);
if (icon != null) {
break;
}
}
if (icon == null) {
icon = defaultFavicon;
}
return icon;
}
}
package com.commafeed.backend.service;
import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
import com.commafeed.backend.favicon.Favicon;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.Models;
import com.google.common.io.Resources;
import io.quarkus.arc.All;
@Singleton
public class FeedService {
private final FeedDAO feedDAO;
private final List<AbstractFaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
public FeedService(FeedDAO feedDAO, @All List<AbstractFaviconFetcher> faviconFetchers) {
this.feedDAO = feedDAO;
this.faviconFetchers = faviconFetchers;
try {
defaultFavicon = new Favicon(
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif");
} catch (IOException e) {
throw new RuntimeException("could not load default favicon", e);
}
}
public synchronized Feed findOrCreate(String url) {
String normalizedUrl = FeedUtils.normalizeURL(url);
String normalizedUrlHash = Digests.sha1Hex(normalizedUrl);
Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash);
if (feed == null) {
feed = new Feed();
feed.setUrl(url);
feed.setNormalizedUrl(normalizedUrl);
feed.setNormalizedUrlHash(normalizedUrlHash);
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
feedDAO.persist(feed);
}
return feed;
}
public void update(Feed feed) {
String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(Digests.sha1Hex(normalized));
feed.setLastUpdated(Instant.now());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.merge(feed);
}
public Favicon fetchFavicon(Feed feed) {
Favicon icon = null;
for (AbstractFaviconFetcher faviconFetcher : faviconFetchers) {
icon = faviconFetcher.fetch(feed);
if (icon != null) {
break;
}
}
if (icon == null) {
icon = defaultFavicon;
}
return icon;
}
}

296
commafeed-server/src/main/java/com/commafeed/backend/service/FeedSubscriptionService.java
View File

@@ -1,148 +1,148 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.UnreadCount;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedSubscriptionService {
private final FeedDAO feedDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService;
private final FeedRefreshEngine feedRefreshEngine;
private final CommaFeedConfiguration config;
public FeedSubscriptionService(FeedDAO feedDAO, FeedEntryStatusDAO feedEntryStatusDAO, FeedSubscriptionDAO feedSubscriptionDAO,
FeedService feedService, FeedRefreshEngine feedRefreshEngine, CommaFeedConfiguration config) {
this.feedDAO = feedDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.feedService = feedService;
this.feedRefreshEngine = feedRefreshEngine;
this.config = config;
// automatically refresh new feeds after they are subscribed to
// we need to use this hook because the feed needs to have been persisted before being processed by the feed engine
feedSubscriptionDAO.onPostCommitInsert(sub -> {
Feed feed = sub.getFeed();
if (feed.getDisabledUntil() == null || feed.getDisabledUntil().isBefore(Instant.now())) {
feedRefreshEngine.refreshImmediately(feed);
}
});
}
public long subscribe(User user, String url, String title) {
return subscribe(user, url, title, null, 0);
}
public long subscribe(User user, String url, String title, FeedCategory parent) {
return subscribe(user, url, title, parent, 0);
}
public long subscribe(User user, String url, String title, FeedCategory category, int position) {
Integer maxFeedsPerUser = config.database().cleanup().maxFeedsPerUser();
if (maxFeedsPerUser > 0 && feedSubscriptionDAO.count(user) >= maxFeedsPerUser) {
String message = String.format("You cannot subscribe to more feeds on this CommaFeed instance (max %s feeds per user)",
maxFeedsPerUser);
throw new FeedSubscriptionException(message);
}
Feed feed = feedService.findOrCreate(url);
// upgrade feed to https if it was using http
if (FeedUtils.isHttp(feed.getUrl()) && FeedUtils.isHttps(url)) {
feed.setUrl(url);
feedDAO.saveOrUpdate(feed);
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
if (sub == null) {
sub = new FeedSubscription();
sub.setFeed(feed);
sub.setUser(user);
}
sub.setCategory(category);
sub.setPosition(position);
sub.setTitle(FeedUtils.truncate(title, 128));
feedSubscriptionDAO.saveOrUpdate(sub);
return sub.getId();
}
public boolean unsubscribe(User user, Long subId) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subId);
if (sub != null) {
feedSubscriptionDAO.delete(sub);
return true;
} else {
return false;
}
}
public void refreshAll(User user) throws ForceFeedRefreshTooSoonException {
Instant lastForceRefresh = user.getLastForceRefresh();
if (lastForceRefresh != null && lastForceRefresh.plus(config.feedRefresh().forceRefreshCooldownDuration()).isAfter(Instant.now())) {
throw new ForceFeedRefreshTooSoonException();
}
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
user.setLastForceRefresh(Instant.now());
}
public void refreshAllUpForRefresh(User user) {
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Instant disabledUntil = sub.getFeed().getDisabledUntil();
if (disabledUntil == null || disabledUntil.isBefore(Instant.now())) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
}
}
public Map<Long, UnreadCount> getUnreadCount(User user) {
return feedSubscriptionDAO.findAll(user)
.stream()
.collect(Collectors.toMap(FeedSubscription::getId, feedEntryStatusDAO::getUnreadCount));
}
@SuppressWarnings("serial")
public static class FeedSubscriptionException extends RuntimeException {
private FeedSubscriptionException(String msg) {
super(msg);
}
}
@SuppressWarnings("serial")
public static class ForceFeedRefreshTooSoonException extends Exception {
private ForceFeedRefreshTooSoonException() {
super();
}
}
}
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.UnreadCount;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedSubscriptionService {
private final FeedDAO feedDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService;
private final FeedRefreshEngine feedRefreshEngine;
private final CommaFeedConfiguration config;
public FeedSubscriptionService(FeedDAO feedDAO, FeedEntryStatusDAO feedEntryStatusDAO, FeedSubscriptionDAO feedSubscriptionDAO,
FeedService feedService, FeedRefreshEngine feedRefreshEngine, CommaFeedConfiguration config) {
this.feedDAO = feedDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.feedService = feedService;
this.feedRefreshEngine = feedRefreshEngine;
this.config = config;
// automatically refresh new feeds after they are subscribed to
// we need to use this hook because the feed needs to have been persisted before being processed by the feed engine
feedSubscriptionDAO.onPostCommitInsert(sub -> {
Feed feed = sub.getFeed();
if (feed.getDisabledUntil() == null || feed.getDisabledUntil().isBefore(Instant.now())) {
feedRefreshEngine.refreshImmediately(feed);
}
});
}
public long subscribe(User user, String url, String title) {
return subscribe(user, url, title, null, 0);
}
public long subscribe(User user, String url, String title, FeedCategory parent) {
return subscribe(user, url, title, parent, 0);
}
public long subscribe(User user, String url, String title, FeedCategory category, int position) {
Integer maxFeedsPerUser = config.database().cleanup().maxFeedsPerUser();
if (maxFeedsPerUser > 0 && feedSubscriptionDAO.count(user) >= maxFeedsPerUser) {
String message = String.format("You cannot subscribe to more feeds on this CommaFeed instance (max %s feeds per user)",
maxFeedsPerUser);
throw new FeedSubscriptionException(message);
}
Feed feed = feedService.findOrCreate(url);
// upgrade feed to https if it was using http
if (FeedUtils.isHttp(feed.getUrl()) && FeedUtils.isHttps(url)) {
feed.setUrl(url);
feedDAO.saveOrUpdate(feed);
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
if (sub == null) {
sub = new FeedSubscription();
sub.setFeed(feed);
sub.setUser(user);
}
sub.setCategory(category);
sub.setPosition(position);
sub.setTitle(FeedUtils.truncate(title, 128));
feedSubscriptionDAO.saveOrUpdate(sub);
return sub.getId();
}
public boolean unsubscribe(User user, Long subId) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subId);
if (sub != null) {
feedSubscriptionDAO.delete(sub);
return true;
} else {
return false;
}
}
public void refreshAll(User user) throws ForceFeedRefreshTooSoonException {
Instant lastForceRefresh = user.getLastForceRefresh();
if (lastForceRefresh != null && lastForceRefresh.plus(config.feedRefresh().forceRefreshCooldownDuration()).isAfter(Instant.now())) {
throw new ForceFeedRefreshTooSoonException();
}
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
user.setLastForceRefresh(Instant.now());
}
public void refreshAllUpForRefresh(User user) {
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Instant disabledUntil = sub.getFeed().getDisabledUntil();
if (disabledUntil == null || disabledUntil.isBefore(Instant.now())) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
}
}
public Map<Long, UnreadCount> getUnreadCount(User user) {
return feedSubscriptionDAO.findAll(user)
.stream()
.collect(Collectors.toMap(FeedSubscription::getId, feedEntryStatusDAO::getUnreadCount));
}
@SuppressWarnings("serial")
public static class FeedSubscriptionException extends RuntimeException {
private FeedSubscriptionException(String msg) {
super(msg);
}
}
@SuppressWarnings("serial")
public static class ForceFeedRefreshTooSoonException extends Exception {
private ForceFeedRefreshTooSoonException() {
super();
}
}
}

42
commafeed-server/src/main/java/com/commafeed/backend/service/MailService.java
View File

@@ -1,21 +1,21 @@
package com.commafeed.backend.service;
import jakarta.inject.Singleton;
import com.commafeed.backend.model.User;
import io.quarkus.mailer.Mail;
import io.quarkus.mailer.Mailer;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class MailService {
private final Mailer mailer;
public void sendMail(User user, String subject, String content) {
Mail mail = Mail.withHtml(user.getEmail(), "CommaFeed - " + subject, content);
mailer.send(mail);
}
}
package com.commafeed.backend.service;
import jakarta.inject.Singleton;
import com.commafeed.backend.model.User;
import io.quarkus.mailer.Mail;
import io.quarkus.mailer.Mailer;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class MailService {
private final Mailer mailer;
public void sendMail(User user, String subject, String content) {
Mail mail = Mail.withHtml(user.getEmail(), "CommaFeed - " + subject, content);
mailer.send(mail);
}
}

190
commafeed-server/src/main/java/com/commafeed/backend/service/PasswordEncryptionService.java
View File

@@ -1,95 +1,95 @@
package com.commafeed.backend.service;
import java.io.Serializable;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.spec.KeySpec;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
// taken from http://www.javacodegeeks.com/2012/05/secure-password-storage-donts-dos-and.html
@SuppressWarnings("serial")
@Slf4j
@RequiredArgsConstructor
@Singleton
public class PasswordEncryptionService implements Serializable {
public boolean authenticate(String attemptedPassword, byte[] encryptedPassword, byte[] salt) {
if (StringUtils.isBlank(attemptedPassword)) {
return false;
}
// Encrypt the clear-text password using the same salt that was used to
// encrypt the original password
byte[] encryptedAttemptedPassword = null;
try {
encryptedAttemptedPassword = getEncryptedPassword(attemptedPassword, salt);
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
if (encryptedAttemptedPassword == null) {
return false;
}
// Authentication succeeds if encrypted password that the user entered
// is equal to the stored hash
return MessageDigest.isEqual(encryptedPassword, encryptedAttemptedPassword);
}
public byte[] getEncryptedPassword(String password, byte[] salt) {
// PBKDF2 with SHA-1 as the hashing algorithm. Note that the NIST
// specifically names SHA-1 as an acceptable hashing algorithm for
// PBKDF2
String algorithm = "PBKDF2WithHmacSHA1";
// SHA-1 generates 160 bit hashes, so that's what makes sense here
int derivedKeyLength = 160;
// Pick an iteration count that works for you. The NIST recommends at
// least 1,000 iterations:
// http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf
// iOS 4.x reportedly uses 10,000:
// http://blog.crackpassword.com/2010/09/smartphone-forensics-cracking-blackberry-backup-passwords/
int iterations = 20000;
KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, derivedKeyLength);
byte[] bytes = null;
try {
SecretKeyFactory f = SecretKeyFactory.getInstance(algorithm);
SecretKey key = f.generateSecret(spec);
bytes = key.getEncoded();
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
return bytes;
}
public byte[] generateSalt() {
// VERY important to use SecureRandom instead of just Random
byte[] salt = null;
try {
SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
// Generate a 8 byte (64 bit) salt as recommended by RSA PKCS5
salt = new byte[8];
random.nextBytes(salt);
} catch (NoSuchAlgorithmException e) {
// should never happen
log.error(e.getMessage(), e);
}
return salt;
}
}
package com.commafeed.backend.service;
import java.io.Serializable;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.spec.KeySpec;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
// taken from http://www.javacodegeeks.com/2012/05/secure-password-storage-donts-dos-and.html
@SuppressWarnings("serial")
@Slf4j
@RequiredArgsConstructor
@Singleton
public class PasswordEncryptionService implements Serializable {
public boolean authenticate(String attemptedPassword, byte[] encryptedPassword, byte[] salt) {
if (StringUtils.isBlank(attemptedPassword)) {
return false;
}
// Encrypt the clear-text password using the same salt that was used to
// encrypt the original password
byte[] encryptedAttemptedPassword = null;
try {
encryptedAttemptedPassword = getEncryptedPassword(attemptedPassword, salt);
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
if (encryptedAttemptedPassword == null) {
return false;
}
// Authentication succeeds if encrypted password that the user entered
// is equal to the stored hash
return MessageDigest.isEqual(encryptedPassword, encryptedAttemptedPassword);
}
public byte[] getEncryptedPassword(String password, byte[] salt) {
// PBKDF2 with SHA-1 as the hashing algorithm. Note that the NIST
// specifically names SHA-1 as an acceptable hashing algorithm for
// PBKDF2
String algorithm = "PBKDF2WithHmacSHA1";
// SHA-1 generates 160 bit hashes, so that's what makes sense here
int derivedKeyLength = 160;
// Pick an iteration count that works for you. The NIST recommends at
// least 1,000 iterations:
// http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf
// iOS 4.x reportedly uses 10,000:
// http://blog.crackpassword.com/2010/09/smartphone-forensics-cracking-blackberry-backup-passwords/
int iterations = 20000;
KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, derivedKeyLength);
byte[] bytes = null;
try {
SecretKeyFactory f = SecretKeyFactory.getInstance(algorithm);
SecretKey key = f.generateSecret(spec);
bytes = key.getEncoded();
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
return bytes;
}
public byte[] generateSalt() {
// VERY important to use SecureRandom instead of just Random
byte[] salt = null;
try {
SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
// Generate a 8 byte (64 bit) salt as recommended by RSA PKCS5
salt = new byte[8];
random.nextBytes(salt);
} catch (NoSuchAlgorithmException e) {
// should never happen
log.error(e.getMessage(), e);
}
return salt;
}
}

332
commafeed-server/src/main/java/com/commafeed/backend/service/UserService.java
View File

@@ -1,166 +1,166 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.dao.UserRoleDAO;
import com.commafeed.backend.dao.UserSettingsDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.internal.PostLoginActivities;
import com.google.common.base.Preconditions;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class UserService {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UserDAO userDAO;
private final UserRoleDAO userRoleDAO;
private final UserSettingsDAO userSettingsDAO;
private final PasswordEncryptionService encryptionService;
private final CommaFeedConfiguration config;
private final PostLoginActivities postLoginActivities;
/**
* try to log in with given credentials
*/
public Optional<User> login(String nameOrEmail, String password) {
if (nameOrEmail == null || password == null) {
return Optional.empty();
}
User user = userDAO.findByName(nameOrEmail);
if (user == null) {
user = userDAO.findByEmail(nameOrEmail);
}
if (user != null && !user.isDisabled()) {
boolean authenticated = encryptionService.authenticate(password, user.getPassword(), user.getSalt());
if (authenticated) {
performPostLoginActivities(user);
return Optional.of(user);
}
}
return Optional.empty();
}
/**
* try to log in with given api key
*/
public Optional<User> login(String apiKey) {
if (apiKey == null) {
return Optional.empty();
}
User user = userDAO.findByApiKey(apiKey);
if (user != null && !user.isDisabled()) {
performPostLoginActivities(user);
return Optional.of(user);
}
return Optional.empty();
}
/**
* try to log in with given fever api key
*/
public Optional<User> login(long userId, String feverApiKey) {
if (feverApiKey == null) {
return Optional.empty();
}
User user = userDAO.findById(userId);
if (user == null || user.isDisabled() || user.getApiKey() == null) {
return Optional.empty();
}
String computedFeverApiKey = Digests.md5Hex(user.getName() + ":" + user.getApiKey());
if (!computedFeverApiKey.equals(feverApiKey)) {
return Optional.empty();
}
performPostLoginActivities(user);
return Optional.of(user);
}
/**
* should triggers after successful login
*/
public void performPostLoginActivities(User user) {
postLoginActivities.executeFor(user);
}
public User register(String name, String password, String email, Collection<Role> roles) {
return register(name, password, email, roles, false);
}
public User register(String name, String password, String email, Collection<Role> roles, boolean forceRegistration) {
if (!forceRegistration) {
Preconditions.checkState(config.users().allowRegistrations(), "Registrations are closed on this CommaFeed instance");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");
if (StringUtils.isNotBlank(email)) {
Preconditions.checkArgument(userDAO.findByEmail(email) == null, "Email already taken");
}
User user = new User();
byte[] salt = encryptionService.generateSalt();
user.setName(name);
user.setEmail(email);
user.setCreated(Instant.now());
user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.saveOrUpdate(user);
for (Role role : roles) {
userRoleDAO.saveOrUpdate(new UserRole(user, role));
}
return user;
}
public void createAdminUser() {
register(CommaFeedApplication.USERNAME_ADMIN, "admin", "admin@commafeed.com", Arrays.asList(Role.ADMIN, Role.USER), true);
}
public void createDemoUser() {
register(CommaFeedApplication.USERNAME_DEMO, "demo", "demo@commafeed.com", Collections.singletonList(Role.USER), true);
}
public void unregister(User user) {
userSettingsDAO.delete(userSettingsDAO.findByUser(user));
userRoleDAO.delete(userRoleDAO.findAll(user));
feedSubscriptionDAO.delete(feedSubscriptionDAO.findAll(user));
feedCategoryDAO.delete(feedCategoryDAO.findAll(user));
userDAO.delete(user);
}
public String generateApiKey(User user) {
byte[] key = encryptionService.getEncryptedPassword(UUID.randomUUID().toString(), user.getSalt());
return Digests.sha1Hex(key);
}
public Set<Role> getRoles(User user) {
return userRoleDAO.findRoles(user);
}
}
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.dao.UserRoleDAO;
import com.commafeed.backend.dao.UserSettingsDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.internal.PostLoginActivities;
import com.google.common.base.Preconditions;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class UserService {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UserDAO userDAO;
private final UserRoleDAO userRoleDAO;
private final UserSettingsDAO userSettingsDAO;
private final PasswordEncryptionService encryptionService;
private final CommaFeedConfiguration config;
private final PostLoginActivities postLoginActivities;
/**
* try to log in with given credentials
*/
public Optional<User> login(String nameOrEmail, String password) {
if (nameOrEmail == null || password == null) {
return Optional.empty();
}
User user = userDAO.findByName(nameOrEmail);
if (user == null) {
user = userDAO.findByEmail(nameOrEmail);
}
if (user != null && !user.isDisabled()) {
boolean authenticated = encryptionService.authenticate(password, user.getPassword(), user.getSalt());
if (authenticated) {
performPostLoginActivities(user);
return Optional.of(user);
}
}
return Optional.empty();
}
/**
* try to log in with given api key
*/
public Optional<User> login(String apiKey) {
if (apiKey == null) {
return Optional.empty();
}
User user = userDAO.findByApiKey(apiKey);
if (user != null && !user.isDisabled()) {
performPostLoginActivities(user);
return Optional.of(user);
}
return Optional.empty();
}
/**
* try to log in with given fever api key
*/
public Optional<User> login(long userId, String feverApiKey) {
if (feverApiKey == null) {
return Optional.empty();
}
User user = userDAO.findById(userId);
if (user == null || user.isDisabled() || user.getApiKey() == null) {
return Optional.empty();
}
String computedFeverApiKey = Digests.md5Hex(user.getName() + ":" + user.getApiKey());
if (!computedFeverApiKey.equals(feverApiKey)) {
return Optional.empty();
}
performPostLoginActivities(user);
return Optional.of(user);
}
/**
* should triggers after successful login
*/
public void performPostLoginActivities(User user) {
postLoginActivities.executeFor(user);
}
public User register(String name, String password, String email, Collection<Role> roles) {
return register(name, password, email, roles, false);
}
public User register(String name, String password, String email, Collection<Role> roles, boolean forceRegistration) {
if (!forceRegistration) {
Preconditions.checkState(config.users().allowRegistrations(), "Registrations are closed on this CommaFeed instance");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");
if (StringUtils.isNotBlank(email)) {
Preconditions.checkArgument(userDAO.findByEmail(email) == null, "Email already taken");
}
User user = new User();
byte[] salt = encryptionService.generateSalt();
user.setName(name);
user.setEmail(email);
user.setCreated(Instant.now());
user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.saveOrUpdate(user);
for (Role role : roles) {
userRoleDAO.saveOrUpdate(new UserRole(user, role));
}
return user;
}
public void createAdminUser() {
register(CommaFeedApplication.USERNAME_ADMIN, "admin", "admin@commafeed.com", Arrays.asList(Role.ADMIN, Role.USER), true);
}
public void createDemoUser() {
register(CommaFeedApplication.USERNAME_DEMO, "demo", "demo@commafeed.com", Collections.singletonList(Role.USER), true);
}
public void unregister(User user) {
userSettingsDAO.delete(userSettingsDAO.findByUser(user));
userRoleDAO.delete(userRoleDAO.findAll(user));
feedSubscriptionDAO.delete(feedSubscriptionDAO.findAll(user));
feedCategoryDAO.delete(feedCategoryDAO.findAll(user));
userDAO.delete(user);
}
public String generateApiKey(User user) {
byte[] key = encryptionService.getEncryptedPassword(UUID.randomUUID().toString(), user.getSalt());
return Digests.sha1Hex(key);
}
public Set<Role> getRoles(User user) {
return userRoleDAO.findRoles(user);
}
}

266
commafeed-server/src/main/java/com/commafeed/backend/service/db/DatabaseCleaningService.java
View File

@@ -1,133 +1,133 @@
package com.commafeed.backend.service.db;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryDAO.FeedCapacity;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
/**
* Contains utility methods for cleaning the database
*
*/
@Slf4j
@Singleton
public class DatabaseCleaningService {
private final int batchSize;
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final Meter entriesDeletedMeter;
public DatabaseCleaningService(CommaFeedConfiguration config, UnitOfWork unitOfWork, FeedDAO feedDAO, FeedEntryDAO feedEntryDAO,
FeedEntryContentDAO feedEntryContentDAO, FeedEntryStatusDAO feedEntryStatusDAO, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.feedEntryDAO = feedEntryDAO;
this.feedEntryContentDAO = feedEntryContentDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.batchSize = config.database().cleanup().batchSize();
this.entriesDeletedMeter = metrics.meter(MetricRegistry.name(getClass(), "entriesDeleted"));
}
public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
int deleted;
long entriesTotal = 0;
do {
List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) {
long entriesDeleted;
do {
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesDeletedMeter.mark(entriesDeleted);
entriesTotal += entriesDeleted;
log.debug("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
}
deleted = unitOfWork.call(() -> feedDAO.delete(feedDAO.findByIds(feeds.stream().map(AbstractModel::getId).toList())));
total += deleted;
log.debug("removed {} feeds without subscriptions", total);
} while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total);
}
public void cleanContentsWithoutEntries() {
log.info("cleaning contents without entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize));
total += deleted;
log.debug("removed {} contents without entries", total);
} while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total);
}
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
log.info("cleaning entries exceeding feed capacity");
long total = 0;
while (true) {
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize));
if (feeds.isEmpty()) {
break;
}
for (final FeedCapacity feed : feeds) {
long remaining = feed.getCapacity() - maxFeedCapacity;
do {
final long rem = remaining;
int deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(batchSize, rem)));
entriesDeletedMeter.mark(deleted);
total += deleted;
remaining -= deleted;
log.debug("removed {} entries for feeds exceeding capacity", total);
} while (remaining > 0);
}
}
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
}
public void cleanEntriesOlderThan(final Instant olderThan) {
log.info("cleaning old entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize));
entriesDeletedMeter.mark(deleted);
total += deleted;
log.debug("removed {} old entries", total);
} while (deleted != 0);
log.info("cleanup done: {} old entries deleted", total);
}
public void cleanStatusesOlderThan(final Instant olderThan) {
log.info("cleaning old read statuses");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
total += deleted;
log.debug("removed {} old read statuses", total);
} while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
}
}
package com.commafeed.backend.service.db;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryDAO.FeedCapacity;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
/**
* Contains utility methods for cleaning the database
*
*/
@Slf4j
@Singleton
public class DatabaseCleaningService {
private final int batchSize;
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final Meter entriesDeletedMeter;
public DatabaseCleaningService(CommaFeedConfiguration config, UnitOfWork unitOfWork, FeedDAO feedDAO, FeedEntryDAO feedEntryDAO,
FeedEntryContentDAO feedEntryContentDAO, FeedEntryStatusDAO feedEntryStatusDAO, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.feedEntryDAO = feedEntryDAO;
this.feedEntryContentDAO = feedEntryContentDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.batchSize = config.database().cleanup().batchSize();
this.entriesDeletedMeter = metrics.meter(MetricRegistry.name(getClass(), "entriesDeleted"));
}
public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
int deleted;
long entriesTotal = 0;
do {
List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) {
long entriesDeleted;
do {
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesDeletedMeter.mark(entriesDeleted);
entriesTotal += entriesDeleted;
log.debug("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
}
deleted = unitOfWork.call(() -> feedDAO.delete(feedDAO.findByIds(feeds.stream().map(AbstractModel::getId).toList())));
total += deleted;
log.debug("removed {} feeds without subscriptions", total);
} while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total);
}
public void cleanContentsWithoutEntries() {
log.info("cleaning contents without entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize));
total += deleted;
log.debug("removed {} contents without entries", total);
} while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total);
}
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
log.info("cleaning entries exceeding feed capacity");
long total = 0;
while (true) {
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize));
if (feeds.isEmpty()) {
break;
}
for (final FeedCapacity feed : feeds) {
long remaining = feed.getCapacity() - maxFeedCapacity;
do {
final long rem = remaining;
int deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(batchSize, rem)));
entriesDeletedMeter.mark(deleted);
total += deleted;
remaining -= deleted;
log.debug("removed {} entries for feeds exceeding capacity", total);
} while (remaining > 0);
}
}
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
}
public void cleanEntriesOlderThan(final Instant olderThan) {
log.info("cleaning old entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize));
entriesDeletedMeter.mark(deleted);
total += deleted;
log.debug("removed {} old entries", total);
} while (deleted != 0);
log.info("cleanup done: {} old entries deleted", total);
}
public void cleanStatusesOlderThan(final Instant olderThan) {
log.info("cleaning old read statuses");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
total += deleted;
log.debug("removed {} old read statuses", total);
} while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
}
}

126
commafeed-server/src/main/java/com/commafeed/backend/service/db/DatabaseStartupService.java
View File

@@ -1,63 +1,63 @@
package com.commafeed.backend.service.db;
import jakarta.inject.Singleton;
import org.kohsuke.MetaInfServices;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.service.UserService;
import liquibase.database.Database;
import liquibase.database.core.PostgresDatabase;
import liquibase.structure.DatabaseObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class DatabaseStartupService {
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserService userService;
private final CommaFeedConfiguration config;
public void populateInitialData() {
long count = unitOfWork.call(userDAO::count);
if (count == 0) {
unitOfWork.run(this::initialData);
}
}
private void initialData() {
log.info("populating database with default values");
try {
userService.createAdminUser();
if (config.users().createDemoAccount()) {
userService.createDemoUser();
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
/**
* Register a postgresql database in liquibase that doesn't escape columns, so that we can use lower case columns
*/
@MetaInfServices(Database.class)
public static class LowerCaseColumnsPostgresDatabase extends PostgresDatabase {
@Override
public String escapeObjectName(String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}
@Override
public int getPriority() {
return super.getPriority() + 1;
}
}
}
package com.commafeed.backend.service.db;
import jakarta.inject.Singleton;
import org.kohsuke.MetaInfServices;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.service.UserService;
import liquibase.database.Database;
import liquibase.database.core.PostgresDatabase;
import liquibase.structure.DatabaseObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class DatabaseStartupService {
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserService userService;
private final CommaFeedConfiguration config;
public void populateInitialData() {
long count = unitOfWork.call(userDAO::count);
if (count == 0) {
unitOfWork.run(this::initialData);
}
}
private void initialData() {
log.info("populating database with default values");
try {
userService.createAdminUser();
if (config.users().createDemoAccount()) {
userService.createDemoUser();
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
/**
* Register a postgresql database in liquibase that doesn't escape columns, so that we can use lower case columns
*/
@MetaInfServices(Database.class)
public static class LowerCaseColumnsPostgresDatabase extends PostgresDatabase {
@Override
public String escapeObjectName(String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}
@Override
public int getPriority() {
return super.getPriority() + 1;
}
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/service/internal/PostLoginActivities.java
View File

@@ -1,30 +1,30 @@
package com.commafeed.backend.service.internal;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class PostLoginActivities {
private final UserDAO userDAO;
private final UnitOfWork unitOfWork;
public void executeFor(User user) {
// only update lastLogin every once in a while in order to avoid invalidating the cache every time someone logs in
Instant now = Instant.now();
Instant lastLogin = user.getLastLogin();
if (lastLogin == null || ChronoUnit.MINUTES.between(lastLogin, now) >= 30) {
user.setLastLogin(now);
unitOfWork.run(() -> userDAO.saveOrUpdate(user));
}
}
}
package com.commafeed.backend.service.internal;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class PostLoginActivities {
private final UserDAO userDAO;
private final UnitOfWork unitOfWork;
public void executeFor(User user) {
// only update lastLogin every once in a while in order to avoid invalidating the cache every time someone logs in
Instant now = Instant.now();
Instant lastLogin = user.getLastLogin();
if (lastLogin == null || ChronoUnit.MINUTES.between(lastLogin, now) >= 30) {
user.setLastLogin(now);
unitOfWork.run(() -> userDAO.saveOrUpdate(user));
}
}
}

122
commafeed-server/src/main/java/com/commafeed/backend/task/DemoAccountCleanupTask.java
View File

@@ -1,61 +1,61 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.UserService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Singleton
@Slf4j
public class DemoAccountCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserService userService;
@Override
protected void run() {
if (!config.users().createDemoAccount()) {
return;
}
log.info("recreating demo user account");
unitOfWork.run(() -> {
User demoUser = userDAO.findByName(CommaFeedApplication.USERNAME_DEMO);
if (demoUser == null) {
return;
}
userService.unregister(demoUser);
userService.createDemoUser();
});
}
@Override
protected long getInitialDelay() {
return 1;
}
@Override
protected long getPeriod() {
return getTimeUnit().convert(24, TimeUnit.HOURS);
}
@Override
protected TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.UserService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Singleton
@Slf4j
public class DemoAccountCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserService userService;
@Override
protected void run() {
if (!config.users().createDemoAccount()) {
return;
}
log.info("recreating demo user account");
unitOfWork.run(() -> {
User demoUser = userDAO.findByName(CommaFeedApplication.USERNAME_DEMO);
if (demoUser == null) {
return;
}
userService.unregister(demoUser);
userService.createDemoUser();
});
}
@Override
protected long getInitialDelay() {
return 1;
}
@Override
protected long getPeriod() {
return getTimeUnit().convert(24, TimeUnit.HOURS);
}
@Override
protected TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

84
commafeed-server/src/main/java/com/commafeed/backend/task/EntriesExceedingFeedCapacityCleanupTask.java
View File

@@ -1,42 +1,42 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
}
}
@Override
public long getInitialDelay() {
return 10;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class EntriesExceedingFeedCapacityCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
cleaner.cleanEntriesForFeedsExceedingCapacity(maxFeedCapacity);
}
}
@Override
public long getInitialDelay() {
return 10;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

90
commafeed-server/src/main/java/com/commafeed/backend/task/OldEntriesCleanupTask.java
View File

@@ -1,45 +1,45 @@
package com.commafeed.backend.task;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OldEntriesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
cleaner.cleanEntriesOlderThan(threshold);
}
}
@Override
public long getInitialDelay() {
return 5;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}
package com.commafeed.backend.task;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OldEntriesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
cleaner.cleanEntriesOlderThan(threshold);
}
}
@Override
public long getInitialDelay() {
return 5;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

86
commafeed-server/src/main/java/com/commafeed/backend/task/OldStatusesCleanupTask.java
View File

@@ -1,43 +1,43 @@
package com.commafeed.backend.task;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OldStatusesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
Instant threshold = config.database().cleanup().statusesInstantThreshold();
if (threshold != null) {
cleaner.cleanStatusesOlderThan(threshold);
}
}
@Override
public long getInitialDelay() {
return 15;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}
package com.commafeed.backend.task;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OldStatusesCleanupTask extends ScheduledTask {
private final CommaFeedConfiguration config;
private final DatabaseCleaningService cleaner;
@Override
public void run() {
Instant threshold = config.database().cleanup().statusesInstantThreshold();
if (threshold != null) {
cleaner.cleanStatusesOlderThan(threshold);
}
}
@Override
public long getInitialDelay() {
return 15;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

74
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedContentsCleanupTask.java
View File

@@ -1,37 +1,37 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OrphanedContentsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanContentsWithoutEntries();
}
@Override
public long getInitialDelay() {
return 25;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OrphanedContentsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanContentsWithoutEntries();
}
@Override
public long getInitialDelay() {
return 25;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

74
commafeed-server/src/main/java/com/commafeed/backend/task/OrphanedFeedsCleanupTask.java
View File

@@ -1,37 +1,37 @@
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OrphanedFeedsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanFeedsWithoutSubscriptions();
}
@Override
public long getInitialDelay() {
return 20;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}
package com.commafeed.backend.task;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.service.db.DatabaseCleaningService;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class OrphanedFeedsCleanupTask extends ScheduledTask {
private final DatabaseCleaningService cleaner;
@Override
public void run() {
cleaner.cleanFeedsWithoutSubscriptions();
}
@Override
public long getInitialDelay() {
return 20;
}
@Override
public long getPeriod() {
return 60;
}
@Override
public TimeUnit getTimeUnit() {
return TimeUnit.MINUTES;
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/task/ScheduledTask.java
View File

@@ -1,30 +1,30 @@
package com.commafeed.backend.task;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class ScheduledTask {
protected abstract void run();
protected abstract long getInitialDelay();
protected abstract long getPeriod();
protected abstract TimeUnit getTimeUnit();
public void register(ScheduledExecutorService executor) {
Runnable runnable = () -> {
try {
ScheduledTask.this.run();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
};
log.info("registering task {} for execution every {} {}, starting in {} {}", getClass().getSimpleName(), getPeriod(), getTimeUnit(),
getInitialDelay(), getTimeUnit());
executor.scheduleWithFixedDelay(runnable, getInitialDelay(), getPeriod(), getTimeUnit());
}
}
package com.commafeed.backend.task;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class ScheduledTask {
protected abstract void run();
protected abstract long getInitialDelay();
protected abstract long getPeriod();
protected abstract TimeUnit getTimeUnit();
public void register(ScheduledExecutorService executor) {
Runnable runnable = () -> {
try {
ScheduledTask.this.run();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
};
log.info("registering task {} for execution every {} {}, starting in {} {}", getClass().getSimpleName(), getPeriod(), getTimeUnit(),
getInitialDelay(), getTimeUnit());
executor.scheduleWithFixedDelay(runnable, getInitialDelay(), getPeriod(), getTimeUnit());
}
}

58
commafeed-server/src/main/java/com/commafeed/backend/task/TaskScheduler.java
View File

@@ -1,29 +1,29 @@
package com.commafeed.backend.task;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import jakarta.inject.Singleton;
import io.quarkus.arc.All;
@Singleton
public class TaskScheduler {
private final List<ScheduledTask> tasks;
private final ScheduledExecutorService executor;
public TaskScheduler(@All List<ScheduledTask> tasks) {
this.tasks = tasks;
this.executor = Executors.newScheduledThreadPool(tasks.size());
}
public void start() {
tasks.forEach(task -> task.register(executor));
}
public void stop() {
executor.shutdownNow();
}
}
package com.commafeed.backend.task;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import jakarta.inject.Singleton;
import io.quarkus.arc.All;
@Singleton
public class TaskScheduler {
private final List<ScheduledTask> tasks;
private final ScheduledExecutorService executor;
public TaskScheduler(@All List<ScheduledTask> tasks) {
this.tasks = tasks;
this.executor = Executors.newScheduledThreadPool(tasks.size());
}
public void start() {
tasks.forEach(task -> task.register(executor));
}
public void stop() {
executor.shutdownNow();
}
}

20
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/FeedURLProvider.java
View File

@@ -1,10 +1,10 @@
package com.commafeed.backend.urlprovider;
/**
* Tries to find a feed url given the url and page content
*/
public interface FeedURLProvider {
String get(String url, String urlContent);
}
package com.commafeed.backend.urlprovider;
/**
* Tries to find a feed url given the url and page content
*/
public interface FeedURLProvider {
String get(String url, String urlContent);
}

62
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/InPageReferenceFeedURLProvider.java
View File

@@ -1,31 +1,31 @@
package com.commafeed.backend.urlprovider;
import jakarta.inject.Singleton;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
@Singleton
public class InPageReferenceFeedURLProvider implements FeedURLProvider {
@Override
public String get(String url, String urlContent) {
String foundUrl = null;
Document doc = Jsoup.parse(urlContent, url);
String root = doc.children().get(0).tagName();
if ("html".equals(root)) {
Elements atom = doc.select("link[type=application/atom+xml]");
Elements rss = doc.select("link[type=application/rss+xml]");
if (!atom.isEmpty()) {
foundUrl = atom.get(0).attr("abs:href");
} else if (!rss.isEmpty()) {
foundUrl = rss.get(0).attr("abs:href");
}
}
return foundUrl;
}
}
package com.commafeed.backend.urlprovider;
import jakarta.inject.Singleton;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
@Singleton
public class InPageReferenceFeedURLProvider implements FeedURLProvider {
@Override
public String get(String url, String urlContent) {
String foundUrl = null;
Document doc = Jsoup.parse(urlContent, url);
String root = doc.children().get(0).tagName();
if ("html".equals(root)) {
Elements atom = doc.select("link[type=application/atom+xml]");
Elements rss = doc.select("link[type=application/rss+xml]");
if (!atom.isEmpty()) {
foundUrl = atom.get(0).attr("abs:href");
} else if (!rss.isEmpty()) {
foundUrl = rss.get(0).attr("abs:href");
}
}
return foundUrl;
}
}

56
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/YoutubeFeedURLProvider.java
View File

@@ -1,28 +1,28 @@
package com.commafeed.backend.urlprovider;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
/**
* Workaround for Youtube channels
*
* converts the channel URL https://www.youtube.com/channel/CHANNEL_ID to the valid feed URL
* https://www.youtube.com/feeds/videos.xml?channel_id=CHANNEL_ID
*/
@Singleton
public class YoutubeFeedURLProvider implements FeedURLProvider {
private static final String PREFIX = "https://www.youtube.com/channel/";
private static final String REPLACEMENT_PREFIX = "https://www.youtube.com/feeds/videos.xml?channel_id=";
@Override
public String get(String url, String urlContent) {
if (!StringUtils.startsWithIgnoreCase(url, PREFIX)) {
return null;
}
return REPLACEMENT_PREFIX + url.substring(PREFIX.length());
}
}
package com.commafeed.backend.urlprovider;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
/**
* Workaround for Youtube channels
*
* converts the channel URL https://www.youtube.com/channel/CHANNEL_ID to the valid feed URL
* https://www.youtube.com/feeds/videos.xml?channel_id=CHANNEL_ID
*/
@Singleton
public class YoutubeFeedURLProvider implements FeedURLProvider {
private static final String PREFIX = "https://www.youtube.com/channel/";
private static final String REPLACEMENT_PREFIX = "https://www.youtube.com/feeds/videos.xml?channel_id=";
@Override
public String get(String url, String urlContent) {
if (!StringUtils.startsWithIgnoreCase(url, PREFIX)) {
return null;
}
return REPLACEMENT_PREFIX + url.substring(PREFIX.length());
}
}

80
commafeed-server/src/main/java/com/commafeed/frontend/model/Category.java
View File

@@ -1,41 +1,41 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Entry details")
@Data
@RegisterForReflection
public class Category implements Serializable {
@Schema(description = "category id", requiredMode = RequiredMode.REQUIRED)
private String id;
@Schema(description = "parent category id")
private String parentId;
@Schema(description = "parent category name")
private String parentName;
@Schema(description = "category id", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "category children categories", requiredMode = RequiredMode.REQUIRED)
private List<Category> children = new ArrayList<>();
@Schema(description = "category feeds", requiredMode = RequiredMode.REQUIRED)
private List<Subscription> feeds = new ArrayList<>();
@Schema(description = "whether the category is expanded or collapsed", requiredMode = RequiredMode.REQUIRED)
private boolean expanded;
@Schema(description = "position of the category in the list", requiredMode = RequiredMode.REQUIRED)
private int position;
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Entry details")
@Data
@RegisterForReflection
public class Category implements Serializable {
@Schema(description = "category id", requiredMode = RequiredMode.REQUIRED)
private String id;
@Schema(description = "parent category id")
private String parentId;
@Schema(description = "parent category name")
private String parentName;
@Schema(description = "category id", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "category children categories", requiredMode = RequiredMode.REQUIRED)
private List<Category> children = new ArrayList<>();
@Schema(description = "category feeds", requiredMode = RequiredMode.REQUIRED)
private List<Subscription> feeds = new ArrayList<>();
@Schema(description = "whether the category is expanded or collapsed", requiredMode = RequiredMode.REQUIRED)
private boolean expanded;
@Schema(description = "position of the category in the list", requiredMode = RequiredMode.REQUIRED)
private int position;
}

100
commafeed-server/src/main/java/com/commafeed/frontend/model/Entries.java
View File

@@ -1,50 +1,50 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "List of entries with some metadata")
@Data
@RegisterForReflection
public class Entries implements Serializable {
@Schema(description = "name of the feed or the category requested", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "error or warning message")
private String message;
@Schema(description = "times the server tried to refresh the feed and failed", requiredMode = RequiredMode.REQUIRED)
private int errorCount;
@Schema(description = "URL of the website, extracted from the feed, only filled if querying for feed entries, not category entries")
private String feedLink;
@Schema(description = "list generation timestamp", requiredMode = RequiredMode.REQUIRED)
private long timestamp;
@Schema(description = "if the query has more elements", requiredMode = RequiredMode.REQUIRED)
private boolean hasMore;
@Schema(description = "the requested offset")
private int offset;
@Schema(description = "the requested limit")
private int limit;
@Schema(description = "list of entries", requiredMode = RequiredMode.REQUIRED)
private List<Entry> entries = new ArrayList<>();
@Schema(
description = "if true, the unread flag was ignored in the request, all entries are returned regardless of their read status",
requiredMode = RequiredMode.REQUIRED)
private boolean ignoredReadStatus;
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "List of entries with some metadata")
@Data
@RegisterForReflection
public class Entries implements Serializable {
@Schema(description = "name of the feed or the category requested", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "error or warning message")
private String message;
@Schema(description = "times the server tried to refresh the feed and failed", requiredMode = RequiredMode.REQUIRED)
private int errorCount;
@Schema(description = "URL of the website, extracted from the feed, only filled if querying for feed entries, not category entries")
private String feedLink;
@Schema(description = "list generation timestamp", requiredMode = RequiredMode.REQUIRED)
private long timestamp;
@Schema(description = "if the query has more elements", requiredMode = RequiredMode.REQUIRED)
private boolean hasMore;
@Schema(description = "the requested offset")
private int offset;
@Schema(description = "the requested limit")
private int limit;
@Schema(description = "list of entries", requiredMode = RequiredMode.REQUIRED)
private List<Entry> entries = new ArrayList<>();
@Schema(
description = "if true, the unread flag was ignored in the request, all entries are returned regardless of their read status",
requiredMode = RequiredMode.REQUIRED)
private boolean ignoredReadStatus;
}

348
commafeed-server/src/main/java/com/commafeed/frontend/model/Entry.java
View File

@@ -1,174 +1,174 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEnclosureImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndEntryImpl;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Entry details")
@Data
@RegisterForReflection
public class Entry implements Serializable {
@Schema(description = "entry id", requiredMode = RequiredMode.REQUIRED)
private String id;
@Schema(description = "entry guid", requiredMode = RequiredMode.REQUIRED)
private String guid;
@Schema(description = "entry title", requiredMode = RequiredMode.REQUIRED)
private String title;
@Schema(description = "entry content", requiredMode = RequiredMode.REQUIRED)
private String content;
@Schema(description = "comma-separated list of categories")
private String categories;
@Schema(description = "whether entry content and title are rtl", requiredMode = RequiredMode.REQUIRED)
private boolean rtl;
@Schema(description = "entry author")
private String author;
@Schema(description = "entry enclosure url, if any")
private String enclosureUrl;
@Schema(description = "entry enclosure mime type, if any")
private String enclosureType;
@Schema(description = "entry media description, if any")
private String mediaDescription;
@Schema(description = "entry media thumbnail url, if any")
private String mediaThumbnailUrl;
@Schema(description = "entry media thumbnail width, if any")
private Integer mediaThumbnailWidth;
@Schema(description = "entry media thumbnail height, if any")
private Integer mediaThumbnailHeight;
@Schema(description = "entry publication date", type = "number", requiredMode = RequiredMode.REQUIRED)
private Instant date;
@Schema(description = "entry insertion date in the database", type = "number", requiredMode = RequiredMode.REQUIRED)
private Instant insertedDate;
@Schema(description = "feed id", requiredMode = RequiredMode.REQUIRED)
private String feedId;
@Schema(description = "feed name", requiredMode = RequiredMode.REQUIRED)
private String feedName;
@Schema(description = "this entry's feed url", requiredMode = RequiredMode.REQUIRED)
private String feedUrl;
@Schema(description = "this entry's website url", requiredMode = RequiredMode.REQUIRED)
private String feedLink;
@Schema(description = "The favicon url to use for this feed", requiredMode = RequiredMode.REQUIRED)
private String iconUrl;
@Schema(description = "entry url", requiredMode = RequiredMode.REQUIRED)
private String url;
@Schema(description = "read status", requiredMode = RequiredMode.REQUIRED)
private boolean read;
@Schema(description = "starred status", requiredMode = RequiredMode.REQUIRED)
private boolean starred;
@Schema(description = "whether the entry is still markable (old entry statuses are discarded)", requiredMode = RequiredMode.REQUIRED)
private boolean markable;
@Schema(description = "tags", requiredMode = RequiredMode.REQUIRED)
private List<String> tags;
public static Entry build(FeedEntryStatus status, boolean proxyImages) {
Entry entry = new Entry();
FeedEntry feedEntry = status.getEntry();
FeedSubscription sub = status.getSubscription();
FeedEntryContent content = feedEntry.getContent();
entry.setId(String.valueOf(feedEntry.getId()));
entry.setGuid(feedEntry.getGuid());
entry.setRead(status.isRead());
entry.setStarred(status.isStarred());
entry.setMarkable(status.isMarkable());
entry.setDate(feedEntry.getPublished());
entry.setInsertedDate(feedEntry.getInserted());
entry.setUrl(feedEntry.getUrl());
entry.setFeedName(sub.getTitle());
entry.setFeedId(String.valueOf(sub.getId()));
entry.setFeedUrl(sub.getFeed().getUrl());
entry.setFeedLink(sub.getFeed().getLink());
entry.setIconUrl(FeedUtils.getFaviconUrl(sub));
entry.setTags(status.getTags().stream().map(FeedEntryTag::getName).toList());
if (content != null) {
entry.setRtl(content.isRTL());
entry.setTitle(content.getTitle());
entry.setContent(proxyImages ? FeedUtils.proxyImages(content.getContent()) : content.getContent());
entry.setAuthor(content.getAuthor());
entry.setEnclosureType(content.getEnclosureType());
entry.setEnclosureUrl(proxyImages && StringUtils.contains(content.getEnclosureType(), "image")
? FeedUtils.proxyImage(content.getEnclosureUrl())
: content.getEnclosureUrl());
entry.setMediaDescription(content.getMediaDescription());
entry.setMediaThumbnailUrl(proxyImages ? FeedUtils.proxyImage(content.getMediaThumbnailUrl()) : content.getMediaThumbnailUrl());
entry.setMediaThumbnailWidth(content.getMediaThumbnailWidth());
entry.setMediaThumbnailHeight(content.getMediaThumbnailHeight());
entry.setCategories(content.getCategories());
}
return entry;
}
public SyndEntry asRss() {
SyndEntry entry = new SyndEntryImpl();
entry.setUri(getGuid());
entry.setTitle(getTitle());
entry.setAuthor(getAuthor());
SyndContentImpl content = new SyndContentImpl();
content.setValue(getContent());
entry.setContents(Collections.singletonList(content));
if (getEnclosureUrl() != null) {
SyndEnclosureImpl enclosure = new SyndEnclosureImpl();
enclosure.setType(getEnclosureType());
enclosure.setUrl(getEnclosureUrl());
entry.setEnclosures(Collections.singletonList(enclosure));
}
entry.setLink(getUrl());
entry.setPublishedDate(getDate() == null ? null : Date.from(getDate()));
return entry;
}
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.FeedSubscription;
import com.rometools.rome.feed.synd.SyndContentImpl;
import com.rometools.rome.feed.synd.SyndEnclosureImpl;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndEntryImpl;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Entry details")
@Data
@RegisterForReflection
public class Entry implements Serializable {
@Schema(description = "entry id", requiredMode = RequiredMode.REQUIRED)
private String id;
@Schema(description = "entry guid", requiredMode = RequiredMode.REQUIRED)
private String guid;
@Schema(description = "entry title", requiredMode = RequiredMode.REQUIRED)
private String title;
@Schema(description = "entry content", requiredMode = RequiredMode.REQUIRED)
private String content;
@Schema(description = "comma-separated list of categories")
private String categories;
@Schema(description = "whether entry content and title are rtl", requiredMode = RequiredMode.REQUIRED)
private boolean rtl;
@Schema(description = "entry author")
private String author;
@Schema(description = "entry enclosure url, if any")
private String enclosureUrl;
@Schema(description = "entry enclosure mime type, if any")
private String enclosureType;
@Schema(description = "entry media description, if any")
private String mediaDescription;
@Schema(description = "entry media thumbnail url, if any")
private String mediaThumbnailUrl;
@Schema(description = "entry media thumbnail width, if any")
private Integer mediaThumbnailWidth;
@Schema(description = "entry media thumbnail height, if any")
private Integer mediaThumbnailHeight;
@Schema(description = "entry publication date", type = "number", requiredMode = RequiredMode.REQUIRED)
private Instant date;
@Schema(description = "entry insertion date in the database", type = "number", requiredMode = RequiredMode.REQUIRED)
private Instant insertedDate;
@Schema(description = "feed id", requiredMode = RequiredMode.REQUIRED)
private String feedId;
@Schema(description = "feed name", requiredMode = RequiredMode.REQUIRED)
private String feedName;
@Schema(description = "this entry's feed url", requiredMode = RequiredMode.REQUIRED)
private String feedUrl;
@Schema(description = "this entry's website url", requiredMode = RequiredMode.REQUIRED)
private String feedLink;
@Schema(description = "The favicon url to use for this feed", requiredMode = RequiredMode.REQUIRED)
private String iconUrl;
@Schema(description = "entry url", requiredMode = RequiredMode.REQUIRED)
private String url;
@Schema(description = "read status", requiredMode = RequiredMode.REQUIRED)
private boolean read;
@Schema(description = "starred status", requiredMode = RequiredMode.REQUIRED)
private boolean starred;
@Schema(description = "whether the entry is still markable (old entry statuses are discarded)", requiredMode = RequiredMode.REQUIRED)
private boolean markable;
@Schema(description = "tags", requiredMode = RequiredMode.REQUIRED)
private List<String> tags;
public static Entry build(FeedEntryStatus status, boolean proxyImages) {
Entry entry = new Entry();
FeedEntry feedEntry = status.getEntry();
FeedSubscription sub = status.getSubscription();
FeedEntryContent content = feedEntry.getContent();
entry.setId(String.valueOf(feedEntry.getId()));
entry.setGuid(feedEntry.getGuid());
entry.setRead(status.isRead());
entry.setStarred(status.isStarred());
entry.setMarkable(status.isMarkable());
entry.setDate(feedEntry.getPublished());
entry.setInsertedDate(feedEntry.getInserted());
entry.setUrl(feedEntry.getUrl());
entry.setFeedName(sub.getTitle());
entry.setFeedId(String.valueOf(sub.getId()));
entry.setFeedUrl(sub.getFeed().getUrl());
entry.setFeedLink(sub.getFeed().getLink());
entry.setIconUrl(FeedUtils.getFaviconUrl(sub));
entry.setTags(status.getTags().stream().map(FeedEntryTag::getName).toList());
if (content != null) {
entry.setRtl(content.isRTL());
entry.setTitle(content.getTitle());
entry.setContent(proxyImages ? FeedUtils.proxyImages(content.getContent()) : content.getContent());
entry.setAuthor(content.getAuthor());
entry.setEnclosureType(content.getEnclosureType());
entry.setEnclosureUrl(proxyImages && StringUtils.contains(content.getEnclosureType(), "image")
? FeedUtils.proxyImage(content.getEnclosureUrl())
: content.getEnclosureUrl());
entry.setMediaDescription(content.getMediaDescription());
entry.setMediaThumbnailUrl(proxyImages ? FeedUtils.proxyImage(content.getMediaThumbnailUrl()) : content.getMediaThumbnailUrl());
entry.setMediaThumbnailWidth(content.getMediaThumbnailWidth());
entry.setMediaThumbnailHeight(content.getMediaThumbnailHeight());
entry.setCategories(content.getCategories());
}
return entry;
}
public SyndEntry asRss() {
SyndEntry entry = new SyndEntryImpl();
entry.setUri(getGuid());
entry.setTitle(getTitle());
entry.setAuthor(getAuthor());
SyndContentImpl content = new SyndContentImpl();
content.setValue(getContent());
entry.setContents(Collections.singletonList(content));
if (getEnclosureUrl() != null) {
SyndEnclosureImpl enclosure = new SyndEnclosureImpl();
enclosure.setType(getEnclosureType());
enclosure.setUrl(getEnclosureUrl());
entry.setEnclosures(Collections.singletonList(enclosure));
}
entry.setLink(getUrl());
entry.setPublishedDate(getDate() == null ? null : Date.from(getDate()));
return entry;
}
}

44
commafeed-server/src/main/java/com/commafeed/frontend/model/FeedInfo.java
View File

@@ -1,22 +1,22 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Feed details")
@Data
@RegisterForReflection
public class FeedInfo implements Serializable {
@Schema(description = "url", requiredMode = RequiredMode.REQUIRED)
private String url;
@Schema(description = "title", requiredMode = RequiredMode.REQUIRED)
private String title;
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Feed details")
@Data
@RegisterForReflection
public class FeedInfo implements Serializable {
@Schema(description = "url", requiredMode = RequiredMode.REQUIRED)
private String url;
@Schema(description = "title", requiredMode = RequiredMode.REQUIRED)
private String title;
}

98
commafeed-server/src/main/java/com/commafeed/frontend/model/ServerInfo.java
View File

@@ -1,49 +1,49 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Server infos")
@Data
@RegisterForReflection
public class ServerInfo implements Serializable {
@Schema
private String announcement;
@Schema(requiredMode = RequiredMode.REQUIRED)
private String version;
@Schema(requiredMode = RequiredMode.REQUIRED)
private String gitCommit;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean allowRegistrations;
@Schema
private String googleAnalyticsCode;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean smtpEnabled;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean demoAccountEnabled;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean websocketEnabled;
@Schema(requiredMode = RequiredMode.REQUIRED)
private long websocketPingInterval;
@Schema(requiredMode = RequiredMode.REQUIRED)
private long treeReloadInterval;
@Schema(requiredMode = RequiredMode.REQUIRED)
private long forceRefreshCooldownDuration;
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Server infos")
@Data
@RegisterForReflection
public class ServerInfo implements Serializable {
@Schema
private String announcement;
@Schema(requiredMode = RequiredMode.REQUIRED)
private String version;
@Schema(requiredMode = RequiredMode.REQUIRED)
private String gitCommit;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean allowRegistrations;
@Schema
private String googleAnalyticsCode;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean smtpEnabled;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean demoAccountEnabled;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean websocketEnabled;
@Schema(requiredMode = RequiredMode.REQUIRED)
private long websocketPingInterval;
@Schema(requiredMode = RequiredMode.REQUIRED)
private long treeReloadInterval;
@Schema(requiredMode = RequiredMode.REQUIRED)
private long forceRefreshCooldownDuration;
}

224
commafeed-server/src/main/java/com/commafeed/frontend/model/Settings.java
View File

@@ -1,112 +1,112 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "User settings")
@Data
@RegisterForReflection
public class Settings implements Serializable {
@Schema(description = "user's preferred language, english if none", requiredMode = RequiredMode.REQUIRED)
private String language;
@Schema(
description = "user reads all entries or unread entries only",
allowableValues = "all,unread",
requiredMode = RequiredMode.REQUIRED)
private String readingMode;
@Schema(
description = "user reads entries in ascending or descending order",
allowableValues = "asc,desc",
requiredMode = RequiredMode.REQUIRED)
private String readingOrder;
@Schema(description = "user wants category and feeds with no unread entries shown", requiredMode = RequiredMode.REQUIRED)
private boolean showRead;
@Schema(description = "In expanded view, scroll through entries mark them as read", requiredMode = RequiredMode.REQUIRED)
private boolean scrollMarks;
@Schema(description = "user's custom css for the website")
private String customCss;
@Schema(description = "user's custom js for the website")
private String customJs;
@Schema(description = "user's preferred scroll speed when navigating between entries", requiredMode = RequiredMode.REQUIRED)
private int scrollSpeed;
@Schema(
description = "whether to scroll to the selected entry",
allowableValues = "always,never,if_needed",
requiredMode = RequiredMode.REQUIRED)
private String scrollMode;
@Schema(description = "number of entries to keep above the selected entry when scrolling", requiredMode = RequiredMode.REQUIRED)
private int entriesToKeepOnTopWhenScrolling;
@Schema(
description = "whether to show the star icon in the header of entries",
allowableValues = "always,never,on_desktop,on_mobile",
requiredMode = RequiredMode.REQUIRED)
private String starIconDisplayMode;
@Schema(
description = "whether to show the external link icon in the header of entries",
allowableValues = "always,never,on_desktop,on_mobile",
requiredMode = RequiredMode.REQUIRED)
private String externalLinkIconDisplayMode;
@Schema(description = "ask for confirmation when marking all entries as read", requiredMode = RequiredMode.REQUIRED)
private boolean markAllAsReadConfirmation;
@Schema(description = "show commafeed's own context menu on right click", requiredMode = RequiredMode.REQUIRED)
private boolean customContextMenu;
@Schema(description = "on mobile, show action buttons at the bottom of the screen", requiredMode = RequiredMode.REQUIRED)
private boolean mobileFooter;
@Schema(description = "show unread count in the title", requiredMode = RequiredMode.REQUIRED)
private boolean unreadCountTitle;
@Schema(description = "show unread count in the favicon", requiredMode = RequiredMode.REQUIRED)
private boolean unreadCountFavicon;
@Schema(description = "sharing settings", requiredMode = RequiredMode.REQUIRED)
private SharingSettings sharingSettings = new SharingSettings();
@Schema(description = "User sharing settings")
@Data
public static class SharingSettings implements Serializable {
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean email;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean gmail;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean facebook;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean twitter;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean tumblr;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean pocket;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean instapaper;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean buffer;
}
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "User settings")
@Data
@RegisterForReflection
public class Settings implements Serializable {
@Schema(description = "user's preferred language, english if none", requiredMode = RequiredMode.REQUIRED)
private String language;
@Schema(
description = "user reads all entries or unread entries only",
allowableValues = "all,unread",
requiredMode = RequiredMode.REQUIRED)
private String readingMode;
@Schema(
description = "user reads entries in ascending or descending order",
allowableValues = "asc,desc",
requiredMode = RequiredMode.REQUIRED)
private String readingOrder;
@Schema(description = "user wants category and feeds with no unread entries shown", requiredMode = RequiredMode.REQUIRED)
private boolean showRead;
@Schema(description = "In expanded view, scroll through entries mark them as read", requiredMode = RequiredMode.REQUIRED)
private boolean scrollMarks;
@Schema(description = "user's custom css for the website")
private String customCss;
@Schema(description = "user's custom js for the website")
private String customJs;
@Schema(description = "user's preferred scroll speed when navigating between entries", requiredMode = RequiredMode.REQUIRED)
private int scrollSpeed;
@Schema(
description = "whether to scroll to the selected entry",
allowableValues = "always,never,if_needed",
requiredMode = RequiredMode.REQUIRED)
private String scrollMode;
@Schema(description = "number of entries to keep above the selected entry when scrolling", requiredMode = RequiredMode.REQUIRED)
private int entriesToKeepOnTopWhenScrolling;
@Schema(
description = "whether to show the star icon in the header of entries",
allowableValues = "always,never,on_desktop,on_mobile",
requiredMode = RequiredMode.REQUIRED)
private String starIconDisplayMode;
@Schema(
description = "whether to show the external link icon in the header of entries",
allowableValues = "always,never,on_desktop,on_mobile",
requiredMode = RequiredMode.REQUIRED)
private String externalLinkIconDisplayMode;
@Schema(description = "ask for confirmation when marking all entries as read", requiredMode = RequiredMode.REQUIRED)
private boolean markAllAsReadConfirmation;
@Schema(description = "show commafeed's own context menu on right click", requiredMode = RequiredMode.REQUIRED)
private boolean customContextMenu;
@Schema(description = "on mobile, show action buttons at the bottom of the screen", requiredMode = RequiredMode.REQUIRED)
private boolean mobileFooter;
@Schema(description = "show unread count in the title", requiredMode = RequiredMode.REQUIRED)
private boolean unreadCountTitle;
@Schema(description = "show unread count in the favicon", requiredMode = RequiredMode.REQUIRED)
private boolean unreadCountFavicon;
@Schema(description = "sharing settings", requiredMode = RequiredMode.REQUIRED)
private SharingSettings sharingSettings = new SharingSettings();
@Schema(description = "User sharing settings")
@Data
public static class SharingSettings implements Serializable {
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean email;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean gmail;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean facebook;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean twitter;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean tumblr;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean pocket;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean instapaper;
@Schema(requiredMode = RequiredMode.REQUIRED)
private boolean buffer;
}
}

170
commafeed-server/src/main/java/com/commafeed/frontend/model/Subscription.java
View File

@@ -1,86 +1,86 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "User information")
@Data
@RegisterForReflection
public class Subscription implements Serializable {
@Schema(description = "subscription id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "subscription name", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "error message while fetching the feed")
private String message;
@Schema(description = "error count", requiredMode = RequiredMode.REQUIRED)
private int errorCount;
@Schema(description = "last time the feed was refreshed", type = "number")
private Instant lastRefresh;
@Schema(description = "next time the feed refresh is planned, null if refresh is already queued", type = "number")
private Instant nextRefresh;
@Schema(description = "this subscription's feed url", requiredMode = RequiredMode.REQUIRED)
private String feedUrl;
@Schema(description = "this subscription's website url", requiredMode = RequiredMode.REQUIRED)
private String feedLink;
@Schema(description = "The favicon url to use for this feed", requiredMode = RequiredMode.REQUIRED)
private String iconUrl;
@Schema(description = "unread count", requiredMode = RequiredMode.REQUIRED)
private long unread;
@Schema(description = "category id")
private String categoryId;
@Schema(description = "position of the subscription's in the list")
private int position;
@Schema(description = "date of the newest item", type = "number")
private Instant newestItemTime;
@Schema(description = "JEXL string evaluated on new entries to mark them as read if they do not match")
private String filter;
public static Subscription build(FeedSubscription subscription, UnreadCount unreadCount) {
FeedCategory category = subscription.getCategory();
Feed feed = subscription.getFeed();
Subscription sub = new Subscription();
sub.setId(subscription.getId());
sub.setName(subscription.getTitle());
sub.setPosition(subscription.getPosition());
sub.setMessage(feed.getMessage());
sub.setErrorCount(feed.getErrorCount());
sub.setFeedUrl(feed.getUrl());
sub.setFeedLink(feed.getLink());
sub.setIconUrl(FeedUtils.getFaviconUrl(subscription));
sub.setLastRefresh(feed.getLastUpdated());
sub.setNextRefresh(
(feed.getDisabledUntil() != null && feed.getDisabledUntil().isBefore(Instant.now())) ? null : feed.getDisabledUntil());
sub.setUnread(unreadCount.getUnreadCount());
sub.setNewestItemTime(unreadCount.getNewestItemTime());
sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));
sub.setFilter(subscription.getFilter());
return sub;
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "User information")
@Data
@RegisterForReflection
public class Subscription implements Serializable {
@Schema(description = "subscription id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "subscription name", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "error message while fetching the feed")
private String message;
@Schema(description = "error count", requiredMode = RequiredMode.REQUIRED)
private int errorCount;
@Schema(description = "last time the feed was refreshed", type = "number")
private Instant lastRefresh;
@Schema(description = "next time the feed refresh is planned, null if refresh is already queued", type = "number")
private Instant nextRefresh;
@Schema(description = "this subscription's feed url", requiredMode = RequiredMode.REQUIRED)
private String feedUrl;
@Schema(description = "this subscription's website url", requiredMode = RequiredMode.REQUIRED)
private String feedLink;
@Schema(description = "The favicon url to use for this feed", requiredMode = RequiredMode.REQUIRED)
private String iconUrl;
@Schema(description = "unread count", requiredMode = RequiredMode.REQUIRED)
private long unread;
@Schema(description = "category id")
private String categoryId;
@Schema(description = "position of the subscription's in the list")
private int position;
@Schema(description = "date of the newest item", type = "number")
private Instant newestItemTime;
@Schema(description = "JEXL string evaluated on new entries to mark them as read if they do not match")
private String filter;
public static Subscription build(FeedSubscription subscription, UnreadCount unreadCount) {
FeedCategory category = subscription.getCategory();
Feed feed = subscription.getFeed();
Subscription sub = new Subscription();
sub.setId(subscription.getId());
sub.setName(subscription.getTitle());
sub.setPosition(subscription.getPosition());
sub.setMessage(feed.getMessage());
sub.setErrorCount(feed.getErrorCount());
sub.setFeedUrl(feed.getUrl());
sub.setFeedLink(feed.getLink());
sub.setIconUrl(FeedUtils.getFaviconUrl(subscription));
sub.setLastRefresh(feed.getLastUpdated());
sub.setNextRefresh(
(feed.getDisabledUntil() != null && feed.getDisabledUntil().isBefore(Instant.now())) ? null : feed.getDisabledUntil());
sub.setUnread(unreadCount.getUnreadCount());
sub.setNewestItemTime(unreadCount.getNewestItemTime());
sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));
sub.setFilter(subscription.getFilter());
return sub;
}
}

68
commafeed-server/src/main/java/com/commafeed/frontend/model/UnreadCount.java
View File

@@ -1,34 +1,34 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Unread count")
@Data
@RegisterForReflection
public class UnreadCount implements Serializable {
@Schema
private long feedId;
@Schema
private long unreadCount;
@Schema(type = "number")
private Instant newestItemTime;
public UnreadCount() {
}
public UnreadCount(long feedId, long unreadCount, Instant newestItemTime) {
this.feedId = feedId;
this.unreadCount = unreadCount;
this.newestItemTime = newestItemTime;
}
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Unread count")
@Data
@RegisterForReflection
public class UnreadCount implements Serializable {
@Schema
private long feedId;
@Schema
private long unreadCount;
@Schema(type = "number")
private Instant newestItemTime;
public UnreadCount() {
}
public UnreadCount(long feedId, long unreadCount, Instant newestItemTime) {
this.feedId = feedId;
this.unreadCount = unreadCount;
this.newestItemTime = newestItemTime;
}
}

94
commafeed-server/src/main/java/com/commafeed/frontend/model/UserModel.java
View File

@@ -1,47 +1,47 @@
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "User information")
@Data
@RegisterForReflection
public class UserModel implements Serializable {
@Schema(description = "user id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "user name", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "user email, if any")
private String email;
@Schema(description = "api key")
private String apiKey;
@Schema(description = "user password, never returned by the api")
private String password;
@Schema(description = "account status", requiredMode = RequiredMode.REQUIRED)
private boolean enabled;
@Schema(description = "account creation date", type = "number")
private Instant created;
@Schema(description = "last login date", type = "number")
private Instant lastLogin;
@Schema(description = "user is admin", requiredMode = RequiredMode.REQUIRED)
private boolean admin;
@Schema(description = "user last force refresh", type = "number")
private Instant lastForceRefresh;
}
package com.commafeed.frontend.model;
import java.io.Serializable;
import java.time.Instant;
import io.quarkus.runtime.annotations.RegisterForReflection;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "User information")
@Data
@RegisterForReflection
public class UserModel implements Serializable {
@Schema(description = "user id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "user name", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "user email, if any")
private String email;
@Schema(description = "api key")
private String apiKey;
@Schema(description = "user password, never returned by the api")
private String password;
@Schema(description = "account status", requiredMode = RequiredMode.REQUIRED)
private boolean enabled;
@Schema(description = "account creation date", type = "number")
private Instant created;
@Schema(description = "last login date", type = "number")
private Instant lastLogin;
@Schema(description = "user is admin", requiredMode = RequiredMode.REQUIRED)
private boolean admin;
@Schema(description = "user last force refresh", type = "number")
private Instant lastForceRefresh;
}

52
commafeed-server/src/main/java/com/commafeed/frontend/model/request/AddCategoryRequest.java
View File

@@ -1,26 +1,26 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Add Category Request")
@Data
public class AddCategoryRequest implements Serializable {
@Schema(description = "name", requiredMode = RequiredMode.REQUIRED)
@NotEmpty
@Size(max = 128)
private String name;
@Schema(description = "parent category id, if any")
@Size(max = 128)
private String parentId;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Add Category Request")
@Data
public class AddCategoryRequest implements Serializable {
@Schema(description = "name", requiredMode = RequiredMode.REQUIRED)
@NotEmpty
@Size(max = 128)
private String name;
@Schema(description = "parent category id, if any")
@Size(max = 128)
private String parentId;
}

62
commafeed-server/src/main/java/com/commafeed/frontend/model/request/AdminSaveUserRequest.java
View File

@@ -1,31 +1,31 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Save User information")
@Data
public class AdminSaveUserRequest implements Serializable {
@Schema(description = "user id")
private Long id;
@Schema(description = "user name", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "user email, if any")
private String email;
@Schema(description = "user password")
private String password;
@Schema(description = "account status", requiredMode = RequiredMode.REQUIRED)
private boolean enabled;
@Schema(description = "user is admin", requiredMode = RequiredMode.REQUIRED)
private boolean admin;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Save User information")
@Data
public class AdminSaveUserRequest implements Serializable {
@Schema(description = "user id")
private Long id;
@Schema(description = "user name", requiredMode = RequiredMode.REQUIRED)
private String name;
@Schema(description = "user email, if any")
private String email;
@Schema(description = "user password")
private String password;
@Schema(description = "account status", requiredMode = RequiredMode.REQUIRED)
private boolean enabled;
@Schema(description = "user is admin", requiredMode = RequiredMode.REQUIRED)
private boolean admin;
}

60
commafeed-server/src/main/java/com/commafeed/frontend/model/request/CategoryModificationRequest.java
View File

@@ -1,30 +1,30 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Category modification request")
@Data
public class CategoryModificationRequest implements Serializable {
@Schema(description = "id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "new name, null if not changed")
@Size(max = 128)
private String name;
@Schema(description = "new parent category id")
@Size(max = 128)
private String parentId;
@Schema(description = "new display position, null if not changed")
private Integer position;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Category modification request")
@Data
public class CategoryModificationRequest implements Serializable {
@Schema(description = "id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "new name, null if not changed")
@Size(max = 128)
private String name;
@Schema(description = "new parent category id")
@Size(max = 128)
private String parentId;
@Schema(description = "new display position, null if not changed")
private Integer position;
}

40
commafeed-server/src/main/java/com/commafeed/frontend/model/request/CollapseRequest.java
View File

@@ -1,20 +1,20 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Mark Request")
@Data
public class CollapseRequest implements Serializable {
@Schema(description = "category id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "collapse", requiredMode = RequiredMode.REQUIRED)
private boolean collapse;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Mark Request")
@Data
public class CollapseRequest implements Serializable {
@Schema(description = "category id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "collapse", requiredMode = RequiredMode.REQUIRED)
private boolean collapse;
}

44
commafeed-server/src/main/java/com/commafeed/frontend/model/request/FeedInfoRequest.java
View File

@@ -1,22 +1,22 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Feed information request")
@Data
public class FeedInfoRequest implements Serializable {
@Schema(description = "feed url", requiredMode = RequiredMode.REQUIRED)
@NotEmpty
@Size(max = 4096)
private String url;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.NotEmpty;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Feed information request")
@Data
public class FeedInfoRequest implements Serializable {
@Schema(description = "feed url", requiredMode = RequiredMode.REQUIRED)
@NotEmpty
@Size(max = 4096)
private String url;
}

68
commafeed-server/src/main/java/com/commafeed/frontend/model/request/FeedModificationRequest.java
View File

@@ -1,34 +1,34 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Feed modification request")
@Data
public class FeedModificationRequest implements Serializable {
@Schema(description = "id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "new name, null if not changed")
@Size(max = 128)
private String name;
@Schema(description = "new parent category id")
@Size(max = 128)
private String categoryId;
@Schema(description = "new display position, null if not changed")
private Integer position;
@Schema(description = "JEXL string evaluated on new entries to mark them as read if they do not match")
@Size(max = 4096)
private String filter;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import jakarta.validation.constraints.Size;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema(description = "Feed modification request")
@Data
public class FeedModificationRequest implements Serializable {
@Schema(description = "id", requiredMode = RequiredMode.REQUIRED)
private Long id;
@Schema(description = "new name, null if not changed")
@Size(max = 128)
private String name;
@Schema(description = "new parent category id")
@Size(max = 128)
private String categoryId;
@Schema(description = "new display position, null if not changed")
private Integer position;
@Schema(description = "JEXL string evaluated on new entries to mark them as read if they do not match")
@Size(max = 4096)
private String filter;
}

34
commafeed-server/src/main/java/com/commafeed/frontend/model/request/IDRequest.java
View File

@@ -1,17 +1,17 @@
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema
@Data
public class IDRequest implements Serializable {
@Schema(requiredMode = RequiredMode.REQUIRED)
private Long id;
}
package com.commafeed.frontend.model.request;
import java.io.Serializable;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.RequiredMode;
import lombok.Data;
@SuppressWarnings("serial")
@Schema
@Data
public class IDRequest implements Serializable {
@Schema(requiredMode = RequiredMode.REQUIRED)
private Long id;
}

Some files were not shown because too many files have changed in this diff Show More