Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-24 04:57:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

normalize line endings

Browse Source
This commit is contained in:
Athou
2025-03-10 08:38:21 +01:00
parent ec4554c76e
commit fb7f041454
223 changed files with 18091 additions and 18093 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedEntryKeyword.java
View File

@@ -1,39 +1,39 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
/**
* A keyword used in a search query
*/
@Getter
@RequiredArgsConstructor
public class FeedEntryKeyword {
public enum Mode {
INCLUDE, EXCLUDE
}
private final String keyword;
private final Mode mode;
public static List<FeedEntryKeyword> fromQueryString(String keywords) {
List<FeedEntryKeyword> list = new ArrayList<>();
if (keywords != null) {
for (String keyword : StringUtils.split(keywords)) {
boolean not = false;
if (keyword.startsWith("-") || keyword.startsWith("!")) {
not = true;
keyword = keyword.substring(1);
}
list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
}
}
return list;
}
}
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
/**
* A keyword used in a search query
*/
@Getter
@RequiredArgsConstructor
public class FeedEntryKeyword {
public enum Mode {
INCLUDE, EXCLUDE
}
private final String keyword;
private final Mode mode;
public static List<FeedEntryKeyword> fromQueryString(String keywords) {
List<FeedEntryKeyword> list = new ArrayList<>();
if (keywords != null) {
for (String keyword : StringUtils.split(keywords)) {
boolean not = false;
if (keyword.startsWith("-") || keyword.startsWith("!")) {
not = true;
keyword = keyword.substring(1);
}
list.add(new FeedEntryKeyword(keyword, not ? Mode.EXCLUDE : Mode.INCLUDE));
}
}
return list;
}
}

246
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
View File

@@ -1,123 +1,123 @@
package com.commafeed.backend.feed;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HostNotAllowedException;
import com.commafeed.backend.HttpGetter.HttpRequest;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.SchemeNotAllowedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.parser.FeedParser;
import com.commafeed.backend.feed.parser.FeedParser.FeedParsingException;
import com.commafeed.backend.feed.parser.FeedParserResult;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import io.quarkus.arc.All;
import lombok.extern.slf4j.Slf4j;
/**
* Fetches a feed then parses it
*/
@Slf4j
@Singleton
public class FeedFetcher {
private final FeedParser parser;
private final HttpGetter getter;
private final List<FeedURLProvider> urlProviders;
public FeedFetcher(FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
this.parser = parser;
this.getter = getter;
this.urlProviders = urlProviders;
}
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
Instant lastPublishedDate, String lastContentHash) throws FeedParsingException, IOException, NotModifiedException,
TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException, NoFeedFoundException {
log.debug("Fetching feed {}", feedUrl);
HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
byte[] content = result.getContent();
FeedParserResult parserResult;
try {
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
} catch (FeedParsingException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, new String(result.getContent(), StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
result = getter.get(HttpRequest.builder(extractedUrl).lastModified(lastModified).eTag(eTag).build());
content = result.getContent();
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
} else {
throw new NoFeedFoundException(e);
}
} else {
throw e;
}
}
if (content == null) {
throw new IOException("Feed content is empty.");
}
boolean lastModifiedHeaderValueChanged = !StringUtils.equals(lastModified, result.getLastModifiedSince());
boolean etagHeaderValueChanged = !StringUtils.equals(eTag, result.getETag());
String hash = Digests.sha1Hex(content);
if (lastContentHash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException("content hash not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
}
if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
}
return new FeedFetcherResult(parserResult, result.getUrlAfterRedirect(), result.getLastModifiedSince(), result.getETag(), hash,
result.getValidFor());
}
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) {
for (FeedURLProvider urlProvider : urlProviders) {
String feedUrl = urlProvider.get(url, urlContent);
if (feedUrl != null) {
return feedUrl;
}
}
return null;
}
public record FeedFetcherResult(FeedParserResult feed, String urlAfterRedirect, String lastModifiedHeader, String lastETagHeader,
String contentHash, Duration validFor) {
}
public static class NoFeedFoundException extends Exception {
private static final long serialVersionUID = 1L;
public NoFeedFoundException(Throwable cause) {
super("This URL does not point to an RSS feed or a website with an RSS feed.", cause);
}
}
}
package com.commafeed.backend.feed;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HostNotAllowedException;
import com.commafeed.backend.HttpGetter.HttpRequest;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.SchemeNotAllowedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.parser.FeedParser;
import com.commafeed.backend.feed.parser.FeedParser.FeedParsingException;
import com.commafeed.backend.feed.parser.FeedParserResult;
import com.commafeed.backend.urlprovider.FeedURLProvider;
import io.quarkus.arc.All;
import lombok.extern.slf4j.Slf4j;
/**
* Fetches a feed then parses it
*/
@Slf4j
@Singleton
public class FeedFetcher {
private final FeedParser parser;
private final HttpGetter getter;
private final List<FeedURLProvider> urlProviders;
public FeedFetcher(FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
this.parser = parser;
this.getter = getter;
this.urlProviders = urlProviders;
}
public FeedFetcherResult fetch(String feedUrl, boolean extractFeedUrlFromHtml, String lastModified, String eTag,
Instant lastPublishedDate, String lastContentHash) throws FeedParsingException, IOException, NotModifiedException,
TooManyRequestsException, SchemeNotAllowedException, HostNotAllowedException, NoFeedFoundException {
log.debug("Fetching feed {}", feedUrl);
HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
byte[] content = result.getContent();
FeedParserResult parserResult;
try {
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
} catch (FeedParsingException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl = extractFeedUrl(urlProviders, feedUrl, new String(result.getContent(), StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
result = getter.get(HttpRequest.builder(extractedUrl).lastModified(lastModified).eTag(eTag).build());
content = result.getContent();
parserResult = parser.parse(result.getUrlAfterRedirect(), content);
} else {
throw new NoFeedFoundException(e);
}
} else {
throw e;
}
}
if (content == null) {
throw new IOException("Feed content is empty.");
}
boolean lastModifiedHeaderValueChanged = !StringUtils.equals(lastModified, result.getLastModifiedSince());
boolean etagHeaderValueChanged = !StringUtils.equals(eTag, result.getETag());
String hash = Digests.sha1Hex(content);
if (lastContentHash != null && lastContentHash.equals(hash)) {
log.debug("content hash not modified: {}", feedUrl);
throw new NotModifiedException("content hash not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
}
if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException("publishedDate not modified",
lastModifiedHeaderValueChanged ? result.getLastModifiedSince() : null,
etagHeaderValueChanged ? result.getETag() : null);
}
return new FeedFetcherResult(parserResult, result.getUrlAfterRedirect(), result.getLastModifiedSince(), result.getETag(), hash,
result.getValidFor());
}
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) {
for (FeedURLProvider urlProvider : urlProviders) {
String feedUrl = urlProvider.get(url, urlContent);
if (feedUrl != null) {
return feedUrl;
}
}
return null;
}
public record FeedFetcherResult(FeedParserResult feed, String urlAfterRedirect, String lastModifiedHeader, String lastETagHeader,
String contentHash, Duration validFor) {
}
public static class NoFeedFoundException extends Exception {
private static final long serialVersionUID = 1L;
public NoFeedFoundException(Throwable cause) {
super("This URL does not point to an RSS feed or a website with an RSS feed.", cause);
}
}
}

428
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshEngine.java
View File

@@ -1,214 +1,214 @@
package com.commafeed.backend.feed;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedRefreshEngine {
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedRefreshWorker worker;
private final FeedRefreshUpdater updater;
private final CommaFeedConfiguration config;
private final Meter refill;
private final BlockingDeque<Feed> queue;
private final ExecutorService feedProcessingLoopExecutor;
private final ExecutorService refillLoopExecutor;
private final ExecutorService refillExecutor;
private final ThreadPoolExecutor workerExecutor;
private final ThreadPoolExecutor databaseUpdaterExecutor;
public FeedRefreshEngine(UnitOfWork unitOfWork, FeedDAO feedDAO, FeedRefreshWorker worker, FeedRefreshUpdater updater,
CommaFeedConfiguration config, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.worker = worker;
this.updater = updater;
this.config = config;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
this.queue = new LinkedBlockingDeque<>();
this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor();
this.refillLoopExecutor = Executors.newSingleThreadExecutor();
this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.feedRefresh().databaseThreads());
metrics.register(MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(MetricRegistry.name(getClass(), "worker", "active"), (Gauge<Integer>) workerExecutor::getActiveCount);
metrics.register(MetricRegistry.name(getClass(), "updater", "active"), (Gauge<Integer>) databaseUpdaterExecutor::getActiveCount);
}
public void start() {
startFeedProcessingLoop();
startRefillLoop();
}
private void startFeedProcessingLoop() {
// take a feed from the queue, process it, rince, repeat
feedProcessingLoopExecutor.submit(() -> {
while (!feedProcessingLoopExecutor.isShutdown()) {
try {
// take() is blocking until a feed is available from the queue
Feed feed = queue.take();
// send the feed to be processed
log.debug("got feed {} from the queue, send it for processing", feed.getId());
processFeedAsync(feed);
// we removed a feed from the queue, try to refill it as it may now be empty
if (queue.isEmpty()) {
log.debug("took the last feed from the queue, try to refill");
refillQueueAsync();
}
} catch (InterruptedException e) {
log.debug("interrupted while waiting for a feed in the queue");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
private void startRefillLoop() {
// refill the queue at regular intervals if it's empty
refillLoopExecutor.submit(() -> {
while (!refillLoopExecutor.isShutdown()) {
try {
if (queue.isEmpty()) {
log.debug("refilling queue");
refillQueueAsync();
}
log.debug("sleeping for 15s");
TimeUnit.SECONDS.sleep(15);
} catch (InterruptedException e) {
log.debug("interrupted while sleeping");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
public void refreshImmediately(Feed feed) {
log.debug("add feed {} at the start of the queue", feed.getId());
// remove the feed from the queue if it was already queued to avoid refreshing it twice
queue.removeIf(f -> f.getId().equals(feed.getId()));
queue.addFirst(feed);
}
private void refillQueueAsync() {
CompletableFuture.runAsync(() -> {
if (!queue.isEmpty()) {
return;
}
refill.mark();
List<Feed> nextUpdatableFeeds = getNextUpdatableFeeds(getBatchSize());
log.debug("found {} feeds that are up for refresh", nextUpdatableFeeds.size());
for (Feed feed : nextUpdatableFeeds) {
// add the feed only if it was not already queued
if (queue.stream().noneMatch(f -> f.getId().equals(feed.getId()))) {
queue.addLast(feed);
}
}
}, refillExecutor).whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while refilling the queue", ex);
}
});
}
private void processFeedAsync(Feed feed) {
CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
.thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
.whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while processing feed {}", feed.getUrl(), ex);
}
});
}
private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(() -> {
Instant lastLoginThreshold = config.feedRefresh().userInactivityPeriod().isZero() ? null
: Instant.now().minus(config.feedRefresh().userInactivityPeriod());
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
// update disabledUntil to prevent feeds from being returned again by feedDAO.findNextUpdatable()
Instant nextUpdateDate = Instant.now().plus(config.feedRefresh().interval());
feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
return feeds;
});
}
private int getBatchSize() {
return Math.min(100, 3 * config.feedRefresh().httpThreads());
}
public void stop() {
this.feedProcessingLoopExecutor.shutdownNow();
this.refillLoopExecutor.shutdownNow();
this.refillExecutor.shutdownNow();
this.workerExecutor.shutdownNow();
this.databaseUpdaterExecutor.shutdownNow();
}
/**
* returns an ExecutorService with a single thread that discards tasks if a task is already running
*/
private ThreadPoolExecutor newDiscardingSingleThreadExecutorService() {
ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/**
* returns an ExecutorService that blocks submissions until a thread is available
*/
private ThreadPoolExecutor newBlockingExecutorService(int threads) {
ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler((r, e) -> {
if (e.isShutdown()) {
return;
}
try {
e.getQueue().put(r);
} catch (InterruptedException ex) {
log.debug("interrupted while waiting for a slot in the queue.", ex);
Thread.currentThread().interrupt();
}
});
return pool;
}
}
package com.commafeed.backend.feed;
import java.time.Instant;
import java.util.List;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import jakarta.inject.Singleton;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedRefreshEngine {
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedRefreshWorker worker;
private final FeedRefreshUpdater updater;
private final CommaFeedConfiguration config;
private final Meter refill;
private final BlockingDeque<Feed> queue;
private final ExecutorService feedProcessingLoopExecutor;
private final ExecutorService refillLoopExecutor;
private final ExecutorService refillExecutor;
private final ThreadPoolExecutor workerExecutor;
private final ThreadPoolExecutor databaseUpdaterExecutor;
public FeedRefreshEngine(UnitOfWork unitOfWork, FeedDAO feedDAO, FeedRefreshWorker worker, FeedRefreshUpdater updater,
CommaFeedConfiguration config, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.worker = worker;
this.updater = updater;
this.config = config;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
this.queue = new LinkedBlockingDeque<>();
this.feedProcessingLoopExecutor = Executors.newSingleThreadExecutor();
this.refillLoopExecutor = Executors.newSingleThreadExecutor();
this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.feedRefresh().databaseThreads());
metrics.register(MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(MetricRegistry.name(getClass(), "worker", "active"), (Gauge<Integer>) workerExecutor::getActiveCount);
metrics.register(MetricRegistry.name(getClass(), "updater", "active"), (Gauge<Integer>) databaseUpdaterExecutor::getActiveCount);
}
public void start() {
startFeedProcessingLoop();
startRefillLoop();
}
private void startFeedProcessingLoop() {
// take a feed from the queue, process it, rince, repeat
feedProcessingLoopExecutor.submit(() -> {
while (!feedProcessingLoopExecutor.isShutdown()) {
try {
// take() is blocking until a feed is available from the queue
Feed feed = queue.take();
// send the feed to be processed
log.debug("got feed {} from the queue, send it for processing", feed.getId());
processFeedAsync(feed);
// we removed a feed from the queue, try to refill it as it may now be empty
if (queue.isEmpty()) {
log.debug("took the last feed from the queue, try to refill");
refillQueueAsync();
}
} catch (InterruptedException e) {
log.debug("interrupted while waiting for a feed in the queue");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
private void startRefillLoop() {
// refill the queue at regular intervals if it's empty
refillLoopExecutor.submit(() -> {
while (!refillLoopExecutor.isShutdown()) {
try {
if (queue.isEmpty()) {
log.debug("refilling queue");
refillQueueAsync();
}
log.debug("sleeping for 15s");
TimeUnit.SECONDS.sleep(15);
} catch (InterruptedException e) {
log.debug("interrupted while sleeping");
Thread.currentThread().interrupt();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
});
}
public void refreshImmediately(Feed feed) {
log.debug("add feed {} at the start of the queue", feed.getId());
// remove the feed from the queue if it was already queued to avoid refreshing it twice
queue.removeIf(f -> f.getId().equals(feed.getId()));
queue.addFirst(feed);
}
private void refillQueueAsync() {
CompletableFuture.runAsync(() -> {
if (!queue.isEmpty()) {
return;
}
refill.mark();
List<Feed> nextUpdatableFeeds = getNextUpdatableFeeds(getBatchSize());
log.debug("found {} feeds that are up for refresh", nextUpdatableFeeds.size());
for (Feed feed : nextUpdatableFeeds) {
// add the feed only if it was not already queued
if (queue.stream().noneMatch(f -> f.getId().equals(feed.getId()))) {
queue.addLast(feed);
}
}
}, refillExecutor).whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while refilling the queue", ex);
}
});
}
private void processFeedAsync(Feed feed) {
CompletableFuture.supplyAsync(() -> worker.update(feed), workerExecutor)
.thenApplyAsync(r -> updater.update(r.feed(), r.entries()), databaseUpdaterExecutor)
.whenComplete((data, ex) -> {
if (ex != null) {
log.error("error while processing feed {}", feed.getUrl(), ex);
}
});
}
private List<Feed> getNextUpdatableFeeds(int max) {
return unitOfWork.call(() -> {
Instant lastLoginThreshold = config.feedRefresh().userInactivityPeriod().isZero() ? null
: Instant.now().minus(config.feedRefresh().userInactivityPeriod());
List<Feed> feeds = feedDAO.findNextUpdatable(max, lastLoginThreshold);
// update disabledUntil to prevent feeds from being returned again by feedDAO.findNextUpdatable()
Instant nextUpdateDate = Instant.now().plus(config.feedRefresh().interval());
feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
return feeds;
});
}
private int getBatchSize() {
return Math.min(100, 3 * config.feedRefresh().httpThreads());
}
public void stop() {
this.feedProcessingLoopExecutor.shutdownNow();
this.refillLoopExecutor.shutdownNow();
this.refillExecutor.shutdownNow();
this.workerExecutor.shutdownNow();
this.databaseUpdaterExecutor.shutdownNow();
}
/**
* returns an ExecutorService with a single thread that discards tasks if a task is already running
*/
private ThreadPoolExecutor newDiscardingSingleThreadExecutorService() {
ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
/**
* returns an ExecutorService that blocks submissions until a thread is available
*/
private ThreadPoolExecutor newBlockingExecutorService(int threads) {
ThreadPoolExecutor pool = new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler((r, e) -> {
if (e.isShutdown()) {
return;
}
try {
e.getQueue().put(r);
} catch (InterruptedException ex) {
log.debug("interrupted while waiting for a slot in the queue.", ex);
Thread.currentThread().interrupt();
}
});
return pool;
}
}

168
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshIntervalCalculator.java
View File

@@ -1,84 +1,84 @@
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.FeedRefreshErrorHandling;
import com.google.common.primitives.Longs;
@Singleton
public class FeedRefreshIntervalCalculator {
private final Duration interval;
private final Duration maxInterval;
private final boolean empirical;
private final FeedRefreshErrorHandling errorHandling;
private final InstantSource instantSource;
public FeedRefreshIntervalCalculator(CommaFeedConfiguration config, InstantSource instantSource) {
this.interval = config.feedRefresh().interval();
this.maxInterval = config.feedRefresh().maxInterval();
this.empirical = config.feedRefresh().intervalEmpirical();
this.errorHandling = config.feedRefresh().errors();
this.instantSource = instantSource;
}
public Instant onFetchSuccess(Instant publishedDate, Long averageEntryInterval, Duration validFor) {
Instant instant = empirical ? computeEmpiricalRefreshInterval(publishedDate, averageEntryInterval)
: instantSource.instant().plus(interval);
return constrainToBounds(ObjectUtils.max(instant, instantSource.instant().plus(validFor)));
}
public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) {
return onFetchSuccess(publishedDate, averageEntryInterval, Duration.ZERO);
}
public Instant onTooManyRequests(Instant retryAfter, int errorCount) {
return constrainToBounds(ObjectUtils.max(retryAfter, onFetchError(errorCount)));
}
public Instant onFetchError(int errorCount) {
if (errorCount < errorHandling.retriesBeforeBackoff()) {
return constrainToBounds(instantSource.instant().plus(interval));
}
Duration retryInterval = errorHandling.backoffInterval().multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L);
return constrainToBounds(instantSource.instant().plus(retryInterval));
}
private Instant computeEmpiricalRefreshInterval(Instant publishedDate, Long averageEntryInterval) {
Instant now = instantSource.instant();
if (publishedDate == null) {
return now.plus(maxInterval);
}
long daysSinceLastPublication = ChronoUnit.DAYS.between(publishedDate, now);
if (daysSinceLastPublication >= 30) {
return now.plus(maxInterval);
} else if (daysSinceLastPublication >= 14) {
return now.plus(maxInterval.dividedBy(2));
} else if (daysSinceLastPublication >= 7) {
return now.plus(maxInterval.dividedBy(4));
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor
int factor = 2;
long millis = Longs.constrainToRange(averageEntryInterval / factor, interval.toMillis(), maxInterval.dividedBy(4).toMillis());
return now.plusMillis(millis);
} else {
// unknown case
return now.plus(maxInterval);
}
}
private Instant constrainToBounds(Instant instant) {
return ObjectUtils.max(ObjectUtils.min(instant, instantSource.instant().plus(maxInterval)), instantSource.instant().plus(interval));
}
}
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.time.InstantSource;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ObjectUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.CommaFeedConfiguration.FeedRefreshErrorHandling;
import com.google.common.primitives.Longs;
@Singleton
public class FeedRefreshIntervalCalculator {
private final Duration interval;
private final Duration maxInterval;
private final boolean empirical;
private final FeedRefreshErrorHandling errorHandling;
private final InstantSource instantSource;
public FeedRefreshIntervalCalculator(CommaFeedConfiguration config, InstantSource instantSource) {
this.interval = config.feedRefresh().interval();
this.maxInterval = config.feedRefresh().maxInterval();
this.empirical = config.feedRefresh().intervalEmpirical();
this.errorHandling = config.feedRefresh().errors();
this.instantSource = instantSource;
}
public Instant onFetchSuccess(Instant publishedDate, Long averageEntryInterval, Duration validFor) {
Instant instant = empirical ? computeEmpiricalRefreshInterval(publishedDate, averageEntryInterval)
: instantSource.instant().plus(interval);
return constrainToBounds(ObjectUtils.max(instant, instantSource.instant().plus(validFor)));
}
public Instant onFeedNotModified(Instant publishedDate, Long averageEntryInterval) {
return onFetchSuccess(publishedDate, averageEntryInterval, Duration.ZERO);
}
public Instant onTooManyRequests(Instant retryAfter, int errorCount) {
return constrainToBounds(ObjectUtils.max(retryAfter, onFetchError(errorCount)));
}
public Instant onFetchError(int errorCount) {
if (errorCount < errorHandling.retriesBeforeBackoff()) {
return constrainToBounds(instantSource.instant().plus(interval));
}
Duration retryInterval = errorHandling.backoffInterval().multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L);
return constrainToBounds(instantSource.instant().plus(retryInterval));
}
private Instant computeEmpiricalRefreshInterval(Instant publishedDate, Long averageEntryInterval) {
Instant now = instantSource.instant();
if (publishedDate == null) {
return now.plus(maxInterval);
}
long daysSinceLastPublication = ChronoUnit.DAYS.between(publishedDate, now);
if (daysSinceLastPublication >= 30) {
return now.plus(maxInterval);
} else if (daysSinceLastPublication >= 14) {
return now.plus(maxInterval.dividedBy(2));
} else if (daysSinceLastPublication >= 7) {
return now.plus(maxInterval.dividedBy(4));
} else if (averageEntryInterval != null) {
// use average time between entries to decide when to refresh next, divided by factor
int factor = 2;
long millis = Longs.constrainToRange(averageEntryInterval / factor, interval.toMillis(), maxInterval.dividedBy(4).toMillis());
return now.plusMillis(millis);
} else {
// unknown case
return now.plus(maxInterval);
}
}
private Instant constrainToBounds(Instant instant) {
return ObjectUtils.max(ObjectUtils.min(instant, instantSource.instant().plus(maxInterval)), instantSource.instant().plus(interval));
}
}

360
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
View File

@@ -1,180 +1,180 @@
package com.commafeed.backend.feed;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions;
import com.google.common.util.concurrent.Striped;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Updates the feed in the database and inserts new entries
*/
@Slf4j
@Singleton
public class FeedRefreshUpdater {
private final UnitOfWork unitOfWork;
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final WebSocketSessions webSocketSessions;
private final Striped<Lock> locks;
private final Meter feedUpdated;
private final Meter entryInserted;
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO, WebSocketSessions webSocketSessions) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.webSocketSessions = webSocketSessions;
locks = Striped.lazyWeakLock(100000);
feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
private AddEntryResult addEntry(final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
boolean processed = false;
boolean inserted = false;
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread = new HashSet<>();
// lock on feed, make sure we are not updating the same feed twice at
// the same time
String key1 = StringUtils.trimToEmpty(String.valueOf(feed.getId()));
// lock on content, make sure we are not updating the same entry
// twice at the same time
Content content = entry.content();
String key2 = Digests.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title()));
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
Lock lock2 = iterator.next();
boolean locked1 = false;
boolean locked2 = false;
try {
// try to lock, give up after 1 minute
locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
processed = true;
inserted = unitOfWork.call(() -> {
boolean newEntry = false;
FeedEntry feedEntry = feedEntryService.find(feed, entry);
if (feedEntry == null) {
feedEntry = feedEntryService.create(feed, entry);
newEntry = true;
}
if (newEntry) {
entryInserted.mark();
for (FeedSubscription sub : subscriptions) {
boolean unread = feedEntryService.applyFilter(sub, feedEntry);
if (unread) {
subscriptionsForWhichEntryIsUnread.add(sub);
}
}
}
return newEntry;
});
} else {
log.error("lock timeout for {} - {}", feed.getUrl(), key1);
}
} catch (InterruptedException e) {
log.error("interrupted while waiting for lock for {} : {}", feed.getUrl(), e.getMessage(), e);
} finally {
if (locked1) {
lock1.unlock();
}
if (locked2) {
lock2.unlock();
}
}
return new AddEntryResult(processed, inserted, subscriptionsForWhichEntryIsUnread);
}
public boolean update(Feed feed, List<Entry> entries) {
boolean processed = true;
long inserted = 0;
Map<FeedSubscription, Long> unreadCountBySubscription = new HashMap<>();
if (!entries.isEmpty()) {
List<FeedSubscription> subscriptions = null;
for (Entry entry : entries) {
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
processed &= addEntryResult.processed;
inserted += addEntryResult.inserted ? 1 : 0;
addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(sub -> unreadCountBySubscription.merge(sub, 1L, Long::sum));
}
if (inserted == 0) {
feed.setMessage("No new entries found");
} else if (inserted > 0) {
feed.setMessage("Found %s new entries".formatted(inserted));
}
}
if (!processed) {
// requeue asap
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
}
if (inserted > 0) {
feedUpdated.mark();
}
unitOfWork.run(() -> feedService.update(feed));
notifyOverWebsocket(unreadCountBySubscription);
return processed;
}
private void notifyOverWebsocket(Map<FeedSubscription, Long> unreadCountBySubscription) {
unreadCountBySubscription.forEach((sub, unreadCount) -> webSocketSessions.sendMessage(sub.getUser(),
WebSocketMessageBuilder.newFeedEntries(sub, unreadCount)));
}
@AllArgsConstructor
private static class AddEntryResult {
private final boolean processed;
private final boolean inserted;
private final Set<FeedSubscription> subscriptionsForWhichEntryIsUnread;
}
}
package com.commafeed.backend.feed;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.commafeed.frontend.ws.WebSocketMessageBuilder;
import com.commafeed.frontend.ws.WebSocketSessions;
import com.google.common.util.concurrent.Striped;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Updates the feed in the database and inserts new entries
*/
@Slf4j
@Singleton
public class FeedRefreshUpdater {
private final UnitOfWork unitOfWork;
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final WebSocketSessions webSocketSessions;
private final Striped<Lock> locks;
private final Meter feedUpdated;
private final Meter entryInserted;
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO, WebSocketSessions webSocketSessions) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.webSocketSessions = webSocketSessions;
locks = Striped.lazyWeakLock(100000);
feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
private AddEntryResult addEntry(final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
boolean processed = false;
boolean inserted = false;
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread = new HashSet<>();
// lock on feed, make sure we are not updating the same feed twice at
// the same time
String key1 = StringUtils.trimToEmpty(String.valueOf(feed.getId()));
// lock on content, make sure we are not updating the same entry
// twice at the same time
Content content = entry.content();
String key2 = Digests.sha1Hex(StringUtils.trimToEmpty(content.content() + content.title()));
Iterator<Lock> iterator = locks.bulkGet(Arrays.asList(key1, key2)).iterator();
Lock lock1 = iterator.next();
Lock lock2 = iterator.next();
boolean locked1 = false;
boolean locked2 = false;
try {
// try to lock, give up after 1 minute
locked1 = lock1.tryLock(1, TimeUnit.MINUTES);
locked2 = lock2.tryLock(1, TimeUnit.MINUTES);
if (locked1 && locked2) {
processed = true;
inserted = unitOfWork.call(() -> {
boolean newEntry = false;
FeedEntry feedEntry = feedEntryService.find(feed, entry);
if (feedEntry == null) {
feedEntry = feedEntryService.create(feed, entry);
newEntry = true;
}
if (newEntry) {
entryInserted.mark();
for (FeedSubscription sub : subscriptions) {
boolean unread = feedEntryService.applyFilter(sub, feedEntry);
if (unread) {
subscriptionsForWhichEntryIsUnread.add(sub);
}
}
}
return newEntry;
});
} else {
log.error("lock timeout for {} - {}", feed.getUrl(), key1);
}
} catch (InterruptedException e) {
log.error("interrupted while waiting for lock for {} : {}", feed.getUrl(), e.getMessage(), e);
} finally {
if (locked1) {
lock1.unlock();
}
if (locked2) {
lock2.unlock();
}
}
return new AddEntryResult(processed, inserted, subscriptionsForWhichEntryIsUnread);
}
public boolean update(Feed feed, List<Entry> entries) {
boolean processed = true;
long inserted = 0;
Map<FeedSubscription, Long> unreadCountBySubscription = new HashMap<>();
if (!entries.isEmpty()) {
List<FeedSubscription> subscriptions = null;
for (Entry entry : entries) {
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
processed &= addEntryResult.processed;
inserted += addEntryResult.inserted ? 1 : 0;
addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(sub -> unreadCountBySubscription.merge(sub, 1L, Long::sum));
}
if (inserted == 0) {
feed.setMessage("No new entries found");
} else if (inserted > 0) {
feed.setMessage("Found %s new entries".formatted(inserted));
}
}
if (!processed) {
// requeue asap
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
}
if (inserted > 0) {
feedUpdated.mark();
}
unitOfWork.run(() -> feedService.update(feed));
notifyOverWebsocket(unreadCountBySubscription);
return processed;
}
private void notifyOverWebsocket(Map<FeedSubscription, Long> unreadCountBySubscription) {
unreadCountBySubscription.forEach((sub, unreadCount) -> webSocketSessions.sendMessage(sub.getUser(),
WebSocketMessageBuilder.newFeedEntries(sub, unreadCount)));
}
@AllArgsConstructor
private static class AddEntryResult {
private final boolean processed;
private final boolean inserted;
private final Set<FeedSubscription> subscriptionsForWhichEntryIsUnread;
}
}

250
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
View File

@@ -1,125 +1,125 @@
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
/**
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link FeedRefreshUpdater} does that)
*/
@Slf4j
@Singleton
public class FeedRefreshWorker {
private final FeedRefreshIntervalCalculator refreshIntervalCalculator;
private final FeedFetcher fetcher;
private final CommaFeedConfiguration config;
private final Meter feedFetched;
public FeedRefreshWorker(FeedRefreshIntervalCalculator refreshIntervalCalculator, FeedFetcher fetcher, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.refreshIntervalCalculator = refreshIntervalCalculator;
this.fetcher = fetcher;
this.config = config;
this.feedFetched = metrics.meter(MetricRegistry.name(getClass(), "feedFetched"));
}
public FeedRefreshWorkerResult update(Feed feed) {
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FeedFetcherResult result = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
List<Entry> entries = result.feed().entries();
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
entries = entries.stream().filter(entry -> entry.published().isAfter(threshold)).toList();
}
String urlAfterRedirect = result.urlAfterRedirect();
if (StringUtils.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;
}
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(result.feed().link());
feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(result.contentHash());
feed.setLastPublishedDate(result.feed().lastPublishedDate());
feed.setAverageEntryInterval(result.feed().averageEntryInterval());
feed.setLastEntryDate(result.feed().lastEntryDate());
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(),
result.feed().averageEntryInterval(), result.validFor()));
return new FeedRefreshWorkerResult(feed, entries);
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
feed.setErrorCount(0);
feed.setMessage(e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFeedNotModified(feed.getLastPublishedDate(), feed.getAverageEntryInterval()));
if (e.getNewLastModifiedHeader() != null) {
feed.setLastModifiedHeader(e.getNewLastModifiedHeader());
}
if (e.getNewEtagHeader() != null) {
feed.setEtagHeader(e.getNewEtagHeader());
}
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (TooManyRequestsException e) {
log.debug("Too many requests : {}", feed.getUrl());
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Server indicated that we are sending too many requests");
feed.setDisabledUntil(refreshIntervalCalculator.onTooManyRequests(e.getRetryAfter(), feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (Exception e) {
log.debug("unable to refresh feed {}", feed.getUrl(), e);
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Unable to refresh feed : " + e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} finally {
feedFetched.mark();
}
}
public record FeedRefreshWorkerResult(Feed feed, List<Entry> entries) {
}
}
package com.commafeed.backend.feed;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter.NotModifiedException;
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
import com.commafeed.backend.feed.FeedFetcher.FeedFetcherResult;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
/**
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link FeedRefreshUpdater} does that)
*/
@Slf4j
@Singleton
public class FeedRefreshWorker {
private final FeedRefreshIntervalCalculator refreshIntervalCalculator;
private final FeedFetcher fetcher;
private final CommaFeedConfiguration config;
private final Meter feedFetched;
public FeedRefreshWorker(FeedRefreshIntervalCalculator refreshIntervalCalculator, FeedFetcher fetcher, CommaFeedConfiguration config,
MetricRegistry metrics) {
this.refreshIntervalCalculator = refreshIntervalCalculator;
this.fetcher = fetcher;
this.config = config;
this.feedFetched = metrics.meter(MetricRegistry.name(getClass(), "feedFetched"));
}
public FeedRefreshWorkerResult update(Feed feed) {
try {
String url = Optional.ofNullable(feed.getUrlAfterRedirect()).orElse(feed.getUrl());
FeedFetcherResult result = fetcher.fetch(url, false, feed.getLastModifiedHeader(), feed.getEtagHeader(),
feed.getLastPublishedDate(), feed.getLastContentHash());
// stops here if NotModifiedException or any other exception is thrown
List<Entry> entries = result.feed().entries();
int maxFeedCapacity = config.database().cleanup().maxFeedCapacity();
if (maxFeedCapacity > 0) {
entries = entries.stream().limit(maxFeedCapacity).toList();
}
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
entries = entries.stream().filter(entry -> entry.published().isAfter(threshold)).toList();
}
String urlAfterRedirect = result.urlAfterRedirect();
if (StringUtils.equals(url, urlAfterRedirect)) {
urlAfterRedirect = null;
}
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(result.feed().link());
feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(result.contentHash());
feed.setLastPublishedDate(result.feed().lastPublishedDate());
feed.setAverageEntryInterval(result.feed().averageEntryInterval());
feed.setLastEntryDate(result.feed().lastEntryDate());
feed.setErrorCount(0);
feed.setMessage(null);
feed.setDisabledUntil(refreshIntervalCalculator.onFetchSuccess(result.feed().lastPublishedDate(),
result.feed().averageEntryInterval(), result.validFor()));
return new FeedRefreshWorkerResult(feed, entries);
} catch (NotModifiedException e) {
log.debug("Feed not modified : {} - {}", feed.getUrl(), e.getMessage());
feed.setErrorCount(0);
feed.setMessage(e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFeedNotModified(feed.getLastPublishedDate(), feed.getAverageEntryInterval()));
if (e.getNewLastModifiedHeader() != null) {
feed.setLastModifiedHeader(e.getNewLastModifiedHeader());
}
if (e.getNewEtagHeader() != null) {
feed.setEtagHeader(e.getNewEtagHeader());
}
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (TooManyRequestsException e) {
log.debug("Too many requests : {}", feed.getUrl());
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Server indicated that we are sending too many requests");
feed.setDisabledUntil(refreshIntervalCalculator.onTooManyRequests(e.getRetryAfter(), feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (Exception e) {
log.debug("unable to refresh feed {}", feed.getUrl(), e);
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Unable to refresh feed : " + e.getMessage());
feed.setDisabledUntil(refreshIntervalCalculator.onFetchError(feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} finally {
feedFetched.mark();
}
}
public record FeedRefreshWorkerResult(Feed feed, List<Entry> entries) {
}
}

436
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java
View File

@@ -1,218 +1,218 @@
package com.commafeed.backend.feed;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.utils.Base64;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.netpreserve.urlcanon.Canonicalizer;
import org.netpreserve.urlcanon.ParsedUrl;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.feed.parser.TextDirectionDetector;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry;
import lombok.extern.slf4j.Slf4j;
/**
* Utility methods related to feed handling
*
*/
@Slf4j
public class FeedUtils {
private static final String ESCAPED_QUESTION_MARK = Pattern.quote("?");
public static String truncate(String string, int length) {
if (string != null) {
string = string.substring(0, Math.min(length, string.length()));
}
return string;
}
public static boolean isHttp(String url) {
return url.startsWith("http://");
}
public static boolean isHttps(String url) {
return url.startsWith("https://");
}
public static boolean isAbsoluteUrl(String url) {
return isHttp(url) || isHttps(url);
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates
*/
public static String normalizeURL(String url) {
if (url == null) {
return null;
}
ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
String normalized = parsedUrl.toString();
if (normalized == null) {
normalized = url;
}
// convert to lower case, the url probably won't work in some cases
// after that but we don't care we just want to compare urls to avoid
// duplicates
normalized = normalized.toLowerCase();
// store all urls as http
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
// remove the www. part
normalized = normalized.replace("//www.", "//");
// feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedburner feeds have a special treatment
if (normalized.split(ESCAPED_QUESTION_MARK)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = normalized.split(ESCAPED_QUESTION_MARK)[0];
normalized = StringUtils.removeEnd(normalized, "/");
}
return normalized;
}
public static boolean isRTL(String title, String content) {
String text = StringUtils.isNotBlank(content) ? content : title;
if (StringUtils.isBlank(text)) {
return false;
}
String stripped = Jsoup.parse(text).text();
if (StringUtils.isBlank(stripped)) {
return false;
}
return TextDirectionDetector.detect(stripped) == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
public static String removeTrailingSlash(String url) {
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}
return url;
}
/**
*
* @param relativeUrl
* the url of the entry
* @param feedLink
* the url of the feed as described in the feed
* @param feedUrl
* the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry
*/
public static String toAbsoluteUrl(String relativeUrl, String feedLink, String feedUrl) {
String baseUrl = (feedLink != null && isAbsoluteUrl(feedLink)) ? feedLink : feedUrl;
if (baseUrl == null) {
return null;
}
try {
return new URL(new URL(baseUrl), relativeUrl).toString();
} catch (MalformedURLException e) {
log.debug("could not parse url : {}", e.getMessage(), e);
return null;
}
}
public static String getFaviconUrl(FeedSubscription subscription) {
return "rest/feed/favicon/" + subscription.getId();
}
public static String proxyImages(String content) {
if (StringUtils.isBlank(content)) {
return content;
}
Document doc = Jsoup.parse(content);
Elements elements = doc.select("img");
for (Element element : elements) {
String href = element.attr("src");
if (StringUtils.isNotBlank(href)) {
String proxy = proxyImage(href);
element.attr("src", proxy);
}
}
return doc.body().html();
}
public static String proxyImage(String url) {
if (StringUtils.isBlank(url)) {
return url;
}
return "rest/server/proxy?u=" + imageProxyEncoder(url);
}
public static String rot13(String msg) {
StringBuilder message = new StringBuilder();
for (char c : msg.toCharArray()) {
if (c >= 'a' && c <= 'm') {
c += 13;
} else if (c >= 'n' && c <= 'z') {
c -= 13;
} else if (c >= 'A' && c <= 'M') {
c += 13;
} else if (c >= 'N' && c <= 'Z') {
c -= 13;
}
message.append(c);
}
return message.toString();
}
public static String imageProxyEncoder(String url) {
return Base64.encodeBase64String(rot13(url).getBytes());
}
public static String imageProxyDecoder(String code) {
return rot13(new String(Base64.decodeBase64(code)));
}
public static void removeUnwantedFromSearch(List<Entry> entries, List<FeedEntryKeyword> keywords) {
Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
Entry entry = it.next();
boolean keep = true;
for (FeedEntryKeyword keyword : keywords) {
String title = entry.getTitle() == null ? null : Jsoup.parse(entry.getTitle()).text();
String content = entry.getContent() == null ? null : Jsoup.parse(entry.getContent()).text();
boolean condition = !StringUtils.containsIgnoreCase(content, keyword.getKeyword())
&& !StringUtils.containsIgnoreCase(title, keyword.getKeyword());
if (keyword.getMode() == Mode.EXCLUDE) {
condition = !condition;
}
if (condition) {
keep = false;
break;
}
}
if (!keep) {
it.remove();
}
}
}
}
package com.commafeed.backend.feed;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.utils.Base64;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.netpreserve.urlcanon.Canonicalizer;
import org.netpreserve.urlcanon.ParsedUrl;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.feed.parser.TextDirectionDetector;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.frontend.model.Entry;
import lombok.extern.slf4j.Slf4j;
/**
* Utility methods related to feed handling
*
*/
@Slf4j
public class FeedUtils {
private static final String ESCAPED_QUESTION_MARK = Pattern.quote("?");
public static String truncate(String string, int length) {
if (string != null) {
string = string.substring(0, Math.min(length, string.length()));
}
return string;
}
public static boolean isHttp(String url) {
return url.startsWith("http://");
}
public static boolean isHttps(String url) {
return url.startsWith("https://");
}
public static boolean isAbsoluteUrl(String url) {
return isHttp(url) || isHttps(url);
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates
*/
public static String normalizeURL(String url) {
if (url == null) {
return null;
}
ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
String normalized = parsedUrl.toString();
if (normalized == null) {
normalized = url;
}
// convert to lower case, the url probably won't work in some cases
// after that but we don't care we just want to compare urls to avoid
// duplicates
normalized = normalized.toLowerCase();
// store all urls as http
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
// remove the www. part
normalized = normalized.replace("//www.", "//");
// feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedburner feeds have a special treatment
if (normalized.split(ESCAPED_QUESTION_MARK)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = normalized.split(ESCAPED_QUESTION_MARK)[0];
normalized = StringUtils.removeEnd(normalized, "/");
}
return normalized;
}
public static boolean isRTL(String title, String content) {
String text = StringUtils.isNotBlank(content) ? content : title;
if (StringUtils.isBlank(text)) {
return false;
}
String stripped = Jsoup.parse(text).text();
if (StringUtils.isBlank(stripped)) {
return false;
}
return TextDirectionDetector.detect(stripped) == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
public static String removeTrailingSlash(String url) {
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}
return url;
}
/**
*
* @param relativeUrl
* the url of the entry
* @param feedLink
* the url of the feed as described in the feed
* @param feedUrl
* the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry
*/
public static String toAbsoluteUrl(String relativeUrl, String feedLink, String feedUrl) {
String baseUrl = (feedLink != null && isAbsoluteUrl(feedLink)) ? feedLink : feedUrl;
if (baseUrl == null) {
return null;
}
try {
return new URL(new URL(baseUrl), relativeUrl).toString();
} catch (MalformedURLException e) {
log.debug("could not parse url : {}", e.getMessage(), e);
return null;
}
}
public static String getFaviconUrl(FeedSubscription subscription) {
return "rest/feed/favicon/" + subscription.getId();
}
public static String proxyImages(String content) {
if (StringUtils.isBlank(content)) {
return content;
}
Document doc = Jsoup.parse(content);
Elements elements = doc.select("img");
for (Element element : elements) {
String href = element.attr("src");
if (StringUtils.isNotBlank(href)) {
String proxy = proxyImage(href);
element.attr("src", proxy);
}
}
return doc.body().html();
}
public static String proxyImage(String url) {
if (StringUtils.isBlank(url)) {
return url;
}
return "rest/server/proxy?u=" + imageProxyEncoder(url);
}
public static String rot13(String msg) {
StringBuilder message = new StringBuilder();
for (char c : msg.toCharArray()) {
if (c >= 'a' && c <= 'm') {
c += 13;
} else if (c >= 'n' && c <= 'z') {
c -= 13;
} else if (c >= 'A' && c <= 'M') {
c += 13;
} else if (c >= 'N' && c <= 'Z') {
c -= 13;
}
message.append(c);
}
return message.toString();
}
public static String imageProxyEncoder(String url) {
return Base64.encodeBase64String(rot13(url).getBytes());
}
public static String imageProxyDecoder(String code) {
return rot13(new String(Base64.decodeBase64(code)));
}
public static void removeUnwantedFromSearch(List<Entry> entries, List<FeedEntryKeyword> keywords) {
Iterator<Entry> it = entries.iterator();
while (it.hasNext()) {
Entry entry = it.next();
boolean keep = true;
for (FeedEntryKeyword keyword : keywords) {
String title = entry.getTitle() == null ? null : Jsoup.parse(entry.getTitle()).text();
String content = entry.getContent() == null ? null : Jsoup.parse(entry.getContent()).text();
boolean condition = !StringUtils.containsIgnoreCase(content, keyword.getKeyword())
&& !StringUtils.containsIgnoreCase(title, keyword.getKeyword());
if (keyword.getMode() == Mode.EXCLUDE) {
condition = !condition;
}
if (condition) {
keep = false;
break;
}
}
if (!keep) {
it.remove();
}
}
}
}

140
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/EncodingDetector.java
View File

@@ -1,70 +1,70 @@
package com.commafeed.backend.feed.parser;
import java.nio.charset.Charset;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
@Singleton
class EncodingDetector {
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public Charset getEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (StringUtils.startsWithIgnoreCase(extracted, "iso-8859-")) {
if (!StringUtils.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (StringUtils.startsWithIgnoreCase(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Extract the declared encoding from the xml
*/
public String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = StringUtils.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
/**
* Detect encoding by analyzing characters in the array
*/
private Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
}
package com.commafeed.backend.feed.parser;
import java.nio.charset.Charset;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import com.ibm.icu.text.CharsetDetector;
import com.ibm.icu.text.CharsetMatch;
@Singleton
class EncodingDetector {
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the characters used in the
* feed
*
*/
public Charset getEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);
if (StringUtils.startsWithIgnoreCase(extracted, "iso-8859-")) {
if (!StringUtils.endsWith(extracted, "1")) {
return Charset.forName(extracted);
}
} else if (StringUtils.startsWithIgnoreCase(extracted, "windows-")) {
return Charset.forName(extracted);
}
return detectEncoding(bytes);
}
/**
* Extract the declared encoding from the xml
*/
public String extractDeclaredEncoding(byte[] bytes) {
int index = ArrayUtils.indexOf(bytes, (byte) '>');
if (index == -1) {
return null;
}
String pi = new String(ArrayUtils.subarray(bytes, 0, index + 1)).replace('\'', '"');
index = StringUtils.indexOf(pi, "encoding=\"");
if (index == -1) {
return null;
}
String encoding = pi.substring(index + 10);
encoding = encoding.substring(0, encoding.indexOf('"'));
return encoding;
}
/**
* Detect encoding by analyzing characters in the array
*/
private Charset detectEncoding(byte[] bytes) {
String encoding = "UTF-8";
CharsetDetector detector = new CharsetDetector();
detector.setText(bytes);
CharsetMatch match = detector.detect();
if (match != null) {
encoding = match.getName();
}
if (encoding.equalsIgnoreCase("ISO-8859-1")) {
encoding = "windows-1252";
}
return Charset.forName(encoding);
}
}

140
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedCleaner.java
View File

@@ -1,70 +1,70 @@
package com.commafeed.backend.feed.parser;
import java.util.Collection;
import java.util.regex.Pattern;
import jakarta.inject.Singleton;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.apache.commons.lang3.StringUtils;
@Singleton
class FeedCleaner {
private static final Pattern DOCTYPE_PATTERN = Pattern.compile("<!DOCTYPE[^>]*>", Pattern.CASE_INSENSITIVE);
public String trimInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
StringBuilder sb = new StringBuilder();
boolean firstTagFound = false;
for (int i = 0; i < xml.length(); i++) {
char c = xml.charAt(i);
if (!firstTagFound) {
if (c == '<') {
firstTagFound = true;
} else {
continue;
}
}
if (c >= 32 || c == 9 || c == 10 || c == 13) {
if (!Character.isHighSurrogate(c) && !Character.isLowSurrogate(c)) {
sb.append(c);
}
}
}
return sb.toString();
}
// https://stackoverflow.com/a/40836618
public String replaceHtmlEntitiesWithNumericEntities(String source) {
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
Collection<Emit> emits = Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build().parseText(source);
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
return sb.toString();
}
public String removeDoctypeDeclarations(String xml) {
return DOCTYPE_PATTERN.matcher(xml).replaceAll("");
}
}
package com.commafeed.backend.feed.parser;
import java.util.Collection;
import java.util.regex.Pattern;
import jakarta.inject.Singleton;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.apache.commons.lang3.StringUtils;
@Singleton
class FeedCleaner {
private static final Pattern DOCTYPE_PATTERN = Pattern.compile("<!DOCTYPE[^>]*>", Pattern.CASE_INSENSITIVE);
public String trimInvalidXmlCharacters(String xml) {
if (StringUtils.isBlank(xml)) {
return null;
}
StringBuilder sb = new StringBuilder();
boolean firstTagFound = false;
for (int i = 0; i < xml.length(); i++) {
char c = xml.charAt(i);
if (!firstTagFound) {
if (c == '<') {
firstTagFound = true;
} else {
continue;
}
}
if (c >= 32 || c == 9 || c == 10 || c == 13) {
if (!Character.isHighSurrogate(c) && !Character.isLowSurrogate(c)) {
sb.append(c);
}
}
}
return sb.toString();
}
// https://stackoverflow.com/a/40836618
public String replaceHtmlEntitiesWithNumericEntities(String source) {
// Create a buffer sufficiently large that re-allocations are minimized.
StringBuilder sb = new StringBuilder(source.length() << 1);
Collection<Emit> emits = Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build().parseText(source);
int prevIndex = 0;
for (Emit emit : emits) {
int matchIndex = emit.getStart();
sb.append(source, prevIndex, matchIndex);
sb.append(HtmlEntities.HTML_TO_NUMERIC_MAP.get(emit.getKeyword()));
prevIndex = emit.getEnd() + 1;
}
// Add the remainder of the string (contains no more matches).
sb.append(source.substring(prevIndex));
return sb.toString();
}
public String removeDoctypeDeclarations(String xml) {
return DOCTYPE_PATTERN.matcher(xml).replaceAll("");
}
}

570
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParser.java
View File

@@ -1,285 +1,285 @@
package com.commafeed.backend.feed.parser;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.google.common.collect.Iterables;
import com.rometools.modules.mediarss.MediaEntryModule;
import com.rometools.modules.mediarss.MediaModule;
import com.rometools.modules.mediarss.types.MediaGroup;
import com.rometools.modules.mediarss.types.Metadata;
import com.rometools.modules.mediarss.types.Thumbnail;
import com.rometools.rome.feed.synd.SyndCategory;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.SyndFeedInput;
import lombok.RequiredArgsConstructor;
/**
* Parses raw xml into a FeedParserResult object
*/
@RequiredArgsConstructor
@Singleton
public class FeedParser {
private static final Namespace ATOM_10_NS = Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Instant START = Instant.ofEpochMilli(86400000);
private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
private final EncodingDetector encodingDetector;
private final FeedCleaner feedCleaner;
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedParsingException {
try {
Charset encoding = encodingDetector.getEncoding(xml);
String xmlString = feedCleaner.trimInvalidXmlCharacters(new String(xml, encoding));
if (xmlString == null) {
throw new FeedParsingException("Input string is null for url " + feedUrl);
}
xmlString = feedCleaner.replaceHtmlEntitiesWithNumericEntities(xmlString);
xmlString = feedCleaner.removeDoctypeDeclarations(xmlString);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed feed = new SyndFeedInput().build(source);
handleForeignMarkup(feed);
String title = feed.getTitle();
String link = feed.getLink();
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
if (lastPublishedDate == null || lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
lastPublishedDate = lastEntryDate;
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
return new FeedParserResult(title, link, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
} catch (FeedParsingException e) {
throw e;
} catch (Exception e) {
throw new FeedParsingException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
/**
* Adds atom links for rss feeds
*/
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
private List<Entry> buildEntries(SyndFeed feed, String feedUrl) {
List<Entry> entries = new ArrayList<>();
for (SyndEntry item : feed.getEntries()) {
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
String url = buildEntryUrl(feed, feedUrl, item);
if (StringUtils.isBlank(url) && FeedUtils.isAbsoluteUrl(guid)) {
// if link is empty but guid is used as url, use guid
url = guid;
}
Instant publishedDate = buildEntryPublishedDate(item);
Content content = buildContent(item);
entries.add(new Entry(guid, url, publishedDate, content));
}
entries.sort(Comparator.comparing(Entry::published).reversed());
return entries;
}
private Content buildContent(SyndEntry item) {
String title = getTitle(item);
String content = getContent(item);
String author = StringUtils.trimToNull(item.getAuthor());
String categories = StringUtils
.trimToNull(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")));
Enclosure enclosure = buildEnclosure(item);
Media media = buildMedia(item);
return new Content(title, content, author, categories, enclosure, media);
}
private Enclosure buildEnclosure(SyndEntry item) {
SyndEnclosure enclosure = Iterables.getFirst(item.getEnclosures(), null);
if (enclosure == null) {
return null;
}
return new Enclosure(enclosure.getUrl(), enclosure.getType());
}
private Instant buildEntryPublishedDate(SyndEntry item) {
Date date = item.getPublishedDate();
if (date == null) {
date = item.getUpdatedDate();
}
return toValidInstant(date, true);
}
private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
if (url == null || FeedUtils.isAbsoluteUrl(url)) {
// url is absolute, nothing to do
return url;
}
// url is relative, trying to resolve it
String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
return FeedUtils.toAbsoluteUrl(url, feedLink, feedUrl);
}
private Instant toValidInstant(Date date, boolean nullToNow) {
Instant now = Instant.now();
if (date == null) {
return nullToNow ? now : null;
}
Instant instant = date.toInstant();
if (instant.isBefore(START) || instant.isAfter(END)) {
return now;
}
if (instant.isAfter(now)) {
return now;
}
return instant;
}
private String getContent(SyndEntry item) {
String content;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private Media buildMedia(SyndEntry item) {
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module == null) {
return null;
}
Media media = buildMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = buildMedia(group.getMetadata());
}
return media;
}
private Media buildMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
String description = metadata.getDescription();
String thumbnailUrl = null;
Integer thumbnailWidth = null;
Integer thumbnailHeight = null;
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
thumbnailWidth = thumbnail.getWidth();
thumbnailHeight = thumbnail.getHeight();
if (thumbnail.getUrl() != null) {
thumbnailUrl = thumbnail.getUrl().toString();
}
}
if (description == null && thumbnailUrl == null) {
return null;
}
return new Media(description, thumbnailUrl, thumbnailWidth, thumbnailHeight);
}
private Long averageTimeBetweenEntries(List<Entry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < entries.size() - 1; i++) {
long diff = Math.abs(entries.get(i).published().toEpochMilli() - entries.get(i + 1).published().toEpochMilli());
stats.addValue(diff);
}
return (long) stats.getMean();
}
public static class FeedParsingException extends Exception {
private static final long serialVersionUID = 1L;
public FeedParsingException(String message) {
super(message);
}
public FeedParsingException(String message, Throwable cause) {
super(message, cause);
}
}
}
package com.commafeed.backend.feed.parser;
import java.io.StringReader;
import java.nio.charset.Charset;
import java.text.DateFormat;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.google.common.collect.Iterables;
import com.rometools.modules.mediarss.MediaEntryModule;
import com.rometools.modules.mediarss.MediaModule;
import com.rometools.modules.mediarss.types.MediaGroup;
import com.rometools.modules.mediarss.types.Metadata;
import com.rometools.modules.mediarss.types.Thumbnail;
import com.rometools.rome.feed.synd.SyndCategory;
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEnclosure;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.rome.feed.synd.SyndLink;
import com.rometools.rome.feed.synd.SyndLinkImpl;
import com.rometools.rome.io.SyndFeedInput;
import lombok.RequiredArgsConstructor;
/**
* Parses raw xml into a FeedParserResult object
*/
@RequiredArgsConstructor
@Singleton
public class FeedParser {
private static final Namespace ATOM_10_NS = Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Instant START = Instant.ofEpochMilli(86400000);
private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
private final EncodingDetector encodingDetector;
private final FeedCleaner feedCleaner;
public FeedParserResult parse(String feedUrl, byte[] xml) throws FeedParsingException {
try {
Charset encoding = encodingDetector.getEncoding(xml);
String xmlString = feedCleaner.trimInvalidXmlCharacters(new String(xml, encoding));
if (xmlString == null) {
throw new FeedParsingException("Input string is null for url " + feedUrl);
}
xmlString = feedCleaner.replaceHtmlEntitiesWithNumericEntities(xmlString);
xmlString = feedCleaner.removeDoctypeDeclarations(xmlString);
InputSource source = new InputSource(new StringReader(xmlString));
SyndFeed feed = new SyndFeedInput().build(source);
handleForeignMarkup(feed);
String title = feed.getTitle();
String link = feed.getLink();
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
if (lastPublishedDate == null || lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
lastPublishedDate = lastEntryDate;
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
return new FeedParserResult(title, link, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
} catch (FeedParsingException e) {
throw e;
} catch (Exception e) {
throw new FeedParsingException(String.format("Could not parse feed from %s : %s", feedUrl, e.getMessage()), e);
}
}
/**
* Adds atom links for rss feeds
*/
private void handleForeignMarkup(SyndFeed feed) {
List<Element> foreignMarkup = feed.getForeignMarkup();
if (foreignMarkup == null) {
return;
}
for (Element element : foreignMarkup) {
if ("link".equals(element.getName()) && ATOM_10_NS.equals(element.getNamespace())) {
SyndLink link = new SyndLinkImpl();
link.setRel(element.getAttributeValue("rel"));
link.setHref(element.getAttributeValue("href"));
feed.getLinks().add(link);
}
}
}
private List<Entry> buildEntries(SyndFeed feed, String feedUrl) {
List<Entry> entries = new ArrayList<>();
for (SyndEntry item : feed.getEntries()) {
String guid = item.getUri();
if (StringUtils.isBlank(guid)) {
guid = item.getLink();
}
if (StringUtils.isBlank(guid)) {
// no guid and no link, skip entry
continue;
}
String url = buildEntryUrl(feed, feedUrl, item);
if (StringUtils.isBlank(url) && FeedUtils.isAbsoluteUrl(guid)) {
// if link is empty but guid is used as url, use guid
url = guid;
}
Instant publishedDate = buildEntryPublishedDate(item);
Content content = buildContent(item);
entries.add(new Entry(guid, url, publishedDate, content));
}
entries.sort(Comparator.comparing(Entry::published).reversed());
return entries;
}
private Content buildContent(SyndEntry item) {
String title = getTitle(item);
String content = getContent(item);
String author = StringUtils.trimToNull(item.getAuthor());
String categories = StringUtils
.trimToNull(item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")));
Enclosure enclosure = buildEnclosure(item);
Media media = buildMedia(item);
return new Content(title, content, author, categories, enclosure, media);
}
private Enclosure buildEnclosure(SyndEntry item) {
SyndEnclosure enclosure = Iterables.getFirst(item.getEnclosures(), null);
if (enclosure == null) {
return null;
}
return new Enclosure(enclosure.getUrl(), enclosure.getType());
}
private Instant buildEntryPublishedDate(SyndEntry item) {
Date date = item.getPublishedDate();
if (date == null) {
date = item.getUpdatedDate();
}
return toValidInstant(date, true);
}
private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
if (url == null || FeedUtils.isAbsoluteUrl(url)) {
// url is absolute, nothing to do
return url;
}
// url is relative, trying to resolve it
String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
return FeedUtils.toAbsoluteUrl(url, feedLink, feedUrl);
}
private Instant toValidInstant(Date date, boolean nullToNow) {
Instant now = Instant.now();
if (date == null) {
return nullToNow ? now : null;
}
Instant instant = date.toInstant();
if (instant.isBefore(START) || instant.isAfter(END)) {
return now;
}
if (instant.isAfter(now)) {
return now;
}
return instant;
}
private String getContent(SyndEntry item) {
String content;
if (item.getContents().isEmpty()) {
content = item.getDescription() == null ? null : item.getDescription().getValue();
} else {
content = item.getContents().stream().map(SyndContent::getValue).collect(Collectors.joining(System.lineSeparator()));
}
return StringUtils.trimToNull(content);
}
private String getTitle(SyndEntry item) {
String title = item.getTitle();
if (StringUtils.isBlank(title)) {
Date date = item.getPublishedDate();
if (date != null) {
title = DateFormat.getInstance().format(date);
} else {
title = "(no title)";
}
}
return StringUtils.trimToNull(title);
}
private Media buildMedia(SyndEntry item) {
MediaEntryModule module = (MediaEntryModule) item.getModule(MediaModule.URI);
if (module == null) {
return null;
}
Media media = buildMedia(module.getMetadata());
if (media == null && ArrayUtils.isNotEmpty(module.getMediaGroups())) {
MediaGroup group = module.getMediaGroups()[0];
media = buildMedia(group.getMetadata());
}
return media;
}
private Media buildMedia(Metadata metadata) {
if (metadata == null) {
return null;
}
String description = metadata.getDescription();
String thumbnailUrl = null;
Integer thumbnailWidth = null;
Integer thumbnailHeight = null;
if (ArrayUtils.isNotEmpty(metadata.getThumbnail())) {
Thumbnail thumbnail = metadata.getThumbnail()[0];
thumbnailWidth = thumbnail.getWidth();
thumbnailHeight = thumbnail.getHeight();
if (thumbnail.getUrl() != null) {
thumbnailUrl = thumbnail.getUrl().toString();
}
}
if (description == null && thumbnailUrl == null) {
return null;
}
return new Media(description, thumbnailUrl, thumbnailWidth, thumbnailHeight);
}
private Long averageTimeBetweenEntries(List<Entry> entries) {
if (entries.isEmpty() || entries.size() == 1) {
return null;
}
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < entries.size() - 1; i++) {
long diff = Math.abs(entries.get(i).published().toEpochMilli() - entries.get(i + 1).published().toEpochMilli());
stats.addValue(diff);
}
return (long) stats.getMean();
}
public static class FeedParsingException extends Exception {
private static final long serialVersionUID = 1L;
public FeedParsingException(String message) {
super(message);
}
public FeedParsingException(String message, Throwable cause) {
super(message, cause);
}
}
}

40
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParserResult.java
View File

@@ -1,20 +1,20 @@
package com.commafeed.backend.feed.parser;
import java.time.Instant;
import java.util.List;
public record FeedParserResult(String title, String link, Instant lastPublishedDate, Long averageEntryInterval, Instant lastEntryDate,
List<Entry> entries) {
public record Entry(String guid, String url, Instant published, Content content) {
}
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {
}
public record Enclosure(String url, String type) {
}
public record Media(String description, String thumbnailUrl, Integer thumbnailWidth, Integer thumbnailHeight) {
}
}
package com.commafeed.backend.feed.parser;
import java.time.Instant;
import java.util.List;
public record FeedParserResult(String title, String link, Instant lastPublishedDate, Long averageEntryInterval, Instant lastEntryDate,
List<Entry> entries) {
public record Entry(String guid, String url, Instant published, Content content) {
}
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {
}
public record Enclosure(String url, String type) {
}
public record Media(String description, String thumbnailUrl, Integer thumbnailWidth, Integer thumbnailHeight) {
}
}

544
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/HtmlEntities.java
View File

@@ -1,272 +1,272 @@
package com.commafeed.backend.feed.parser;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import lombok.experimental.UtilityClass;
@UtilityClass
class HtmlEntities {
public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final String[] HTML_ENTITIES;
public static final String[] NUMERIC_ENTITIES;
static {
Map<String, String> map = new LinkedHashMap<>();
map.put("&Aacute;", "&#193;");
map.put("&aacute;", "&#225;");
map.put("&Acirc;", "&#194;");
map.put("&acirc;", "&#226;");
map.put("&acute;", "&#180;");
map.put("&AElig;", "&#198;");
map.put("&aelig;", "&#230;");
map.put("&Agrave;", "&#192;");
map.put("&agrave;", "&#224;");
map.put("&alefsym;", "&#8501;");
map.put("&Alpha;", "&#913;");
map.put("&alpha;", "&#945;");
map.put("&amp;", "&#38;");
map.put("&and;", "&#8743;");
map.put("&ang;", "&#8736;");
map.put("&Aring;", "&#197;");
map.put("&aring;", "&#229;");
map.put("&asymp;", "&#8776;");
map.put("&Atilde;", "&#195;");
map.put("&atilde;", "&#227;");
map.put("&Auml;", "&#196;");
map.put("&auml;", "&#228;");
map.put("&bdquo;", "&#8222;");
map.put("&Beta;", "&#914;");
map.put("&beta;", "&#946;");
map.put("&brvbar;", "&#166;");
map.put("&bull;", "&#8226;");
map.put("&cap;", "&#8745;");
map.put("&Ccedil;", "&#199;");
map.put("&ccedil;", "&#231;");
map.put("&cedil;", "&#184;");
map.put("&cent;", "&#162;");
map.put("&Chi;", "&#935;");
map.put("&chi;", "&#967;");
map.put("&circ;", "&#710;");
map.put("&clubs;", "&#9827;");
map.put("&cong;", "&#8773;");
map.put("&copy;", "&#169;");
map.put("&crarr;", "&#8629;");
map.put("&cup;", "&#8746;");
map.put("&curren;", "&#164;");
map.put("&dagger;", "&#8224;");
map.put("&Dagger;", "&#8225;");
map.put("&darr;", "&#8595;");
map.put("&dArr;", "&#8659;");
map.put("&deg;", "&#176;");
map.put("&Delta;", "&#916;");
map.put("&delta;", "&#948;");
map.put("&diams;", "&#9830;");
map.put("&divide;", "&#247;");
map.put("&Eacute;", "&#201;");
map.put("&eacute;", "&#233;");
map.put("&Ecirc;", "&#202;");
map.put("&ecirc;", "&#234;");
map.put("&Egrave;", "&#200;");
map.put("&egrave;", "&#232;");
map.put("&empty;", "&#8709;");
map.put("&emsp;", "&#8195;");
map.put("&ensp;", "&#8194;");
map.put("&Epsilon;", "&#917;");
map.put("&epsilon;", "&#949;");
map.put("&equiv;", "&#8801;");
map.put("&Eta;", "&#919;");
map.put("&eta;", "&#951;");
map.put("&ETH;", "&#208;");
map.put("&eth;", "&#240;");
map.put("&Euml;", "&#203;");
map.put("&euml;", "&#235;");
map.put("&euro;", "&#8364;");
map.put("&exist;", "&#8707;");
map.put("&fnof;", "&#402;");
map.put("&forall;", "&#8704;");
map.put("&frac12;", "&#189;");
map.put("&frac14;", "&#188;");
map.put("&frac34;", "&#190;");
map.put("&frasl;", "&#8260;");
map.put("&Gamma;", "&#915;");
map.put("&gamma;", "&#947;");
map.put("&ge;", "&#8805;");
map.put("&harr;", "&#8596;");
map.put("&hArr;", "&#8660;");
map.put("&hearts;", "&#9829;");
map.put("&hellip;", "&#8230;");
map.put("&Iacute;", "&#205;");
map.put("&iacute;", "&#237;");
map.put("&Icirc;", "&#206;");
map.put("&icirc;", "&#238;");
map.put("&iexcl;", "&#161;");
map.put("&Igrave;", "&#204;");
map.put("&igrave;", "&#236;");
map.put("&image;", "&#8465;");
map.put("&infin;", "&#8734;");
map.put("&int;", "&#8747;");
map.put("&Iota;", "&#921;");
map.put("&iota;", "&#953;");
map.put("&iquest;", "&#191;");
map.put("&isin;", "&#8712;");
map.put("&Iuml;", "&#207;");
map.put("&iuml;", "&#239;");
map.put("&Kappa;", "&#922;");
map.put("&kappa;", "&#954;");
map.put("&Lambda;", "&#923;");
map.put("&lambda;", "&#955;");
map.put("&lang;", "&#9001;");
map.put("&laquo;", "&#171;");
map.put("&larr;", "&#8592;");
map.put("&lArr;", "&#8656;");
map.put("&lceil;", "&#8968;");
map.put("&ldquo;", "&#8220;");
map.put("&le;", "&#8804;");
map.put("&lfloor;", "&#8970;");
map.put("&lowast;", "&#8727;");
map.put("&loz;", "&#9674;");
map.put("&lrm;", "&#8206;");
map.put("&lsaquo;", "&#8249;");
map.put("&lsquo;", "&#8216;");
map.put("&macr;", "&#175;");
map.put("&mdash;", "&#8212;");
map.put("&micro;", "&#181;");
map.put("&middot;", "&#183;");
map.put("&minus;", "&#8722;");
map.put("&Mu;", "&#924;");
map.put("&mu;", "&#956;");
map.put("&nabla;", "&#8711;");
map.put("&nbsp;", "&#160;");
map.put("&ndash;", "&#8211;");
map.put("&ne;", "&#8800;");
map.put("&ni;", "&#8715;");
map.put("&not;", "&#172;");
map.put("&notin;", "&#8713;");
map.put("&nsub;", "&#8836;");
map.put("&Ntilde;", "&#209;");
map.put("&ntilde;", "&#241;");
map.put("&Nu;", "&#925;");
map.put("&nu;", "&#957;");
map.put("&Oacute;", "&#211;");
map.put("&oacute;", "&#243;");
map.put("&Ocirc;", "&#212;");
map.put("&ocirc;", "&#244;");
map.put("&OElig;", "&#338;");
map.put("&oelig;", "&#339;");
map.put("&Ograve;", "&#210;");
map.put("&ograve;", "&#242;");
map.put("&oline;", "&#8254;");
map.put("&Omega;", "&#937;");
map.put("&omega;", "&#969;");
map.put("&Omicron;", "&#927;");
map.put("&omicron;", "&#959;");
map.put("&oplus;", "&#8853;");
map.put("&or;", "&#8744;");
map.put("&ordf;", "&#170;");
map.put("&ordm;", "&#186;");
map.put("&Oslash;", "&#216;");
map.put("&oslash;", "&#248;");
map.put("&Otilde;", "&#213;");
map.put("&otilde;", "&#245;");
map.put("&otimes;", "&#8855;");
map.put("&Ouml;", "&#214;");
map.put("&ouml;", "&#246;");
map.put("&para;", "&#182;");
map.put("&part;", "&#8706;");
map.put("&permil;", "&#8240;");
map.put("&perp;", "&#8869;");
map.put("&Phi;", "&#934;");
map.put("&phi;", "&#966;");
map.put("&Pi;", "&#928;");
map.put("&pi;", "&#960;");
map.put("&piv;", "&#982;");
map.put("&plusmn;", "&#177;");
map.put("&pound;", "&#163;");
map.put("&prime;", "&#8242;");
map.put("&Prime;", "&#8243;");
map.put("&prod;", "&#8719;");
map.put("&prop;", "&#8733;");
map.put("&Psi;", "&#936;");
map.put("&psi;", "&#968;");
map.put("&quot;", "&#34;");
map.put("&radic;", "&#8730;");
map.put("&rang;", "&#9002;");
map.put("&raquo;", "&#187;");
map.put("&rarr;", "&#8594;");
map.put("&rArr;", "&#8658;");
map.put("&rceil;", "&#8969;");
map.put("&rdquo;", "&#8221;");
map.put("&real;", "&#8476;");
map.put("&reg;", "&#174;");
map.put("&rfloor;", "&#8971;");
map.put("&Rho;", "&#929;");
map.put("&rho;", "&#961;");
map.put("&rlm;", "&#8207;");
map.put("&rsaquo;", "&#8250;");
map.put("&rsquo;", "&#8217;");
map.put("&sbquo;", "&#8218;");
map.put("&Scaron;", "&#352;");
map.put("&scaron;", "&#353;");
map.put("&sdot;", "&#8901;");
map.put("&sect;", "&#167;");
map.put("&shy;", "&#173;");
map.put("&Sigma;", "&#931;");
map.put("&sigma;", "&#963;");
map.put("&sigmaf;", "&#962;");
map.put("&sim;", "&#8764;");
map.put("&spades;", "&#9824;");
map.put("&sub;", "&#8834;");
map.put("&sube;", "&#8838;");
map.put("&sum;", "&#8721;");
map.put("&sup1;", "&#185;");
map.put("&sup2;", "&#178;");
map.put("&sup3;", "&#179;");
map.put("&sup;", "&#8835;");
map.put("&supe;", "&#8839;");
map.put("&szlig;", "&#223;");
map.put("&Tau;", "&#932;");
map.put("&tau;", "&#964;");
map.put("&there4;", "&#8756;");
map.put("&Theta;", "&#920;");
map.put("&theta;", "&#952;");
map.put("&thetasym;", "&#977;");
map.put("&thinsp;", "&#8201;");
map.put("&THORN;", "&#222;");
map.put("&thorn;", "&#254;");
map.put("&tilde;", "&#732;");
map.put("&times;", "&#215;");
map.put("&trade;", "&#8482;");
map.put("&Uacute;", "&#218;");
map.put("&uacute;", "&#250;");
map.put("&uarr;", "&#8593;");
map.put("&uArr;", "&#8657;");
map.put("&Ucirc;", "&#219;");
map.put("&ucirc;", "&#251;");
map.put("&Ugrave;", "&#217;");
map.put("&ugrave;", "&#249;");
map.put("&uml;", "&#168;");
map.put("&upsih;", "&#978;");
map.put("&Upsilon;", "&#933;");
map.put("&upsilon;", "&#965;");
map.put("&Uuml;", "&#220;");
map.put("&uuml;", "&#252;");
map.put("&weierp;", "&#8472;");
map.put("&Xi;", "&#926;");
map.put("&xi;", "&#958;");
map.put("&Yacute;", "&#221;");
map.put("&yacute;", "&#253;");
map.put("&yen;", "&#165;");
map.put("&yuml;", "&#255;");
map.put("&Yuml;", "&#376;");
map.put("&Zeta;", "&#918;");
map.put("&zeta;", "&#950;");
map.put("&zwj;", "&#8205;");
map.put("&zwnj;", "&#8204;");
HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map);
HTML_ENTITIES = map.keySet().toArray(new String[0]);
NUMERIC_ENTITIES = map.values().toArray(new String[0]);
}
}
package com.commafeed.backend.feed.parser;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import lombok.experimental.UtilityClass;
@UtilityClass
class HtmlEntities {
public static final Map<String, String> HTML_TO_NUMERIC_MAP;
public static final String[] HTML_ENTITIES;
public static final String[] NUMERIC_ENTITIES;
static {
Map<String, String> map = new LinkedHashMap<>();
map.put("&Aacute;", "&#193;");
map.put("&aacute;", "&#225;");
map.put("&Acirc;", "&#194;");
map.put("&acirc;", "&#226;");
map.put("&acute;", "&#180;");
map.put("&AElig;", "&#198;");
map.put("&aelig;", "&#230;");
map.put("&Agrave;", "&#192;");
map.put("&agrave;", "&#224;");
map.put("&alefsym;", "&#8501;");
map.put("&Alpha;", "&#913;");
map.put("&alpha;", "&#945;");
map.put("&amp;", "&#38;");
map.put("&and;", "&#8743;");
map.put("&ang;", "&#8736;");
map.put("&Aring;", "&#197;");
map.put("&aring;", "&#229;");
map.put("&asymp;", "&#8776;");
map.put("&Atilde;", "&#195;");
map.put("&atilde;", "&#227;");
map.put("&Auml;", "&#196;");
map.put("&auml;", "&#228;");
map.put("&bdquo;", "&#8222;");
map.put("&Beta;", "&#914;");
map.put("&beta;", "&#946;");
map.put("&brvbar;", "&#166;");
map.put("&bull;", "&#8226;");
map.put("&cap;", "&#8745;");
map.put("&Ccedil;", "&#199;");
map.put("&ccedil;", "&#231;");
map.put("&cedil;", "&#184;");
map.put("&cent;", "&#162;");
map.put("&Chi;", "&#935;");
map.put("&chi;", "&#967;");
map.put("&circ;", "&#710;");
map.put("&clubs;", "&#9827;");
map.put("&cong;", "&#8773;");
map.put("&copy;", "&#169;");
map.put("&crarr;", "&#8629;");
map.put("&cup;", "&#8746;");
map.put("&curren;", "&#164;");
map.put("&dagger;", "&#8224;");
map.put("&Dagger;", "&#8225;");
map.put("&darr;", "&#8595;");
map.put("&dArr;", "&#8659;");
map.put("&deg;", "&#176;");
map.put("&Delta;", "&#916;");
map.put("&delta;", "&#948;");
map.put("&diams;", "&#9830;");
map.put("&divide;", "&#247;");
map.put("&Eacute;", "&#201;");
map.put("&eacute;", "&#233;");
map.put("&Ecirc;", "&#202;");
map.put("&ecirc;", "&#234;");
map.put("&Egrave;", "&#200;");
map.put("&egrave;", "&#232;");
map.put("&empty;", "&#8709;");
map.put("&emsp;", "&#8195;");
map.put("&ensp;", "&#8194;");
map.put("&Epsilon;", "&#917;");
map.put("&epsilon;", "&#949;");
map.put("&equiv;", "&#8801;");
map.put("&Eta;", "&#919;");
map.put("&eta;", "&#951;");
map.put("&ETH;", "&#208;");
map.put("&eth;", "&#240;");
map.put("&Euml;", "&#203;");
map.put("&euml;", "&#235;");
map.put("&euro;", "&#8364;");
map.put("&exist;", "&#8707;");
map.put("&fnof;", "&#402;");
map.put("&forall;", "&#8704;");
map.put("&frac12;", "&#189;");
map.put("&frac14;", "&#188;");
map.put("&frac34;", "&#190;");
map.put("&frasl;", "&#8260;");
map.put("&Gamma;", "&#915;");
map.put("&gamma;", "&#947;");
map.put("&ge;", "&#8805;");
map.put("&harr;", "&#8596;");
map.put("&hArr;", "&#8660;");
map.put("&hearts;", "&#9829;");
map.put("&hellip;", "&#8230;");
map.put("&Iacute;", "&#205;");
map.put("&iacute;", "&#237;");
map.put("&Icirc;", "&#206;");
map.put("&icirc;", "&#238;");
map.put("&iexcl;", "&#161;");
map.put("&Igrave;", "&#204;");
map.put("&igrave;", "&#236;");
map.put("&image;", "&#8465;");
map.put("&infin;", "&#8734;");
map.put("&int;", "&#8747;");
map.put("&Iota;", "&#921;");
map.put("&iota;", "&#953;");
map.put("&iquest;", "&#191;");
map.put("&isin;", "&#8712;");
map.put("&Iuml;", "&#207;");
map.put("&iuml;", "&#239;");
map.put("&Kappa;", "&#922;");
map.put("&kappa;", "&#954;");
map.put("&Lambda;", "&#923;");
map.put("&lambda;", "&#955;");
map.put("&lang;", "&#9001;");
map.put("&laquo;", "&#171;");
map.put("&larr;", "&#8592;");
map.put("&lArr;", "&#8656;");
map.put("&lceil;", "&#8968;");
map.put("&ldquo;", "&#8220;");
map.put("&le;", "&#8804;");
map.put("&lfloor;", "&#8970;");
map.put("&lowast;", "&#8727;");
map.put("&loz;", "&#9674;");
map.put("&lrm;", "&#8206;");
map.put("&lsaquo;", "&#8249;");
map.put("&lsquo;", "&#8216;");
map.put("&macr;", "&#175;");
map.put("&mdash;", "&#8212;");
map.put("&micro;", "&#181;");
map.put("&middot;", "&#183;");
map.put("&minus;", "&#8722;");
map.put("&Mu;", "&#924;");
map.put("&mu;", "&#956;");
map.put("&nabla;", "&#8711;");
map.put("&nbsp;", "&#160;");
map.put("&ndash;", "&#8211;");
map.put("&ne;", "&#8800;");
map.put("&ni;", "&#8715;");
map.put("&not;", "&#172;");
map.put("&notin;", "&#8713;");
map.put("&nsub;", "&#8836;");
map.put("&Ntilde;", "&#209;");
map.put("&ntilde;", "&#241;");
map.put("&Nu;", "&#925;");
map.put("&nu;", "&#957;");
map.put("&Oacute;", "&#211;");
map.put("&oacute;", "&#243;");
map.put("&Ocirc;", "&#212;");
map.put("&ocirc;", "&#244;");
map.put("&OElig;", "&#338;");
map.put("&oelig;", "&#339;");
map.put("&Ograve;", "&#210;");
map.put("&ograve;", "&#242;");
map.put("&oline;", "&#8254;");
map.put("&Omega;", "&#937;");
map.put("&omega;", "&#969;");
map.put("&Omicron;", "&#927;");
map.put("&omicron;", "&#959;");
map.put("&oplus;", "&#8853;");
map.put("&or;", "&#8744;");
map.put("&ordf;", "&#170;");
map.put("&ordm;", "&#186;");
map.put("&Oslash;", "&#216;");
map.put("&oslash;", "&#248;");
map.put("&Otilde;", "&#213;");
map.put("&otilde;", "&#245;");
map.put("&otimes;", "&#8855;");
map.put("&Ouml;", "&#214;");
map.put("&ouml;", "&#246;");
map.put("&para;", "&#182;");
map.put("&part;", "&#8706;");
map.put("&permil;", "&#8240;");
map.put("&perp;", "&#8869;");
map.put("&Phi;", "&#934;");
map.put("&phi;", "&#966;");
map.put("&Pi;", "&#928;");
map.put("&pi;", "&#960;");
map.put("&piv;", "&#982;");
map.put("&plusmn;", "&#177;");
map.put("&pound;", "&#163;");
map.put("&prime;", "&#8242;");
map.put("&Prime;", "&#8243;");
map.put("&prod;", "&#8719;");
map.put("&prop;", "&#8733;");
map.put("&Psi;", "&#936;");
map.put("&psi;", "&#968;");
map.put("&quot;", "&#34;");
map.put("&radic;", "&#8730;");
map.put("&rang;", "&#9002;");
map.put("&raquo;", "&#187;");
map.put("&rarr;", "&#8594;");
map.put("&rArr;", "&#8658;");
map.put("&rceil;", "&#8969;");
map.put("&rdquo;", "&#8221;");
map.put("&real;", "&#8476;");
map.put("&reg;", "&#174;");
map.put("&rfloor;", "&#8971;");
map.put("&Rho;", "&#929;");
map.put("&rho;", "&#961;");
map.put("&rlm;", "&#8207;");
map.put("&rsaquo;", "&#8250;");
map.put("&rsquo;", "&#8217;");
map.put("&sbquo;", "&#8218;");
map.put("&Scaron;", "&#352;");
map.put("&scaron;", "&#353;");
map.put("&sdot;", "&#8901;");
map.put("&sect;", "&#167;");
map.put("&shy;", "&#173;");
map.put("&Sigma;", "&#931;");
map.put("&sigma;", "&#963;");
map.put("&sigmaf;", "&#962;");
map.put("&sim;", "&#8764;");
map.put("&spades;", "&#9824;");
map.put("&sub;", "&#8834;");
map.put("&sube;", "&#8838;");
map.put("&sum;", "&#8721;");
map.put("&sup1;", "&#185;");
map.put("&sup2;", "&#178;");
map.put("&sup3;", "&#179;");
map.put("&sup;", "&#8835;");
map.put("&supe;", "&#8839;");
map.put("&szlig;", "&#223;");
map.put("&Tau;", "&#932;");
map.put("&tau;", "&#964;");
map.put("&there4;", "&#8756;");
map.put("&Theta;", "&#920;");
map.put("&theta;", "&#952;");
map.put("&thetasym;", "&#977;");
map.put("&thinsp;", "&#8201;");
map.put("&THORN;", "&#222;");
map.put("&thorn;", "&#254;");
map.put("&tilde;", "&#732;");
map.put("&times;", "&#215;");
map.put("&trade;", "&#8482;");
map.put("&Uacute;", "&#218;");
map.put("&uacute;", "&#250;");
map.put("&uarr;", "&#8593;");
map.put("&uArr;", "&#8657;");
map.put("&Ucirc;", "&#219;");
map.put("&ucirc;", "&#251;");
map.put("&Ugrave;", "&#217;");
map.put("&ugrave;", "&#249;");
map.put("&uml;", "&#168;");
map.put("&upsih;", "&#978;");
map.put("&Upsilon;", "&#933;");
map.put("&upsilon;", "&#965;");
map.put("&Uuml;", "&#220;");
map.put("&uuml;", "&#252;");
map.put("&weierp;", "&#8472;");
map.put("&Xi;", "&#926;");
map.put("&xi;", "&#958;");
map.put("&Yacute;", "&#221;");
map.put("&yacute;", "&#253;");
map.put("&yen;", "&#165;");
map.put("&yuml;", "&#255;");
map.put("&Yuml;", "&#376;");
map.put("&Zeta;", "&#918;");
map.put("&zeta;", "&#950;");
map.put("&zwj;", "&#8205;");
map.put("&zwnj;", "&#8204;");
HTML_TO_NUMERIC_MAP = Collections.unmodifiableMap(map);
HTML_ENTITIES = map.keySet().toArray(new String[0]);
NUMERIC_ENTITIES = map.values().toArray(new String[0]);
}
}

112
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/TextDirectionDetector.java
View File

@@ -1,56 +1,56 @@
package com.commafeed.backend.feed.parser;
import java.text.Bidi;
import java.util.regex.Pattern;
import org.apache.commons.lang3.math.NumberUtils;
public class TextDirectionDetector {
private static final Pattern WORDS_PATTERN = Pattern.compile("\\s+");
private static final Pattern URL_PATTERN = Pattern.compile("^https?://.*");
private static final double RTL_THRESHOLD = 0.4D;
public enum Direction {
LEFT_TO_RIGHT, RIGHT_TO_LEFT
}
public static Direction detect(String input) {
if (input == null || input.isBlank()) {
return Direction.LEFT_TO_RIGHT;
}
long rtl = 0;
long total = 0;
for (String token : WORDS_PATTERN.split(input)) {
// skip urls
if (URL_PATTERN.matcher(token).matches()) {
continue;
}
// skip numbers
if (NumberUtils.isCreatable(token)) {
continue;
}
boolean requiresBidi = Bidi.requiresBidi(token.toCharArray(), 0, token.length());
if (requiresBidi) {
Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT);
if (bidi.getBaseLevel() == 1) {
rtl++;
}
}
total++;
}
if (total == 0) {
return Direction.LEFT_TO_RIGHT;
}
double ratio = (double) rtl / total;
return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
}
}
package com.commafeed.backend.feed.parser;
import java.text.Bidi;
import java.util.regex.Pattern;
import org.apache.commons.lang3.math.NumberUtils;
public class TextDirectionDetector {
private static final Pattern WORDS_PATTERN = Pattern.compile("\\s+");
private static final Pattern URL_PATTERN = Pattern.compile("^https?://.*");
private static final double RTL_THRESHOLD = 0.4D;
public enum Direction {
LEFT_TO_RIGHT, RIGHT_TO_LEFT
}
public static Direction detect(String input) {
if (input == null || input.isBlank()) {
return Direction.LEFT_TO_RIGHT;
}
long rtl = 0;
long total = 0;
for (String token : WORDS_PATTERN.split(input)) {
// skip urls
if (URL_PATTERN.matcher(token).matches()) {
continue;
}
// skip numbers
if (NumberUtils.isCreatable(token)) {
continue;
}
boolean requiresBidi = Bidi.requiresBidi(token.toCharArray(), 0, token.length());
if (requiresBidi) {
Bidi bidi = new Bidi(token, Bidi.DIRECTION_DEFAULT_LEFT_TO_RIGHT);
if (bidi.getBaseLevel() == 1) {
rtl++;
}
}
total++;
}
if (total == 0) {
return Direction.LEFT_TO_RIGHT;
}
double ratio = (double) rtl / total;
return ratio > RTL_THRESHOLD ? Direction.RIGHT_TO_LEFT : Direction.LEFT_TO_RIGHT;
}
}