Archives/Athou_commafeed
1
0
Fork 0
mirror of https://github.com/Athou/commafeed.git synced 2026-11-06 04:16:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

use 120 max line length for both server and client code (was 100 and 140)

Browse Source
This commit is contained in:
Athou
2026-11-03 06:48:11 +01:00
parent 29987a460d
commit 69cf674a02
157 changed files with 1228 additions and 1958 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
commafeed-server/src/main/java/com/commafeed/backend/HttpClientFactory.java
View File

@@ -56,29 +56,24 @@ public class HttpClientFactory {
private static final DnsResolver DNS_RESOLVER = SystemDefaultDnsResolver.INSTANCE;
private static final IPAddress CGNAT_RANGE = new IPAddressString("100.64.0.0/10").getAddress();
private static final IPAddress NAT64_LOCAL_USE_RANGE =
new IPAddressString("64:ff9b:1::/48").getAddress();
private static final IPAddress NAT64_LOCAL_USE_RANGE = new IPAddressString("64:ff9b:1::/48").getAddress();
private final CommaFeedConfiguration config;
private final CommaFeedVersion version;
public CloseableHttpAsyncClient newClient(int poolSize) {
PoolingAsyncClientConnectionManager connectionManager =
newConnectionManager(config, poolSize);
PoolingAsyncClientConnectionManager connectionManager = newConnectionManager(config, poolSize);
String userAgent =
config.httpClient()
.userAgent()
.orElseGet(() ->
String.format(
"CommaFeed/%s (https://github.com/Athou/commafeed)",
version.getVersion()));
"CommaFeed/%s (https://github.com/Athou/commafeed)", version.getVersion()));
return newClient(config, connectionManager, userAgent);
}
private CloseableHttpAsyncClient newClient(
CommaFeedConfiguration config,
PoolingAsyncClientConnectionManager connectionManager,
String userAgent) {
CommaFeedConfiguration config, PoolingAsyncClientConnectionManager connectionManager, String userAgent) {
List<Header> headers = new ArrayList<>();
headers.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "en"));
headers.add(new BasicHeader(HttpHeaders.PRAGMA, "No-cache"));
@@ -86,10 +81,8 @@ public class HttpClientFactory {
headers.add(new BasicHeader(HttpHeaders.ACCEPT, "*/*"));
headers.add(new BasicHeader(HttpHeaders.SEC_FETCH_MODE, "navigate"));
SequencedMap<String, UnaryOperator<AsyncDataConsumer>> contentDecoderMap =
new LinkedHashMap<>();
contentDecoderMap.put(
ContentCoding.DEFLATE.token(), d -> new InflatingAsyncDataConsumer(d, null));
SequencedMap<String, UnaryOperator<AsyncDataConsumer>> contentDecoderMap = new LinkedHashMap<>();
contentDecoderMap.put(ContentCoding.DEFLATE.token(), d -> new InflatingAsyncDataConsumer(d, null));
contentDecoderMap.put(ContentCoding.GZIP.token(), InflatingGzipDataConsumer::new);
RedirectStrategy redirectStrategy =
@@ -107,8 +100,7 @@ public class HttpClientFactory {
.setDefaultHeaders(headers)
.setConnectionManager(connectionManager)
.evictExpiredConnections()
.evictIdleConnections(
TimeValue.of(config.httpClient().idleConnectionsEvictionInterval()))
.evictIdleConnections(TimeValue.of(config.httpClient().idleConnectionsEvictionInterval()))
.setContentDecoderMap(new LinkedHashMap<>(contentDecoderMap))
.setRedirectStrategy(redirectStrategy)
.build();
@@ -116,10 +108,8 @@ public class HttpClientFactory {
return client;
}
private PoolingAsyncClientConnectionManager newConnectionManager(
CommaFeedConfiguration config, int poolSize) {
SSLFactory sslFactory =
SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
private PoolingAsyncClientConnectionManager newConnectionManager(CommaFeedConfiguration config, int poolSize) {
SSLFactory sslFactory = SSLFactory.builder().withUnsafeTrustMaterial().withUnsafeHostnameVerifier().build();
DnsResolver dnsResolver =
config.httpClient().blockLocalAddresses()
? new BlockLocalAddressesDnsResolver(DNS_RESOLVER)
@@ -131,14 +121,12 @@ public class HttpClientFactory {
ConnectionConfig.custom()
.setConnectTimeout(Timeout.of(config.httpClient().connectTimeout()))
.setSocketTimeout(Timeout.of(config.httpClient().socketTimeout()))
.setTimeToLive(
Timeout.of(config.httpClient().connectionTimeToLive()))
.setTimeToLive(Timeout.of(config.httpClient().connectionTimeToLive()))
.build())
.setDefaultTlsConfig(
TlsConfig.custom()
.setVersionPolicy(HttpVersionPolicy.NEGOTIATE)
.setHandshakeTimeout(
Timeout.of(config.httpClient().sslHandshakeTimeout()))
.setHandshakeTimeout(Timeout.of(config.httpClient().sslHandshakeTimeout()))
.build())
.setMaxConnPerRoute(poolSize)
.setMaxConnTotal(poolSize)
@@ -191,8 +179,7 @@ public class HttpClientFactory {
InetAddress[] addresses = delegate.resolve(host);
for (InetAddress addr : addresses) {
if (isLocalAddress(addr)) {
throw new UnknownHostException(
"Access to local address blocked: " + addr.getHostAddress());
throw new UnknownHostException("Access to local address blocked: " + addr.getHostAddress());
}
}
return addresses;
@@ -228,8 +215,7 @@ public class HttpClientFactory {
for (InetAddress addr : addresses) {
if (isLocalAddress(addr)) {
throw new HttpException(
"Access to local address blocked: " + addr.getHostAddress());
throw new HttpException("Access to local address blocked: " + addr.getHostAddress());
}
}

34
commafeed-server/src/main/java/com/commafeed/backend/HttpGetter.java
View File

@@ -75,32 +75,22 @@ public class HttpGetter {
this.client = httpClientFactory.newClient(config.feedRefresh().httpThreads());
this.cache = newCache(config);
metrics.registerGauge(
MetricRegistry.name(getClass(), "cache", "size"),
() -> cache == null ? 0 : cache.size());
metrics.registerGauge(MetricRegistry.name(getClass(), "cache", "size"), () -> cache == null ? 0 : cache.size());
metrics.registerGauge(
MetricRegistry.name(getClass(), "cache", "memoryUsage"),
() ->
cache == null
? 0
: cache.asMap().values().stream()
.mapToInt(e -> ArrayUtils.getLength(e.content))
.sum());
: cache.asMap().values().stream().mapToInt(e -> ArrayUtils.getLength(e.content)).sum());
}
public HttpResult get(String url)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
return get(HttpRequest.builder(url).build());
}
public HttpResult get(HttpRequest request)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
URI uri = URI.create(request.getUrl());
ensureHttpScheme(uri.getScheme());
@@ -178,8 +168,7 @@ public class HttpGetter {
FutureCallback<Message<org.apache.hc.core5.http.HttpResponse, byte[]>> callback =
new FutureCallback<>() {
@Override
public void completed(
Message<org.apache.hc.core5.http.HttpResponse, byte[]> result) {}
public void completed(Message<org.apache.hc.core5.http.HttpResponse, byte[]> result) {}
@Override
public void failed(Exception ex) {}
@@ -283,15 +272,13 @@ public class HttpGetter {
}
return CacheBuilder.newBuilder()
.weigher((HttpRequest key, HttpResponse value) ->
value.content() != null ? value.content().length : 0)
.weigher((HttpRequest key, HttpResponse value) -> value.content() != null ? value.content().length : 0)
.maximumWeight(cacheConfig.maximumMemorySize().asLongValue())
.expireAfterWrite(cacheConfig.expiration())
.build();
}
private record LimitedContentLengthEntityConsumer(
@Delegate AsyncEntityConsumer<byte[]> delegate, long maxBytes)
private record LimitedContentLengthEntityConsumer(@Delegate AsyncEntityConsumer<byte[]> delegate, long maxBytes)
implements AsyncEntityConsumer<byte[]> {
@Override
@@ -324,9 +311,7 @@ public class HttpGetter {
@Override
protected void data(ByteBuffer src, boolean endOfStream) throws IOException {
if (out.size() + src.remaining() > maxBytes) {
throw new IOException(
"Response size exceeds the maximum allowed size (%s bytes)"
.formatted(maxBytes));
throw new IOException("Response size exceeds the maximum allowed size (%s bytes)".formatted(maxBytes));
}
byte[] chunk = new byte[src.remaining()];
src.get(chunk);
@@ -366,8 +351,7 @@ public class HttpGetter {
this(message, null, null);
}
public NotModifiedException(
String message, String newLastModifiedHeader, String newEtagHeader) {
public NotModifiedException(String message, String newLastModifiedHeader, String newEtagHeader) {
super(message);
this.newLastModifiedHeader = newLastModifiedHeader;
this.newEtagHeader = newEtagHeader;

10
commafeed-server/src/main/java/com/commafeed/backend/Urls.java
View File

@@ -65,11 +65,7 @@ public class Urls {
try {
return URI.create(baseUrl).resolve(relativeUrl).toString();
} catch (IllegalArgumentException e) {
log.debug(
"Unable to create absolute url from relative url: {} base: {}",
relativeUrl,
baseUrl,
e);
log.debug("Unable to create absolute url from relative url: {} base: {}", relativeUrl, baseUrl, e);
return null;
}
}
@@ -86,8 +82,8 @@ public class Urls {
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to
* identify a feed and avoid duplicates
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and
* avoid duplicates
*/
public static String normalize(String url) {
if (url == null) {

4
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedCategoryDAO.java
View File

@@ -31,9 +31,7 @@ public class FeedCategoryDAO extends GenericDAO<FeedCategory> {
}
public FeedCategory findById(User user, Long id) {
return query().selectFrom(CATEGORY)
.where(CATEGORY.user.eq(user), CATEGORY.id.eq(id))
.fetchOne();
return query().selectFrom(CATEGORY).where(CATEGORY.user.eq(user), CATEGORY.id.eq(id)).fetchOne();
}
public FeedCategory findByName(User user, String name, FeedCategory parent) {

11
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedDAO.java
View File

@@ -36,10 +36,7 @@ public class FeedDAO extends GenericDAO<Feed> {
// join on subscriptions to only refresh feeds that have subscribers
.join(SUBSCRIPTION)
.on(SUBSCRIPTION.feed.eq(FEED))
.where(
FEED.disabledUntil
.isNull()
.or(FEED.disabledUntil.lt(Instant.now())));
.where(FEED.disabledUntil.isNull().or(FEED.disabledUntil.lt(Instant.now())));
if (lastLoginThreshold != null) {
query.join(SUBSCRIPTION.user).where(SUBSCRIPTION.user.lastLogin.gt(lastLoginThreshold));
@@ -53,11 +50,7 @@ public class FeedDAO extends GenericDAO<Feed> {
}
public Feed findByUrl(String normalizedUrl, String normalizedUrlHash) {
return query()
.selectFrom(FEED)
.where(FEED.normalizedUrlHash.eq(normalizedUrlHash))
.fetch()
.stream()
return query().selectFrom(FEED).where(FEED.normalizedUrlHash.eq(normalizedUrlHash)).fetch().stream()
.filter(f -> Strings.CS.equals(normalizedUrl, f.getNormalizedUrl()))
.findFirst()
.orElse(null);

14
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryDAO.java
View File

@@ -42,8 +42,7 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
}
Set<String> result = new HashSet<>();
for (List<String> batch :
Lists.partition(new ArrayList<>(guidHashes), IN_CLAUSE_BATCH_SIZE)) {
for (List<String> batch : Lists.partition(new ArrayList<>(guidHashes), IN_CLAUSE_BATCH_SIZE)) {
result.addAll(
query().select(ENTRY.guidHash)
.from(ENTRY)
@@ -53,8 +52,7 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
return result;
}
public List<FeedCapacity> findFeedsExceedingCapacity(
long maxCapacity, long max, boolean keepStarredEntries) {
public List<FeedCapacity> findFeedsExceedingCapacity(long maxCapacity, long max, boolean keepStarredEntries) {
NumberExpression<Long> count = ENTRY.id.count();
JPAQuery<Tuple> query = query().select(ENTRY.feed.id, count).from(ENTRY);
@@ -68,8 +66,7 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
}
public int delete(Long feedId, long max) {
List<FeedEntry> list =
query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).limit(max).fetch();
List<FeedEntry> list = query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).limit(max).fetch();
return delete(list);
}
@@ -91,10 +88,7 @@ public class FeedEntryDAO extends GenericDAO<FeedEntry> {
/** Delete the oldest entries of a feed */
public int deleteOldEntries(Long feedId, long max, boolean keepStarredEntries) {
JPAQuery<FeedEntry> query =
query().selectFrom(ENTRY)
.where(ENTRY.feed.id.eq(feedId))
.orderBy(ENTRY.published.asc())
.limit(max);
query().selectFrom(ENTRY).where(ENTRY.feed.id.eq(feedId)).orderBy(ENTRY.published.asc()).limit(max);
if (keepStarredEntries) {
query.where(Predicates.isNotStarred(ENTRY));

34
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryStatusDAO.java
View File

@@ -48,9 +48,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
private final CommaFeedConfiguration config;
public FeedEntryStatusDAO(
EntityManager entityManager,
FeedEntryTagDAO feedEntryTagDAO,
CommaFeedConfiguration config) {
EntityManager entityManager, FeedEntryTagDAO feedEntryTagDAO, CommaFeedConfiguration config) {
super(entityManager, FeedEntryStatus.class, config);
this.feedEntryTagDAO = feedEntryTagDAO;
this.config = config;
@@ -58,22 +56,16 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
public FeedEntryStatus getStatus(User user, FeedSubscription sub, FeedEntry entry) {
List<FeedEntryStatus> statuses =
query().selectFrom(STATUS)
.where(STATUS.entry.eq(entry), STATUS.subscription.eq(sub))
.fetch();
query().selectFrom(STATUS).where(STATUS.entry.eq(entry), STATUS.subscription.eq(sub)).fetch();
FeedEntryStatus status = statuses.stream().findFirst().orElse(null);
return handleStatus(user, status, sub, entry);
}
/** creates an artificial "unread" status if status is null */
private FeedEntryStatus handleStatus(
User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
private FeedEntryStatus handleStatus(User user, FeedEntryStatus status, FeedSubscription sub, FeedEntry entry) {
if (status == null) {
Instant statusesInstantThreshold =
config.database().cleanup().statusesInstantThreshold();
boolean read =
statusesInstantThreshold != null
&& entry.getPublished().isBefore(statusesInstantThreshold);
Instant statusesInstantThreshold = config.database().cleanup().statusesInstantThreshold();
boolean read = statusesInstantThreshold != null && entry.getPublished().isBefore(statusesInstantThreshold);
status = new FeedEntryStatus(user, sub, entry);
status.setRead(read);
status.setMarkable(!read);
@@ -85,8 +77,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
private void fetchTags(User user, List<FeedEntryStatus> statuses) {
Map<Long, List<FeedEntryTag>> tagsByEntryIds =
feedEntryTagDAO.findByEntries(
user, statuses.stream().map(FeedEntryStatus::getEntry).toList());
feedEntryTagDAO.findByEntries(user, statuses.stream().map(FeedEntryStatus::getEntry).toList());
for (FeedEntryStatus status : statuses) {
List<FeedEntryTag> tags = tagsByEntryIds.get(status.getEntry().getId());
status.setTags(tags == null ? List.of() : tags);
@@ -258,9 +249,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
Long count = tuple.get(ENTRY.count());
Instant published = tuple.get(ENTRY.published.max());
for (FeedSubscription sub : subsByFeedId.get(feedId)) {
result.put(
sub.getId(),
new UnreadCount(sub.getId(), count == null ? 0 : count, published));
result.put(sub.getId(), new UnreadCount(sub.getId(), count == null ? 0 : count, published));
}
}
@@ -303,8 +292,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
where.and(SUBSCRIPTION.autoMarkAsReadAfterDays.gt(0));
NumberExpression<Integer> daysDiff =
Expressions.numberTemplate(
Integer.class, "TIMESTAMPDIFF(DAY, {0}, {1})", ENTRY.published, now);
Expressions.numberTemplate(Integer.class, "TIMESTAMPDIFF(DAY, {0}, {1})", ENTRY.published, now);
where.and(daysDiff.goe(SUBSCRIPTION.autoMarkAsReadAfterDays));
where.and(buildUnreadPredicate());
@@ -333,11 +321,7 @@ public class FeedEntryStatusDAO extends GenericDAO<FeedEntryStatus> {
.toList();
if (!statusIdsToUpdate.isEmpty()) {
updated +=
updateQuery(STATUS)
.where(STATUS.id.in(statusIdsToUpdate))
.set(STATUS.read, true)
.execute();
updated += updateQuery(STATUS).where(STATUS.id.in(statusIdsToUpdate)).set(STATUS.read, true).execute();
}
// Insert new statuses for entries without existing status

6
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedEntryTagDAO.java
View File

@@ -31,11 +31,7 @@ public class FeedEntryTagDAO extends GenericDAO<FeedEntryTag> {
}
public Map<Long, List<FeedEntryTag>> findByEntries(User user, List<FeedEntry> entries) {
return query()
.selectFrom(TAG)
.where(TAG.user.eq(user), TAG.entry.in(entries))
.fetch()
.stream()
return query().selectFrom(TAG).where(TAG.user.eq(user), TAG.entry.in(entries)).fetch().stream()
.collect(Collectors.groupingBy(t -> t.getEntry().getId()));
}
}

18
commafeed-server/src/main/java/com/commafeed/backend/dao/FeedSubscriptionDAO.java
View File

@@ -84,9 +84,7 @@ public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> {
public FeedSubscription findByFeed(User user, Feed feed) {
List<FeedSubscription> subs =
query().selectFrom(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.feed.eq(feed))
.fetch();
query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user), SUBSCRIPTION.feed.eq(feed)).fetch();
FeedSubscription sub = subs.stream().findFirst().orElse(null);
return initRelations(sub);
}
@@ -104,15 +102,11 @@ public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> {
}
public Long count(User user) {
return query().select(SUBSCRIPTION.count())
.from(SUBSCRIPTION)
.where(SUBSCRIPTION.user.eq(user))
.fetchOne();
return query().select(SUBSCRIPTION.count()).from(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user)).fetchOne();
}
public List<FeedSubscription> findByCategory(User user, FeedCategory category) {
JPQLQuery<FeedSubscription> query =
query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user));
JPQLQuery<FeedSubscription> query = query().selectFrom(SUBSCRIPTION).where(SUBSCRIPTION.user.eq(user));
if (category == null) {
query.where(SUBSCRIPTION.category.isNull());
} else {
@@ -122,11 +116,9 @@ public class FeedSubscriptionDAO extends GenericDAO<FeedSubscription> {
}
public List<FeedSubscription> findByCategories(User user, List<FeedCategory> categories) {
Set<Long> categoryIds =
categories.stream().map(AbstractModel::getId).collect(Collectors.toSet());
Set<Long> categoryIds = categories.stream().map(AbstractModel::getId).collect(Collectors.toSet());
return findAll(user).stream()
.filter(s ->
s.getCategory() != null && categoryIds.contains(s.getCategory().getId()))
.filter(s -> s.getCategory() != null && categoryIds.contains(s.getCategory().getId()))
.toList();
}

4
commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
View File

@@ -37,9 +37,7 @@ public class FacebookFaviconFetcher implements FaviconFetcher {
return null;
}
String iconUrl =
String.format(
"https://graph.facebook.com/%s/picture?type=square&height=16", userName);
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
try {
log.debug("Getting Facebook user's icon, {}", url);

3
commafeed-server/src/main/java/com/commafeed/backend/favicon/Favicon.java
View File

@@ -17,8 +17,7 @@ public record Favicon(byte[] icon, MediaType mediaType) {
try {
return MediaType.valueOf(contentType);
} catch (Exception e) {
log.debug(
"invalid content type '{}' received, returning default value", contentType, e);
log.debug("invalid content type '{}' received, returning default value", contentType, e);
return DEFAULT_MEDIA_TYPE;
}
}

5
commafeed-server/src/main/java/com/commafeed/backend/favicon/RootFaviconFetcher.java
View File

@@ -32,10 +32,7 @@ public class RootFaviconFetcher implements FaviconFetcher {
URI uri = URI.create(url.trim());
String faviconUrl =
"%s://%s%s/favicon.ico"
.formatted(
uri.getScheme(),
uri.getHost(),
uri.getPort() > 0 ? ":" + uri.getPort() : "");
.formatted(uri.getScheme(), uri.getHost(), uri.getPort() > 0 ? ":" + uri.getPort() : "");
log.debug("getting root icon at {}", faviconUrl);
HttpResult result = getter.get(faviconUrl);

30
commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
View File

@@ -37,8 +37,7 @@ public class YoutubeFaviconFetcher implements FaviconFetcher {
private static final JsonPointer CHANNEL_THUMBNAIL_URL =
JsonPointer.compile("/items/0/snippet/thumbnails/default/url");
private static final JsonPointer PLAYLIST_CHANNEL_ID =
JsonPointer.compile("/items/0/snippet/channelId");
private static final JsonPointer PLAYLIST_CHANNEL_ID = JsonPointer.compile("/items/0/snippet/channelId");
private final HttpGetter getter;
private final CommaFeedConfiguration config;
@@ -60,17 +59,11 @@ public class YoutubeFaviconFetcher implements FaviconFetcher {
try {
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
Optional<NameValuePair> userId =
params.stream()
.filter(nvp -> nvp.getName().equalsIgnoreCase("user"))
.findFirst();
params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
Optional<NameValuePair> channelId =
params.stream()
.filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id"))
.findFirst();
params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id")).findFirst();
Optional<NameValuePair> playlistId =
params.stream()
.filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id"))
.findFirst();
params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id")).findFirst();
byte[] response = null;
if (userId.isPresent()) {
@@ -103,10 +96,7 @@ public class YoutubeFaviconFetcher implements FaviconFetcher {
}
private byte[] fetchForUser(String googleAuthKey, String userId)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
URI uri =
UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", PART_SNIPPET)
@@ -117,10 +107,7 @@ public class YoutubeFaviconFetcher implements FaviconFetcher {
}
private byte[] fetchForChannel(String googleAuthKey, String channelId)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
URI uri =
UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
.queryParam("part", PART_SNIPPET)
@@ -131,10 +118,7 @@ public class YoutubeFaviconFetcher implements FaviconFetcher {
}
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
throws IOException,
NotModifiedException,
TooManyRequestsException,
SchemeNotAllowedException {
throws IOException, NotModifiedException, TooManyRequestsException, SchemeNotAllowedException {
URI uri =
UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
.queryParam("part", PART_SNIPPET)

27
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedFetcher.java
View File

@@ -36,8 +36,7 @@ public class FeedFetcher {
private final HttpGetter getter;
private final List<FeedURLProvider> urlProviders;
public FeedFetcher(
FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
public FeedFetcher(FeedParser parser, HttpGetter getter, @All List<FeedURLProvider> urlProviders) {
this.parser = parser;
this.getter = getter;
this.urlProviders = urlProviders;
@@ -58,9 +57,7 @@ public class FeedFetcher {
NoFeedFoundException {
log.debug("Fetching feed {}", feedUrl);
HttpResult result =
getter.get(
HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
HttpResult result = getter.get(HttpRequest.builder(feedUrl).lastModified(lastModified).eTag(eTag).build());
byte[] content = result.content();
FeedParserResult parserResult;
@@ -69,19 +66,12 @@ public class FeedFetcher {
} catch (FeedParsingException e) {
if (extractFeedUrlFromHtml) {
String extractedUrl =
extractFeedUrl(
urlProviders,
feedUrl,
new String(result.content(), StandardCharsets.UTF_8));
extractFeedUrl(urlProviders, feedUrl, new String(result.content(), StandardCharsets.UTF_8));
if (StringUtils.isNotBlank(extractedUrl)) {
feedUrl = extractedUrl;
result =
getter.get(
HttpRequest.builder(extractedUrl)
.lastModified(lastModified)
.eTag(eTag)
.build());
getter.get(HttpRequest.builder(extractedUrl).lastModified(lastModified).eTag(eTag).build());
content = result.content();
parserResult = parser.parse(result.urlAfterRedirect(), content);
} else {
@@ -96,8 +86,7 @@ public class FeedFetcher {
throw new IOException("Feed content is empty.");
}
boolean lastModifiedHeaderValueChanged =
!Strings.CS.equals(lastModified, result.lastModifiedSince());
boolean lastModifiedHeaderValueChanged = !Strings.CS.equals(lastModified, result.lastModifiedSince());
boolean etagHeaderValueChanged = !Strings.CS.equals(eTag, result.eTag());
String hash = Digests.sha1Hex(content);
@@ -109,8 +98,7 @@ public class FeedFetcher {
etagHeaderValueChanged ? result.eTag() : null);
}
if (lastPublishedDate != null
&& lastPublishedDate.equals(parserResult.lastPublishedDate())) {
if (lastPublishedDate != null && lastPublishedDate.equals(parserResult.lastPublishedDate())) {
log.debug("publishedDate not modified: {}", feedUrl);
throw new NotModifiedException(
"publishedDate not modified",
@@ -127,8 +115,7 @@ public class FeedFetcher {
result.validFor());
}
private static String extractFeedUrl(
List<FeedURLProvider> urlProviders, String url, String urlContent) {
private static String extractFeedUrl(List<FeedURLProvider> urlProviders, String url, String urlContent) {
return urlProviders.stream()
.flatMap(provider -> provider.get(url, urlContent).stream())
.filter(StringUtils::isNotBlank)

45
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshEngine.java
View File

@@ -67,8 +67,7 @@ public class FeedRefreshEngine {
this.queue = new LinkedBlockingDeque<>();
metrics.register(
MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(MetricRegistry.name(getClass(), "queue", "size"), (Gauge<Integer>) queue::size);
metrics.register(
MetricRegistry.name(getClass(), "worker", "active"),
(Gauge<Integer>) () -> workerExecutor.getActiveCount());
@@ -88,12 +87,10 @@ public class FeedRefreshEngine {
this.refillLoopExecutor = Executors.newSingleThreadExecutor();
this.refillExecutor = newDiscardingSingleThreadExecutorService();
this.workerExecutor = newBlockingExecutorService(config.feedRefresh().httpThreads());
this.databaseUpdaterExecutor =
newBlockingExecutorService(config.feedRefresh().databaseThreads());
this.databaseUpdaterExecutor = newBlockingExecutorService(config.feedRefresh().databaseThreads());
this.notifierExecutor =
newDiscardingExecutorService(
config.pushNotifications().threads(),
config.pushNotifications().queueCapacity());
config.pushNotifications().threads(), config.pushNotifications().queueCapacity());
}
public void start() {
@@ -169,9 +166,7 @@ public class FeedRefreshEngine {
refill.mark();
List<Feed> nextUpdatableFeeds = getNextUpdatableFeeds(getBatchSize());
log.debug(
"found {} feeds that are up for refresh",
nextUpdatableFeeds.size());
log.debug("found {} feeds that are up for refresh", nextUpdatableFeeds.size());
for (Feed feed : nextUpdatableFeeds) {
// add the feed only if it was not already queued
if (queue.stream().noneMatch(f -> f.getId().equals(feed.getId()))) {
@@ -199,8 +194,7 @@ public class FeedRefreshEngine {
notifier.notifyOverWebsocket(sub, entries);
return CompletableFuture.runAsync(
() -> notifier.sendPushNotifications(sub, entries),
notifierExecutor);
() -> notifier.sendPushNotifications(sub, entries), notifierExecutor);
})
.toList();
return CompletableFuture.allOf(futures.toArray(CompletableFuture[]::new));
@@ -222,8 +216,7 @@ public class FeedRefreshEngine {
// update disabledUntil to prevent feeds from being returned again by
// feedDAO.findNextUpdatable()
Instant nextUpdateDate = Instant.now().plus(config.feedRefresh().interval());
feedDAO.setDisabledUntil(
feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
feedDAO.setDisabledUntil(feeds.stream().map(AbstractModel::getId).toList(), nextUpdateDate);
}
return feeds;
});
@@ -234,26 +227,19 @@ public class FeedRefreshEngine {
}
public void stop() {
MoreExecutors.shutdownAndAwaitTermination(
this.feedProcessingLoopExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(
this.refillLoopExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.feedProcessingLoopExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.refillLoopExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.refillExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.workerExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(
this.databaseUpdaterExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.databaseUpdaterExecutor, config.shutdownTimeout());
MoreExecutors.shutdownAndAwaitTermination(this.notifierExecutor, config.shutdownTimeout());
queue.clear();
}
/**
* returns an ExecutorService with a single thread that discards tasks if a task is already
* running
*/
/** returns an ExecutorService with a single thread that discards tasks if a task is already running */
private ThreadPoolExecutor newDiscardingSingleThreadExecutorService() {
ThreadPoolExecutor pool =
new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
ThreadPoolExecutor pool = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
@@ -262,11 +248,7 @@ public class FeedRefreshEngine {
private ThreadPoolExecutor newDiscardingExecutorService(int threads, int queueCapacity) {
ThreadPoolExecutor pool =
new ThreadPoolExecutor(
threads,
threads,
0L,
TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<>(queueCapacity));
threads, threads, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(queueCapacity));
pool.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
return pool;
}
@@ -274,8 +256,7 @@ public class FeedRefreshEngine {
/** returns an ExecutorService that blocks submissions until a thread is available */
private ThreadPoolExecutor newBlockingExecutorService(int threads) {
ThreadPoolExecutor pool =
new ThreadPoolExecutor(
threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new SynchronousQueue<>());
pool.setRejectedExecutionHandler((r, e) -> {
if (e.isShutdown()) {
return;

17
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshIntervalCalculator.java
View File

@@ -22,8 +22,7 @@ public class FeedRefreshIntervalCalculator {
private final FeedRefreshErrorHandling errorHandling;
private final InstantSource instantSource;
public FeedRefreshIntervalCalculator(
CommaFeedConfiguration config, InstantSource instantSource) {
public FeedRefreshIntervalCalculator(CommaFeedConfiguration config, InstantSource instantSource) {
this.interval = config.feedRefresh().interval();
this.maxInterval = config.feedRefresh().maxInterval();
this.empirical = config.feedRefresh().intervalEmpirical();
@@ -31,8 +30,7 @@ public class FeedRefreshIntervalCalculator {
this.instantSource = instantSource;
}
public Instant onFetchSuccess(
Instant publishedDate, Long averageEntryInterval, Duration validFor) {
public Instant onFetchSuccess(Instant publishedDate, Long averageEntryInterval, Duration validFor) {
Instant instant =
empirical
? computeEmpiricalRefreshInterval(publishedDate, averageEntryInterval)
@@ -54,14 +52,11 @@ public class FeedRefreshIntervalCalculator {
}
Duration retryInterval =
errorHandling
.backoffInterval()
.multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L);
errorHandling.backoffInterval().multipliedBy(errorCount - errorHandling.retriesBeforeBackoff() + 1L);
return constrainToBounds(instantSource.instant().plus(retryInterval));
}
private Instant computeEmpiricalRefreshInterval(
Instant publishedDate, Long averageEntryInterval) {
private Instant computeEmpiricalRefreshInterval(Instant publishedDate, Long averageEntryInterval) {
Instant now = instantSource.instant();
if (publishedDate == null) {
@@ -80,9 +75,7 @@ public class FeedRefreshIntervalCalculator {
int factor = 2;
long millis =
Longs.constrainToRange(
averageEntryInterval / factor,
interval.toMillis(),
maxInterval.dividedBy(4).toMillis());
averageEntryInterval / factor, interval.toMillis(), maxInterval.dividedBy(4).toMillis());
return now.plusMillis(millis);
} else {
// unknown case

19
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
View File

@@ -64,8 +64,7 @@ public class FeedRefreshUpdater {
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
private AddEntryResult addEntry(
final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
private AddEntryResult addEntry(final Feed feed, final Entry entry, final List<FeedSubscription> subscriptions) {
boolean processed = false;
FeedEntry insertedEntry = null;
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread = new HashSet<>();
@@ -111,11 +110,7 @@ public class FeedRefreshUpdater {
log.error("lock timeout for {} - {}", feed.getUrl(), key1);
}
} catch (InterruptedException e) {
log.error(
"interrupted while waiting for lock for {} : {}",
feed.getUrl(),
e.getMessage(),
e);
log.error("interrupted while waiting for lock for {} : {}", feed.getUrl(), e.getMessage(), e);
Thread.currentThread().interrupt();
} finally {
if (locked1) {
@@ -131,13 +126,11 @@ public class FeedRefreshUpdater {
public FeedRefreshUpdaterResult update(Feed feed, List<Entry> entries) {
boolean processed = true;
long inserted = 0;
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription =
new HashMap<>();
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription = new HashMap<>();
if (!entries.isEmpty()) {
List<FeedSubscription> subscriptions = null;
List<Entry> newEntries =
unitOfWork.call(() -> feedEntryService.removeExistingEntries(feed, entries));
List<Entry> newEntries = unitOfWork.call(() -> feedEntryService.removeExistingEntries(feed, entries));
for (Entry entry : newEntries) {
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
@@ -176,9 +169,7 @@ public class FeedRefreshUpdater {
}
private record AddEntryResult(
boolean processed,
FeedEntry insertedEntry,
Set<FeedSubscription> subscriptionsForWhichEntryIsUnread) {}
boolean processed, FeedEntry insertedEntry, Set<FeedSubscription> subscriptionsForWhichEntryIsUnread) {}
public record FeedRefreshUpdaterResult(
Map<FeedSubscription, List<FeedEntry>> insertedUnreadEntriesBySubscription) {}

13
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
View File

@@ -22,8 +22,8 @@ import java.util.List;
import java.util.Optional;
/**
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link
* FeedRefreshUpdater} does that)
* Calls {@link FeedFetcher} and updates the Feed object, but does not update the database, ({@link FeedRefreshUpdater}
* does that)
*/
@Slf4j
@Singleton
@@ -68,10 +68,7 @@ public class FeedRefreshWorker {
Duration entriesMaxAge = config.database().cleanup().entriesMaxAge();
if (!entriesMaxAge.isZero()) {
Instant threshold = Instant.now().minus(entriesMaxAge);
entries =
entries.stream()
.filter(entry -> entry.published().isAfter(threshold))
.toList();
entries = entries.stream().filter(entry -> entry.published().isAfter(threshold)).toList();
}
String urlAfterRedirect = result.urlAfterRedirect();
@@ -121,9 +118,7 @@ public class FeedRefreshWorker {
feed.setErrorCount(feed.getErrorCount() + 1);
feed.setMessage("Server indicated that we are sending too many requests");
feed.setDisabledUntil(
refreshIntervalCalculator.onTooManyRequests(
e.getRetryAfter(), feed.getErrorCount()));
feed.setDisabledUntil(refreshIntervalCalculator.onTooManyRequests(e.getRetryAfter(), feed.getErrorCount()));
return new FeedRefreshWorkerResult(feed, Collections.emptyList());
} catch (Exception e) {

7
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUpdateNotifier.java
View File

@@ -30,15 +30,12 @@ public class FeedUpdateNotifier {
public void notifyOverWebsocket(FeedSubscription sub, List<FeedEntry> entries) {
if (!entries.isEmpty()) {
webSocketSessions.sendMessage(
sub.getUser(), WebSocketMessageBuilder.newFeedEntries(sub, entries.size()));
webSocketSessions.sendMessage(sub.getUser(), WebSocketMessageBuilder.newFeedEntries(sub, entries.size()));
}
}
public void sendPushNotifications(FeedSubscription sub, List<FeedEntry> entries) {
if (!config.pushNotifications().enabled()
|| !sub.isPushNotificationsEnabled()
|| entries.isEmpty()) {
if (!config.pushNotifications().enabled() || !sub.isPushNotificationsEnabled() || entries.isEmpty()) {
return;
}

3
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java
View File

@@ -40,8 +40,7 @@ public class FeedUtils {
return false;
}
return TextDirectionDetector.detect(stripped)
== TextDirectionDetector.Direction.RIGHT_TO_LEFT;
return TextDirectionDetector.detect(stripped) == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
public static String getFaviconUrl(FeedSubscription subscription) {

4
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/EncodingDetector.java
View File

@@ -14,8 +14,8 @@ import java.nio.charset.Charset;
public class EncodingDetector {
/**
* Detect feed encoding by using the declared encoding in the xml processing instruction and by
* detecting the characters used in the feed
* Detect feed encoding by using the declared encoding in the xml processing instruction and by detecting the
* characters used in the feed
*/
public Charset getEncoding(byte[] bytes) {
String extracted = extractDeclaredEncoding(bytes);

25
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParser.java
View File

@@ -42,14 +42,12 @@ import java.util.stream.Collectors;
@Singleton
public class FeedParser {
private static final Namespace ATOM_10_NS =
Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Namespace ATOM_10_NS = Namespace.getNamespace("http://www.w3.org/2005/Atom");
private static final Instant START = Instant.ofEpochMilli(86400000);
private static final Instant END = Instant.ofEpochMilli(1000L * Integer.MAX_VALUE - 86400000);
private static final Comparator<Entry> ENTRY_COMPARATOR =
Comparator.comparing(Entry::published).reversed();
private static final Comparator<Entry> ENTRY_COMPARATOR = Comparator.comparing(Entry::published).reversed();
private final EncodingDetector encodingDetector;
private final XMLCleaner xmlCleaner;
@@ -78,20 +76,13 @@ public class FeedParser {
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
if (lastPublishedDate == null
|| lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
if (lastPublishedDate == null || lastEntryDate != null && lastPublishedDate.isBefore(lastEntryDate)) {
lastPublishedDate = lastEntryDate;
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
return new FeedParserResult(
title,
link,
iconUrl,
lastPublishedDate,
averageEntryInterval,
lastEntryDate,
entries);
title, link, iconUrl, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
} catch (FeedParsingException e) {
throw e;
} catch (Exception e) {
@@ -151,9 +142,7 @@ public class FeedParser {
String author = StringUtils.trimToNull(item.getAuthor());
String categories =
StringUtils.trimToNull(
item.getCategories().stream()
.map(SyndCategory::getName)
.collect(Collectors.joining(", ")));
item.getCategories().stream().map(SyndCategory::getName).collect(Collectors.joining(", ")));
Enclosure enclosure = buildEnclosure(item);
Media media = buildMedia(item);
@@ -281,9 +270,7 @@ public class FeedParser {
SummaryStatistics stats = new SummaryStatistics();
for (int i = 0; i < entries.size() - 1; i++) {
long diff =
Math.abs(
entries.get(i).published().toEpochMilli()
- entries.get(i + 1).published().toEpochMilli());
Math.abs(entries.get(i).published().toEpochMilli() - entries.get(i + 1).published().toEpochMilli());
stats.addValue(diff);
}
return (long) stats.getMean();

13
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParserResult.java
View File

@@ -14,18 +14,9 @@ public record FeedParserResult(
public record Entry(String guid, String url, Instant published, Content content) {}
public record Content(
String title,
String content,
String author,
String categories,
Enclosure enclosure,
Media media) {}
String title, String content, String author, String categories, Enclosure enclosure, Media media) {}
public record Enclosure(String url, String type) {}
public record Media(
String description,
String thumbnailUrl,
Integer thumbnailWidth,
Integer thumbnailHeight) {}
public record Media(String description, String thumbnailUrl, Integer thumbnailWidth, Integer thumbnailHeight) {}
}

6
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/XMLCleaner.java
View File

@@ -13,11 +13,9 @@ import java.util.regex.Pattern;
@Singleton
public class XMLCleaner {
private static final Pattern DOCTYPE_PATTERN =
Pattern.compile("<!DOCTYPE[^>]*>", Pattern.CASE_INSENSITIVE);
private static final Pattern DOCTYPE_PATTERN = Pattern.compile("<!DOCTYPE[^>]*>", Pattern.CASE_INSENSITIVE);
private final Trie trie =
Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build();
private final Trie trie = Trie.builder().ignoreOverlaps().addKeywords(HtmlEntities.HTML_ENTITIES).build();
public String clean(String xml) {
xml = removeCharactersBeforeFirstXmlTag(xml);

3
commafeed-server/src/main/java/com/commafeed/backend/model/UserSettings.java
View File

@@ -155,8 +155,7 @@ public class UserSettings extends AbstractModel {
private boolean instapaper;
private boolean buffer;
@Embedded
private PushNotificationUserSettings pushNotifications = new PushNotificationUserSettings();
@Embedded private PushNotificationUserSettings pushNotifications = new PushNotificationUserSettings();
@Embeddable
@SuppressWarnings("serial")

10
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLExporter.java
View File

@@ -49,8 +49,7 @@ public class OPMLExporter {
}
// export root subscriptions
for (FeedSubscription sub :
subscriptions.stream().filter(s -> s.getCategory() == null).toList()) {
for (FeedSubscription sub : subscriptions.stream().filter(s -> s.getCategory() == null).toList()) {
opml.getOutlines().add(buildSubscriptionOutline(sub));
}
@@ -65,17 +64,14 @@ public class OPMLExporter {
for (FeedCategory child :
categories.stream()
.filter(c ->
c.getParent() != null && c.getParent().getId().equals(cat.getId()))
.filter(c -> c.getParent() != null && c.getParent().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildCategoryOutline(child, categories, subscriptions));
}
for (FeedSubscription sub :
subscriptions.stream()
.filter(s ->
s.getCategory() != null
&& s.getCategory().getId().equals(cat.getId()))
.filter(s -> s.getCategory() != null && s.getCategory().getId().equals(cat.getId()))
.toList()) {
outline.getChildren().add(buildSubscriptionOutline(sub));
}

3
commafeed-server/src/main/java/com/commafeed/backend/opml/OPMLImporter.java
View File

@@ -79,8 +79,7 @@ public class OPMLImporter {
}
// make sure we continue with the import process even if a feed failed
try {
feedSubscriptionService.subscribe(
user, outline.getXmlUrl(), name, parent, position);
feedSubscriptionService.subscribe(user, outline.getXmlUrl(), name, parent, position);
} catch (Exception e) {
log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage());
}

40
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentCleaningService.java
View File

@@ -33,10 +33,8 @@ import java.util.List;
public class FeedEntryContentCleaningService {
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES =
Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES =
Arrays.asList("display", "width", "height");
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] {'(', ')'};
public String clean(String content, String baseUri, boolean keepTextOnly) {
@@ -59,8 +57,7 @@ public class FeedEntryContentCleaningService {
e.attr("style", escaped);
}
clean.outputSettings(
new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
@@ -126,24 +123,13 @@ public class FeedEntryContentCleaningService {
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes(
"iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes(
"td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes(
"th",
"border",
"bordercolor",
"abbr",
"axis",
"colspan",
"rowspan",
"scope",
"width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
@@ -161,8 +147,7 @@ public class FeedEntryContentCleaningService {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl =
buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
@@ -187,8 +172,7 @@ public class FeedEntryContentCleaningService {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl =
buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
@@ -216,8 +200,7 @@ public class FeedEntryContentCleaningService {
new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug(
"warning while parsing css: {}", exception.getMessage(), exception);
log.debug("warning while parsing css: {}", exception.getMessage(), exception);
}
@Override
@@ -227,10 +210,7 @@ public class FeedEntryContentCleaningService {
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug(
"fatal error while parsing css: {}",
exception.getMessage(),
exception);
log.debug("fatal error while parsing css: {}", exception.getMessage(), exception);
}
});

13
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentService.java
View File

@@ -27,9 +27,7 @@ public class FeedEntryContentService {
public FeedEntryContent findOrCreate(Content content, String baseUrl) {
FeedEntryContent entryContent = buildContent(content, baseUrl);
Optional<FeedEntryContent> existing =
feedEntryContentDAO
.findExisting(entryContent.getContentHash(), entryContent.getTitleHash())
.stream()
feedEntryContentDAO.findExisting(entryContent.getContentHash(), entryContent.getTitleHash()).stream()
.filter(entryContent::equivalentTo)
.findFirst();
if (existing.isPresent()) {
@@ -44,11 +42,9 @@ public class FeedEntryContentService {
FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitleHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.title())));
entryContent.setContentHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.content())));
entryContent.setTitle(
FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setTitle(FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false));
entryContent.setAuthor(
FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
entryContent.setAuthor(FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
entryContent.setDirection(
FeedUtils.isRTL(content.title(), content.content())
@@ -63,8 +59,7 @@ public class FeedEntryContentService {
Media media = content.media();
if (media != null) {
entryContent.setMediaDescription(
cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaDescription(cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048));
entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
entryContent.setMediaThumbnailHeight(media.thumbnailHeight());

26
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryFilteringService.java
View File

@@ -50,27 +50,17 @@ public class FeedEntryFilteringService {
private final ExecutorService executor = Executors.newCachedThreadPool();
private final CommaFeedConfiguration config;
public boolean filterMatchesEntry(String filter, FeedEntry entry)
throws FeedEntryFilterException {
public boolean filterMatchesEntry(String filter, FeedEntry entry) throws FeedEntryFilterException {
if (StringUtils.isBlank(filter)) {
return true;
}
String title =
entry.getContent().getTitle() == null
? ""
: Jsoup.parse(entry.getContent().getTitle()).text();
String author =
entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor();
String title = entry.getContent().getTitle() == null ? "" : Jsoup.parse(entry.getContent().getTitle()).text();
String author = entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor();
String content =
entry.getContent().getContent() == null
? ""
: Jsoup.parse(entry.getContent().getContent()).text();
entry.getContent().getContent() == null ? "" : Jsoup.parse(entry.getContent().getContent()).text();
String url = entry.getUrl() == null ? "" : entry.getUrl();
String categories =
entry.getContent().getCategories() == null
? ""
: entry.getContent().getCategories();
String categories = entry.getContent().getCategories() == null ? "" : entry.getContent().getCategories();
Map<String, Object> data = new HashMap<>();
data.put("title", title);
@@ -97,11 +87,9 @@ public class FeedEntryFilteringService {
TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FeedEntryFilterException(
"interrupted while evaluating expression " + filter, e);
throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e);
} catch (ExecutionException e) {
throw new FeedEntryFilterException(
"Exception while evaluating expression " + filter, e);
throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}

30
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java
View File

@@ -55,12 +55,9 @@ public class FeedEntryService {
}
public List<Entry> removeExistingEntries(Feed feed, List<Entry> entries) {
Set<String> guidHashes =
entries.stream().map(e -> Digests.sha1Hex(e.guid())).collect(Collectors.toSet());
Set<String> guidHashes = entries.stream().map(e -> Digests.sha1Hex(e.guid())).collect(Collectors.toSet());
Set<String> existingGuidHashes = feedEntryDAO.findExistingGuidHashes(guidHashes, feed);
return entries.stream()
.filter(e -> !existingGuidHashes.contains(Digests.sha1Hex(e.guid())))
.toList();
return entries.stream().filter(e -> !existingGuidHashes.contains(Digests.sha1Hex(e.guid()))).toList();
}
public boolean applyFilter(FeedSubscription sub, FeedEntry entry) {
@@ -122,37 +119,22 @@ public class FeedEntryService {
List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses =
feedEntryStatusDAO.findBySubscriptions(
user,
subscriptions,
true,
keywords,
null,
-1,
-1,
null,
false,
null,
null,
null);
user, subscriptions, true, keywords, null, -1, -1, null, false, null, null, null);
markList(statuses, olderThan, insertedBefore);
}
public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statuses =
feedEntryStatusDAO.findStarred(user, null, null, -1, -1, null, false);
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, null, -1, -1, null, false);
markList(statuses, olderThan, insertedBefore);
}
private void markList(
List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
private void markList(List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statusesToMark =
statuses.stream()
.filter(FeedEntryStatus::isMarkable)
.filter(s -> {
Instant entryDate = s.getEntry().getPublished();
return olderThan == null
|| entryDate == null
|| entryDate.isBefore(olderThan);
return olderThan == null || entryDate == null || entryDate.isBefore(olderThan);
})
.filter(s -> {
Instant insertedDate = s.getEntry().getInserted();

6
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryTagService.java
View File

@@ -36,16 +36,14 @@ public class FeedEntryTagService {
}
List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry);
Set<String> existingTagNames =
existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
Set<String> existingTagNames = existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
List<FeedEntryTag> addList =
tagNames.stream()
.filter(name -> !existingTagNames.contains(name))
.map(name -> new FeedEntryTag(user, entry, name))
.toList();
List<FeedEntryTag> removeList =
existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).toList();
List<FeedEntryTag> removeList = existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).toList();
addList.forEach(feedEntryTagDAO::persist);
feedEntryTagDAO.delete(removeList);

3
commafeed-server/src/main/java/com/commafeed/backend/service/FeedFaviconService.java
View File

@@ -36,8 +36,7 @@ public class FeedFaviconService {
this.defaultFavicon =
new Favicon(
Resources.toByteArray(
Objects.requireNonNull(
getClass().getResource("/images/default_favicon.gif"))),
Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))),
"image/gif");
}

10
commafeed-server/src/main/java/com/commafeed/backend/service/FeedSubscriptionService.java
View File

@@ -48,15 +48,13 @@ public class FeedSubscriptionService {
// by the feed engine
feedSubscriptionDAO.onPostCommitInsert(sub -> {
Feed feed = sub.getFeed();
if (feed.getDisabledUntil() == null
|| feed.getDisabledUntil().isBefore(Instant.now())) {
if (feed.getDisabledUntil() == null || feed.getDisabledUntil().isBefore(Instant.now())) {
feedRefreshEngine.refreshImmediately(feed);
}
});
}
public long subscribe(
User user, String url, String title, FeedCategory category, int position) {
public long subscribe(User user, String url, String title, FeedCategory category, int position) {
Integer maxFeedsPerUser = config.database().cleanup().maxFeedsPerUser();
if (maxFeedsPerUser > 0 && feedSubscriptionDAO.count(user) >= maxFeedsPerUser) {
String message =
@@ -98,9 +96,7 @@ public class FeedSubscriptionService {
public void refreshAll(User user) throws ForceFeedRefreshTooSoonException {
Instant lastForceRefresh = user.getLastForceRefresh();
if (lastForceRefresh != null
&& lastForceRefresh
.plus(config.feedRefresh().forceRefreshCooldownDuration())
.isAfter(Instant.now())) {
&& lastForceRefresh.plus(config.feedRefresh().forceRefreshCooldownDuration()).isAfter(Instant.now())) {
throw new ForceFeedRefreshTooSoonException();
}

40
commafeed-server/src/main/java/com/commafeed/backend/service/PushNotificationService.java
View File

@@ -47,16 +47,13 @@ public class PushNotificationService {
private final CommaFeedConfiguration config;
public PushNotificationService(
HttpClientFactory httpClientFactory,
MetricRegistry metrics,
CommaFeedConfiguration config) {
HttpClientFactory httpClientFactory, MetricRegistry metrics, CommaFeedConfiguration config) {
this.httpClient = httpClientFactory.newClient(config.pushNotifications().threads());
this.meter = metrics.meter(MetricRegistry.name(getClass(), "notify"));
this.config = config;
}
public void notify(
PushNotificationUserSettings settings, FeedSubscription subscription, FeedEntry entry) {
public void notify(PushNotificationUserSettings settings, FeedSubscription subscription, FeedEntry entry) {
if (!config.pushNotifications().enabled() || settings.getType() == null) {
return;
}
@@ -79,9 +76,7 @@ public class PushNotificationService {
case NTFY -> sendNtfy(settings, feedTitle, entryTitle, entryUrl);
case GOTIFY -> sendGotify(settings, feedTitle, entryTitle, entryUrl);
case PUSHOVER -> sendPushover(settings, feedTitle, entryTitle, entryUrl);
default ->
throw new IllegalStateException(
"unsupported notification type: " + settings.getType());
default -> throw new IllegalStateException("unsupported notification type: " + settings.getType());
}
} catch (IOException e) {
throw new PushNotificationException("Failed to send external notification", e);
@@ -90,11 +85,7 @@ public class PushNotificationService {
meter.mark();
}
private void sendNtfy(
PushNotificationUserSettings settings,
String feedTitle,
String entryTitle,
String entryUrl)
private void sendNtfy(PushNotificationUserSettings settings, String feedTitle, String entryTitle, String entryUrl)
throws IOException {
String serverUrl = Urls.removeTrailingSlash(settings.getServerUrl());
String topic = settings.getTopic();
@@ -119,11 +110,7 @@ public class PushNotificationService {
send("ntfy", request.build());
}
private void sendGotify(
PushNotificationUserSettings settings,
String feedTitle,
String entryTitle,
String entryUrl)
private void sendGotify(PushNotificationUserSettings settings, String feedTitle, String entryTitle, String entryUrl)
throws IOException {
String serverUrl = Urls.removeTrailingSlash(settings.getServerUrl());
String token = settings.getUserSecret();
@@ -143,8 +130,7 @@ public class PushNotificationService {
new JsonObject()
.put(
"client::notification",
new JsonObject()
.put("click", new JsonObject().put("url", entryUrl))));
new JsonObject().put("click", new JsonObject().put("url", entryUrl))));
}
SimpleRequestBuilder request = SimpleRequestBuilder.post(serverUrl + "/message");
@@ -155,10 +141,7 @@ public class PushNotificationService {
}
private void sendPushover(
PushNotificationUserSettings settings,
String feedTitle,
String entryTitle,
String entryUrl)
PushNotificationUserSettings settings, String feedTitle, String entryTitle, String entryUrl)
throws IOException {
String token = settings.getUserSecret();
String userKey = settings.getUserId();
@@ -177,8 +160,7 @@ public class PushNotificationService {
params.add(new BasicNameValuePair("url", entryUrl));
}
SimpleRequestBuilder request =
SimpleRequestBuilder.post("https://api.pushover.net/1/messages.json");
SimpleRequestBuilder request = SimpleRequestBuilder.post("https://api.pushover.net/1/messages.json");
request.setBody(
WWWFormCodec.format(params, StandardCharsets.UTF_8),
ContentType.APPLICATION_FORM_URLENCODED.withCharset(StandardCharsets.UTF_8));
@@ -194,11 +176,9 @@ public class PushNotificationService {
null);
try {
HttpResponse response =
future.get(config.httpClient().responseTimeout().toSeconds(), TimeUnit.SECONDS)
.getHead();
future.get(config.httpClient().responseTimeout().toSeconds(), TimeUnit.SECONDS).getHead();
if (response.getCode() >= 400) {
throw new PushNotificationException(
name + " notification failed with status " + response.getCode());
throw new PushNotificationException(name + " notification failed with status " + response.getCode());
}
} catch (InterruptedException e) {
future.cancel(true);

16
commafeed-server/src/main/java/com/commafeed/backend/service/UserService.java
View File

@@ -53,8 +53,7 @@ public class UserService {
user = userDAO.findByEmail(nameOrEmail);
}
if (user != null && !user.isDisabled()) {
boolean authenticated =
encryptionService.authenticate(password, user.getPassword(), user.getSalt());
boolean authenticated = encryptionService.authenticate(password, user.getPassword(), user.getSalt());
if (authenticated) {
performPostLoginActivities(user);
return Optional.of(user);
@@ -107,16 +106,11 @@ public class UserService {
}
public User register(
String name,
String password,
String email,
Collection<Role> roles,
boolean forceRegistration) {
String name, String password, String email, Collection<Role> roles, boolean forceRegistration) {
if (!forceRegistration) {
Preconditions.checkState(
config.users().allowRegistrations(),
"Registrations are closed on this CommaFeed instance");
config.users().allowRegistrations(), "Registrations are closed on this CommaFeed instance");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");
@@ -167,9 +161,7 @@ public class UserService {
}
public String generateApiKey(User user) {
byte[] key =
encryptionService.getEncryptedPassword(
UUID.randomUUID().toString(), user.getSalt());
byte[] key = encryptionService.getEncryptedPassword(UUID.randomUUID().toString(), user.getSalt());
return Digests.sha1Hex(key);
}

28
commafeed-server/src/main/java/com/commafeed/backend/service/db/DatabaseCleaningService.java
View File

@@ -71,18 +71,15 @@ public class DatabaseCleaningService {
return;
}
entriesDeleted =
unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesDeletedMeter.mark(entriesDeleted);
entriesTotal += entriesDeleted;
log.debug("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
}
deleted =
unitOfWork.call(() ->
feedDAO.delete(
feedDAO.findByIds(
feeds.stream().map(AbstractModel::getId).toList())));
unitOfWork.call(
() -> feedDAO.delete(feedDAO.findByIds(feeds.stream().map(AbstractModel::getId).toList())));
total += deleted;
log.debug("removed {} feeds without subscriptions", total);
} while (deleted != 0);
@@ -117,8 +114,7 @@ public class DatabaseCleaningService {
List<FeedCapacity> feeds =
unitOfWork.call(() ->
feedEntryDAO.findFeedsExceedingCapacity(
maxFeedCapacity, batchSize, keepStarredEntries));
feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize, keepStarredEntries));
if (feeds.isEmpty()) {
break;
}
@@ -128,8 +124,7 @@ public class DatabaseCleaningService {
int deleted;
do {
if (Thread.currentThread().isInterrupted()) {
log.info(
"interrupted, stopping cleanup of entries exceeding feed capacity");
log.info("interrupted, stopping cleanup of entries exceeding feed capacity");
return;
}
@@ -137,9 +132,7 @@ public class DatabaseCleaningService {
deleted =
unitOfWork.call(() ->
feedEntryDAO.deleteOldEntries(
feed.id(),
Math.min(batchSize, rem),
keepStarredEntries));
feed.id(), Math.min(batchSize, rem), keepStarredEntries));
entriesDeletedMeter.mark(deleted);
total += deleted;
remaining -= deleted;
@@ -161,9 +154,8 @@ public class DatabaseCleaningService {
}
deleted =
unitOfWork.call(() ->
feedEntryDAO.deleteEntriesOlderThan(
olderThan, batchSize, keepStarredEntries));
unitOfWork.call(
() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize, keepStarredEntries));
entriesDeletedMeter.mark(deleted);
total += deleted;
log.debug("removed {} old entries", total);
@@ -181,9 +173,7 @@ public class DatabaseCleaningService {
return;
}
deleted =
unitOfWork.call(
() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
total += deleted;
log.debug("removed {} old read statuses", total);
} while (deleted != 0);

6
commafeed-server/src/main/java/com/commafeed/backend/service/db/DatabaseStartupService.java
View File

@@ -27,14 +27,12 @@ public class DatabaseStartupService {
}
/**
* Register a postgresql database in liquibase that doesn't escape columns, so that we can use
* lower case columns
* Register a postgresql database in liquibase that doesn't escape columns, so that we can use lower case columns
*/
@MetaInfServices(Database.class)
public static class LowerCaseColumnsPostgresDatabase extends PostgresDatabase {
@Override
public String escapeObjectName(
String objectName, Class<? extends DatabaseObject> objectType) {
public String escapeObjectName(String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}

3
commafeed-server/src/main/java/com/commafeed/backend/urlprovider/YoutubeFeedURLProvider.java
View File

@@ -16,8 +16,7 @@ import java.util.List;
public class YoutubeFeedURLProvider implements FeedURLProvider {
private static final String PREFIX = "https://www.youtube.com/channel/";
private static final String REPLACEMENT_PREFIX =
"https://www.youtube.com/feeds/videos.xml?channel_id=";
private static final String REPLACEMENT_PREFIX = "https://www.youtube.com/feeds/videos.xml?channel_id=";
@Override
public List<String> get(String url, String urlContent) {