Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-22 20:14:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

add support for declared icons in feeds (#2048)

Browse Source
This commit is contained in:
Athou
2026-05-10 16:24:25 +02:00
parent 1622eb642a
commit 4a8be29616
27 changed files with 715 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
commafeed-server/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java
View File

@@ -1,49 +0,0 @@
package com.commafeed.backend.favicon;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class AbstractFaviconFetcher {
private static final List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
public abstract Favicon fetch(Feed feed);
protected boolean isValidIconResponse(byte[] content, String contentType) {
if (content == null) {
return false;
}
long length = content.length;
if (StringUtils.isNotBlank(contentType)) {
contentType = contentType.split(";")[0];
}
if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
log.debug("Content-Type {} is blacklisted", contentType);
return false;
}
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
return true;
}
}

130
commafeed-server/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java
View File

@@ -1,130 +0,0 @@
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.Urls;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Inspired/Ported from https://github.com/potatolondon/getfavicon
*
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(Integer.MIN_VALUE)
public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
Favicon icon = fetch(feed.getLink());
if (icon == null) {
icon = fetch(feed.getUrl());
}
return icon;
}
private Favicon fetch(String url) {
if (url == null) {
log.debug("url is null");
return null;
}
int doubleSlash = url.indexOf("//");
if (doubleSlash == -1) {
doubleSlash = 0;
} else {
doubleSlash += 2;
}
int firstSlash = url.indexOf('/', doubleSlash);
if (firstSlash != -1) {
url = url.substring(0, firstSlash);
}
Favicon icon = getIconAtRoot(url);
if (icon == null) {
icon = getIconInPage(url);
}
return icon;
}
private Favicon getIconAtRoot(String url) {
byte[] bytes = null;
String contentType = null;
try {
url = Urls.removeTrailingSlash(url) + "/favicon.ico";
log.debug("getting root icon at {}", url);
HttpResult result = getter.get(url);
bytes = result.content();
contentType = result.contentType();
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private Favicon getIconInPage(String url) {
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.content()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon", e);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
String href = icons.getFirst().attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
log.debug("Found unconfirmed iconInPage at {}", href);
byte[] bytes;
String contentType;
try {
HttpResult result = getter.get(href);
bytes = result.content();
contentType = result.contentType();
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href, e);
return null;
}
if (!isValidIconResponse(bytes, contentType)) {
log.debug("Invalid icon found for {}", href);
return null;
}
return new Favicon(bytes, contentType);
}
}

15
commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
View File

@@ -4,6 +4,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.hc.core5.http.NameValuePair;
@@ -19,14 +20,14 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
@Priority(3)
public class FacebookFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("www.facebook.com")) {
return null;
}
@@ -38,23 +39,15 @@ public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
byte[] bytes = null;
String contentType = null;
try {
log.debug("Getting Facebook user's icon, {}", url);
HttpResult iconResult = getter.get(iconUrl);
bytes = iconResult.content();
contentType = iconResult.contentType();
return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private String extractUserName(String url) {

9
commafeed-server/src/main/java/com/commafeed/backend/favicon/FaviconFetcher.java Normal file
View File

@@ -0,0 +1,9 @@
package com.commafeed.backend.favicon;
import com.commafeed.backend.model.Feed;
public interface FaviconFetcher {
Favicon fetch(Feed feed);
}

39
commafeed-server/src/main/java/com/commafeed/backend/favicon/FeedFaviconFetcher.java Normal file
View File

@@ -0,0 +1,39 @@
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Fetch favicon from the url declared in the feed.
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(2)
public class FeedFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getIconUrl();
if (url == null) {
return null;
}
try {
HttpResult result = getter.get(url);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve icon declared in the feed {}", url, e);
return null;
}
}
}

65
commafeed-server/src/main/java/com/commafeed/backend/favicon/HtmlFaviconFetcher.java Normal file
View File

@@ -0,0 +1,65 @@
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Extracts favicon url from html page.
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(1)
public class HtmlFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getLink();
if (url == null) {
return null;
}
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.content()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon", e);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
String href = icons.getFirst().attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
try {
HttpResult result = getter.get(href);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href, e);
return null;
}
}
}

44
commafeed-server/src/main/java/com/commafeed/backend/favicon/RootFaviconFetcher.java Normal file
View File

@@ -0,0 +1,44 @@
package com.commafeed.backend.favicon;
import java.net.URI;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Fetches favicon from root of the domain (e.g. https://example.com/favicon.ico)
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(0)
public class RootFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getLink();
if (url == null) {
url = feed.getUrl();
}
URI uri = URI.create(url);
String faviconUrl = "%s://%s/favicon.ico".formatted(uri.getScheme(), uri.getHost());
try {
log.debug("getting root icon at {}", faviconUrl);
HttpResult result = getter.get(faviconUrl);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
return null;
}
}
}

13
commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
View File

@@ -5,6 +5,7 @@ import java.net.URI;
import java.util.List;
import java.util.Optional;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.UriBuilder;
@@ -29,7 +30,8 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
@Priority(3)
public class YoutubeFaviconFetcher implements FaviconFetcher {
private static final String PART_SNIPPET = "snippet";
@@ -53,8 +55,6 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
return null;
}
byte[] bytes = null;
String contentType = null;
try {
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
@@ -84,16 +84,11 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
}
HttpResult iconResult = getter.get(thumbnailUrl.asText());
bytes = iconResult.content();
contentType = iconResult.contentType();
return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve YouTube icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private byte[] fetchForUser(String googleAuthKey, String userId)

1
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
View File

@@ -69,6 +69,7 @@ public class FeedRefreshWorker {
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(result.feed().link());
feed.setIconUrl(result.feed().iconUrl());
feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(result.contentHash());

3
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParser.java
View File

@@ -81,6 +81,7 @@ public class FeedParser {
String title = feed.getTitle();
String link = feed.getLink();
String iconUrl = feed.getIcon() != null ? feed.getIcon().getUrl() : null;
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
@@ -89,7 +90,7 @@ public class FeedParser {
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
return new FeedParserResult(title, link, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
return new FeedParserResult(title, link, iconUrl, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
} catch (FeedParsingException e) {
throw e;
} catch (Exception e) {

4
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParserResult.java
View File

@@ -3,8 +3,8 @@ package com.commafeed.backend.feed.parser;
import java.time.Instant;
import java.util.List;
public record FeedParserResult(String title, String link, Instant lastPublishedDate, Long averageEntryInterval, Instant lastEntryDate,
List<Entry> entries) {
public record FeedParserResult(String title, String link, String iconUrl, Instant lastPublishedDate, Long averageEntryInterval,
Instant lastEntryDate, List<Entry> entries) {
public record Entry(String guid, String url, Instant published, Content content) {}
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {}

5
commafeed-server/src/main/java/com/commafeed/backend/model/Feed.java
View File

@@ -48,6 +48,11 @@ public class Feed extends AbstractModel {
@JdbcTypeCode(Types.LONGVARCHAR)
private String link;
@Lob
@Column(name = "icon_url", length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String iconUrl;
/**
* Last time we tried to fetch the feed
*/

71
commafeed-server/src/main/java/com/commafeed/backend/service/FeedFaviconService.java Normal file
View File

@@ -0,0 +1,71 @@
package com.commafeed.backend.service;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.MediaType;
import org.apache.commons.lang3.ArrayUtils;
import com.commafeed.backend.favicon.Favicon;
import com.commafeed.backend.favicon.FaviconFetcher;
import com.commafeed.backend.model.Feed;
import com.google.common.io.Resources;
import io.quarkus.arc.All;
import lombok.extern.slf4j.Slf4j;
@Singleton
@Slf4j
public class FeedFaviconService {
private static final Set<MediaType> ICON_MIMETYPE_BLACKLIST = Set.of(MediaType.APPLICATION_XML_TYPE, MediaType.TEXT_HTML_TYPE);
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
private final List<FaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
public FeedFaviconService(@All List<FaviconFetcher> faviconFetchers) throws IOException {
this.faviconFetchers = faviconFetchers;
this.defaultFavicon = new Favicon(
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif");
}
public Favicon fetchFavicon(Feed feed) {
for (FaviconFetcher faviconFetcher : faviconFetchers) {
Favicon icon = faviconFetcher.fetch(feed);
if (isFaviconValid(icon)) {
return icon;
}
}
return defaultFavicon;
}
private static boolean isFaviconValid(Favicon favicon) {
if (favicon == null || ArrayUtils.isEmpty(favicon.icon())) {
return false;
}
long length = favicon.icon().length;
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
if (ICON_MIMETYPE_BLACKLIST.stream().anyMatch(bl -> bl.isCompatible(favicon.mediaType()))) {
log.debug("Content-Type {} is blacklisted", favicon.mediaType());
return false;
}
return true;
}
}

33
commafeed-server/src/main/java/com/commafeed/backend/service/FeedService.java
View File

@@ -1,37 +1,23 @@
package com.commafeed.backend.service;
import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.Urls;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
import com.commafeed.backend.favicon.Favicon;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.Models;
import com.google.common.io.Resources;
import io.quarkus.arc.All;
import lombok.RequiredArgsConstructor;
@Singleton
@RequiredArgsConstructor
public class FeedService {
private final FeedDAO feedDAO;
private final List<AbstractFaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
public FeedService(FeedDAO feedDAO, @All List<AbstractFaviconFetcher> faviconFetchers) throws IOException {
this.feedDAO = feedDAO;
this.faviconFetchers = faviconFetchers;
this.defaultFavicon = new Favicon(
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif");
}
public synchronized Feed findOrCreate(String url) {
String normalizedUrl = Urls.normalize(url);
@@ -57,19 +43,4 @@ public class FeedService {
feedDAO.merge(feed);
}
public Favicon fetchFavicon(Feed feed) {
Favicon icon = null;
for (AbstractFaviconFetcher faviconFetcher : faviconFetchers) {
icon = faviconFetcher.fetch(feed);
if (icon != null) {
break;
}
}
if (icon == null) {
icon = defaultFavicon;
}
return icon;
}
}

18
commafeed-server/src/main/java/com/commafeed/frontend/resource/FeedREST.java
View File

@@ -62,7 +62,7 @@ import com.commafeed.backend.opml.OPMLImporter;
import com.commafeed.backend.service.FeedEntryFilteringService;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.commafeed.backend.service.FeedFaviconService;
import com.commafeed.backend.service.FeedSubscriptionService;
import com.commafeed.backend.service.FeedSubscriptionService.ForceFeedRefreshTooSoonException;
import com.commafeed.frontend.model.Entries;
@@ -106,7 +106,7 @@ public class FeedREST {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedFetcher feedFetcher;
private final FeedService feedService;
private final FeedFaviconService feedFaviconService;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionService feedSubscriptionService;
private final FeedEntryFilteringService feedEntryFilteringService;
@@ -339,7 +339,19 @@ public class FeedREST {
}
Feed feed = subscription.getFeed();
Favicon icon = feedService.fetchFavicon(feed);
if (feed.getLastUpdated() == null) {
// the feed has never been fetched yet so the iconUrl field hasn't been updated yet, and the icon can't be fetched
// this can happen for newly subscribed feeds:
// the feed has been added in the tree in the web client and the favicon is being fetched,
// but the feed has not been fetched yet
// the client is configured to retry in that case
return Response.status(Status.SERVICE_UNAVAILABLE)
.entity("Feed has not been fetched yet, please retry in a bit")
.type(MediaType.TEXT_PLAIN)
.build();
}
Favicon icon = feedFaviconService.fetchFavicon(feed);
return Response.ok(icon.icon(), icon.mediaType()).build();
}

6
commafeed-server/src/main/java/com/commafeed/frontend/resource/fever/FeverREST.java
View File

@@ -45,7 +45,7 @@ import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings.ReadingOrder;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.commafeed.backend.service.FeedFaviconService;
import com.commafeed.backend.service.UserService;
import com.commafeed.frontend.resource.fever.FeverResponse.FeverFavicon;
import com.commafeed.frontend.resource.fever.FeverResponse.FeverFeed;
@@ -80,7 +80,7 @@ public class FeverREST {
private final UserService userService;
private final FeedEntryService feedEntryService;
private final FeedService feedService;
private final FeedFaviconService feedFaviconService;
private final FeedEntryDAO feedEntryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedCategoryDAO feedCategoryDAO;
@@ -303,7 +303,7 @@ public class FeverREST {
private List<FeverFavicon> buildFavicons(List<FeedSubscription> subscriptions) {
return subscriptions.stream().map(s -> {
Favicon favicon = feedService.fetchFavicon(s.getFeed());
Favicon favicon = feedFaviconService.fetchFavicon(s.getFeed());
FeverFavicon f = new FeverFavicon();
f.setId(s.getFeed().getId());

14
commafeed-server/src/main/resources/changelogs/db.changelog-7.2.xml Normal file
View File

@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd">
<changeSet id="feed-icons" author="athou">
<addColumn tableName="FEEDS">
<column name="icon_url" type="CLOB">
<constraints nullable="true" />
</column>
</addColumn>
</changeSet>
</databaseChangeLog>

1
commafeed-server/src/main/resources/migrations.xml
View File

@@ -38,5 +38,6 @@
<include file="changelogs/db.changelog-5.11.xml" />
<include file="changelogs/db.changelog-5.12.xml" />
<include file="changelogs/db.changelog-7.0.xml" />
<include file="changelogs/db.changelog-7.2.xml" />
</databaseChangeLog>