Archives/Athou_commafeed
1
0
Fork 0
mirror of https://github.com/Athou/commafeed.git synced 2026-11-04 03:14:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

add support for declared icons in feeds (#2048)

Browse Source
This commit is contained in:
Athou
2026-05-10 16:24:25 +02:00
parent 1622eb642a
commit 4a8be29616
27 changed files with 715 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
commafeed-server/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java
View File

@@ -1,49 +0,0 @@
package com.commafeed.backend.favicon;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class AbstractFaviconFetcher {
private static final List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
public abstract Favicon fetch(Feed feed);
protected boolean isValidIconResponse(byte[] content, String contentType) {
if (content == null) {
return false;
}
long length = content.length;
if (StringUtils.isNotBlank(contentType)) {
contentType = contentType.split(";")[0];
}
if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
log.debug("Content-Type {} is blacklisted", contentType);
return false;
}
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
return true;
}
}

130
commafeed-server/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java
View File

@@ -1,130 +0,0 @@
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.Urls;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Inspired/Ported from https://github.com/potatolondon/getfavicon
*
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(Integer.MIN_VALUE)
public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
Favicon icon = fetch(feed.getLink());
if (icon == null) {
icon = fetch(feed.getUrl());
}
return icon;
}
private Favicon fetch(String url) {
if (url == null) {
log.debug("url is null");
return null;
}
int doubleSlash = url.indexOf("//");
if (doubleSlash == -1) {
doubleSlash = 0;
} else {
doubleSlash += 2;
}
int firstSlash = url.indexOf('/', doubleSlash);
if (firstSlash != -1) {
url = url.substring(0, firstSlash);
}
Favicon icon = getIconAtRoot(url);
if (icon == null) {
icon = getIconInPage(url);
}
return icon;
}
private Favicon getIconAtRoot(String url) {
byte[] bytes = null;
String contentType = null;
try {
url = Urls.removeTrailingSlash(url) + "/favicon.ico";
log.debug("getting root icon at {}", url);
HttpResult result = getter.get(url);
bytes = result.content();
contentType = result.contentType();
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private Favicon getIconInPage(String url) {
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.content()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon", e);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
String href = icons.getFirst().attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
log.debug("Found unconfirmed iconInPage at {}", href);
byte[] bytes;
String contentType;
try {
HttpResult result = getter.get(href);
bytes = result.content();
contentType = result.contentType();
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href, e);
return null;
}
if (!isValidIconResponse(bytes, contentType)) {
log.debug("Invalid icon found for {}", href);
return null;
}
return new Favicon(bytes, contentType);
}
}

15
commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
View File

@@ -4,6 +4,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.hc.core5.http.NameValuePair;
@@ -19,14 +20,14 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
@Priority(3)
public class FacebookFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("www.facebook.com")) {
return null;
}
@@ -38,23 +39,15 @@ public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
byte[] bytes = null;
String contentType = null;
try {
log.debug("Getting Facebook user's icon, {}", url);
HttpResult iconResult = getter.get(iconUrl);
bytes = iconResult.content();
contentType = iconResult.contentType();
return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private String extractUserName(String url) {

9
commafeed-server/src/main/java/com/commafeed/backend/favicon/FaviconFetcher.java Normal file
View File

@@ -0,0 +1,9 @@
package com.commafeed.backend.favicon;
import com.commafeed.backend.model.Feed;
public interface FaviconFetcher {
Favicon fetch(Feed feed);
}

39
commafeed-server/src/main/java/com/commafeed/backend/favicon/FeedFaviconFetcher.java Normal file
View File

@@ -0,0 +1,39 @@
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Fetch favicon from the url declared in the feed.
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(2)
public class FeedFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getIconUrl();
if (url == null) {
return null;
}
try {
HttpResult result = getter.get(url);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve icon declared in the feed {}", url, e);
return null;
}
}
}

65
commafeed-server/src/main/java/com/commafeed/backend/favicon/HtmlFaviconFetcher.java Normal file
View File

@@ -0,0 +1,65 @@
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Extracts favicon url from html page.
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(1)
public class HtmlFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getLink();
if (url == null) {
return null;
}
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.content()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon", e);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
String href = icons.getFirst().attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
try {
HttpResult result = getter.get(href);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href, e);
return null;
}
}
}

44
commafeed-server/src/main/java/com/commafeed/backend/favicon/RootFaviconFetcher.java Normal file
View File

@@ -0,0 +1,44 @@
package com.commafeed.backend.favicon;
import java.net.URI;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Fetches favicon from root of the domain (e.g. https://example.com/favicon.ico)
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(0)
public class RootFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getLink();
if (url == null) {
url = feed.getUrl();
}
URI uri = URI.create(url);
String faviconUrl = "%s://%s/favicon.ico".formatted(uri.getScheme(), uri.getHost());
try {
log.debug("getting root icon at {}", faviconUrl);
HttpResult result = getter.get(faviconUrl);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
return null;
}
}
}

13
commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
View File

@@ -5,6 +5,7 @@ import java.net.URI;
import java.util.List;
import java.util.Optional;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.UriBuilder;
@@ -29,7 +30,8 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
@Priority(3)
public class YoutubeFaviconFetcher implements FaviconFetcher {
private static final String PART_SNIPPET = "snippet";
@@ -53,8 +55,6 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
return null;
}
byte[] bytes = null;
String contentType = null;
try {
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
@@ -84,16 +84,11 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
}
HttpResult iconResult = getter.get(thumbnailUrl.asText());
bytes = iconResult.content();
contentType = iconResult.contentType();
return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve YouTube icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private byte[] fetchForUser(String googleAuthKey, String userId)