Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-22 03:54:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

add support for declared icons in feeds (#2048)

Browse Source
This commit is contained in:
Athou
2026-05-10 16:24:25 +02:00
parent 1622eb642a
commit 4a8be29616
27 changed files with 715 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
commafeed-server/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java
View File

@@ -1,49 +0,0 @@
package com.commafeed.backend.favicon;
import java.util.Arrays;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public abstract class AbstractFaviconFetcher {
private static final List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
public abstract Favicon fetch(Feed feed);
protected boolean isValidIconResponse(byte[] content, String contentType) {
if (content == null) {
return false;
}
long length = content.length;
if (StringUtils.isNotBlank(contentType)) {
contentType = contentType.split(";")[0];
}
if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
log.debug("Content-Type {} is blacklisted", contentType);
return false;
}
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
return true;
}
}

130
commafeed-server/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java
View File

@@ -1,130 +0,0 @@
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.Urls;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Inspired/Ported from https://github.com/potatolondon/getfavicon
*
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(Integer.MIN_VALUE)
public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
Favicon icon = fetch(feed.getLink());
if (icon == null) {
icon = fetch(feed.getUrl());
}
return icon;
}
private Favicon fetch(String url) {
if (url == null) {
log.debug("url is null");
return null;
}
int doubleSlash = url.indexOf("//");
if (doubleSlash == -1) {
doubleSlash = 0;
} else {
doubleSlash += 2;
}
int firstSlash = url.indexOf('/', doubleSlash);
if (firstSlash != -1) {
url = url.substring(0, firstSlash);
}
Favicon icon = getIconAtRoot(url);
if (icon == null) {
icon = getIconInPage(url);
}
return icon;
}
private Favicon getIconAtRoot(String url) {
byte[] bytes = null;
String contentType = null;
try {
url = Urls.removeTrailingSlash(url) + "/favicon.ico";
log.debug("getting root icon at {}", url);
HttpResult result = getter.get(url);
bytes = result.content();
contentType = result.contentType();
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private Favicon getIconInPage(String url) {
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.content()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon", e);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
String href = icons.getFirst().attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
log.debug("Found unconfirmed iconInPage at {}", href);
byte[] bytes;
String contentType;
try {
HttpResult result = getter.get(href);
bytes = result.content();
contentType = result.contentType();
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href, e);
return null;
}
if (!isValidIconResponse(bytes, contentType)) {
log.debug("Invalid icon found for {}", href);
return null;
}
return new Favicon(bytes, contentType);
}
}

15
commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
View File

@@ -4,6 +4,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.util.List;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.hc.core5.http.NameValuePair;
@@ -19,14 +20,14 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
@Priority(3)
public class FacebookFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("www.facebook.com")) {
return null;
}
@@ -38,23 +39,15 @@ public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
byte[] bytes = null;
String contentType = null;
try {
log.debug("Getting Facebook user's icon, {}", url);
HttpResult iconResult = getter.get(iconUrl);
bytes = iconResult.content();
contentType = iconResult.contentType();
return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve Facebook icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private String extractUserName(String url) {

9
commafeed-server/src/main/java/com/commafeed/backend/favicon/FaviconFetcher.java Normal file
View File

@@ -0,0 +1,9 @@
package com.commafeed.backend.favicon;
import com.commafeed.backend.model.Feed;
public interface FaviconFetcher {
Favicon fetch(Feed feed);
}

39
commafeed-server/src/main/java/com/commafeed/backend/favicon/FeedFaviconFetcher.java Normal file
View File

@@ -0,0 +1,39 @@
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Fetch favicon from the url declared in the feed.
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(2)
public class FeedFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getIconUrl();
if (url == null) {
return null;
}
try {
HttpResult result = getter.get(url);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve icon declared in the feed {}", url, e);
return null;
}
}
}

65
commafeed-server/src/main/java/com/commafeed/backend/favicon/HtmlFaviconFetcher.java Normal file
View File

@@ -0,0 +1,65 @@
package com.commafeed.backend.favicon;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Extracts favicon url from html page.
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(1)
public class HtmlFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getLink();
if (url == null) {
return null;
}
Document doc;
try {
HttpResult result = getter.get(url);
doc = Jsoup.parse(new String(result.content()), url);
} catch (Exception e) {
log.debug("Failed to retrieve page to find icon", e);
return null;
}
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
if (icons.isEmpty()) {
log.debug("No icon found in page {}", url);
return null;
}
String href = icons.getFirst().attr("abs:href");
if (StringUtils.isBlank(href)) {
log.debug("No icon found in page");
return null;
}
try {
HttpResult result = getter.get(href);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve icon found in page {}", href, e);
return null;
}
}
}

44
commafeed-server/src/main/java/com/commafeed/backend/favicon/RootFaviconFetcher.java Normal file
View File

@@ -0,0 +1,44 @@
package com.commafeed.backend.favicon;
import java.net.URI;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
/**
* Fetches favicon from root of the domain (e.g. https://example.com/favicon.ico)
*/
@Slf4j
@RequiredArgsConstructor
@Singleton
@Priority(0)
public class RootFaviconFetcher implements FaviconFetcher {
private final HttpGetter getter;
@Override
public Favicon fetch(Feed feed) {
String url = feed.getLink();
if (url == null) {
url = feed.getUrl();
}
URI uri = URI.create(url);
String faviconUrl = "%s://%s/favicon.ico".formatted(uri.getScheme(), uri.getHost());
try {
log.debug("getting root icon at {}", faviconUrl);
HttpResult result = getter.get(faviconUrl);
return new Favicon(result.content(), result.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
return null;
}
}
}

13
commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
View File

@@ -5,6 +5,7 @@ import java.net.URI;
import java.util.List;
import java.util.Optional;
import jakarta.annotation.Priority;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.UriBuilder;
@@ -29,7 +30,8 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
@Priority(3)
public class YoutubeFaviconFetcher implements FaviconFetcher {
private static final String PART_SNIPPET = "snippet";
@@ -53,8 +55,6 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
return null;
}
byte[] bytes = null;
String contentType = null;
try {
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
@@ -84,16 +84,11 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
}
HttpResult iconResult = getter.get(thumbnailUrl.asText());
bytes = iconResult.content();
contentType = iconResult.contentType();
return new Favicon(iconResult.content(), iconResult.contentType());
} catch (Exception e) {
log.debug("Failed to retrieve YouTube icon", e);
}
if (!isValidIconResponse(bytes, contentType)) {
return null;
}
return new Favicon(bytes, contentType);
}
private byte[] fetchForUser(String googleAuthKey, String userId)

1
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshWorker.java
View File

@@ -69,6 +69,7 @@ public class FeedRefreshWorker {
feed.setUrlAfterRedirect(urlAfterRedirect);
feed.setLink(result.feed().link());
feed.setIconUrl(result.feed().iconUrl());
feed.setLastModifiedHeader(result.lastModifiedHeader());
feed.setEtagHeader(result.lastETagHeader());
feed.setLastContentHash(result.contentHash());

3
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParser.java
View File

@@ -81,6 +81,7 @@ public class FeedParser {
String title = feed.getTitle();
String link = feed.getLink();
String iconUrl = feed.getIcon() != null ? feed.getIcon().getUrl() : null;
List<Entry> entries = buildEntries(feed, feedUrl);
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
@@ -89,7 +90,7 @@ public class FeedParser {
}
Long averageEntryInterval = averageTimeBetweenEntries(entries);
return new FeedParserResult(title, link, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
return new FeedParserResult(title, link, iconUrl, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
} catch (FeedParsingException e) {
throw e;
} catch (Exception e) {

4
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParserResult.java
View File

@@ -3,8 +3,8 @@ package com.commafeed.backend.feed.parser;
import java.time.Instant;
import java.util.List;
public record FeedParserResult(String title, String link, Instant lastPublishedDate, Long averageEntryInterval, Instant lastEntryDate,
List<Entry> entries) {
public record FeedParserResult(String title, String link, String iconUrl, Instant lastPublishedDate, Long averageEntryInterval,
Instant lastEntryDate, List<Entry> entries) {
public record Entry(String guid, String url, Instant published, Content content) {}
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {}

5
commafeed-server/src/main/java/com/commafeed/backend/model/Feed.java
View File

@@ -48,6 +48,11 @@ public class Feed extends AbstractModel {
@JdbcTypeCode(Types.LONGVARCHAR)
private String link;
@Lob
@Column(name = "icon_url", length = Integer.MAX_VALUE)
@JdbcTypeCode(Types.LONGVARCHAR)
private String iconUrl;
/**
* Last time we tried to fetch the feed
*/

71
commafeed-server/src/main/java/com/commafeed/backend/service/FeedFaviconService.java Normal file
View File

@@ -0,0 +1,71 @@
package com.commafeed.backend.service;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import jakarta.inject.Singleton;
import jakarta.ws.rs.core.MediaType;
import org.apache.commons.lang3.ArrayUtils;
import com.commafeed.backend.favicon.Favicon;
import com.commafeed.backend.favicon.FaviconFetcher;
import com.commafeed.backend.model.Feed;
import com.google.common.io.Resources;
import io.quarkus.arc.All;
import lombok.extern.slf4j.Slf4j;
@Singleton
@Slf4j
public class FeedFaviconService {
private static final Set<MediaType> ICON_MIMETYPE_BLACKLIST = Set.of(MediaType.APPLICATION_XML_TYPE, MediaType.TEXT_HTML_TYPE);
private static final long MIN_ICON_LENGTH = 100;
private static final long MAX_ICON_LENGTH = 100000;
private final List<FaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
public FeedFaviconService(@All List<FaviconFetcher> faviconFetchers) throws IOException {
this.faviconFetchers = faviconFetchers;
this.defaultFavicon = new Favicon(
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif");
}
public Favicon fetchFavicon(Feed feed) {
for (FaviconFetcher faviconFetcher : faviconFetchers) {
Favicon icon = faviconFetcher.fetch(feed);
if (isFaviconValid(icon)) {
return icon;
}
}
return defaultFavicon;
}
private static boolean isFaviconValid(Favicon favicon) {
if (favicon == null || ArrayUtils.isEmpty(favicon.icon())) {
return false;
}
long length = favicon.icon().length;
if (length < MIN_ICON_LENGTH) {
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
return false;
}
if (length > MAX_ICON_LENGTH) {
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
return false;
}
if (ICON_MIMETYPE_BLACKLIST.stream().anyMatch(bl -> bl.isCompatible(favicon.mediaType()))) {
log.debug("Content-Type {} is blacklisted", favicon.mediaType());
return false;
}
return true;
}
}

33
commafeed-server/src/main/java/com/commafeed/backend/service/FeedService.java
View File

@@ -1,37 +1,23 @@
package com.commafeed.backend.service;
import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.Urls;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
import com.commafeed.backend.favicon.Favicon;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.Models;
import com.google.common.io.Resources;
import io.quarkus.arc.All;
import lombok.RequiredArgsConstructor;
@Singleton
@RequiredArgsConstructor
public class FeedService {
private final FeedDAO feedDAO;
private final List<AbstractFaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
public FeedService(FeedDAO feedDAO, @All List<AbstractFaviconFetcher> faviconFetchers) throws IOException {
this.feedDAO = feedDAO;
this.faviconFetchers = faviconFetchers;
this.defaultFavicon = new Favicon(
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif");
}
public synchronized Feed findOrCreate(String url) {
String normalizedUrl = Urls.normalize(url);
@@ -57,19 +43,4 @@ public class FeedService {
feedDAO.merge(feed);
}
public Favicon fetchFavicon(Feed feed) {
Favicon icon = null;
for (AbstractFaviconFetcher faviconFetcher : faviconFetchers) {
icon = faviconFetcher.fetch(feed);
if (icon != null) {
break;
}
}
if (icon == null) {
icon = defaultFavicon;
}
return icon;
}
}

18
commafeed-server/src/main/java/com/commafeed/frontend/resource/FeedREST.java
View File

@@ -62,7 +62,7 @@ import com.commafeed.backend.opml.OPMLImporter;
import com.commafeed.backend.service.FeedEntryFilteringService;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.commafeed.backend.service.FeedFaviconService;
import com.commafeed.backend.service.FeedSubscriptionService;
import com.commafeed.backend.service.FeedSubscriptionService.ForceFeedRefreshTooSoonException;
import com.commafeed.frontend.model.Entries;
@@ -106,7 +106,7 @@ public class FeedREST {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedFetcher feedFetcher;
private final FeedService feedService;
private final FeedFaviconService feedFaviconService;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionService feedSubscriptionService;
private final FeedEntryFilteringService feedEntryFilteringService;
@@ -339,7 +339,19 @@ public class FeedREST {
}
Feed feed = subscription.getFeed();
Favicon icon = feedService.fetchFavicon(feed);
if (feed.getLastUpdated() == null) {
// the feed has never been fetched yet so the iconUrl field hasn't been updated yet, and the icon can't be fetched
// this can happen for newly subscribed feeds:
// the feed has been added in the tree in the web client and the favicon is being fetched,
// but the feed has not been fetched yet
// the client is configured to retry in that case
return Response.status(Status.SERVICE_UNAVAILABLE)
.entity("Feed has not been fetched yet, please retry in a bit")
.type(MediaType.TEXT_PLAIN)
.build();
}
Favicon icon = feedFaviconService.fetchFavicon(feed);
return Response.ok(icon.icon(), icon.mediaType()).build();
}

6
commafeed-server/src/main/java/com/commafeed/frontend/resource/fever/FeverREST.java
View File

@@ -45,7 +45,7 @@ import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserSettings.ReadingOrder;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.commafeed.backend.service.FeedFaviconService;
import com.commafeed.backend.service.UserService;
import com.commafeed.frontend.resource.fever.FeverResponse.FeverFavicon;
import com.commafeed.frontend.resource.fever.FeverResponse.FeverFeed;
@@ -80,7 +80,7 @@ public class FeverREST {
private final UserService userService;
private final FeedEntryService feedEntryService;
private final FeedService feedService;
private final FeedFaviconService feedFaviconService;
private final FeedEntryDAO feedEntryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedCategoryDAO feedCategoryDAO;
@@ -303,7 +303,7 @@ public class FeverREST {
private List<FeverFavicon> buildFavicons(List<FeedSubscription> subscriptions) {
return subscriptions.stream().map(s -> {
Favicon favicon = feedService.fetchFavicon(s.getFeed());
Favicon favicon = feedFaviconService.fetchFavicon(s.getFeed());
FeverFavicon f = new FeverFavicon();
f.setId(s.getFeed().getId());

14
commafeed-server/src/main/resources/changelogs/db.changelog-7.2.xml Normal file
View File

@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd">
<changeSet id="feed-icons" author="athou">
<addColumn tableName="FEEDS">
<column name="icon_url" type="CLOB">
<constraints nullable="true" />
</column>
</addColumn>
</changeSet>
</databaseChangeLog>

1
commafeed-server/src/main/resources/migrations.xml
View File

@@ -38,5 +38,6 @@
<include file="changelogs/db.changelog-5.11.xml" />
<include file="changelogs/db.changelog-5.12.xml" />
<include file="changelogs/db.changelog-7.0.xml" />
<include file="changelogs/db.changelog-7.2.xml" />
</databaseChangeLog>

29
commafeed-server/src/test/java/com/commafeed/backend/favicon/FacebookFaviconFetcherTest.java
View File

@@ -75,33 +75,4 @@ class FacebookFaviconFetcherTest {
Assertions.assertNull(faviconFetcher.fetch(feed));
}
@Test
void testFetchWithInvalidIconResponse() throws Exception {
Feed feed = new Feed();
feed.setUrl("https://www.facebook.com/something?id=validUserId");
// Create a byte array that's too small
byte[] iconBytes = new byte[50];
String contentType = "image/png";
HttpResult httpResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
Mockito.when(httpGetter.get("https://graph.facebook.com/validUserId/picture?type=square&height=16")).thenReturn(httpResult);
Assertions.assertNull(faviconFetcher.fetch(feed));
}
@Test
void testFetchWithBlacklistedContentType() throws Exception {
Feed feed = new Feed();
feed.setUrl("https://www.facebook.com/something?id=validUserId");
byte[] iconBytes = new byte[1000];
String contentType = "application/xml"; // Blacklisted content type
HttpResult httpResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
Mockito.when(httpGetter.get("https://graph.facebook.com/validUserId/picture?type=square&height=16")).thenReturn(httpResult);
Assertions.assertNull(faviconFetcher.fetch(feed));
}
}

69
commafeed-server/src/test/java/com/commafeed/backend/favicon/FeedFaviconFetcherTest.java Normal file
View File

@@ -0,0 +1,69 @@
package com.commafeed.backend.favicon;
import java.time.Duration;
import jakarta.ws.rs.core.MediaType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
@ExtendWith(MockitoExtension.class)
class FeedFaviconFetcherTest {
@Mock
private HttpGetter httpGetter;
private FeedFaviconFetcher faviconFetcher;
@BeforeEach
void init() {
faviconFetcher = new FeedFaviconFetcher(httpGetter);
}
@Test
void testFetchWithNullIconUrl() {
Feed feed = new Feed();
feed.setUrl("https://example.com/feed");
Assertions.assertNull(faviconFetcher.fetch(feed));
Mockito.verifyNoInteractions(httpGetter);
}
@Test
void testFetchWithValidIconUrl() throws Exception {
Feed feed = new Feed();
feed.setUrl("https://example.com/feed");
feed.setIconUrl("https://example.com/icon.png");
byte[] iconBytes = new byte[1000];
String contentType = "image/png";
HttpResult httpResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
Mockito.when(httpGetter.get("https://example.com/icon.png")).thenReturn(httpResult);
Favicon result = faviconFetcher.fetch(feed);
Assertions.assertNotNull(result);
Assertions.assertEquals(iconBytes, result.icon());
Assertions.assertTrue(result.mediaType().isCompatible(MediaType.valueOf(contentType)));
}
@Test
void testFetchWithHttpGetterException() throws Exception {
Feed feed = new Feed();
feed.setUrl("https://example.com/feed");
feed.setIconUrl("https://example.com/icon.png");
Mockito.when(httpGetter.get("https://example.com/icon.png")).thenThrow(new RuntimeException("Network error"));
Assertions.assertNull(faviconFetcher.fetch(feed));
}
}

122
commafeed-server/src/test/java/com/commafeed/backend/favicon/HtmlFaviconFetcherTest.java Normal file
View File

@@ -0,0 +1,122 @@
package com.commafeed.backend.favicon;
import java.time.Duration;
import jakarta.ws.rs.core.MediaType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
@ExtendWith(MockitoExtension.class)
class HtmlFaviconFetcherTest {
@Mock
private HttpGetter httpGetter;
private HtmlFaviconFetcher faviconFetcher;
@BeforeEach
void init() {
faviconFetcher = new HtmlFaviconFetcher(httpGetter);
}
@Test
void testFetchWithNullLink() {
Feed feed = new Feed();
feed.setUrl("https://example.com/feed");
Assertions.assertNull(faviconFetcher.fetch(feed));
Mockito.verifyNoInteractions(httpGetter);
}
@Test
void testFetchWithValidIconLink() throws Exception {
Feed feed = new Feed();
feed.setUrl("https://example.com/feed");
feed.setLink("https://example.com");
String html = "<html><head><link rel=\"icon\" href=\"/favicon.png\" /></head><body></body></html>";
HttpResult pageResult = new HttpResult(html.getBytes(), "text/html", null, null, null, Duration.ZERO);
Mockito.when(httpGetter.get("https://example.com")).thenReturn(pageResult);
byte[] iconBytes = new byte[1000];
String contentType = "image/png";
HttpResult iconResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
Mockito.when(httpGetter.get("https://example.com/favicon.png")).thenReturn(iconResult);
Favicon result = faviconFetcher.fetch(feed);
Assertions.assertNotNull(result);
Assertions.assertEquals(iconBytes, result.icon());
Assertions.assertTrue(result.mediaType().isCompatible(MediaType.valueOf(contentType)));
}
@Test
void testFetchWithShortcutIconLink() throws Exception {
Feed feed = new Feed();
feed.setUrl("https://example.com/feed");
feed.setLink("https://example.com");
String html = "<html><head><link rel=\"shortcut icon\" href=\"https://example.com/shortcut-favicon.ico\" /></head><body></body></html>";
HttpResult pageResult = new HttpResult(html.getBytes(), "text/html", null, null, null, Duration.ZERO);
Mockito.when(httpGetter.get("https://example.com")).thenReturn(pageResult);
byte[] iconBytes = new byte[1000];
String contentType = "image/x-icon";
HttpResult iconResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
Mockito.when(httpGetter.get("https://example.com/shortcut-favicon.ico")).thenReturn(iconResult);
Favicon result = faviconFetcher.fetch(feed);
Assertions.assertNotNull(result);
Assertions.assertEquals(iconBytes, result.icon());
Assertions.assertTrue(result.mediaType().isCompatible(MediaType.valueOf(contentType)));
}
@Test
void testFetchWithNoIconInPage() throws Exception {
Feed feed = new Feed();
feed.setUrl("https://example.com/feed");
feed.setLink("https://example.com");
String html = "<html><head></head><body></body></html>";
HttpResult pageResult = new HttpResult(html.getBytes(), "text/html", null, null, null, Duration.ZERO);
Mockito.when(httpGetter.get("https://example.com")).thenReturn(pageResult);
Assertions.assertNull(faviconFetcher.fetch(feed));
}
@Test
void testFetchWithPageFetchException() throws Exception {
Feed feed = new Feed();
feed.setUrl("https://example.com/feed");
feed.setLink("https://example.com");
Mockito.when(httpGetter.get("https://example.com")).thenThrow(new RuntimeException("Network error"));
Assertions.assertNull(faviconFetcher.fetch(feed));
}
@Test
void testFetchWithIconFetchException() throws Exception {
Feed feed = new Feed();
feed.setUrl("https://example.com/feed");
feed.setLink("https://example.com");
String html = "<html><head><link rel=\"icon\" href=\"https://example.com/favicon.png\" /></head><body></body></html>";
HttpResult pageResult = new HttpResult(html.getBytes(), "text/html", null, null, null, Duration.ZERO);
Mockito.when(httpGetter.get("https://example.com")).thenReturn(pageResult);
Mockito.when(httpGetter.get("https://example.com/favicon.png")).thenThrow(new RuntimeException("Network error"));
Assertions.assertNull(faviconFetcher.fetch(feed));
}
}

77
commafeed-server/src/test/java/com/commafeed/backend/favicon/RootFaviconFetcherTest.java Normal file
View File

@@ -0,0 +1,77 @@
package com.commafeed.backend.favicon;
import java.time.Duration;
import jakarta.ws.rs.core.MediaType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
@ExtendWith(MockitoExtension.class)
class RootFaviconFetcherTest {
@Mock
private HttpGetter httpGetter;
private RootFaviconFetcher faviconFetcher;
@BeforeEach
void init() {
faviconFetcher = new RootFaviconFetcher(httpGetter);
}
@Test
void testFetchUsesLinkWhenAvailable() throws Exception {
Feed feed = new Feed();
feed.setUrl("https://feeds.example.com/feed");
feed.setLink("https://www.example.com/blog");
byte[] iconBytes = new byte[1000];
String contentType = "image/x-icon";
HttpResult httpResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
Mockito.when(httpGetter.get("https://www.example.com/favicon.ico")).thenReturn(httpResult);
Favicon result = faviconFetcher.fetch(feed);
Assertions.assertNotNull(result);
Assertions.assertEquals(iconBytes, result.icon());
Assertions.assertTrue(result.mediaType().isCompatible(MediaType.valueOf(contentType)));
}
@Test
void testFetchFallsBackToUrlWhenLinkIsNull() throws Exception {
Feed feed = new Feed();
feed.setUrl("https://example.com/feed.xml");
byte[] iconBytes = new byte[1000];
String contentType = "image/x-icon";
HttpResult httpResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
Mockito.when(httpGetter.get("https://example.com/favicon.ico")).thenReturn(httpResult);
Favicon result = faviconFetcher.fetch(feed);
Assertions.assertNotNull(result);
Assertions.assertEquals(iconBytes, result.icon());
Assertions.assertTrue(result.mediaType().isCompatible(MediaType.valueOf(contentType)));
}
@Test
void testFetchWithHttpGetterException() throws Exception {
Feed feed = new Feed();
feed.setUrl("https://example.com/feed.xml");
feed.setLink("https://example.com");
Mockito.when(httpGetter.get("https://example.com/favicon.ico")).thenThrow(new RuntimeException("Network error"));
Assertions.assertNull(faviconFetcher.fetch(feed));
}
}

25
commafeed-server/src/test/java/com/commafeed/backend/favicon/YoutubeFaviconFetcherTest.java
View File

@@ -168,31 +168,6 @@ class YoutubeFaviconFetcherTest {
Assertions.assertNull(faviconFetcher.fetch(feed));
}
@Test
void testFetchWithInvalidIconResponse() throws Exception {
Feed feed = new Feed();
feed.setUrl("https://youtube.com/feeds/videos.xml?user=testUser");
Mockito.when(config.googleAuthKey()).thenReturn(Optional.of("test-api-key"));
byte[] apiResponse = """
{"items":[{"snippet":{"thumbnails":{"default":{"url":"https://example.com/icon.png"}}}}]}""".getBytes();
HttpResult apiHttpResult = new HttpResult(apiResponse, "application/json", null, null, null, Duration.ZERO);
Mockito.when(httpGetter.get("https://www.googleapis.com/youtube/v3/channels?part=snippet&key=test-api-key&forUsername=testUser"))
.thenReturn(apiHttpResult);
JsonNode jsonNode = new ObjectMapper().readTree(apiResponse);
Mockito.when(objectMapper.readTree(apiResponse)).thenReturn(jsonNode);
// Create a byte array that's too small
byte[] iconBytes = new byte[50];
String contentType = "image/png";
HttpResult iconHttpResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
Mockito.when(httpGetter.get("https://example.com/icon.png")).thenReturn(iconHttpResult);
Assertions.assertNull(faviconFetcher.fetch(feed));
}
@Test
void testFetchWithEmptyApiResponse() throws Exception {
Feed feed = new Feed();

2
commafeed-server/src/test/java/com/commafeed/backend/feed/FeedFetcherTest.java
View File

@@ -53,7 +53,7 @@ class FeedFetcherTest {
byte[] feed = "feed".getBytes();
Mockito.when(getter.get(HttpGetter.HttpRequest.builder(feedUrl).build()))
.thenReturn(new HttpResult(feed, "application/atom+xml", null, null, feedUrl, Duration.ZERO));
Mockito.when(parser.parse(feedUrl, feed)).thenReturn(new FeedParserResult("title", "link", null, null, null, null));
Mockito.when(parser.parse(feedUrl, feed)).thenReturn(new FeedParserResult("title", "link", "iconUrl", null, null, null, null));
Mockito.when(urlProvider.get(htmlUrl, new String(html))).thenReturn(List.of(feedUrl));

151
commafeed-server/src/test/java/com/commafeed/backend/service/FeedFaviconServiceTest.java Normal file
View File

@@ -0,0 +1,151 @@
package com.commafeed.backend.service;
import java.io.IOException;
import java.util.List;
import jakarta.ws.rs.core.MediaType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import com.commafeed.backend.favicon.Favicon;
import com.commafeed.backend.favicon.FaviconFetcher;
import com.commafeed.backend.model.Feed;
@ExtendWith(MockitoExtension.class)
class FeedFaviconServiceTest {
@Mock
private FaviconFetcher fetcher1;
@Mock
private FaviconFetcher fetcher2;
private FeedFaviconService service;
private Feed feed;
@BeforeEach
void init() throws IOException {
service = new FeedFaviconService(List.of(fetcher1, fetcher2));
feed = new Feed();
feed.setUrl("https://example.com/feed");
}
@Test
void testReturnsFirstValidFavicon() {
byte[] iconBytes = new byte[1000];
Favicon validFavicon = new Favicon(iconBytes, "image/png");
Mockito.when(fetcher1.fetch(feed)).thenReturn(validFavicon);
Favicon result = service.fetchFavicon(feed);
Assertions.assertEquals(validFavicon, result);
Mockito.verify(fetcher1).fetch(feed);
Mockito.verifyNoInteractions(fetcher2);
}
@Test
void testFallsBackToNextFetcherWhenFirstReturnsNull() {
byte[] iconBytes = new byte[1000];
Favicon validFavicon = new Favicon(iconBytes, "image/png");
Mockito.when(fetcher1.fetch(feed)).thenReturn(null);
Mockito.when(fetcher2.fetch(feed)).thenReturn(validFavicon);
Favicon result = service.fetchFavicon(feed);
Assertions.assertEquals(validFavicon, result);
}
@Test
void testFallsBackToNextFetcherWhenFirstReturnsTooSmallIcon() {
byte[] tinyIcon = new byte[50];
Favicon tinyFavicon = new Favicon(tinyIcon, "image/png");
byte[] validIcon = new byte[1000];
Favicon validFavicon = new Favicon(validIcon, "image/png");
Mockito.when(fetcher1.fetch(feed)).thenReturn(tinyFavicon);
Mockito.when(fetcher2.fetch(feed)).thenReturn(validFavicon);
Favicon result = service.fetchFavicon(feed);
Assertions.assertEquals(validFavicon, result);
}
@Test
void testFallsBackToNextFetcherWhenFirstReturnsTooLargeIcon() {
byte[] hugeIcon = new byte[100001];
Favicon hugeFavicon = new Favicon(hugeIcon, "image/png");
byte[] validIcon = new byte[1000];
Favicon validFavicon = new Favicon(validIcon, "image/png");
Mockito.when(fetcher1.fetch(feed)).thenReturn(hugeFavicon);
Mockito.when(fetcher2.fetch(feed)).thenReturn(validFavicon);
Favicon result = service.fetchFavicon(feed);
Assertions.assertEquals(validFavicon, result);
}
@Test
void testFallsBackToNextFetcherWhenFirstReturnsBlacklistedContentType() {
byte[] iconBytes = new byte[1000];
Favicon xmlFavicon = new Favicon(iconBytes, "application/xml");
byte[] validIcon = new byte[1000];
Favicon validFavicon = new Favicon(validIcon, "image/png");
Mockito.when(fetcher1.fetch(feed)).thenReturn(xmlFavicon);
Mockito.when(fetcher2.fetch(feed)).thenReturn(validFavicon);
Favicon result = service.fetchFavicon(feed);
Assertions.assertEquals(validFavicon, result);
}
@Test
void testFallsBackToNextFetcherWhenFirstReturnsHtmlContentType() {
byte[] iconBytes = new byte[1000];
Favicon htmlFavicon = new Favicon(iconBytes, "text/html");
byte[] validIcon = new byte[1000];
Favicon validFavicon = new Favicon(validIcon, "image/png");
Mockito.when(fetcher1.fetch(feed)).thenReturn(htmlFavicon);
Mockito.when(fetcher2.fetch(feed)).thenReturn(validFavicon);
Favicon result = service.fetchFavicon(feed);
Assertions.assertEquals(validFavicon, result);
}
@Test
void testReturnsDefaultFaviconWhenAllFetchersFail() {
Mockito.when(fetcher1.fetch(feed)).thenReturn(null);
Mockito.when(fetcher2.fetch(feed)).thenReturn(null);
Favicon result = service.fetchFavicon(feed);
Assertions.assertNotNull(result);
Assertions.assertTrue(result.mediaType().isCompatible(MediaType.valueOf("image/gif")));
Assertions.assertTrue(result.icon().length > 0);
}
@Test
void testReturnsDefaultFaviconWhenNoFetchersRegistered() throws IOException {
FeedFaviconService emptyService = new FeedFaviconService(List.of());
Favicon result = emptyService.fetchFavicon(feed);
Assertions.assertNotNull(result);
Assertions.assertTrue(result.mediaType().isCompatible(MediaType.valueOf("image/gif")));
}
}