mirror of
https://github.com/Athou/commafeed.git
synced 2026-09-29 15:34:51 +00:00
add support for declared icons in feeds (#2048)
This commit is contained in:
@@ -14,6 +14,7 @@ interface ImageWithPlaceholderWhileLoadingProps {
|
|||||||
placeholderHeight?: number
|
placeholderHeight?: number
|
||||||
placeholderBackgroundColor?: string
|
placeholderBackgroundColor?: string
|
||||||
placeholderIconSize?: number
|
placeholderIconSize?: number
|
||||||
|
onError?: (e: React.SyntheticEvent<HTMLImageElement, Event>) => void
|
||||||
}
|
}
|
||||||
|
|
||||||
const useStyles = tss
|
const useStyles = tss
|
||||||
@@ -44,6 +45,7 @@ export function ImageWithPlaceholderWhileLoading({
|
|||||||
title,
|
title,
|
||||||
width,
|
width,
|
||||||
style,
|
style,
|
||||||
|
onError,
|
||||||
}: Readonly<ImageWithPlaceholderWhileLoadingProps>) {
|
}: Readonly<ImageWithPlaceholderWhileLoadingProps>) {
|
||||||
const { classes } = useStyles({
|
const { classes } = useStyles({
|
||||||
placeholderWidth,
|
placeholderWidth,
|
||||||
@@ -70,6 +72,7 @@ export function ImageWithPlaceholderWhileLoading({
|
|||||||
width={width}
|
width={width}
|
||||||
height={height}
|
height={height}
|
||||||
onLoad={() => setLoading(false)}
|
onLoad={() => setLoading(false)}
|
||||||
|
onError={onError}
|
||||||
style={{
|
style={{
|
||||||
...style,
|
...style,
|
||||||
display: loading ? "none" : (style?.display ?? "initial"),
|
display: loading ? "none" : (style?.display ?? "initial"),
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import { useTimeout } from "@mantine/hooks"
|
||||||
|
import { useState } from "react"
|
||||||
import { ImageWithPlaceholderWhileLoading } from "@/components/ImageWithPlaceholderWhileLoading"
|
import { ImageWithPlaceholderWhileLoading } from "@/components/ImageWithPlaceholderWhileLoading"
|
||||||
|
|
||||||
export interface FeedFaviconProps {
|
export interface FeedFaviconProps {
|
||||||
@@ -6,9 +8,16 @@ export interface FeedFaviconProps {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function FeedFavicon({ url, size = 18 }: Readonly<FeedFaviconProps>) {
|
export function FeedFavicon({ url, size = 18 }: Readonly<FeedFaviconProps>) {
|
||||||
|
// the backend always returns a favicon except when the feed has never been fetched
|
||||||
|
// this can happen when the user subscribes to a feed, the feed is added to the tree but the feed has not been fetched yet
|
||||||
|
// in this case we just retry every second until the feed is fetched and the favicon is available
|
||||||
|
const [timestamp, setTimestamp] = useState(0)
|
||||||
|
const { start: retry } = useTimeout(() => setTimestamp(Date.now()), 1000)
|
||||||
|
|
||||||
|
const urlWithTimestamp = url + (timestamp === 0 ? "" : `?t=${timestamp}`)
|
||||||
return (
|
return (
|
||||||
<ImageWithPlaceholderWhileLoading
|
<ImageWithPlaceholderWhileLoading
|
||||||
src={url}
|
src={urlWithTimestamp}
|
||||||
alt="feed favicon"
|
alt="feed favicon"
|
||||||
width={size}
|
width={size}
|
||||||
height={size}
|
height={size}
|
||||||
@@ -16,6 +25,7 @@ export function FeedFavicon({ url, size = 18 }: Readonly<FeedFaviconProps>) {
|
|||||||
placeholderHeight={size}
|
placeholderHeight={size}
|
||||||
placeholderBackgroundColor="inherit"
|
placeholderBackgroundColor="inherit"
|
||||||
placeholderIconSize={size}
|
placeholderIconSize={size}
|
||||||
|
onError={retry}
|
||||||
/>
|
/>
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,49 +0,0 @@
|
|||||||
package com.commafeed.backend.favicon;
|
|
||||||
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.List;
|
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
|
|
||||||
import com.commafeed.backend.model.Feed;
|
|
||||||
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
@Slf4j
|
|
||||||
public abstract class AbstractFaviconFetcher {
|
|
||||||
|
|
||||||
private static final List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
|
|
||||||
private static final long MIN_ICON_LENGTH = 100;
|
|
||||||
private static final long MAX_ICON_LENGTH = 100000;
|
|
||||||
|
|
||||||
public abstract Favicon fetch(Feed feed);
|
|
||||||
|
|
||||||
protected boolean isValidIconResponse(byte[] content, String contentType) {
|
|
||||||
if (content == null) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
long length = content.length;
|
|
||||||
|
|
||||||
if (StringUtils.isNotBlank(contentType)) {
|
|
||||||
contentType = contentType.split(";")[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
|
|
||||||
log.debug("Content-Type {} is blacklisted", contentType);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (length < MIN_ICON_LENGTH) {
|
|
||||||
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (length > MAX_ICON_LENGTH) {
|
|
||||||
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,130 +0,0 @@
|
|||||||
package com.commafeed.backend.favicon;
|
|
||||||
|
|
||||||
import jakarta.annotation.Priority;
|
|
||||||
import jakarta.inject.Singleton;
|
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
import org.jsoup.Jsoup;
|
|
||||||
import org.jsoup.nodes.Document;
|
|
||||||
import org.jsoup.select.Elements;
|
|
||||||
|
|
||||||
import com.commafeed.backend.HttpGetter;
|
|
||||||
import com.commafeed.backend.HttpGetter.HttpResult;
|
|
||||||
import com.commafeed.backend.Urls;
|
|
||||||
import com.commafeed.backend.model.Feed;
|
|
||||||
|
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
import lombok.extern.slf4j.Slf4j;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Inspired/Ported from https://github.com/potatolondon/getfavicon
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
@Slf4j
|
|
||||||
@RequiredArgsConstructor
|
|
||||||
@Singleton
|
|
||||||
@Priority(Integer.MIN_VALUE)
|
|
||||||
public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
|
|
||||||
|
|
||||||
private final HttpGetter getter;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Favicon fetch(Feed feed) {
|
|
||||||
Favicon icon = fetch(feed.getLink());
|
|
||||||
if (icon == null) {
|
|
||||||
icon = fetch(feed.getUrl());
|
|
||||||
}
|
|
||||||
return icon;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Favicon fetch(String url) {
|
|
||||||
if (url == null) {
|
|
||||||
log.debug("url is null");
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
int doubleSlash = url.indexOf("//");
|
|
||||||
if (doubleSlash == -1) {
|
|
||||||
doubleSlash = 0;
|
|
||||||
} else {
|
|
||||||
doubleSlash += 2;
|
|
||||||
}
|
|
||||||
int firstSlash = url.indexOf('/', doubleSlash);
|
|
||||||
if (firstSlash != -1) {
|
|
||||||
url = url.substring(0, firstSlash);
|
|
||||||
}
|
|
||||||
|
|
||||||
Favicon icon = getIconAtRoot(url);
|
|
||||||
|
|
||||||
if (icon == null) {
|
|
||||||
icon = getIconInPage(url);
|
|
||||||
}
|
|
||||||
|
|
||||||
return icon;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Favicon getIconAtRoot(String url) {
|
|
||||||
byte[] bytes = null;
|
|
||||||
String contentType = null;
|
|
||||||
|
|
||||||
try {
|
|
||||||
url = Urls.removeTrailingSlash(url) + "/favicon.ico";
|
|
||||||
log.debug("getting root icon at {}", url);
|
|
||||||
HttpResult result = getter.get(url);
|
|
||||||
bytes = result.content();
|
|
||||||
contentType = result.contentType();
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isValidIconResponse(bytes, contentType)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return new Favicon(bytes, contentType);
|
|
||||||
}
|
|
||||||
|
|
||||||
private Favicon getIconInPage(String url) {
|
|
||||||
|
|
||||||
Document doc;
|
|
||||||
try {
|
|
||||||
HttpResult result = getter.get(url);
|
|
||||||
doc = Jsoup.parse(new String(result.content()), url);
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.debug("Failed to retrieve page to find icon", e);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
|
|
||||||
|
|
||||||
if (icons.isEmpty()) {
|
|
||||||
log.debug("No icon found in page {}", url);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
String href = icons.getFirst().attr("abs:href");
|
|
||||||
if (StringUtils.isBlank(href)) {
|
|
||||||
log.debug("No icon found in page");
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug("Found unconfirmed iconInPage at {}", href);
|
|
||||||
|
|
||||||
byte[] bytes;
|
|
||||||
String contentType;
|
|
||||||
try {
|
|
||||||
HttpResult result = getter.get(href);
|
|
||||||
bytes = result.content();
|
|
||||||
contentType = result.contentType();
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.debug("Failed to retrieve icon found in page {}", href, e);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!isValidIconResponse(bytes, contentType)) {
|
|
||||||
log.debug("Invalid icon found for {}", href);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return new Favicon(bytes, contentType);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -4,6 +4,7 @@ import java.net.URI;
|
|||||||
import java.net.URISyntaxException;
|
import java.net.URISyntaxException;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
|
import jakarta.annotation.Priority;
|
||||||
import jakarta.inject.Singleton;
|
import jakarta.inject.Singleton;
|
||||||
|
|
||||||
import org.apache.hc.core5.http.NameValuePair;
|
import org.apache.hc.core5.http.NameValuePair;
|
||||||
@@ -19,14 +20,14 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
@Slf4j
|
@Slf4j
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@Singleton
|
@Singleton
|
||||||
public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
|
@Priority(3)
|
||||||
|
public class FacebookFaviconFetcher implements FaviconFetcher {
|
||||||
|
|
||||||
private final HttpGetter getter;
|
private final HttpGetter getter;
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Favicon fetch(Feed feed) {
|
public Favicon fetch(Feed feed) {
|
||||||
String url = feed.getUrl();
|
String url = feed.getUrl();
|
||||||
|
|
||||||
if (!url.toLowerCase().contains("www.facebook.com")) {
|
if (!url.toLowerCase().contains("www.facebook.com")) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -38,23 +39,15 @@ public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
|
|||||||
|
|
||||||
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
|
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
|
||||||
|
|
||||||
byte[] bytes = null;
|
|
||||||
String contentType = null;
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
log.debug("Getting Facebook user's icon, {}", url);
|
log.debug("Getting Facebook user's icon, {}", url);
|
||||||
|
|
||||||
HttpResult iconResult = getter.get(iconUrl);
|
HttpResult iconResult = getter.get(iconUrl);
|
||||||
bytes = iconResult.content();
|
return new Favicon(iconResult.content(), iconResult.contentType());
|
||||||
contentType = iconResult.contentType();
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.debug("Failed to retrieve Facebook icon", e);
|
log.debug("Failed to retrieve Facebook icon", e);
|
||||||
}
|
|
||||||
|
|
||||||
if (!isValidIconResponse(bytes, contentType)) {
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return new Favicon(bytes, contentType);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private String extractUserName(String url) {
|
private String extractUserName(String url) {
|
||||||
|
|||||||
@@ -0,0 +1,9 @@
|
|||||||
|
package com.commafeed.backend.favicon;
|
||||||
|
|
||||||
|
import com.commafeed.backend.model.Feed;
|
||||||
|
|
||||||
|
public interface FaviconFetcher {
|
||||||
|
|
||||||
|
Favicon fetch(Feed feed);
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package com.commafeed.backend.favicon;
|
||||||
|
|
||||||
|
import jakarta.annotation.Priority;
|
||||||
|
import jakarta.inject.Singleton;
|
||||||
|
|
||||||
|
import com.commafeed.backend.HttpGetter;
|
||||||
|
import com.commafeed.backend.HttpGetter.HttpResult;
|
||||||
|
import com.commafeed.backend.model.Feed;
|
||||||
|
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetch favicon from the url declared in the feed.
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Singleton
|
||||||
|
@Priority(2)
|
||||||
|
public class FeedFaviconFetcher implements FaviconFetcher {
|
||||||
|
|
||||||
|
private final HttpGetter getter;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Favicon fetch(Feed feed) {
|
||||||
|
String url = feed.getIconUrl();
|
||||||
|
if (url == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
HttpResult result = getter.get(url);
|
||||||
|
return new Favicon(result.content(), result.contentType());
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.debug("Failed to retrieve icon declared in the feed {}", url, e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
package com.commafeed.backend.favicon;
|
||||||
|
|
||||||
|
import jakarta.annotation.Priority;
|
||||||
|
import jakarta.inject.Singleton;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.jsoup.Jsoup;
|
||||||
|
import org.jsoup.nodes.Document;
|
||||||
|
import org.jsoup.select.Elements;
|
||||||
|
|
||||||
|
import com.commafeed.backend.HttpGetter;
|
||||||
|
import com.commafeed.backend.HttpGetter.HttpResult;
|
||||||
|
import com.commafeed.backend.model.Feed;
|
||||||
|
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extracts favicon url from html page.
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Singleton
|
||||||
|
@Priority(1)
|
||||||
|
public class HtmlFaviconFetcher implements FaviconFetcher {
|
||||||
|
|
||||||
|
private final HttpGetter getter;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Favicon fetch(Feed feed) {
|
||||||
|
String url = feed.getLink();
|
||||||
|
if (url == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
Document doc;
|
||||||
|
try {
|
||||||
|
HttpResult result = getter.get(url);
|
||||||
|
doc = Jsoup.parse(new String(result.content()), url);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.debug("Failed to retrieve page to find icon", e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
|
||||||
|
if (icons.isEmpty()) {
|
||||||
|
log.debug("No icon found in page {}", url);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
String href = icons.getFirst().attr("abs:href");
|
||||||
|
if (StringUtils.isBlank(href)) {
|
||||||
|
log.debug("No icon found in page");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
HttpResult result = getter.get(href);
|
||||||
|
return new Favicon(result.content(), result.contentType());
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.debug("Failed to retrieve icon found in page {}", href, e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
package com.commafeed.backend.favicon;
|
||||||
|
|
||||||
|
import java.net.URI;
|
||||||
|
|
||||||
|
import jakarta.annotation.Priority;
|
||||||
|
import jakarta.inject.Singleton;
|
||||||
|
|
||||||
|
import com.commafeed.backend.HttpGetter;
|
||||||
|
import com.commafeed.backend.HttpGetter.HttpResult;
|
||||||
|
import com.commafeed.backend.model.Feed;
|
||||||
|
|
||||||
|
import lombok.RequiredArgsConstructor;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fetches favicon from root of the domain (e.g. https://example.com/favicon.ico)
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@RequiredArgsConstructor
|
||||||
|
@Singleton
|
||||||
|
@Priority(0)
|
||||||
|
public class RootFaviconFetcher implements FaviconFetcher {
|
||||||
|
|
||||||
|
private final HttpGetter getter;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Favicon fetch(Feed feed) {
|
||||||
|
String url = feed.getLink();
|
||||||
|
if (url == null) {
|
||||||
|
url = feed.getUrl();
|
||||||
|
}
|
||||||
|
|
||||||
|
URI uri = URI.create(url);
|
||||||
|
String faviconUrl = "%s://%s/favicon.ico".formatted(uri.getScheme(), uri.getHost());
|
||||||
|
try {
|
||||||
|
log.debug("getting root icon at {}", faviconUrl);
|
||||||
|
HttpResult result = getter.get(faviconUrl);
|
||||||
|
return new Favicon(result.content(), result.contentType());
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import java.net.URI;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Optional;
|
import java.util.Optional;
|
||||||
|
|
||||||
|
import jakarta.annotation.Priority;
|
||||||
import jakarta.inject.Singleton;
|
import jakarta.inject.Singleton;
|
||||||
import jakarta.ws.rs.core.UriBuilder;
|
import jakarta.ws.rs.core.UriBuilder;
|
||||||
|
|
||||||
@@ -29,7 +30,8 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
@Slf4j
|
@Slf4j
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
@Singleton
|
@Singleton
|
||||||
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
|
@Priority(3)
|
||||||
|
public class YoutubeFaviconFetcher implements FaviconFetcher {
|
||||||
|
|
||||||
private static final String PART_SNIPPET = "snippet";
|
private static final String PART_SNIPPET = "snippet";
|
||||||
|
|
||||||
@@ -53,8 +55,6 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
byte[] bytes = null;
|
|
||||||
String contentType = null;
|
|
||||||
try {
|
try {
|
||||||
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
|
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
|
||||||
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
|
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
|
||||||
@@ -84,16 +84,11 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
|
|||||||
}
|
}
|
||||||
|
|
||||||
HttpResult iconResult = getter.get(thumbnailUrl.asText());
|
HttpResult iconResult = getter.get(thumbnailUrl.asText());
|
||||||
bytes = iconResult.content();
|
return new Favicon(iconResult.content(), iconResult.contentType());
|
||||||
contentType = iconResult.contentType();
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.debug("Failed to retrieve YouTube icon", e);
|
log.debug("Failed to retrieve YouTube icon", e);
|
||||||
}
|
|
||||||
|
|
||||||
if (!isValidIconResponse(bytes, contentType)) {
|
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
return new Favicon(bytes, contentType);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private byte[] fetchForUser(String googleAuthKey, String userId)
|
private byte[] fetchForUser(String googleAuthKey, String userId)
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ public class FeedRefreshWorker {
|
|||||||
|
|
||||||
feed.setUrlAfterRedirect(urlAfterRedirect);
|
feed.setUrlAfterRedirect(urlAfterRedirect);
|
||||||
feed.setLink(result.feed().link());
|
feed.setLink(result.feed().link());
|
||||||
|
feed.setIconUrl(result.feed().iconUrl());
|
||||||
feed.setLastModifiedHeader(result.lastModifiedHeader());
|
feed.setLastModifiedHeader(result.lastModifiedHeader());
|
||||||
feed.setEtagHeader(result.lastETagHeader());
|
feed.setEtagHeader(result.lastETagHeader());
|
||||||
feed.setLastContentHash(result.contentHash());
|
feed.setLastContentHash(result.contentHash());
|
||||||
|
|||||||
@@ -81,6 +81,7 @@ public class FeedParser {
|
|||||||
|
|
||||||
String title = feed.getTitle();
|
String title = feed.getTitle();
|
||||||
String link = feed.getLink();
|
String link = feed.getLink();
|
||||||
|
String iconUrl = feed.getIcon() != null ? feed.getIcon().getUrl() : null;
|
||||||
List<Entry> entries = buildEntries(feed, feedUrl);
|
List<Entry> entries = buildEntries(feed, feedUrl);
|
||||||
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
|
Instant lastEntryDate = entries.stream().findFirst().map(Entry::published).orElse(null);
|
||||||
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
|
Instant lastPublishedDate = toValidInstant(feed.getPublishedDate(), false);
|
||||||
@@ -89,7 +90,7 @@ public class FeedParser {
|
|||||||
}
|
}
|
||||||
Long averageEntryInterval = averageTimeBetweenEntries(entries);
|
Long averageEntryInterval = averageTimeBetweenEntries(entries);
|
||||||
|
|
||||||
return new FeedParserResult(title, link, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
|
return new FeedParserResult(title, link, iconUrl, lastPublishedDate, averageEntryInterval, lastEntryDate, entries);
|
||||||
} catch (FeedParsingException e) {
|
} catch (FeedParsingException e) {
|
||||||
throw e;
|
throw e;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|||||||
@@ -3,8 +3,8 @@ package com.commafeed.backend.feed.parser;
|
|||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
public record FeedParserResult(String title, String link, Instant lastPublishedDate, Long averageEntryInterval, Instant lastEntryDate,
|
public record FeedParserResult(String title, String link, String iconUrl, Instant lastPublishedDate, Long averageEntryInterval,
|
||||||
List<Entry> entries) {
|
Instant lastEntryDate, List<Entry> entries) {
|
||||||
public record Entry(String guid, String url, Instant published, Content content) {}
|
public record Entry(String guid, String url, Instant published, Content content) {}
|
||||||
|
|
||||||
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {}
|
public record Content(String title, String content, String author, String categories, Enclosure enclosure, Media media) {}
|
||||||
|
|||||||
@@ -48,6 +48,11 @@ public class Feed extends AbstractModel {
|
|||||||
@JdbcTypeCode(Types.LONGVARCHAR)
|
@JdbcTypeCode(Types.LONGVARCHAR)
|
||||||
private String link;
|
private String link;
|
||||||
|
|
||||||
|
@Lob
|
||||||
|
@Column(name = "icon_url", length = Integer.MAX_VALUE)
|
||||||
|
@JdbcTypeCode(Types.LONGVARCHAR)
|
||||||
|
private String iconUrl;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Last time we tried to fetch the feed
|
* Last time we tried to fetch the feed
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package com.commafeed.backend.service;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Objects;
|
||||||
|
import java.util.Set;
|
||||||
|
|
||||||
|
import jakarta.inject.Singleton;
|
||||||
|
import jakarta.ws.rs.core.MediaType;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.ArrayUtils;
|
||||||
|
|
||||||
|
import com.commafeed.backend.favicon.Favicon;
|
||||||
|
import com.commafeed.backend.favicon.FaviconFetcher;
|
||||||
|
import com.commafeed.backend.model.Feed;
|
||||||
|
import com.google.common.io.Resources;
|
||||||
|
|
||||||
|
import io.quarkus.arc.All;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
|
||||||
|
@Singleton
|
||||||
|
@Slf4j
|
||||||
|
public class FeedFaviconService {
|
||||||
|
|
||||||
|
private static final Set<MediaType> ICON_MIMETYPE_BLACKLIST = Set.of(MediaType.APPLICATION_XML_TYPE, MediaType.TEXT_HTML_TYPE);
|
||||||
|
private static final long MIN_ICON_LENGTH = 100;
|
||||||
|
private static final long MAX_ICON_LENGTH = 100000;
|
||||||
|
|
||||||
|
private final List<FaviconFetcher> faviconFetchers;
|
||||||
|
private final Favicon defaultFavicon;
|
||||||
|
|
||||||
|
public FeedFaviconService(@All List<FaviconFetcher> faviconFetchers) throws IOException {
|
||||||
|
this.faviconFetchers = faviconFetchers;
|
||||||
|
this.defaultFavicon = new Favicon(
|
||||||
|
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif");
|
||||||
|
}
|
||||||
|
|
||||||
|
public Favicon fetchFavicon(Feed feed) {
|
||||||
|
for (FaviconFetcher faviconFetcher : faviconFetchers) {
|
||||||
|
Favicon icon = faviconFetcher.fetch(feed);
|
||||||
|
if (isFaviconValid(icon)) {
|
||||||
|
return icon;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return defaultFavicon;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static boolean isFaviconValid(Favicon favicon) {
|
||||||
|
if (favicon == null || ArrayUtils.isEmpty(favicon.icon())) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
long length = favicon.icon().length;
|
||||||
|
if (length < MIN_ICON_LENGTH) {
|
||||||
|
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (length > MAX_ICON_LENGTH) {
|
||||||
|
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ICON_MIMETYPE_BLACKLIST.stream().anyMatch(bl -> bl.isCompatible(favicon.mediaType()))) {
|
||||||
|
log.debug("Content-Type {} is blacklisted", favicon.mediaType());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,37 +1,23 @@
|
|||||||
package com.commafeed.backend.service;
|
package com.commafeed.backend.service;
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.time.Instant;
|
import java.time.Instant;
|
||||||
import java.util.List;
|
|
||||||
import java.util.Objects;
|
|
||||||
|
|
||||||
import jakarta.inject.Singleton;
|
import jakarta.inject.Singleton;
|
||||||
|
|
||||||
import com.commafeed.backend.Digests;
|
import com.commafeed.backend.Digests;
|
||||||
import com.commafeed.backend.Urls;
|
import com.commafeed.backend.Urls;
|
||||||
import com.commafeed.backend.dao.FeedDAO;
|
import com.commafeed.backend.dao.FeedDAO;
|
||||||
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
|
|
||||||
import com.commafeed.backend.favicon.Favicon;
|
|
||||||
import com.commafeed.backend.feed.FeedUtils;
|
import com.commafeed.backend.feed.FeedUtils;
|
||||||
import com.commafeed.backend.model.Feed;
|
import com.commafeed.backend.model.Feed;
|
||||||
import com.commafeed.backend.model.Models;
|
import com.commafeed.backend.model.Models;
|
||||||
import com.google.common.io.Resources;
|
|
||||||
|
|
||||||
import io.quarkus.arc.All;
|
import lombok.RequiredArgsConstructor;
|
||||||
|
|
||||||
@Singleton
|
@Singleton
|
||||||
|
@RequiredArgsConstructor
|
||||||
public class FeedService {
|
public class FeedService {
|
||||||
|
|
||||||
private final FeedDAO feedDAO;
|
private final FeedDAO feedDAO;
|
||||||
private final List<AbstractFaviconFetcher> faviconFetchers;
|
|
||||||
private final Favicon defaultFavicon;
|
|
||||||
|
|
||||||
public FeedService(FeedDAO feedDAO, @All List<AbstractFaviconFetcher> faviconFetchers) throws IOException {
|
|
||||||
this.feedDAO = feedDAO;
|
|
||||||
this.faviconFetchers = faviconFetchers;
|
|
||||||
this.defaultFavicon = new Favicon(
|
|
||||||
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif");
|
|
||||||
}
|
|
||||||
|
|
||||||
public synchronized Feed findOrCreate(String url) {
|
public synchronized Feed findOrCreate(String url) {
|
||||||
String normalizedUrl = Urls.normalize(url);
|
String normalizedUrl = Urls.normalize(url);
|
||||||
@@ -57,19 +43,4 @@ public class FeedService {
|
|||||||
feedDAO.merge(feed);
|
feedDAO.merge(feed);
|
||||||
}
|
}
|
||||||
|
|
||||||
public Favicon fetchFavicon(Feed feed) {
|
|
||||||
|
|
||||||
Favicon icon = null;
|
|
||||||
for (AbstractFaviconFetcher faviconFetcher : faviconFetchers) {
|
|
||||||
icon = faviconFetcher.fetch(feed);
|
|
||||||
if (icon != null) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (icon == null) {
|
|
||||||
icon = defaultFavicon;
|
|
||||||
}
|
|
||||||
return icon;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ import com.commafeed.backend.opml.OPMLImporter;
|
|||||||
import com.commafeed.backend.service.FeedEntryFilteringService;
|
import com.commafeed.backend.service.FeedEntryFilteringService;
|
||||||
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
|
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
|
||||||
import com.commafeed.backend.service.FeedEntryService;
|
import com.commafeed.backend.service.FeedEntryService;
|
||||||
import com.commafeed.backend.service.FeedService;
|
import com.commafeed.backend.service.FeedFaviconService;
|
||||||
import com.commafeed.backend.service.FeedSubscriptionService;
|
import com.commafeed.backend.service.FeedSubscriptionService;
|
||||||
import com.commafeed.backend.service.FeedSubscriptionService.ForceFeedRefreshTooSoonException;
|
import com.commafeed.backend.service.FeedSubscriptionService.ForceFeedRefreshTooSoonException;
|
||||||
import com.commafeed.frontend.model.Entries;
|
import com.commafeed.frontend.model.Entries;
|
||||||
@@ -106,7 +106,7 @@ public class FeedREST {
|
|||||||
private final FeedCategoryDAO feedCategoryDAO;
|
private final FeedCategoryDAO feedCategoryDAO;
|
||||||
private final FeedEntryStatusDAO feedEntryStatusDAO;
|
private final FeedEntryStatusDAO feedEntryStatusDAO;
|
||||||
private final FeedFetcher feedFetcher;
|
private final FeedFetcher feedFetcher;
|
||||||
private final FeedService feedService;
|
private final FeedFaviconService feedFaviconService;
|
||||||
private final FeedEntryService feedEntryService;
|
private final FeedEntryService feedEntryService;
|
||||||
private final FeedSubscriptionService feedSubscriptionService;
|
private final FeedSubscriptionService feedSubscriptionService;
|
||||||
private final FeedEntryFilteringService feedEntryFilteringService;
|
private final FeedEntryFilteringService feedEntryFilteringService;
|
||||||
@@ -339,7 +339,19 @@ public class FeedREST {
|
|||||||
}
|
}
|
||||||
|
|
||||||
Feed feed = subscription.getFeed();
|
Feed feed = subscription.getFeed();
|
||||||
Favicon icon = feedService.fetchFavicon(feed);
|
if (feed.getLastUpdated() == null) {
|
||||||
|
// the feed has never been fetched yet so the iconUrl field hasn't been updated yet, and the icon can't be fetched
|
||||||
|
// this can happen for newly subscribed feeds:
|
||||||
|
// the feed has been added in the tree in the web client and the favicon is being fetched,
|
||||||
|
// but the feed has not been fetched yet
|
||||||
|
// the client is configured to retry in that case
|
||||||
|
return Response.status(Status.SERVICE_UNAVAILABLE)
|
||||||
|
.entity("Feed has not been fetched yet, please retry in a bit")
|
||||||
|
.type(MediaType.TEXT_PLAIN)
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
Favicon icon = feedFaviconService.fetchFavicon(feed);
|
||||||
return Response.ok(icon.icon(), icon.mediaType()).build();
|
return Response.ok(icon.icon(), icon.mediaType()).build();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ import com.commafeed.backend.model.FeedSubscription;
|
|||||||
import com.commafeed.backend.model.User;
|
import com.commafeed.backend.model.User;
|
||||||
import com.commafeed.backend.model.UserSettings.ReadingOrder;
|
import com.commafeed.backend.model.UserSettings.ReadingOrder;
|
||||||
import com.commafeed.backend.service.FeedEntryService;
|
import com.commafeed.backend.service.FeedEntryService;
|
||||||
import com.commafeed.backend.service.FeedService;
|
import com.commafeed.backend.service.FeedFaviconService;
|
||||||
import com.commafeed.backend.service.UserService;
|
import com.commafeed.backend.service.UserService;
|
||||||
import com.commafeed.frontend.resource.fever.FeverResponse.FeverFavicon;
|
import com.commafeed.frontend.resource.fever.FeverResponse.FeverFavicon;
|
||||||
import com.commafeed.frontend.resource.fever.FeverResponse.FeverFeed;
|
import com.commafeed.frontend.resource.fever.FeverResponse.FeverFeed;
|
||||||
@@ -80,7 +80,7 @@ public class FeverREST {
|
|||||||
|
|
||||||
private final UserService userService;
|
private final UserService userService;
|
||||||
private final FeedEntryService feedEntryService;
|
private final FeedEntryService feedEntryService;
|
||||||
private final FeedService feedService;
|
private final FeedFaviconService feedFaviconService;
|
||||||
private final FeedEntryDAO feedEntryDAO;
|
private final FeedEntryDAO feedEntryDAO;
|
||||||
private final FeedSubscriptionDAO feedSubscriptionDAO;
|
private final FeedSubscriptionDAO feedSubscriptionDAO;
|
||||||
private final FeedCategoryDAO feedCategoryDAO;
|
private final FeedCategoryDAO feedCategoryDAO;
|
||||||
@@ -303,7 +303,7 @@ public class FeverREST {
|
|||||||
|
|
||||||
private List<FeverFavicon> buildFavicons(List<FeedSubscription> subscriptions) {
|
private List<FeverFavicon> buildFavicons(List<FeedSubscription> subscriptions) {
|
||||||
return subscriptions.stream().map(s -> {
|
return subscriptions.stream().map(s -> {
|
||||||
Favicon favicon = feedService.fetchFavicon(s.getFeed());
|
Favicon favicon = feedFaviconService.fetchFavicon(s.getFeed());
|
||||||
|
|
||||||
FeverFavicon f = new FeverFavicon();
|
FeverFavicon f = new FeverFavicon();
|
||||||
f.setId(s.getFeed().getId());
|
f.setId(s.getFeed().getId());
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||||
|
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd">
|
||||||
|
|
||||||
|
<changeSet id="feed-icons" author="athou">
|
||||||
|
<addColumn tableName="FEEDS">
|
||||||
|
<column name="icon_url" type="CLOB">
|
||||||
|
<constraints nullable="true" />
|
||||||
|
</column>
|
||||||
|
</addColumn>
|
||||||
|
</changeSet>
|
||||||
|
|
||||||
|
</databaseChangeLog>
|
||||||
@@ -38,5 +38,6 @@
|
|||||||
<include file="changelogs/db.changelog-5.11.xml" />
|
<include file="changelogs/db.changelog-5.11.xml" />
|
||||||
<include file="changelogs/db.changelog-5.12.xml" />
|
<include file="changelogs/db.changelog-5.12.xml" />
|
||||||
<include file="changelogs/db.changelog-7.0.xml" />
|
<include file="changelogs/db.changelog-7.0.xml" />
|
||||||
|
<include file="changelogs/db.changelog-7.2.xml" />
|
||||||
|
|
||||||
</databaseChangeLog>
|
</databaseChangeLog>
|
||||||
@@ -75,33 +75,4 @@ class FacebookFaviconFetcherTest {
|
|||||||
|
|
||||||
Assertions.assertNull(faviconFetcher.fetch(feed));
|
Assertions.assertNull(faviconFetcher.fetch(feed));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
void testFetchWithInvalidIconResponse() throws Exception {
|
|
||||||
Feed feed = new Feed();
|
|
||||||
feed.setUrl("https://www.facebook.com/something?id=validUserId");
|
|
||||||
|
|
||||||
// Create a byte array that's too small
|
|
||||||
byte[] iconBytes = new byte[50];
|
|
||||||
String contentType = "image/png";
|
|
||||||
|
|
||||||
HttpResult httpResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
|
|
||||||
Mockito.when(httpGetter.get("https://graph.facebook.com/validUserId/picture?type=square&height=16")).thenReturn(httpResult);
|
|
||||||
|
|
||||||
Assertions.assertNull(faviconFetcher.fetch(feed));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
void testFetchWithBlacklistedContentType() throws Exception {
|
|
||||||
Feed feed = new Feed();
|
|
||||||
feed.setUrl("https://www.facebook.com/something?id=validUserId");
|
|
||||||
|
|
||||||
byte[] iconBytes = new byte[1000];
|
|
||||||
String contentType = "application/xml"; // Blacklisted content type
|
|
||||||
|
|
||||||
HttpResult httpResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
|
|
||||||
Mockito.when(httpGetter.get("https://graph.facebook.com/validUserId/picture?type=square&height=16")).thenReturn(httpResult);
|
|
||||||
|
|
||||||
Assertions.assertNull(faviconFetcher.fetch(feed));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package com.commafeed.backend.favicon;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
|
||||||
|
import jakarta.ws.rs.core.MediaType;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.mockito.Mock;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
|
||||||
|
import com.commafeed.backend.HttpGetter;
|
||||||
|
import com.commafeed.backend.HttpGetter.HttpResult;
|
||||||
|
import com.commafeed.backend.model.Feed;
|
||||||
|
|
||||||
|
@ExtendWith(MockitoExtension.class)
|
||||||
|
class FeedFaviconFetcherTest {
|
||||||
|
|
||||||
|
@Mock
|
||||||
|
private HttpGetter httpGetter;
|
||||||
|
|
||||||
|
private FeedFaviconFetcher faviconFetcher;
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void init() {
|
||||||
|
faviconFetcher = new FeedFaviconFetcher(httpGetter);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFetchWithNullIconUrl() {
|
||||||
|
Feed feed = new Feed();
|
||||||
|
feed.setUrl("https://example.com/feed");
|
||||||
|
|
||||||
|
Assertions.assertNull(faviconFetcher.fetch(feed));
|
||||||
|
Mockito.verifyNoInteractions(httpGetter);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFetchWithValidIconUrl() throws Exception {
|
||||||
|
Feed feed = new Feed();
|
||||||
|
feed.setUrl("https://example.com/feed");
|
||||||
|
feed.setIconUrl("https://example.com/icon.png");
|
||||||
|
|
||||||
|
byte[] iconBytes = new byte[1000];
|
||||||
|
String contentType = "image/png";
|
||||||
|
HttpResult httpResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
|
||||||
|
Mockito.when(httpGetter.get("https://example.com/icon.png")).thenReturn(httpResult);
|
||||||
|
|
||||||
|
Favicon result = faviconFetcher.fetch(feed);
|
||||||
|
|
||||||
|
Assertions.assertNotNull(result);
|
||||||
|
Assertions.assertEquals(iconBytes, result.icon());
|
||||||
|
Assertions.assertTrue(result.mediaType().isCompatible(MediaType.valueOf(contentType)));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFetchWithHttpGetterException() throws Exception {
|
||||||
|
Feed feed = new Feed();
|
||||||
|
feed.setUrl("https://example.com/feed");
|
||||||
|
feed.setIconUrl("https://example.com/icon.png");
|
||||||
|
|
||||||
|
Mockito.when(httpGetter.get("https://example.com/icon.png")).thenThrow(new RuntimeException("Network error"));
|
||||||
|
|
||||||
|
Assertions.assertNull(faviconFetcher.fetch(feed));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
package com.commafeed.backend.favicon;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
|
||||||
|
import jakarta.ws.rs.core.MediaType;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.mockito.Mock;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
|
||||||
|
import com.commafeed.backend.HttpGetter;
|
||||||
|
import com.commafeed.backend.HttpGetter.HttpResult;
|
||||||
|
import com.commafeed.backend.model.Feed;
|
||||||
|
|
||||||
|
@ExtendWith(MockitoExtension.class)
|
||||||
|
class HtmlFaviconFetcherTest {
|
||||||
|
|
||||||
|
@Mock
|
||||||
|
private HttpGetter httpGetter;
|
||||||
|
|
||||||
|
private HtmlFaviconFetcher faviconFetcher;
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void init() {
|
||||||
|
faviconFetcher = new HtmlFaviconFetcher(httpGetter);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFetchWithNullLink() {
|
||||||
|
Feed feed = new Feed();
|
||||||
|
feed.setUrl("https://example.com/feed");
|
||||||
|
|
||||||
|
Assertions.assertNull(faviconFetcher.fetch(feed));
|
||||||
|
Mockito.verifyNoInteractions(httpGetter);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFetchWithValidIconLink() throws Exception {
|
||||||
|
Feed feed = new Feed();
|
||||||
|
feed.setUrl("https://example.com/feed");
|
||||||
|
feed.setLink("https://example.com");
|
||||||
|
|
||||||
|
String html = "<html><head><link rel=\"icon\" href=\"/favicon.png\" /></head><body></body></html>";
|
||||||
|
HttpResult pageResult = new HttpResult(html.getBytes(), "text/html", null, null, null, Duration.ZERO);
|
||||||
|
Mockito.when(httpGetter.get("https://example.com")).thenReturn(pageResult);
|
||||||
|
|
||||||
|
byte[] iconBytes = new byte[1000];
|
||||||
|
String contentType = "image/png";
|
||||||
|
HttpResult iconResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
|
||||||
|
Mockito.when(httpGetter.get("https://example.com/favicon.png")).thenReturn(iconResult);
|
||||||
|
|
||||||
|
Favicon result = faviconFetcher.fetch(feed);
|
||||||
|
|
||||||
|
Assertions.assertNotNull(result);
|
||||||
|
Assertions.assertEquals(iconBytes, result.icon());
|
||||||
|
Assertions.assertTrue(result.mediaType().isCompatible(MediaType.valueOf(contentType)));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFetchWithShortcutIconLink() throws Exception {
|
||||||
|
Feed feed = new Feed();
|
||||||
|
feed.setUrl("https://example.com/feed");
|
||||||
|
feed.setLink("https://example.com");
|
||||||
|
|
||||||
|
String html = "<html><head><link rel=\"shortcut icon\" href=\"https://example.com/shortcut-favicon.ico\" /></head><body></body></html>";
|
||||||
|
HttpResult pageResult = new HttpResult(html.getBytes(), "text/html", null, null, null, Duration.ZERO);
|
||||||
|
Mockito.when(httpGetter.get("https://example.com")).thenReturn(pageResult);
|
||||||
|
|
||||||
|
byte[] iconBytes = new byte[1000];
|
||||||
|
String contentType = "image/x-icon";
|
||||||
|
HttpResult iconResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
|
||||||
|
Mockito.when(httpGetter.get("https://example.com/shortcut-favicon.ico")).thenReturn(iconResult);
|
||||||
|
|
||||||
|
Favicon result = faviconFetcher.fetch(feed);
|
||||||
|
|
||||||
|
Assertions.assertNotNull(result);
|
||||||
|
Assertions.assertEquals(iconBytes, result.icon());
|
||||||
|
Assertions.assertTrue(result.mediaType().isCompatible(MediaType.valueOf(contentType)));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFetchWithNoIconInPage() throws Exception {
|
||||||
|
Feed feed = new Feed();
|
||||||
|
feed.setUrl("https://example.com/feed");
|
||||||
|
feed.setLink("https://example.com");
|
||||||
|
|
||||||
|
String html = "<html><head></head><body></body></html>";
|
||||||
|
HttpResult pageResult = new HttpResult(html.getBytes(), "text/html", null, null, null, Duration.ZERO);
|
||||||
|
Mockito.when(httpGetter.get("https://example.com")).thenReturn(pageResult);
|
||||||
|
|
||||||
|
Assertions.assertNull(faviconFetcher.fetch(feed));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFetchWithPageFetchException() throws Exception {
|
||||||
|
Feed feed = new Feed();
|
||||||
|
feed.setUrl("https://example.com/feed");
|
||||||
|
feed.setLink("https://example.com");
|
||||||
|
|
||||||
|
Mockito.when(httpGetter.get("https://example.com")).thenThrow(new RuntimeException("Network error"));
|
||||||
|
|
||||||
|
Assertions.assertNull(faviconFetcher.fetch(feed));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFetchWithIconFetchException() throws Exception {
|
||||||
|
Feed feed = new Feed();
|
||||||
|
feed.setUrl("https://example.com/feed");
|
||||||
|
feed.setLink("https://example.com");
|
||||||
|
|
||||||
|
String html = "<html><head><link rel=\"icon\" href=\"https://example.com/favicon.png\" /></head><body></body></html>";
|
||||||
|
HttpResult pageResult = new HttpResult(html.getBytes(), "text/html", null, null, null, Duration.ZERO);
|
||||||
|
Mockito.when(httpGetter.get("https://example.com")).thenReturn(pageResult);
|
||||||
|
Mockito.when(httpGetter.get("https://example.com/favicon.png")).thenThrow(new RuntimeException("Network error"));
|
||||||
|
|
||||||
|
Assertions.assertNull(faviconFetcher.fetch(feed));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
package com.commafeed.backend.favicon;
|
||||||
|
|
||||||
|
import java.time.Duration;
|
||||||
|
|
||||||
|
import jakarta.ws.rs.core.MediaType;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.mockito.Mock;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
|
||||||
|
import com.commafeed.backend.HttpGetter;
|
||||||
|
import com.commafeed.backend.HttpGetter.HttpResult;
|
||||||
|
import com.commafeed.backend.model.Feed;
|
||||||
|
|
||||||
|
@ExtendWith(MockitoExtension.class)
|
||||||
|
class RootFaviconFetcherTest {
|
||||||
|
|
||||||
|
@Mock
|
||||||
|
private HttpGetter httpGetter;
|
||||||
|
|
||||||
|
private RootFaviconFetcher faviconFetcher;
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void init() {
|
||||||
|
faviconFetcher = new RootFaviconFetcher(httpGetter);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFetchUsesLinkWhenAvailable() throws Exception {
|
||||||
|
Feed feed = new Feed();
|
||||||
|
feed.setUrl("https://feeds.example.com/feed");
|
||||||
|
feed.setLink("https://www.example.com/blog");
|
||||||
|
|
||||||
|
byte[] iconBytes = new byte[1000];
|
||||||
|
String contentType = "image/x-icon";
|
||||||
|
HttpResult httpResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
|
||||||
|
Mockito.when(httpGetter.get("https://www.example.com/favicon.ico")).thenReturn(httpResult);
|
||||||
|
|
||||||
|
Favicon result = faviconFetcher.fetch(feed);
|
||||||
|
|
||||||
|
Assertions.assertNotNull(result);
|
||||||
|
Assertions.assertEquals(iconBytes, result.icon());
|
||||||
|
Assertions.assertTrue(result.mediaType().isCompatible(MediaType.valueOf(contentType)));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFetchFallsBackToUrlWhenLinkIsNull() throws Exception {
|
||||||
|
Feed feed = new Feed();
|
||||||
|
feed.setUrl("https://example.com/feed.xml");
|
||||||
|
|
||||||
|
byte[] iconBytes = new byte[1000];
|
||||||
|
String contentType = "image/x-icon";
|
||||||
|
HttpResult httpResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
|
||||||
|
Mockito.when(httpGetter.get("https://example.com/favicon.ico")).thenReturn(httpResult);
|
||||||
|
|
||||||
|
Favicon result = faviconFetcher.fetch(feed);
|
||||||
|
|
||||||
|
Assertions.assertNotNull(result);
|
||||||
|
Assertions.assertEquals(iconBytes, result.icon());
|
||||||
|
Assertions.assertTrue(result.mediaType().isCompatible(MediaType.valueOf(contentType)));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFetchWithHttpGetterException() throws Exception {
|
||||||
|
Feed feed = new Feed();
|
||||||
|
feed.setUrl("https://example.com/feed.xml");
|
||||||
|
feed.setLink("https://example.com");
|
||||||
|
|
||||||
|
Mockito.when(httpGetter.get("https://example.com/favicon.ico")).thenThrow(new RuntimeException("Network error"));
|
||||||
|
|
||||||
|
Assertions.assertNull(faviconFetcher.fetch(feed));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -168,31 +168,6 @@ class YoutubeFaviconFetcherTest {
|
|||||||
Assertions.assertNull(faviconFetcher.fetch(feed));
|
Assertions.assertNull(faviconFetcher.fetch(feed));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
|
||||||
void testFetchWithInvalidIconResponse() throws Exception {
|
|
||||||
Feed feed = new Feed();
|
|
||||||
feed.setUrl("https://youtube.com/feeds/videos.xml?user=testUser");
|
|
||||||
|
|
||||||
Mockito.when(config.googleAuthKey()).thenReturn(Optional.of("test-api-key"));
|
|
||||||
|
|
||||||
byte[] apiResponse = """
|
|
||||||
{"items":[{"snippet":{"thumbnails":{"default":{"url":"https://example.com/icon.png"}}}}]}""".getBytes();
|
|
||||||
HttpResult apiHttpResult = new HttpResult(apiResponse, "application/json", null, null, null, Duration.ZERO);
|
|
||||||
Mockito.when(httpGetter.get("https://www.googleapis.com/youtube/v3/channels?part=snippet&key=test-api-key&forUsername=testUser"))
|
|
||||||
.thenReturn(apiHttpResult);
|
|
||||||
|
|
||||||
JsonNode jsonNode = new ObjectMapper().readTree(apiResponse);
|
|
||||||
Mockito.when(objectMapper.readTree(apiResponse)).thenReturn(jsonNode);
|
|
||||||
|
|
||||||
// Create a byte array that's too small
|
|
||||||
byte[] iconBytes = new byte[50];
|
|
||||||
String contentType = "image/png";
|
|
||||||
HttpResult iconHttpResult = new HttpResult(iconBytes, contentType, null, null, null, Duration.ZERO);
|
|
||||||
Mockito.when(httpGetter.get("https://example.com/icon.png")).thenReturn(iconHttpResult);
|
|
||||||
|
|
||||||
Assertions.assertNull(faviconFetcher.fetch(feed));
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void testFetchWithEmptyApiResponse() throws Exception {
|
void testFetchWithEmptyApiResponse() throws Exception {
|
||||||
Feed feed = new Feed();
|
Feed feed = new Feed();
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ class FeedFetcherTest {
|
|||||||
byte[] feed = "feed".getBytes();
|
byte[] feed = "feed".getBytes();
|
||||||
Mockito.when(getter.get(HttpGetter.HttpRequest.builder(feedUrl).build()))
|
Mockito.when(getter.get(HttpGetter.HttpRequest.builder(feedUrl).build()))
|
||||||
.thenReturn(new HttpResult(feed, "application/atom+xml", null, null, feedUrl, Duration.ZERO));
|
.thenReturn(new HttpResult(feed, "application/atom+xml", null, null, feedUrl, Duration.ZERO));
|
||||||
Mockito.when(parser.parse(feedUrl, feed)).thenReturn(new FeedParserResult("title", "link", null, null, null, null));
|
Mockito.when(parser.parse(feedUrl, feed)).thenReturn(new FeedParserResult("title", "link", "iconUrl", null, null, null, null));
|
||||||
|
|
||||||
Mockito.when(urlProvider.get(htmlUrl, new String(html))).thenReturn(List.of(feedUrl));
|
Mockito.when(urlProvider.get(htmlUrl, new String(html))).thenReturn(List.of(feedUrl));
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,151 @@
|
|||||||
|
package com.commafeed.backend.service;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import jakarta.ws.rs.core.MediaType;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.extension.ExtendWith;
|
||||||
|
import org.mockito.Mock;
|
||||||
|
import org.mockito.Mockito;
|
||||||
|
import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
|
||||||
|
import com.commafeed.backend.favicon.Favicon;
|
||||||
|
import com.commafeed.backend.favicon.FaviconFetcher;
|
||||||
|
import com.commafeed.backend.model.Feed;
|
||||||
|
|
||||||
|
@ExtendWith(MockitoExtension.class)
|
||||||
|
class FeedFaviconServiceTest {
|
||||||
|
|
||||||
|
@Mock
|
||||||
|
private FaviconFetcher fetcher1;
|
||||||
|
|
||||||
|
@Mock
|
||||||
|
private FaviconFetcher fetcher2;
|
||||||
|
|
||||||
|
private FeedFaviconService service;
|
||||||
|
private Feed feed;
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
void init() throws IOException {
|
||||||
|
service = new FeedFaviconService(List.of(fetcher1, fetcher2));
|
||||||
|
feed = new Feed();
|
||||||
|
feed.setUrl("https://example.com/feed");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testReturnsFirstValidFavicon() {
|
||||||
|
byte[] iconBytes = new byte[1000];
|
||||||
|
Favicon validFavicon = new Favicon(iconBytes, "image/png");
|
||||||
|
|
||||||
|
Mockito.when(fetcher1.fetch(feed)).thenReturn(validFavicon);
|
||||||
|
|
||||||
|
Favicon result = service.fetchFavicon(feed);
|
||||||
|
|
||||||
|
Assertions.assertEquals(validFavicon, result);
|
||||||
|
Mockito.verify(fetcher1).fetch(feed);
|
||||||
|
Mockito.verifyNoInteractions(fetcher2);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFallsBackToNextFetcherWhenFirstReturnsNull() {
|
||||||
|
byte[] iconBytes = new byte[1000];
|
||||||
|
Favicon validFavicon = new Favicon(iconBytes, "image/png");
|
||||||
|
|
||||||
|
Mockito.when(fetcher1.fetch(feed)).thenReturn(null);
|
||||||
|
Mockito.when(fetcher2.fetch(feed)).thenReturn(validFavicon);
|
||||||
|
|
||||||
|
Favicon result = service.fetchFavicon(feed);
|
||||||
|
|
||||||
|
Assertions.assertEquals(validFavicon, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFallsBackToNextFetcherWhenFirstReturnsTooSmallIcon() {
|
||||||
|
byte[] tinyIcon = new byte[50];
|
||||||
|
Favicon tinyFavicon = new Favicon(tinyIcon, "image/png");
|
||||||
|
|
||||||
|
byte[] validIcon = new byte[1000];
|
||||||
|
Favicon validFavicon = new Favicon(validIcon, "image/png");
|
||||||
|
|
||||||
|
Mockito.when(fetcher1.fetch(feed)).thenReturn(tinyFavicon);
|
||||||
|
Mockito.when(fetcher2.fetch(feed)).thenReturn(validFavicon);
|
||||||
|
|
||||||
|
Favicon result = service.fetchFavicon(feed);
|
||||||
|
|
||||||
|
Assertions.assertEquals(validFavicon, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFallsBackToNextFetcherWhenFirstReturnsTooLargeIcon() {
|
||||||
|
byte[] hugeIcon = new byte[100001];
|
||||||
|
Favicon hugeFavicon = new Favicon(hugeIcon, "image/png");
|
||||||
|
|
||||||
|
byte[] validIcon = new byte[1000];
|
||||||
|
Favicon validFavicon = new Favicon(validIcon, "image/png");
|
||||||
|
|
||||||
|
Mockito.when(fetcher1.fetch(feed)).thenReturn(hugeFavicon);
|
||||||
|
Mockito.when(fetcher2.fetch(feed)).thenReturn(validFavicon);
|
||||||
|
|
||||||
|
Favicon result = service.fetchFavicon(feed);
|
||||||
|
|
||||||
|
Assertions.assertEquals(validFavicon, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFallsBackToNextFetcherWhenFirstReturnsBlacklistedContentType() {
|
||||||
|
byte[] iconBytes = new byte[1000];
|
||||||
|
Favicon xmlFavicon = new Favicon(iconBytes, "application/xml");
|
||||||
|
|
||||||
|
byte[] validIcon = new byte[1000];
|
||||||
|
Favicon validFavicon = new Favicon(validIcon, "image/png");
|
||||||
|
|
||||||
|
Mockito.when(fetcher1.fetch(feed)).thenReturn(xmlFavicon);
|
||||||
|
Mockito.when(fetcher2.fetch(feed)).thenReturn(validFavicon);
|
||||||
|
|
||||||
|
Favicon result = service.fetchFavicon(feed);
|
||||||
|
|
||||||
|
Assertions.assertEquals(validFavicon, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testFallsBackToNextFetcherWhenFirstReturnsHtmlContentType() {
|
||||||
|
byte[] iconBytes = new byte[1000];
|
||||||
|
Favicon htmlFavicon = new Favicon(iconBytes, "text/html");
|
||||||
|
|
||||||
|
byte[] validIcon = new byte[1000];
|
||||||
|
Favicon validFavicon = new Favicon(validIcon, "image/png");
|
||||||
|
|
||||||
|
Mockito.when(fetcher1.fetch(feed)).thenReturn(htmlFavicon);
|
||||||
|
Mockito.when(fetcher2.fetch(feed)).thenReturn(validFavicon);
|
||||||
|
|
||||||
|
Favicon result = service.fetchFavicon(feed);
|
||||||
|
|
||||||
|
Assertions.assertEquals(validFavicon, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testReturnsDefaultFaviconWhenAllFetchersFail() {
|
||||||
|
Mockito.when(fetcher1.fetch(feed)).thenReturn(null);
|
||||||
|
Mockito.when(fetcher2.fetch(feed)).thenReturn(null);
|
||||||
|
|
||||||
|
Favicon result = service.fetchFavicon(feed);
|
||||||
|
|
||||||
|
Assertions.assertNotNull(result);
|
||||||
|
Assertions.assertTrue(result.mediaType().isCompatible(MediaType.valueOf("image/gif")));
|
||||||
|
Assertions.assertTrue(result.icon().length > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testReturnsDefaultFaviconWhenNoFetchersRegistered() throws IOException {
|
||||||
|
FeedFaviconService emptyService = new FeedFaviconService(List.of());
|
||||||
|
|
||||||
|
Favicon result = emptyService.fetchFavicon(feed);
|
||||||
|
|
||||||
|
Assertions.assertNotNull(result);
|
||||||
|
Assertions.assertTrue(result.mediaType().isCompatible(MediaType.valueOf("image/gif")));
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user