split client and server into maven modules

2026-03-21 21:37:29 +00:00 · 2022-08-13 10:34:59 +02:00
parent 4c4868a2b6
commit ac7b6eeb21
277 changed files with 645 additions and 521 deletions
--- a/commafeed-server/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java
+++ b/commafeed-server/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java
@@ -0,0 +1,60 @@
+package com.commafeed.backend.favicon;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.commafeed.backend.model.Feed;
+
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public abstract class AbstractFaviconFetcher {
+
+	protected static final int TIMEOUT = 4000;
+
+	private static final List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
+	private static final long MIN_ICON_LENGTH = 100;
+	private static final long MAX_ICON_LENGTH = 100000;
+
+	public abstract Favicon fetch(Feed feed);
+
+	protected boolean isValidIconResponse(byte[] content, String contentType) {
+		if (content == null) {
+			return false;
+		}
+
+		long length = content.length;
+
+		if (StringUtils.isNotBlank(contentType)) {
+			contentType = contentType.split(";")[0];
+		}
+
+		if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
+			log.debug("Content-Type {} is blacklisted", contentType);
+			return false;
+		}
+
+		if (length < MIN_ICON_LENGTH) {
+			log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
+			return false;
+		}
+
+		if (length > MAX_ICON_LENGTH) {
+			log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
+			return false;
+		}
+
+		return true;
+	}
+
+	@RequiredArgsConstructor
+	@Getter
+	public static class Favicon {
+		private final byte[] icon;
+		private final String mediaType;
+	}
+}
--- a/commafeed-server/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java
+++ b/commafeed-server/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java
@@ -0,0 +1,132 @@
+package com.commafeed.backend.favicon;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.apache.commons.lang3.StringUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.select.Elements;
+
+import com.commafeed.backend.HttpGetter;
+import com.commafeed.backend.HttpGetter.HttpResult;
+import com.commafeed.backend.feed.FeedUtils;
+import com.commafeed.backend.model.Feed;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Inspired/Ported from https://github.com/potatolondon/getfavicon
+ * 
+ */
+@Slf4j
+@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
+@Singleton
+public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
+
+	private final HttpGetter getter;
+
+	@Override
+	public Favicon fetch(Feed feed) {
+		Favicon icon = fetch(feed.getLink());
+		if (icon == null) {
+			icon = fetch(feed.getUrl());
+		}
+		return icon;
+	}
+
+	private Favicon fetch(String url) {
+		if (url == null) {
+			log.debug("url is null");
+			return null;
+		}
+
+		int doubleSlash = url.indexOf("//");
+		if (doubleSlash == -1) {
+			doubleSlash = 0;
+		} else {
+			doubleSlash += 2;
+		}
+		int firstSlash = url.indexOf('/', doubleSlash);
+		if (firstSlash != -1) {
+			url = url.substring(0, firstSlash);
+		}
+
+		Favicon icon = getIconAtRoot(url);
+
+		if (icon == null) {
+			icon = getIconInPage(url);
+		}
+
+		return icon;
+	}
+
+	private Favicon getIconAtRoot(String url) {
+		byte[] bytes = null;
+		String contentType = null;
+
+		try {
+			url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico";
+			log.debug("getting root icon at {}", url);
+			HttpResult result = getter.getBinary(url, TIMEOUT);
+			bytes = result.getContent();
+			contentType = result.getContentType();
+		} catch (Exception e) {
+			log.debug("Failed to retrieve iconAtRoot for url {}: ", url);
+			log.trace("Failed to retrieve iconAtRoot for url {}: ", url, e);
+		}
+
+		if (!isValidIconResponse(bytes, contentType)) {
+			return null;
+		}
+		return new Favicon(bytes, contentType);
+	}
+
+	private Favicon getIconInPage(String url) {
+
+		Document doc = null;
+		try {
+			HttpResult result = getter.getBinary(url, TIMEOUT);
+			doc = Jsoup.parse(new String(result.getContent()), url);
+		} catch (Exception e) {
+			log.debug("Failed to retrieve page to find icon");
+			log.trace("Failed to retrieve page to find icon", e);
+			return null;
+		}
+
+		Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
+
+		if (icons.isEmpty()) {
+			log.debug("No icon found in page {}", url);
+			return null;
+		}
+
+		String href = icons.get(0).attr("abs:href");
+		if (StringUtils.isBlank(href)) {
+			log.debug("No icon found in page");
+			return null;
+		}
+
+		log.debug("Found unconfirmed iconInPage at {}", href);
+
+		byte[] bytes = null;
+		String contentType = null;
+		try {
+			HttpResult result = getter.getBinary(href, TIMEOUT);
+			bytes = result.getContent();
+			contentType = result.getContentType();
+		} catch (Exception e) {
+			log.debug("Failed to retrieve icon found in page {}", href);
+			log.trace("Failed to retrieve icon found in page {}", href, e);
+			return null;
+		}
+
+		if (!isValidIconResponse(bytes, contentType)) {
+			log.debug("Invalid icon found for {}", href);
+			return null;
+		}
+
+		return new Favicon(bytes, contentType);
+	}
+}
--- a/commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
+++ b/commafeed-server/src/main/java/com/commafeed/backend/favicon/FacebookFaviconFetcher.java
@@ -0,0 +1,79 @@
+package com.commafeed.backend.favicon;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URLEncodedUtils;
+
+import com.commafeed.backend.HttpGetter;
+import com.commafeed.backend.HttpGetter.HttpResult;
+import com.commafeed.backend.model.Feed;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
+@Singleton
+public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
+
+	private final HttpGetter getter;
+
+	@Override
+	public Favicon fetch(Feed feed) {
+		String url = feed.getUrl();
+
+		if (!url.toLowerCase().contains("www.facebook.com")) {
+			return null;
+		}
+
+		String userName = extractUserName(url);
+		if (userName == null) {
+			return null;
+		}
+
+		String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
+
+		byte[] bytes = null;
+		String contentType = null;
+
+		try {
+			log.debug("Getting Facebook user's icon, {}", url);
+
+			HttpResult iconResult = getter.getBinary(iconUrl, TIMEOUT);
+			bytes = iconResult.getContent();
+			contentType = iconResult.getContentType();
+		} catch (Exception e) {
+			log.debug("Failed to retrieve Facebook icon", e);
+		}
+
+		if (!isValidIconResponse(bytes, contentType)) {
+			return null;
+		}
+		return new Favicon(bytes, contentType);
+	}
+
+	private String extractUserName(String url) {
+		URI uri = null;
+		try {
+			uri = new URI(url);
+		} catch (URISyntaxException e) {
+			log.debug("could not parse url", e);
+			return null;
+		}
+		List<NameValuePair> params = URLEncodedUtils.parse(uri, StandardCharsets.UTF_8);
+		for (NameValuePair param : params) {
+			if ("id".equals(param.getName())) {
+				return param.getValue();
+			}
+		}
+		return null;
+	}
+
+}
--- a/commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
+++ b/commafeed-server/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
@@ -0,0 +1,100 @@
+package com.commafeed.backend.favicon;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Optional;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URLEncodedUtils;
+
+import com.commafeed.CommaFeedConfiguration;
+import com.commafeed.backend.HttpGetter;
+import com.commafeed.backend.HttpGetter.HttpResult;
+import com.commafeed.backend.model.Feed;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpRequestInitializer;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.services.youtube.YouTube;
+import com.google.api.services.youtube.model.Channel;
+import com.google.api.services.youtube.model.ChannelListResponse;
+import com.google.api.services.youtube.model.Thumbnail;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
+@Singleton
+public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
+
+	private final HttpGetter getter;
+	private final CommaFeedConfiguration config;
+
+	@Override
+	public Favicon fetch(Feed feed) {
+		String url = feed.getUrl();
+
+		if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
+			return null;
+		}
+
+		String googleAuthKey = config.getApplicationSettings().getGoogleAuthKey();
+		if (googleAuthKey == null) {
+			log.debug("no google auth key configured");
+			return null;
+		}
+
+		byte[] bytes = null;
+		String contentType = null;
+		try {
+			List<NameValuePair> params = URLEncodedUtils.parse(url.substring(url.indexOf("?") + 1), StandardCharsets.UTF_8);
+			Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
+			Optional<NameValuePair> channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id")).findFirst();
+			if (!userId.isPresent() && !channelId.isPresent()) {
+				return null;
+			}
+
+			YouTube youtube = new YouTube.Builder(new NetHttpTransport(), JacksonFactory.getDefaultInstance(),
+					new HttpRequestInitializer() {
+						@Override
+						public void initialize(HttpRequest request) throws IOException {
+						}
+					}).setApplicationName("CommaFeed").build();
+
+			YouTube.Channels.List list = youtube.channels().list("snippet");
+			list.setKey(googleAuthKey);
+			if (userId.isPresent()) {
+				list.setForUsername(userId.get().getValue());
+			} else {
+				list.setId(channelId.get().getValue());
+			}
+
+			log.debug("contacting youtube api");
+			ChannelListResponse response = list.execute();
+			if (response.getItems().isEmpty()) {
+				log.debug("youtube api returned no items");
+				return null;
+			}
+
+			Channel channel = response.getItems().get(0);
+			Thumbnail thumbnail = channel.getSnippet().getThumbnails().getDefault();
+
+			log.debug("fetching favicon");
+			HttpResult iconResult = getter.getBinary(thumbnail.getUrl(), TIMEOUT);
+			bytes = iconResult.getContent();
+			contentType = iconResult.getContentType();
+		} catch (Exception e) {
+			log.debug("Failed to retrieve YouTube icon", e);
+		}
+
+		if (!isValidIconResponse(bytes, contentType)) {
+			return null;
+		}
+		return new Favicon(bytes, contentType);
+	}
+}