mirror of
https://github.com/Athou/commafeed.git
synced 2026-09-23 20:45:07 +00:00
normalize line endings
This commit is contained in:
Athou
@@ -1,49 +1,49 @@
|
||||
package com.commafeed.backend.favicon;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import com.commafeed.backend.model.Feed;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public abstract class AbstractFaviconFetcher {
|
||||
|
||||
private static final List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
|
||||
private static final long MIN_ICON_LENGTH = 100;
|
||||
private static final long MAX_ICON_LENGTH = 100000;
|
||||
|
||||
public abstract Favicon fetch(Feed feed);
|
||||
|
||||
protected boolean isValidIconResponse(byte[] content, String contentType) {
|
||||
if (content == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
long length = content.length;
|
||||
|
||||
if (StringUtils.isNotBlank(contentType)) {
|
||||
contentType = contentType.split(";")[0];
|
||||
}
|
||||
|
||||
if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
|
||||
log.debug("Content-Type {} is blacklisted", contentType);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (length < MIN_ICON_LENGTH) {
|
||||
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (length > MAX_ICON_LENGTH) {
|
||||
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
package com.commafeed.backend.favicon;
|
||||
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import com.commafeed.backend.model.Feed;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
public abstract class AbstractFaviconFetcher {
|
||||
|
||||
private static final List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
|
||||
private static final long MIN_ICON_LENGTH = 100;
|
||||
private static final long MAX_ICON_LENGTH = 100000;
|
||||
|
||||
public abstract Favicon fetch(Feed feed);
|
||||
|
||||
protected boolean isValidIconResponse(byte[] content, String contentType) {
|
||||
if (content == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
long length = content.length;
|
||||
|
||||
if (StringUtils.isNotBlank(contentType)) {
|
||||
contentType = contentType.split(";")[0];
|
||||
}
|
||||
|
||||
if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
|
||||
log.debug("Content-Type {} is blacklisted", contentType);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (length < MIN_ICON_LENGTH) {
|
||||
log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (length > MAX_ICON_LENGTH) {
|
||||
log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,133 +1,133 @@
|
||||
package com.commafeed.backend.favicon;
|
||||
|
||||
import jakarta.annotation.Priority;
|
||||
import jakarta.inject.Singleton;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.select.Elements;
|
||||
|
||||
import com.commafeed.backend.HttpGetter;
|
||||
import com.commafeed.backend.HttpGetter.HttpResult;
|
||||
import com.commafeed.backend.feed.FeedUtils;
|
||||
import com.commafeed.backend.model.Feed;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Inspired/Ported from https://github.com/potatolondon/getfavicon
|
||||
*
|
||||
*/
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
@Singleton
|
||||
@Priority(Integer.MIN_VALUE)
|
||||
public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
|
||||
|
||||
private final HttpGetter getter;
|
||||
|
||||
@Override
|
||||
public Favicon fetch(Feed feed) {
|
||||
Favicon icon = fetch(feed.getLink());
|
||||
if (icon == null) {
|
||||
icon = fetch(feed.getUrl());
|
||||
}
|
||||
return icon;
|
||||
}
|
||||
|
||||
private Favicon fetch(String url) {
|
||||
if (url == null) {
|
||||
log.debug("url is null");
|
||||
return null;
|
||||
}
|
||||
|
||||
int doubleSlash = url.indexOf("//");
|
||||
if (doubleSlash == -1) {
|
||||
doubleSlash = 0;
|
||||
} else {
|
||||
doubleSlash += 2;
|
||||
}
|
||||
int firstSlash = url.indexOf('/', doubleSlash);
|
||||
if (firstSlash != -1) {
|
||||
url = url.substring(0, firstSlash);
|
||||
}
|
||||
|
||||
Favicon icon = getIconAtRoot(url);
|
||||
|
||||
if (icon == null) {
|
||||
icon = getIconInPage(url);
|
||||
}
|
||||
|
||||
return icon;
|
||||
}
|
||||
|
||||
private Favicon getIconAtRoot(String url) {
|
||||
byte[] bytes = null;
|
||||
String contentType = null;
|
||||
|
||||
try {
|
||||
url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico";
|
||||
log.debug("getting root icon at {}", url);
|
||||
HttpResult result = getter.get(url);
|
||||
bytes = result.getContent();
|
||||
contentType = result.getContentType();
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to retrieve iconAtRoot for url {}: ", url);
|
||||
log.trace("Failed to retrieve iconAtRoot for url {}: ", url, e);
|
||||
}
|
||||
|
||||
if (!isValidIconResponse(bytes, contentType)) {
|
||||
return null;
|
||||
}
|
||||
return new Favicon(bytes, contentType);
|
||||
}
|
||||
|
||||
private Favicon getIconInPage(String url) {
|
||||
|
||||
Document doc;
|
||||
try {
|
||||
HttpResult result = getter.get(url);
|
||||
doc = Jsoup.parse(new String(result.getContent()), url);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to retrieve page to find icon");
|
||||
log.trace("Failed to retrieve page to find icon", e);
|
||||
return null;
|
||||
}
|
||||
|
||||
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
|
||||
|
||||
if (icons.isEmpty()) {
|
||||
log.debug("No icon found in page {}", url);
|
||||
return null;
|
||||
}
|
||||
|
||||
String href = icons.get(0).attr("abs:href");
|
||||
if (StringUtils.isBlank(href)) {
|
||||
log.debug("No icon found in page");
|
||||
return null;
|
||||
}
|
||||
|
||||
log.debug("Found unconfirmed iconInPage at {}", href);
|
||||
|
||||
byte[] bytes;
|
||||
String contentType;
|
||||
try {
|
||||
HttpResult result = getter.get(href);
|
||||
bytes = result.getContent();
|
||||
contentType = result.getContentType();
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to retrieve icon found in page {}", href);
|
||||
log.trace("Failed to retrieve icon found in page {}", href, e);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!isValidIconResponse(bytes, contentType)) {
|
||||
log.debug("Invalid icon found for {}", href);
|
||||
return null;
|
||||
}
|
||||
|
||||
return new Favicon(bytes, contentType);
|
||||
}
|
||||
}
|
||||
package com.commafeed.backend.favicon;
|
||||
|
||||
import jakarta.annotation.Priority;
|
||||
import jakarta.inject.Singleton;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.jsoup.Jsoup;
|
||||
import org.jsoup.nodes.Document;
|
||||
import org.jsoup.select.Elements;
|
||||
|
||||
import com.commafeed.backend.HttpGetter;
|
||||
import com.commafeed.backend.HttpGetter.HttpResult;
|
||||
import com.commafeed.backend.feed.FeedUtils;
|
||||
import com.commafeed.backend.model.Feed;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
/**
|
||||
* Inspired/Ported from https://github.com/potatolondon/getfavicon
|
||||
*
|
||||
*/
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
@Singleton
|
||||
@Priority(Integer.MIN_VALUE)
|
||||
public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
|
||||
|
||||
private final HttpGetter getter;
|
||||
|
||||
@Override
|
||||
public Favicon fetch(Feed feed) {
|
||||
Favicon icon = fetch(feed.getLink());
|
||||
if (icon == null) {
|
||||
icon = fetch(feed.getUrl());
|
||||
}
|
||||
return icon;
|
||||
}
|
||||
|
||||
private Favicon fetch(String url) {
|
||||
if (url == null) {
|
||||
log.debug("url is null");
|
||||
return null;
|
||||
}
|
||||
|
||||
int doubleSlash = url.indexOf("//");
|
||||
if (doubleSlash == -1) {
|
||||
doubleSlash = 0;
|
||||
} else {
|
||||
doubleSlash += 2;
|
||||
}
|
||||
int firstSlash = url.indexOf('/', doubleSlash);
|
||||
if (firstSlash != -1) {
|
||||
url = url.substring(0, firstSlash);
|
||||
}
|
||||
|
||||
Favicon icon = getIconAtRoot(url);
|
||||
|
||||
if (icon == null) {
|
||||
icon = getIconInPage(url);
|
||||
}
|
||||
|
||||
return icon;
|
||||
}
|
||||
|
||||
private Favicon getIconAtRoot(String url) {
|
||||
byte[] bytes = null;
|
||||
String contentType = null;
|
||||
|
||||
try {
|
||||
url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico";
|
||||
log.debug("getting root icon at {}", url);
|
||||
HttpResult result = getter.get(url);
|
||||
bytes = result.getContent();
|
||||
contentType = result.getContentType();
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to retrieve iconAtRoot for url {}: ", url);
|
||||
log.trace("Failed to retrieve iconAtRoot for url {}: ", url, e);
|
||||
}
|
||||
|
||||
if (!isValidIconResponse(bytes, contentType)) {
|
||||
return null;
|
||||
}
|
||||
return new Favicon(bytes, contentType);
|
||||
}
|
||||
|
||||
private Favicon getIconInPage(String url) {
|
||||
|
||||
Document doc;
|
||||
try {
|
||||
HttpResult result = getter.get(url);
|
||||
doc = Jsoup.parse(new String(result.getContent()), url);
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to retrieve page to find icon");
|
||||
log.trace("Failed to retrieve page to find icon", e);
|
||||
return null;
|
||||
}
|
||||
|
||||
Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
|
||||
|
||||
if (icons.isEmpty()) {
|
||||
log.debug("No icon found in page {}", url);
|
||||
return null;
|
||||
}
|
||||
|
||||
String href = icons.get(0).attr("abs:href");
|
||||
if (StringUtils.isBlank(href)) {
|
||||
log.debug("No icon found in page");
|
||||
return null;
|
||||
}
|
||||
|
||||
log.debug("Found unconfirmed iconInPage at {}", href);
|
||||
|
||||
byte[] bytes;
|
||||
String contentType;
|
||||
try {
|
||||
HttpResult result = getter.get(href);
|
||||
bytes = result.getContent();
|
||||
contentType = result.getContentType();
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to retrieve icon found in page {}", href);
|
||||
log.trace("Failed to retrieve icon found in page {}", href, e);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!isValidIconResponse(bytes, contentType)) {
|
||||
log.debug("Invalid icon found for {}", href);
|
||||
return null;
|
||||
}
|
||||
|
||||
return new Favicon(bytes, contentType);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,73 +1,73 @@
|
||||
package com.commafeed.backend.favicon;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.List;
|
||||
|
||||
import jakarta.inject.Singleton;
|
||||
|
||||
import org.apache.hc.core5.http.NameValuePair;
|
||||
import org.apache.hc.core5.net.URIBuilder;
|
||||
|
||||
import com.commafeed.backend.HttpGetter;
|
||||
import com.commafeed.backend.HttpGetter.HttpResult;
|
||||
import com.commafeed.backend.model.Feed;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
@Singleton
|
||||
public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
|
||||
|
||||
private final HttpGetter getter;
|
||||
|
||||
@Override
|
||||
public Favicon fetch(Feed feed) {
|
||||
String url = feed.getUrl();
|
||||
|
||||
if (!url.toLowerCase().contains("www.facebook.com")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String userName = extractUserName(url);
|
||||
if (userName == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
|
||||
|
||||
byte[] bytes = null;
|
||||
String contentType = null;
|
||||
|
||||
try {
|
||||
log.debug("Getting Facebook user's icon, {}", url);
|
||||
|
||||
HttpResult iconResult = getter.get(iconUrl);
|
||||
bytes = iconResult.getContent();
|
||||
contentType = iconResult.getContentType();
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to retrieve Facebook icon", e);
|
||||
}
|
||||
|
||||
if (!isValidIconResponse(bytes, contentType)) {
|
||||
return null;
|
||||
}
|
||||
return new Favicon(bytes, contentType);
|
||||
}
|
||||
|
||||
private String extractUserName(String url) {
|
||||
URI uri;
|
||||
try {
|
||||
uri = new URI(url);
|
||||
} catch (URISyntaxException e) {
|
||||
log.debug("could not parse url", e);
|
||||
return null;
|
||||
}
|
||||
|
||||
List<NameValuePair> params = new URIBuilder(uri).getQueryParams();
|
||||
return params.stream().filter(p -> "id".equals(p.getName())).map(NameValuePair::getValue).findFirst().orElse(null);
|
||||
}
|
||||
|
||||
}
|
||||
package com.commafeed.backend.favicon;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.util.List;
|
||||
|
||||
import jakarta.inject.Singleton;
|
||||
|
||||
import org.apache.hc.core5.http.NameValuePair;
|
||||
import org.apache.hc.core5.net.URIBuilder;
|
||||
|
||||
import com.commafeed.backend.HttpGetter;
|
||||
import com.commafeed.backend.HttpGetter.HttpResult;
|
||||
import com.commafeed.backend.model.Feed;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
@Singleton
|
||||
public class FacebookFaviconFetcher extends AbstractFaviconFetcher {
|
||||
|
||||
private final HttpGetter getter;
|
||||
|
||||
@Override
|
||||
public Favicon fetch(Feed feed) {
|
||||
String url = feed.getUrl();
|
||||
|
||||
if (!url.toLowerCase().contains("www.facebook.com")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String userName = extractUserName(url);
|
||||
if (userName == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
String iconUrl = String.format("https://graph.facebook.com/%s/picture?type=square&height=16", userName);
|
||||
|
||||
byte[] bytes = null;
|
||||
String contentType = null;
|
||||
|
||||
try {
|
||||
log.debug("Getting Facebook user's icon, {}", url);
|
||||
|
||||
HttpResult iconResult = getter.get(iconUrl);
|
||||
bytes = iconResult.getContent();
|
||||
contentType = iconResult.getContentType();
|
||||
} catch (Exception e) {
|
||||
log.debug("Failed to retrieve Facebook icon", e);
|
||||
}
|
||||
|
||||
if (!isValidIconResponse(bytes, contentType)) {
|
||||
return null;
|
||||
}
|
||||
return new Favicon(bytes, contentType);
|
||||
}
|
||||
|
||||
private String extractUserName(String url) {
|
||||
URI uri;
|
||||
try {
|
||||
uri = new URI(url);
|
||||
} catch (URISyntaxException e) {
|
||||
log.debug("could not parse url", e);
|
||||
return null;
|
||||
}
|
||||
|
||||
List<NameValuePair> params = new URIBuilder(uri).getQueryParams();
|
||||
return params.stream().filter(p -> "id".equals(p.getName())).map(NameValuePair::getValue).findFirst().orElse(null);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,31 +1,31 @@
|
||||
package com.commafeed.backend.favicon;
|
||||
|
||||
import jakarta.ws.rs.core.MediaType;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@RequiredArgsConstructor
|
||||
@Getter
|
||||
@Slf4j
|
||||
public class Favicon {
|
||||
|
||||
private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.valueOf("image/x-icon");
|
||||
|
||||
private final byte[] icon;
|
||||
private final MediaType mediaType;
|
||||
|
||||
public Favicon(byte[] icon, String contentType) {
|
||||
this(icon, parseMediaType(contentType));
|
||||
}
|
||||
|
||||
private static MediaType parseMediaType(String contentType) {
|
||||
try {
|
||||
return MediaType.valueOf(contentType);
|
||||
} catch (Exception e) {
|
||||
log.debug("invalid content type '{}' received, returning default value", contentType);
|
||||
return DEFAULT_MEDIA_TYPE;
|
||||
}
|
||||
}
|
||||
package com.commafeed.backend.favicon;
|
||||
|
||||
import jakarta.ws.rs.core.MediaType;
|
||||
|
||||
import lombok.Getter;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@RequiredArgsConstructor
|
||||
@Getter
|
||||
@Slf4j
|
||||
public class Favicon {
|
||||
|
||||
private static final MediaType DEFAULT_MEDIA_TYPE = MediaType.valueOf("image/x-icon");
|
||||
|
||||
private final byte[] icon;
|
||||
private final MediaType mediaType;
|
||||
|
||||
public Favicon(byte[] icon, String contentType) {
|
||||
this(icon, parseMediaType(contentType));
|
||||
}
|
||||
|
||||
private static MediaType parseMediaType(String contentType) {
|
||||
try {
|
||||
return MediaType.valueOf(contentType);
|
||||
} catch (Exception e) {
|
||||
log.debug("invalid content type '{}' received, returning default value", contentType);
|
||||
return DEFAULT_MEDIA_TYPE;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,135 +1,135 @@
|
||||
package com.commafeed.backend.favicon;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import jakarta.inject.Singleton;
|
||||
import jakarta.ws.rs.core.UriBuilder;
|
||||
|
||||
import org.apache.commons.lang3.ArrayUtils;
|
||||
import org.apache.hc.core5.http.NameValuePair;
|
||||
import org.apache.hc.core5.net.URIBuilder;
|
||||
|
||||
import com.commafeed.CommaFeedConfiguration;
|
||||
import com.commafeed.backend.HttpGetter;
|
||||
import com.commafeed.backend.HttpGetter.HostNotAllowedException;
|
||||
import com.commafeed.backend.HttpGetter.HttpResult;
|
||||
import com.commafeed.backend.HttpGetter.NotModifiedException;
|
||||
import com.commafeed.backend.HttpGetter.SchemeNotAllowedException;
|
||||
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
|
||||
import com.commafeed.backend.model.Feed;
|
||||
import com.fasterxml.jackson.core.JsonPointer;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
@Singleton
|
||||
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
|
||||
|
||||
private static final JsonPointer CHANNEL_THUMBNAIL_URL = JsonPointer.compile("/items/0/snippet/thumbnails/default/url");
|
||||
private static final JsonPointer PLAYLIST_CHANNEL_ID = JsonPointer.compile("/items/0/snippet/channelId");
|
||||
|
||||
private final HttpGetter getter;
|
||||
private final CommaFeedConfiguration config;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@Override
|
||||
public Favicon fetch(Feed feed) {
|
||||
String url = feed.getUrl();
|
||||
if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Optional<String> googleAuthKey = config.googleAuthKey();
|
||||
if (googleAuthKey.isEmpty()) {
|
||||
log.debug("no google auth key configured");
|
||||
return null;
|
||||
}
|
||||
|
||||
byte[] bytes = null;
|
||||
String contentType = null;
|
||||
try {
|
||||
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
|
||||
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
|
||||
Optional<NameValuePair> channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id")).findFirst();
|
||||
Optional<NameValuePair> playlistId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id")).findFirst();
|
||||
|
||||
byte[] response = null;
|
||||
if (userId.isPresent()) {
|
||||
log.debug("contacting youtube api for user {}", userId.get().getValue());
|
||||
response = fetchForUser(googleAuthKey.get(), userId.get().getValue());
|
||||
} else if (channelId.isPresent()) {
|
||||
log.debug("contacting youtube api for channel {}", channelId.get().getValue());
|
||||
response = fetchForChannel(googleAuthKey.get(), channelId.get().getValue());
|
||||
} else if (playlistId.isPresent()) {
|
||||
log.debug("contacting youtube api for playlist {}", playlistId.get().getValue());
|
||||
response = fetchForPlaylist(googleAuthKey.get(), playlistId.get().getValue());
|
||||
}
|
||||
if (ArrayUtils.isEmpty(response)) {
|
||||
log.debug("youtube api returned empty response");
|
||||
return null;
|
||||
}
|
||||
|
||||
JsonNode thumbnailUrl = objectMapper.readTree(response).at(CHANNEL_THUMBNAIL_URL);
|
||||
if (thumbnailUrl.isMissingNode()) {
|
||||
log.debug("youtube api returned invalid response");
|
||||
return null;
|
||||
}
|
||||
|
||||
HttpResult iconResult = getter.get(thumbnailUrl.asText());
|
||||
bytes = iconResult.getContent();
|
||||
contentType = iconResult.getContentType();
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to retrieve YouTube icon", e);
|
||||
}
|
||||
|
||||
if (!isValidIconResponse(bytes, contentType)) {
|
||||
return null;
|
||||
}
|
||||
return new Favicon(bytes, contentType);
|
||||
}
|
||||
|
||||
private byte[] fetchForUser(String googleAuthKey, String userId)
|
||||
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
|
||||
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
|
||||
.queryParam("part", "snippet")
|
||||
.queryParam("key", googleAuthKey)
|
||||
.queryParam("forUsername", userId)
|
||||
.build();
|
||||
return getter.get(uri.toString()).getContent();
|
||||
}
|
||||
|
||||
private byte[] fetchForChannel(String googleAuthKey, String channelId)
|
||||
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
|
||||
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
|
||||
.queryParam("part", "snippet")
|
||||
.queryParam("key", googleAuthKey)
|
||||
.queryParam("id", channelId)
|
||||
.build();
|
||||
return getter.get(uri.toString()).getContent();
|
||||
}
|
||||
|
||||
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
|
||||
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
|
||||
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
|
||||
.queryParam("part", "snippet")
|
||||
.queryParam("key", googleAuthKey)
|
||||
.queryParam("id", playlistId)
|
||||
.build();
|
||||
byte[] playlistBytes = getter.get(uri.toString()).getContent();
|
||||
|
||||
JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID);
|
||||
if (channelId.isMissingNode()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return fetchForChannel(googleAuthKey, channelId.asText());
|
||||
}
|
||||
|
||||
}
|
||||
package com.commafeed.backend.favicon;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import jakarta.inject.Singleton;
|
||||
import jakarta.ws.rs.core.UriBuilder;
|
||||
|
||||
import org.apache.commons.lang3.ArrayUtils;
|
||||
import org.apache.hc.core5.http.NameValuePair;
|
||||
import org.apache.hc.core5.net.URIBuilder;
|
||||
|
||||
import com.commafeed.CommaFeedConfiguration;
|
||||
import com.commafeed.backend.HttpGetter;
|
||||
import com.commafeed.backend.HttpGetter.HostNotAllowedException;
|
||||
import com.commafeed.backend.HttpGetter.HttpResult;
|
||||
import com.commafeed.backend.HttpGetter.NotModifiedException;
|
||||
import com.commafeed.backend.HttpGetter.SchemeNotAllowedException;
|
||||
import com.commafeed.backend.HttpGetter.TooManyRequestsException;
|
||||
import com.commafeed.backend.model.Feed;
|
||||
import com.fasterxml.jackson.core.JsonPointer;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
@Slf4j
|
||||
@RequiredArgsConstructor
|
||||
@Singleton
|
||||
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
|
||||
|
||||
private static final JsonPointer CHANNEL_THUMBNAIL_URL = JsonPointer.compile("/items/0/snippet/thumbnails/default/url");
|
||||
private static final JsonPointer PLAYLIST_CHANNEL_ID = JsonPointer.compile("/items/0/snippet/channelId");
|
||||
|
||||
private final HttpGetter getter;
|
||||
private final CommaFeedConfiguration config;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
@Override
|
||||
public Favicon fetch(Feed feed) {
|
||||
String url = feed.getUrl();
|
||||
if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Optional<String> googleAuthKey = config.googleAuthKey();
|
||||
if (googleAuthKey.isEmpty()) {
|
||||
log.debug("no google auth key configured");
|
||||
return null;
|
||||
}
|
||||
|
||||
byte[] bytes = null;
|
||||
String contentType = null;
|
||||
try {
|
||||
List<NameValuePair> params = new URIBuilder(url).getQueryParams();
|
||||
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
|
||||
Optional<NameValuePair> channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel_id")).findFirst();
|
||||
Optional<NameValuePair> playlistId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("playlist_id")).findFirst();
|
||||
|
||||
byte[] response = null;
|
||||
if (userId.isPresent()) {
|
||||
log.debug("contacting youtube api for user {}", userId.get().getValue());
|
||||
response = fetchForUser(googleAuthKey.get(), userId.get().getValue());
|
||||
} else if (channelId.isPresent()) {
|
||||
log.debug("contacting youtube api for channel {}", channelId.get().getValue());
|
||||
response = fetchForChannel(googleAuthKey.get(), channelId.get().getValue());
|
||||
} else if (playlistId.isPresent()) {
|
||||
log.debug("contacting youtube api for playlist {}", playlistId.get().getValue());
|
||||
response = fetchForPlaylist(googleAuthKey.get(), playlistId.get().getValue());
|
||||
}
|
||||
if (ArrayUtils.isEmpty(response)) {
|
||||
log.debug("youtube api returned empty response");
|
||||
return null;
|
||||
}
|
||||
|
||||
JsonNode thumbnailUrl = objectMapper.readTree(response).at(CHANNEL_THUMBNAIL_URL);
|
||||
if (thumbnailUrl.isMissingNode()) {
|
||||
log.debug("youtube api returned invalid response");
|
||||
return null;
|
||||
}
|
||||
|
||||
HttpResult iconResult = getter.get(thumbnailUrl.asText());
|
||||
bytes = iconResult.getContent();
|
||||
contentType = iconResult.getContentType();
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to retrieve YouTube icon", e);
|
||||
}
|
||||
|
||||
if (!isValidIconResponse(bytes, contentType)) {
|
||||
return null;
|
||||
}
|
||||
return new Favicon(bytes, contentType);
|
||||
}
|
||||
|
||||
private byte[] fetchForUser(String googleAuthKey, String userId)
|
||||
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
|
||||
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
|
||||
.queryParam("part", "snippet")
|
||||
.queryParam("key", googleAuthKey)
|
||||
.queryParam("forUsername", userId)
|
||||
.build();
|
||||
return getter.get(uri.toString()).getContent();
|
||||
}
|
||||
|
||||
private byte[] fetchForChannel(String googleAuthKey, String channelId)
|
||||
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
|
||||
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/channels")
|
||||
.queryParam("part", "snippet")
|
||||
.queryParam("key", googleAuthKey)
|
||||
.queryParam("id", channelId)
|
||||
.build();
|
||||
return getter.get(uri.toString()).getContent();
|
||||
}
|
||||
|
||||
private byte[] fetchForPlaylist(String googleAuthKey, String playlistId)
|
||||
throws IOException, NotModifiedException, TooManyRequestsException, HostNotAllowedException, SchemeNotAllowedException {
|
||||
URI uri = UriBuilder.fromUri("https://www.googleapis.com/youtube/v3/playlists")
|
||||
.queryParam("part", "snippet")
|
||||
.queryParam("key", googleAuthKey)
|
||||
.queryParam("id", playlistId)
|
||||
.build();
|
||||
byte[] playlistBytes = getter.get(uri.toString()).getContent();
|
||||
|
||||
JsonNode channelId = objectMapper.readTree(playlistBytes).at(PLAYLIST_CHANNEL_ID);
|
||||
if (channelId.isMissingNode()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return fetchForChannel(googleAuthKey, channelId.asText());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user