From 3794d61a77a3b4b1c4639c51452aa75f59fb41b9 Mon Sep 17 00:00:00 2001 From: Athou Date: Fri, 24 Oct 2014 11:15:17 +0200 Subject: [PATCH 1/3] readme update (fix #654) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 13016281..6715a593 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ You also need Maven 3.x (and a Java 1.7+ JDK) installed in order to build the ap To install maven and openjdk on Ubuntu, issue the following commands - sudo apt-get install openjdk-7-jdk maven + sudo apt-get install build-essential openjdk-7-jdk maven On Windows and other operating systems, just download maven 3.x from the [official site](http://maven.apache.org/), extract it somewhere and add the `bin` directory to your `PATH` environment variable. From 9e0c94f1a4d85bbde6806dc33248ccafc4d35f63 Mon Sep 17 00:00:00 2001 From: Tyler Gebhard Date: Sun, 26 Oct 2014 03:03:02 -0400 Subject: [PATCH 2/3] changes to the way favicons are retrieved for YouTube feeds. Now instead of fetching the YouTube logo, it fetches the YouTube user's custom thumbnail. --- .../backend/feed/FaviconFetcher.java | 59 +++++++++++++++++++ .../com/commafeed/backend/feed/FeedUtils.java | 14 +++++ .../commafeed/frontend/resource/FeedREST.java | 2 +- 3 files changed, 74 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/commafeed/backend/feed/FaviconFetcher.java b/src/main/java/com/commafeed/backend/feed/FaviconFetcher.java index a7349422..aeeec33e 100644 --- a/src/main/java/com/commafeed/backend/feed/FaviconFetcher.java +++ b/src/main/java/com/commafeed/backend/feed/FaviconFetcher.java @@ -39,6 +39,12 @@ public class FaviconFetcher { log.debug("url is null"); return null; } + + // Get YouTube Icon here + if (url.toLowerCase().contains("://gdata.youtube.com/")) { + byte[] icon = getYouTubeIcon(url); + return icon; + } int doubleSlash = url.indexOf("//"); if (doubleSlash == -1) { @@ -154,4 +160,57 @@ public class FaviconFetcher { return bytes; } + /* + * Instead of grabbing the actual favicon, grab the user's icon + * This prevents a whole bunch of repeated YouTube icons, replacing + * each with identifiable user icons. + */ + private byte[] getYouTubeIcon(String url) { + byte[] bytes = null; + String contentType = null; + String username = null; + String imageUrl = null; + String thumbnailUrl = null; + try { + int apiOrBase = url.indexOf("/users/"); + int userEndSlash = url.indexOf('/', apiOrBase + "/users/".length()); + if (userEndSlash != -1) { + username = url.substring(apiOrBase + "/users/".length(), userEndSlash); + } + imageUrl = "https://gdata.youtube.com/feeds/users/" + username; + log.debug("Getting YouTube user's icon, {}", url); + + //initial get to translate username to obscure user thumbnail URL + HttpResult result = getter.getBinary(imageUrl, TIMEOUT); + bytes = result.getContent(); + contentType = result.getContentType(); + thumbnailUrl = FeedUtils.parseForImageUrl(bytes); + + int thumbnailStart = thumbnailUrl.indexOf("", thumbnailStart); + if (thumbnailStart != -1) { + thumbnailUrl = thumbnailUrl.substring(thumbnailStart+"", thumbnailStart); + if (thumbnailStart != -1) { + thumbnailUrl = thumbnailUrl.substring(thumbnailStart + " ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html"); - private static long MIN_ICON_LENGTH = 100; - private static long MAX_ICON_LENGTH = 100000; - private static int TIMEOUT = 4000; - - private final HttpGetter getter; - - public byte[] fetch(String url) { - - if (url == null) { - log.debug("url is null"); - return null; - } - - // Get YouTube Icon here - if (url.toLowerCase().contains("://gdata.youtube.com/")) { - byte[] icon = getYouTubeIcon(url); - return icon; - } - - int doubleSlash = url.indexOf("//"); - if (doubleSlash == -1) { - doubleSlash = 0; - } else { - doubleSlash += 2; - } - int firstSlash = url.indexOf('/', doubleSlash); - if (firstSlash != -1) { - url = url.substring(0, firstSlash); - } - - byte[] icon = getIconAtRoot(url); - - if (icon == null) { - icon = getIconInPage(url); - } - - return icon; - } - - private byte[] getIconAtRoot(String url) { - byte[] bytes = null; - String contentType = null; - - try { - url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico"; - log.debug("getting root icon at {}", url); - HttpResult result = getter.getBinary(url, TIMEOUT); - bytes = result.getContent(); - contentType = result.getContentType(); - } catch (Exception e) { - log.debug("Failed to retrieve iconAtRoot: " + e.getMessage(), e); - } - - if (!isValidIconResponse(bytes, contentType)) { - bytes = null; - } - return bytes; - } - - private boolean isValidIconResponse(byte[] content, String contentType) { - if (content == null) { - return false; - } - - long length = content.length; - - if (StringUtils.isNotBlank(contentType)) { - contentType = contentType.split(";")[0]; - } - - if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) { - log.debug("Content-Type {} is blacklisted", contentType); - return false; - } - - if (length < MIN_ICON_LENGTH) { - log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH); - return false; - } - - if (length > MAX_ICON_LENGTH) { - log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH); - return false; - } - - return true; - } - - private byte[] getIconInPage(String url) { - - Document doc = null; - try { - HttpResult result = getter.getBinary(url, TIMEOUT); - doc = Jsoup.parse(new String(result.getContent()), url); - } catch (Exception e) { - log.debug("Failed to retrieve page to find icon"); - return null; - } - - Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]"); - - if (icons.isEmpty()) { - log.debug("No icon found in page {}", url); - return null; - } - - String href = icons.get(0).attr("abs:href"); - if (StringUtils.isBlank(href)) { - log.debug("No icon found in page"); - return null; - } - - log.debug("Found unconfirmed iconInPage at {}", href); - - byte[] bytes = null; - String contentType = null; - try { - HttpResult result = getter.getBinary(href, TIMEOUT); - bytes = result.getContent(); - contentType = result.getContentType(); - } catch (Exception e) { - log.debug("Failed to retrieve icon found in page {}", href); - return null; - } - - if (!isValidIconResponse(bytes, contentType)) { - log.debug("Invalid icon found for {}", href); - return null; - } - - return bytes; - } - - /* - * Instead of grabbing the actual favicon, grab the user's icon - * This prevents a whole bunch of repeated YouTube icons, replacing - * each with identifiable user icons. - */ - private byte[] getYouTubeIcon(String url) { - byte[] bytes = null; - String contentType = null; - String username = null; - String imageUrl = null; - String thumbnailUrl = null; - try { - int apiOrBase = url.indexOf("/users/"); - int userEndSlash = url.indexOf('/', apiOrBase + "/users/".length()); - if (userEndSlash != -1) { - username = url.substring(apiOrBase + "/users/".length(), userEndSlash); - } - imageUrl = "https://gdata.youtube.com/feeds/users/" + username; - log.debug("Getting YouTube user's icon, {}", url); - - //initial get to translate username to obscure user thumbnail URL - HttpResult result = getter.getBinary(imageUrl, TIMEOUT); - bytes = result.getContent(); - contentType = result.getContentType(); - thumbnailUrl = FeedUtils.parseForImageUrl(bytes); - - int thumbnailStart = thumbnailUrl.indexOf("", thumbnailStart); - if (thumbnailStart != -1) { - thumbnailUrl = thumbnailUrl.substring(thumbnailStart+"