diff --git a/CHANGELOG b/CHANGELOG
index 511b1ad0..d714369d 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -2,6 +2,7 @@ v 2.0.3
- internet explorer ajax cache workaround
- categories are now deletable again
- openshift support is back
+ - youtube feeds now show user favicon instead of youtube favicon
v 2.0.2
- api using the api key is now working again
- context path is now configurable in config.yml (see app.contextPath in config.yml.example)
diff --git a/README.md b/README.md
index 6715a593..a57e0205 100644
--- a/README.md
+++ b/README.md
@@ -22,6 +22,9 @@ You also need Maven 3.x (and a Java 1.7+ JDK) installed in order to build the ap
To install maven and openjdk on Ubuntu, issue the following commands
sudo apt-get install build-essential openjdk-7-jdk maven
+ # Make sure java7 is the selected java version
+ sudo update-alternatives --config java
+ sudo update-alternatives --config javac
On Windows and other operating systems, just download maven 3.x from the [official site](http://maven.apache.org/), extract it somewhere and add the `bin` directory to your `PATH` environment variable.
diff --git a/pom.xml b/pom.xml
index 7276eca5..e2c8314d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -180,6 +180,11 @@
guice
3.0
+
+ com.google.inject.extensions
+ guice-multibindings
+ 3.0
+
io.dropwizard
diff --git a/src/main/app/images/google_reader_icon.png b/src/main/app/images/google_reader_icon.png
deleted file mode 100644
index 086b89d1..00000000
Binary files a/src/main/app/images/google_reader_icon.png and /dev/null differ
diff --git a/src/main/app/images/logo.png b/src/main/app/images/logo.png
deleted file mode 100644
index cb17cb87..00000000
Binary files a/src/main/app/images/logo.png and /dev/null differ
diff --git a/src/main/java/com/commafeed/CommaFeedApplication.java b/src/main/java/com/commafeed/CommaFeedApplication.java
index ff90d4cf..f15ec6c3 100644
--- a/src/main/java/com/commafeed/CommaFeedApplication.java
+++ b/src/main/java/com/commafeed/CommaFeedApplication.java
@@ -9,11 +9,17 @@ import io.dropwizard.servlets.CacheBustingFilter;
import io.dropwizard.setup.Bootstrap;
import io.dropwizard.setup.Environment;
+import java.io.IOException;
import java.util.Date;
import java.util.EnumSet;
import java.util.concurrent.ScheduledExecutorService;
import javax.servlet.DispatcherType;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
import org.eclipse.jetty.server.session.SessionHandler;
@@ -150,8 +156,18 @@ public class CommaFeedApplication extends Application {
swaggerConfig.setBasePath("/rest");
// cache configuration
- environment.servlets().addFilter("cache-filter", new CacheBustingFilter())
- .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/*");
+ // prevent caching on REST resources, except for favicons
+ environment.servlets().addFilter("cache-filter", new CacheBustingFilter() {
+ @Override
+ public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
+ String path = ((HttpServletRequest) request).getRequestURI();
+ if (path.contains("/feed/favicon")) {
+ chain.doFilter(request, response);
+ } else {
+ super.doFilter(request, response, chain);
+ }
+ }
+ }).addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/rest/*");
}
public static void main(String[] args) throws Exception {
diff --git a/src/main/java/com/commafeed/CommaFeedModule.java b/src/main/java/com/commafeed/CommaFeedModule.java
index 1524865b..a18b0be3 100644
--- a/src/main/java/com/commafeed/CommaFeedModule.java
+++ b/src/main/java/com/commafeed/CommaFeedModule.java
@@ -11,8 +11,12 @@ import com.commafeed.CommaFeedConfiguration.CacheType;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.cache.NoopCacheService;
import com.commafeed.backend.cache.RedisCacheService;
+import com.commafeed.backend.favicon.DefaultFaviconFetcher;
+import com.commafeed.backend.favicon.AbstractFaviconFetcher;
+import com.commafeed.backend.favicon.YoutubeFaviconFetcher;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
+import com.google.inject.multibindings.Multibinder;
@RequiredArgsConstructor
@Slf4j
@@ -33,5 +37,9 @@ public class CommaFeedModule extends AbstractModule {
: new RedisCacheService(config.getRedisPoolFactory().build());
log.info("using cache {}", cacheService.getClass());
bind(CacheService.class).toInstance(cacheService);
+
+ Multibinder multibinder = Multibinder.newSetBinder(binder(), AbstractFaviconFetcher.class);
+ multibinder.addBinding().to(YoutubeFaviconFetcher.class);
+ multibinder.addBinding().to(DefaultFaviconFetcher.class);
}
}
diff --git a/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java b/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java
new file mode 100644
index 00000000..bd10f197
--- /dev/null
+++ b/src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java
@@ -0,0 +1,51 @@
+package com.commafeed.backend.favicon;
+
+import java.util.Arrays;
+import java.util.List;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.commafeed.backend.model.Feed;
+
+@Slf4j
+public abstract class AbstractFaviconFetcher {
+
+ private static List ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
+ private static long MIN_ICON_LENGTH = 100;
+ private static long MAX_ICON_LENGTH = 100000;
+
+ protected static int TIMEOUT = 4000;
+
+ public abstract byte[] fetch(Feed feed);
+
+ protected boolean isValidIconResponse(byte[] content, String contentType) {
+ if (content == null) {
+ return false;
+ }
+
+ long length = content.length;
+
+ if (StringUtils.isNotBlank(contentType)) {
+ contentType = contentType.split(";")[0];
+ }
+
+ if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
+ log.debug("Content-Type {} is blacklisted", contentType);
+ return false;
+ }
+
+ if (length < MIN_ICON_LENGTH) {
+ log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
+ return false;
+ }
+
+ if (length > MAX_ICON_LENGTH) {
+ log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
+ return false;
+ }
+
+ return true;
+ }
+}
diff --git a/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java b/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java
new file mode 100644
index 00000000..d6b90f8b
--- /dev/null
+++ b/src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java
@@ -0,0 +1,123 @@
+package com.commafeed.backend.favicon;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang.StringUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.select.Elements;
+
+import com.commafeed.backend.HttpGetter;
+import com.commafeed.backend.HttpGetter.HttpResult;
+import com.commafeed.backend.feed.FeedUtils;
+import com.commafeed.backend.model.Feed;
+
+/**
+ * Inspired/Ported from https://github.com/potatolondon/getfavicon
+ *
+ */
+@Slf4j
+@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
+@Singleton
+public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
+
+ private final HttpGetter getter;
+
+ @Override
+ public byte[] fetch(Feed feed) {
+ String url = feed.getLink() != null ? feed.getLink() : feed.getUrl();
+
+ if (url == null) {
+ log.debug("url is null");
+ return null;
+ }
+
+ int doubleSlash = url.indexOf("//");
+ if (doubleSlash == -1) {
+ doubleSlash = 0;
+ } else {
+ doubleSlash += 2;
+ }
+ int firstSlash = url.indexOf('/', doubleSlash);
+ if (firstSlash != -1) {
+ url = url.substring(0, firstSlash);
+ }
+
+ byte[] icon = getIconAtRoot(url);
+
+ if (icon == null) {
+ icon = getIconInPage(url);
+ }
+
+ return icon;
+ }
+
+ private byte[] getIconAtRoot(String url) {
+ byte[] bytes = null;
+ String contentType = null;
+
+ try {
+ url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico";
+ log.debug("getting root icon at {}", url);
+ HttpResult result = getter.getBinary(url, TIMEOUT);
+ bytes = result.getContent();
+ contentType = result.getContentType();
+ } catch (Exception e) {
+ log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
+ }
+
+ if (!isValidIconResponse(bytes, contentType)) {
+ bytes = null;
+ }
+ return bytes;
+ }
+
+ private byte[] getIconInPage(String url) {
+
+ Document doc = null;
+ try {
+ HttpResult result = getter.getBinary(url, TIMEOUT);
+ doc = Jsoup.parse(new String(result.getContent()), url);
+ } catch (Exception e) {
+ log.debug("Failed to retrieve page to find icon", e);
+ return null;
+ }
+
+ Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
+
+ if (icons.isEmpty()) {
+ log.debug("No icon found in page {}", url);
+ return null;
+ }
+
+ String href = icons.get(0).attr("abs:href");
+ if (StringUtils.isBlank(href)) {
+ log.debug("No icon found in page");
+ return null;
+ }
+
+ log.debug("Found unconfirmed iconInPage at {}", href);
+
+ byte[] bytes = null;
+ String contentType = null;
+ try {
+ HttpResult result = getter.getBinary(href, TIMEOUT);
+ bytes = result.getContent();
+ contentType = result.getContentType();
+ } catch (Exception e) {
+ log.debug("Failed to retrieve icon found in page {}", href, e);
+ return null;
+ }
+
+ if (!isValidIconResponse(bytes, contentType)) {
+ log.debug("Invalid icon found for {}", href);
+ return null;
+ }
+
+ return bytes;
+ }
+}
diff --git a/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java b/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
new file mode 100644
index 00000000..7998d03d
--- /dev/null
+++ b/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
@@ -0,0 +1,90 @@
+package com.commafeed.backend.favicon;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.select.Elements;
+
+import com.commafeed.backend.HttpGetter;
+import com.commafeed.backend.HttpGetter.HttpResult;
+import com.commafeed.backend.model.Feed;
+
+@Slf4j
+@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
+@Singleton
+public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
+
+ private final HttpGetter getter;
+
+ @Override
+ public byte[] fetch(Feed feed) {
+ String url = feed.getUrl();
+
+ if (!url.toLowerCase().contains("://gdata.youtube.com/")) {
+ return null;
+ }
+
+ String userName = extractUserName(url);
+ if (userName == null) {
+ return null;
+ }
+
+ String profileUrl = "https://gdata.youtube.com/feeds/users/" + userName;
+
+ byte[] bytes = null;
+ String contentType = null;
+
+ try {
+ log.debug("Getting YouTube user's icon, {}", url);
+
+ // initial get to translate username to obscure user thumbnail URL
+ HttpResult profileResult = getter.getBinary(profileUrl, TIMEOUT);
+ Document doc = Jsoup.parse(new String(profileResult.getContent()), profileUrl);
+
+ Elements thumbnails = doc.select("media|thumbnail");
+ if (thumbnails.isEmpty()) {
+ return null;
+ }
+
+ String thumbnailUrl = thumbnails.get(0).attr("abs:url");
+
+ int thumbnailStart = thumbnailUrl.indexOf("", thumbnailStart);
+ if (thumbnailStart != -1) {
+ thumbnailUrl = thumbnailUrl.substring(thumbnailStart + " ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
- private static long MIN_ICON_LENGTH = 100;
- private static long MAX_ICON_LENGTH = 100000;
- private static int TIMEOUT = 4000;
-
- private final HttpGetter getter;
-
- public byte[] fetch(String url) {
-
- if (url == null) {
- log.debug("url is null");
- return null;
- }
-
- // Get YouTube Icon here
- if (url.toLowerCase().contains("://gdata.youtube.com/")) {
- byte[] icon = getYouTubeIcon(url);
- return icon;
- }
-
- int doubleSlash = url.indexOf("//");
- if (doubleSlash == -1) {
- doubleSlash = 0;
- } else {
- doubleSlash += 2;
- }
- int firstSlash = url.indexOf('/', doubleSlash);
- if (firstSlash != -1) {
- url = url.substring(0, firstSlash);
- }
-
- byte[] icon = getIconAtRoot(url);
-
- if (icon == null) {
- icon = getIconInPage(url);
- }
-
- return icon;
- }
-
- private byte[] getIconAtRoot(String url) {
- byte[] bytes = null;
- String contentType = null;
-
- try {
- url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico";
- log.debug("getting root icon at {}", url);
- HttpResult result = getter.getBinary(url, TIMEOUT);
- bytes = result.getContent();
- contentType = result.getContentType();
- } catch (Exception e) {
- log.debug("Failed to retrieve iconAtRoot: " + e.getMessage(), e);
- }
-
- if (!isValidIconResponse(bytes, contentType)) {
- bytes = null;
- }
- return bytes;
- }
-
- private boolean isValidIconResponse(byte[] content, String contentType) {
- if (content == null) {
- return false;
- }
-
- long length = content.length;
-
- if (StringUtils.isNotBlank(contentType)) {
- contentType = contentType.split(";")[0];
- }
-
- if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
- log.debug("Content-Type {} is blacklisted", contentType);
- return false;
- }
-
- if (length < MIN_ICON_LENGTH) {
- log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
- return false;
- }
-
- if (length > MAX_ICON_LENGTH) {
- log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
- return false;
- }
-
- return true;
- }
-
- private byte[] getIconInPage(String url) {
-
- Document doc = null;
- try {
- HttpResult result = getter.getBinary(url, TIMEOUT);
- doc = Jsoup.parse(new String(result.getContent()), url);
- } catch (Exception e) {
- log.debug("Failed to retrieve page to find icon");
- return null;
- }
-
- Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
-
- if (icons.isEmpty()) {
- log.debug("No icon found in page {}", url);
- return null;
- }
-
- String href = icons.get(0).attr("abs:href");
- if (StringUtils.isBlank(href)) {
- log.debug("No icon found in page");
- return null;
- }
-
- log.debug("Found unconfirmed iconInPage at {}", href);
-
- byte[] bytes = null;
- String contentType = null;
- try {
- HttpResult result = getter.getBinary(href, TIMEOUT);
- bytes = result.getContent();
- contentType = result.getContentType();
- } catch (Exception e) {
- log.debug("Failed to retrieve icon found in page {}", href);
- return null;
- }
-
- if (!isValidIconResponse(bytes, contentType)) {
- log.debug("Invalid icon found for {}", href);
- return null;
- }
-
- return bytes;
- }
-
- /*
- * Instead of grabbing the actual favicon, grab the user's icon
- * This prevents a whole bunch of repeated YouTube icons, replacing
- * each with identifiable user icons.
- */
- private byte[] getYouTubeIcon(String url) {
- byte[] bytes = null;
- String contentType = null;
- String username = null;
- String imageUrl = null;
- String thumbnailUrl = null;
- try {
- int apiOrBase = url.indexOf("/users/");
- int userEndSlash = url.indexOf('/', apiOrBase + "/users/".length());
- if (userEndSlash != -1) {
- username = url.substring(apiOrBase + "/users/".length(), userEndSlash);
- }
- imageUrl = "https://gdata.youtube.com/feeds/users/" + username;
- log.debug("Getting YouTube user's icon, {}", url);
-
- //initial get to translate username to obscure user thumbnail URL
- HttpResult result = getter.getBinary(imageUrl, TIMEOUT);
- bytes = result.getContent();
- contentType = result.getContentType();
- thumbnailUrl = FeedUtils.parseForImageUrl(bytes);
-
- int thumbnailStart = thumbnailUrl.indexOf("", thumbnailStart);
- if (thumbnailStart != -1) {
- thumbnailUrl = thumbnailUrl.substring(thumbnailStart+"