Merge pull request #1 from Athou/master

Merge latest changes from Athou to Hubcapp

CHANGELOG

@@ -2,6 +2,7 @@ v 2.0.3
   - internet explorer ajax cache workaround
   - categories are now deletable again
   - openshift support is back
+  - youtube feeds now show user favicon instead of youtube favicon
 v 2.0.2
   - api using the api key is now working again
   - context path is now configurable in config.yml (see app.contextPath in config.yml.example)

README.md

@@ -22,6 +22,9 @@ You also need Maven 3.x (and a Java 1.7+ JDK) installed in order to build the ap
 To install maven and openjdk on Ubuntu, issue the following commands
 
     sudo apt-get install build-essential openjdk-7-jdk maven
+    # Make sure java7 is the selected java version
+    sudo update-alternatives --config java
+    sudo update-alternatives --config javac
     
     
 On Windows and other operating systems, just download maven 3.x from the [official site](http://maven.apache.org/), extract it somewhere and add the `bin` directory to your `PATH` environment variable.

pom.xml

@@ -180,6 +180,11 @@
 			<artifactId>guice</artifactId>
 			<version>3.0</version>
 		</dependency>
+		<dependency>
+			<groupId>com.google.inject.extensions</groupId>
+			<artifactId>guice-multibindings</artifactId>
+			<version>3.0</version>
+		</dependency>
 
 		<dependency>
 			<groupId>io.dropwizard</groupId>

src/main/java/com/commafeed/CommaFeedApplication.java

@@ -9,11 +9,17 @@ import io.dropwizard.servlets.CacheBustingFilter;
 import io.dropwizard.setup.Bootstrap;
 import io.dropwizard.setup.Environment;
 
+import java.io.IOException;
 import java.util.Date;
 import java.util.EnumSet;
 import java.util.concurrent.ScheduledExecutorService;
 
 import javax.servlet.DispatcherType;
+import javax.servlet.FilterChain;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
 
 import org.eclipse.jetty.server.session.SessionHandler;
 
@@ -150,8 +156,18 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
 		swaggerConfig.setBasePath("/rest");
 
 		// cache configuration
-		environment.servlets().addFilter("cache-filter", new CacheBustingFilter())
+		// prevent caching on REST resources, except for favicons
-				.addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/*");
+		environment.servlets().addFilter("cache-filter", new CacheBustingFilter() {
+			@Override
+			public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException {
+				String path = ((HttpServletRequest) request).getRequestURI();
+				if (path.contains("/feed/favicon")) {
+					chain.doFilter(request, response);
+				} else {
+					super.doFilter(request, response, chain);
+				}
+			}
+		}).addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), false, "/rest/*");
 	}
 
 	public static void main(String[] args) throws Exception {

src/main/java/com/commafeed/CommaFeedModule.java

@@ -11,8 +11,12 @@ import com.commafeed.CommaFeedConfiguration.CacheType;
 import com.commafeed.backend.cache.CacheService;
 import com.commafeed.backend.cache.NoopCacheService;
 import com.commafeed.backend.cache.RedisCacheService;
+import com.commafeed.backend.favicon.DefaultFaviconFetcher;
+import com.commafeed.backend.favicon.AbstractFaviconFetcher;
+import com.commafeed.backend.favicon.YoutubeFaviconFetcher;
 import com.google.inject.AbstractModule;
 import com.google.inject.Provides;
+import com.google.inject.multibindings.Multibinder;
 
 @RequiredArgsConstructor
 @Slf4j
@@ -33,5 +37,9 @@ public class CommaFeedModule extends AbstractModule {
 				: new RedisCacheService(config.getRedisPoolFactory().build());
 		log.info("using cache {}", cacheService.getClass());
 		bind(CacheService.class).toInstance(cacheService);
+
+		Multibinder<AbstractFaviconFetcher> multibinder = Multibinder.newSetBinder(binder(), AbstractFaviconFetcher.class);
+		multibinder.addBinding().to(YoutubeFaviconFetcher.class);
+		multibinder.addBinding().to(DefaultFaviconFetcher.class);
 	}
 }

src/main/java/com/commafeed/backend/favicon/AbstractFaviconFetcher.java

@@ -0,0 +1,51 @@
+package com.commafeed.backend.favicon;
+
+import java.util.Arrays;
+import java.util.List;
+
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang.StringUtils;
+
+import com.commafeed.backend.model.Feed;
+
+@Slf4j
+public abstract class AbstractFaviconFetcher {
+
+	private static List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
+	private static long MIN_ICON_LENGTH = 100;
+	private static long MAX_ICON_LENGTH = 100000;
+
+	protected static int TIMEOUT = 4000;
+
+	public abstract byte[] fetch(Feed feed);
+
+	protected boolean isValidIconResponse(byte[] content, String contentType) {
+		if (content == null) {
+			return false;
+		}
+
+		long length = content.length;
+
+		if (StringUtils.isNotBlank(contentType)) {
+			contentType = contentType.split(";")[0];
+		}
+
+		if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
+			log.debug("Content-Type {} is blacklisted", contentType);
+			return false;
+		}
+
+		if (length < MIN_ICON_LENGTH) {
+			log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
+			return false;
+		}
+
+		if (length > MAX_ICON_LENGTH) {
+			log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
+			return false;
+		}
+
+		return true;
+	}
+}

src/main/java/com/commafeed/backend/favicon/DefaultFaviconFetcher.java

@@ -0,0 +1,123 @@
+package com.commafeed.backend.favicon;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import org.apache.commons.lang.StringUtils;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.select.Elements;
+
+import com.commafeed.backend.HttpGetter;
+import com.commafeed.backend.HttpGetter.HttpResult;
+import com.commafeed.backend.feed.FeedUtils;
+import com.commafeed.backend.model.Feed;
+
+/**
+ * Inspired/Ported from https://github.com/potatolondon/getfavicon
+ * 
+ */
+@Slf4j
+@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
+@Singleton
+public class DefaultFaviconFetcher extends AbstractFaviconFetcher {
+
+	private final HttpGetter getter;
+
+	@Override
+	public byte[] fetch(Feed feed) {
+		String url = feed.getLink() != null ? feed.getLink() : feed.getUrl();
+
+		if (url == null) {
+			log.debug("url is null");
+			return null;
+		}
+
+		int doubleSlash = url.indexOf("//");
+		if (doubleSlash == -1) {
+			doubleSlash = 0;
+		} else {
+			doubleSlash += 2;
+		}
+		int firstSlash = url.indexOf('/', doubleSlash);
+		if (firstSlash != -1) {
+			url = url.substring(0, firstSlash);
+		}
+
+		byte[] icon = getIconAtRoot(url);
+
+		if (icon == null) {
+			icon = getIconInPage(url);
+		}
+
+		return icon;
+	}
+
+	private byte[] getIconAtRoot(String url) {
+		byte[] bytes = null;
+		String contentType = null;
+
+		try {
+			url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico";
+			log.debug("getting root icon at {}", url);
+			HttpResult result = getter.getBinary(url, TIMEOUT);
+			bytes = result.getContent();
+			contentType = result.getContentType();
+		} catch (Exception e) {
+			log.debug("Failed to retrieve iconAtRoot for url {}: ", url, e);
+		}
+
+		if (!isValidIconResponse(bytes, contentType)) {
+			bytes = null;
+		}
+		return bytes;
+	}
+
+	private byte[] getIconInPage(String url) {
+
+		Document doc = null;
+		try {
+			HttpResult result = getter.getBinary(url, TIMEOUT);
+			doc = Jsoup.parse(new String(result.getContent()), url);
+		} catch (Exception e) {
+			log.debug("Failed to retrieve page to find icon", e);
+			return null;
+		}
+
+		Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
+
+		if (icons.isEmpty()) {
+			log.debug("No icon found in page {}", url);
+			return null;
+		}
+
+		String href = icons.get(0).attr("abs:href");
+		if (StringUtils.isBlank(href)) {
+			log.debug("No icon found in page");
+			return null;
+		}
+
+		log.debug("Found unconfirmed iconInPage at {}", href);
+
+		byte[] bytes = null;
+		String contentType = null;
+		try {
+			HttpResult result = getter.getBinary(href, TIMEOUT);
+			bytes = result.getContent();
+			contentType = result.getContentType();
+		} catch (Exception e) {
+			log.debug("Failed to retrieve icon found in page {}", href, e);
+			return null;
+		}
+
+		if (!isValidIconResponse(bytes, contentType)) {
+			log.debug("Invalid icon found for {}", href);
+			return null;
+		}
+
+		return bytes;
+	}
+}

src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java

@@ -0,0 +1,90 @@
+package com.commafeed.backend.favicon;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.select.Elements;
+
+import com.commafeed.backend.HttpGetter;
+import com.commafeed.backend.HttpGetter.HttpResult;
+import com.commafeed.backend.model.Feed;
+
+@Slf4j
+@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
+@Singleton
+public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
+
+	private final HttpGetter getter;
+
+	@Override
+	public byte[] fetch(Feed feed) {
+		String url = feed.getUrl();
+
+		if (!url.toLowerCase().contains("://gdata.youtube.com/")) {
+			return null;
+		}
+
+		String userName = extractUserName(url);
+		if (userName == null) {
+			return null;
+		}
+
+		String profileUrl = "https://gdata.youtube.com/feeds/users/" + userName;
+
+		byte[] bytes = null;
+		String contentType = null;
+
+		try {
+			log.debug("Getting YouTube user's icon, {}", url);
+
+			// initial get to translate username to obscure user thumbnail URL
+			HttpResult profileResult = getter.getBinary(profileUrl, TIMEOUT);
+			Document doc = Jsoup.parse(new String(profileResult.getContent()), profileUrl);
+
+			Elements thumbnails = doc.select("media|thumbnail");
+			if (thumbnails.isEmpty()) {
+				return null;
+			}
+
+			String thumbnailUrl = thumbnails.get(0).attr("abs:url");
+
+			int thumbnailStart = thumbnailUrl.indexOf("<media:thumbnail url='");
+			int thumbnailEnd = thumbnailUrl.indexOf("'/>", thumbnailStart);
+			if (thumbnailStart != -1) {
+				thumbnailUrl = thumbnailUrl.substring(thumbnailStart + "<media:thumbnail url='".length(), thumbnailEnd);
+			}
+
+			// final get to actually retrieve the thumbnail
+			HttpResult iconResult = getter.getBinary(thumbnailUrl, TIMEOUT);
+			bytes = iconResult.getContent();
+			contentType = iconResult.getContentType();
+		} catch (Exception e) {
+			log.debug("Failed to retrieve YouTube icon", e);
+		}
+
+		if (!isValidIconResponse(bytes, contentType)) {
+			bytes = null;
+		}
+		return bytes;
+	}
+
+	private String extractUserName(String url) {
+		int apiOrBase = url.indexOf("/users/");
+		if (apiOrBase == -1) {
+			return null;
+		}
+
+		int userEndSlash = url.indexOf('/', apiOrBase + "/users/".length());
+		if (userEndSlash == -1) {
+			return null;
+		}
+
+		return url.substring(apiOrBase + "/users/".length(), userEndSlash);
+	}
+
+}

src/main/java/com/commafeed/backend/feed/FaviconFetcher.java

@@ -1,216 +0,0 @@
-package com.commafeed.backend.feed;
-
-import java.util.Arrays;
-import java.util.List;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-
-import lombok.RequiredArgsConstructor;
-import lombok.extern.slf4j.Slf4j;
-
-import org.apache.commons.lang.StringUtils;
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.select.Elements;
-
-import com.commafeed.backend.HttpGetter;
-import com.commafeed.backend.HttpGetter.HttpResult;
-
-/**
- * Inspired/Ported from https://github.com/potatolondon/getfavicon
- * 
- */
-@Slf4j
-@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
-@Singleton
-public class FaviconFetcher {
-
-	private static List<String> ICON_MIMETYPE_BLACKLIST = Arrays.asList("application/xml", "text/html");
-	private static long MIN_ICON_LENGTH = 100;
-	private static long MAX_ICON_LENGTH = 100000;
-	private static int TIMEOUT = 4000;
-
-	private final HttpGetter getter;
-
-	public byte[] fetch(String url) {
-
-		if (url == null) {
-			log.debug("url is null");
-			return null;
-		}
-		
-		// Get YouTube Icon here
-		if (url.toLowerCase().contains("://gdata.youtube.com/")) {
-			byte[] icon = getYouTubeIcon(url);
-			return icon;
-		}
-
-		int doubleSlash = url.indexOf("//");
-		if (doubleSlash == -1) {
-			doubleSlash = 0;
-		} else {
-			doubleSlash += 2;
-		}
-		int firstSlash = url.indexOf('/', doubleSlash);
-		if (firstSlash != -1) {
-			url = url.substring(0, firstSlash);
-		}
-
-		byte[] icon = getIconAtRoot(url);
-
-		if (icon == null) {
-			icon = getIconInPage(url);
-		}
-
-		return icon;
-	}
-
-	private byte[] getIconAtRoot(String url) {
-		byte[] bytes = null;
-		String contentType = null;
-
-		try {
-			url = FeedUtils.removeTrailingSlash(url) + "/favicon.ico";
-			log.debug("getting root icon at {}", url);
-			HttpResult result = getter.getBinary(url, TIMEOUT);
-			bytes = result.getContent();
-			contentType = result.getContentType();
-		} catch (Exception e) {
-			log.debug("Failed to retrieve iconAtRoot: " + e.getMessage(), e);
-		}
-
-		if (!isValidIconResponse(bytes, contentType)) {
-			bytes = null;
-		}
-		return bytes;
-	}
-
-	private boolean isValidIconResponse(byte[] content, String contentType) {
-		if (content == null) {
-			return false;
-		}
-
-		long length = content.length;
-
-		if (StringUtils.isNotBlank(contentType)) {
-			contentType = contentType.split(";")[0];
-		}
-
-		if (ICON_MIMETYPE_BLACKLIST.contains(contentType)) {
-			log.debug("Content-Type {} is blacklisted", contentType);
-			return false;
-		}
-
-		if (length < MIN_ICON_LENGTH) {
-			log.debug("Length {} below MIN_ICON_LENGTH {}", length, MIN_ICON_LENGTH);
-			return false;
-		}
-
-		if (length > MAX_ICON_LENGTH) {
-			log.debug("Length {} greater than MAX_ICON_LENGTH {}", length, MAX_ICON_LENGTH);
-			return false;
-		}
-
-		return true;
-	}
-
-	private byte[] getIconInPage(String url) {
-
-		Document doc = null;
-		try {
-			HttpResult result = getter.getBinary(url, TIMEOUT);
-			doc = Jsoup.parse(new String(result.getContent()), url);
-		} catch (Exception e) {
-			log.debug("Failed to retrieve page to find icon");
-			return null;
-		}
-
-		Elements icons = doc.select("link[rel~=(?i)^(shortcut|icon|shortcut icon)$]");
-
-		if (icons.isEmpty()) {
-			log.debug("No icon found in page {}", url);
-			return null;
-		}
-
-		String href = icons.get(0).attr("abs:href");
-		if (StringUtils.isBlank(href)) {
-			log.debug("No icon found in page");
-			return null;
-		}
-
-		log.debug("Found unconfirmed iconInPage at {}", href);
-
-		byte[] bytes = null;
-		String contentType = null;
-		try {
-			HttpResult result = getter.getBinary(href, TIMEOUT);
-			bytes = result.getContent();
-			contentType = result.getContentType();
-		} catch (Exception e) {
-			log.debug("Failed to retrieve icon found in page {}", href);
-			return null;
-		}
-
-		if (!isValidIconResponse(bytes, contentType)) {
-			log.debug("Invalid icon found for {}", href);
-			return null;
-		}
-
-		return bytes;
-	}
-
-	/*
-	 * Instead of grabbing the actual favicon, grab the user's icon
-	 * This prevents a whole bunch of repeated YouTube icons, replacing
-	 * each with identifiable user icons.
-	 */
-	private byte[] getYouTubeIcon(String url) {
-		byte[] bytes = null;
-		String contentType = null;
-		String username = null;
-		String imageUrl = null;
-		String thumbnailUrl = null;
-		try {
-			int apiOrBase = url.indexOf("/users/");
-			int userEndSlash = url.indexOf('/', apiOrBase + "/users/".length());
-			if (userEndSlash != -1) {
-				username = url.substring(apiOrBase + "/users/".length(), userEndSlash);
-			}
-			imageUrl = "https://gdata.youtube.com/feeds/users/" + username;
-			log.debug("Getting YouTube user's icon, {}", url);
-			
-			//initial get to translate username to obscure user thumbnail URL
-			HttpResult result = getter.getBinary(imageUrl, TIMEOUT);
-			bytes = result.getContent();
-			contentType = result.getContentType();
-			thumbnailUrl = FeedUtils.parseForImageUrl(bytes);
-
-			int thumbnailStart = thumbnailUrl.indexOf("<media:thumbnail url='");
-			int thumbnailEnd = thumbnailUrl.indexOf("'/>", thumbnailStart);
-			if (thumbnailStart != -1) {
-				thumbnailUrl = thumbnailUrl.substring(thumbnailStart+"<media:thumbnail url='".length(), thumbnailEnd);
-			}
-			
-			//final get to actually retrieve the thumbnail
-			result = getter.getBinary(thumbnailUrl, TIMEOUT);
-			bytes = result.getContent();
-			contentType = result.getContentType();
-		} catch (Exception e) {
-			log.debug("Failed to retrieve YouTubeIcon, instead retrieving default YouTube favicon: " + e.getMessage(), e);
-			return fetch("http://www.youtube.com/");
-		}
-
-		if (!isValidIconResponse(bytes, contentType)) {
-			bytes = null;
-		}
-		return bytes;
-	}
-	
-	public boolean exceptionUrl(String url) {
-		if (url.toLowerCase().contains("://gdata.youtube.com/")) {
-			return true;
-		}
-		return false;
-	}
-}

src/main/java/com/commafeed/backend/service/FeedService.java

@@ -1,23 +1,39 @@
 package com.commafeed.backend.service;
 
+import java.io.IOException;
 import java.util.Date;
+import java.util.Set;
 
 import javax.inject.Inject;
 import javax.inject.Singleton;
 
-import lombok.RequiredArgsConstructor;
-
 import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.io.IOUtils;
 
 import com.commafeed.backend.dao.FeedDAO;
+import com.commafeed.backend.favicon.AbstractFaviconFetcher;
 import com.commafeed.backend.feed.FeedUtils;
 import com.commafeed.backend.model.Feed;
 
-@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
 @Singleton
 public class FeedService {
 
 	private final FeedDAO feedDAO;
+	private final Set<AbstractFaviconFetcher> faviconFetchers;
+
+	private byte[] defaultFavicon;
+
+	@Inject
+	public FeedService(FeedDAO feedDAO, Set<AbstractFaviconFetcher> faviconFetchers) {
+		this.feedDAO = feedDAO;
+		this.faviconFetchers = faviconFetchers;
+
+		try {
+			defaultFavicon = IOUtils.toByteArray(getClass().getResource("/images/default_favicon.gif"));
+		} catch (IOException e) {
+			throw new RuntimeException("could not load default favicon", e);
+		}
+	}
 
 	public synchronized Feed findOrCreate(String url) {
 		String normalized = FeedUtils.normalizeURL(url);
@@ -33,4 +49,19 @@ public class FeedService {
 		return feed;
 	}
 
+	public byte[] fetchFavicon(Feed feed) {
+
+		byte[] icon = null;
+		for (AbstractFaviconFetcher faviconFetcher : faviconFetchers) {
+			icon = faviconFetcher.fetch(feed);
+			if (icon != null) {
+				break;
+			}
+		}
+		if (icon == null) {
+			icon = defaultFavicon;
+		}
+		return icon;
+	}
+
 }

src/main/java/com/commafeed/frontend/resource/FeedREST.java

@@ -42,7 +42,6 @@ import com.commafeed.backend.cache.CacheService;
 import com.commafeed.backend.dao.FeedCategoryDAO;
 import com.commafeed.backend.dao.FeedEntryStatusDAO;
 import com.commafeed.backend.dao.FeedSubscriptionDAO;
-import com.commafeed.backend.feed.FaviconFetcher;
 import com.commafeed.backend.feed.FeedFetcher;
 import com.commafeed.backend.feed.FeedQueues;
 import com.commafeed.backend.feed.FeedUtils;
@@ -57,6 +56,7 @@ import com.commafeed.backend.model.UserSettings.ReadingOrder;
 import com.commafeed.backend.opml.OPMLExporter;
 import com.commafeed.backend.opml.OPMLImporter;
 import com.commafeed.backend.service.FeedEntryService;
+import com.commafeed.backend.service.FeedService;
 import com.commafeed.backend.service.FeedSubscriptionService;
 import com.commafeed.frontend.auth.SecurityCheck;
 import com.commafeed.frontend.model.Entries;
@@ -95,8 +95,8 @@ public class FeedREST {
 	private final FeedSubscriptionDAO feedSubscriptionDAO;
 	private final FeedCategoryDAO feedCategoryDAO;
 	private final FeedEntryStatusDAO feedEntryStatusDAO;
-	private final FaviconFetcher faviconFetcher;
 	private final FeedFetcher feedFetcher;
+	private final FeedService feedService;
 	private final FeedEntryService feedEntryService;
 	private final FeedSubscriptionService feedSubscriptionService;
 	private final FeedQueues queues;
@@ -322,21 +322,13 @@ public class FeedREST {
 			return Response.status(Status.NOT_FOUND).build();
 		}
 		Feed feed = subscription.getFeed();
-		String url = faviconFetcher.exceptionUrl(feed.getUrl()) ? feed.getUrl() : (feed.getLink() != null ? feed.getLink() : feed.getUrl());
+		byte[] icon = feedService.fetchFavicon(feed);
-		byte[] icon = faviconFetcher.fetch(url);
 
-		ResponseBuilder builder = null;
+		ResponseBuilder builder = Response.ok(icon, "image/x-icon");
-		if (icon == null) {
-			String baseUrl = FeedUtils.removeTrailingSlash(config.getApplicationSettings().getPublicUrl());
-			builder = Response.status(Status.MOVED_PERMANENTLY).location(URI.create(baseUrl + "/images/default_favicon.gif"));
-		} else {
-			builder = Response.ok(icon, "image/x-icon");
-		}
 
 		CacheControl cacheControl = new CacheControl();
 		cacheControl.setMaxAge(2592000);
-		cacheControl.setPrivate(true);
+		cacheControl.setPrivate(false);
-		// trying to replicate "public, max-age=2592000"
 		builder.cacheControl(cacheControl);
 
 		Calendar calendar = Calendar.getInstance();