diff --git a/.openshift/config.mysql.yml b/.openshift/config.mysql.yml index e4b9cc63..c312270d 100644 --- a/.openshift/config.mysql.yml +++ b/.openshift/config.mysql.yml @@ -13,6 +13,9 @@ app: # put your google analytics tracking code here googleAnalyticsTrackingCode: + # put your google server key (used for youtube favicon fetching) + googleAuthKey: + # number of http threads backgroundThreads: 3 diff --git a/config.dev.yml b/config.dev.yml index 5330ee20..a7771204 100644 --- a/config.dev.yml +++ b/config.dev.yml @@ -13,6 +13,9 @@ app: # put your google analytics tracking code here googleAnalyticsTrackingCode: + # put your google server key (used for youtube favicon fetching) + googleAuthKey: + # number of http threads backgroundThreads: 3 diff --git a/config.yml.example b/config.yml.example index 5f6e096d..5f6d1674 100644 --- a/config.yml.example +++ b/config.yml.example @@ -13,6 +13,9 @@ app: # put your google analytics tracking code here googleAnalyticsTrackingCode: + # put your google server key (used for youtube favicon fetching) + googleAuthKey: + # number of http threads backgroundThreads: 3 diff --git a/pom.xml b/pom.xml index a28e5ff6..6619cf8f 100644 --- a/pom.xml +++ b/pom.xml @@ -353,6 +353,12 @@ 0.9.15 + + com.google.apis + google-api-services-youtube + v3-rev138-1.20.0 + + com.h2database h2 diff --git a/src/main/java/com/commafeed/CommaFeedConfiguration.java b/src/main/java/com/commafeed/CommaFeedConfiguration.java index 7b7b619f..018b991e 100644 --- a/src/main/java/com/commafeed/CommaFeedConfiguration.java +++ b/src/main/java/com/commafeed/CommaFeedConfiguration.java @@ -77,6 +77,8 @@ public class CommaFeedConfiguration extends Configuration { private String googleAnalyticsTrackingCode; + private String googleAuthKey; + @NotNull @Min(1) @Valid diff --git a/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java b/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java index 81e7d3ee..2ae17c1f 100644 --- a/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java +++ b/src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java @@ -1,18 +1,31 @@ package com.commafeed.backend.favicon; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Optional; + import javax.inject.Inject; import javax.inject.Singleton; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; -import org.jsoup.select.Elements; +import org.apache.http.NameValuePair; +import org.apache.http.client.utils.URLEncodedUtils; +import com.commafeed.CommaFeedConfiguration; import com.commafeed.backend.HttpGetter; import com.commafeed.backend.HttpGetter.HttpResult; import com.commafeed.backend.model.Feed; +import com.google.api.client.http.HttpRequest; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.youtube.YouTube; +import com.google.api.services.youtube.model.Channel; +import com.google.api.services.youtube.model.ChannelListResponse; +import com.google.api.services.youtube.model.Thumbnail; @Slf4j @RequiredArgsConstructor(onConstructor = @__({ @Inject })) @@ -20,40 +33,60 @@ import com.commafeed.backend.model.Feed; public class YoutubeFaviconFetcher extends AbstractFaviconFetcher { private final HttpGetter getter; + private final CommaFeedConfiguration config; @Override public byte[] fetch(Feed feed) { String url = feed.getUrl(); - if (!url.toLowerCase().contains("://gdata.youtube.com/")) { + if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) { return null; } - String userName = extractUserName(url); - if (userName == null) { + String googleAuthKey = config.getApplicationSettings().getGoogleAuthKey(); + if (googleAuthKey == null) { + log.debug("no google auth key configured"); return null; } - String profileUrl = "https://gdata.youtube.com/feeds/users/" + userName; - byte[] bytes = null; String contentType = null; - try { - log.debug("Getting YouTube user's icon, {}", url); - - // initial get to translate username to obscure user thumbnail URL - HttpResult profileResult = getter.getBinary(profileUrl, TIMEOUT); - Document doc = Jsoup.parse(new String(profileResult.getContent()), profileUrl); - - Elements thumbnails = doc.select("media|thumbnail"); - if (thumbnails.isEmpty()) { + List params = URLEncodedUtils.parse(url.substring(url.indexOf("?") + 1), StandardCharsets.UTF_8); + Optional userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst(); + Optional channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel")).findFirst(); + System.out.println(userId.isPresent()); + if (!userId.isPresent() && !channelId.isPresent()) { return null; } - String thumbnailUrl = thumbnails.get(0).attr("abs:url"); - // final get to actually retrieve the thumbnail - HttpResult iconResult = getter.getBinary(thumbnailUrl, TIMEOUT); + YouTube youtube = new YouTube.Builder(new NetHttpTransport(), JacksonFactory.getDefaultInstance(), + new HttpRequestInitializer() { + @Override + public void initialize(HttpRequest request) throws IOException { + } + }).setApplicationName("CommaFeed").build(); + + YouTube.Channels.List list = youtube.channels().list("snippet"); + list.setKey(googleAuthKey); + if (userId.isPresent()) { + list.setForUsername(userId.get().getValue()); + } else { + list.setId(channelId.get().getValue()); + } + + log.debug("contacting youtube api"); + ChannelListResponse response = list.execute(); + if (response.getItems().isEmpty()) { + log.debug("youtube api returned no items"); + return null; + } + + Channel channel = response.getItems().get(0); + Thumbnail thumbnail = channel.getSnippet().getThumbnails().getDefault(); + + log.debug("fetching favicon"); + HttpResult iconResult = getter.getBinary(thumbnail.getUrl(), TIMEOUT); bytes = iconResult.getContent(); contentType = iconResult.getContentType(); } catch (Exception e) { @@ -65,19 +98,4 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher { } return bytes; } - - private String extractUserName(String url) { - int apiOrBase = url.indexOf("/users/"); - if (apiOrBase == -1) { - return null; - } - - int userEndSlash = url.indexOf('/', apiOrBase + "/users/".length()); - if (userEndSlash == -1) { - return null; - } - - return url.substring(apiOrBase + "/users/".length(), userEndSlash); - } - }