garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

fix youtube favicon fetching

Browse Source
This commit is contained in:
Athou
2015-06-04 12:28:20 +02:00
parent dcd5f3d529
commit 9adc993472
6 changed files with 70 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.openshift/config.mysql.yml
View File

@@ -13,6 +13,9 @@ app:
# put your google analytics tracking code here
googleAnalyticsTrackingCode:
# put your google server key (used for youtube favicon fetching)
googleAuthKey:
# number of http threads
backgroundThreads: 3

3
config.dev.yml
View File

@@ -13,6 +13,9 @@ app:
# put your google analytics tracking code here
googleAnalyticsTrackingCode:
# put your google server key (used for youtube favicon fetching)
googleAuthKey:
# number of http threads
backgroundThreads: 3

3
config.yml.example
View File

@@ -13,6 +13,9 @@ app:
# put your google analytics tracking code here
googleAnalyticsTrackingCode:
# put your google server key (used for youtube favicon fetching)
googleAuthKey:
# number of http threads
backgroundThreads: 3

6
pom.xml
View File

@@ -353,6 +353,12 @@
<version>0.9.15</version>
</dependency>
<dependency>
<groupId>com.google.apis</groupId>
<artifactId>google-api-services-youtube</artifactId>
<version>v3-rev138-1.20.0</version>
</dependency>
<dependency>
<groupId>com.h2database</groupId>
<artifactId>h2</artifactId>

2
src/main/java/com/commafeed/CommaFeedConfiguration.java
View File

@@ -77,6 +77,8 @@ public class CommaFeedConfiguration extends Configuration {
private String googleAnalyticsTrackingCode;
private String googleAuthKey;
@NotNull
@Min(1)
@Valid

88
src/main/java/com/commafeed/backend/favicon/YoutubeFaviconFetcher.java
View File

@@ -1,18 +1,31 @@
package com.commafeed.backend.favicon;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Optional;
import javax.inject.Inject;
import javax.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.HttpGetter;
import com.commafeed.backend.HttpGetter.HttpResult;
import com.commafeed.backend.model.Feed;
import com.google.api.client.http.HttpRequest;
import com.google.api.client.http.HttpRequestInitializer;
import com.google.api.client.http.javanet.NetHttpTransport;
import com.google.api.client.json.jackson2.JacksonFactory;
import com.google.api.services.youtube.YouTube;
import com.google.api.services.youtube.model.Channel;
import com.google.api.services.youtube.model.ChannelListResponse;
import com.google.api.services.youtube.model.Thumbnail;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@@ -20,40 +33,60 @@ import com.commafeed.backend.model.Feed;
public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
private final HttpGetter getter;
private final CommaFeedConfiguration config;
@Override
public byte[] fetch(Feed feed) {
String url = feed.getUrl();
if (!url.toLowerCase().contains("://gdata.youtube.com/")) {
if (!url.toLowerCase().contains("youtube.com/feeds/videos.xml")) {
return null;
}
String userName = extractUserName(url);
if (userName == null) {
String googleAuthKey = config.getApplicationSettings().getGoogleAuthKey();
if (googleAuthKey == null) {
log.debug("no google auth key configured");
return null;
}
String profileUrl = "https://gdata.youtube.com/feeds/users/" + userName;
byte[] bytes = null;
String contentType = null;
try {
log.debug("Getting YouTube user's icon, {}", url);
// initial get to translate username to obscure user thumbnail URL
HttpResult profileResult = getter.getBinary(profileUrl, TIMEOUT);
Document doc = Jsoup.parse(new String(profileResult.getContent()), profileUrl);
Elements thumbnails = doc.select("media|thumbnail");
if (thumbnails.isEmpty()) {
List<NameValuePair> params = URLEncodedUtils.parse(url.substring(url.indexOf("?") + 1), StandardCharsets.UTF_8);
Optional<NameValuePair> userId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("user")).findFirst();
Optional<NameValuePair> channelId = params.stream().filter(nvp -> nvp.getName().equalsIgnoreCase("channel")).findFirst();
System.out.println(userId.isPresent());
if (!userId.isPresent() && !channelId.isPresent()) {
return null;
}
String thumbnailUrl = thumbnails.get(0).attr("abs:url");
// final get to actually retrieve the thumbnail
HttpResult iconResult = getter.getBinary(thumbnailUrl, TIMEOUT);
YouTube youtube = new YouTube.Builder(new NetHttpTransport(), JacksonFactory.getDefaultInstance(),
new HttpRequestInitializer() {
@Override
public void initialize(HttpRequest request) throws IOException {
}
}).setApplicationName("CommaFeed").build();
YouTube.Channels.List list = youtube.channels().list("snippet");
list.setKey(googleAuthKey);
if (userId.isPresent()) {
list.setForUsername(userId.get().getValue());
} else {
list.setId(channelId.get().getValue());
}
log.debug("contacting youtube api");
ChannelListResponse response = list.execute();
if (response.getItems().isEmpty()) {
log.debug("youtube api returned no items");
return null;
}
Channel channel = response.getItems().get(0);
Thumbnail thumbnail = channel.getSnippet().getThumbnails().getDefault();
log.debug("fetching favicon");
HttpResult iconResult = getter.getBinary(thumbnail.getUrl(), TIMEOUT);
bytes = iconResult.getContent();
contentType = iconResult.getContentType();
} catch (Exception e) {
@@ -65,19 +98,4 @@ public class YoutubeFaviconFetcher extends AbstractFaviconFetcher {
}
return bytes;
}
private String extractUserName(String url) {
int apiOrBase = url.indexOf("/users/");
if (apiOrBase == -1) {
return null;
}
int userEndSlash = url.indexOf('/', apiOrBase + "/users/".length());
if (userEndSlash == -1) {
return null;
}
return url.substring(apiOrBase + "/users/".length(), userEndSlash);
}
}