Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-03-21 21:37:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

store normalized urls

Browse Source
This commit is contained in:
Athou
2013-07-02 14:33:53 +02:00
parent d8a9022c97
commit f230ad74b1
6 changed files with 67 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java
View File

@@ -12,6 +12,7 @@ import javax.annotation.PreDestroy;
import javax.enterprise.context.ApplicationScoped;
import javax.inject.Inject;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -158,6 +159,9 @@ public class FeedRefreshTaskGiver {
}
public void giveBack(Feed feed) {
String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
giveBackQueue.add(feed);
}

16
src/main/java/com/commafeed/backend/feeds/FeedUtils.java
View File

@@ -87,11 +87,25 @@ public class FeedUtils {
return encoding;
}
/**
* Normalize the url. The resulting url is not meant to be used but rather
* as a mean to identify a feed and avoid duplicates
*/
public static String normalizeURL(String url) {
if (url == null) {
return null;
}
return URLCanonicalizer.getCanonicalURL(url);
String normalized = URLCanonicalizer.getCanonicalURL(url);
if (normalized == null) {
return url;
}
normalized = normalized.toLowerCase();
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
return normalized;
}
/**