store normalized urls

This commit is contained in:
Athou
2013-07-02 14:33:53 +02:00
parent d8a9022c97
commit f230ad74b1
6 changed files with 67 additions and 1 deletions

View File

@@ -87,11 +87,25 @@ public class FeedUtils {
return encoding;
}
/**
* Normalize the url. The resulting url is not meant to be used but rather
* as a mean to identify a feed and avoid duplicates
*/
public static String normalizeURL(String url) {
if (url == null) {
return null;
}
return URLCanonicalizer.getCanonicalURL(url);
String normalized = URLCanonicalizer.getCanonicalURL(url);
if (normalized == null) {
return url;
}
normalized = normalized.toLowerCase();
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
return normalized;
}
/**