Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-24 04:57:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

extract url utils from FeedUtils

Browse Source
This commit is contained in:
Athou
2025-07-20 07:23:24 +02:00
parent 0b0a964a90
commit f894fdf564
8 changed files with 130 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedUtils.java
View File

@@ -1,8 +1,6 @@
package com.commafeed.backend.feed;
import java.net.URI;
import java.util.List;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.hc.client5.http.utils.Base64;
@@ -10,8 +8,6 @@ import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.netpreserve.urlcanon.Canonicalizer;
import org.netpreserve.urlcanon.ParsedUrl;
import com.commafeed.backend.feed.FeedEntryKeyword.Mode;
import com.commafeed.backend.feed.parser.TextDirectionDetector;
@@ -29,8 +25,6 @@ import lombok.extern.slf4j.Slf4j;
@Slf4j
public class FeedUtils {
private static final String ESCAPED_QUESTION_MARK = Pattern.quote("?");
public static String truncate(String string, int length) {
if (string != null) {
string = string.substring(0, Math.min(length, string.length()));
@@ -38,59 +32,6 @@ public class FeedUtils {
return string;
}
public static boolean isHttp(String url) {
return url.startsWith("http://");
}
public static boolean isHttps(String url) {
return url.startsWith("https://");
}
public static boolean isAbsoluteUrl(String url) {
return isHttp(url) || isHttps(url);
}
/**
* Normalize the url. The resulting url is not meant to be fetched but rather used as a mean to identify a feed and avoid duplicates
*/
public static String normalizeURL(String url) {
if (url == null) {
return null;
}
ParsedUrl parsedUrl = ParsedUrl.parseUrl(url);
Canonicalizer.AGGRESSIVE.canonicalize(parsedUrl);
String normalized = parsedUrl.toString();
if (normalized == null) {
normalized = url;
}
// convert to lower case, the url probably won't work in some cases
// after that but we don't care we just want to compare urls to avoid
// duplicates
normalized = normalized.toLowerCase();
// store all urls as http
if (normalized.startsWith("https")) {
normalized = "http" + normalized.substring(5);
}
// remove the www. part
normalized = normalized.replace("//www.", "//");
// feedproxy redirects to feedburner
normalized = normalized.replace("feedproxy.google.com", "feeds.feedburner.com");
// feedburner feeds have a special treatment
if (normalized.split(ESCAPED_QUESTION_MARK)[0].contains("feedburner.com")) {
normalized = normalized.replace("feeds2.feedburner.com", "feeds.feedburner.com");
normalized = normalized.split(ESCAPED_QUESTION_MARK)[0];
normalized = StringUtils.removeEnd(normalized, "/");
}
return normalized;
}
public static boolean isRTL(String title, String content) {
String text = StringUtils.isNotBlank(content) ? content : title;
if (StringUtils.isBlank(text)) {
@@ -105,32 +46,6 @@ public class FeedUtils {
return TextDirectionDetector.detect(stripped) == TextDirectionDetector.Direction.RIGHT_TO_LEFT;
}
public static String removeTrailingSlash(String url) {
if (url.endsWith("/")) {
url = url.substring(0, url.length() - 1);
}
return url;
}
/**
*
* @param relativeUrl
* the url of the entry
* @param feedLink
* the url of the feed as described in the feed
* @param feedUrl
* the url of the feed that we used to fetch the feed
* @return an absolute url pointing to the entry
*/
public static String toAbsoluteUrl(String relativeUrl, String feedLink, String feedUrl) {
String baseUrl = (feedLink != null && isAbsoluteUrl(feedLink)) ? feedLink : feedUrl;
if (baseUrl == null) {
return null;
}
return URI.create(baseUrl).resolve(relativeUrl).toString();
}
public static String getFaviconUrl(FeedSubscription subscription) {
return "rest/feed/favicon/" + subscription.getId();
}

8
commafeed-server/src/main/java/com/commafeed/backend/feed/parser/FeedParser.java
View File

@@ -19,7 +19,7 @@ import org.jdom2.Element;
import org.jdom2.Namespace;
import org.xml.sax.InputSource;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.Urls;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
@@ -119,7 +119,7 @@ public class FeedParser {
}
String url = buildEntryUrl(feed, feedUrl, item);
if (StringUtils.isBlank(url) && FeedUtils.isAbsoluteUrl(guid)) {
if (StringUtils.isBlank(url) && Urls.isAbsolute(guid)) {
// if link is empty but guid is used as url, use guid
url = guid;
}
@@ -165,14 +165,14 @@ public class FeedParser {
private String buildEntryUrl(SyndFeed feed, String feedUrl, SyndEntry item) {
String url = StringUtils.trimToNull(StringUtils.normalizeSpace(item.getLink()));
if (url == null || FeedUtils.isAbsoluteUrl(url)) {
if (url == null || Urls.isAbsolute(url)) {
// url is absolute, nothing to do
return url;
}
// url is relative, trying to resolve it
String feedLink = StringUtils.trimToNull(StringUtils.normalizeSpace(feed.getLink()));
return FeedUtils.toAbsoluteUrl(url, feedLink, feedUrl);
return Urls.toAbsolute(url, feedLink, feedUrl);
}
private Instant toValidInstant(Date date, boolean nullToNow) {