mirror of
https://github.com/Athou/commafeed.git
synced 2026-11-20 10:43:37 +00:00
Merge pull request #2390 from jump-oss/fix/feed-discovery-redirect
Fix feed discovery after HTML redirects
This commit is contained in:
@@ -66,7 +66,10 @@ public class FeedFetcher {
|
||||
} catch (FeedParsingException e) {
|
||||
if (extractFeedUrlFromHtml) {
|
||||
String extractedUrl =
|
||||
extractFeedUrl(urlProviders, feedUrl, new String(result.content(), StandardCharsets.UTF_8));
|
||||
extractFeedUrl(
|
||||
urlProviders,
|
||||
result.urlAfterRedirect(),
|
||||
new String(result.content(), StandardCharsets.UTF_8));
|
||||
if (StringUtils.isNotBlank(extractedUrl)) {
|
||||
feedUrl = extractedUrl;
|
||||
|
||||
|
||||
@@ -9,15 +9,19 @@ import com.commafeed.backend.feed.parser.FeedParser;
|
||||
import com.commafeed.backend.feed.parser.FeedParser.FeedParsingException;
|
||||
import com.commafeed.backend.feed.parser.FeedParserResult;
|
||||
import com.commafeed.backend.urlprovider.FeedURLProvider;
|
||||
import com.commafeed.backend.urlprovider.InPageReferenceFeedURLProvider;
|
||||
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.CsvSource;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.List;
|
||||
@@ -59,6 +63,37 @@ class FeedFetcherTest {
|
||||
Assertions.assertEquals("title", result.feed().title());
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@CsvSource({
|
||||
"https://example.com/blog/, feed.xml, https://example.com/blog/feed.xml",
|
||||
"https://other.example.com/blog/, feed.xml, https://other.example.com/blog/feed.xml",
|
||||
"https://other.example.com/blog/, /feed.xml, https://other.example.com/feed.xml",
|
||||
"https://example.com/blog/, https://feeds.example.com/rss, https://feeds.example.com/rss"
|
||||
})
|
||||
void findsUrlInRedirectedPage(String redirectedUrl, String href, String expectedFeedUrl) throws Exception {
|
||||
String originalUrl = "https://example.com/start";
|
||||
byte[] html =
|
||||
"<html><head><link type='application/rss+xml' href='%s'></head></html>"
|
||||
.formatted(href)
|
||||
.getBytes(StandardCharsets.UTF_8);
|
||||
Mockito.when(getter.get(HttpGetter.HttpRequest.builder(originalUrl).build()))
|
||||
.thenReturn(new HttpResult(html, "text/html", null, null, redirectedUrl, Duration.ZERO));
|
||||
Mockito.when(parser.parse(redirectedUrl, html)).thenThrow(new FeedParsingException("invalid feed"));
|
||||
|
||||
byte[] feed = "feed".getBytes(StandardCharsets.UTF_8);
|
||||
Mockito.when(getter.get(HttpGetter.HttpRequest.builder(expectedFeedUrl).build()))
|
||||
.thenReturn(new HttpResult(feed, "application/rss+xml", null, null, expectedFeedUrl, Duration.ZERO));
|
||||
FeedParserResult parsedFeed = new FeedParserResult("title", "link", "iconUrl", null, null, null, null);
|
||||
Mockito.when(parser.parse(expectedFeedUrl, feed)).thenReturn(parsedFeed);
|
||||
|
||||
FeedFetcher redirectedFetcher = new FeedFetcher(parser, getter, List.of(new InPageReferenceFeedURLProvider()));
|
||||
FeedFetcherResult result = redirectedFetcher.fetch(originalUrl, true, null, null, null, null);
|
||||
|
||||
Assertions.assertEquals(parsedFeed, result.feed());
|
||||
Assertions.assertEquals(expectedFeedUrl, result.urlAfterRedirect());
|
||||
Mockito.verify(getter).get(HttpGetter.HttpRequest.builder(expectedFeedUrl).build());
|
||||
}
|
||||
|
||||
@Test
|
||||
void updatesHeaderWhenContentDitNotChange() throws Exception {
|
||||
String url = "https://aaa.com";
|
||||
|
||||
Reference in New Issue
Block a user