Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-25 21:45:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

feed refresh engine now uses its own immutable model

Browse Source
This commit is contained in:
Athou
2024-01-07 10:51:35 +01:00
parent 60b6c69020
commit 7b335e2fd4
24 changed files with 623 additions and 565 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentService.java
View File

@@ -8,6 +8,7 @@ import java.util.Optional;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Document.OutputSettings;
@@ -23,6 +24,9 @@ import org.w3c.dom.css.CSSStyleDeclaration;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.commafeed.backend.model.FeedEntryContent;
import com.steadystate.css.parser.CSSOMParser;
@@ -46,26 +50,65 @@ public class FeedEntryContentService {
/**
* this is NOT thread-safe
*/
public FeedEntryContent findOrCreate(FeedEntryContent content, String baseUrl) {
content.setAuthor(FeedUtils.truncate(handleContent(content.getAuthor(), baseUrl, true), 128));
content.setTitle(FeedUtils.truncate(handleContent(content.getTitle(), baseUrl, true), 2048));
content.setContent(handleContent(content.getContent(), baseUrl, false));
content.setMediaDescription(handleContent(content.getMediaDescription(), baseUrl, false));
public FeedEntryContent findOrCreate(Content content, String baseUrl) {
String title = FeedUtils.truncate(handleContent(content.title(), baseUrl, true), 2048);
String titleHash = DigestUtils.sha1Hex(StringUtils.trimToEmpty(title));
String contentHash = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getContent()));
content.setContentHash(contentHash);
String titleHash = DigestUtils.sha1Hex(StringUtils.trimToEmpty(content.getTitle()));
content.setTitleHash(titleHash);
String contentString = handleContent(content.content(), baseUrl, false);
String contentHash = DigestUtils.sha1Hex(StringUtils.trimToEmpty(contentString));
List<FeedEntryContent> existing = feedEntryContentDAO.findExisting(contentHash, titleHash);
Optional<FeedEntryContent> equivalentContent = existing.stream().filter(content::equivalentTo).findFirst();
Optional<FeedEntryContent> equivalentContent = existing.stream()
.filter(c -> isEquivalent(c, content, title, contentString))
.findFirst();
if (equivalentContent.isPresent()) {
return equivalentContent.get();
}
feedEntryContentDAO.saveOrUpdate(content);
return content;
FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitle(title);
entryContent.setTitleHash(titleHash);
entryContent.setContent(contentString);
entryContent.setContentHash(contentHash);
entryContent.setAuthor(FeedUtils.truncate(handleContent(content.author(), baseUrl, true), 128));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
Enclosure enclosure = content.enclosure();
if (enclosure != null) {
entryContent.setEnclosureUrl(enclosure.url());
entryContent.setEnclosureType(enclosure.type());
}
Media media = content.media();
if (media != null) {
entryContent.setMediaDescription(handleContent(media.description(), baseUrl, false));
entryContent.setMediaThumbnailUrl(media.thumbnailUrl());
entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
entryContent.setMediaThumbnailHeight(media.thumbnailHeight());
}
feedEntryContentDAO.saveOrUpdate(entryContent);
return entryContent;
}
private boolean isEquivalent(FeedEntryContent content, Content c, String title, String contentString) {
EqualsBuilder builder = new EqualsBuilder().append(content.getTitle(), title)
.append(content.getContent(), contentString)
.append(content.getAuthor(), c.author())
.append(content.getCategories(), c.categories());
if (c.enclosure() != null) {
builder.append(content.getEnclosureUrl(), c.enclosure().url()).append(content.getEnclosureType(), c.enclosure().type());
}
if (c.media() != null) {
builder.append(content.getMediaDescription(), c.media().description())
.append(content.getMediaThumbnailUrl(), c.media().thumbnailUrl())
.append(content.getMediaThumbnailWidth(), c.media().thumbnailWidth())
.append(content.getMediaThumbnailHeight(), c.media().thumbnailHeight());
}
return builder.build();
}
private static Safelist buildWhiteList() {

36
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java
View File

@@ -10,9 +10,10 @@ import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
@@ -37,30 +38,27 @@ public class FeedEntryService {
/**
* this is NOT thread-safe
*/
public boolean addEntry(Feed feed, FeedEntry entry, List<FeedSubscription> subscriptions) {
Long existing = feedEntryDAO.findExisting(entry.getGuid(), feed);
public boolean addEntry(Feed feed, Entry entry, List<FeedSubscription> subscriptions) {
String guid = FeedUtils.truncate(entry.guid(), 2048);
String guidHash = DigestUtils.sha1Hex(entry.guid());
Long existing = feedEntryDAO.findExisting(guidHash, feed);
if (existing != null) {
return false;
}
FeedEntryContent content = feedEntryContentService.findOrCreate(entry.getContent(), feed.getLink());
entry.setGuidHash(DigestUtils.sha1Hex(entry.getGuid()));
entry.setContent(content);
entry.setInserted(new Date());
entry.setFeed(feed);
feedEntryDAO.saveOrUpdate(entry);
FeedEntry feedEntry = buildEntry(feed, entry, guid, guidHash);
feedEntryDAO.saveOrUpdate(feedEntry);
// if filter does not match the entry, mark it as read
for (FeedSubscription sub : subscriptions) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), feedEntry);
} catch (FeedEntryFilteringService.FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, feedEntry);
status.setRead(true);
feedEntryStatusDAO.saveOrUpdate(status);
}
@@ -69,8 +67,20 @@ public class FeedEntryService {
return true;
}
public void markEntry(User user, Long entryId, boolean read) {
private FeedEntry buildEntry(Feed feed, Entry e, String guid, String guidHash) {
FeedEntry entry = new FeedEntry();
entry.setGuid(guid);
entry.setGuidHash(guidHash);
entry.setUrl(FeedUtils.truncate(e.url(), 2048));
entry.setUpdated(e.updated());
entry.setInserted(new Date());
entry.setFeed(feed);
entry.setContent(feedEntryContentService.findOrCreate(e.content(), feed.getLink()));
return entry;
}
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;

10
commafeed-server/src/main/java/com/commafeed/backend/service/FeedService.java
View File

@@ -37,13 +37,14 @@ public class FeedService {
}
public synchronized Feed findOrCreate(String url) {
String normalized = FeedUtils.normalizeURL(url);
Feed feed = feedDAO.findByUrl(normalized);
String normalizedUrl = FeedUtils.normalizeURL(url);
String normalizedUrlHash = DigestUtils.sha1Hex(normalizedUrl);
Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash);
if (feed == null) {
feed = new Feed();
feed.setUrl(url);
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
feed.setNormalizedUrl(normalizedUrl);
feed.setNormalizedUrlHash(normalizedUrlHash);
feed.setDisabledUntil(new Date(0));
feedDAO.saveOrUpdate(feed);
}
@@ -55,6 +56,7 @@ public class FeedService {
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
feed.setLastUpdated(new Date());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.saveOrUpdate(feed);
}