Archives/Athou_commafeed
1
0
Fork 1
mirror of https://github.com/Athou/commafeed.git synced 2026-09-24 04:57:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

normalize line endings

Browse Source
This commit is contained in:
Athou
2025-03-10 08:38:21 +01:00
parent ec4554c76e
commit fb7f041454
223 changed files with 18091 additions and 18093 deletions
Download Patch File Download Diff File Expand all files Collapse all files

350
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentCleaningService.java
View File

@@ -1,175 +1,175 @@
package com.commafeed.backend.service;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Document.OutputSettings;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Safelist;
import org.w3c.css.sac.CSSException;
import org.w3c.css.sac.CSSParseException;
import org.w3c.css.sac.ErrorHandler;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
import com.steadystate.css.parser.CSSOMParser;
import com.steadystate.css.parser.SACParserCSS21;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
@Singleton
public class FeedEntryContentCleaningService {
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
public String clean(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
}
return content;
}
private static Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1",
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
private String escapeIFrameCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private String escapeImgCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser(new SACParserCSS21());
parser.setErrorHandler(new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug("warning while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug("fatal error while parsing css: {}", exception.getMessage(), exception);
}
});
return parser;
}
}
package com.commafeed.backend.service;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Document.OutputSettings;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Entities.EscapeMode;
import org.jsoup.safety.Cleaner;
import org.jsoup.safety.Safelist;
import org.w3c.css.sac.CSSException;
import org.w3c.css.sac.CSSParseException;
import org.w3c.css.sac.ErrorHandler;
import org.w3c.css.sac.InputSource;
import org.w3c.dom.css.CSSStyleDeclaration;
import com.steadystate.css.parser.CSSOMParser;
import com.steadystate.css.parser.SACParserCSS21;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@RequiredArgsConstructor
@Slf4j
@Singleton
public class FeedEntryContentCleaningService {
private static final Safelist HTML_WHITELIST = buildWhiteList();
private static final List<String> ALLOWED_IFRAME_CSS_RULES = Arrays.asList("height", "width", "border");
private static final List<String> ALLOWED_IMG_CSS_RULES = Arrays.asList("display", "width", "height");
private static final char[] FORBIDDEN_CSS_RULE_CHARACTERS = new char[] { '(', ')' };
public String clean(String content, String baseUri, boolean keepTextOnly) {
if (StringUtils.isNotBlank(content)) {
baseUri = StringUtils.trimToEmpty(baseUri);
Document dirty = Jsoup.parseBodyFragment(content, baseUri);
Cleaner cleaner = new Cleaner(HTML_WHITELIST);
Document clean = cleaner.clean(dirty);
for (Element e : clean.select("iframe[style]")) {
String style = e.attr("style");
String escaped = escapeIFrameCss(style);
e.attr("style", escaped);
}
for (Element e : clean.select("img[style]")) {
String style = e.attr("style");
String escaped = escapeImgCss(style);
e.attr("style", escaped);
}
clean.outputSettings(new OutputSettings().escapeMode(EscapeMode.base).prettyPrint(false));
Element body = clean.body();
if (keepTextOnly) {
content = body.text();
} else {
content = body.html();
}
}
return content;
}
private static Safelist buildWhiteList() {
Safelist whitelist = new Safelist();
whitelist.addTags("a", "b", "blockquote", "br", "caption", "cite", "code", "col", "colgroup", "dd", "div", "dl", "dt", "em", "h1",
"h2", "h3", "h4", "h5", "h6", "i", "iframe", "img", "li", "ol", "p", "pre", "q", "small", "strike", "strong", "sub", "sup",
"table", "tbody", "td", "tfoot", "th", "thead", "tr", "u", "ul");
whitelist.addAttributes("div", "dir");
whitelist.addAttributes("pre", "dir");
whitelist.addAttributes("code", "dir");
whitelist.addAttributes("table", "dir");
whitelist.addAttributes("p", "dir");
whitelist.addAttributes("a", "href", "title");
whitelist.addAttributes("blockquote", "cite");
whitelist.addAttributes("col", "span", "width");
whitelist.addAttributes("colgroup", "span", "width");
whitelist.addAttributes("iframe", "src", "height", "width", "allowfullscreen", "frameborder", "style");
whitelist.addAttributes("img", "align", "alt", "height", "src", "title", "width", "style");
whitelist.addAttributes("ol", "start", "type");
whitelist.addAttributes("q", "cite");
whitelist.addAttributes("table", "border", "bordercolor", "summary", "width");
whitelist.addAttributes("td", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "width");
whitelist.addAttributes("th", "border", "bordercolor", "abbr", "axis", "colspan", "rowspan", "scope", "width");
whitelist.addAttributes("ul", "type");
whitelist.addProtocols("a", "href", "ftp", "http", "https", "magnet", "mailto");
whitelist.addProtocols("blockquote", "cite", "http", "https");
whitelist.addProtocols("img", "src", "http", "https");
whitelist.addProtocols("q", "cite", "http", "https");
whitelist.addEnforcedAttribute("a", "target", "_blank");
whitelist.addEnforcedAttribute("a", "rel", "noreferrer");
return whitelist;
}
private String escapeIFrameCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IFRAME_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private String escapeImgCss(String orig) {
String rule = "";
try {
List<String> rules = new ArrayList<>();
CSSStyleDeclaration decl = buildCssParser().parseStyleDeclaration(new InputSource(new StringReader(orig)));
for (int i = 0; i < decl.getLength(); i++) {
String property = decl.item(i);
String value = decl.getPropertyValue(property);
if (StringUtils.isBlank(property) || StringUtils.isBlank(value)) {
continue;
}
if (ALLOWED_IMG_CSS_RULES.contains(property) && StringUtils.containsNone(value, FORBIDDEN_CSS_RULE_CHARACTERS)) {
rules.add(property + ":" + decl.getPropertyValue(property) + ";");
}
}
rule = StringUtils.join(rules, "");
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return rule;
}
private CSSOMParser buildCssParser() {
CSSOMParser parser = new CSSOMParser(new SACParserCSS21());
parser.setErrorHandler(new ErrorHandler() {
@Override
public void warning(CSSParseException exception) throws CSSException {
log.debug("warning while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void error(CSSParseException exception) throws CSSException {
log.debug("error while parsing css: {}", exception.getMessage(), exception);
}
@Override
public void fatalError(CSSParseException exception) throws CSSException {
log.debug("fatal error while parsing css: {}", exception.getMessage(), exception);
}
});
return parser;
}
}

142
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryContentService.java
View File

@@ -1,71 +1,71 @@
package com.commafeed.backend.service;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.commafeed.backend.model.FeedEntryContent;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryContentService {
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryContentCleaningService cleaningService;
/**
* this is NOT thread-safe
*/
public FeedEntryContent findOrCreate(Content content, String baseUrl) {
FeedEntryContent entryContent = buildContent(content, baseUrl);
Optional<FeedEntryContent> existing = feedEntryContentDAO.findExisting(entryContent.getContentHash(), entryContent.getTitleHash())
.stream()
.filter(entryContent::equivalentTo)
.findFirst();
if (existing.isPresent()) {
return existing.get();
} else {
feedEntryContentDAO.saveOrUpdate(entryContent);
return entryContent;
}
}
private FeedEntryContent buildContent(Content content, String baseUrl) {
FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitleHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.title())));
entryContent.setContentHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.content())));
entryContent.setTitle(FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false));
entryContent.setAuthor(FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
entryContent.setDirection(
FeedUtils.isRTL(content.title(), content.content()) ? FeedEntryContent.Direction.rtl : FeedEntryContent.Direction.ltr);
Enclosure enclosure = content.enclosure();
if (enclosure != null) {
entryContent.setEnclosureUrl(FeedUtils.truncate(enclosure.url(), 2048));
entryContent.setEnclosureType(enclosure.type());
}
Media media = content.media();
if (media != null) {
entryContent.setMediaDescription(cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048));
entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
entryContent.setMediaThumbnailHeight(media.thumbnailHeight());
}
return entryContent;
}
}
package com.commafeed.backend.service;
import java.util.Optional;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
import com.commafeed.backend.feed.parser.FeedParserResult.Enclosure;
import com.commafeed.backend.feed.parser.FeedParserResult.Media;
import com.commafeed.backend.model.FeedEntryContent;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryContentService {
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryContentCleaningService cleaningService;
/**
* this is NOT thread-safe
*/
public FeedEntryContent findOrCreate(Content content, String baseUrl) {
FeedEntryContent entryContent = buildContent(content, baseUrl);
Optional<FeedEntryContent> existing = feedEntryContentDAO.findExisting(entryContent.getContentHash(), entryContent.getTitleHash())
.stream()
.filter(entryContent::equivalentTo)
.findFirst();
if (existing.isPresent()) {
return existing.get();
} else {
feedEntryContentDAO.saveOrUpdate(entryContent);
return entryContent;
}
}
private FeedEntryContent buildContent(Content content, String baseUrl) {
FeedEntryContent entryContent = new FeedEntryContent();
entryContent.setTitleHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.title())));
entryContent.setContentHash(Digests.sha1Hex(StringUtils.trimToEmpty(content.content())));
entryContent.setTitle(FeedUtils.truncate(cleaningService.clean(content.title(), baseUrl, true), 2048));
entryContent.setContent(cleaningService.clean(content.content(), baseUrl, false));
entryContent.setAuthor(FeedUtils.truncate(cleaningService.clean(content.author(), baseUrl, true), 128));
entryContent.setCategories(FeedUtils.truncate(content.categories(), 4096));
entryContent.setDirection(
FeedUtils.isRTL(content.title(), content.content()) ? FeedEntryContent.Direction.rtl : FeedEntryContent.Direction.ltr);
Enclosure enclosure = content.enclosure();
if (enclosure != null) {
entryContent.setEnclosureUrl(FeedUtils.truncate(enclosure.url(), 2048));
entryContent.setEnclosureType(enclosure.type());
}
Media media = content.media();
if (media != null) {
entryContent.setMediaDescription(cleaningService.clean(media.description(), baseUrl, false));
entryContent.setMediaThumbnailUrl(FeedUtils.truncate(media.thumbnailUrl(), 2048));
entryContent.setMediaThumbnailWidth(media.thumbnailWidth());
entryContent.setMediaThumbnailHeight(media.thumbnailHeight());
}
return entryContent;
}
}

248
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryFilteringService.java
View File

@@ -1,124 +1,124 @@
package com.commafeed.backend.service;
import java.time.Year;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import jakarta.inject.Singleton;
import org.apache.commons.jexl2.JexlContext;
import org.apache.commons.jexl2.JexlEngine;
import org.apache.commons.jexl2.JexlException;
import org.apache.commons.jexl2.JexlInfo;
import org.apache.commons.jexl2.MapContext;
import org.apache.commons.jexl2.Script;
import org.apache.commons.jexl2.introspection.JexlMethod;
import org.apache.commons.jexl2.introspection.JexlPropertyGet;
import org.apache.commons.jexl2.introspection.Uberspect;
import org.apache.commons.jexl2.introspection.UberspectImpl;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.model.FeedEntry;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryFilteringService {
private static final JexlEngine ENGINE = initEngine();
private final ExecutorService executor = Executors.newCachedThreadPool();
private final CommaFeedConfiguration config;
private static JexlEngine initEngine() {
// classloader that prevents object creation
ClassLoader cl = new ClassLoader() {
@Override
protected Class<?> loadClass(String name, boolean resolve) {
return null;
}
};
// uberspect that prevents access to .class and .getClass()
Uberspect uberspect = new UberspectImpl(LogFactory.getLog(JexlEngine.class)) {
@Override
public JexlPropertyGet getPropertyGet(Object obj, Object identifier, JexlInfo info) {
if ("class".equals(identifier)) {
return null;
}
return super.getPropertyGet(obj, identifier, info);
}
@Override
public JexlMethod getMethod(Object obj, String method, Object[] args, JexlInfo info) {
if ("getClass".equals(method)) {
return null;
}
return super.getMethod(obj, method, args, info);
}
};
JexlEngine engine = new JexlEngine(uberspect, null, null, null);
engine.setStrict(true);
engine.setClassLoader(cl);
return engine;
}
public boolean filterMatchesEntry(String filter, FeedEntry entry) throws FeedEntryFilterException {
if (StringUtils.isBlank(filter)) {
return true;
}
Script script;
try {
script = ENGINE.createScript(filter);
} catch (JexlException e) {
throw new FeedEntryFilterException("Exception while parsing expression " + filter, e);
}
JexlContext context = new MapContext();
context.set("title", entry.getContent().getTitle() == null ? "" : Jsoup.parse(entry.getContent().getTitle()).text().toLowerCase());
context.set("author", entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor().toLowerCase());
context.set("content",
entry.getContent().getContent() == null ? "" : Jsoup.parse(entry.getContent().getContent()).text().toLowerCase());
context.set("url", entry.getUrl() == null ? "" : entry.getUrl().toLowerCase());
context.set("categories", entry.getContent().getCategories() == null ? "" : entry.getContent().getCategories().toLowerCase());
context.set("year", Year.now().getValue());
Callable<Object> callable = script.callable(context);
Future<Object> future = executor.submit(callable);
Object result;
try {
result = future.get(config.feedRefresh().filteringExpressionEvaluationTimeout().toMillis(), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e);
} catch (ExecutionException e) {
throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}
try {
return (boolean) result;
} catch (ClassCastException e) {
throw new FeedEntryFilterException(e.getMessage(), e);
}
}
@SuppressWarnings("serial")
public static class FeedEntryFilterException extends Exception {
public FeedEntryFilterException(String message, Throwable t) {
super(message, t);
}
}
}
package com.commafeed.backend.service;
import java.time.Year;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import jakarta.inject.Singleton;
import org.apache.commons.jexl2.JexlContext;
import org.apache.commons.jexl2.JexlEngine;
import org.apache.commons.jexl2.JexlException;
import org.apache.commons.jexl2.JexlInfo;
import org.apache.commons.jexl2.MapContext;
import org.apache.commons.jexl2.Script;
import org.apache.commons.jexl2.introspection.JexlMethod;
import org.apache.commons.jexl2.introspection.JexlPropertyGet;
import org.apache.commons.jexl2.introspection.Uberspect;
import org.apache.commons.jexl2.introspection.UberspectImpl;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.model.FeedEntry;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryFilteringService {
private static final JexlEngine ENGINE = initEngine();
private final ExecutorService executor = Executors.newCachedThreadPool();
private final CommaFeedConfiguration config;
private static JexlEngine initEngine() {
// classloader that prevents object creation
ClassLoader cl = new ClassLoader() {
@Override
protected Class<?> loadClass(String name, boolean resolve) {
return null;
}
};
// uberspect that prevents access to .class and .getClass()
Uberspect uberspect = new UberspectImpl(LogFactory.getLog(JexlEngine.class)) {
@Override
public JexlPropertyGet getPropertyGet(Object obj, Object identifier, JexlInfo info) {
if ("class".equals(identifier)) {
return null;
}
return super.getPropertyGet(obj, identifier, info);
}
@Override
public JexlMethod getMethod(Object obj, String method, Object[] args, JexlInfo info) {
if ("getClass".equals(method)) {
return null;
}
return super.getMethod(obj, method, args, info);
}
};
JexlEngine engine = new JexlEngine(uberspect, null, null, null);
engine.setStrict(true);
engine.setClassLoader(cl);
return engine;
}
public boolean filterMatchesEntry(String filter, FeedEntry entry) throws FeedEntryFilterException {
if (StringUtils.isBlank(filter)) {
return true;
}
Script script;
try {
script = ENGINE.createScript(filter);
} catch (JexlException e) {
throw new FeedEntryFilterException("Exception while parsing expression " + filter, e);
}
JexlContext context = new MapContext();
context.set("title", entry.getContent().getTitle() == null ? "" : Jsoup.parse(entry.getContent().getTitle()).text().toLowerCase());
context.set("author", entry.getContent().getAuthor() == null ? "" : entry.getContent().getAuthor().toLowerCase());
context.set("content",
entry.getContent().getContent() == null ? "" : Jsoup.parse(entry.getContent().getContent()).text().toLowerCase());
context.set("url", entry.getUrl() == null ? "" : entry.getUrl().toLowerCase());
context.set("categories", entry.getContent().getCategories() == null ? "" : entry.getContent().getCategories().toLowerCase());
context.set("year", Year.now().getValue());
Callable<Object> callable = script.callable(context);
Future<Object> future = executor.submit(callable);
Object result;
try {
result = future.get(config.feedRefresh().filteringExpressionEvaluationTimeout().toMillis(), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e);
} catch (ExecutionException e) {
throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}
try {
return (boolean) result;
} catch (ClassCastException e) {
throw new FeedEntryFilterException(e.getMessage(), e);
}
}
@SuppressWarnings("serial")
public static class FeedEntryFilterException extends Exception {
public FeedEntryFilterException(String message, Throwable t) {
super(message, t);
}
}
}

262
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryService.java
View File

@@ -1,131 +1,131 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class FeedEntryService {
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
public FeedEntry find(Feed feed, Entry entry) {
String guidHash = Digests.sha1Hex(entry.guid());
return feedEntryDAO.findExisting(guidHash, feed);
}
public FeedEntry create(Feed feed, Entry entry) {
FeedEntry feedEntry = new FeedEntry();
feedEntry.setGuid(FeedUtils.truncate(entry.guid(), 2048));
feedEntry.setGuidHash(Digests.sha1Hex(entry.guid()));
feedEntry.setUrl(FeedUtils.truncate(entry.url(), 2048));
feedEntry.setPublished(entry.published());
feedEntry.setInserted(Instant.now());
feedEntry.setFeed(feed);
feedEntry.setContent(feedEntryContentService.findOrCreate(entry.content(), feed.getLink()));
feedEntryDAO.saveOrUpdate(feedEntry);
return feedEntry;
}
public boolean applyFilter(FeedSubscription sub, FeedEntry entry) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.saveOrUpdate(status);
}
return matches;
}
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, entry.getFeed());
if (sub == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
if (status.isMarkable()) {
status.setRead(read);
feedEntryStatusDAO.saveOrUpdate(status);
}
}
public void starEntry(User user, Long entryId, Long subscriptionId, boolean starred) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subscriptionId);
if (sub == null) {
return;
}
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
status.setStarred(starred);
feedEntryStatusDAO.saveOrUpdate(status);
}
public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Instant olderThan, Instant insertedBefore,
List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null,
false, null, null, null);
markList(statuses, olderThan, insertedBefore);
}
public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, -1, -1, null, false);
markList(statuses, olderThan, insertedBefore);
}
private void markList(List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statusesToMark = statuses.stream().filter(FeedEntryStatus::isMarkable).filter(s -> {
Instant entryDate = s.getEntry().getPublished();
return olderThan == null || entryDate == null || entryDate.isBefore(olderThan);
}).filter(s -> {
Instant insertedDate = s.getEntry().getInserted();
return insertedBefore == null || insertedDate == null || insertedDate.isBefore(insertedBefore);
}).toList();
statusesToMark.forEach(s -> s.setRead(true));
feedEntryStatusDAO.saveOrUpdate(statusesToMark);
}
}
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedEntryFilteringService.FeedEntryFilterException;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class FeedEntryService {
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
public FeedEntry find(Feed feed, Entry entry) {
String guidHash = Digests.sha1Hex(entry.guid());
return feedEntryDAO.findExisting(guidHash, feed);
}
public FeedEntry create(Feed feed, Entry entry) {
FeedEntry feedEntry = new FeedEntry();
feedEntry.setGuid(FeedUtils.truncate(entry.guid(), 2048));
feedEntry.setGuidHash(Digests.sha1Hex(entry.guid()));
feedEntry.setUrl(FeedUtils.truncate(entry.url(), 2048));
feedEntry.setPublished(entry.published());
feedEntry.setInserted(Instant.now());
feedEntry.setFeed(feed);
feedEntry.setContent(feedEntryContentService.findOrCreate(entry.content(), feed.getLink()));
feedEntryDAO.saveOrUpdate(feedEntry);
return feedEntry;
}
public boolean applyFilter(FeedSubscription sub, FeedEntry entry) {
boolean matches = true;
try {
matches = feedEntryFilteringService.filterMatchesEntry(sub.getFilter(), entry);
} catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.saveOrUpdate(status);
}
return matches;
}
public void markEntry(User user, Long entryId, boolean read) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, entry.getFeed());
if (sub == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
if (status.isMarkable()) {
status.setRead(read);
feedEntryStatusDAO.saveOrUpdate(status);
}
}
public void starEntry(User user, Long entryId, Long subscriptionId, boolean starred) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subscriptionId);
if (sub == null) {
return;
}
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
FeedEntryStatus status = feedEntryStatusDAO.getStatus(user, sub, entry);
status.setStarred(starred);
feedEntryStatusDAO.saveOrUpdate(status);
}
public void markSubscriptionEntries(User user, List<FeedSubscription> subscriptions, Instant olderThan, Instant insertedBefore,
List<FeedEntryKeyword> keywords) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null,
false, null, null, null);
markList(statuses, olderThan, insertedBefore);
}
public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findStarred(user, null, -1, -1, null, false);
markList(statuses, olderThan, insertedBefore);
}
private void markList(List<FeedEntryStatus> statuses, Instant olderThan, Instant insertedBefore) {
List<FeedEntryStatus> statusesToMark = statuses.stream().filter(FeedEntryStatus::isMarkable).filter(s -> {
Instant entryDate = s.getEntry().getPublished();
return olderThan == null || entryDate == null || entryDate.isBefore(olderThan);
}).filter(s -> {
Instant insertedDate = s.getEntry().getInserted();
return insertedBefore == null || insertedDate == null || insertedDate.isBefore(insertedBefore);
}).toList();
statusesToMark.forEach(s -> s.setRead(true));
feedEntryStatusDAO.saveOrUpdate(statusesToMark);
}
}

86
commafeed-server/src/main/java/com/commafeed/backend/service/FeedEntryTagService.java
View File

@@ -1,43 +1,43 @@
package com.commafeed.backend.service;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryTagDAO;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryTagService {
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO;
public void updateTags(User user, Long entryId, List<String> tagNames) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry);
Set<String> existingTagNames = existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
List<FeedEntryTag> addList = tagNames.stream()
.filter(name -> !existingTagNames.contains(name))
.map(name -> new FeedEntryTag(user, entry, name))
.toList();
List<FeedEntryTag> removeList = existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).toList();
feedEntryTagDAO.saveOrUpdate(addList);
feedEntryTagDAO.delete(removeList);
}
}
package com.commafeed.backend.service;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryTagDAO;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryTag;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class FeedEntryTagService {
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryTagDAO feedEntryTagDAO;
public void updateTags(User user, Long entryId, List<String> tagNames) {
FeedEntry entry = feedEntryDAO.findById(entryId);
if (entry == null) {
return;
}
List<FeedEntryTag> existingTags = feedEntryTagDAO.findByEntry(user, entry);
Set<String> existingTagNames = existingTags.stream().map(FeedEntryTag::getName).collect(Collectors.toSet());
List<FeedEntryTag> addList = tagNames.stream()
.filter(name -> !existingTagNames.contains(name))
.map(name -> new FeedEntryTag(user, entry, name))
.toList();
List<FeedEntryTag> removeList = existingTags.stream().filter(tag -> !tagNames.contains(tag.getName())).toList();
feedEntryTagDAO.saveOrUpdate(addList);
feedEntryTagDAO.delete(removeList);
}
}

160
commafeed-server/src/main/java/com/commafeed/backend/service/FeedService.java
View File

@@ -1,80 +1,80 @@
package com.commafeed.backend.service;
import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
import com.commafeed.backend.favicon.Favicon;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.Models;
import com.google.common.io.Resources;
import io.quarkus.arc.All;
@Singleton
public class FeedService {
private final FeedDAO feedDAO;
private final List<AbstractFaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
public FeedService(FeedDAO feedDAO, @All List<AbstractFaviconFetcher> faviconFetchers) {
this.feedDAO = feedDAO;
this.faviconFetchers = faviconFetchers;
try {
defaultFavicon = new Favicon(
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif");
} catch (IOException e) {
throw new RuntimeException("could not load default favicon", e);
}
}
public synchronized Feed findOrCreate(String url) {
String normalizedUrl = FeedUtils.normalizeURL(url);
String normalizedUrlHash = Digests.sha1Hex(normalizedUrl);
Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash);
if (feed == null) {
feed = new Feed();
feed.setUrl(url);
feed.setNormalizedUrl(normalizedUrl);
feed.setNormalizedUrlHash(normalizedUrlHash);
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
feedDAO.persist(feed);
}
return feed;
}
public void update(Feed feed) {
String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(Digests.sha1Hex(normalized));
feed.setLastUpdated(Instant.now());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.merge(feed);
}
public Favicon fetchFavicon(Feed feed) {
Favicon icon = null;
for (AbstractFaviconFetcher faviconFetcher : faviconFetchers) {
icon = faviconFetcher.fetch(feed);
if (icon != null) {
break;
}
}
if (icon == null) {
icon = defaultFavicon;
}
return icon;
}
}
package com.commafeed.backend.service;
import java.io.IOException;
import java.time.Instant;
import java.util.List;
import java.util.Objects;
import jakarta.inject.Singleton;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.favicon.AbstractFaviconFetcher;
import com.commafeed.backend.favicon.Favicon;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.Models;
import com.google.common.io.Resources;
import io.quarkus.arc.All;
@Singleton
public class FeedService {
private final FeedDAO feedDAO;
private final List<AbstractFaviconFetcher> faviconFetchers;
private final Favicon defaultFavicon;
public FeedService(FeedDAO feedDAO, @All List<AbstractFaviconFetcher> faviconFetchers) {
this.feedDAO = feedDAO;
this.faviconFetchers = faviconFetchers;
try {
defaultFavicon = new Favicon(
Resources.toByteArray(Objects.requireNonNull(getClass().getResource("/images/default_favicon.gif"))), "image/gif");
} catch (IOException e) {
throw new RuntimeException("could not load default favicon", e);
}
}
public synchronized Feed findOrCreate(String url) {
String normalizedUrl = FeedUtils.normalizeURL(url);
String normalizedUrlHash = Digests.sha1Hex(normalizedUrl);
Feed feed = feedDAO.findByUrl(normalizedUrl, normalizedUrlHash);
if (feed == null) {
feed = new Feed();
feed.setUrl(url);
feed.setNormalizedUrl(normalizedUrl);
feed.setNormalizedUrlHash(normalizedUrlHash);
feed.setDisabledUntil(Models.MINIMUM_INSTANT);
feedDAO.persist(feed);
}
return feed;
}
public void update(Feed feed) {
String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(Digests.sha1Hex(normalized));
feed.setLastUpdated(Instant.now());
feed.setEtagHeader(FeedUtils.truncate(feed.getEtagHeader(), 255));
feedDAO.merge(feed);
}
public Favicon fetchFavicon(Feed feed) {
Favicon icon = null;
for (AbstractFaviconFetcher faviconFetcher : faviconFetchers) {
icon = faviconFetcher.fetch(feed);
if (icon != null) {
break;
}
}
if (icon == null) {
icon = defaultFavicon;
}
return icon;
}
}

296
commafeed-server/src/main/java/com/commafeed/backend/service/FeedSubscriptionService.java
View File

@@ -1,148 +1,148 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.UnreadCount;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedSubscriptionService {
private final FeedDAO feedDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService;
private final FeedRefreshEngine feedRefreshEngine;
private final CommaFeedConfiguration config;
public FeedSubscriptionService(FeedDAO feedDAO, FeedEntryStatusDAO feedEntryStatusDAO, FeedSubscriptionDAO feedSubscriptionDAO,
FeedService feedService, FeedRefreshEngine feedRefreshEngine, CommaFeedConfiguration config) {
this.feedDAO = feedDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.feedService = feedService;
this.feedRefreshEngine = feedRefreshEngine;
this.config = config;
// automatically refresh new feeds after they are subscribed to
// we need to use this hook because the feed needs to have been persisted before being processed by the feed engine
feedSubscriptionDAO.onPostCommitInsert(sub -> {
Feed feed = sub.getFeed();
if (feed.getDisabledUntil() == null || feed.getDisabledUntil().isBefore(Instant.now())) {
feedRefreshEngine.refreshImmediately(feed);
}
});
}
public long subscribe(User user, String url, String title) {
return subscribe(user, url, title, null, 0);
}
public long subscribe(User user, String url, String title, FeedCategory parent) {
return subscribe(user, url, title, parent, 0);
}
public long subscribe(User user, String url, String title, FeedCategory category, int position) {
Integer maxFeedsPerUser = config.database().cleanup().maxFeedsPerUser();
if (maxFeedsPerUser > 0 && feedSubscriptionDAO.count(user) >= maxFeedsPerUser) {
String message = String.format("You cannot subscribe to more feeds on this CommaFeed instance (max %s feeds per user)",
maxFeedsPerUser);
throw new FeedSubscriptionException(message);
}
Feed feed = feedService.findOrCreate(url);
// upgrade feed to https if it was using http
if (FeedUtils.isHttp(feed.getUrl()) && FeedUtils.isHttps(url)) {
feed.setUrl(url);
feedDAO.saveOrUpdate(feed);
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
if (sub == null) {
sub = new FeedSubscription();
sub.setFeed(feed);
sub.setUser(user);
}
sub.setCategory(category);
sub.setPosition(position);
sub.setTitle(FeedUtils.truncate(title, 128));
feedSubscriptionDAO.saveOrUpdate(sub);
return sub.getId();
}
public boolean unsubscribe(User user, Long subId) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subId);
if (sub != null) {
feedSubscriptionDAO.delete(sub);
return true;
} else {
return false;
}
}
public void refreshAll(User user) throws ForceFeedRefreshTooSoonException {
Instant lastForceRefresh = user.getLastForceRefresh();
if (lastForceRefresh != null && lastForceRefresh.plus(config.feedRefresh().forceRefreshCooldownDuration()).isAfter(Instant.now())) {
throw new ForceFeedRefreshTooSoonException();
}
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
user.setLastForceRefresh(Instant.now());
}
public void refreshAllUpForRefresh(User user) {
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Instant disabledUntil = sub.getFeed().getDisabledUntil();
if (disabledUntil == null || disabledUntil.isBefore(Instant.now())) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
}
}
public Map<Long, UnreadCount> getUnreadCount(User user) {
return feedSubscriptionDAO.findAll(user)
.stream()
.collect(Collectors.toMap(FeedSubscription::getId, feedEntryStatusDAO::getUnreadCount));
}
@SuppressWarnings("serial")
public static class FeedSubscriptionException extends RuntimeException {
private FeedSubscriptionException(String msg) {
super(msg);
}
}
@SuppressWarnings("serial")
public static class ForceFeedRefreshTooSoonException extends Exception {
private ForceFeedRefreshTooSoonException() {
super();
}
}
}
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import jakarta.inject.Singleton;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedRefreshEngine;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.UnreadCount;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Singleton
public class FeedSubscriptionService {
private final FeedDAO feedDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService;
private final FeedRefreshEngine feedRefreshEngine;
private final CommaFeedConfiguration config;
public FeedSubscriptionService(FeedDAO feedDAO, FeedEntryStatusDAO feedEntryStatusDAO, FeedSubscriptionDAO feedSubscriptionDAO,
FeedService feedService, FeedRefreshEngine feedRefreshEngine, CommaFeedConfiguration config) {
this.feedDAO = feedDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.feedService = feedService;
this.feedRefreshEngine = feedRefreshEngine;
this.config = config;
// automatically refresh new feeds after they are subscribed to
// we need to use this hook because the feed needs to have been persisted before being processed by the feed engine
feedSubscriptionDAO.onPostCommitInsert(sub -> {
Feed feed = sub.getFeed();
if (feed.getDisabledUntil() == null || feed.getDisabledUntil().isBefore(Instant.now())) {
feedRefreshEngine.refreshImmediately(feed);
}
});
}
public long subscribe(User user, String url, String title) {
return subscribe(user, url, title, null, 0);
}
public long subscribe(User user, String url, String title, FeedCategory parent) {
return subscribe(user, url, title, parent, 0);
}
public long subscribe(User user, String url, String title, FeedCategory category, int position) {
Integer maxFeedsPerUser = config.database().cleanup().maxFeedsPerUser();
if (maxFeedsPerUser > 0 && feedSubscriptionDAO.count(user) >= maxFeedsPerUser) {
String message = String.format("You cannot subscribe to more feeds on this CommaFeed instance (max %s feeds per user)",
maxFeedsPerUser);
throw new FeedSubscriptionException(message);
}
Feed feed = feedService.findOrCreate(url);
// upgrade feed to https if it was using http
if (FeedUtils.isHttp(feed.getUrl()) && FeedUtils.isHttps(url)) {
feed.setUrl(url);
feedDAO.saveOrUpdate(feed);
}
FeedSubscription sub = feedSubscriptionDAO.findByFeed(user, feed);
if (sub == null) {
sub = new FeedSubscription();
sub.setFeed(feed);
sub.setUser(user);
}
sub.setCategory(category);
sub.setPosition(position);
sub.setTitle(FeedUtils.truncate(title, 128));
feedSubscriptionDAO.saveOrUpdate(sub);
return sub.getId();
}
public boolean unsubscribe(User user, Long subId) {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subId);
if (sub != null) {
feedSubscriptionDAO.delete(sub);
return true;
} else {
return false;
}
}
public void refreshAll(User user) throws ForceFeedRefreshTooSoonException {
Instant lastForceRefresh = user.getLastForceRefresh();
if (lastForceRefresh != null && lastForceRefresh.plus(config.feedRefresh().forceRefreshCooldownDuration()).isAfter(Instant.now())) {
throw new ForceFeedRefreshTooSoonException();
}
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
user.setLastForceRefresh(Instant.now());
}
public void refreshAllUpForRefresh(User user) {
List<FeedSubscription> subs = feedSubscriptionDAO.findAll(user);
for (FeedSubscription sub : subs) {
Instant disabledUntil = sub.getFeed().getDisabledUntil();
if (disabledUntil == null || disabledUntil.isBefore(Instant.now())) {
Feed feed = sub.getFeed();
feedRefreshEngine.refreshImmediately(feed);
}
}
}
public Map<Long, UnreadCount> getUnreadCount(User user) {
return feedSubscriptionDAO.findAll(user)
.stream()
.collect(Collectors.toMap(FeedSubscription::getId, feedEntryStatusDAO::getUnreadCount));
}
@SuppressWarnings("serial")
public static class FeedSubscriptionException extends RuntimeException {
private FeedSubscriptionException(String msg) {
super(msg);
}
}
@SuppressWarnings("serial")
public static class ForceFeedRefreshTooSoonException extends Exception {
private ForceFeedRefreshTooSoonException() {
super();
}
}
}

42
commafeed-server/src/main/java/com/commafeed/backend/service/MailService.java
View File

@@ -1,21 +1,21 @@
package com.commafeed.backend.service;
import jakarta.inject.Singleton;
import com.commafeed.backend.model.User;
import io.quarkus.mailer.Mail;
import io.quarkus.mailer.Mailer;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class MailService {
private final Mailer mailer;
public void sendMail(User user, String subject, String content) {
Mail mail = Mail.withHtml(user.getEmail(), "CommaFeed - " + subject, content);
mailer.send(mail);
}
}
package com.commafeed.backend.service;
import jakarta.inject.Singleton;
import com.commafeed.backend.model.User;
import io.quarkus.mailer.Mail;
import io.quarkus.mailer.Mailer;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class MailService {
private final Mailer mailer;
public void sendMail(User user, String subject, String content) {
Mail mail = Mail.withHtml(user.getEmail(), "CommaFeed - " + subject, content);
mailer.send(mail);
}
}

190
commafeed-server/src/main/java/com/commafeed/backend/service/PasswordEncryptionService.java
View File

@@ -1,95 +1,95 @@
package com.commafeed.backend.service;
import java.io.Serializable;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.spec.KeySpec;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
// taken from http://www.javacodegeeks.com/2012/05/secure-password-storage-donts-dos-and.html
@SuppressWarnings("serial")
@Slf4j
@RequiredArgsConstructor
@Singleton
public class PasswordEncryptionService implements Serializable {
public boolean authenticate(String attemptedPassword, byte[] encryptedPassword, byte[] salt) {
if (StringUtils.isBlank(attemptedPassword)) {
return false;
}
// Encrypt the clear-text password using the same salt that was used to
// encrypt the original password
byte[] encryptedAttemptedPassword = null;
try {
encryptedAttemptedPassword = getEncryptedPassword(attemptedPassword, salt);
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
if (encryptedAttemptedPassword == null) {
return false;
}
// Authentication succeeds if encrypted password that the user entered
// is equal to the stored hash
return MessageDigest.isEqual(encryptedPassword, encryptedAttemptedPassword);
}
public byte[] getEncryptedPassword(String password, byte[] salt) {
// PBKDF2 with SHA-1 as the hashing algorithm. Note that the NIST
// specifically names SHA-1 as an acceptable hashing algorithm for
// PBKDF2
String algorithm = "PBKDF2WithHmacSHA1";
// SHA-1 generates 160 bit hashes, so that's what makes sense here
int derivedKeyLength = 160;
// Pick an iteration count that works for you. The NIST recommends at
// least 1,000 iterations:
// http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf
// iOS 4.x reportedly uses 10,000:
// http://blog.crackpassword.com/2010/09/smartphone-forensics-cracking-blackberry-backup-passwords/
int iterations = 20000;
KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, derivedKeyLength);
byte[] bytes = null;
try {
SecretKeyFactory f = SecretKeyFactory.getInstance(algorithm);
SecretKey key = f.generateSecret(spec);
bytes = key.getEncoded();
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
return bytes;
}
public byte[] generateSalt() {
// VERY important to use SecureRandom instead of just Random
byte[] salt = null;
try {
SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
// Generate a 8 byte (64 bit) salt as recommended by RSA PKCS5
salt = new byte[8];
random.nextBytes(salt);
} catch (NoSuchAlgorithmException e) {
// should never happen
log.error(e.getMessage(), e);
}
return salt;
}
}
package com.commafeed.backend.service;
import java.io.Serializable;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.spec.KeySpec;
import javax.crypto.SecretKey;
import javax.crypto.SecretKeyFactory;
import javax.crypto.spec.PBEKeySpec;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
// taken from http://www.javacodegeeks.com/2012/05/secure-password-storage-donts-dos-and.html
@SuppressWarnings("serial")
@Slf4j
@RequiredArgsConstructor
@Singleton
public class PasswordEncryptionService implements Serializable {
public boolean authenticate(String attemptedPassword, byte[] encryptedPassword, byte[] salt) {
if (StringUtils.isBlank(attemptedPassword)) {
return false;
}
// Encrypt the clear-text password using the same salt that was used to
// encrypt the original password
byte[] encryptedAttemptedPassword = null;
try {
encryptedAttemptedPassword = getEncryptedPassword(attemptedPassword, salt);
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
if (encryptedAttemptedPassword == null) {
return false;
}
// Authentication succeeds if encrypted password that the user entered
// is equal to the stored hash
return MessageDigest.isEqual(encryptedPassword, encryptedAttemptedPassword);
}
public byte[] getEncryptedPassword(String password, byte[] salt) {
// PBKDF2 with SHA-1 as the hashing algorithm. Note that the NIST
// specifically names SHA-1 as an acceptable hashing algorithm for
// PBKDF2
String algorithm = "PBKDF2WithHmacSHA1";
// SHA-1 generates 160 bit hashes, so that's what makes sense here
int derivedKeyLength = 160;
// Pick an iteration count that works for you. The NIST recommends at
// least 1,000 iterations:
// http://csrc.nist.gov/publications/nistpubs/800-132/nist-sp800-132.pdf
// iOS 4.x reportedly uses 10,000:
// http://blog.crackpassword.com/2010/09/smartphone-forensics-cracking-blackberry-backup-passwords/
int iterations = 20000;
KeySpec spec = new PBEKeySpec(password.toCharArray(), salt, iterations, derivedKeyLength);
byte[] bytes = null;
try {
SecretKeyFactory f = SecretKeyFactory.getInstance(algorithm);
SecretKey key = f.generateSecret(spec);
bytes = key.getEncoded();
} catch (Exception e) {
// should never happen
log.error(e.getMessage(), e);
}
return bytes;
}
public byte[] generateSalt() {
// VERY important to use SecureRandom instead of just Random
byte[] salt = null;
try {
SecureRandom random = SecureRandom.getInstance("SHA1PRNG");
// Generate a 8 byte (64 bit) salt as recommended by RSA PKCS5
salt = new byte[8];
random.nextBytes(salt);
} catch (NoSuchAlgorithmException e) {
// should never happen
log.error(e.getMessage(), e);
}
return salt;
}
}

332
commafeed-server/src/main/java/com/commafeed/backend/service/UserService.java
View File

@@ -1,166 +1,166 @@
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.dao.UserRoleDAO;
import com.commafeed.backend.dao.UserSettingsDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.internal.PostLoginActivities;
import com.google.common.base.Preconditions;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class UserService {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UserDAO userDAO;
private final UserRoleDAO userRoleDAO;
private final UserSettingsDAO userSettingsDAO;
private final PasswordEncryptionService encryptionService;
private final CommaFeedConfiguration config;
private final PostLoginActivities postLoginActivities;
/**
* try to log in with given credentials
*/
public Optional<User> login(String nameOrEmail, String password) {
if (nameOrEmail == null || password == null) {
return Optional.empty();
}
User user = userDAO.findByName(nameOrEmail);
if (user == null) {
user = userDAO.findByEmail(nameOrEmail);
}
if (user != null && !user.isDisabled()) {
boolean authenticated = encryptionService.authenticate(password, user.getPassword(), user.getSalt());
if (authenticated) {
performPostLoginActivities(user);
return Optional.of(user);
}
}
return Optional.empty();
}
/**
* try to log in with given api key
*/
public Optional<User> login(String apiKey) {
if (apiKey == null) {
return Optional.empty();
}
User user = userDAO.findByApiKey(apiKey);
if (user != null && !user.isDisabled()) {
performPostLoginActivities(user);
return Optional.of(user);
}
return Optional.empty();
}
/**
* try to log in with given fever api key
*/
public Optional<User> login(long userId, String feverApiKey) {
if (feverApiKey == null) {
return Optional.empty();
}
User user = userDAO.findById(userId);
if (user == null || user.isDisabled() || user.getApiKey() == null) {
return Optional.empty();
}
String computedFeverApiKey = Digests.md5Hex(user.getName() + ":" + user.getApiKey());
if (!computedFeverApiKey.equals(feverApiKey)) {
return Optional.empty();
}
performPostLoginActivities(user);
return Optional.of(user);
}
/**
* should triggers after successful login
*/
public void performPostLoginActivities(User user) {
postLoginActivities.executeFor(user);
}
public User register(String name, String password, String email, Collection<Role> roles) {
return register(name, password, email, roles, false);
}
public User register(String name, String password, String email, Collection<Role> roles, boolean forceRegistration) {
if (!forceRegistration) {
Preconditions.checkState(config.users().allowRegistrations(), "Registrations are closed on this CommaFeed instance");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");
if (StringUtils.isNotBlank(email)) {
Preconditions.checkArgument(userDAO.findByEmail(email) == null, "Email already taken");
}
User user = new User();
byte[] salt = encryptionService.generateSalt();
user.setName(name);
user.setEmail(email);
user.setCreated(Instant.now());
user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.saveOrUpdate(user);
for (Role role : roles) {
userRoleDAO.saveOrUpdate(new UserRole(user, role));
}
return user;
}
public void createAdminUser() {
register(CommaFeedApplication.USERNAME_ADMIN, "admin", "admin@commafeed.com", Arrays.asList(Role.ADMIN, Role.USER), true);
}
public void createDemoUser() {
register(CommaFeedApplication.USERNAME_DEMO, "demo", "demo@commafeed.com", Collections.singletonList(Role.USER), true);
}
public void unregister(User user) {
userSettingsDAO.delete(userSettingsDAO.findByUser(user));
userRoleDAO.delete(userRoleDAO.findAll(user));
feedSubscriptionDAO.delete(feedSubscriptionDAO.findAll(user));
feedCategoryDAO.delete(feedCategoryDAO.findAll(user));
userDAO.delete(user);
}
public String generateApiKey(User user) {
byte[] key = encryptionService.getEncryptedPassword(UUID.randomUUID().toString(), user.getSalt());
return Digests.sha1Hex(key);
}
public Set<Role> getRoles(User user) {
return userRoleDAO.findRoles(user);
}
}
package com.commafeed.backend.service;
import java.time.Instant;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import jakarta.inject.Singleton;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.Digests;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.dao.UserRoleDAO;
import com.commafeed.backend.dao.UserSettingsDAO;
import com.commafeed.backend.model.User;
import com.commafeed.backend.model.UserRole;
import com.commafeed.backend.model.UserRole.Role;
import com.commafeed.backend.service.internal.PostLoginActivities;
import com.google.common.base.Preconditions;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class UserService {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final UserDAO userDAO;
private final UserRoleDAO userRoleDAO;
private final UserSettingsDAO userSettingsDAO;
private final PasswordEncryptionService encryptionService;
private final CommaFeedConfiguration config;
private final PostLoginActivities postLoginActivities;
/**
* try to log in with given credentials
*/
public Optional<User> login(String nameOrEmail, String password) {
if (nameOrEmail == null || password == null) {
return Optional.empty();
}
User user = userDAO.findByName(nameOrEmail);
if (user == null) {
user = userDAO.findByEmail(nameOrEmail);
}
if (user != null && !user.isDisabled()) {
boolean authenticated = encryptionService.authenticate(password, user.getPassword(), user.getSalt());
if (authenticated) {
performPostLoginActivities(user);
return Optional.of(user);
}
}
return Optional.empty();
}
/**
* try to log in with given api key
*/
public Optional<User> login(String apiKey) {
if (apiKey == null) {
return Optional.empty();
}
User user = userDAO.findByApiKey(apiKey);
if (user != null && !user.isDisabled()) {
performPostLoginActivities(user);
return Optional.of(user);
}
return Optional.empty();
}
/**
* try to log in with given fever api key
*/
public Optional<User> login(long userId, String feverApiKey) {
if (feverApiKey == null) {
return Optional.empty();
}
User user = userDAO.findById(userId);
if (user == null || user.isDisabled() || user.getApiKey() == null) {
return Optional.empty();
}
String computedFeverApiKey = Digests.md5Hex(user.getName() + ":" + user.getApiKey());
if (!computedFeverApiKey.equals(feverApiKey)) {
return Optional.empty();
}
performPostLoginActivities(user);
return Optional.of(user);
}
/**
* should triggers after successful login
*/
public void performPostLoginActivities(User user) {
postLoginActivities.executeFor(user);
}
public User register(String name, String password, String email, Collection<Role> roles) {
return register(name, password, email, roles, false);
}
public User register(String name, String password, String email, Collection<Role> roles, boolean forceRegistration) {
if (!forceRegistration) {
Preconditions.checkState(config.users().allowRegistrations(), "Registrations are closed on this CommaFeed instance");
}
Preconditions.checkArgument(userDAO.findByName(name) == null, "Name already taken");
if (StringUtils.isNotBlank(email)) {
Preconditions.checkArgument(userDAO.findByEmail(email) == null, "Email already taken");
}
User user = new User();
byte[] salt = encryptionService.generateSalt();
user.setName(name);
user.setEmail(email);
user.setCreated(Instant.now());
user.setSalt(salt);
user.setPassword(encryptionService.getEncryptedPassword(password, salt));
userDAO.saveOrUpdate(user);
for (Role role : roles) {
userRoleDAO.saveOrUpdate(new UserRole(user, role));
}
return user;
}
public void createAdminUser() {
register(CommaFeedApplication.USERNAME_ADMIN, "admin", "admin@commafeed.com", Arrays.asList(Role.ADMIN, Role.USER), true);
}
public void createDemoUser() {
register(CommaFeedApplication.USERNAME_DEMO, "demo", "demo@commafeed.com", Collections.singletonList(Role.USER), true);
}
public void unregister(User user) {
userSettingsDAO.delete(userSettingsDAO.findByUser(user));
userRoleDAO.delete(userRoleDAO.findAll(user));
feedSubscriptionDAO.delete(feedSubscriptionDAO.findAll(user));
feedCategoryDAO.delete(feedCategoryDAO.findAll(user));
userDAO.delete(user);
}
public String generateApiKey(User user) {
byte[] key = encryptionService.getEncryptedPassword(UUID.randomUUID().toString(), user.getSalt());
return Digests.sha1Hex(key);
}
public Set<Role> getRoles(User user) {
return userRoleDAO.findRoles(user);
}
}

266
commafeed-server/src/main/java/com/commafeed/backend/service/db/DatabaseCleaningService.java
View File

@@ -1,133 +1,133 @@
package com.commafeed.backend.service.db;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryDAO.FeedCapacity;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
/**
* Contains utility methods for cleaning the database
*
*/
@Slf4j
@Singleton
public class DatabaseCleaningService {
private final int batchSize;
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final Meter entriesDeletedMeter;
public DatabaseCleaningService(CommaFeedConfiguration config, UnitOfWork unitOfWork, FeedDAO feedDAO, FeedEntryDAO feedEntryDAO,
FeedEntryContentDAO feedEntryContentDAO, FeedEntryStatusDAO feedEntryStatusDAO, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.feedEntryDAO = feedEntryDAO;
this.feedEntryContentDAO = feedEntryContentDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.batchSize = config.database().cleanup().batchSize();
this.entriesDeletedMeter = metrics.meter(MetricRegistry.name(getClass(), "entriesDeleted"));
}
public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
int deleted;
long entriesTotal = 0;
do {
List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) {
long entriesDeleted;
do {
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesDeletedMeter.mark(entriesDeleted);
entriesTotal += entriesDeleted;
log.debug("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
}
deleted = unitOfWork.call(() -> feedDAO.delete(feedDAO.findByIds(feeds.stream().map(AbstractModel::getId).toList())));
total += deleted;
log.debug("removed {} feeds without subscriptions", total);
} while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total);
}
public void cleanContentsWithoutEntries() {
log.info("cleaning contents without entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize));
total += deleted;
log.debug("removed {} contents without entries", total);
} while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total);
}
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
log.info("cleaning entries exceeding feed capacity");
long total = 0;
while (true) {
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize));
if (feeds.isEmpty()) {
break;
}
for (final FeedCapacity feed : feeds) {
long remaining = feed.getCapacity() - maxFeedCapacity;
do {
final long rem = remaining;
int deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(batchSize, rem)));
entriesDeletedMeter.mark(deleted);
total += deleted;
remaining -= deleted;
log.debug("removed {} entries for feeds exceeding capacity", total);
} while (remaining > 0);
}
}
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
}
public void cleanEntriesOlderThan(final Instant olderThan) {
log.info("cleaning old entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize));
entriesDeletedMeter.mark(deleted);
total += deleted;
log.debug("removed {} old entries", total);
} while (deleted != 0);
log.info("cleanup done: {} old entries deleted", total);
}
public void cleanStatusesOlderThan(final Instant olderThan) {
log.info("cleaning old read statuses");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
total += deleted;
log.debug("removed {} old read statuses", total);
} while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
}
}
package com.commafeed.backend.service.db;
import java.time.Instant;
import java.util.List;
import jakarta.inject.Singleton;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryContentDAO;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryDAO.FeedCapacity;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.model.AbstractModel;
import com.commafeed.backend.model.Feed;
import lombok.extern.slf4j.Slf4j;
/**
* Contains utility methods for cleaning the database
*
*/
@Slf4j
@Singleton
public class DatabaseCleaningService {
private final int batchSize;
private final UnitOfWork unitOfWork;
private final FeedDAO feedDAO;
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryContentDAO feedEntryContentDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final Meter entriesDeletedMeter;
public DatabaseCleaningService(CommaFeedConfiguration config, UnitOfWork unitOfWork, FeedDAO feedDAO, FeedEntryDAO feedEntryDAO,
FeedEntryContentDAO feedEntryContentDAO, FeedEntryStatusDAO feedEntryStatusDAO, MetricRegistry metrics) {
this.unitOfWork = unitOfWork;
this.feedDAO = feedDAO;
this.feedEntryDAO = feedEntryDAO;
this.feedEntryContentDAO = feedEntryContentDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.batchSize = config.database().cleanup().batchSize();
this.entriesDeletedMeter = metrics.meter(MetricRegistry.name(getClass(), "entriesDeleted"));
}
public void cleanFeedsWithoutSubscriptions() {
log.info("cleaning feeds without subscriptions");
long total = 0;
int deleted;
long entriesTotal = 0;
do {
List<Feed> feeds = unitOfWork.call(() -> feedDAO.findWithoutSubscriptions(1));
for (Feed feed : feeds) {
long entriesDeleted;
do {
entriesDeleted = unitOfWork.call(() -> feedEntryDAO.delete(feed.getId(), batchSize));
entriesDeletedMeter.mark(entriesDeleted);
entriesTotal += entriesDeleted;
log.debug("removed {} entries for feeds without subscriptions", entriesTotal);
} while (entriesDeleted > 0);
}
deleted = unitOfWork.call(() -> feedDAO.delete(feedDAO.findByIds(feeds.stream().map(AbstractModel::getId).toList())));
total += deleted;
log.debug("removed {} feeds without subscriptions", total);
} while (deleted != 0);
log.info("cleanup done: {} feeds without subscriptions deleted", total);
}
public void cleanContentsWithoutEntries() {
log.info("cleaning contents without entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryContentDAO.deleteWithoutEntries(batchSize));
total += deleted;
log.debug("removed {} contents without entries", total);
} while (deleted != 0);
log.info("cleanup done: {} contents without entries deleted", total);
}
public void cleanEntriesForFeedsExceedingCapacity(final int maxFeedCapacity) {
log.info("cleaning entries exceeding feed capacity");
long total = 0;
while (true) {
List<FeedCapacity> feeds = unitOfWork.call(() -> feedEntryDAO.findFeedsExceedingCapacity(maxFeedCapacity, batchSize));
if (feeds.isEmpty()) {
break;
}
for (final FeedCapacity feed : feeds) {
long remaining = feed.getCapacity() - maxFeedCapacity;
do {
final long rem = remaining;
int deleted = unitOfWork.call(() -> feedEntryDAO.deleteOldEntries(feed.getId(), Math.min(batchSize, rem)));
entriesDeletedMeter.mark(deleted);
total += deleted;
remaining -= deleted;
log.debug("removed {} entries for feeds exceeding capacity", total);
} while (remaining > 0);
}
}
log.info("cleanup done: {} entries for feeds exceeding capacity deleted", total);
}
public void cleanEntriesOlderThan(final Instant olderThan) {
log.info("cleaning old entries");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryDAO.deleteEntriesOlderThan(olderThan, batchSize));
entriesDeletedMeter.mark(deleted);
total += deleted;
log.debug("removed {} old entries", total);
} while (deleted != 0);
log.info("cleanup done: {} old entries deleted", total);
}
public void cleanStatusesOlderThan(final Instant olderThan) {
log.info("cleaning old read statuses");
long total = 0;
long deleted;
do {
deleted = unitOfWork.call(() -> feedEntryStatusDAO.deleteOldStatuses(olderThan, batchSize));
total += deleted;
log.debug("removed {} old read statuses", total);
} while (deleted != 0);
log.info("cleanup done: {} old read statuses deleted", total);
}
}

126
commafeed-server/src/main/java/com/commafeed/backend/service/db/DatabaseStartupService.java
View File

@@ -1,63 +1,63 @@
package com.commafeed.backend.service.db;
import jakarta.inject.Singleton;
import org.kohsuke.MetaInfServices;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.service.UserService;
import liquibase.database.Database;
import liquibase.database.core.PostgresDatabase;
import liquibase.structure.DatabaseObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class DatabaseStartupService {
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserService userService;
private final CommaFeedConfiguration config;
public void populateInitialData() {
long count = unitOfWork.call(userDAO::count);
if (count == 0) {
unitOfWork.run(this::initialData);
}
}
private void initialData() {
log.info("populating database with default values");
try {
userService.createAdminUser();
if (config.users().createDemoAccount()) {
userService.createDemoUser();
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
/**
* Register a postgresql database in liquibase that doesn't escape columns, so that we can use lower case columns
*/
@MetaInfServices(Database.class)
public static class LowerCaseColumnsPostgresDatabase extends PostgresDatabase {
@Override
public String escapeObjectName(String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}
@Override
public int getPriority() {
return super.getPriority() + 1;
}
}
}
package com.commafeed.backend.service.db;
import jakarta.inject.Singleton;
import org.kohsuke.MetaInfServices;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.service.UserService;
import liquibase.database.Database;
import liquibase.database.core.PostgresDatabase;
import liquibase.structure.DatabaseObject;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RequiredArgsConstructor
@Singleton
public class DatabaseStartupService {
private final UnitOfWork unitOfWork;
private final UserDAO userDAO;
private final UserService userService;
private final CommaFeedConfiguration config;
public void populateInitialData() {
long count = unitOfWork.call(userDAO::count);
if (count == 0) {
unitOfWork.run(this::initialData);
}
}
private void initialData() {
log.info("populating database with default values");
try {
userService.createAdminUser();
if (config.users().createDemoAccount()) {
userService.createDemoUser();
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
/**
* Register a postgresql database in liquibase that doesn't escape columns, so that we can use lower case columns
*/
@MetaInfServices(Database.class)
public static class LowerCaseColumnsPostgresDatabase extends PostgresDatabase {
@Override
public String escapeObjectName(String objectName, Class<? extends DatabaseObject> objectType) {
return objectName;
}
@Override
public int getPriority() {
return super.getPriority() + 1;
}
}
}

60
commafeed-server/src/main/java/com/commafeed/backend/service/internal/PostLoginActivities.java
View File

@@ -1,30 +1,30 @@
package com.commafeed.backend.service.internal;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class PostLoginActivities {
private final UserDAO userDAO;
private final UnitOfWork unitOfWork;
public void executeFor(User user) {
// only update lastLogin every once in a while in order to avoid invalidating the cache every time someone logs in
Instant now = Instant.now();
Instant lastLogin = user.getLastLogin();
if (lastLogin == null || ChronoUnit.MINUTES.between(lastLogin, now) >= 30) {
user.setLastLogin(now);
unitOfWork.run(() -> userDAO.saveOrUpdate(user));
}
}
}
package com.commafeed.backend.service.internal;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import jakarta.inject.Singleton;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.dao.UserDAO;
import com.commafeed.backend.model.User;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
@Singleton
public class PostLoginActivities {
private final UserDAO userDAO;
private final UnitOfWork unitOfWork;
public void executeFor(User user) {
// only update lastLogin every once in a while in order to avoid invalidating the cache every time someone logs in
Instant now = Instant.now();
Instant lastLogin = user.getLastLogin();
if (lastLogin == null || ChronoUnit.MINUTES.between(lastLogin, now) >= 30) {
user.setLastLogin(now);
unitOfWork.run(() -> userDAO.saveOrUpdate(user));
}
}
}