diff --git a/pom.xml b/pom.xml index 9aa2ab5e..8fefd3f1 100644 --- a/pom.xml +++ b/pom.xml @@ -178,6 +178,12 @@ 1.7.7 + + org.apache.commons + commons-jexl + 2.1.1 + + com.google.inject guice diff --git a/src/main/app/i18n/en.js b/src/main/app/i18n/en.js index f5d9e90d..f7d01143 100644 --- a/src/main/app/i18n/en.js +++ b/src/main/app/i18n/en.js @@ -98,6 +98,8 @@ "next_refresh" : "Next refresh", "queued_for_refresh" : "Queued for refresh", "feed_url" : "Feed URL", + "filtering_expression" : "Filtering expression", + "filtering_expression_help" : "If not empty, an expression evaluating to 'true' or 'false'. If false, new entries for this feed will be marked as read automatically.\nAvailable variables are 'title', 'content', 'url' and 'author' and their content is converted to lower case for convenience.\nExample: url.contains('youtube') or (author eq 'athou' and title.contains('github').\nComplete available syntax is available here.", "generate_api_key_first" : "Generate an API key in your profile first.", "unsubscribe" : "Unsubscribe", "unsubscribe_confirmation" : "Are you sure you want to unsubscribe from this feed?", diff --git a/src/main/app/js/controllers.js b/src/main/app/js/controllers.js index 7ae8d904..9eb1747d 100644 --- a/src/main/app/js/controllers.js +++ b/src/main/app/js/controllers.js @@ -322,17 +322,21 @@ module.controller('FeedDetailsCtrl', ['$scope', '$state', '$stateParams', 'FeedS $scope.save = function() { var sub = $scope.sub; + $scope.error = null; FeedService.modify({ id : sub.id, name : sub.name, position : sub.position, - categoryId : sub.categoryId + categoryId : sub.categoryId, + filter : sub.filter }, function() { CategoryService.init(); $state.transitionTo('feeds.view', { _id : 'all', _type : 'category' }); + }, function(e) { + $scope.error = e.data; }); }; }]); @@ -489,7 +493,7 @@ module.controller('ToolbarCtrl', [ type : $stateParams._type, id : $stateParams._id, olderThan : olderThan, - keywords: $location.search().q, + keywords : $location.search().q, read : true }); }; @@ -882,7 +886,7 @@ module.controller('FeedListCtrl', [ service.mark({ id : $scope.selectedId, olderThan : olderThan || $scope.timestamp, - keywords: $location.search().q, + keywords : $location.search().q, read : true }, function() { CategoryService.refresh(function() { diff --git a/src/main/app/sass/generic/_misc.scss b/src/main/app/sass/generic/_misc.scss index a60f5dbe..29fcd431 100644 --- a/src/main/app/sass/generic/_misc.scss +++ b/src/main/app/sass/generic/_misc.scss @@ -43,6 +43,10 @@ label { display: block; } +.pre-wrap { + white-space: pre-wrap; +} + .form-horizontal .control-group { margin-bottom: 10px; } diff --git a/src/main/app/templates/feeds.feed_details.html b/src/main/app/templates/feeds.feed_details.html index 699b25f9..cee5d685 100644 --- a/src/main/app/templates/feeds.feed_details.html +++ b/src/main/app/templates/feeds.feed_details.html @@ -3,6 +3,7 @@

{{ 'details.feed_details' | translate }}

+
{{ error }}
@@ -69,6 +70,14 @@
+
+ +
+ +

+
+
+
diff --git a/src/main/java/com/commafeed/backend/feed/FeedEntryFilter.java b/src/main/java/com/commafeed/backend/feed/FeedEntryFilter.java new file mode 100644 index 00000000..f9ca6a5e --- /dev/null +++ b/src/main/java/com/commafeed/backend/feed/FeedEntryFilter.java @@ -0,0 +1,113 @@ +package com.commafeed.backend.feed; + +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import lombok.RequiredArgsConstructor; + +import org.apache.commons.jexl2.JexlContext; +import org.apache.commons.jexl2.JexlEngine; +import org.apache.commons.jexl2.JexlException; +import org.apache.commons.jexl2.JexlInfo; +import org.apache.commons.jexl2.MapContext; +import org.apache.commons.jexl2.Script; +import org.apache.commons.jexl2.introspection.JexlMethod; +import org.apache.commons.jexl2.introspection.JexlPropertyGet; +import org.apache.commons.jexl2.introspection.Uberspect; +import org.apache.commons.jexl2.introspection.UberspectImpl; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.logging.LogFactory; +import org.jsoup.Jsoup; + +import com.commafeed.backend.model.FeedEntry; + +@RequiredArgsConstructor +public class FeedEntryFilter { + + private static final JexlEngine ENGINE = initEngine(); + + private static JexlEngine initEngine() { + // classloader that prevents object creation + ClassLoader cl = new ClassLoader() { + @Override + public Class loadClass(String name) throws ClassNotFoundException { + return null; + } + + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + return null; + } + }; + + // uberspect that prevents access to .class and .getClass() + Uberspect uberspect = new UberspectImpl(LogFactory.getLog(JexlEngine.class)) { + @Override + public JexlPropertyGet getPropertyGet(Object obj, Object identifier, JexlInfo info) { + if ("class".equals(identifier)) { + return null; + } + return super.getPropertyGet(obj, identifier, info); + } + + @Override + public JexlMethod getMethod(Object obj, String method, Object[] args, JexlInfo info) { + if ("getClass".equals(method)) { + return null; + } + return super.getMethod(obj, method, args, info); + } + }; + + JexlEngine engine = new JexlEngine(uberspect, null, null, null); + engine.setStrict(true); + engine.setClassLoader(cl); + return engine; + } + + private final String filter; + + public boolean matchesEntry(FeedEntry entry) throws FeedEntryFilterException { + if (StringUtils.isBlank(filter)) { + return true; + } + + Script script = null; + try { + script = ENGINE.createScript(filter); + } catch (JexlException e) { + throw new FeedEntryFilterException("Exception while parsing expression " + filter, e); + } + + JexlContext context = new MapContext(); + context.set("title", Jsoup.parse(entry.getContent().getTitle()).text().toLowerCase()); + context.set("author", entry.getContent().getAuthor().toLowerCase()); + context.set("content", Jsoup.parse(entry.getContent().getContent()).text().toLowerCase()); + context.set("url", entry.getUrl().toLowerCase()); + + Callable callable = script.callable(context); + Future future = Executors.newFixedThreadPool(1).submit(callable); + Object result = null; + try { + result = future.get(500, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e); + } catch (ExecutionException e) { + throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e); + } catch (TimeoutException e) { + throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e); + } + return (boolean) result; + } + + @SuppressWarnings("serial") + public static class FeedEntryFilterException extends Exception { + public FeedEntryFilterException(String message, Throwable t) { + super(message, t); + } + } +} diff --git a/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java b/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java index b80de7c0..52beb7ca 100644 --- a/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java +++ b/src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java @@ -193,7 +193,7 @@ public class FeedRefreshUpdater implements Managed { boolean inserted = new UnitOfWork(sessionFactory) { @Override protected Boolean runInSession() throws Exception { - return feedUpdateService.addEntry(feed, entry); + return feedUpdateService.addEntry(feed, entry, subscriptions); } }.run(); if (inserted) { diff --git a/src/main/java/com/commafeed/backend/model/FeedSubscription.java b/src/main/java/com/commafeed/backend/model/FeedSubscription.java index 363665eb..92a08cec 100644 --- a/src/main/java/com/commafeed/backend/model/FeedSubscription.java +++ b/src/main/java/com/commafeed/backend/model/FeedSubscription.java @@ -40,4 +40,7 @@ public class FeedSubscription extends AbstractModel { private Integer position; + @Column(length = 4096) + private String filter; + } diff --git a/src/main/java/com/commafeed/backend/service/FeedUpdateService.java b/src/main/java/com/commafeed/backend/service/FeedUpdateService.java index e0c68e17..437268af 100644 --- a/src/main/java/com/commafeed/backend/service/FeedUpdateService.java +++ b/src/main/java/com/commafeed/backend/service/FeedUpdateService.java @@ -1,30 +1,39 @@ package com.commafeed.backend.service; import java.util.Date; +import java.util.List; import javax.inject.Inject; import javax.inject.Singleton; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import com.commafeed.backend.dao.FeedEntryDAO; +import com.commafeed.backend.dao.FeedEntryStatusDAO; +import com.commafeed.backend.feed.FeedEntryFilter; +import com.commafeed.backend.feed.FeedEntryFilter.FeedEntryFilterException; import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.FeedEntry; import com.commafeed.backend.model.FeedEntryContent; +import com.commafeed.backend.model.FeedEntryStatus; +import com.commafeed.backend.model.FeedSubscription; +@Slf4j @RequiredArgsConstructor(onConstructor = @__({ @Inject })) @Singleton public class FeedUpdateService { private final FeedEntryDAO feedEntryDAO; + private final FeedEntryStatusDAO feedEntryStatusDAO; private final FeedEntryContentService feedEntryContentService; /** * this is NOT thread-safe */ - public boolean addEntry(Feed feed, FeedEntry entry) { + public boolean addEntry(Feed feed, FeedEntry entry, List subscriptions) { Long existing = feedEntryDAO.findExisting(entry.getGuid(), feed); if (existing != null) { @@ -36,8 +45,24 @@ public class FeedUpdateService { entry.setContent(content); entry.setInserted(new Date()); entry.setFeed(feed); - feedEntryDAO.saveOrUpdate(entry); + + // if filter does not match the entry, mark it as read + for (FeedSubscription sub : subscriptions) { + FeedEntryFilter filter = new FeedEntryFilter(sub.getFilter()); + boolean matches = true; + try { + matches = filter.matchesEntry(entry); + } catch (FeedEntryFilterException e) { + log.error("could not evaluate filter {}", sub.getFilter(), e); + } + if (!matches) { + FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry); + status.setRead(true); + feedEntryStatusDAO.saveOrUpdate(status); + } + } + return true; } } diff --git a/src/main/java/com/commafeed/frontend/model/Subscription.java b/src/main/java/com/commafeed/frontend/model/Subscription.java index a395b184..99e8d387 100644 --- a/src/main/java/com/commafeed/frontend/model/Subscription.java +++ b/src/main/java/com/commafeed/frontend/model/Subscription.java @@ -35,6 +35,7 @@ public class Subscription implements Serializable { sub.setUnread(unreadCount.getUnreadCount()); sub.setNewestItemTime(unreadCount.getNewestItemTime()); sub.setCategoryId(category == null ? null : String.valueOf(category.getId())); + sub.setFilter(subscription.getFilter()); return sub; } @@ -77,4 +78,7 @@ public class Subscription implements Serializable { @ApiModelProperty("date of the newest item") private Date newestItemTime; + @ApiModelProperty(value = "JEXL string evaluated on new entries to mark them as read if they do not match") + private String filter; + } \ No newline at end of file diff --git a/src/main/java/com/commafeed/frontend/model/request/FeedModificationRequest.java b/src/main/java/com/commafeed/frontend/model/request/FeedModificationRequest.java index c8b44876..595693e4 100644 --- a/src/main/java/com/commafeed/frontend/model/request/FeedModificationRequest.java +++ b/src/main/java/com/commafeed/frontend/model/request/FeedModificationRequest.java @@ -24,4 +24,7 @@ public class FeedModificationRequest implements Serializable { @ApiModelProperty(value = "new display position, null if not changed") private Integer position; + @ApiModelProperty(value = "JEXL string evaluated on new entries to mark them as read if they do not match") + private String filter; + } diff --git a/src/main/java/com/commafeed/frontend/resource/FeedREST.java b/src/main/java/com/commafeed/frontend/resource/FeedREST.java index edf4ac8c..ccc52bcd 100644 --- a/src/main/java/com/commafeed/frontend/resource/FeedREST.java +++ b/src/main/java/com/commafeed/frontend/resource/FeedREST.java @@ -44,6 +44,8 @@ import com.commafeed.backend.cache.CacheService; import com.commafeed.backend.dao.FeedCategoryDAO; import com.commafeed.backend.dao.FeedEntryStatusDAO; import com.commafeed.backend.dao.FeedSubscriptionDAO; +import com.commafeed.backend.feed.FeedEntryFilter; +import com.commafeed.backend.feed.FeedEntryFilter.FeedEntryFilterException; import com.commafeed.backend.feed.FeedEntryKeyword; import com.commafeed.backend.feed.FeedFetcher; import com.commafeed.backend.feed.FeedQueues; @@ -51,6 +53,8 @@ import com.commafeed.backend.feed.FeedUtils; import com.commafeed.backend.feed.FetchedFeed; import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.FeedCategory; +import com.commafeed.backend.model.FeedEntry; +import com.commafeed.backend.model.FeedEntryContent; import com.commafeed.backend.model.FeedEntryStatus; import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.model.User; @@ -94,6 +98,20 @@ import com.wordnik.swagger.annotations.ApiParam; @Singleton public class FeedREST { + private static final FeedEntry TEST_ENTRY = initTestEntry(); + + private static FeedEntry initTestEntry() { + FeedEntry entry = new FeedEntry(); + entry.setUrl("https://github.com/Athou/commafeed"); + + FeedEntryContent content = new FeedEntryContent(); + content.setAuthor("Athou"); + content.setTitle("Merge pull request #662 from Athou/dw8"); + content.setContent("Merge pull request #662 from Athou/dw8"); + entry.setContent(content); + return entry; + } + private final FeedSubscriptionDAO feedSubscriptionDAO; private final FeedCategoryDAO feedCategoryDAO; private final FeedEntryStatusDAO feedEntryStatusDAO; @@ -421,7 +439,15 @@ public class FeedREST { Preconditions.checkNotNull(req); Preconditions.checkNotNull(req.getId()); + try { + new FeedEntryFilter(req.getFilter()).matchesEntry(TEST_ENTRY); + } catch (FeedEntryFilterException e) { + Throwable root = Throwables.getRootCause(e); + return Response.status(Status.BAD_REQUEST).entity(root.getMessage()).build(); + } + FeedSubscription subscription = feedSubscriptionDAO.findById(user, req.getId()); + subscription.setFilter(req.getFilter()); if (StringUtils.isNotBlank(req.getName())) { subscription.setTitle(req.getName()); diff --git a/src/main/resources/changelogs/db.changelog-2.1.xml b/src/main/resources/changelogs/db.changelog-2.1.xml new file mode 100644 index 00000000..df158560 --- /dev/null +++ b/src/main/resources/changelogs/db.changelog-2.1.xml @@ -0,0 +1,11 @@ + + + + + + + + + + diff --git a/src/main/resources/migrations.xml b/src/main/resources/migrations.xml index acc1dafa..8da3a73d 100644 --- a/src/main/resources/migrations.xml +++ b/src/main/resources/migrations.xml @@ -9,5 +9,6 @@ + \ No newline at end of file diff --git a/src/test/java/com/commafeed/backend/feed/FeedEntryFilterTest.java b/src/test/java/com/commafeed/backend/feed/FeedEntryFilterTest.java new file mode 100644 index 00000000..869710b6 --- /dev/null +++ b/src/test/java/com/commafeed/backend/feed/FeedEntryFilterTest.java @@ -0,0 +1,71 @@ +package com.commafeed.backend.feed; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import com.commafeed.backend.feed.FeedEntryFilter.FeedEntryFilterException; +import com.commafeed.backend.model.FeedEntry; +import com.commafeed.backend.model.FeedEntryContent; + +public class FeedEntryFilterTest { + + private FeedEntry entry; + private FeedEntryContent content; + + @Before + public void init() { + entry = new FeedEntry(); + entry.setUrl("https://github.com/Athou/commafeed"); + + content = new FeedEntryContent(); + content.setAuthor("Athou"); + content.setTitle("Merge pull request #662 from Athou/dw8"); + content.setContent("Merge pull request #662 from Athou/dw8"); + entry.setContent(content); + + } + + @Test + public void emptyFilterMatchesFilter() throws FeedEntryFilterException { + FeedEntryFilter filter = new FeedEntryFilter(null); + Assert.assertTrue(filter.matchesEntry(entry)); + } + + @Test + public void blankFilterMatchesFilter() throws FeedEntryFilterException { + FeedEntryFilter filter = new FeedEntryFilter(""); + Assert.assertTrue(filter.matchesEntry(entry)); + } + + @Test + public void simpleExpression() throws FeedEntryFilterException { + FeedEntryFilter filter = new FeedEntryFilter("author eq 'athou'"); + Assert.assertTrue(filter.matchesEntry(entry)); + } + + @Test(expected = FeedEntryFilterException.class) + public void newIsDisabled() throws FeedEntryFilterException { + FeedEntryFilter filter = new FeedEntryFilter("null eq new ('java.lang.String', 'athou')"); + filter.matchesEntry(entry); + } + + @Test(expected = FeedEntryFilterException.class) + public void getClassMethodIsDisabled() throws FeedEntryFilterException { + FeedEntryFilter filter = new FeedEntryFilter("null eq ''.getClass()"); + filter.matchesEntry(entry); + } + + @Test + public void dotClassIsDisabled() throws FeedEntryFilterException { + FeedEntryFilter filter = new FeedEntryFilter("null eq ''.class"); + Assert.assertTrue(filter.matchesEntry(entry)); + } + + @Test(expected = FeedEntryFilterException.class) + public void cannotLoopForever() throws FeedEntryFilterException { + FeedEntryFilter filter = new FeedEntryFilter("while(true) {}"); + filter.matchesEntry(entry); + } + +}