garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

Merge branch 'entry-filtering'

Browse Source
This commit is contained in:
Athou
2014-11-10 09:20:26 +01:00
parent 5825a16aff 281e015376
commit 4ea5ebbf9e
15 changed files with 288 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/app/i18n/en.js
View File

@@ -98,6 +98,8 @@
"next_refresh" : "Next refresh",
"queued_for_refresh" : "Queued for refresh",
"feed_url" : "Feed URL",
"filtering_expression" : "Filtering expression",
"filtering_expression_help" : "If not empty, an expression evaluating to 'true' or 'false'. If false, new entries for this feed will be marked as read automatically.\nAvailable variables are 'title', 'content', 'url' and 'author' and their content is converted to lower case for convenience.\nExample: url.contains('youtube') or (author eq 'athou' and title.contains('github').\nComplete available syntax is available <a href='http://commons.apache.org/proper/commons-jexl/reference/syntax.html' target='_blank'>here</a>.",
"generate_api_key_first" : "Generate an API key in your profile first.",
"unsubscribe" : "Unsubscribe",
"unsubscribe_confirmation" : "Are you sure you want to unsubscribe from this feed?",

10
src/main/app/js/controllers.js
View File

@@ -322,17 +322,21 @@ module.controller('FeedDetailsCtrl', ['$scope', '$state', '$stateParams', 'FeedS
$scope.save = function() {
var sub = $scope.sub;
$scope.error = null;
FeedService.modify({
id : sub.id,
name : sub.name,
position : sub.position,
categoryId : sub.categoryId
categoryId : sub.categoryId,
filter : sub.filter
}, function() {
CategoryService.init();
$state.transitionTo('feeds.view', {
_id : 'all',
_type : 'category'
});
}, function(e) {
$scope.error = e.data;
});
};
}]);
@@ -489,7 +493,7 @@ module.controller('ToolbarCtrl', [
type : $stateParams._type,
id : $stateParams._id,
olderThan : olderThan,
keywords: $location.search().q,
keywords : $location.search().q,
read : true
});
};
@@ -882,7 +886,7 @@ module.controller('FeedListCtrl', [
service.mark({
id : $scope.selectedId,
olderThan : olderThan || $scope.timestamp,
keywords: $location.search().q,
keywords : $location.search().q,
read : true
}, function() {
CategoryService.refresh(function() {

4
src/main/app/sass/generic/_misc.scss
View File

@@ -43,6 +43,10 @@ label {
display: block;
}
.pre-wrap {
white-space: pre-wrap;
}
.form-horizontal .control-group {
margin-bottom: 10px;
}

9
src/main/app/templates/feeds.feed_details.html
View File

@@ -3,6 +3,7 @@
<h3>{{ 'details.feed_details' | translate }}</h3>
</div>
<form name="form" class="form-horizontal" ng-submit="save()">
<div class="alert alert-danger" ng-if="error">{{ error }}</div>
<div class="form-group">
<label class="col-sm-2 control-label">{{ 'details.url' | translate }}</label>
<div class="col-sm-10 checkbox">
@@ -69,6 +70,14 @@
</div>
</div>
<div class="form-group">
<label class="col-sm-2 control-label">{{ 'details.filtering_expression' | translate }}</label>
<div class="col-sm-10 form-control-static">
<input type="text" name="filter" ng-model="sub.filter" class="form-control"></input>
<p class="help-block pre-wrap" ng-bind-html="'details.filtering_expression_help' | translate"></p>
</div>
</div>
<div class="form-group">
<div class="col-sm-offset-2 col-sm-10">
<button type="submit" class="btn btn-primary">{{ 'global.save' | translate }}</button>

113
src/main/java/com/commafeed/backend/feed/FeedEntryFilter.java Normal file
View File

@@ -0,0 +1,113 @@
package com.commafeed.backend.feed;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import lombok.RequiredArgsConstructor;
import org.apache.commons.jexl2.JexlContext;
import org.apache.commons.jexl2.JexlEngine;
import org.apache.commons.jexl2.JexlException;
import org.apache.commons.jexl2.JexlInfo;
import org.apache.commons.jexl2.MapContext;
import org.apache.commons.jexl2.Script;
import org.apache.commons.jexl2.introspection.JexlMethod;
import org.apache.commons.jexl2.introspection.JexlPropertyGet;
import org.apache.commons.jexl2.introspection.Uberspect;
import org.apache.commons.jexl2.introspection.UberspectImpl;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import com.commafeed.backend.model.FeedEntry;
@RequiredArgsConstructor
public class FeedEntryFilter {
private static final JexlEngine ENGINE = initEngine();
private static JexlEngine initEngine() {
// classloader that prevents object creation
ClassLoader cl = new ClassLoader() {
@Override
public Class<?> loadClass(String name) throws ClassNotFoundException {
return null;
}
@Override
protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
return null;
}
};
// uberspect that prevents access to .class and .getClass()
Uberspect uberspect = new UberspectImpl(LogFactory.getLog(JexlEngine.class)) {
@Override
public JexlPropertyGet getPropertyGet(Object obj, Object identifier, JexlInfo info) {
if ("class".equals(identifier)) {
return null;
}
return super.getPropertyGet(obj, identifier, info);
}
@Override
public JexlMethod getMethod(Object obj, String method, Object[] args, JexlInfo info) {
if ("getClass".equals(method)) {
return null;
}
return super.getMethod(obj, method, args, info);
}
};
JexlEngine engine = new JexlEngine(uberspect, null, null, null);
engine.setStrict(true);
engine.setClassLoader(cl);
return engine;
}
private final String filter;
public boolean matchesEntry(FeedEntry entry) throws FeedEntryFilterException {
if (StringUtils.isBlank(filter)) {
return true;
}
Script script = null;
try {
script = ENGINE.createScript(filter);
} catch (JexlException e) {
throw new FeedEntryFilterException("Exception while parsing expression " + filter, e);
}
JexlContext context = new MapContext();
context.set("title", Jsoup.parse(entry.getContent().getTitle()).text().toLowerCase());
context.set("author", entry.getContent().getAuthor().toLowerCase());
context.set("content", Jsoup.parse(entry.getContent().getContent()).text().toLowerCase());
context.set("url", entry.getUrl().toLowerCase());
Callable<Object> callable = script.callable(context);
Future<Object> future = Executors.newFixedThreadPool(1).submit(callable);
Object result = null;
try {
result = future.get(500, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
throw new FeedEntryFilterException("interrupted while evaluating expression " + filter, e);
} catch (ExecutionException e) {
throw new FeedEntryFilterException("Exception while evaluating expression " + filter, e);
} catch (TimeoutException e) {
throw new FeedEntryFilterException("Took too long evaluating expression " + filter, e);
}
return (boolean) result;
}
@SuppressWarnings("serial")
public static class FeedEntryFilterException extends Exception {
public FeedEntryFilterException(String message, Throwable t) {
super(message, t);
}
}
}

2
src/main/java/com/commafeed/backend/feed/FeedRefreshUpdater.java
View File

@@ -193,7 +193,7 @@ public class FeedRefreshUpdater implements Managed {
boolean inserted = new UnitOfWork<Boolean>(sessionFactory) {
@Override
protected Boolean runInSession() throws Exception {
return feedUpdateService.addEntry(feed, entry);
return feedUpdateService.addEntry(feed, entry, subscriptions);
}
}.run();
if (inserted) {

3
src/main/java/com/commafeed/backend/model/FeedSubscription.java
View File

@@ -40,4 +40,7 @@ public class FeedSubscription extends AbstractModel {
private Integer position;
@Column(length = 4096)
private String filter;
}

29
src/main/java/com/commafeed/backend/service/FeedUpdateService.java
View File

@@ -1,30 +1,39 @@
package com.commafeed.backend.service;
import java.util.Date;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Singleton;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.feed.FeedEntryFilter;
import com.commafeed.backend.feed.FeedEntryFilter.FeedEntryFilterException;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
@Slf4j
@RequiredArgsConstructor(onConstructor = @__({ @Inject }))
@Singleton
public class FeedUpdateService {
private final FeedEntryDAO feedEntryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
/**
* this is NOT thread-safe
*/
public boolean addEntry(Feed feed, FeedEntry entry) {
public boolean addEntry(Feed feed, FeedEntry entry, List<FeedSubscription> subscriptions) {
Long existing = feedEntryDAO.findExisting(entry.getGuid(), feed);
if (existing != null) {
@@ -36,8 +45,24 @@ public class FeedUpdateService {
entry.setContent(content);
entry.setInserted(new Date());
entry.setFeed(feed);
feedEntryDAO.saveOrUpdate(entry);
// if filter does not match the entry, mark it as read
for (FeedSubscription sub : subscriptions) {
FeedEntryFilter filter = new FeedEntryFilter(sub.getFilter());
boolean matches = true;
try {
matches = filter.matchesEntry(entry);
} catch (FeedEntryFilterException e) {
log.error("could not evaluate filter {}", sub.getFilter(), e);
}
if (!matches) {
FeedEntryStatus status = new FeedEntryStatus(sub.getUser(), sub, entry);
status.setRead(true);
feedEntryStatusDAO.saveOrUpdate(status);
}
}
return true;
}
}

4
src/main/java/com/commafeed/frontend/model/Subscription.java
View File

@@ -35,6 +35,7 @@ public class Subscription implements Serializable {
sub.setUnread(unreadCount.getUnreadCount());
sub.setNewestItemTime(unreadCount.getNewestItemTime());
sub.setCategoryId(category == null ? null : String.valueOf(category.getId()));
sub.setFilter(subscription.getFilter());
return sub;
}
@@ -77,4 +78,7 @@ public class Subscription implements Serializable {
@ApiModelProperty("date of the newest item")
private Date newestItemTime;
@ApiModelProperty(value = "JEXL string evaluated on new entries to mark them as read if they do not match")
private String filter;
}

3
src/main/java/com/commafeed/frontend/model/request/FeedModificationRequest.java
View File

@@ -24,4 +24,7 @@ public class FeedModificationRequest implements Serializable {
@ApiModelProperty(value = "new display position, null if not changed")
private Integer position;
@ApiModelProperty(value = "JEXL string evaluated on new entries to mark them as read if they do not match")
private String filter;
}

26
src/main/java/com/commafeed/frontend/resource/FeedREST.java
View File

@@ -44,6 +44,8 @@ import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.feed.FeedEntryFilter;
import com.commafeed.backend.feed.FeedEntryFilter.FeedEntryFilterException;
import com.commafeed.backend.feed.FeedEntryKeyword;
import com.commafeed.backend.feed.FeedFetcher;
import com.commafeed.backend.feed.FeedQueues;
@@ -51,6 +53,8 @@ import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.feed.FetchedFeed;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
import com.commafeed.backend.model.FeedEntryStatus;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
@@ -94,6 +98,20 @@ import com.wordnik.swagger.annotations.ApiParam;
@Singleton
public class FeedREST {
private static final FeedEntry TEST_ENTRY = initTestEntry();
private static FeedEntry initTestEntry() {
FeedEntry entry = new FeedEntry();
entry.setUrl("https://github.com/Athou/commafeed");
FeedEntryContent content = new FeedEntryContent();
content.setAuthor("Athou");
content.setTitle("Merge pull request #662 from Athou/dw8");
content.setContent("Merge pull request #662 from Athou/dw8");
entry.setContent(content);
return entry;
}
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedCategoryDAO feedCategoryDAO;
private final FeedEntryStatusDAO feedEntryStatusDAO;
@@ -421,7 +439,15 @@ public class FeedREST {
Preconditions.checkNotNull(req);
Preconditions.checkNotNull(req.getId());
try {
new FeedEntryFilter(req.getFilter()).matchesEntry(TEST_ENTRY);
} catch (FeedEntryFilterException e) {
Throwable root = Throwables.getRootCause(e);
return Response.status(Status.BAD_REQUEST).entity(root.getMessage()).build();
}
FeedSubscription subscription = feedSubscriptionDAO.findById(user, req.getId());
subscription.setFilter(req.getFilter());
if (StringUtils.isNotBlank(req.getName())) {
subscription.setTitle(req.getName());

11
src/main/resources/changelogs/db.changelog-2.1.xml Normal file
View File

@@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.1.xsd">
<changeSet id="add-sub-filter" author="athou">
<addColumn tableName="FEEDSUBSCRIPTIONS">
<column name="filter" type="varchar(4096)" />
</addColumn>
</changeSet>
</databaseChangeLog>

1
src/main/resources/migrations.xml
View File

@@ -9,5 +9,6 @@
<include file="changelogs/db.changelog-1.3.xml" />
<include file="changelogs/db.changelog-1.4.xml" />
<include file="changelogs/db.changelog-1.5.xml" />
<include file="changelogs/db.changelog-2.1.xml" />
</databaseChangeLog>

71
src/test/java/com/commafeed/backend/feed/FeedEntryFilterTest.java Normal file
View File

@@ -0,0 +1,71 @@
package com.commafeed.backend.feed;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import com.commafeed.backend.feed.FeedEntryFilter.FeedEntryFilterException;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedEntryContent;
public class FeedEntryFilterTest {
private FeedEntry entry;
private FeedEntryContent content;
@Before
public void init() {
entry = new FeedEntry();
entry.setUrl("https://github.com/Athou/commafeed");
content = new FeedEntryContent();
content.setAuthor("Athou");
content.setTitle("Merge pull request #662 from Athou/dw8");
content.setContent("Merge pull request #662 from Athou/dw8");
entry.setContent(content);
}
@Test
public void emptyFilterMatchesFilter() throws FeedEntryFilterException {
FeedEntryFilter filter = new FeedEntryFilter(null);
Assert.assertTrue(filter.matchesEntry(entry));
}
@Test
public void blankFilterMatchesFilter() throws FeedEntryFilterException {
FeedEntryFilter filter = new FeedEntryFilter("");
Assert.assertTrue(filter.matchesEntry(entry));
}
@Test
public void simpleExpression() throws FeedEntryFilterException {
FeedEntryFilter filter = new FeedEntryFilter("author eq 'athou'");
Assert.assertTrue(filter.matchesEntry(entry));
}
@Test(expected = FeedEntryFilterException.class)
public void newIsDisabled() throws FeedEntryFilterException {
FeedEntryFilter filter = new FeedEntryFilter("null eq new ('java.lang.String', 'athou')");
filter.matchesEntry(entry);
}
@Test(expected = FeedEntryFilterException.class)
public void getClassMethodIsDisabled() throws FeedEntryFilterException {
FeedEntryFilter filter = new FeedEntryFilter("null eq ''.getClass()");
filter.matchesEntry(entry);
}
@Test
public void dotClassIsDisabled() throws FeedEntryFilterException {
FeedEntryFilter filter = new FeedEntryFilter("null eq ''.class");
Assert.assertTrue(filter.matchesEntry(entry));
}
@Test(expected = FeedEntryFilterException.class)
public void cannotLoopForever() throws FeedEntryFilterException {
FeedEntryFilter filter = new FeedEntryFilter("while(true) {}");
filter.matchesEntry(entry);
}
}