diff --git a/pom.xml b/pom.xml
index c4fc413d..7ee9af7e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,6 +19,7 @@
java:openejb/Resource/My DataSource
org.hibernate.dialect.HSQLDialect
false
+ com.commafeed.backend.cache.InMemoryCacheService
@@ -69,6 +70,16 @@
2.3
false
+
+
+ ${basedir}/src/main/webapp/WEB-INF
+ WEB-INF
+ true
+
+ **/beans.xml
+
+
+
@@ -172,6 +183,11 @@
+
+ redis.clients
+ jedis
+ 2.0.0
+
org.liquibase
liquibase-core
@@ -443,6 +459,12 @@
true
+
+ redis
+
+ com.commafeed.backend.cache.RedisCacheService
+
+
mysql
diff --git a/src/main/java/com/commafeed/backend/cache/CacheService.java b/src/main/java/com/commafeed/backend/cache/CacheService.java
index 73edccb0..aa1dc339 100644
--- a/src/main/java/com/commafeed/backend/cache/CacheService.java
+++ b/src/main/java/com/commafeed/backend/cache/CacheService.java
@@ -1,31 +1,21 @@
package com.commafeed.backend.cache;
-import java.util.concurrent.TimeUnit;
+import java.util.List;
+
+import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-public class CacheService {
+public abstract class CacheService {
- Cache entryCache = CacheBuilder.newBuilder()
- .maximumSize(1000000).expireAfterWrite(24, TimeUnit.HOURS).build();
+ public abstract List getLastEntries(Feed feed);
- private static enum Marker {
- INSTANCE
- }
+ public abstract void setLastEntries(Feed feed, List entries);
- public boolean hasFeedEntry(Feed feed, FeedEntry entry) {
- return entryCache.getIfPresent(buildKey(feed, entry)) == Marker.INSTANCE;
- }
-
- public void putFeedEntry(Feed feed, FeedEntry entry) {
- entryCache.put(buildKey(feed, entry), Marker.INSTANCE);
- }
-
- private String buildKey(Feed feed, FeedEntry entry) {
- return String.format("%s:%s:%s", feed.getId(), entry.getGuid(),
+ public String buildKey(Feed feed, FeedEntry entry) {
+ return DigestUtils.sha1Hex(entry.getGuid() +
entry.getUrl());
}
+
}
diff --git a/src/main/java/com/commafeed/backend/cache/InMemoryCacheService.java b/src/main/java/com/commafeed/backend/cache/InMemoryCacheService.java
new file mode 100644
index 00000000..a59dba3d
--- /dev/null
+++ b/src/main/java/com/commafeed/backend/cache/InMemoryCacheService.java
@@ -0,0 +1,46 @@
+package com.commafeed.backend.cache;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.PostConstruct;
+import javax.enterprise.context.ApplicationScoped;
+import javax.enterprise.inject.Alternative;
+import javax.inject.Inject;
+
+import com.commafeed.backend.model.Feed;
+import com.commafeed.backend.services.ApplicationSettingsService;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+
+@Alternative
+@ApplicationScoped
+public class InMemoryCacheService extends CacheService {
+
+ @Inject
+ ApplicationSettingsService applicationSettingsService;
+
+ private Cache> entryCache;
+
+ @PostConstruct
+ private void init() {
+ int capacity = applicationSettingsService.get().isHeavyLoad() ? 1000000 : 100;
+ entryCache = CacheBuilder.newBuilder()
+ .maximumSize(capacity).expireAfterWrite(24, TimeUnit.HOURS).build();
+ }
+
+ @Override
+ public List getLastEntries(Feed feed) {
+ List list = entryCache.getIfPresent(feed.getId());
+ if (list == null) {
+ list = Collections.emptyList();
+ }
+ return list;
+ }
+
+ @Override
+ public void setLastEntries(Feed feed, List entries) {
+ entryCache.put(feed.getId(), entries);
+ }
+}
diff --git a/src/main/java/com/commafeed/backend/cache/RedisCacheService.java b/src/main/java/com/commafeed/backend/cache/RedisCacheService.java
new file mode 100644
index 00000000..2f1a744c
--- /dev/null
+++ b/src/main/java/com/commafeed/backend/cache/RedisCacheService.java
@@ -0,0 +1,62 @@
+package com.commafeed.backend.cache;
+
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import javax.enterprise.context.ApplicationScoped;
+import javax.enterprise.inject.Alternative;
+
+import redis.clients.jedis.Jedis;
+import redis.clients.jedis.JedisPool;
+import redis.clients.jedis.JedisPoolConfig;
+import redis.clients.jedis.Pipeline;
+
+import com.commafeed.backend.model.Feed;
+import com.google.api.client.util.Lists;
+
+@Alternative
+@ApplicationScoped
+public class RedisCacheService extends CacheService {
+
+ private JedisPool pool = new JedisPool(new JedisPoolConfig(), "localhost");
+
+ @Override
+ public List getLastEntries(Feed feed) {
+ List list = Lists.newArrayList();
+ Jedis jedis = pool.getResource();
+ try {
+ String key = buildKey(feed);
+ Set members = jedis.smembers(key);
+ for (String member : members) {
+ list.add(member);
+ }
+ } finally {
+ pool.returnResource(jedis);
+ }
+ return list;
+ }
+
+ @Override
+ public void setLastEntries(Feed feed, List entries) {
+ Jedis jedis = pool.getResource();
+ try {
+ String key = buildKey(feed);
+
+ Pipeline pipe = jedis.pipelined();
+ pipe.del(key);
+ for (String entry : entries) {
+ pipe.sadd(key, entry);
+ }
+ pipe.expire(key, (int) TimeUnit.HOURS.toSeconds(24));
+ pipe.sync();
+ } finally {
+ pool.returnResource(jedis);
+ }
+ }
+
+ private String buildKey(Feed feed) {
+ return "feed:" + feed.getId();
+ }
+
+}
diff --git a/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java b/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java
index 2b40b341..d8c0b5d9 100644
--- a/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java
+++ b/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java
@@ -10,6 +10,8 @@ import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.time.DateUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.commafeed.backend.MetricsBean;
import com.commafeed.backend.dao.FeedDAO;
@@ -21,6 +23,8 @@ import com.google.common.collect.Queues;
@Singleton
public class FeedRefreshTaskGiver {
+ protected static final Logger log = LoggerFactory.getLogger(FeedRefreshTaskGiver.class);
+
@Inject
FeedDAO feedDAO;
diff --git a/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java b/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java
index 6e6b556f..1376576e 100644
--- a/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java
+++ b/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java
@@ -31,6 +31,7 @@ import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.pubsubhubbub.SubscriptionHandler;
import com.commafeed.backend.services.ApplicationSettingsService;
import com.commafeed.backend.services.FeedUpdateService;
+import com.google.api.client.util.Lists;
import com.google.common.util.concurrent.Striped;
@Singleton
@@ -141,9 +142,21 @@ public class FeedRefreshUpdater {
if (entries.isEmpty() == false) {
List subscriptions = feedSubscriptionDAO
.findByFeed(feed);
+ List lastEntries = cache.getLastEntries(feed);
+ List currentEntries = Lists.newArrayList();
for (FeedEntry entry : entries) {
- ok &= updateEntry(feed, entry, subscriptions);
+ String cacheKey = cache.buildKey(feed, entry);
+ if (!lastEntries.contains(cacheKey)) {
+ log.debug("cache miss for {}", entry.getUrl());
+ ok &= updateEntry(feed, entry, subscriptions);
+ metricsBean.entryCacheMiss();
+ } else {
+ log.debug("cache hit for {}", entry.getUrl());
+ metricsBean.entryCacheHit();
+ }
+ currentEntries.add(cacheKey);
}
+ cache.setLastEntries(feed, currentEntries);
}
if (applicationSettingsService.get().isPubsubhubbub()) {
@@ -171,15 +184,7 @@ public class FeedRefreshUpdater {
try {
locked = lock.tryLock(1, TimeUnit.MINUTES);
if (locked) {
- if (!cache.hasFeedEntry(feed, entry)) {
- log.debug("cache miss for {}", entry.getUrl());
- feedUpdateService.updateEntry(feed, entry, subscriptions);
- cache.putFeedEntry(feed, entry);
- metricsBean.entryCacheMiss();
- } else {
- log.debug("cache hit for {}", entry.getUrl());
- metricsBean.entryCacheHit();
- }
+ feedUpdateService.updateEntry(feed, entry, subscriptions);
success = true;
} else {
log.error("lock timeout for " + feed.getUrl() + " - " + key);
diff --git a/src/main/webapp/WEB-INF/beans.xml b/src/main/webapp/WEB-INF/beans.xml
index ca7cef2c..55e533bf 100644
--- a/src/main/webapp/WEB-INF/beans.xml
+++ b/src/main/webapp/WEB-INF/beans.xml
@@ -2,5 +2,7 @@
-
+
+ ${cache_service.class}
+
\ No newline at end of file