diff --git a/pom.xml b/pom.xml index c4fc413d..7ee9af7e 100644 --- a/pom.xml +++ b/pom.xml @@ -19,6 +19,7 @@ java:openejb/Resource/My DataSource org.hibernate.dialect.HSQLDialect false + com.commafeed.backend.cache.InMemoryCacheService @@ -69,6 +70,16 @@ 2.3 false + + + ${basedir}/src/main/webapp/WEB-INF + WEB-INF + true + + **/beans.xml + + + @@ -172,6 +183,11 @@ + + redis.clients + jedis + 2.0.0 + org.liquibase liquibase-core @@ -443,6 +459,12 @@ true + + redis + + com.commafeed.backend.cache.RedisCacheService + + mysql diff --git a/src/main/java/com/commafeed/backend/cache/CacheService.java b/src/main/java/com/commafeed/backend/cache/CacheService.java index 73edccb0..aa1dc339 100644 --- a/src/main/java/com/commafeed/backend/cache/CacheService.java +++ b/src/main/java/com/commafeed/backend/cache/CacheService.java @@ -1,31 +1,21 @@ package com.commafeed.backend.cache; -import java.util.concurrent.TimeUnit; +import java.util.List; + +import org.apache.commons.codec.digest.DigestUtils; import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.FeedEntry; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -public class CacheService { +public abstract class CacheService { - Cache entryCache = CacheBuilder.newBuilder() - .maximumSize(1000000).expireAfterWrite(24, TimeUnit.HOURS).build(); + public abstract List getLastEntries(Feed feed); - private static enum Marker { - INSTANCE - } + public abstract void setLastEntries(Feed feed, List entries); - public boolean hasFeedEntry(Feed feed, FeedEntry entry) { - return entryCache.getIfPresent(buildKey(feed, entry)) == Marker.INSTANCE; - } - - public void putFeedEntry(Feed feed, FeedEntry entry) { - entryCache.put(buildKey(feed, entry), Marker.INSTANCE); - } - - private String buildKey(Feed feed, FeedEntry entry) { - return String.format("%s:%s:%s", feed.getId(), entry.getGuid(), + public String buildKey(Feed feed, FeedEntry entry) { + return DigestUtils.sha1Hex(entry.getGuid() + entry.getUrl()); } + } diff --git a/src/main/java/com/commafeed/backend/cache/InMemoryCacheService.java b/src/main/java/com/commafeed/backend/cache/InMemoryCacheService.java new file mode 100644 index 00000000..a59dba3d --- /dev/null +++ b/src/main/java/com/commafeed/backend/cache/InMemoryCacheService.java @@ -0,0 +1,46 @@ +package com.commafeed.backend.cache; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import javax.annotation.PostConstruct; +import javax.enterprise.context.ApplicationScoped; +import javax.enterprise.inject.Alternative; +import javax.inject.Inject; + +import com.commafeed.backend.model.Feed; +import com.commafeed.backend.services.ApplicationSettingsService; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; + +@Alternative +@ApplicationScoped +public class InMemoryCacheService extends CacheService { + + @Inject + ApplicationSettingsService applicationSettingsService; + + private Cache> entryCache; + + @PostConstruct + private void init() { + int capacity = applicationSettingsService.get().isHeavyLoad() ? 1000000 : 100; + entryCache = CacheBuilder.newBuilder() + .maximumSize(capacity).expireAfterWrite(24, TimeUnit.HOURS).build(); + } + + @Override + public List getLastEntries(Feed feed) { + List list = entryCache.getIfPresent(feed.getId()); + if (list == null) { + list = Collections.emptyList(); + } + return list; + } + + @Override + public void setLastEntries(Feed feed, List entries) { + entryCache.put(feed.getId(), entries); + } +} diff --git a/src/main/java/com/commafeed/backend/cache/RedisCacheService.java b/src/main/java/com/commafeed/backend/cache/RedisCacheService.java new file mode 100644 index 00000000..2f1a744c --- /dev/null +++ b/src/main/java/com/commafeed/backend/cache/RedisCacheService.java @@ -0,0 +1,62 @@ +package com.commafeed.backend.cache; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import javax.enterprise.context.ApplicationScoped; +import javax.enterprise.inject.Alternative; + +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.JedisPoolConfig; +import redis.clients.jedis.Pipeline; + +import com.commafeed.backend.model.Feed; +import com.google.api.client.util.Lists; + +@Alternative +@ApplicationScoped +public class RedisCacheService extends CacheService { + + private JedisPool pool = new JedisPool(new JedisPoolConfig(), "localhost"); + + @Override + public List getLastEntries(Feed feed) { + List list = Lists.newArrayList(); + Jedis jedis = pool.getResource(); + try { + String key = buildKey(feed); + Set members = jedis.smembers(key); + for (String member : members) { + list.add(member); + } + } finally { + pool.returnResource(jedis); + } + return list; + } + + @Override + public void setLastEntries(Feed feed, List entries) { + Jedis jedis = pool.getResource(); + try { + String key = buildKey(feed); + + Pipeline pipe = jedis.pipelined(); + pipe.del(key); + for (String entry : entries) { + pipe.sadd(key, entry); + } + pipe.expire(key, (int) TimeUnit.HOURS.toSeconds(24)); + pipe.sync(); + } finally { + pool.returnResource(jedis); + } + } + + private String buildKey(Feed feed) { + return "feed:" + feed.getId(); + } + +} diff --git a/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java b/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java index 2b40b341..d8c0b5d9 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java @@ -10,6 +10,8 @@ import javax.inject.Inject; import javax.inject.Singleton; import org.apache.commons.lang3.time.DateUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.commafeed.backend.MetricsBean; import com.commafeed.backend.dao.FeedDAO; @@ -21,6 +23,8 @@ import com.google.common.collect.Queues; @Singleton public class FeedRefreshTaskGiver { + protected static final Logger log = LoggerFactory.getLogger(FeedRefreshTaskGiver.class); + @Inject FeedDAO feedDAO; diff --git a/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java b/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java index 6e6b556f..1376576e 100644 --- a/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java +++ b/src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java @@ -31,6 +31,7 @@ import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.pubsubhubbub.SubscriptionHandler; import com.commafeed.backend.services.ApplicationSettingsService; import com.commafeed.backend.services.FeedUpdateService; +import com.google.api.client.util.Lists; import com.google.common.util.concurrent.Striped; @Singleton @@ -141,9 +142,21 @@ public class FeedRefreshUpdater { if (entries.isEmpty() == false) { List subscriptions = feedSubscriptionDAO .findByFeed(feed); + List lastEntries = cache.getLastEntries(feed); + List currentEntries = Lists.newArrayList(); for (FeedEntry entry : entries) { - ok &= updateEntry(feed, entry, subscriptions); + String cacheKey = cache.buildKey(feed, entry); + if (!lastEntries.contains(cacheKey)) { + log.debug("cache miss for {}", entry.getUrl()); + ok &= updateEntry(feed, entry, subscriptions); + metricsBean.entryCacheMiss(); + } else { + log.debug("cache hit for {}", entry.getUrl()); + metricsBean.entryCacheHit(); + } + currentEntries.add(cacheKey); } + cache.setLastEntries(feed, currentEntries); } if (applicationSettingsService.get().isPubsubhubbub()) { @@ -171,15 +184,7 @@ public class FeedRefreshUpdater { try { locked = lock.tryLock(1, TimeUnit.MINUTES); if (locked) { - if (!cache.hasFeedEntry(feed, entry)) { - log.debug("cache miss for {}", entry.getUrl()); - feedUpdateService.updateEntry(feed, entry, subscriptions); - cache.putFeedEntry(feed, entry); - metricsBean.entryCacheMiss(); - } else { - log.debug("cache hit for {}", entry.getUrl()); - metricsBean.entryCacheHit(); - } + feedUpdateService.updateEntry(feed, entry, subscriptions); success = true; } else { log.error("lock timeout for " + feed.getUrl() + " - " + key); diff --git a/src/main/webapp/WEB-INF/beans.xml b/src/main/webapp/WEB-INF/beans.xml index ca7cef2c..55e533bf 100644 --- a/src/main/webapp/WEB-INF/beans.xml +++ b/src/main/webapp/WEB-INF/beans.xml @@ -2,5 +2,7 @@ - + + ${cache_service.class} + \ No newline at end of file