garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

added in memory cache and redis cache and marked them as application scoped

Browse Source
This commit is contained in:
Athou
2013-06-29 11:20:53 +02:00
parent c2af4de96b
commit 27e03e3038
7 changed files with 161 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
pom.xml
View File

@@ -19,6 +19,7 @@
<jpa.datasource.name>java:openejb/Resource/My DataSource</jpa.datasource.name>
<jpa.dialect>org.hibernate.dialect.HSQLDialect</jpa.dialect>
<jpa.cache>false</jpa.cache>
<cache_service.class>com.commafeed.backend.cache.InMemoryCacheService</cache_service.class>
</properties>
<repositories>
@@ -69,6 +70,16 @@
<version>2.3</version>
<configuration>
<failOnMissingWebXml>false</failOnMissingWebXml>
<webResources>
<resource>
<directory>${basedir}/src/main/webapp/WEB-INF</directory>
<targetPath>WEB-INF</targetPath>
<filtering>true</filtering>
<includes>
<include>**/beans.xml</include>
</includes>
</resource>
</webResources>
</configuration>
</plugin>
<plugin>
@@ -172,6 +183,11 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>org.liquibase</groupId>
<artifactId>liquibase-core</artifactId>
@@ -443,6 +459,12 @@
<jpa.cache>true</jpa.cache>
</properties>
</profile>
<profile>
<id>redis</id>
<properties>
<cache_service.class>com.commafeed.backend.cache.RedisCacheService</cache_service.class>
</properties>
</profile>
<profile>
<id>mysql</id>
<properties>

28
src/main/java/com/commafeed/backend/cache/CacheService.java vendored
View File

@@ -1,31 +1,21 @@
package com.commafeed.backend.cache;
import java.util.concurrent.TimeUnit;
import java.util.List;
import org.apache.commons.codec.digest.DigestUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
public class CacheService {
public abstract class CacheService {
Cache<String, Marker> entryCache = CacheBuilder.newBuilder()
.maximumSize(1000000).expireAfterWrite(24, TimeUnit.HOURS).build();
public abstract List<String> getLastEntries(Feed feed);
private static enum Marker {
INSTANCE
}
public abstract void setLastEntries(Feed feed, List<String> entries);
public boolean hasFeedEntry(Feed feed, FeedEntry entry) {
return entryCache.getIfPresent(buildKey(feed, entry)) == Marker.INSTANCE;
}
public void putFeedEntry(Feed feed, FeedEntry entry) {
entryCache.put(buildKey(feed, entry), Marker.INSTANCE);
}
private String buildKey(Feed feed, FeedEntry entry) {
return String.format("%s:%s:%s", feed.getId(), entry.getGuid(),
public String buildKey(Feed feed, FeedEntry entry) {
return DigestUtils.sha1Hex(entry.getGuid() +
entry.getUrl());
}
}

46
src/main/java/com/commafeed/backend/cache/InMemoryCacheService.java vendored Normal file
View File

@@ -0,0 +1,46 @@
package com.commafeed.backend.cache;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.TimeUnit;
import javax.annotation.PostConstruct;
import javax.enterprise.context.ApplicationScoped;
import javax.enterprise.inject.Alternative;
import javax.inject.Inject;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.services.ApplicationSettingsService;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
@Alternative
@ApplicationScoped
public class InMemoryCacheService extends CacheService {
@Inject
ApplicationSettingsService applicationSettingsService;
private Cache<Long, List<String>> entryCache;
@PostConstruct
private void init() {
int capacity = applicationSettingsService.get().isHeavyLoad() ? 1000000 : 100;
entryCache = CacheBuilder.newBuilder()
.maximumSize(capacity).expireAfterWrite(24, TimeUnit.HOURS).build();
}
@Override
public List<String> getLastEntries(Feed feed) {
List<String> list = entryCache.getIfPresent(feed.getId());
if (list == null) {
list = Collections.emptyList();
}
return list;
}
@Override
public void setLastEntries(Feed feed, List<String> entries) {
entryCache.put(feed.getId(), entries);
}
}

62
src/main/java/com/commafeed/backend/cache/RedisCacheService.java vendored Normal file
View File

@@ -0,0 +1,62 @@
package com.commafeed.backend.cache;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.enterprise.context.ApplicationScoped;
import javax.enterprise.inject.Alternative;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.Pipeline;
import com.commafeed.backend.model.Feed;
import com.google.api.client.util.Lists;
@Alternative
@ApplicationScoped
public class RedisCacheService extends CacheService {
private JedisPool pool = new JedisPool(new JedisPoolConfig(), "localhost");
@Override
public List<String> getLastEntries(Feed feed) {
List<String> list = Lists.newArrayList();
Jedis jedis = pool.getResource();
try {
String key = buildKey(feed);
Set<String> members = jedis.smembers(key);
for (String member : members) {
list.add(member);
}
} finally {
pool.returnResource(jedis);
}
return list;
}
@Override
public void setLastEntries(Feed feed, List<String> entries) {
Jedis jedis = pool.getResource();
try {
String key = buildKey(feed);
Pipeline pipe = jedis.pipelined();
pipe.del(key);
for (String entry : entries) {
pipe.sadd(key, entry);
}
pipe.expire(key, (int) TimeUnit.HOURS.toSeconds(24));
pipe.sync();
} finally {
pool.returnResource(jedis);
}
}
private String buildKey(Feed feed) {
return "feed:" + feed.getId();
}
}

4
src/main/java/com/commafeed/backend/feeds/FeedRefreshTaskGiver.java
View File

@@ -10,6 +10,8 @@ import javax.inject.Inject;
import javax.inject.Singleton;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.commafeed.backend.MetricsBean;
import com.commafeed.backend.dao.FeedDAO;
@@ -21,6 +23,8 @@ import com.google.common.collect.Queues;
@Singleton
public class FeedRefreshTaskGiver {
protected static final Logger log = LoggerFactory.getLogger(FeedRefreshTaskGiver.class);
@Inject
FeedDAO feedDAO;

25
src/main/java/com/commafeed/backend/feeds/FeedRefreshUpdater.java
View File

@@ -31,6 +31,7 @@ import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.pubsubhubbub.SubscriptionHandler;
import com.commafeed.backend.services.ApplicationSettingsService;
import com.commafeed.backend.services.FeedUpdateService;
import com.google.api.client.util.Lists;
import com.google.common.util.concurrent.Striped;
@Singleton
@@ -141,9 +142,21 @@ public class FeedRefreshUpdater {
if (entries.isEmpty() == false) {
List<FeedSubscription> subscriptions = feedSubscriptionDAO
.findByFeed(feed);
List<String> lastEntries = cache.getLastEntries(feed);
List<String> currentEntries = Lists.newArrayList();
for (FeedEntry entry : entries) {
ok &= updateEntry(feed, entry, subscriptions);
String cacheKey = cache.buildKey(feed, entry);
if (!lastEntries.contains(cacheKey)) {
log.debug("cache miss for {}", entry.getUrl());
ok &= updateEntry(feed, entry, subscriptions);
metricsBean.entryCacheMiss();
} else {
log.debug("cache hit for {}", entry.getUrl());
metricsBean.entryCacheHit();
}
currentEntries.add(cacheKey);
}
cache.setLastEntries(feed, currentEntries);
}
if (applicationSettingsService.get().isPubsubhubbub()) {
@@ -171,15 +184,7 @@ public class FeedRefreshUpdater {
try {
locked = lock.tryLock(1, TimeUnit.MINUTES);
if (locked) {
if (!cache.hasFeedEntry(feed, entry)) {
log.debug("cache miss for {}", entry.getUrl());
feedUpdateService.updateEntry(feed, entry, subscriptions);
cache.putFeedEntry(feed, entry);
metricsBean.entryCacheMiss();
} else {
log.debug("cache hit for {}", entry.getUrl());
metricsBean.entryCacheHit();
}
feedUpdateService.updateEntry(feed, entry, subscriptions);
success = true;
} else {
log.error("lock timeout for " + feed.getUrl() + " - " + key);

4
src/main/webapp/WEB-INF/beans.xml
View File

@@ -2,5 +2,7 @@
<beans xmlns="http://java.sun.com/xml/ns/javaee" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="
http://java.sun.com/xml/ns/javaee http://www.oracle.com/webfolder/technetwork/jsc/xml/ns/javaee/beans_1_0.xsd">
<alternatives>
<class>${cache_service.class}</class>
</alternatives>
</beans>