remove redis as caching is no longer needed now

This commit is contained in:
Athou
2024-08-12 21:27:21 +02:00
parent 3af8485326
commit 044694487d
15 changed files with 22 additions and 410 deletions

View File

@@ -12,12 +12,6 @@ import io.smallrye.config.WithDefault;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.Positive;
import redis.clients.jedis.DefaultJedisClientConfig;
import redis.clients.jedis.HostAndPort;
import redis.clients.jedis.JedisClientConfig;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import redis.clients.jedis.Protocol;
/**
* CommaFeed configuration
@@ -92,11 +86,6 @@ public interface CommaFeedConfiguration {
*/
Websocket websocket();
/**
* Redis settings to enable caching. This is only really useful on instances with a lot of users.
*/
Redis redis();
interface FeedRefresh {
/**
* Amount of time CommaFeed will wait before refreshing the same feed.
@@ -230,47 +219,4 @@ public interface CommaFeedConfiguration {
Duration treeReloadInterval();
}
interface Redis {
Optional<String> host();
@WithDefault("" + Protocol.DEFAULT_PORT)
int port();
/**
* Username is only required when using Redis ACLs
*/
Optional<String> username();
Optional<String> password();
@WithDefault("" + Protocol.DEFAULT_TIMEOUT)
int timeout();
@WithDefault("" + Protocol.DEFAULT_DATABASE)
int database();
@WithDefault("500")
int maxTotal();
default JedisPool build() {
Optional<String> host = host();
if (host.isEmpty()) {
throw new IllegalStateException("Redis host is required");
}
JedisPoolConfig poolConfig = new JedisPoolConfig();
poolConfig.setMaxTotal(maxTotal());
JedisClientConfig clientConfig = DefaultJedisClientConfig.builder()
.user(username().orElse(null))
.password(password().orElse(null))
.timeoutMillis(timeout())
.database(database())
.build();
return new JedisPool(poolConfig, new HostAndPort(host.get(), port()), clientConfig);
}
}
}

View File

@@ -1,10 +1,6 @@
package com.commafeed;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.CommaFeedConfiguration.Redis;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.cache.NoopCacheService;
import com.commafeed.backend.cache.RedisCacheService;
import jakarta.enterprise.inject.Produces;
import jakarta.inject.Singleton;
@@ -12,17 +8,6 @@ import jakarta.inject.Singleton;
@Singleton
public class CommaFeedProducers {
@Produces
@Singleton
public CacheService cacheService(CommaFeedConfiguration config) {
Redis redis = config.redis();
if (redis.host().isEmpty()) {
return new NoopCacheService();
}
return new RedisCacheService(redis.build());
}
@Produces
@Singleton
public MetricRegistry metricRegistry() {

View File

@@ -1,39 +0,0 @@
package com.commafeed.backend.cache;
import java.util.List;
import java.util.Set;
import com.commafeed.backend.Digests;
import com.commafeed.backend.feed.parser.FeedParserResult.Entry;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.Category;
import com.commafeed.frontend.model.UnreadCount;
public abstract class CacheService {
// feed entries for faster refresh
public abstract Set<String> getLastEntries(Feed feed);
public abstract void setLastEntries(Feed feed, List<String> entries);
public String buildUniqueEntryKey(Entry entry) {
return Digests.sha1Hex(entry.guid() + entry.url());
}
// user categories
public abstract Category getUserRootCategory(User user);
public abstract void setUserRootCategory(User user, Category category);
public abstract void invalidateUserRootCategory(User... users);
// unread count
public abstract UnreadCount getUnreadCount(FeedSubscription sub);
public abstract void setUnreadCount(FeedSubscription sub, UnreadCount count);
public abstract void invalidateUnreadCount(FeedSubscription... subs);
}

View File

@@ -1,54 +0,0 @@
package com.commafeed.backend.cache;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.Category;
import com.commafeed.frontend.model.UnreadCount;
public class NoopCacheService extends CacheService {
@Override
public Set<String> getLastEntries(Feed feed) {
return Collections.emptySet();
}
@Override
public void setLastEntries(Feed feed, List<String> entries) {
}
@Override
public UnreadCount getUnreadCount(FeedSubscription sub) {
return null;
}
@Override
public void setUnreadCount(FeedSubscription sub, UnreadCount count) {
}
@Override
public void invalidateUnreadCount(FeedSubscription... subs) {
}
@Override
public Category getUserRootCategory(User user) {
return null;
}
@Override
public void setUserRootCategory(User user, Category category) {
}
@Override
public void invalidateUserRootCategory(User... users) {
}
}

View File

@@ -1,154 +0,0 @@
package com.commafeed.backend.cache;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.Category;
import com.commafeed.frontend.model.UnreadCount;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.Pipeline;
@Slf4j
@RequiredArgsConstructor
public class RedisCacheService extends CacheService {
private static final ObjectMapper MAPPER = new ObjectMapper().registerModule(new JavaTimeModule());
private final JedisPool pool;
@Override
public Set<String> getLastEntries(Feed feed) {
try (Jedis jedis = pool.getResource()) {
String key = buildRedisEntryKey(feed);
return jedis.smembers(key);
}
}
@Override
public void setLastEntries(Feed feed, List<String> entries) {
try (Jedis jedis = pool.getResource()) {
String key = buildRedisEntryKey(feed);
Pipeline pipe = jedis.pipelined();
pipe.del(key);
for (String entry : entries) {
pipe.sadd(key, entry);
}
pipe.expire(key, (int) TimeUnit.DAYS.toSeconds(7));
pipe.sync();
}
}
@Override
public Category getUserRootCategory(User user) {
Category cat = null;
try (Jedis jedis = pool.getResource()) {
String key = buildRedisUserRootCategoryKey(user);
String json = jedis.get(key);
if (json != null) {
cat = MAPPER.readValue(json, Category.class);
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return cat;
}
@Override
public void setUserRootCategory(User user, Category category) {
try (Jedis jedis = pool.getResource()) {
String key = buildRedisUserRootCategoryKey(user);
Pipeline pipe = jedis.pipelined();
pipe.del(key);
pipe.set(key, MAPPER.writeValueAsString(category));
pipe.expire(key, (int) TimeUnit.MINUTES.toSeconds(30));
pipe.sync();
} catch (JsonProcessingException e) {
log.error(e.getMessage(), e);
}
}
@Override
public UnreadCount getUnreadCount(FeedSubscription sub) {
UnreadCount count = null;
try (Jedis jedis = pool.getResource()) {
String key = buildRedisUnreadCountKey(sub);
String json = jedis.get(key);
if (json != null) {
count = MAPPER.readValue(json, UnreadCount.class);
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
return count;
}
@Override
public void setUnreadCount(FeedSubscription sub, UnreadCount count) {
try (Jedis jedis = pool.getResource()) {
String key = buildRedisUnreadCountKey(sub);
Pipeline pipe = jedis.pipelined();
pipe.del(key);
pipe.set(key, MAPPER.writeValueAsString(count));
pipe.expire(key, (int) TimeUnit.MINUTES.toSeconds(30));
pipe.sync();
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
@Override
public void invalidateUserRootCategory(User... users) {
try (Jedis jedis = pool.getResource()) {
Pipeline pipe = jedis.pipelined();
if (users != null) {
for (User user : users) {
String key = buildRedisUserRootCategoryKey(user);
pipe.del(key);
}
}
pipe.sync();
}
}
@Override
public void invalidateUnreadCount(FeedSubscription... subs) {
try (Jedis jedis = pool.getResource()) {
Pipeline pipe = jedis.pipelined();
if (subs != null) {
for (FeedSubscription sub : subs) {
String key = buildRedisUnreadCountKey(sub);
pipe.del(key);
}
}
pipe.sync();
}
}
private String buildRedisEntryKey(Feed feed) {
return "f:" + Models.getId(feed);
}
private String buildRedisUserRootCategoryKey(User user) {
return "c:" + Models.getId(user);
}
private String buildRedisUnreadCountKey(FeedSubscription sub) {
return "u:" + Models.getId(sub);
}
}

View File

@@ -1,6 +1,5 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -16,7 +15,6 @@ import org.apache.commons.lang3.StringUtils;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.commafeed.backend.Digests;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
import com.commafeed.backend.dao.UnitOfWork;
import com.commafeed.backend.feed.parser.FeedParserResult.Content;
@@ -25,7 +23,6 @@ import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedEntry;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.User;
import com.commafeed.backend.service.FeedEntryService;
import com.commafeed.backend.service.FeedService;
import com.commafeed.frontend.ws.WebSocketMessageBuilder;
@@ -47,29 +44,23 @@ public class FeedRefreshUpdater {
private final FeedService feedService;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final CacheService cache;
private final WebSocketSessions webSocketSessions;
private final Striped<Lock> locks;
private final Meter entryCacheMiss;
private final Meter entryCacheHit;
private final Meter feedUpdated;
private final Meter entryInserted;
public FeedRefreshUpdater(UnitOfWork unitOfWork, FeedService feedService, FeedEntryService feedEntryService, MetricRegistry metrics,
FeedSubscriptionDAO feedSubscriptionDAO, CacheService cache, WebSocketSessions webSocketSessions) {
FeedSubscriptionDAO feedSubscriptionDAO, WebSocketSessions webSocketSessions) {
this.unitOfWork = unitOfWork;
this.feedService = feedService;
this.feedEntryService = feedEntryService;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.cache = cache;
this.webSocketSessions = webSocketSessions;
locks = Striped.lazyWeakLock(100000);
entryCacheMiss = metrics.meter(MetricRegistry.name(getClass(), "entryCacheMiss"));
entryCacheHit = metrics.meter(MetricRegistry.name(getClass(), "entryCacheHit"));
feedUpdated = metrics.meter(MetricRegistry.name(getClass(), "feedUpdated"));
entryInserted = metrics.meter(MetricRegistry.name(getClass(), "entryInserted"));
}
@@ -139,39 +130,21 @@ public class FeedRefreshUpdater {
Map<FeedSubscription, Long> unreadCountBySubscription = new HashMap<>();
if (!entries.isEmpty()) {
Set<String> lastEntries = cache.getLastEntries(feed);
List<String> currentEntries = new ArrayList<>();
List<FeedSubscription> subscriptions = null;
for (Entry entry : entries) {
String cacheKey = cache.buildUniqueEntryKey(entry);
if (!lastEntries.contains(cacheKey)) {
log.debug("cache miss for {}", entry.url());
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
processed &= addEntryResult.processed;
inserted += addEntryResult.inserted ? 1 : 0;
addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(sub -> unreadCountBySubscription.merge(sub, 1L, Long::sum));
entryCacheMiss.mark();
} else {
log.debug("cache hit for {}", entry.url());
entryCacheHit.mark();
if (subscriptions == null) {
subscriptions = unitOfWork.call(() -> feedSubscriptionDAO.findByFeed(feed));
}
currentEntries.add(cacheKey);
AddEntryResult addEntryResult = addEntry(feed, entry, subscriptions);
processed &= addEntryResult.processed;
inserted += addEntryResult.inserted ? 1 : 0;
addEntryResult.subscriptionsForWhichEntryIsUnread.forEach(sub -> unreadCountBySubscription.merge(sub, 1L, Long::sum));
}
cache.setLastEntries(feed, currentEntries);
if (subscriptions == null) {
if (inserted == 0) {
feed.setMessage("No new entries found");
} else if (inserted > 0) {
log.debug("inserted {} entries for feed {}", inserted, feed.getId());
List<User> users = subscriptions.stream().map(FeedSubscription::getUser).toList();
cache.invalidateUnreadCount(subscriptions.toArray(new FeedSubscription[0]));
cache.invalidateUserRootCategory(users.toArray(new User[0]));
feed.setMessage("Found %s new entries".formatted(inserted));
}
}

View File

@@ -6,7 +6,6 @@ import java.util.List;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.FeedCategory;
@@ -28,7 +27,6 @@ public class OPMLImporter {
private final FeedCategoryDAO feedCategoryDAO;
private final FeedSubscriptionService feedSubscriptionService;
private final CacheService cache;
public void importOpml(User user, String xml) throws IllegalArgumentException, FeedException {
xml = xml.substring(xml.indexOf('<'));
@@ -79,6 +77,5 @@ public class OPMLImporter {
log.error("error while importing {}: {}", outline.getXmlUrl(), e.getMessage());
}
}
cache.invalidateUserRootCategory(user);
}
}

View File

@@ -4,7 +4,6 @@ import java.time.Instant;
import java.util.List;
import com.commafeed.backend.Digests;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedEntryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
@@ -32,7 +31,6 @@ public class FeedEntryService {
private final FeedEntryStatusDAO feedEntryStatusDAO;
private final FeedEntryContentService feedEntryContentService;
private final FeedEntryFilteringService feedEntryFilteringService;
private final CacheService cache;
public FeedEntry find(Feed feed, Entry entry) {
String guidHash = Digests.sha1Hex(entry.guid());
@@ -85,8 +83,6 @@ public class FeedEntryService {
if (status.isMarkable()) {
status.setRead(read);
feedEntryStatusDAO.saveOrUpdate(status);
cache.invalidateUnreadCount(sub);
cache.invalidateUserRootCategory(user);
}
}
@@ -112,8 +108,6 @@ public class FeedEntryService {
List<FeedEntryStatus> statuses = feedEntryStatusDAO.findBySubscriptions(user, subscriptions, true, keywords, null, -1, -1, null,
false, null, null, null);
markList(statuses, olderThan, insertedBefore);
cache.invalidateUnreadCount(subscriptions.toArray(new FeedSubscription[0]));
cache.invalidateUserRootCategory(user);
}
public void markStarredEntries(User user, Instant olderThan, Instant insertedBefore) {

View File

@@ -8,7 +8,6 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
@@ -17,7 +16,6 @@ import com.commafeed.backend.feed.FeedUtils;
import com.commafeed.backend.model.Feed;
import com.commafeed.backend.model.FeedCategory;
import com.commafeed.backend.model.FeedSubscription;
import com.commafeed.backend.model.Models;
import com.commafeed.backend.model.User;
import com.commafeed.frontend.model.UnreadCount;
@@ -33,17 +31,15 @@ public class FeedSubscriptionService {
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedService feedService;
private final FeedRefreshEngine feedRefreshEngine;
private final CacheService cache;
private final CommaFeedConfiguration config;
public FeedSubscriptionService(FeedDAO feedDAO, FeedEntryStatusDAO feedEntryStatusDAO, FeedSubscriptionDAO feedSubscriptionDAO,
FeedService feedService, FeedRefreshEngine feedRefreshEngine, CacheService cache, CommaFeedConfiguration config) {
FeedService feedService, FeedRefreshEngine feedRefreshEngine, CommaFeedConfiguration config) {
this.feedDAO = feedDAO;
this.feedEntryStatusDAO = feedEntryStatusDAO;
this.feedSubscriptionDAO = feedSubscriptionDAO;
this.feedService = feedService;
this.feedRefreshEngine = feedRefreshEngine;
this.cache = cache;
this.config = config;
// automatically refresh feeds after they are subscribed to
@@ -95,7 +91,6 @@ public class FeedSubscriptionService {
sub.setTitle(FeedUtils.truncate(title, 128));
feedSubscriptionDAO.saveOrUpdate(sub);
cache.invalidateUserRootCategory(user);
return sub.getId();
}
@@ -103,7 +98,6 @@ public class FeedSubscriptionService {
FeedSubscription sub = feedSubscriptionDAO.findById(user, subId);
if (sub != null) {
feedSubscriptionDAO.delete(sub);
cache.invalidateUserRootCategory(user);
return true;
} else {
return false;
@@ -130,17 +124,9 @@ public class FeedSubscriptionService {
}
public Map<Long, UnreadCount> getUnreadCount(User user) {
return feedSubscriptionDAO.findAll(user).stream().collect(Collectors.toMap(FeedSubscription::getId, this::getUnreadCount));
}
private UnreadCount getUnreadCount(FeedSubscription sub) {
UnreadCount count = cache.getUnreadCount(sub);
if (count == null) {
log.debug("unread count cache miss for {}", Models.getId(sub));
count = feedEntryStatusDAO.getUnreadCount(sub);
cache.setUnreadCount(sub, count);
}
return count;
return feedSubscriptionDAO.findAll(user)
.stream()
.collect(Collectors.toMap(FeedSubscription::getId, feedEntryStatusDAO::getUnreadCount));
}
@SuppressWarnings("serial")

View File

@@ -14,7 +14,6 @@ import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
@@ -89,7 +88,6 @@ public class CategoryREST {
private final FeedSubscriptionDAO feedSubscriptionDAO;
private final FeedEntryService feedEntryService;
private final FeedSubscriptionService feedSubscriptionService;
private final CacheService cache;
private final CommaFeedConfiguration config;
@Path("/entries")
@@ -291,7 +289,6 @@ public class CategoryREST {
cat.setParent(parent);
}
feedCategoryDAO.saveOrUpdate(cat);
cache.invalidateUserRootCategory(user);
return Response.ok(cat.getId()).build();
}
@@ -321,7 +318,6 @@ public class CategoryREST {
feedCategoryDAO.saveOrUpdate(categories);
feedCategoryDAO.delete(cat);
cache.invalidateUserRootCategory(user);
return Response.ok().build();
} else {
return Response.status(Status.NOT_FOUND).build();
@@ -374,7 +370,6 @@ public class CategoryREST {
}
feedCategoryDAO.saveOrUpdate(category);
cache.invalidateUserRootCategory(user);
return Response.ok().build();
}
@@ -393,7 +388,6 @@ public class CategoryREST {
}
category.setCollapsed(req.isCollapse());
feedCategoryDAO.saveOrUpdate(category);
cache.invalidateUserRootCategory(user);
return Response.ok().build();
}
@@ -418,18 +412,14 @@ public class CategoryREST {
responses = { @ApiResponse(content = @Content(schema = @Schema(implementation = Category.class))) })
public Response getRootCategory() {
User user = authenticationContext.getCurrentUser();
Category root = cache.getUserRootCategory(user);
if (root == null) {
log.debug("tree cache miss for {}", user.getId());
List<FeedCategory> categories = feedCategoryDAO.findAll(user);
List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user);
Map<Long, UnreadCount> unreadCount = feedSubscriptionService.getUnreadCount(user);
root = buildCategory(null, categories, subscriptions, unreadCount);
root.setId("all");
root.setName("All");
cache.setUserRootCategory(user, root);
}
List<FeedCategory> categories = feedCategoryDAO.findAll(user);
List<FeedSubscription> subscriptions = feedSubscriptionDAO.findAll(user);
Map<Long, UnreadCount> unreadCount = feedSubscriptionService.getUnreadCount(user);
Category root = buildCategory(null, categories, subscriptions, unreadCount);
root.setId("all");
root.setName("All");
return Response.ok(root).build();
}

View File

@@ -17,7 +17,6 @@ import org.jboss.resteasy.reactive.RestForm;
import com.commafeed.CommaFeedApplication;
import com.commafeed.CommaFeedConfiguration;
import com.commafeed.backend.cache.CacheService;
import com.commafeed.backend.dao.FeedCategoryDAO;
import com.commafeed.backend.dao.FeedEntryStatusDAO;
import com.commafeed.backend.dao.FeedSubscriptionDAO;
@@ -114,7 +113,6 @@ public class FeedREST {
private final FeedRefreshEngine feedRefreshEngine;
private final OPMLImporter opmlImporter;
private final OPMLExporter opmlExporter;
private final CacheService cache;
private final CommaFeedConfiguration config;
private static FeedEntry initTestEntry() {
@@ -492,7 +490,6 @@ public class FeedREST {
} else {
feedSubscriptionDAO.saveOrUpdate(subscription);
}
cache.invalidateUserRootCategory(user);
return Response.ok().build();
}