garrettmills/commafeed
1
0
Fork 0
forked from Archives/Athou_commafeed
Code Pull Requests Activity

feeds added manually to the queue now refresh immediately instead of waiting up to 15s (#1036)

Browse Source
This commit is contained in:
Athou
2023-01-16 16:19:51 +01:00
parent 347fc4f2c8
commit 33e3f7ea3c
3 changed files with 50 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedQueues.java
View File

@@ -1,13 +1,10 @@
package com.commafeed.backend.feed;
import java.util.ArrayList;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.stream.Collectors;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.TimeUnit;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -27,123 +24,90 @@ import com.commafeed.backend.model.Feed;
@Singleton
public class FeedQueues {
private SessionFactory sessionFactory;
private final SessionFactory sessionFactory;
private final FeedDAO feedDAO;
private final CommaFeedConfiguration config;
private Queue<FeedRefreshContext> addQueue = new ConcurrentLinkedQueue<>();
private Queue<FeedRefreshContext> takeQueue = new ConcurrentLinkedQueue<>();
private Queue<Feed> giveBackQueue = new ConcurrentLinkedQueue<>();
private Meter refill;
private final BlockingDeque<FeedRefreshContext> queue = new LinkedBlockingDeque<>();
private final Meter refill;
@Inject
public FeedQueues(SessionFactory sessionFactory, FeedDAO feedDAO, CommaFeedConfiguration config, MetricRegistry metrics) {
this.sessionFactory = sessionFactory;
this.config = config;
this.feedDAO = feedDAO;
this.refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
refill = metrics.meter(MetricRegistry.name(getClass(), "refill"));
metrics.register(MetricRegistry.name(getClass(), "addQueue"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return addQueue.size();
}
});
metrics.register(MetricRegistry.name(getClass(), "takeQueue"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return takeQueue.size();
}
});
metrics.register(MetricRegistry.name(getClass(), "giveBackQueue"), new Gauge<Integer>() {
@Override
public Integer getValue() {
return giveBackQueue.size();
}
});
metrics.register(MetricRegistry.name(getClass(), "queue"), (Gauge<Integer>) queue::size);
}
/**
* take a feed from the refresh queue
*/
public synchronized FeedRefreshContext take() {
FeedRefreshContext context = takeQueue.poll();
if (context == null) {
refill();
context = takeQueue.poll();
FeedRefreshContext context = queue.poll();
if (context != null) {
return context;
}
refill();
try {
// try to get something from the queue
// if the queue is empty, wait a bit
// polling the queue instead of sleeping gives us the opportunity to process a feed immediately if it was added manually with
// add()
return queue.poll(15, TimeUnit.SECONDS);
} catch (InterruptedException e) {
throw new RuntimeException("interrupted while waiting for a feed in the queue", e);
}
return context;
}
/**
* add a feed to the refresh queue
*/
public void add(Feed feed, boolean urgent) {
boolean alreadyQueued = addQueue.stream().anyMatch(c -> c.getFeed().getId().equals(feed.getId()));
if (!alreadyQueued) {
addQueue.add(new FeedRefreshContext(feed, urgent));
if (isFeedAlreadyQueued(feed)) {
return;
}
FeedRefreshContext context = new FeedRefreshContext(feed, urgent);
if (urgent) {
queue.addFirst(context);
} else {
queue.addLast(context);
}
}
/**
* refills the refresh queue and empties the giveBack queue while at it
* refills the refresh queue
*/
private void refill() {
refill.mark();
List<FeedRefreshContext> contexts = new ArrayList<>();
int batchSize = Math.min(100, 3 * config.getApplicationSettings().getBackgroundThreads());
// add feeds we got from the add() method
int addQueueSize = addQueue.size();
for (int i = 0; i < Math.min(batchSize, addQueueSize); i++) {
contexts.add(addQueue.poll());
}
// add feeds that are up to refresh from the database
int count = batchSize - contexts.size();
if (count > 0) {
List<Feed> feeds = UnitOfWork.call(sessionFactory, () -> feedDAO.findNextUpdatable(count, getLastLoginThreshold()));
for (Feed feed : feeds) {
contexts.add(new FeedRefreshContext(feed, false));
}
}
int batchSize = Math.min(100, 3 * config.getApplicationSettings().getBackgroundThreads());
List<Feed> feeds = UnitOfWork.call(sessionFactory, () -> {
List<Feed> list = feedDAO.findNextUpdatable(batchSize, getLastLoginThreshold());
// set the disabledDate as we use it in feedDAO to decide what to refresh next. We also use a map to remove
// duplicates.
Map<Long, FeedRefreshContext> map = new LinkedHashMap<>();
for (FeedRefreshContext context : contexts) {
Feed feed = context.getFeed();
feed.setDisabledUntil(DateUtils.addMinutes(new Date(), config.getApplicationSettings().getRefreshIntervalMinutes()));
map.put(feed.getId(), context);
}
// set the disabledDate as we use it in feedDAO.findNextUpdatable() to decide what to refresh next
Date nextRefreshDate = DateUtils.addMinutes(new Date(), config.getApplicationSettings().getRefreshIntervalMinutes());
list.forEach(f -> f.setDisabledUntil(nextRefreshDate));
feedDAO.saveOrUpdate(list);
// refill the queue
takeQueue.addAll(map.values());
return list;
});
// add feeds from the giveBack queue to the map, overriding duplicates
int giveBackQueueSize = giveBackQueue.size();
for (int i = 0; i < giveBackQueueSize; i++) {
Feed feed = giveBackQueue.poll();
map.put(feed.getId(), new FeedRefreshContext(feed, false));
}
// update all feeds in the database
List<Feed> feeds = map.values().stream().map(FeedRefreshContext::getFeed).collect(Collectors.toList());
UnitOfWork.run(sessionFactory, () -> feedDAO.saveOrUpdate(feeds));
feeds.forEach(f -> add(f, false));
}
/**
* give a feed back, updating it to the database during the next refill()
*/
public void giveBack(Feed feed) {
String normalized = FeedUtils.normalizeURL(feed.getUrl());
feed.setNormalizedUrl(normalized);
feed.setNormalizedUrlHash(DigestUtils.sha1Hex(normalized));
feed.setLastUpdated(new Date());
giveBackQueue.add(feed);
UnitOfWork.run(sessionFactory, () -> feedDAO.saveOrUpdate(feed));
// we just finished updating the feed, remove it from the queue
queue.removeIf(c -> isFeedAlreadyQueued(c.getFeed()));
}
private Date getLastLoginThreshold() {
@@ -154,4 +118,7 @@ public class FeedQueues {
}
}
private boolean isFeedAlreadyQueued(Feed feed) {
return queue.stream().anyMatch(c -> c.getFeed().getId().equals(feed.getId()));
}
}

10
commafeed-server/src/main/java/com/commafeed/backend/feed/FeedRefreshTaskGiver.java
View File

@@ -28,7 +28,6 @@ public class FeedRefreshTaskGiver implements Managed {
private final ExecutorService executor;
private final Meter feedRefreshed;
private final Meter threadWaited;
@Inject
public FeedRefreshTaskGiver(FeedQueues queues, FeedDAO feedDAO, FeedRefreshWorker worker, CommaFeedConfiguration config,
@@ -38,7 +37,6 @@ public class FeedRefreshTaskGiver implements Managed {
executor = Executors.newFixedThreadPool(1);
feedRefreshed = metrics.meter(MetricRegistry.name(getClass(), "feedRefreshed"));
threadWaited = metrics.meter(MetricRegistry.name(getClass(), "threadWaited"));
}
@Override
@@ -66,14 +64,6 @@ public class FeedRefreshTaskGiver implements Managed {
if (context != null) {
feedRefreshed.mark();
worker.updateFeed(context);
} else {
log.debug("nothing to do, sleeping for 15s");
threadWaited.mark();
try {
Thread.sleep(15000);
} catch (InterruptedException e) {
log.debug("interrupted while sleeping");
}
}
} catch (Exception e) {
log.error(e.getMessage(), e);