diff --git a/src/main/java/com/commafeed/backend/DatabaseCleaner.java b/src/main/java/com/commafeed/backend/DatabaseCleaner.java index 39179c3b..fc5c036d 100644 --- a/src/main/java/com/commafeed/backend/DatabaseCleaner.java +++ b/src/main/java/com/commafeed/backend/DatabaseCleaner.java @@ -10,13 +10,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.commafeed.backend.dao.FeedDAO; -import com.commafeed.backend.dao.FeedDAO.FeedCount; import com.commafeed.backend.dao.FeedEntryDAO; import com.commafeed.backend.dao.FeedSubscriptionDAO; import com.commafeed.backend.model.Feed; import com.commafeed.backend.model.FeedSubscription; import com.commafeed.backend.services.ApplicationSettingsService; -import com.google.common.collect.Lists; public class DatabaseCleaner { @@ -75,44 +73,6 @@ public class DatabaseCleaner { return total; } - public long cleanDuplicateFeeds() { - long total = 0; - int deleted = -1; - do { - List fcs = feedDAO - .findDuplicates(0, applicationSettingsService.get() - .getDatabaseUpdateThreads(), 1); - deleted = fcs.size(); - - List threads = Lists.newArrayList(); - for (final FeedCount fc : fcs) { - Thread thread = new Thread() { - public void run() { - Feed into = feedDAO.findById(fc.feeds.get(0).getId()); - List feeds = Lists.newArrayList(); - for (Feed feed : fc.feeds) { - feeds.add(feedDAO.findById(feed.getId())); - } - mergeFeeds(into, feeds); - }; - }; - thread.start(); - threads.add(thread); - } - for (Thread thread : threads) { - try { - thread.join(); - } catch (InterruptedException e) { - log.error(e.getMessage(), e); - } - } - total += deleted; - log.info("merged {} feeds", total); - } while (deleted != 0); - log.info("cleanup done: {} feeds merged", total); - return total; - } - public void mergeFeeds(Feed into, List feeds) { for (Feed feed : feeds) { if (into.getId().equals(feed.getId())) { diff --git a/src/main/java/com/commafeed/backend/dao/FeedDAO.java b/src/main/java/com/commafeed/backend/dao/FeedDAO.java index 203e8004..3813ebec 100644 --- a/src/main/java/com/commafeed/backend/dao/FeedDAO.java +++ b/src/main/java/com/commafeed/backend/dao/FeedDAO.java @@ -13,6 +13,7 @@ import javax.persistence.criteria.Path; import javax.persistence.criteria.Predicate; import javax.persistence.criteria.Root; import javax.persistence.criteria.SetJoin; +import javax.persistence.metamodel.SingularAttribute; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; @@ -34,7 +35,7 @@ public class FeedDAO extends GenericDAO { @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) public static class FeedCount { - public String normalizedUrlHash; + public String value; public List feeds; } @@ -136,28 +137,43 @@ public class FeedDAO extends GenericDAO { } - public List findDuplicates(int offset, int limit, long minCount) { + public static enum DuplicateMode { + NORMALIZED_URL(Feed_.normalizedUrlHash), LAST_CONTENT( + Feed_.lastContentHash), PUSH_TOPIC(Feed_.pushTopicHash); + private SingularAttribute path; + + private DuplicateMode(SingularAttribute path) { + this.path = path; + } + + public SingularAttribute getPath() { + return path; + } + } + + public List findDuplicates(DuplicateMode mode, int offset, + int limit, long minCount) { CriteriaQuery query = builder.createQuery(String.class); Root root = query.from(getType()); - Path hashPath = root.get(Feed_.normalizedUrlHash); - Expression count = builder.count(hashPath); + Path path = root.get(mode.getPath()); + Expression count = builder.count(path); - query.select(hashPath); + query.select(path); - query.groupBy(hashPath); + query.groupBy(path); query.having(builder.greaterThan(count, minCount)); TypedQuery q = em.createQuery(query); limit(q, offset, limit); - List normalizedUrlHashes = q.getResultList(); + List pathValues = q.getResultList(); List result = Lists.newArrayList(); - for (String hash : normalizedUrlHashes) { + for (String pathValue : pathValues) { FeedCount fc = new FeedCount(); - fc.normalizedUrlHash = hash; + fc.value = pathValue; fc.feeds = Lists.newArrayList(); - for (Feed feed : findByField(Feed_.normalizedUrlHash, hash)) { + for (Feed feed : findByField(mode.getPath(), pathValue)) { Feed f = new Feed(); f.setId(feed.getId()); f.setUrl(feed.getUrl()); diff --git a/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java b/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java index 793ec0b7..9d8a9f54 100644 --- a/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java +++ b/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java @@ -21,6 +21,7 @@ import com.commafeed.backend.DatabaseCleaner; import com.commafeed.backend.MetricsBean; import com.commafeed.backend.StartupBean; import com.commafeed.backend.dao.FeedDAO; +import com.commafeed.backend.dao.FeedDAO.DuplicateMode; import com.commafeed.backend.dao.FeedDAO.FeedCount; import com.commafeed.backend.dao.UserDAO; import com.commafeed.backend.dao.UserRoleDAO; @@ -272,9 +273,11 @@ public class AdminREST extends AbstractResourceREST { @Path("/cleanup/findDuplicateFeeds") @GET @ApiOperation(value = "Find duplicate feeds") - public Response findDuplicateFeeds(@QueryParam("page") int page, - @QueryParam("limit") int limit, @QueryParam("minCount") long minCount) { - List list = feedDAO.findDuplicates(limit * page, limit, minCount); + public Response findDuplicateFeeds(@QueryParam("mode") DuplicateMode mode, + @QueryParam("page") int page, @QueryParam("limit") int limit, + @QueryParam("minCount") long minCount) { + List list = feedDAO.findDuplicates(mode, limit * page, + limit, minCount); return Response.ok(list).build(); } @@ -303,15 +306,4 @@ public class AdminREST extends AbstractResourceREST { cleaner.mergeFeeds(into, feeds); return Response.ok().build(); } - - @Path("/cleanup/automerge") - @GET - @ApiOperation(value = "Automatically merge feeds", notes = "Merge feeds together") - public Response autoMergeFeeds() { - Map map = Maps.newHashMap(); - map.put("merged feeds", - cleaner.cleanDuplicateFeeds()); - return Response.ok(map).build(); - } - } diff --git a/src/main/webapp/js/controllers.js b/src/main/webapp/js/controllers.js index 36fe35d0..bd8bc9a9 100644 --- a/src/main/webapp/js/controllers.js +++ b/src/main/webapp/js/controllers.js @@ -1213,9 +1213,11 @@ module.controller('ManageDuplicateFeedsCtrl', [ $scope.limit = 10; $scope.page = 0; $scope.minCount = 1; + $scope.mode = 'NORMALIZED_URL'; $scope.mergeData = {}; $scope.refreshData = function() { AdminCleanupService.findDuplicateFeeds({ + mode: $scope.mode, limit : $scope.limit, page : $scope.page, minCount: $scope.minCount @@ -1342,6 +1344,9 @@ function($scope, $location, $state, AdminSettingsService) { $scope.toUsers = function() { $state.transitionTo('admin.userlist'); }; + $scope.toCleanup = function() { + $state.transitionTo('admin.duplicate_feeds'); + }; }]); module.controller('HelpController', [ '$scope', 'CategoryService', diff --git a/src/main/webapp/js/main.js b/src/main/webapp/js/main.js index 2f63fbf4..7d6a1b64 100644 --- a/src/main/webapp/js/main.js +++ b/src/main/webapp/js/main.js @@ -92,7 +92,7 @@ app.config([ '$routeProvider', '$stateProvider', '$urlRouterProvider', '$httpPro controller : 'ManageUserCtrl' }); $stateProvider.state('admin.duplicate_feeds', { - url : '/feeds/duplicates/', + url : '/feeds/duplicates', templateUrl : 'templates/admin.duplicate_feeds.html', controller : 'ManageDuplicateFeedsCtrl' }); diff --git a/src/main/webapp/templates/admin.duplicate_feeds.html b/src/main/webapp/templates/admin.duplicate_feeds.html index 72555ab3..c42ddae9 100644 --- a/src/main/webapp/templates/admin.duplicate_feeds.html +++ b/src/main/webapp/templates/admin.duplicate_feeds.html @@ -1,10 +1,21 @@
+
Limit Page Min. count - - +
+
+ Mode + + + +
+ diff --git a/src/main/webapp/templates/admin.settings.html b/src/main/webapp/templates/admin.settings.html index 95484428..53f69ff9 100644 --- a/src/main/webapp/templates/admin.settings.html +++ b/src/main/webapp/templates/admin.settings.html @@ -5,6 +5,10 @@ Manage users + - + + Cleanup feeds +