From 52270d50d9822222a92620f0c683abc3de1526da Mon Sep 17 00:00:00 2001 From: Athou Date: Mon, 8 Jul 2013 16:38:09 +0200 Subject: [PATCH] interface for deleting duplicates --- .../com/commafeed/backend/dao/FeedDAO.java | 37 +++++++++++++++++++ .../frontend/rest/resources/AdminREST.java | 24 +++++++++++- src/main/webapp/js/controllers.js | 28 ++++++++++++++ src/main/webapp/js/main.js | 5 +++ src/main/webapp/js/services.js | 20 ++++++++++ .../templates/admin.duplicate_feeds.html | 34 +++++++++++++++++ 6 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 src/main/webapp/templates/admin.duplicate_feeds.html diff --git a/src/main/java/com/commafeed/backend/dao/FeedDAO.java b/src/main/java/com/commafeed/backend/dao/FeedDAO.java index b6c9e046..8a08ec43 100644 --- a/src/main/java/com/commafeed/backend/dao/FeedDAO.java +++ b/src/main/java/com/commafeed/backend/dao/FeedDAO.java @@ -7,10 +7,15 @@ import javax.ejb.Stateless; import javax.persistence.Query; import javax.persistence.TypedQuery; import javax.persistence.criteria.CriteriaQuery; +import javax.persistence.criteria.Expression; import javax.persistence.criteria.JoinType; +import javax.persistence.criteria.Path; import javax.persistence.criteria.Predicate; import javax.persistence.criteria.Root; import javax.persistence.criteria.SetJoin; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang.StringUtils; @@ -26,6 +31,13 @@ import com.google.common.collect.Lists; @Stateless public class FeedDAO extends GenericDAO { + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class FeedCount { + public String normalizedUrlHash; + public List feeds; + } + private List getUpdatablePredicates(Root root, Date threshold) { @@ -123,4 +135,29 @@ public class FeedDAO extends GenericDAO { return deleted; } + + public List findDuplicates(int offset, int limit) { + CriteriaQuery query = builder.createQuery(String.class); + Root root = query.from(getType()); + + Path hashPath = root.get(Feed_.normalizedUrlHash); + Expression count = builder.count(hashPath); + + query.select(hashPath); + + query.groupBy(hashPath); + query.having(builder.greaterThan(count, 1l)); + + TypedQuery q = em.createQuery(query); + limit(q, offset, limit); + List normalizedUrlHashes = q.getResultList(); + + List result = Lists.newArrayList(); + for (String hash : normalizedUrlHashes) { + FeedCount fc = new FeedCount(); + fc.normalizedUrlHash = hash; + fc.feeds = findByField(Feed_.normalizedUrlHash, hash); + } + return result; + } } diff --git a/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java b/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java index a42d5e2f..680753f1 100644 --- a/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java +++ b/src/main/java/com/commafeed/frontend/rest/resources/AdminREST.java @@ -21,6 +21,7 @@ import com.commafeed.backend.DatabaseCleaner; import com.commafeed.backend.MetricsBean; import com.commafeed.backend.StartupBean; import com.commafeed.backend.dao.FeedDAO; +import com.commafeed.backend.dao.FeedDAO.FeedCount; import com.commafeed.backend.dao.UserDAO; import com.commafeed.backend.dao.UserRoleDAO; import com.commafeed.backend.feeds.FeedRefreshTaskGiver; @@ -38,8 +39,8 @@ import com.commafeed.frontend.SecurityCheck; import com.commafeed.frontend.model.UserModel; import com.commafeed.frontend.model.request.FeedMergeRequest; import com.commafeed.frontend.model.request.IDRequest; -import com.google.api.client.util.Lists; import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.wordnik.swagger.annotations.Api; @@ -268,17 +269,36 @@ public class AdminREST extends AbstractResourceREST { return Response.ok(map).build(); } + @Path("/cleanup/findDuplicateFeeds") + @GET + @ApiOperation(value = "Find duplicate feeds") + public Response findDuplicateFeeds(@QueryParam("page") int page, + @QueryParam("limit") int limit) { + List list = feedDAO.findDuplicates(limit * page, limit); + return Response.ok(list).build(); + } + @Path("/cleanup/merge") @POST @ApiOperation(value = "Merge feeds", notes = "Merge feeds together") - public Response mergeFeeds(@ApiParam(required = true) FeedMergeRequest request) { + public Response mergeFeeds( + @ApiParam(required = true) FeedMergeRequest request) { Feed into = feedDAO.findById(request.getIntoFeedId()); + if (into == null) { + return Response.status(Status.BAD_REQUEST) + .entity("'into feed' not found").build(); + } List feeds = Lists.newArrayList(); for (Long feedId : request.getFeedIds()) { Feed feed = feedDAO.findById(feedId); feeds.add(feed); } + + if (feeds.isEmpty()) { + return Response.status(Status.BAD_REQUEST) + .entity("'from feeds' empty").build(); + } cleaner.mergeFeeds(into, feeds); return Response.ok().build(); diff --git a/src/main/webapp/js/controllers.js b/src/main/webapp/js/controllers.js index 6882316c..6e60b64c 100644 --- a/src/main/webapp/js/controllers.js +++ b/src/main/webapp/js/controllers.js @@ -1204,6 +1204,34 @@ function($scope, $state, $stateParams, $dialog, AdminUsersService) { }; }]); +module.controller('ManageDuplicateFeedsCtrl', [ + '$scope', 'AdminCleanupService', + function($scope, AdminCleanupService) { + + $scope.limit = 10; + $scope.page = 0; + $scope.mergeData = {}; + $scope.refreshData = function() { + AdminCleanupService.findDuplicateFeeds({ + limit : $scope.limit, + page : $scope.page + }, function(data) { + $scope.counts = data; + }); + }; + + $scope.focus = function(count) { + $scope.current = count; + }; + + $scope.merge = function() { + AdminCleanupService.mergeFeeds($scope.mergeData, function() { + alert('done!'); + }); + }; + +}]); + module.controller('SettingsCtrl', ['$scope', '$location', 'SettingsService', 'AnalyticsService', 'ServerService', function($scope, $location, SettingsService, AnalyticsService, ServerService) { diff --git a/src/main/webapp/js/main.js b/src/main/webapp/js/main.js index cffbbf17..2f63fbf4 100644 --- a/src/main/webapp/js/main.js +++ b/src/main/webapp/js/main.js @@ -91,6 +91,11 @@ app.config([ '$routeProvider', '$stateProvider', '$urlRouterProvider', '$httpPro templateUrl : 'templates/admin.useredit.html', controller : 'ManageUserCtrl' }); + $stateProvider.state('admin.duplicate_feeds', { + url : '/feeds/duplicates/', + templateUrl : 'templates/admin.duplicate_feeds.html', + controller : 'ManageDuplicateFeedsCtrl' + }); $stateProvider.state('admin.settings', { url : '/settings', templateUrl : 'templates/admin.settings.html', diff --git a/src/main/webapp/js/services.js b/src/main/webapp/js/services.js index 9604b352..9fc82f0d 100644 --- a/src/main/webapp/js/services.js +++ b/src/main/webapp/js/services.js @@ -272,6 +272,26 @@ module.factory('AdminSettingsService', ['$resource', function($resource) { return res; }]); +module.factory('AdminCleanupService', ['$resource', function($resource) { + var actions = { + findDuplicateFeeds : { + method : 'GET', + isArray: true, + params : { + _method : 'findDuplicateFeeds' + } + }, + mergeFeeds : { + method : 'POST', + params : { + _method : 'merge' + } + }, + }; + var res = $resource('rest/admin/cleanup/:_method', {}, actions); + return res; +}]); + module.factory('ServerService', ['$resource', function($resource) { var res = $resource('rest/server/get'); return res; diff --git a/src/main/webapp/templates/admin.duplicate_feeds.html b/src/main/webapp/templates/admin.duplicate_feeds.html new file mode 100644 index 00000000..b0185e80 --- /dev/null +++ b/src/main/webapp/templates/admin.duplicate_feeds.html @@ -0,0 +1,34 @@ +
+ + Limit + Page + + + + + + + + + + + + + +
urlcount +
{{count.normalizedUrlHash}}{{count.feeds.length}}
+ +
+
+ Merge + + + Into + +
+ +
+
\ No newline at end of file