add a setting to hide commafeed from search engines

commafeed-server/config.dev.yml

@@ -3,6 +3,9 @@
 app:
   # url used to access commafeed
   publicUrl: http://localhost:8082/
+
+  # whether to expose a robots.txt file that disallows web crawlers and search engine indexers
+  hideFromWebCrawlers: true
   
   # whether to allow user registrations
   allowRegistrations: true

commafeed-server/config.yml.example

@@ -3,6 +3,9 @@
 app:
   # url used to access commafeed
   publicUrl: http://localhost:8082/
+
+  # whether to expose a robots.txt file that disallows web crawlers and search engine indexers
+  hideFromWebCrawlers: true
   
   # whether to allow user registrations
   allowRegistrations: false

commafeed-server/src/main/java/com/commafeed/CommaFeedApplication.java

@@ -47,6 +47,7 @@ import com.commafeed.frontend.servlet.CustomCssServlet;
 import com.commafeed.frontend.servlet.CustomJsServlet;
 import com.commafeed.frontend.servlet.LogoutServlet;
 import com.commafeed.frontend.servlet.NextUnreadServlet;
+import com.commafeed.frontend.servlet.RobotsTxtDisallowAllServlet;
 import com.commafeed.frontend.session.SessionHelperFactoryProvider;
 import com.commafeed.frontend.ws.WebSocketConfigurator;
 import com.commafeed.frontend.ws.WebSocketEndpoint;
@@ -169,6 +170,11 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
 		environment.servlets().addServlet("customCss", injector.getInstance(CustomCssServlet.class)).addMapping("/custom_css.css");
 		environment.servlets().addServlet("customJs", injector.getInstance(CustomJsServlet.class)).addMapping("/custom_js.js");
 		environment.servlets().addServlet("analytics.js", injector.getInstance(AnalyticsServlet.class)).addMapping("/analytics.js");
+		if (Boolean.TRUE.equals(config.getApplicationSettings().getHideFromWebCrawlers())) {
+			environment.servlets()
+					.addServlet("robots.txt", injector.getInstance(RobotsTxtDisallowAllServlet.class))
+					.addMapping("/robots.txt");
+		}
 
 		// WebSocket endpoint
 		ServerEndpointConfig serverEndpointConfig = ServerEndpointConfig.Builder.create(WebSocketEndpoint.class, "/ws")

commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java

@@ -65,6 +65,10 @@ public class CommaFeedConfiguration extends Configuration {
 		@Valid
 		private String publicUrl;
 
+		@NotNull
+		@Valid
+		private Boolean hideFromWebCrawlers = true;
+
 		@NotNull
 		@Valid
 		private Boolean allowRegistrations;

commafeed-server/src/main/java/com/commafeed/frontend/servlet/RobotsTxtDisallowAllServlet.java

@@ -0,0 +1,20 @@
+package com.commafeed.frontend.servlet;
+
+import java.io.IOException;
+
+import javax.inject.Singleton;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+@Singleton
+public class RobotsTxtDisallowAllServlet extends HttpServlet {
+
+	@Override
+	protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
+		resp.setContentType("text/plain");
+		resp.getWriter().write("User-agent: *");
+		resp.getWriter().write("\n");
+		resp.getWriter().write("Disallow: /");
+	}
+}

commafeed-server/src/test/resources/config.test.yml

@@ -3,6 +3,9 @@
 app:
   # url used to access commafeed
   publicUrl: http://localhost:8082/
+
+  # whether to expose a robots.txt file that disallows web crawlers and search engine indexers
+  hideFromWebCrawlers: true
   
   # whether to allow user registrations
   allowRegistrations: true