From 9156b8b6d0cfd5365658ef89da6cf2ce98773b98 Mon Sep 17 00:00:00 2001
From: Athou <jeremiepanzer@gmail.com>
Date: Tue, 13 Jun 2023 10:47:39 +0200
Subject: [PATCH] add a setting to hide commafeed from search engines

---
 commafeed-server/config.dev.yml               |  3 +++
 commafeed-server/config.yml.example           |  3 +++
 .../com/commafeed/CommaFeedApplication.java   |  6 ++++++
 .../com/commafeed/CommaFeedConfiguration.java |  4 ++++
 .../servlet/RobotsTxtDisallowAllServlet.java  | 20 +++++++++++++++++++
 .../src/test/resources/config.test.yml        |  3 +++
 6 files changed, 39 insertions(+)
 create mode 100644 commafeed-server/src/main/java/com/commafeed/frontend/servlet/RobotsTxtDisallowAllServlet.java

diff --git a/commafeed-server/config.dev.yml b/commafeed-server/config.dev.yml
index 64a4a87c..c1d0c8be 100644
--- a/commafeed-server/config.dev.yml
+++ b/commafeed-server/config.dev.yml
@@ -3,6 +3,9 @@
 app:
   # url used to access commafeed
   publicUrl: http://localhost:8082/
+
+  # whether to expose a robots.txt file that disallows web crawlers and search engine indexers
+  hideFromWebCrawlers: true
   
   # whether to allow user registrations
   allowRegistrations: true
diff --git a/commafeed-server/config.yml.example b/commafeed-server/config.yml.example
index 3030e90f..ad02695b 100644
--- a/commafeed-server/config.yml.example
+++ b/commafeed-server/config.yml.example
@@ -3,6 +3,9 @@
 app:
   # url used to access commafeed
   publicUrl: http://localhost:8082/
+
+  # whether to expose a robots.txt file that disallows web crawlers and search engine indexers
+  hideFromWebCrawlers: true
   
   # whether to allow user registrations
   allowRegistrations: false
diff --git a/commafeed-server/src/main/java/com/commafeed/CommaFeedApplication.java b/commafeed-server/src/main/java/com/commafeed/CommaFeedApplication.java
index 9ba199fb..3fdbd2cd 100644
--- a/commafeed-server/src/main/java/com/commafeed/CommaFeedApplication.java
+++ b/commafeed-server/src/main/java/com/commafeed/CommaFeedApplication.java
@@ -47,6 +47,7 @@ import com.commafeed.frontend.servlet.CustomCssServlet;
 import com.commafeed.frontend.servlet.CustomJsServlet;
 import com.commafeed.frontend.servlet.LogoutServlet;
 import com.commafeed.frontend.servlet.NextUnreadServlet;
+import com.commafeed.frontend.servlet.RobotsTxtDisallowAllServlet;
 import com.commafeed.frontend.session.SessionHelperFactoryProvider;
 import com.commafeed.frontend.ws.WebSocketConfigurator;
 import com.commafeed.frontend.ws.WebSocketEndpoint;
@@ -169,6 +170,11 @@ public class CommaFeedApplication extends Application<CommaFeedConfiguration> {
 		environment.servlets().addServlet("customCss", injector.getInstance(CustomCssServlet.class)).addMapping("/custom_css.css");
 		environment.servlets().addServlet("customJs", injector.getInstance(CustomJsServlet.class)).addMapping("/custom_js.js");
 		environment.servlets().addServlet("analytics.js", injector.getInstance(AnalyticsServlet.class)).addMapping("/analytics.js");
+		if (Boolean.TRUE.equals(config.getApplicationSettings().getHideFromWebCrawlers())) {
+			environment.servlets()
+					.addServlet("robots.txt", injector.getInstance(RobotsTxtDisallowAllServlet.class))
+					.addMapping("/robots.txt");
+		}
 
 		// WebSocket endpoint
 		ServerEndpointConfig serverEndpointConfig = ServerEndpointConfig.Builder.create(WebSocketEndpoint.class, "/ws")
diff --git a/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java b/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java
index 0532f626..229338e0 100644
--- a/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java
+++ b/commafeed-server/src/main/java/com/commafeed/CommaFeedConfiguration.java
@@ -65,6 +65,10 @@ public class CommaFeedConfiguration extends Configuration {
 		@Valid
 		private String publicUrl;
 
+		@NotNull
+		@Valid
+		private Boolean hideFromWebCrawlers = true;
+
 		@NotNull
 		@Valid
 		private Boolean allowRegistrations;
diff --git a/commafeed-server/src/main/java/com/commafeed/frontend/servlet/RobotsTxtDisallowAllServlet.java b/commafeed-server/src/main/java/com/commafeed/frontend/servlet/RobotsTxtDisallowAllServlet.java
new file mode 100644
index 00000000..3ec89703
--- /dev/null
+++ b/commafeed-server/src/main/java/com/commafeed/frontend/servlet/RobotsTxtDisallowAllServlet.java
@@ -0,0 +1,20 @@
+package com.commafeed.frontend.servlet;
+
+import java.io.IOException;
+
+import javax.inject.Singleton;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+@Singleton
+public class RobotsTxtDisallowAllServlet extends HttpServlet {
+
+	@Override
+	protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException {
+		resp.setContentType("text/plain");
+		resp.getWriter().write("User-agent: *");
+		resp.getWriter().write("\n");
+		resp.getWriter().write("Disallow: /");
+	}
+}
diff --git a/commafeed-server/src/test/resources/config.test.yml b/commafeed-server/src/test/resources/config.test.yml
index 3ada8e1b..c772896a 100644
--- a/commafeed-server/src/test/resources/config.test.yml
+++ b/commafeed-server/src/test/resources/config.test.yml
@@ -3,6 +3,9 @@
 app:
   # url used to access commafeed
   publicUrl: http://localhost:8082/
+
+  # whether to expose a robots.txt file that disallows web crawlers and search engine indexers
+  hideFromWebCrawlers: true
   
   # whether to allow user registrations
   allowRegistrations: true