feat: add support for AI crawl bots (#577)

src/parser-platforms.js

@@ -29,6 +29,39 @@ export default [
     },
   },
 
+  /* GPTBot */
+  {
+    test: [/gptbot/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'OpenAI',
+      };
+    },
+  },
+
+  /* ChatGPT-User */
+  {
+    test: [/chatgpt-user/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'OpenAI',
+      };
+    },
+  },
+
+  /* OAI-SearchBot */
+  {
+    test: [/oai-searchbot/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'OpenAI',
+      };
+    },
+  },
+
   /* Baidu */
   {
     test: [/baiduspider/i],
@@ -62,6 +95,72 @@ export default [
     },
   },
 
+  /* ClaudeBot */
+  {
+    test: [/claudebot/i, /claude-web/i, /claude-user/i, /claude-searchbot/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'Anthropic',
+      };
+    },
+  },
+
+  /* Omgilibot */
+  {
+    test: [/omgilibot/i, /webzio-extended/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'Webz.io',
+      };
+    },
+  },
+
+  /* Diffbot */
+  {
+    test: [/diffbot/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'Diffbot',
+      };
+    },
+  },
+
+  /* PerplexityBot */
+  {
+    test: [/perplexitybot/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'Perplexity AI',
+      };
+    },
+  },
+
+  /* Perplexity-User */
+  {
+    test: [/perplexity-user/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'Perplexity AI',
+      };
+    },
+  },
+
+  /* YouBot */
+  {
+    test: [/youbot/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'You.com',
+      };
+    },
+  },
+
   /* Internet Archive Crawler */
   {
     test: [/ia_archiver/i],
@@ -73,6 +172,50 @@ export default [
     },
   },
 
+  /* Meta-WebIndexer */
+  {
+    test: [/meta-webindexer/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'Meta',
+      };
+    },
+  },
+
+  /* Meta-ExternalAds */
+  {
+    test: [/meta-externalads/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'Meta',
+      };
+    },
+  },
+
+  /* Meta-ExternalAgent */
+  {
+    test: [/meta-externalagent/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'Meta',
+      };
+    },
+  },
+
+  /* Meta-ExternalFetcher */
+  {
+    test: [/meta-externalfetcher/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'Meta',
+      };
+    },
+  },
+
   /* Meta Web Crawler */
   {
     test: [/facebookexternalhit/i, /facebookcatalog/i],