feat: add support for AI crawl bots (#577)

2026-09-23 12:34:58 +00:00 · 2025-11-22 18:05:08 +02:00
parent 771dfb2dfe
commit f7d2c0693c
4 changed files with 603 additions and 8 deletions
--- a/src/parser-browsers.js
+++ b/src/parser-browsers.js
@@ -28,6 +28,227 @@ import Utils from './utils.js';
 const commonVersionIdentifier = /version\/(\d+(\.?_?\d+)+)/i;

 const browsersList = [
+  /* GPTBot */
+  {
+    test: [/gptbot/i],
+    describe(ua) {
+      const browser = {
+        name: 'GPTBot',
+      };
+      const version = Utils.getFirstMatch(/gptbot\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* ChatGPT-User */
+  {
+    test: [/chatgpt-user/i],
+    describe(ua) {
+      const browser = {
+        name: 'ChatGPT-User',
+      };
+      const version = Utils.getFirstMatch(/chatgpt-user\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* OAI-SearchBot */
+  {
+    test: [/oai-searchbot/i],
+    describe(ua) {
+      const browser = {
+        name: 'OAI-SearchBot',
+      };
+      const version = Utils.getFirstMatch(/oai-searchbot\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* ClaudeBot */
+  {
+    test: [/claudebot/i, /claude-web/i, /claude-user/i, /claude-searchbot/i],
+    describe(ua) {
+      const browser = {
+        name: 'ClaudeBot',
+      };
+      const version = Utils.getFirstMatch(/(?:claudebot|claude-web|claude-user|claude-searchbot)\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* Omgilibot */
+  {
+    test: [/omgilibot/i, /webzio-extended/i],
+    describe(ua) {
+      const browser = {
+        name: 'Omgilibot',
+      };
+      const version = Utils.getFirstMatch(/(?:omgilibot|webzio-extended)\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* Diffbot */
+  {
+    test: [/diffbot/i],
+    describe(ua) {
+      const browser = {
+        name: 'Diffbot',
+      };
+      const version = Utils.getFirstMatch(/diffbot\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* PerplexityBot */
+  {
+    test: [/perplexitybot/i],
+    describe(ua) {
+      const browser = {
+        name: 'PerplexityBot',
+      };
+      const version = Utils.getFirstMatch(/perplexitybot\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* Perplexity-User */
+  {
+    test: [/perplexity-user/i],
+    describe(ua) {
+      const browser = {
+        name: 'Perplexity-User',
+      };
+      const version = Utils.getFirstMatch(/perplexity-user\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* YouBot */
+  {
+    test: [/youbot/i],
+    describe(ua) {
+      const browser = {
+        name: 'YouBot',
+      };
+      const version = Utils.getFirstMatch(/youbot\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* Meta-WebIndexer */
+  {
+    test: [/meta-webindexer/i],
+    describe(ua) {
+      const browser = {
+        name: 'Meta-WebIndexer',
+      };
+      const version = Utils.getFirstMatch(/meta-webindexer\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* Meta-ExternalAds */
+  {
+    test: [/meta-externalads/i],
+    describe(ua) {
+      const browser = {
+        name: 'Meta-ExternalAds',
+      };
+      const version = Utils.getFirstMatch(/meta-externalads\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* Meta-ExternalAgent */
+  {
+    test: [/meta-externalagent/i],
+    describe(ua) {
+      const browser = {
+        name: 'Meta-ExternalAgent',
+      };
+      const version = Utils.getFirstMatch(/meta-externalagent\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* Meta-ExternalFetcher */
+  {
+    test: [/meta-externalfetcher/i],
+    describe(ua) {
+      const browser = {
+        name: 'Meta-ExternalFetcher',
+      };
+      const version = Utils.getFirstMatch(/meta-externalfetcher\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
  /* Googlebot */
  {
    test: [/googlebot/i],
@@ -130,12 +351,12 @@ const browsersList = [
    },
  },

-  /* MetaWebCrawler */
+  /* FacebookExternalHit */
  {
    test: [/facebookexternalhit/i, /facebookcatalog/i],
    describe() {
      return {
-        name: 'MetaWebCrawler',
+        name: 'FacebookExternalHit',
      };
    },
  },