Add support for AI crawl bots

src/constants.js

@@ -1,6 +1,7 @@
 // NOTE: this list must be up-to-date with browsers listed in
 // test/acceptance/useragentstrings.yml
 export const BROWSER_ALIASES_MAP = {
+  'Applebot-Extended': 'applebot_extended',
   AmazonBot: 'amazonbot',
   'Amazon Silk': 'amazon_silk',
   'Android Browser': 'android',
@@ -9,15 +10,21 @@ export const BROWSER_ALIASES_MAP = {
   BingCrawler: 'bingcrawler',
   BlackBerry: 'blackberry',
   Chrome: 'chrome',
+  'ChatGPT-User': 'chatgpt_user',
+  ClaudeBot: 'claudebot',
   Chromium: 'chromium',
+  Diffbot: 'diffbot',
   DuckDuckBot: 'duckduckbot',
   Electron: 'electron',
   Epiphany: 'epiphany',
+  FacebookBot: 'facebookbot',
   Firefox: 'firefox',
   Focus: 'focus',
   Generic: 'generic',
   'Google Search': 'google_search',
+  'Google-Extended': 'google_extended',
   Googlebot: 'googlebot',
+  GPTBot: 'gptbot',
   'Internet Explorer': 'ie',
   InternetArchiveCrawler: 'internetarchivecrawler',
   'K-Meleon': 'k_meleon',
@@ -26,11 +33,14 @@ export const BROWSER_ALIASES_MAP = {
   'Microsoft Edge': 'edge',
   'MZ Browser': 'mz',
   'NAVER Whale Browser': 'naver',
+  'OAI-SearchBot': 'oai_searchbot',
+  Omgilibot: 'omgilibot',
   Opera: 'opera',
   'Opera Coast': 'opera_coast',
   'Pale Moon': 'pale_moon',
   PhantomJS: 'phantomjs',
   PingdomBot: 'pingdombot',
+  PerplexityBot: 'perplexitybot',
   Puffin: 'puffin',
   QQ: 'qq',
   QQLite: 'qqlite',
@@ -50,9 +60,11 @@ export const BROWSER_ALIASES_MAP = {
   YahooSlurp: 'yahooslurp',
   'Yandex Browser': 'yandex',
   YandexBot: 'yandexbot',
+  YouBot: 'youbot',
 };
 
 export const BROWSER_MAP = {
+  applebot_extended: 'Applebot-Extended',
   amazonbot: 'AmazonBot',
   amazon_silk: 'Amazon Silk',
   android: 'Android Browser',
@@ -61,16 +73,22 @@ export const BROWSER_MAP = {
   bingcrawler: 'BingCrawler',
   blackberry: 'BlackBerry',
   chrome: 'Chrome',
+  chatgpt_user: 'ChatGPT-User',
+  claudebot: 'ClaudeBot',
   chromium: 'Chromium',
+  diffbot: 'Diffbot',
   duckduckbot: 'DuckDuckBot',
   edge: 'Microsoft Edge',
   electron: 'Electron',
   epiphany: 'Epiphany',
+  facebookbot: 'FacebookBot',
   firefox: 'Firefox',
   focus: 'Focus',
   generic: 'Generic',
+  google_extended: 'Google-Extended',
   google_search: 'Google Search',
   googlebot: 'Googlebot',
+  gptbot: 'GPTBot',
   ie: 'Internet Explorer',
   internetarchivecrawler: 'InternetArchiveCrawler',
   k_meleon: 'K-Meleon',
@@ -83,6 +101,9 @@ export const BROWSER_MAP = {
   pale_moon: 'Pale Moon',
   phantomjs: 'PhantomJS',
   pingdombot: 'PingdomBot',
+  perplexitybot: 'PerplexityBot',
+  oai_searchbot: 'OAI-SearchBot',
+  omgilibot: 'Omgilibot',
   puffin: 'Puffin',
   qq: 'QQ Browser',
   qqlite: 'QQ Browser Lite',
@@ -102,6 +123,7 @@ export const BROWSER_MAP = {
   yahooslurp: 'YahooSlurp',
   yandex: 'Yandex Browser',
   yandexbot: 'YandexBot',
+  youbot: 'YouBot',
 };
 
 export const PLATFORMS_MAP = {

src/parser-browsers.js

@@ -28,6 +28,193 @@ import Utils from './utils.js';
 const commonVersionIdentifier = /version\/(\d+(\.?_?\d+)+)/i;
 
 const browsersList = [
+  /* GPTBot */
+  {
+    test: [/gptbot/i],
+    describe(ua) {
+      const browser = {
+        name: 'GPTBot',
+      };
+      const version = Utils.getFirstMatch(/gptbot\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* ChatGPT-User */
+  {
+    test: [/chatgpt-user/i],
+    describe(ua) {
+      const browser = {
+        name: 'ChatGPT-User',
+      };
+      const version = Utils.getFirstMatch(/chatgpt-user\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* OAI-SearchBot */
+  {
+    test: [/oai-searchbot/i],
+    describe(ua) {
+      const browser = {
+        name: 'OAI-SearchBot',
+      };
+      const version = Utils.getFirstMatch(/oai-searchbot\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* Google-Extended */
+  {
+    test: [/google-extended/i],
+    describe(ua) {
+      const browser = {
+        name: 'Google-Extended',
+      };
+      const version = Utils.getFirstMatch(/google-extended\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* Applebot-Extended */
+  {
+    test: [/applebot-extended/i],
+    describe(ua) {
+      const browser = {
+        name: 'Applebot-Extended',
+      };
+      const version = Utils.getFirstMatch(/applebot-extended\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* ClaudeBot */
+  {
+    test: [/claudebot/i, /claude-web/i],
+    describe(ua) {
+      const browser = {
+        name: 'ClaudeBot',
+      };
+      const version = Utils.getFirstMatch(/(?:claudebot|claude-web)\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* Omgilibot */
+  {
+    test: [/omgilibot/i],
+    describe(ua) {
+      const browser = {
+        name: 'Omgilibot',
+      };
+      const version = Utils.getFirstMatch(/omgilibot\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* FacebookBot */
+  {
+    test: [/facebookbot/i],
+    describe(ua) {
+      const browser = {
+        name: 'FacebookBot',
+      };
+      const version = Utils.getFirstMatch(/facebookbot\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* Diffbot */
+  {
+    test: [/diffbot/i],
+    describe(ua) {
+      const browser = {
+        name: 'Diffbot',
+      };
+      const version = Utils.getFirstMatch(/diffbot\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* PerplexityBot */
+  {
+    test: [/perplexitybot/i],
+    describe(ua) {
+      const browser = {
+        name: 'PerplexityBot',
+      };
+      const version = Utils.getFirstMatch(/perplexitybot\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
+  /* YouBot */
+  {
+    test: [/youbot/i],
+    describe(ua) {
+      const browser = {
+        name: 'YouBot',
+      };
+      const version = Utils.getFirstMatch(/youbot\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
   /* Googlebot */
   {
     test: [/googlebot/i],