wip

src/constants.js

@@ -1,7 +1,6 @@
 // NOTE: this list must be up-to-date with browsers listed in
 // test/acceptance/useragentstrings.yml
 export const BROWSER_ALIASES_MAP = {
-  'Applebot-Extended': 'applebot_extended',
   AmazonBot: 'amazonbot',
   'Amazon Silk': 'amazon_silk',
   'Android Browser': 'android',
@@ -17,12 +16,10 @@ export const BROWSER_ALIASES_MAP = {
   DuckDuckBot: 'duckduckbot',
   Electron: 'electron',
   Epiphany: 'epiphany',
-  FacebookBot: 'facebookbot',
   Firefox: 'firefox',
   Focus: 'focus',
   Generic: 'generic',
   'Google Search': 'google_search',
-  'Google-Extended': 'google_extended',
   Googlebot: 'googlebot',
   GPTBot: 'gptbot',
   'Internet Explorer': 'ie',
@@ -30,6 +27,7 @@ export const BROWSER_ALIASES_MAP = {
   'K-Meleon': 'k_meleon',
   Maxthon: 'maxthon',
   MetaWebCrawler: 'metawebcrawler',
+  'Meta-WebIndexer': 'meta_webindexer',
   'Microsoft Edge': 'edge',
   'MZ Browser': 'mz',
   'NAVER Whale Browser': 'naver',
@@ -64,7 +62,6 @@ export const BROWSER_ALIASES_MAP = {
 };
 
 export const BROWSER_MAP = {
-  applebot_extended: 'Applebot-Extended',
   amazonbot: 'AmazonBot',
   amazon_silk: 'Amazon Silk',
   android: 'Android Browser',
@@ -81,11 +78,9 @@ export const BROWSER_MAP = {
   edge: 'Microsoft Edge',
   electron: 'Electron',
   epiphany: 'Epiphany',
-  facebookbot: 'FacebookBot',
   firefox: 'Firefox',
   focus: 'Focus',
   generic: 'Generic',
-  google_extended: 'Google-Extended',
   google_search: 'Google Search',
   googlebot: 'Googlebot',
   gptbot: 'GPTBot',
@@ -94,6 +89,7 @@ export const BROWSER_MAP = {
   k_meleon: 'K-Meleon',
   maxthon: 'Maxthon',
   metawebcrawler: 'MetaWebCrawler',
+  meta_webindexer: 'Meta-WebIndexer',
   mz: 'MZ Browser',
   naver: 'NAVER Whale Browser',
   opera: 'Opera',

src/parser-browsers.js

@@ -79,40 +79,6 @@ const browsersList = [
     },
   },
 
-  /* Google-Extended */
-  {
-    test: [/google-extended/i],
-    describe(ua) {
-      const browser = {
-        name: 'Google-Extended',
-      };
-      const version = Utils.getFirstMatch(/google-extended\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
-
-      if (version) {
-        browser.version = version;
-      }
-
-      return browser;
-    },
-  },
-
-  /* Applebot-Extended */
-  {
-    test: [/applebot-extended/i],
-    describe(ua) {
-      const browser = {
-        name: 'Applebot-Extended',
-      };
-      const version = Utils.getFirstMatch(/applebot-extended\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
-
-      if (version) {
-        browser.version = version;
-      }
-
-      return browser;
-    },
-  },
-
   /* ClaudeBot */
   {
     test: [/claudebot/i, /claude-web/i],
@@ -147,23 +113,6 @@ const browsersList = [
     },
   },
 
-  /* FacebookBot */
-  {
-    test: [/facebookbot/i],
-    describe(ua) {
-      const browser = {
-        name: 'FacebookBot',
-      };
-      const version = Utils.getFirstMatch(/facebookbot\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
-
-      if (version) {
-        browser.version = version;
-      }
-
-      return browser;
-    },
-  },
-
   /* Diffbot */
   {
     test: [/diffbot/i],
@@ -215,6 +164,23 @@ const browsersList = [
     },
   },
 
+  /* Meta-WebIndexer */
+  {
+    test: [/meta-webindexer/i],
+    describe(ua) {
+      const browser = {
+        name: 'Meta-WebIndexer',
+      };
+      const version = Utils.getFirstMatch(/meta-webindexer\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
+
+      if (version) {
+        browser.version = version;
+      }
+
+      return browser;
+    },
+  },
+
   /* Googlebot */
   {
     test: [/googlebot/i],

src/parser-platforms.js

@@ -29,6 +29,39 @@ export default [
     },
   },
 
+  /* GPTBot */
+  {
+    test: [/gptbot/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'OpenAI',
+      };
+    },
+  },
+
+  /* ChatGPT-User */
+  {
+    test: [/chatgpt-user/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'OpenAI',
+      };
+    },
+  },
+
+  /* OAI-SearchBot */
+  {
+    test: [/oai-searchbot/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'OpenAI',
+      };
+    },
+  },
+
   /* Baidu */
   {
     test: [/baiduspider/i],
@@ -62,6 +95,61 @@ export default [
     },
   },
 
+  /* ClaudeBot */
+  {
+    test: [/claudebot/i, /claude-web/i, /claude-user/i, /claude-searchbot/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'Anthropic',
+      };
+    },
+  },
+
+  /* Omgilibot */
+  {
+    test: [/omgilibot/i, /webzio-extended/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'Omgili',
+      };
+    },
+  },
+
+  /* Diffbot */
+  {
+    test: [/diffbot/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'Diffbot',
+      };
+    },
+  },
+
+  /* PerplexityBot */
+  {
+    test: [/perplexitybot/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'Perplexity AI',
+      };
+    },
+  },
+
+  /* YouBot */
+  {
+    test: [/youbot/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'You.com',
+      };
+    },
+  },
+
   /* Internet Archive Crawler */
   {
     test: [/ia_archiver/i],
@@ -73,6 +161,17 @@ export default [
     },
   },
 
+  /* Meta-WebIndexer */
+  {
+    test: [/meta-webindexer/i],
+    describe() {
+      return {
+        type: PLATFORMS_MAP.bot,
+        vendor: 'Meta',
+      };
+    },
+  },
+
   /* Meta Web Crawler */
   {
     test: [/facebookexternalhit/i, /facebookcatalog/i],