Add support for AI crawl bots

test/acceptance/useragentstrings.yml

@@ -2549,6 +2549,141 @@
           vendor: "Google"
         engine:
           name: "Blink"
+  GPTBot:
+    -
+      ua: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.1; +https://openai.com/gptbot)"
+      spec:
+        browser:
+          name: "GPTBot"
+          version: "1.1"
+        os: {}
+        platform:
+          type: "bot"
+          vendor: "OpenAI"
+        engine: {}
+  ChatGPT-User:
+    -
+      ua: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ChatGPT-User/1.0; +https://openai.com/bot)"
+      spec:
+        browser:
+          name: "ChatGPT-User"
+          version: "1.0"
+        os: {}
+        platform:
+          type: "bot"
+          vendor: "OpenAI"
+        engine: {}
+  OAI-SearchBot:
+    -
+      ua: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot)"
+      spec:
+        browser:
+          name: "OAI-SearchBot"
+          version: "1.0"
+        os: {}
+        platform:
+          type: "bot"
+          vendor: "OpenAI"
+        engine: {}
+  Google-Extended:
+    -
+      ua: "Mozilla/5.0 (compatible; Google-Extended; +https://developers.google.com/search/help/google-extended)"
+      spec:
+        browser:
+          name: "Google-Extended"
+        os: {}
+        platform:
+          type: "bot"
+          vendor: "Google"
+        engine: {}
+  Applebot-Extended:
+    -
+      ua: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15 (compatible; Applebot-Extended/1.0; +https://support.apple.com/applebot)"
+      spec:
+        browser:
+          name: "Applebot-Extended"
+          version: "1.0"
+        os:
+          name: "macOS"
+          version: "10.15.7"
+          versionName: "Catalina"
+        platform:
+          type: "bot"
+          vendor: "Apple"
+        engine:
+          name: "WebKit"
+  ClaudeBot:
+    -
+      ua: "Mozilla/5.0 (compatible; ClaudeBot/1.0; +https://www.anthropic.com/claudebot)"
+      spec:
+        browser:
+          name: "ClaudeBot"
+          version: "1.0"
+        os: {}
+        platform:
+          type: "bot"
+          vendor: "Anthropic"
+        engine: {}
+  Omgilibot:
+    -
+      ua: "Mozilla/5.0 (compatible; Omgilibot/1.0; +https://www.omgili.com)"
+      spec:
+        browser:
+          name: "Omgilibot"
+          version: "1.0"
+        os: {}
+        platform:
+          type: "bot"
+          vendor: "Omgili"
+        engine: {}
+  FacebookBot:
+    -
+      ua: "Mozilla/5.0 (compatible; FacebookBot/1.0; +https://developers.facebook.com/docs/sharing/best-practices#crawl)"
+      spec:
+        browser:
+          name: "FacebookBot"
+          version: "1.0"
+        os: {}
+        platform:
+          type: "bot"
+          vendor: "Meta"
+        engine: {}
+  Diffbot:
+    -
+      ua: "Mozilla/5.0 (compatible; Diffbot/3.0; +http://www.diffbot.com)"
+      spec:
+        browser:
+          name: "Diffbot"
+          version: "3.0"
+        os: {}
+        platform:
+          type: "bot"
+          vendor: "Diffbot"
+        engine: {}
+  PerplexityBot:
+    -
+      ua: "Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://www.perplexity.ai/bot)"
+      spec:
+        browser:
+          name: "PerplexityBot"
+          version: "1.0"
+        os: {}
+        platform:
+          type: "bot"
+          vendor: "Perplexity AI"
+        engine: {}
+  YouBot:
+    -
+      ua: "Mozilla/5.0 (compatible; YouBot/1.0; +https://you.com/bot)"
+      spec:
+        browser:
+          name: "YouBot"
+          version: "1.0"
+        os: {}
+        platform:
+          type: "bot"
+          vendor: "You.com"
+        engine: {}
   AmazonBot:
      -
        ua: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)"