Archives/lancedikson_bowser
1
0
Fork 0
mirror of https://github.com/lancedikson/bowser synced 2026-09-22 12:05:23 +00:00
Code Issues Releases Wiki Activity

feat: add recognition for Google and Bing search bot user agents

Browse Source 
Add support for detecting additional Google and Bing crawlers that were
not previously recognized by bowser:

Google bots:
- AdsBot-Google (including AdsBot-Google-Mobile)
- Mediapartners-Google (Google AdSense crawler)
- APIs-Google
- FeedFetcher-Google
- Google-InspectionTool
- GoogleOther

Bing bots:
- BingPreview
- AdIdxBot (adidxbot)

Each new bot is added to parser-browsers.js, parser-platforms.js,
constants.js (BROWSER_ALIASES_MAP and BROWSER_MAP), and acceptance
test fixtures in useragentstrings.yml.

Co-authored-by: naorpeled <6171622+naorpeled@users.noreply.github.com>
This commit is contained in:
copilot-swe-agent[bot]
2026-03-14 14:28:00 +00:00
parent e2f834bb98
commit 4bd1024901
4 changed files with 387 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/constants.js
View File

@@ -1,12 +1,16 @@
// NOTE: this list must be up-to-date with browsers listed in
// test/acceptance/useragentstrings.yml
export const BROWSER_ALIASES_MAP = {
AdIdxBot: 'adidxbot',
AmazonBot: 'amazonbot',
'Amazon Silk': 'amazon_silk',
'Android Browser': 'android',
'AdsBot-Google': 'adsbot_google',
'APIs-Google': 'apis_google',
BaiduSpider: 'baiduspider',
Bada: 'bada',
BingCrawler: 'bingcrawler',
BingPreview: 'bingpreview',
Brave: 'brave',
BlackBerry: 'blackberry',
'ChatGPT-User': 'chatgpt_user',
@@ -19,10 +23,13 @@ export const BROWSER_ALIASES_MAP = {
Electron: 'electron',
Epiphany: 'epiphany',
FacebookExternalHit: 'facebookexternalhit',
'FeedFetcher-Google': 'feedfetcher_google',
Firefox: 'firefox',
Focus: 'focus',
Generic: 'generic',
'Google-InspectionTool': 'google_inspection_tool',
'Google Search': 'google_search',
GoogleOther: 'google_other',
Googlebot: 'googlebot',
GPTBot: 'gptbot',
'Internet Explorer': 'ie',
@@ -31,6 +38,7 @@ export const BROWSER_ALIASES_MAP = {
LibreWolf: 'librewolf',
Linespider: 'linespider',
Maxthon: 'maxthon',
'Mediapartners-Google': 'mediapartners_google',
'Meta-ExternalAds': 'meta_externalads',
'Meta-ExternalAgent': 'meta_externalagent',
'Meta-ExternalFetcher': 'meta_externalfetcher',
@@ -72,12 +80,16 @@ export const BROWSER_ALIASES_MAP = {
};
export const BROWSER_MAP = {
adidxbot: 'AdIdxBot',
adsbot_google: 'AdsBot-Google',
amazonbot: 'AmazonBot',
amazon_silk: 'Amazon Silk',
android: 'Android Browser',
apis_google: 'APIs-Google',
baiduspider: 'BaiduSpider',
bada: 'Bada',
bingcrawler: 'BingCrawler',
bingpreview: 'BingPreview',
blackberry: 'BlackBerry',
brave: 'Brave',
chatgpt_user: 'ChatGPT-User',
@@ -91,9 +103,12 @@ export const BROWSER_MAP = {
electron: 'Electron',
epiphany: 'Epiphany',
facebookexternalhit: 'FacebookExternalHit',
feedfetcher_google: 'FeedFetcher-Google',
firefox: 'Firefox',
focus: 'Focus',
generic: 'Generic',
google_inspection_tool: 'Google-InspectionTool',
google_other: 'GoogleOther',
google_search: 'Google Search',
googlebot: 'Googlebot',
gptbot: 'GPTBot',
@@ -103,6 +118,7 @@ export const BROWSER_MAP = {
librewolf: 'LibreWolf',
linespider: 'Linespider',
maxthon: 'Maxthon',
mediapartners_google: 'Mediapartners-Google',
meta_externalads: 'Meta-ExternalAds',
meta_externalagent: 'Meta-ExternalAgent',
meta_externalfetcher: 'Meta-ExternalFetcher',

136
src/parser-browsers.js
View File

@@ -249,6 +249,108 @@ const browsersList = [
},
},
/* AdsBot-Google */
{
test: [/adsbot-google/i],
describe(ua) {
const browser = {
name: 'AdsBot-Google',
};
const version = Utils.getFirstMatch(/adsbot-google(?:-mobile)?\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
if (version) {
browser.version = version;
}
return browser;
},
},
/* Mediapartners-Google */
{
test: [/mediapartners-google/i],
describe(ua) {
const browser = {
name: 'Mediapartners-Google',
};
const version = Utils.getFirstMatch(/mediapartners-google\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
if (version) {
browser.version = version;
}
return browser;
},
},
/* APIs-Google */
{
test: [/apis-google/i],
describe(ua) {
const browser = {
name: 'APIs-Google',
};
const version = Utils.getFirstMatch(/apis-google\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
if (version) {
browser.version = version;
}
return browser;
},
},
/* FeedFetcher-Google */
{
test: [/feedfetcher-google/i],
describe(ua) {
const browser = {
name: 'FeedFetcher-Google',
};
const version = Utils.getFirstMatch(/feedfetcher-google(?:-calendar)?\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
if (version) {
browser.version = version;
}
return browser;
},
},
/* Google-InspectionTool */
{
test: [/google-inspectiontool/i],
describe(ua) {
const browser = {
name: 'Google-InspectionTool',
};
const version = Utils.getFirstMatch(/google-inspectiontool\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
if (version) {
browser.version = version;
}
return browser;
},
},
/* GoogleOther */
{
test: [/googleother/i],
describe(ua) {
const browser = {
name: 'GoogleOther',
};
const version = Utils.getFirstMatch(/googleother\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
if (version) {
browser.version = version;
}
return browser;
},
},
/* Googlebot */
{
test: [/googlebot/i],
@@ -300,6 +402,40 @@ const browsersList = [
},
},
/* BingPreview */
{
test: [/bingpreview/i],
describe(ua) {
const browser = {
name: 'BingPreview',
};
const version = Utils.getFirstMatch(/bingpreview\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
if (version) {
browser.version = version;
}
return browser;
},
},
/* AdIdxBot */
{
test: [/adidxbot/i],
describe(ua) {
const browser = {
name: 'AdIdxBot',
};
const version = Utils.getFirstMatch(/adidxbot\/(\d+(\.\d+)+)/i, ua) || Utils.getFirstMatch(commonVersionIdentifier, ua);
if (version) {
browser.version = version;
}
return browser;
},
},
/* BingCrawler */
{
test: [/bingbot/i],

88
src/parser-platforms.js
View File

@@ -7,6 +7,72 @@ import { PLATFORMS_MAP } from './constants.js';
*/
export default [
/* AdsBot-Google */
{
test: [/adsbot-google/i],
describe() {
return {
type: PLATFORMS_MAP.bot,
vendor: 'Google',
};
},
},
/* Mediapartners-Google */
{
test: [/mediapartners-google/i],
describe() {
return {
type: PLATFORMS_MAP.bot,
vendor: 'Google',
};
},
},
/* APIs-Google */
{
test: [/apis-google/i],
describe() {
return {
type: PLATFORMS_MAP.bot,
vendor: 'Google',
};
},
},
/* FeedFetcher-Google */
{
test: [/feedfetcher-google/i],
describe() {
return {
type: PLATFORMS_MAP.bot,
vendor: 'Google',
};
},
},
/* Google-InspectionTool */
{
test: [/google-inspectiontool/i],
describe() {
return {
type: PLATFORMS_MAP.bot,
vendor: 'Google',
};
},
},
/* GoogleOther */
{
test: [/googleother/i],
describe() {
return {
type: PLATFORMS_MAP.bot,
vendor: 'Google',
};
},
},
/* Googlebot */
{
test: [/googlebot/i],
@@ -84,6 +150,28 @@ export default [
},
},
/* BingPreview */
{
test: [/bingpreview/i],
describe() {
return {
type: PLATFORMS_MAP.bot,
vendor: 'Bing',
};
},
},
/* AdIdxBot */
{
test: [/adidxbot/i],
describe() {
return {
type: PLATFORMS_MAP.bot,
vendor: 'Bing',
};
},
},
/* Bingbot */
{
test: [/bingbot/i],

147
test/acceptance/useragentstrings.yml
View File

@@ -2699,6 +2699,110 @@
vendor: "Google"
engine:
name: "Blink"
AdsBot-Google:
-
ua: "AdsBot-Google (+http://www.google.com/adsbot.html)"
spec:
browser:
name: "AdsBot-Google"
os: {}
platform:
type: "bot"
vendor: "Google"
engine: {}
-
ua: "Mozilla/5.0 (iPhone; CPU iPhone OS 14_7_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1 (compatible; AdsBot-Google-Mobile/3.0; +http://www.google.com/mobile/adsbot.html)"
spec:
browser:
name: "AdsBot-Google"
version: "3.0"
os:
name: "iOS"
version: "14.7.1"
platform:
type: "bot"
vendor: "Google"
engine:
name: "WebKit"
version: "605.1.15"
Mediapartners-Google:
-
ua: "Mediapartners-Google"
spec:
browser:
name: "Mediapartners-Google"
os: {}
platform:
type: "bot"
vendor: "Google"
engine: {}
-
ua: "Mozilla/5.0 (compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)"
spec:
browser:
name: "Mediapartners-Google"
version: "2.1"
os: {}
platform:
type: "bot"
vendor: "Google"
engine: {}
APIs-Google:
-
ua: "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)"
spec:
browser:
name: "APIs-Google"
os: {}
platform:
type: "bot"
vendor: "Google"
engine: {}
FeedFetcher-Google:
-
ua: "FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)"
spec:
browser:
name: "FeedFetcher-Google"
os: {}
platform:
type: "bot"
vendor: "Google"
engine: {}
Google-InspectionTool:
-
ua: "Mozilla/5.0 (compatible; Google-InspectionTool/1.0; +https://developers.google.com/search/docs/crawling-indexing/overview-google-crawlers)"
spec:
browser:
name: "Google-InspectionTool"
version: "1.0"
os: {}
platform:
type: "bot"
vendor: "Google"
engine: {}
GoogleOther:
-
ua: "GoogleOther"
spec:
browser:
name: "GoogleOther"
os: {}
platform:
type: "bot"
vendor: "Google"
engine: {}
-
ua: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GoogleOther) Chrome/108.0.0.0 Safari/537.36"
spec:
browser:
name: "GoogleOther"
os: {}
platform:
type: "bot"
vendor: "Google"
engine:
name: "Blink"
GPTBot:
-
ua: "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.1; +https://openai.com/gptbot)"
@@ -2960,6 +3064,49 @@
vendor: "Bing"
engine:
name: "Blink"
BingPreview:
-
ua: "Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; BingPreview/1.0b; +http://www.bing.com/bingpreview)"
spec:
browser:
name: "BingPreview"
version: "1.0"
os:
name: "Windows Phone"
version: "8.1"
platform:
type: "bot"
vendor: "Bing"
engine:
name: "Trident"
version: "7.0"
AdIdxBot:
-
ua: "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)"
spec:
browser:
name: "AdIdxBot"
version: "2.0"
os: {}
platform:
type: "bot"
vendor: "Bing"
engine: {}
-
ua: "Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)"
spec:
browser:
name: "AdIdxBot"
version: "2.0"
os:
name: "iOS"
version: "7.0"
platform:
type: "bot"
vendor: "Bing"
engine:
name: "WebKit"
version: "537.51.1"
BaiduSpider:
-
ua: "Baiduspider"