www-ssg/src/robots.njk

---
permalink: robots.txt
---
# __        ___           _     _
# \ \      / / |__   __ _| |_  (_)___  __      ___ __ ___  _ __   __ _
#  \ \ /\ / /| '_ \ / _` | __| | / __| \ \ /\ / / '__/ _ \| '_ \ / _` |
#   \ V  V / | | | | (_| | |_  | \__ \  \ V  V /| | | (_) | | | | (_| |
#    \_/\_/  |_| |_|\__,_|\__| |_|___/   \_/\_/ |_|  \___/|_| |_|\__, |
#                                                                |___/
#           _ _   _
# __      _(_) |_| |__     __ _
# \ \ /\ / / | __| '_ \   / _` |
#  \ V  V /| | |_| | | | | (_| |
#   \_/\_/ |_|\__|_| |_|  \__,_|
#
#                        _             __      _ _     _                   _
#   ___ ___  _   _ _ __ | |_ ___ _ __ / _| ___(_) |_  (_)___   _ __   ___ | |_
#  / __/ _ \| | | | '_ \| __/ _ \ '__| |_ / _ \ | __| | / __| | '_ \ / _ \| __|
# | (_| (_) | |_| | | | | ||  __/ |  |  _|  __/ | |_  | \__ \ | | | | (_) | |_
#  \___\___/ \__,_|_| |_|\__\___|_|  |_|  \___|_|\__| |_|___/ |_| |_|\___/ \__|
#
#           _           _     _ _     _       _ _ _            _           _
# __      _| |__   __ _| |_  (_) |_  (_)___  | (_) | _____    | |__  _   _| |_
# \ \ /\ / / '_ \ / _` | __| | | __| | / __| | | | |/ / _ \   | '_ \| | | | __|
#  \ V  V /| | | | (_| | |_  | | |_  | \__ \ | | |   <  __/_  | |_) | |_| | |_
#   \_/\_/ |_| |_|\__,_|\__| |_|\__| |_|___/ |_|_|_|\_\___( ) |_.__/ \__,_|\__|
#                                                         |/
#  _                     _ _                                             _
# | |__   _____      __ (_) |_  __      ____ _ ___   _ __ ___   __ _  __| | ___
# | '_ \ / _ \ \ /\ / / | | __| \ \ /\ / / _` / __| | '_ ` _ \ / _` |/ _` |/ _ \
# | | | | (_) \ V  V /  | | |_   \ V  V / (_| \__ \ | | | | | | (_| | (_| |  __/_
# |_| |_|\___/ \_/\_/   |_|\__|   \_/\_/ \__,_|___/ |_| |_| |_|\__,_|\__,_|\___(_)
#
#
# Thanks to the excellent Tracy Durnell, from whom this file was lifted :)
# https://tracydurnell.com/robots.txt
# https://tracydurnell.com/2025/06/02/generative-ai-and-the-business-borg-aesthetic/

# Google AI
User-agent: Google-Extended
Disallow: /

User-agent: Google-CloudVertexBot
Disallow: /

User-agent: Gemini-Deep-Research
Disallow: /

# Google

User-agent: GoogleOther
Disallow: /

User-agent: Googlebot-Image
Disallow: /

User-agent: Googlebot-Video
Disallow: /

User-agent: AdsBot-Google
Disallow: /

# Bing
User-agent: bingbot
Disallow: /

# Common Crawl
User-agent: CCBot
Disallow: /

# OpenAI
User-agent: ChatGPT-User
Disallow: /

User-agent: ChatGPT
Disallow: /

User-agent: ChatGPT Agent
Disallow: /

User-agent: GPTBot
Disallow: /

# Omgili
User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

# Facebook
User-agent: FacebookBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: meta-externalfetcher
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: meta-webindexer
Disallow: /

# Anthropic
User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-SearchBot
Disallow: /

User-agent: Claude-User
Disallow: /

# Apple AI
User-agent: Applebot-Extended
Disallow: /

# Other AI training known or suspected
User-agent: AddSearchBot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Ai2Bot-Dolma
Disallow: /

User-agent: aiHitBot
Disallow: /

User-agent: Andibot
Disallow: /

User-agent: Awario
Disallow: /

User-agent: bedrockbot
Disallow: /

User-agent: bigsur.ai
Disallow: /

User-agent: Brightbot 1.0
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: Cotoyogi
Disallow: /

User-agent: Crawlspace
Disallow: /

User-agent: Datenbank Crawler
Disallow: /

User-agent: DeepSeekBot
Disallow: /

User-agent: Devin
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: DuckAssistBot
Disallow: /

User-agent: Echobot Bot
Disallow: /

User-agent: EchoboxBot
Disallow: /

User-agent: Factset_spyderbot
Disallow: /

User-agent: FirecrawlAgent
Disallow: /

User-agent: FriendlyCrawler
Disallow: /

User-agent: iaskspider/2.0
Disallow: /

User-agent: ICC-Crawler
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: img2dataset
Disallow: /

User-agent: ISSCyberRiskCrawler
Disallow: /

User-agent: Kangaroo Bot
Disallow: /

User-agent: LinerBot
Disallow: /

User-agent: MistralAI-User
Disallow: /

User-agent: MistralAI-User/1.0
Disallow: /

User-agent: MyCentralAIScraperBot
Disallow: /

User-agent: netEstate Imprint Crawler
Disallow: /

User-agent: NovaAct
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: Operator
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: Panscient
Disallow: /

User-agent: panscient.com
Disallow: /

User-agent: Perplexity-User
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: PhindBot
Disallow: /

User-agent: Poseidon Research Crawler
Disallow: /

User-agent: QualifiedBot
Disallow: /

User-agent: QuillBot
Disallow: /

User-agent: quillbot.com
Disallow: /

User-agent: SBIntuitionsBot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: SemrushBot-OCOB
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: ShapBot
Disallow: /

User-agent: Sidetrade indexer bot
Disallow: /

User-agent: TerraCotta
Disallow: /

User-agent: The Knowledge AI
Disallow: /

User-agent: Thinkbot
Disallow: /

User-agent: TikTokSpider
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: WARDBot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: wpbot
Disallow: /

User-agent: YaK
Disallow: /

User-agent: YandexAdditional
Disallow: /

User-agent: YandexAdditionalBot
Disallow: /

User-agent: YouBot
Disallow: /

# Siri and Alexa yuck
User-agent: Amazonbot
Disallow: /

User-agent: Applebot
Disallow: /

# Miscellaneous
User-agent: TurnitinBot
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: SlySearch
Disallow: /

User-agent: 360Spider
Disallow: /

User-agent: netEstate NE Crawler
Disallow: /