# Supply It Up LLC — supply-it-up.com
# Search engines and answer engines are WELCOME here, all of them. If a person
# asks where to buy a sign in Austin, we want to be the answer — so every agent
# that can send a buyer is allowed below, by name, on purpose.
#
# The only ones turned away are pure harvesters: they take the catalog, the
# photography and the artwork, and they send nobody. See https://supply-it-up.com/terms/.

User-agent: *
Allow: /

# admin.html carries its own <meta robots="noindex,nofollow"> and is linked
# rel="nofollow". It is deliberately NOT Disallow'd, so Google can read that
# noindex and keep it de-indexed.

# ---- search and answer engines: welcome, and named so there is no doubt ----
# Google Search, and the AI Overviews built on it
User-agent: Googlebot
Allow: /

# so the product photography can rank in image search
User-agent: Googlebot-Image
Allow: /

# Bing, and the Copilot answers built on it
User-agent: Bingbot
Allow: /

# Siri, Spotlight and Safari suggestions
User-agent: Applebot
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# the index ChatGPT search cites from
User-agent: OAI-SearchBot
Allow: /

# ChatGPT fetching a page because a person asked
User-agent: ChatGPT-User
Allow: /

# the index Perplexity cites from — blocking this makes Perplexity-User useless
User-agent: PerplexityBot
Allow: /

# Perplexity fetching a page because a person asked
User-agent: Perplexity-User
Allow: /

# the index Claude search cites from
User-agent: Claude-SearchBot
Allow: /

# Claude fetching a page because a person asked
User-agent: Claude-User
Allow: /

# You.com answers
User-agent: YouBot
Allow: /

# Alexa answers — it can send a buyer, so it is welcome
User-agent: Amazonbot
Allow: /

# ---- harvesters: they send nobody, so they get nothing ----
# OpenAI model training — NOT ChatGPT search, which is OAI-SearchBot above
User-agent: GPTBot
Disallow: /

# Anthropic model training — NOT Claude search, which is Claude-SearchBot above
User-agent: ClaudeBot
Disallow: /

# Anthropic, legacy agent
User-agent: anthropic-ai
Disallow: /

# Anthropic, legacy agent
User-agent: Claude-Web
Disallow: /

# Common Crawl — the corpus most models start from
User-agent: CCBot
Disallow: /

# Gemini training. Does NOT affect Google Search or AI Overviews
User-agent: Google-Extended
Disallow: /

# Apple Intelligence training. Does NOT affect Siri or Spotlight
User-agent: Applebot-Extended
Disallow: /

# Meta model training
User-agent: Meta-ExternalAgent
Disallow: /

# Meta model training
User-agent: FacebookBot
Disallow: /

# ByteDance — aggressive, and ignores this as often as not
User-agent: Bytespider
Disallow: /

# Cohere model training
User-agent: cohere-ai
Disallow: /

# model training
User-agent: Timpibot
Disallow: /

# image harvesting
User-agent: ImagesiftBot
Disallow: /

# commercial scraper, sells structured copies of sites
User-agent: Diffbot
Disallow: /

# sells crawled data for training
User-agent: Omgilibot
Disallow: /

# sells crawled data for training
User-agent: omgili
Disallow: /

# the default scraping framework user agent
User-agent: Scrapy
Disallow: /

Sitemap: https://supply-it-up.com/sitemap.xml