1# https://www.robotstxt.org/robotstxt.html
2User-agent: *
3# Content Signals (https://contentsignals.org/): content on this site may be used to build search
4# results, as AI input (for example, retrieval-augmented generation and agents), and for AI training.
5Content-Signal: search=yes, ai-input=yes, ai-train=yes
6Allow: /
7
8# Sitemap location
9Sitemap: https://spiceai.org/sitemap.xml
10
11# LLM-friendly documentation (llmstxt.org standard)
12# llms.txt: https://spiceai.org/llms.txt
13# llms-full.txt: https://spiceai.org/llms-full.txt
14
15# Crawl-delay for polite crawling (optional, respected by some bots)
16Crawl-delay: 1
17
18# Disallow search results pages
19Disallow: /search
20
21# Allow all documentation
22Allow: /docs/
23Allow: /releases/
24Allow: /cookbook/
25Allow: /llms.txt
26Allow: /llms-full.txt
27
28# Block internal/development paths (if any)
29Disallow: /api/HTTP/sidebar
30

1# https://www.robotstxt.org/robotstxt.html
2User-agent: *
3# Content Signals (https://contentsignals.org/): content on this site may be used to build search
4# results, as AI input (for example, retrieval-augmented generation and agents), and for AI training.
5Content-Signal: search=yes, ai-input=yes, ai-train=yes
6Allow: /
7
8# Sitemap location
9Sitemap: https://spiceai.org/sitemap.xml
10
11# LLM-friendly documentation (llmstxt.org standard)
12# llms.txt: https://spiceai.org/llms.txt
13# llms-full.txt: https://spiceai.org/llms-full.txt
14
15# Crawl-delay for polite crawling (optional, respected by some bots)
16Crawl-delay: 1
17
18# Disallow search results pages
19Disallow: /search
20
21# Allow all documentation
22Allow: /docs/
23Allow: /releases/
24Allow: /cookbook/
25Allow: /llms.txt
26Allow: /llms-full.txt
27
28# Block internal/development paths (if any)
29Disallow: /api/HTTP/sidebar
30