1# https://www.robotstxt.org/robotstxt.html
2User-agent: *
3Allow: /
4
5# Sitemap location
6Sitemap: https://spiceai.org/sitemap.xml
7
8# LLM-friendly documentation (llmstxt.org standard)
9# llms.txt: https://spiceai.org/llms.txt
10# llms-full.txt: https://spiceai.org/llms-full.txt
11
12# Crawl-delay for polite crawling (optional, respected by some bots)
13Crawl-delay: 1
14
15# Disallow search results pages
16Disallow: /search
17
18# Allow all documentation
19Allow: /docs/
20Allow: /releases/
21Allow: /cookbook/
22Allow: /llms.txt
23Allow: /llms-full.txt
24
25# Block internal/development paths (if any)
26Disallow: /api/HTTP/sidebar
27

1# https://www.robotstxt.org/robotstxt.html
2User-agent: *
3Allow: /
4
5# Sitemap location
6Sitemap: https://spiceai.org/sitemap.xml
7
8# LLM-friendly documentation (llmstxt.org standard)
9# llms.txt: https://spiceai.org/llms.txt
10# llms-full.txt: https://spiceai.org/llms-full.txt
11
12# Crawl-delay for polite crawling (optional, respected by some bots)
13Crawl-delay: 1
14
15# Disallow search results pages
16Disallow: /search
17
18# Allow all documentation
19Allow: /docs/
20Allow: /releases/
21Allow: /cookbook/
22Allow: /llms.txt
23Allow: /llms-full.txt
24
25# Block internal/development paths (if any)
26Disallow: /api/HTTP/sidebar
27