User-Agent & AI Bot Parser API
Detect AI training crawlers, RAG bots, headless browsers, and device/OS details from User-Agent.
Audit which AI crawlers your robots.txt allows and generate a valid llms.txt file. Analyze robots.txt for AI crawler permissions (GPTBot, ClaudeBot, PerplexityBot, Google-Extended), lint existing llms.txt files, and generate valid llms.txt.
{
"ok": true,
"data": {
"test_path": "/",
"ai_crawlers": [
{
"bot": "GPTBot",
"vendor": "OpenAI",
"purpose": "ai_training",
"status": "blocked",
"matched_agent": "GPTBot",
"matched_rule": "Disallow: /"
},
{
"bot": "OAI-SearchBot",
"vendor": "OpenAI",
"purpose": "ai_search",
"status": "allowed",
"matched_agent": "OAI-SearchBot",
"matched_rule": "Allow: /"
},
{
"bot": "ChatGPT-User",
"vendor": "OpenAI",
"purpose": "ai_search",
"status": "allowed",
"matched_agent": "*",
"matched_rule": "Allow: /"
},
{
"bot": "ClaudeBot",
"vendor": "Anthropic",
"purpose": "ai_training",
"status": "allowed",
"matched_agent": "*",
"matched_rule": "Allow: /"
},
{
"bot": "Claude-Web",
"vendor": "Anthropic",
"purpose": "ai_search",
"status": "allowed",
"matched_agent": "*",
"matched_rule": "Allow: /"
},
{
"bot": "Google-Extended",
"vendor": "Google",
"purpose": "ai_training",
"status": "allowed",
"matched_agent": "*",
"matched_rule": "Allow: /"
},
{
"bot": "PerplexityBot",
"vendor": "Perplexity",
"purpose": "ai_search",
"status": "allowed",
"matched_agent": "*",
"matched_rule": "Allow: /"
},
{
"bot": "CCBot",
"vendor": "Common Crawl",
"purpose": "ai_training",
"status": "allowed",
"matched_agent": "*",
"matched_rule": "Allow: /"
},
{
"bot": "Bytespider",
"vendor": "ByteDance",
"purpose": "ai_training",
"status": "allowed",
"matched_agent": "*",
"matched_rule": "Allow: /"
},
{
"bot": "Applebot-Extended",
"vendor": "Apple",
"purpose": "ai_training",
"status": "allowed",
"matched_agent": "*",
"matched_rule": "Allow: /"
},
{
"bot": "Meta-ExternalAgent",
"vendor": "Meta",
"purpose": "ai_training",
"status": "allowed",
"matched_agent": "*",
"matched_rule": "Allow: /"
},
{
"bot": "Amazonbot",
"vendor": "Amazon",
"purpose": "ai_search",
"status": "allowed",
"matched_agent": "*",
"matched_rule": "Allow: /"
}
],
"summary": {
"training_bots_allowed": 6,
"training_bots_blocked": 1,
"search_bots_allowed": 5,
"search_bots_blocked": 0,
"sitemaps": []
},
"llms_txt_analysis": {
"present": false,
"valid": false,
"title": null,
"summary": null,
"section_count": 0,
"link_count": 0,
"issues": [
"No llms.txt content provided."
]
},
"generated_robots_ai_snippet": "# --- AI Crawler Policy (Generated by Tinplate) ---\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: OAI-SearchBot\nAllow: /\n\nUser-agent: ChatGPT-User\nAllow: /\n\nUser-agent: ClaudeBot\nDisallow: /\n\nUser-agent: Claude-Web\nAllow: /\n\nUser-agent: Google-Extended\nDisallow: /\n\nUser-agent: PerplexityBot\nAllow: /\n\nUser-agent: CCBot\nDisallow: /\n\nUser-agent: Bytespider\nDisallow: /\n\nUser-agent: Applebot-Extended\nDisallow: /\n\nUser-agent: Meta-ExternalAgent\nDisallow: /\n\nUser-agent: Amazonbot\nAllow: /",
"generated_llms_txt": null
},
"meta": {
"request_id": "req_rla_01",
"ms": 1
}
}
curl -X POST "https://api.tinplate.dev/v1/robots-llms-analyzer" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer $TINPLATE_KEY" \
-d '{"robots_txt":"User-agent: *\nAllow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: OAI-SearchBot\nAllow: /"}'import { TinplateClient } from "@tinplate/sdk";
const tinplate = new TinplateClient({
apiKey: process.env.TINPLATE_API_KEY!,
});
const { data, meta } = await tinplate.call("/v1/robots-llms-analyzer", {
"robots_txt": "User-agent: *\nAllow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: OAI-SearchBot\nAllow: /"
});
console.log(`[${meta.ms} ms]`, data);import requests
response = requests.post(
"https://api.tinplate.dev/v1/robots-llms-analyzer",
headers={
"Content-Type": "application/json",
"Authorization": "Bearer YOUR_TINPLATE_KEY",
},
json={
"robots_txt": "User-agent: *\nAllow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: OAI-SearchBot\nAllow: /"
}
)
print(response.json())const response = await fetch("https://api.tinplate.dev/v1/robots-llms-analyzer", {
method: "POST",
headers: {
"Content-Type": "application/json",
"Authorization": "Bearer YOUR_TINPLATE_KEY",
},
body: JSON.stringify({
"robots_txt": "User-agent: *\nAllow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: OAI-SearchBot\nAllow: /"
}),
});
const result = await response.json();
console.log(result);{
"jsonrpc": "2.0",
"id": 1,
"method": "tools/call",
"params": {
"name": "tinplate_robots_llms_analyzer",
"arguments": {
"robots_txt": "User-agent: *\nAllow: /\n\nUser-agent: GPTBot\nDisallow: /\n\nUser-agent: OAI-SearchBot\nAllow: /"
}
}
}POST /v1/robots-llms-analyzer)| Parameter | Type & Constraints | Description |
|---|---|---|
robots_txt optional |
stringmaxLen: 131072 |
Raw robots.txt content to analyze. |
llms_txt optional |
stringmaxLen: 131072 |
Optional raw llms.txt content to validate against the llmstxt.org specification. |
test_path optional |
stringmaxLen: 512, default: "/" |
URL path to test crawler permissions against (default '/'). |
desired_ai_policy optional |
"allow_all" | "allow_search_block_training" | "block_all_ai"default: "allow_search_block_training" |
Policy used to generate the recommended robots.txt AI crawler snippet. |
site_info optional |
object |
Optional site metadata used to generate a valid llms.txt file. |
data Object Fields (200 OK)| Field | Type | Description |
|---|---|---|
test_path |
string |
Field 'test_path' |
ai_crawlers |
object[] |
Field 'ai_crawlers' |
summary |
object |
Field 'summary' |
llms_txt_analysis |
object |
Field 'llms_txt_analysis' |
generated_robots_ai_snippet |
string |
Field 'generated_robots_ai_snippet' |
generated_llms_txt |
string | null |
Field 'generated_llms_txt' |
Release: v1.0.39 ·
Verified: 16 November 2026 ·
Portfolio Median: 0.318 ms in-Worker V8 compute (add 15–65 ms HTTPS RTT) ·
See Full Portfolio Benchmarks →
Direct Polar All-Access Keys unlock all 42 APIs (49 endpoints), /v1/pipeline, and /mcp on api.tinplate.dev:
Pro ($29/mo · 120 req/min · 100k/mo · 128 KB) or Team ($79/mo · 300 req/min · 500k/mo · 256 KB).
| Plan | Monthly Price | Included Requests | Rate Limit | Max Payload | Best For |
|---|---|---|---|---|---|
| BASIC | $0.00 / mo | 500 / month | 10 req / min | 64 KB | Evaluation & CI smoke tests |
| PRO | $9.99 / mo | 15,000 / month | 60 req / min | 128 KB | Solo devs & production apps |
| ULTRA | $29.99 / mo | 100,000 / month | 300 req / min | 256 KB | Growing SaaS & multi-agent pipelines |
| MEGA | $79.99 / mo | 500,000 / month | 1,000 req / min | 256 KB | High-volume enterprise workloads |
Use policy preset 'allow_search_block_training', which disallows GPTBot and CCBot while allowing OAI-SearchBot and ChatGPT-User.
llms.txt is a Markdown file at the root of a website (/llms.txt) that gives LLMs and coding agents a concise index of your documentation and APIs.
Yes, pass llms_txt to lint the H1 title, summary blockquote, and Markdown link sections.
Stateless in-memory analysis.
Worker Zero-Logging Guarantee: Cloudflare Workers Observability and invocation logging are disabled (observability.enabled = false, head_sampling_rate = 0, invocation_logs = false). Requests via RapidAPI or Zyla traverse marketplace proxies; use a Direct Bearer Key for zero-intermediary routing.
Detect AI training crawlers, RAG bots, headless browsers, and device/OS details from User-Agent.
Convert noisy web pages or raw HTML into clean, token-efficient Markdown for RAG.
Canonicalize URLs and strip 120+ ad-tracking and click-ID parameters for clean deduplication.