{
  "dataset": "crawlers",
  "record": {
    "id": "bytespider",
    "name": "Bytespider",
    "operator": "ByteDance",
    "purpose": "training",
    "ua_substring": "Bytespider",
    "robots_token": "Bytespider",
    "respects_robots": false,
    "verify": "no authoritative published range file; treat unverified Bytespider traffic with suspicion",
    "notes": "Has a reputation for aggressive crawling and inconsistent robots.txt adherence. Rate-limit at the edge if it causes load.",
    "canonical_name": "Bytespider",
    "user_agent_token": "Bytespider",
    "ua_full": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://radar.cloudflare.com/bots/directory/bytespider"
    },
    "bot_type": "training",
    "bot_type_extension": null,
    "opt_out_mechanism": "robots.txt disallow (User-agent: Bytespider) — but adherence is documented as inconsistent; rate-limit / block at the edge if needed",
    "published_ip_range_url": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://radar.cloudflare.com/bots/directory/bytespider",
      "note": "no authoritative ByteDance-published IP-range file confirmed at primary source"
    },
    "asn": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://radar.cloudflare.com/bots/directory/bytespider"
    },
    "reverse_dns_suffix": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://radar.cloudflare.com/bots/directory/bytespider"
    },
    "supports_web_bot_auth": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://radar.cloudflare.com/bots/directory/bytespider"
    },
    "signature_agent_domain": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://radar.cloudflare.com/bots/directory/bytespider"
    },
    "jwks_url": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://radar.cloudflare.com/bots/directory/bytespider"
    },
    "verification_methods": [
      "user-agent-match"
    ],
    "crawl_traffic_share": {
      "value": "10.25%",
      "unit": "% of AI crawler requests",
      "as_of": "2026-05",
      "source": "https://radar.cloudflare.com/bots",
      "source_label": "Cloudflare Radar (May 2026), via research/competitive-research-2026-06.md §1",
      "last_verified": "2026-06-15",
      "note": "second-largest AI crawler in the Radar May 2026 snapshot"
    },
    "targeted_content_type": "HTML, text",
    "documentation_url": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://radar.cloudflare.com/bots/directory/bytespider",
      "note": "no authoritative ByteDance operator documentation page confirmed at primary source"
    },
    "first_seen_date": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://radar.cloudflare.com/bots/directory/bytespider"
    },
    "last_verified_date": "2026-06-15",
    "block_vs_allow_recommendation": "block (default) — training crawler with documented aggressive crawling and inconsistent robots.txt adherence and no authoritative IP file to verify; block/rate-limit at the edge.",
    "citation_referral_value": "low (training; no direct citation or referral)",
    "cloudflare_verified_category": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://radar.cloudflare.com/bots/directory/bytespider"
    },
    "status": "active",
    "triples": [
      [
        "Bytespider",
        "operated_by",
        "ByteDance"
      ],
      [
        "Bytespider",
        "has_bot_type",
        "training"
      ],
      [
        "Bytespider",
        "has_crawl_share",
        "10.25% (Radar 2026-05)"
      ]
    ],
    "attribute_sources": [
      {
        "claims": [
          "crawl_traffic_share"
        ],
        "source": "https://radar.cloudflare.com/bots",
        "last_verified": "2026-06-15"
      }
    ]
  }
}