{
  "dataset": "crawlers",
  "record": {
    "id": "diffbot-user",
    "name": "Diffbot-User",
    "operator": "Diffbot",
    "purpose": "inference",
    "ua_substring": "Diffbot-User",
    "robots_token": "Diffbot-User",
    "respects_robots": true,
    "verify": "no operator-published authoritative IP-range file confirmed; verify by user-agent + edge controls. Diffbot documents the token for on-behalf-of fetches.",
    "notes": "Used for requests made on behalf of human users browsing URLs through Diffbot software, as distinct from Diffbot's proactive Crawlbot. Diffbot documents both 'Diffbot' and 'Diffbot-User' as robots.txt user-agents.",
    "canonical_name": "Diffbot-User",
    "user_agent_token": "Diffbot-User",
    "ua_full": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://docs.diffbot.com/docs/does-crawl-respect-robotstxt"
    },
    "bot_type": "user-action-fetcher",
    "bot_type_extension": null,
    "opt_out_mechanism": "robots.txt disallow (User-agent: Diffbot-User)",
    "published_ip_range_url": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://docs.diffbot.com/docs/does-crawl-respect-robotstxt",
      "note": "no authoritative Diffbot-published IP-range JSON confirmed at primary source"
    },
    "asn": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://docs.diffbot.com/"
    },
    "reverse_dns_suffix": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://docs.diffbot.com/"
    },
    "supports_web_bot_auth": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://docs.diffbot.com/"
    },
    "signature_agent_domain": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://docs.diffbot.com/"
    },
    "jwks_url": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://docs.diffbot.com/"
    },
    "verification_methods": [
      "user-agent-match"
    ],
    "crawl_traffic_share": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://radar.cloudflare.com/bots"
    },
    "targeted_content_type": "HTML, text",
    "documentation_url": "https://docs.diffbot.com/docs/does-crawl-respect-robotstxt",
    "first_seen_date": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://docs.diffbot.com/"
    },
    "last_verified_date": "2026-06-15",
    "block_vs_allow_recommendation": "allow (default) — user-initiated fetch on a human's behalf through Diffbot software; respects robots.txt. Blocking degrades that user's task.",
    "citation_referral_value": "medium (fetches a specific page for a user; can surface it to them)",
    "cloudflare_verified_category": {
      "value": null,
      "verify_status": "verify-against-primary-at-build",
      "source_hint": "https://radar.cloudflare.com/bots/directory/diffbot-user"
    },
    "status": "active",
    "triples": [
      [
        "Diffbot-User",
        "operated_by",
        "Diffbot"
      ],
      [
        "Diffbot-User",
        "has_bot_type",
        "user-action-fetcher"
      ],
      [
        "Diffbot-User",
        "verified_via",
        "user-agent-match"
      ]
    ],
    "attribute_sources": [
      {
        "claims": [
          "user_agent_token",
          "robots_token",
          "respects_robots",
          "documentation_url",
          "opt_out_mechanism"
        ],
        "source": "https://docs.diffbot.com/docs/does-crawl-respect-robotstxt",
        "last_verified": "2026-06-15"
      }
    ]
  }
}