{
 "slug": "web-scraping-crawling",
 "label": "Web scraping and crawling",
 "family": {
  "id": "web-data",
  "label": "Web and data access"
 },
 "url": "https://agentready-tools.com/categories/web-scraping-crawling",
 "catalog_categories": [
  "Web pages and crawlers",
  "Web data, crawling, and search",
  "Web scraping APIs"
 ],
 "tasks": [
  {
   "id": "compare-products",
   "label": "Compare products",
   "url": "https://agentready-tools.com/task/compare-products"
  },
  {
   "id": "search-web",
   "label": "Search the web",
   "url": "https://agentready-tools.com/task/search-web"
  },
  {
   "id": "summarize-webpage",
   "label": "Summarize a webpage",
   "url": "https://agentready-tools.com/task/summarize-webpage"
  },
  {
   "id": "search-social-posts",
   "label": "Search public social posts",
   "url": "https://agentready-tools.com/task/search-social-posts"
  },
  {
   "id": "browse-website",
   "label": "Browse and act on a website",
   "url": "https://agentready-tools.com/task/browse-website"
  }
 ],
 "count": 14,
 "updated": "2026-11-22",
 "note": "Comparison fields come from the catalog listing. \"Not yet verified\" and \"Not enough data\" mean we have no checked value yet; nothing is guessed.",
 "tools": [
  {
   "slug": "agentpay-checker",
   "name": "Agent Payment Checker — x402, MPP, L402, UCP & AP2 Detector",
   "description": "Check which AI-agent payment protocols a URL supports: x402, MPP (Machine Payments Protocol), L402, UCP and A2A/AP2 discovery files.",
   "category": "Web pages and crawlers",
   "url": "https://agentready-tools.com/tools/agentpay-checker",
   "record": "https://agentready-tools.com/tools/agentpay-checker.json",
   "mcp": false,
   "pricing": null,
   "jurisdiction": [],
   "agent_readiness": "Not enough data",
   "verified_checks": 0
  },
  {
   "slug": "apify-mcp",
   "name": "Apify MCP server",
   "description": "Extract data from any website with thousands of scrapers, crawlers, and automations on Apify Store.",
   "category": "Web data, crawling, and search",
   "url": "https://agentready-tools.com/tools/apify-mcp",
   "record": "https://agentready-tools.com/tools/apify-mcp.json",
   "mcp": true,
   "pricing": null,
   "jurisdiction": [],
   "agent_readiness": "Not enough data",
   "verified_checks": 1
  },
  {
   "slug": "apify-cheerio-scraper",
   "name": "Cheerio Scraper",
   "description": "Crawls websites using raw HTTP requests, parses the HTML with the Cheerio library, and extracts data from the pages using a Node.js code.",
   "category": "Web pages and crawlers",
   "url": "https://agentready-tools.com/tools/apify-cheerio-scraper",
   "record": "https://agentready-tools.com/tools/apify-cheerio-scraper.json",
   "mcp": false,
   "pricing": null,
   "jurisdiction": [],
   "agent_readiness": "Not enough data",
   "verified_checks": 0
  },
  {
   "slug": "exa-mcp",
   "name": "Exa MCP",
   "description": "Connect AI agents to Exa for web search, content fetching, and multi-step research.",
   "category": "Web data, crawling, and search",
   "url": "https://agentready-tools.com/tools/exa-mcp",
   "record": "https://agentready-tools.com/tools/exa-mcp.json",
   "mcp": true,
   "pricing": null,
   "jurisdiction": [],
   "agent_readiness": "Not enough data",
   "verified_checks": 1
  },
  {
   "slug": "firecrawl-mcp",
   "name": "Firecrawl MCP server",
   "description": "Firecrawl, the context API for AI agents: search, scrape, interact with the web, plus paper search.",
   "category": "Web data, crawling, and search",
   "url": "https://agentready-tools.com/tools/firecrawl-mcp",
   "record": "https://agentready-tools.com/tools/firecrawl-mcp.json",
   "mcp": true,
   "pricing": null,
   "jurisdiction": [],
   "agent_readiness": "Not enough data",
   "verified_checks": 1
  },
  {
   "slug": "oxylabs-ecommerce-scraper-api",
   "name": "Oxylabs E-Commerce Scraper API",
   "description": "Scraper API for collecting public e-commerce product, search, pricing, and seller data, returned as HTML or structured JSON.",
   "category": "Web scraping APIs",
   "url": "https://agentready-tools.com/tools/oxylabs-ecommerce-scraper-api",
   "record": "https://agentready-tools.com/tools/oxylabs-ecommerce-scraper-api.json",
   "mcp": false,
   "pricing": "$5 in welcome credits for new accounts",
   "jurisdiction": [],
   "agent_readiness": "Not enough data",
   "verified_checks": 0
  },
  {
   "slug": "oxylabs-web-api",
   "name": "Oxylabs Web API",
   "description": "Web scraping API that returns structured data from sites, including JavaScript-heavy pages, with an official MCP server for AI agents.",
   "category": "Web scraping APIs",
   "url": "https://agentready-tools.com/tools/oxylabs-web-api",
   "record": "https://agentready-tools.com/tools/oxylabs-web-api.json",
   "mcp": true,
   "pricing": "$5 in welcome credits for new accounts",
   "jurisdiction": [],
   "agent_readiness": "Not enough data",
   "verified_checks": 1
  },
  {
   "slug": "apify-puppeteer-scraper",
   "name": "Puppeteer Scraper",
   "description": "Crawls websites with the headless Chrome and Puppeteer library using a provided server-side Node.js code.",
   "category": "Web pages and crawlers",
   "url": "https://agentready-tools.com/tools/apify-puppeteer-scraper",
   "record": "https://agentready-tools.com/tools/apify-puppeteer-scraper.json",
   "mcp": false,
   "pricing": null,
   "jurisdiction": [],
   "agent_readiness": "Not enough data",
   "verified_checks": 0
  },
  {
   "slug": "apify-rag-web-browser",
   "name": "RAG Web Browser",
   "description": "Web search and fetch tool for AI agents and RAG pipelines. It queries Google Search, scrapes the top N pages using a full web browser, and returns their…",
   "category": "Web pages and crawlers",
   "url": "https://agentready-tools.com/tools/apify-rag-web-browser",
   "record": "https://agentready-tools.com/tools/apify-rag-web-browser.json",
   "mcp": false,
   "pricing": null,
   "jurisdiction": [],
   "agent_readiness": "Not enough data",
   "verified_checks": 0
  },
  {
   "slug": "scraperapi",
   "name": "ScraperAPI",
   "description": "Web scraping API that handles proxies, browsers and CAPTCHAs.",
   "category": "Web scraping APIs",
   "url": "https://agentready-tools.com/tools/scraperapi",
   "record": "https://agentready-tools.com/tools/scraperapi.json",
   "mcp": false,
   "pricing": null,
   "jurisdiction": [],
   "agent_readiness": "Not enough data",
   "verified_checks": 0
  },
  {
   "slug": "lukaskrivka-article-extractor-smart",
   "name": "Smart Article Extractor",
   "description": "📰 Smart Article Extractor extracts articles from any scientific, academic, or news website with just one click.",
   "category": "Web pages and crawlers",
   "url": "https://agentready-tools.com/tools/lukaskrivka-article-extractor-smart",
   "record": "https://agentready-tools.com/tools/lukaskrivka-article-extractor-smart.json",
   "mcp": false,
   "pricing": null,
   "jurisdiction": [],
   "agent_readiness": "Not enough data",
   "verified_checks": 0
  },
  {
   "slug": "apify-web-scraper",
   "name": "Web Scraper",
   "description": "Crawls arbitrary websites using a web browser and extracts structured data from web pages using a provided JavaScript function.",
   "category": "Web pages and crawlers",
   "url": "https://agentready-tools.com/tools/apify-web-scraper",
   "record": "https://agentready-tools.com/tools/apify-web-scraper.json",
   "mcp": false,
   "pricing": null,
   "jurisdiction": [],
   "agent_readiness": "Not enough data",
   "verified_checks": 0
  },
  {
   "slug": "apify-website-content-crawler",
   "name": "Website Content Crawler",
   "description": "Crawl websites and extract text content to feed AI models, LLM applications, vector databases, or RAG pipelines.",
   "category": "Web pages and crawlers",
   "url": "https://agentready-tools.com/tools/apify-website-content-crawler",
   "record": "https://agentready-tools.com/tools/apify-website-content-crawler.json",
   "mcp": false,
   "pricing": null,
   "jurisdiction": [],
   "agent_readiness": "Not enough data",
   "verified_checks": 0
  },
  {
   "slug": "url-screenshot",
   "name": "Website Screenshot API — URL to PNG/JPEG/PDF for AI Agents",
   "description": "Screenshot any public URL to PNG, JPEG or PDF with Chromium. Predictable JSON output with a direct image URL and explicit errors, built for AI agents, API…",
   "category": "Web pages and crawlers",
   "url": "https://agentready-tools.com/tools/url-screenshot",
   "record": "https://agentready-tools.com/tools/url-screenshot.json",
   "mcp": false,
   "pricing": null,
   "jurisdiction": [],
   "agent_readiness": "60 / 100",
   "verified_checks": 5
  }
 ]
}
