[ { "url": "https://docs.crawl4ai.com", "title": "🚀🤖 Crawl4AI: Open-Source LLM-Friendly Web Crawler & Scraper", "type": "overview", "summary": "Crawl4AI is an open-source, LLM-friendly web crawler and scraper that produces clean Markdown, supports structured extraction, advanced browser control, and high-performance parallel crawling. This…", "sections": [ { "heading": "🚀 Crawl4AI Cloud API — Closed Beta (Launching Soon)", "content": "Reliable, large-scale web extraction, now built to be **drastically more cost-effective** than any of the existing solutions.\n\n👉 **Apply [here](https://forms.gle/E9MyPaNXACnAMaqG7) for early…", "code_blocks": [], "images": [] }, { "heading": "Crawl4AI: Open-Source LLM-Friendly Web Crawler & Scraper", "content": "Crawl4AI is the #1 trending GitHub repository, actively maintained by a vibrant community. It delivers blazing-fast, AI-ready web crawling tailored for large language models, AI agents, and data…", "code_blocks": [], "images": [] }, { "heading": "🆕 AI Assistant Skill Now Available!", "content": "", "code_blocks": [], "images": [] }, { "heading": "🤖 Crawl4AI Skill for Claude & AI Assistants", "content": "Supercharge your AI coding assistant with complete Crawl4AI knowledge! Download our comprehensive skill package that includes:\n\n- 📚 Complete SDK reference (23K+ words)\n- 🚀 Ready-to-use extraction…", "code_blocks": [], "images": [] }, { "heading": "🎯 New: Adaptive Web Crawling", "content": "Crawl4AI now features intelligent adaptive crawling that knows when to stop! Using advanced information foraging algorithms, it determines when sufficient information has been gathered to answer your…", "code_blocks": [], "images": [] }, { "heading": "Quick Start", "content": "Here's a quick example to show you how easy it is to use Crawl4AI with its asynchronous capabilities:", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler\n\nasync def main():\n # Create an instance of AsyncWebCrawler\n async with AsyncWebCrawler() as crawler:\n # Run the crawler on a URL…", "filename": "" } ], "images": [] }, { "heading": "Video Tutorial", "content": "", "code_blocks": [], "images": [] }, { "heading": "What Does Crawl4AI Do?", "content": "Crawl4AI is a feature-rich crawler and scraper that aims to:\n\n1. **Generate Clean Markdown**: Perfect for RAG pipelines or direct ingestion into LLMs.\n2. **Structured Extraction**: Parse repeated…", "code_blocks": [], "images": [] }, { "heading": "Documentation Structure", "content": "To help you get started, we’ve organized our docs into clear sections:\n\n- **Setup & Installation**: Basic instructions to install Crawl4AI via pip or Docker.\n- **Quick Start**: A hands-on…", "code_blocks": [], "images": [] }, { "heading": "How You Can Support", "content": "- **Star & Fork**: If you find Crawl4AI helpful, star the repo on GitHub or fork it to add your own features.\n- **File Issues**: Encounter a bug or missing feature? Let us know by filing an issue, so…", "code_blocks": [], "images": [] }, { "heading": "Quick Links", "content": "- **[GitHub Repo](https://github.com/unclecode/crawl4ai)**\n- **[Installation Guide](core/installation/)**\n- **[Quick Start](core/quickstart/)**\n- **[API Reference](api/async-webcrawler/)**\n-…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "crawler", "scraper", "LLM", "web crawling", "open-source", "Python", "AsyncWebCrawler", "Markdown", "extraction", "AI", "RAG", "adaptive crawling", "documentation" ], "prerequisites": [], "_content_hash": "5ab29c50b5f0b2b408fc95ef0b33dc777bcc6789aa2ccb9b7f220ac50a6cda56", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/", "title": "Home - Crawl4AI Documentation (v0.9.x)", "type": "overview", "summary": "Landing page and overview for Crawl4AI, an open-source, LLM-friendly web crawler and scraper. It introduces the project's mission, key capabilities, quick-start example, documentation structure, and…", "sections": [ { "heading": "🚀🤖 Crawl4AI: Open-Source LLM-Friendly Web Crawler & Scraper", "content": "🚀 Crawl4AI Cloud API — Closed Beta (Launching Soon)\n\nReliable, large-scale web extraction, now built to be drastically more cost-effective than any of the existing solutions.\n\n👉 Apply here for early…", "code_blocks": [], "images": [] }, { "heading": "🆕 AI Assistant Skill Now Available!", "content": "🤖 Crawl4AI Skill for Claude & AI Assistants\n\nSupercharge your AI coding assistant with complete Crawl4AI knowledge! Download our comprehensive skill package that includes:\n\n- 📚 Complete SDK reference…", "code_blocks": [], "images": [] }, { "heading": "🎯 New: Adaptive Web Crawling", "content": "Crawl4AI now features intelligent adaptive crawling that knows when to stop! Using advanced information foraging algorithms, it determines when sufficient information has been gathered to answer your…", "code_blocks": [], "images": [] }, { "heading": "Quick Start", "content": "Here's a quick example to show you how easy it is to use Crawl4AI with its asynchronous capabilities:", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler\n\nasync def main():\n # Create an instance of AsyncWebCrawler\n async with AsyncWebCrawler() as crawler:\n # Run the crawler on a URL…", "filename": "" } ], "images": [] }, { "heading": "Video Tutorial", "content": "", "code_blocks": [], "images": [] }, { "heading": "What Does Crawl4AI Do?", "content": "Crawl4AI is a feature-rich crawler and scraper that aims to:\n\n1. Generate Clean Markdown: Perfect for RAG pipelines or direct ingestion into LLMs.\n2. Structured Extraction: Parse repeated patterns…", "code_blocks": [], "images": [] }, { "heading": "Documentation Structure", "content": "To help you get started, we’ve organized our docs into clear sections:\n\n- Setup & Installation — Basic instructions to install Crawl4AI via pip or Docker.\n- Quick Start — A hands-on introduction…", "code_blocks": [], "images": [] }, { "heading": "How You Can Support", "content": "- Star & Fork: If you find Crawl4AI helpful, star the repo on GitHub or fork it to add your own features.\n- File Issues: Encounter a bug or missing feature? Let us know by filing an issue, so we can…", "code_blocks": [], "images": [] }, { "heading": "Quick Links", "content": "- GitHub Repo\n- Installation Guide\n- Quick Start\n- API Reference\n- Changelog\n\nThank you for joining me on this journey. Let’s keep building an open, democratic approach to data extraction and AI…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "crawl4ai", "web crawler", "scraper", "LLM", "open source", "Python", "async", "AI agents", "RAG", "documentation" ], "prerequisites": [], "_content_hash": "5ab29c50b5f0b2b408fc95ef0b33dc777bcc6789aa2ccb9b7f220ac50a6cda56", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/CONTRIBUTING/", "title": "Contributing Guide - Crawl4AI Documentation (v0.9.x)", "type": "guide", "summary": "Contribution guide for Crawl4AI explaining the branching strategy, contributor workflow, release process, and best practices for submitting pull requests.", "sections": [ { "heading": "Introduction", "content": "Welcome to the Crawl4AI project! As an open-source library for web crawling and AI integration, we value contributions from the community. This guide explains our branching strategy, how to…", "code_blocks": [], "images": [] }, { "heading": "Core Branches", "content": "- **main** : The stable branch containing production-ready code. It's always identical to the latest released version and is tagged for releases. Do not submit PRs directly here.\n\n- **develop** : The…", "code_blocks": [], "images": [] }, { "heading": "Contributor Workflow", "content": "We encourage contributions of all kinds: bug fixes, new features, documentation improvements, tests, or even Docker enhancements. Follow these steps to contribute:\n\n- **Fork the Repository** : Create…", "code_blocks": [ { "language": "shell", "code": "git checkout develop\ngit checkout -b feature/your-feature-name # Or bugfix/your-bugfix-name", "filename": "" } ], "images": [] }, { "heading": "Lead Maintainer's Workflow (For Reference)", "content": "- The lead maintainer (Unclecode) uses the `next` branch for isolated experimental work.\n\n- Features from `next` are periodically merged into `develop` (via rebase and merge) to keep everything in…", "code_blocks": [], "images": [] }, { "heading": "Release Process (High-Level Overview)", "content": "Releases happen bi-weekly to ship improvements regularly. As a contributor, your merged changes in `develop` will be included in the next release unless specified otherwise. Here's a summary of what…", "code_blocks": [], "images": [] }, { "heading": "Benefits of This Approach", "content": "- **Stability** : `main` is always reliable for users.\n\n- **Collaboration** : Fixed PR target (`develop`) makes contributing straightforward.\n\n- **Isolation** : Experimental work in `next` doesn't…", "code_blocks": [], "images": [] }, { "heading": "Checklist for Contributors", "content": "Before submitting a PR:\n\n- [ ] Based on and targeting `develop`.\n\n- [ ] Tests pass (`pytest`).\n\n- [ ] Docs updated if needed (e.g., version refs in mkdocs.yml, Docker files).\n\n- [ ] No breaking…", "code_blocks": [], "images": [] }, { "heading": "Common Issues", "content": "- **Merge Conflicts** : Rebase your branch on latest `develop` before PR.\n\n- **Docker Builds** : Test multi-arch (amd64/arm64) locally if changing Dockerfile.\n\n- **Version Consistency** : Ensure any…", "code_blocks": [], "images": [] }, { "heading": "Communication", "content": "- Open issues for discussions or bugs.\n\n- Join our Discord (link in README) for real-time help.\n\n- After releases, announcements go to GitHub, Discord, and social media.\n\nThanks for contributing to…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "Crawl4AI", "open-source", "contributing", "GitFlow", "GitHub", "Docker", "semantic versioning", "documentation" ], "prerequisites": [], "_content_hash": "8f0f3ec4efafcab7f0c51ce4407652c13de60ea90e78e00b7abc167c28ff9b47", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/adaptive-strategies/", "title": "Adaptive Strategies - Crawl4AI Documentation (v0.9.x)", "type": "guide", "summary": "This guide covers advanced adaptive strategies in Crawl4AI, including the three-layer scoring system, link ranking algorithm, domain-specific configurations, performance optimization, debugging,…", "sections": [ { "heading": "Overview", "content": "While the default adaptive crawling configuration works well for most use cases, understanding the underlying strategies and scoring mechanisms allows you to fine-tune the crawler for specific…", "code_blocks": [], "images": [] }, { "heading": "The Three-Layer Scoring System", "content": "", "code_blocks": [], "images": [] }, { "heading": "1. Coverage Score", "content": "Coverage measures how comprehensively your knowledge base covers the query terms and related concepts.", "code_blocks": [], "images": [] }, { "heading": "Mathematical Foundation", "content": "", "code_blocks": [ { "language": "text", "code": "Coverage(K, Q) = Σ(t ∈ Q) score(t, K) / |Q|\n\nwhere score(t, K) = doc_coverage(t) × (1 + freq_boost(t))", "filename": "" } ], "images": [] }, { "heading": "Components", "content": "- **Document Coverage**: Percentage of documents containing the term\n- **Frequency Boost**: Logarithmic bonus for term frequency\n- **Query Decomposition**: Handles multi-word queries intelligently", "code_blocks": [], "images": [] }, { "heading": "Tuning Coverage", "content": "", "code_blocks": [ { "language": "python", "code": "# For technical documentation with specific terminology\nconfig = AdaptiveConfig(\n confidence_threshold=0.85, # Require high coverage\n top_k_links=5 # Cast wider net\n)\n\n# For…", "filename": "" } ], "images": [] }, { "heading": "2. Consistency Score", "content": "Consistency evaluates whether the information across pages is coherent and non-contradictory.", "code_blocks": [], "images": [] }, { "heading": "How It Works", "content": "- Extracts key statements from each document\n- Compares statements across documents\n- Measures agreement vs. contradiction\n- Returns normalized score (0-1)", "code_blocks": [], "images": [] }, { "heading": "Practical Impact", "content": "- **High consistency (>0.8)**: Information is reliable and coherent\n- **Medium consistency (0.5-0.8)**: Some variation, but generally aligned\n- **Low consistency (<0.5)**: Conflicting information,…", "code_blocks": [], "images": [] }, { "heading": "3. Saturation Score", "content": "Saturation detects when new pages stop providing novel information.", "code_blocks": [], "images": [] }, { "heading": "Detection Algorithm", "content": "", "code_blocks": [ { "language": "python", "code": "# Tracks new unique terms per page\nnew_terms_page_1 = 50\nnew_terms_page_2 = 30 # 60% of first\nnew_terms_page_3 = 15 # 50% of second\nnew_terms_page_4 = 5 # 33% of third\n# Saturation detected:…", "filename": "" } ], "images": [] }, { "heading": "Configuration", "content": "", "code_blocks": [ { "language": "python", "code": "config = AdaptiveConfig(\n min_gain_threshold=0.1 # Stop if <10% new information\n)", "filename": "" } ], "images": [] }, { "heading": "Link Ranking Algorithm", "content": "", "code_blocks": [], "images": [] }, { "heading": "Expected Information Gain", "content": "Each uncrawled link is scored based on:", "code_blocks": [ { "language": "text", "code": "ExpectedGain(link) = Relevance × Novelty × Authority", "filename": "" } ], "images": [] }, { "heading": "1. Relevance Scoring", "content": "Uses BM25 algorithm on link preview text:\n\nFactors:\n- Term frequency in preview\n- Inverse document frequency\n- Preview length normalization", "code_blocks": [ { "language": "text", "code": "relevance = BM25(link.preview_text, query)", "filename": "" } ], "images": [] }, { "heading": "2. Novelty Estimation", "content": "Measures how different the link appears from already-crawled content:\n\nPrevents crawling duplicate or highly similar pages.", "code_blocks": [ { "language": "text", "code": "novelty = 1 - max_similarity(preview, knowledge_base)", "filename": "" } ], "images": [] }, { "heading": "3. Authority Calculation", "content": "URL structure and domain analysis:\n\nFactors:\n- Domain reputation\n- URL depth (fewer slashes = higher authority)\n- Clean URL structure", "code_blocks": [ { "language": "text", "code": "authority = f(domain_rank, url_depth, url_structure)", "filename": "" } ], "images": [] }, { "heading": "Domain-Specific Configurations", "content": "", "code_blocks": [], "images": [] }, { "heading": "Technical Documentation", "content": "Rationale:\n- High threshold ensures comprehensive coverage\n- Lower gain threshold captures edge cases\n- Moderate link following for depth", "code_blocks": [ { "language": "python", "code": "tech_doc_config = AdaptiveConfig(\n confidence_threshold=0.85,\n max_pages=30,\n top_k_links=3,\n min_gain_threshold=0.05 # Keep crawling for small gains\n)", "filename": "" } ], "images": [] }, { "heading": "News & Articles", "content": "Rationale:\n- Lower threshold (articles often repeat information)\n- Higher gain threshold (avoid duplicate stories)\n- More links per page (explore different perspectives)", "code_blocks": [ { "language": "python", "code": "news_config = AdaptiveConfig(\n confidence_threshold=0.6,\n max_pages=10,\n top_k_links=5,\n min_gain_threshold=0.15 # Stop quickly on repetition\n)", "filename": "" } ], "images": [] }, { "heading": "E-commerce", "content": "Rationale:\n- Balanced threshold for product variations\n- Focused link following (avoid infinite products)\n- Standard gain threshold", "code_blocks": [ { "language": "python", "code": "ecommerce_config = AdaptiveConfig(\n confidence_threshold=0.7,\n max_pages=20,\n top_k_links=2,\n min_gain_threshold=0.1\n)", "filename": "" } ], "images": [] }, { "heading": "Research & Academic", "content": "Rationale:\n- Very high threshold for completeness\n- Many pages allowed for thorough research\n- Very low gain threshold to capture references", "code_blocks": [ { "language": "python", "code": "research_config = AdaptiveConfig(\n confidence_threshold=0.9,\n max_pages=50,\n top_k_links=4,\n min_gain_threshold=0.02 # Very low - capture citations\n)", "filename": "" } ], "images": [] }, { "heading": "Performance Optimization", "content": "", "code_blocks": [], "images": [] }, { "heading": "Memory Management", "content": "", "code_blocks": [ { "language": "python", "code": "# For large crawls, use streaming\nconfig = AdaptiveConfig(\n max_pages=100,\n save_state=True,\n state_path=\"large_crawl.json\"\n)\n\n# Periodically clean state\nif len(state.knowledge_base) >…", "filename": "" } ], "images": [] }, { "heading": "Parallel Processing", "content": "", "code_blocks": [ { "language": "python", "code": "# Use multiple start points\nstart_urls = [\n \"https://docs.example.com/intro\",\n \"https://docs.example.com/api\",\n \"https://docs.example.com/guides\"\n]\n\n# Crawl in parallel\ntasks = […", "filename": "" } ], "images": [] }, { "heading": "Debugging & Analysis", "content": "", "code_blocks": [], "images": [] }, { "heading": "Enable Debugging", "content": "", "code_blocks": [ { "language": "python", "code": "import logging\n\nlogging.basicConfig(level=logging.DEBUG)\nadaptive = AdaptiveCrawler(crawler, config, verbose=True)", "filename": "" } ], "images": [] }, { "heading": "Analyze Crawl Patterns", "content": "", "code_blocks": [ { "language": "python", "code": "# After crawling\nstate = await adaptive.digest(start_url, query)\n\n# Analyze link selection\nprint(\"Link selection order:\")\nfor i, url in enumerate(state.crawl_order):\n print(f\"{i+1}. {url}\")\n\n#…", "filename": "" } ], "images": [] }, { "heading": "Export for Analysis", "content": "", "code_blocks": [ { "language": "python", "code": "# Export detailed metrics\nimport json\n\nmetrics = {\n \"query\": query,\n \"total_pages\": len(state.crawled_urls),\n \"confidence\": adaptive.confidence,\n \"coverage_stats\":…", "filename": "" } ], "images": [] }, { "heading": "Custom Strategies", "content": "", "code_blocks": [], "images": [] }, { "heading": "Implementing a Custom Strategy", "content": "", "code_blocks": [ { "language": "python", "code": "from crawl4ai.adaptive_crawler import CrawlStrategy\n\nclass DomainSpecificStrategy(CrawlStrategy):\n def calculate_coverage(self, state: CrawlState) -> float:\n # Custom coverage calculation…", "filename": "" } ], "images": [] }, { "heading": "Combining Strategies", "content": "", "code_blocks": [ { "language": "python", "code": "class HybridStrategy(CrawlStrategy):\n def __init__(self):\n self.strategies = [\n TechnicalDocStrategy(),\n SemanticSimilarityStrategy(),…", "filename": "" } ], "images": [] }, { "heading": "Best Practices", "content": "", "code_blocks": [], "images": [] }, { "heading": "1. Start Conservative", "content": "Begin with default settings and adjust based on results:", "code_blocks": [ { "language": "python", "code": "# Start with defaults\nresult = await adaptive.digest(url, query)\n\n# Analyze and adjust\nif adaptive.confidence < 0.7:\n config.max_pages += 10\n config.confidence_threshold -= 0.1", "filename": "" } ], "images": [] }, { "heading": "2. Monitor Resource Usage", "content": "", "code_blocks": [ { "language": "python", "code": "import psutil\n\n# Check memory before large crawls\nmemory_percent = psutil.virtual_memory().percent\nif memory_percent > 80:\n config.max_pages = min(config.max_pages, 20)", "filename": "" } ], "images": [] }, { "heading": "3. Use Domain Knowledge", "content": "", "code_blocks": [ { "language": "python", "code": "# For API documentation\nif \"api\" in start_url:\n config.top_k_links = 2 # APIs have clear structure\n\n# For blogs\nif \"blog\" in start_url:\n config.min_gain_threshold = 0.2 # Avoid similar posts", "filename": "" } ], "images": [] }, { "heading": "4. Validate Results", "content": "", "code_blocks": [ { "language": "python", "code": "# Always validate the knowledge base\nrelevant_content = adaptive.get_relevant_content(top_k=10)\n\n# Check coverage\nquery_terms = set(query.lower().split())\ncovered_terms = set()\n\nfor doc in…", "filename": "" } ], "images": [] }, { "heading": "Next Steps", "content": "Explore [Custom Strategy Implementation](../tutorials/custom-adaptive-strategies.md)\nLearn about [Knowledge Base Management](../tutorials/knowledge-base-management.md)\nSee [Performance…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "adaptive", "scoring", "coverage", "consistency", "saturation", "link ranking", "BM25", "domain-specific", "performance", "debugging", "custom strategies", "best practices", "Crawl4AI" ], "prerequisites": [], "_content_hash": "a6319cb032b0b93217b594fd31bca2e6fb95064ac47b5ef17ef6e6e3408e8e10", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/advanced-features/", "title": "Overview of Some Important Advanced Features", "type": "guide", "summary": "A guide to advanced Crawl4AI features including proxy usage, PDF/screenshot capture, SSL certificates, custom headers, session persistence, robots.txt compliance, and anti-bot techniques.", "sections": [ { "heading": "Overview of Some Important Advanced Features", "content": "Crawl4AI offers multiple power-user features that go beyond simple crawling. This tutorial covers:\n\n1. **Proxy Usage**\n\n2. **Capturing PDFs & Screenshots**\n\n3. **Handling SSL Certificates**\n\n4.…", "code_blocks": [], "images": [] }, { "heading": "1. Proxy Usage", "content": "If you need to route your crawl traffic through a proxy—whether for IP rotation, geo-testing, or privacy—Crawl4AI supports it via `BrowserConfig.proxy_config`.\n\n**Key Points**\n\n- **`proxy_config`**…", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig\n\nasync def main():\n browser_cfg = BrowserConfig(\n proxy_config={\n \"server\":…", "filename": "" } ], "images": [] }, { "heading": "2. Capturing PDFs & Screenshots", "content": "Sometimes you need a visual record of a page or a PDF “printout.” Crawl4AI can do both in one pass:\n\n**Why PDF + Screenshot?**\n\n- Large or complex pages can be slow or error-prone with “traditional”…", "code_blocks": [ { "language": "python", "code": "import os, asyncio\nfrom base64 import b64decode\nfrom crawl4ai import AsyncWebCrawler, CacheMode, CrawlerRunConfig\n\nasync def main():\n run_config = CrawlerRunConfig(…", "filename": "" } ], "images": [] }, { "heading": "3. Handling SSL Certificates", "content": "If you need to verify or export a site’s SSL certificate—for compliance, debugging, or data analysis—Crawl4AI can fetch it during the crawl:\n\n**Key Points**\n\n- **`fetch_ssl_certificate=True`**…", "code_blocks": [ { "language": "python", "code": "import asyncio, os\nfrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode\n\nasync def main():\n tmp_dir = os.path.join(os.getcwd(), \"tmp\")\n os.makedirs(tmp_dir, exist_ok=True)…", "filename": "" } ], "images": [] }, { "heading": "4. Custom Headers", "content": "Sometimes you need to set custom headers (e.g., language preferences, authentication tokens, or specialized user-agent strings). You can do this in multiple ways:\n\n**Notes**\n\n- Some sites may react…", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler\n\nasync def main():\n # Option 1: Set headers at the crawler strategy level\n crawler1 = AsyncWebCrawler(\n # The underlying strategy can…", "filename": "" } ], "images": [] }, { "heading": "5. Session Persistence & Local Storage", "content": "Crawl4AI can preserve cookies and localStorage so you can continue where you left off—ideal for logging into sites or skipping repeated auth flows.", "code_blocks": [], "images": [] }, { "heading": "5.1 `storage_state`", "content": "", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler\n\nasync def main():\n storage_dict = {\n \"cookies\": [\n {\n \"name\": \"session\",\n \"value\": \"abcd1234\",…", "filename": "" } ], "images": [] }, { "heading": "5.2 Exporting & Reusing State", "content": "You can sign in once, export the browser context, and reuse it later—without re-entering credentials.\n\n- **`await context.storage_state(path=\"my_storage.json\")`**: Exports cookies, localStorage, etc.…", "code_blocks": [], "images": [] }, { "heading": "6. Robots.txt Compliance", "content": "Crawl4AI supports respecting robots.txt rules with efficient caching:\n\n**Key Points**\n\n- Robots.txt files are cached locally for efficiency\n\n- Cache is stored in…", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig\n\nasync def main():\n # Enable robots.txt checking in config\n config = CrawlerRunConfig(\n check_robots_txt=True #…", "filename": "" } ], "images": [] }, { "heading": "Putting It All Together", "content": "Here’s a snippet that combines multiple “advanced” features (proxy, PDF, screenshot, SSL, custom headers, and session reuse) into one run. Normally, you’d tailor each setting to your project’s needs.", "code_blocks": [ { "language": "python", "code": "import os, asyncio\nfrom base64 import b64decode\nfrom crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode\n\nasync def main():\n # 1. Browser config with proxy + headless…", "filename": "" } ], "images": [] }, { "heading": "7. Anti-Bot Features (Stealth Mode & Undetected Browser)", "content": "Crawl4AI provides two powerful features to bypass bot detection:", "code_blocks": [], "images": [] }, { "heading": "7.1 Stealth Mode", "content": "Stealth mode uses playwright-stealth to modify browser fingerprints and behaviors. Enable it with a simple flag:\n\n**When to use**: Sites with basic bot detection (checking navigator.webdriver,…", "code_blocks": [ { "language": "python", "code": "browser_config = BrowserConfig(\n enable_stealth=True, # Activates stealth mode\n headless=False\n)", "filename": "" } ], "images": [] }, { "heading": "7.2 Undetected Browser", "content": "For advanced bot detection, use the undetected browser adapter:\n\n**When to use**: Sites with sophisticated bot detection (Cloudflare, DataDome, etc.)", "code_blocks": [ { "language": "python", "code": "from crawl4ai import UndetectedAdapter\nfrom crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy\n\n# Create undetected adapter\nadapter = UndetectedAdapter()\nstrategy =…", "filename": "" } ], "images": [] }, { "heading": "7.3 Combining Both", "content": "For maximum evasion, combine stealth mode with undetected browser:", "code_blocks": [ { "language": "python", "code": "browser_config = BrowserConfig(\n enable_stealth=True, # Enable stealth\n headless=False\n)\n\nadapter = UndetectedAdapter() # Use undetected browser", "filename": "" } ], "images": [] }, { "heading": "Choosing the Right Approach", "content": "| Detection Level | Recommended Approach |\n| --- | --- |\n| No protection | Regular browser |\n| Basic checks | Regular + Stealth mode |\n| Advanced protection | Undetected browser |\n| Maximum evasion |…", "code_blocks": [], "images": [] }, { "heading": "Conclusion & Next Steps", "content": "You've now explored several **advanced** features:\n\n- **Proxy Usage**\n\n- **PDF & Screenshot** capturing for large or critical pages\n\n- **SSL Certificate** retrieval & exporting\n\n- **Custom Headers**…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "Crawl4AI", "proxy", "PDF", "screenshot", "SSL", "custom headers", "session persistence", "storage state", "robots.txt", "anti-bot", "stealth mode", "undetected browser" ], "prerequisites": [], "_content_hash": "301d3656906ec78dff2743e6608dace0a6bf490db4cdea00846115e9ddc46291", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/anti-bot-and-fallback/", "title": "Anti-Bot Detection & Fallback", "type": "guide", "summary": "Explains how Crawl4AI detects anti-bot blocks and describes the layered retry/fallback system using proxies, retries, and custom fetch functions to retrieve content from protected sites.", "sections": [ { "heading": "How Detection Works", "content": "After each crawl attempt, Crawl4AI inspects the HTTP status code and HTML content for known anti-bot signals:\n\n- **HTTP 403/429** with short or empty response bodies\n- **Challenge pages** —…", "code_blocks": [], "images": [] }, { "heading": "Configuration Options", "content": "All anti-bot retry options live on `CrawlerRunConfig`:\n\n| Parameter | Type | Default | Description |\n| --- | --- | --- | --- |\n| `proxy_config` | `ProxyConfig`, `list[ProxyConfig]`, or `None` |…", "code_blocks": [], "images": [] }, { "heading": "Escalation Chain", "content": "Each retry round tries every proxy in `proxy_config` in order. If all rounds are exhausted and the page is still blocked, the fallback fetch function is called as a last resort.\n\nWorst-case attempts…", "code_blocks": [ { "language": "text", "code": "For each round (1 + max_retries rounds):\n 1. Try proxy_config[0] (or direct if proxy_config is None)\n 2. If blocked → try proxy_config[1]\n 3. If blocked → try proxy_config[2]\n 4. ...…", "filename": "" } ], "images": [] }, { "heading": "Crawl Stats", "content": "Every crawl result includes a `crawl_stats` dict with detailed attempt tracking:", "code_blocks": [ { "language": "python", "code": "result.crawl_stats = {\n \"attempts\": 3, # total browser attempts made\n \"retries\": 1, # retry rounds used (0 = succeeded first round)\n \"proxies_used\": […", "filename": "" } ], "images": [] }, { "heading": "Usage Examples", "content": "", "code_blocks": [], "images": [] }, { "heading": "Simple Retry (No Proxy)", "content": "Retry the crawl up to 3 times when blocking is detected. Useful when blocks are intermittent or IP-based.", "code_blocks": [ { "language": "python", "code": "from crawl4ai import AsyncWebCrawler\nfrom crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig\n\nasync with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:\n result = await…", "filename": "" } ], "images": [] }, { "heading": "Single Proxy", "content": "Pass a single `ProxyConfig` — it's used on every attempt. Same behavior as always.", "code_blocks": [ { "language": "python", "code": "from crawl4ai.async_configs import ProxyConfig\n\nconfig = CrawlerRunConfig(\n max_retries=2,\n proxy_config=ProxyConfig(\n server=\"http://proxy.example.com:8080\",\n username=\"user\",…", "filename": "" } ], "images": [] }, { "heading": "Direct-First, Then Proxies", "content": "Try without a proxy first, then escalate to proxies if blocked. Use `ProxyConfig.DIRECT` (or the string `\"direct\"`) in the list to represent a no-proxy attempt.\n\nWith this setup, each round tries…", "code_blocks": [ { "language": "python", "code": "config = CrawlerRunConfig(\n max_retries=1,\n proxy_config=[\n ProxyConfig.DIRECT, # Try without proxy first\n ProxyConfig(…", "filename": "" } ], "images": [] }, { "heading": "Proxy List (Escalation)", "content": "Pass a list of proxies. They're tried in order — first one that works wins. Within each retry round, the entire list is tried again.\n\nWith this setup, each round tries the datacenter proxy first,…", "code_blocks": [ { "language": "python", "code": "config = CrawlerRunConfig(\n max_retries=1,\n proxy_config=[\n ProxyConfig(\n server=\"http://datacenter-proxy.example.com:8080\",\n username=\"user\",…", "filename": "" } ], "images": [] }, { "heading": "Fallback Fetch Function", "content": "When all browser-based attempts fail, call a custom async function as a last resort. This function receives the URL and must return raw HTML as a string. The returned HTML is processed through the…", "code_blocks": [ { "language": "python", "code": "import aiohttp\n\nasync def my_scraping_api(url: str) -> str:\n \"\"\"Fetch HTML via an external scraping API.\"\"\"\n async with aiohttp.ClientSession() as session:\n async with session.get(…", "filename": "" } ], "images": [] }, { "heading": "Full Escalation (All Features Combined)", "content": "This example combines every layer: stealth mode, a list of proxies tried in order, retries, and a final fetch function.\n\n**What happens step by step:**\n\n| Round | Attempt | What runs |\n| --- | --- |…", "code_blocks": [ { "language": "python", "code": "import aiohttp\nfrom crawl4ai import AsyncWebCrawler\nfrom crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig, ProxyConfig\n\n# Last-resort: fetch HTML via an external service\nasync def…", "filename": "" } ], "images": [] }, { "heading": "Tips", "content": "- Start with `max_retries=0` and a `fallback_fetch_function` if you just want a safety net without burning time on retries.\n- Order proxies cheapest-first — datacenter proxies before residential,…", "code_blocks": [], "images": [] }, { "heading": "See Also", "content": "- [Proxy & Security](../proxy-security/) — Proxy setup, authentication, and rotation\n- [Undetected Browser](../undetected-browser/) — Stealth mode and browser fingerprint evasion\n- [Session…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "Crawl4AI", "anti-bot", "proxy", "retry", "fallback", "Cloudflare", "Akamai", "stealth", "web scraping", "Python" ], "prerequisites": [], "_content_hash": "761a25a8bfc2d3124f85bd552c4c77c1fdab50abd49e44df6d51772cf1c949c9", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/crawl-dispatcher/", "title": "Crawl Dispatcher", "type": "overview", "summary": "This page announces the upcoming Crawl Dispatcher module in Crawl4AI, a feature for handling thousands of crawling tasks simultaneously with efficient resource management and real-time monitoring.", "sections": [ { "heading": "Crawl Dispatcher", "content": "We’re excited to announce a **Crawl Dispatcher** module that can handle **thousands** of crawling tasks simultaneously. By efficiently managing system resources (memory, CPU, network), this…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "Crawl4AI", "Crawl Dispatcher", "multi-URL crawling", "performance monitoring", "scalable" ], "prerequisites": [], "_content_hash": "2bdcb7f67c8ae58365e1b233a4aeb5ce7420f330f604caae09af161537bbb725", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/file-downloading/", "title": "Download Handling in Crawl4AI", "type": "guide", "summary": "This guide explains how to use Crawl4AI to handle file downloads during crawling, including enabling downloads, specifying download locations, triggering downloads, and accessing downloaded files.", "sections": [ { "heading": "Overview", "content": "This guide explains how to use Crawl4AI to handle file downloads during crawling. You'll learn how to trigger downloads, specify download locations, and access downloaded files.", "code_blocks": [], "images": [] }, { "heading": "Enabling Downloads", "content": "To enable downloads, set the `accept_downloads` parameter in the `BrowserConfig` object and pass it to the crawler.", "code_blocks": [ { "language": "", "code": "from crawl4ai.async_configs import BrowserConfig, AsyncWebCrawler\n\nasync def main():\n config = BrowserConfig(accept_downloads=True) # Enable downloads globally\n async with…", "filename": "" } ], "images": [] }, { "heading": "Specifying Download Location", "content": "Specify the download directory using the `downloads_path` attribute in the `BrowserConfig` object. If not provided, Crawl4AI defaults to creating a \"downloads\" directory inside the `.crawl4ai` folder…", "code_blocks": [ { "language": "", "code": "from crawl4ai.async_configs import BrowserConfig\nimport os\n\ndownloads_path = os.path.join(os.getcwd(), \"my_downloads\") # Custom download path\nos.makedirs(downloads_path, exist_ok=True)\n\nconfig =…", "filename": "" } ], "images": [] }, { "heading": "Triggering Downloads", "content": "Downloads are typically triggered by user interactions on a web page, such as clicking a download button. Use `js_code` in `CrawlerRunConfig` to simulate these actions and `wait_for` to allow…", "code_blocks": [ { "language": "", "code": "from crawl4ai.async_configs import CrawlerRunConfig\n\nconfig = CrawlerRunConfig(\n js_code=\"\"\"\n const downloadLink = document.querySelector('a[href$=\".exe\"]');\n if (downloadLink) {…", "filename": "" } ], "images": [] }, { "heading": "Accessing Downloaded Files", "content": "The `downloaded_files` attribute of the `CrawlResult` object contains paths to downloaded files.", "code_blocks": [ { "language": "", "code": "if result.downloaded_files:\n print(\"Downloaded files:\")\n for file_path in result.downloaded_files:\n print(f\"- {file_path}\")\n file_size = os.path.getsize(file_path)…", "filename": "" } ], "images": [] }, { "heading": "Example: Downloading Multiple Files", "content": "", "code_blocks": [ { "language": "", "code": "from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig\nimport os\nfrom pathlib import Path\n\nasync def download_multiple_files(url: str, download_path: str):\n config =…", "filename": "" } ], "images": [] }, { "heading": "Important Considerations", "content": "- **Browser Context:** Downloads are managed within the browser context. Ensure `js_code` correctly targets the download triggers on the webpage.\n\n- **Timing:** Use `wait_for` in `CrawlerRunConfig`…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "Crawl4AI", "file downloads", "BrowserConfig", "CrawlerRunConfig", "AsyncWebCrawler", "downloaded_files", "js_code", "wait_for" ], "prerequisites": [], "_content_hash": "8569f41f8478282952c6e7984fae4060471fe4cebb838f0370bb71cea534f368", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/hooks-auth/", "title": "Hooks & Auth in AsyncWebCrawler", "type": "guide", "summary": "This guide explains how to use hooks in AsyncWebCrawler to customize the crawling pipeline at specific stages, including authentication, route blocking, and pre/post-processing, with a detailed…", "sections": [ { "heading": "Introduction", "content": "##### Hooks & Auth in AsyncWebCrawler\n\n\n\nCrawl4AI’s **hooks** let you customize the crawler at specific points in the pipeline:\n\n\n\n\n1.  **`on_browser_created`** – After browser creation.\n\n2.…", "code_blocks": [], "images": [] }, { "heading": "Example: Using Hooks in AsyncWebCrawler", "content": "", "code_blocks": [ { "language": "python", "code": "import asyncio\nimport json\nfrom crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode\nfrom playwright.async_api import Page, BrowserContext\n\nasync def main():\n print(\"🔗 Hooks…", "filename": "" } ], "images": [] }, { "heading": "Hook Lifecycle Summary", "content": "1.  **`on_browser_created`** :\n\n - Browser is up, but **no** pages or contexts yet.\n\n - Light setup only—don’t try to open or close pages here (that belongs in…", "code_blocks": [], "images": [] }, { "heading": "When to Handle Authentication", "content": "**Recommended** : Use **`on_page_context_created`** if you need to:\n\n\n\n\n- Navigate to a login page or fill forms\n\n- Set cookies or localStorage tokens\n\n- Block resource routes to avoid ads\n\n\n\n\nThis…", "code_blocks": [], "images": [] }, { "heading": "Additional Considerations", "content": "- **Session Management** : If you want multiple `arun()` calls to reuse a single session, pass `session_id=` in your `CrawlerRunConfig`. Hooks remain the same.\n\n- **Performance** : Hooks can slow…", "code_blocks": [], "images": [] }, { "heading": "Conclusion", "content": "Hooks provide **fine-grained** control over:\n\n\n\n\n- **Browser** creation (light tasks only)\n\n- **Page** and **context** creation (auth, route blocking)\n\n- **Navigation** phases\n\n- **Final…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "hooks", "authentication", "AsyncWebCrawler", "Playwright", "Python", "web crawling" ], "prerequisites": [], "_content_hash": "fda9aa77b90df500c4c0773fe126774b815d9350f3f646f41e4ea30aa7631fda", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/identity-based-crawling/", "title": "Identity Based Crawling - Crawl4AI Documentation (v0.9.x)", "type": "guide", "summary": "This guide explains how to use Crawl4AI's Managed Browsers and BrowserProfiler to preserve a user's authentic digital identity while crawling, including persistent profiles, Magic Mode fallback, and…", "sections": [ { "heading": "Preserve Your Identity with Crawl4AI", "content": "Crawl4AI empowers you to navigate and interact with the web using your **authentic digital identity**, ensuring you’re recognized as a human and not mistaken for a bot. This tutorial covers:\n\n1.…", "code_blocks": [], "images": [] }, { "heading": "1. Managed Browsers: Your Digital Identity Solution", "content": "**Managed Browsers** let developers create and use **persistent browser profiles**. These profiles store local storage, cookies, and other session data, letting you browse as your **real self**…", "code_blocks": [], "images": [] }, { "heading": "Key Benefits", "content": "- **Authentic Browsing Experience**: Retain session data and browser fingerprints as though you’re a normal user.\n- **Effortless Configuration**: Once you log in or solve CAPTCHAs in your chosen data…", "code_blocks": [], "images": [] }, { "heading": "Creating a User Data Directory (Command-Line Approach via Playwright)", "content": "If you installed Crawl4AI (which installs Playwright under the hood), you already have a Playwright-managed Chromium on your system. Follow these steps to launch that **Chromium** from your command…", "code_blocks": [ { "language": "bash", "code": "python -m playwright install --dry-run", "filename": "" }, { "language": "bash", "code": "playwright install --dry-run", "filename": "" }, { "language": "bash", "code": "~/.cache/ms-playwright/chromium-1234/chrome-linux/chrome", "filename": "" }, { "language": "bash", "code": "# Linux example\n~/.cache/ms-playwright/chromium-1234/chrome-linux/chrome \\\n --user-data-dir=/home//my_chrome_profile", "filename": "" }, { "language": "bash", "code": "# macOS example (Playwright’s internal binary)\n~/Library/Caches/ms-playwright/chromium-1234/chrome-mac/Chromium.app/Contents/MacOS/Chromium \\\n --user-data-dir=/Users//my_chrome_profile", "filename": "" }, { "language": "powershell", "code": "# Windows example (PowerShell/cmd)\n\"C:\\Users\\\\AppData\\Local\\ms-playwright\\chromium-1234\\chrome-win\\chrome.exe\" ^\n --user-data-dir=\"C:\\Users\\\\my_chrome_profile\"", "filename": "" }, { "language": "python", "code": "from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig\n\nbrowser_config = BrowserConfig(\n headless=True,\n use_managed_browser=True,…", "filename": "" } ], "images": [] }, { "heading": "Creating a Profile Using the Crawl4AI CLI (Easiest)", "content": "If you prefer a guided, interactive setup, use the built-in CLI to create and manage persistent browser profiles.\n\n1. Launch the profile manager (`crwl profiles`).\n\n2. Choose \"Create new profile\" and…", "code_blocks": [ { "language": "bash", "code": "crwl profiles", "filename": "" }, { "language": "python", "code": "from crawl4ai import AsyncWebCrawler, BrowserConfig\n\nprofile_path = \"/home//.crawl4ai/profiles/test_profile_1\"\n\nbrowser_config = BrowserConfig(\n headless=True,\n use_managed_browser=True,…", "filename": "" } ], "images": [] }, { "heading": "3. Using Managed Browsers in Crawl4AI", "content": "Once you have a data directory with your session data, pass it to **`BrowserConfig`**:", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig\n\nasync def main():\n # 1) Reference your persistent data directory\n browser_config = BrowserConfig(…", "filename": "" } ], "images": [] }, { "heading": "Workflow", "content": "1. **Login** externally (via CLI or your normal Chrome with `--user-data-dir=...`).\n2. **Close** that browser.\n3. **Use** the same folder in `user_data_dir=` in Crawl4AI.\n4. **Crawl** – The site sees…", "code_blocks": [], "images": [] }, { "heading": "4. Magic Mode: Simplified Automation", "content": "If you **don’t** need a persistent profile or identity-based approach, **Magic Mode** offers a quick way to simulate human-like browsing without storing long-term data.\n\n**Magic Mode**:\n\n- Simulates…", "code_blocks": [ { "language": "python", "code": "from crawl4ai import AsyncWebCrawler, CrawlerRunConfig\n\nasync with AsyncWebCrawler() as crawler:\n result = await crawler.arun(\n url=\"https://example.com\",\n config=CrawlerRunConfig(…", "filename": "" } ], "images": [] }, { "heading": "5. Comparing Managed Browsers vs. Magic Mode", "content": "| Feature | **Managed Browsers** | **Magic Mode** |\n| --- | --- | --- |\n| **Session Persistence** | Full localStorage/cookies retained in user_data_dir | No persistent data (fresh each run) |\n|…", "code_blocks": [], "images": [] }, { "heading": "6. Using the BrowserProfiler Class", "content": "Crawl4AI provides a dedicated `BrowserProfiler` class for managing browser profiles, making it easy to create, list, and delete profiles for identity-based browsing.", "code_blocks": [], "images": [] }, { "heading": "Creating and Managing Profiles with BrowserProfiler", "content": "The `BrowserProfiler` class offers a comprehensive API for browser profile management:\n\n**How profile creation works:**\n1. A browser window opens for you to interact with\n2. You log in to websites,…", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import BrowserProfiler\n\nasync def manage_profiles():\n # Create a profiler instance\n profiler = BrowserProfiler()\n\n # Create a profile interactively - opens a…", "filename": "" } ], "images": [] }, { "heading": "Interactive Profile Management", "content": "The `BrowserProfiler` also offers an interactive management console that guides you through profile creation, listing, and deletion:", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import BrowserProfiler, AsyncWebCrawler, BrowserConfig\n\n# Define a function to use a profile for crawling\nasync def crawl_with_profile(profile_path, url):…", "filename": "" } ], "images": [] }, { "heading": "Legacy Methods", "content": "For backward compatibility, the previous methods on `ManagedBrowser` are still available, but they delegate to the new `BrowserProfiler` class:", "code_blocks": [ { "language": "python", "code": "from crawl4ai.browser_manager import ManagedBrowser\n\n# These methods still work but use BrowserProfiler internally\nprofiles = ManagedBrowser.list_profiles()", "filename": "" } ], "images": [] }, { "heading": "Complete Example", "content": "See the full example in `docs/examples/identity_based_browsing.py` for a complete demonstration of creating and using profiles for authenticated browsing using the new `BrowserProfiler` class.", "code_blocks": [], "images": [] }, { "heading": "7. Locale, Timezone, and Geolocation Control", "content": "In addition to using persistent profiles, Crawl4AI supports customizing your browser's locale, timezone, and geolocation settings. These features enhance your identity-based browsing experience by…", "code_blocks": [], "images": [] }, { "heading": "Setting Locale and Timezone", "content": "You can set the browser's locale and timezone through `CrawlerRunConfig`:\n\n**How it works:**\n- `locale` affects language preferences, date formats, number formats, etc.\n- `timezone_id` affects…", "code_blocks": [ { "language": "python", "code": "from crawl4ai import AsyncWebCrawler, CrawlerRunConfig\n\nasync with AsyncWebCrawler() as crawler:\n result = await crawler.arun(\n url=\"https://example.com\",\n config=CrawlerRunConfig(…", "filename": "" } ], "images": [] }, { "heading": "Configuring Geolocation", "content": "Control the GPS coordinates reported by the browser's geolocation API:\n\n**Important notes:**\n- When `geolocation` is specified, the browser is automatically granted permission to access location\n-…", "code_blocks": [ { "language": "python", "code": "from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, GeolocationConfig\n\nasync with AsyncWebCrawler() as crawler:\n result = await crawler.arun(\n url=\"https://maps.google.com\", # Or any…", "filename": "" } ], "images": [] }, { "heading": "Combining with Managed Browsers", "content": "These settings work perfectly with managed browsers for a complete identity solution:\n\nCombining persistent profiles with precise geolocation and region settings gives you complete control over your…", "code_blocks": [ { "language": "python", "code": "from crawl4ai import (\n AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, \n GeolocationConfig\n)\n\nbrowser_config = BrowserConfig(\n use_managed_browser=True,…", "filename": "" } ], "images": [] }, { "heading": "8. Summary", "content": "- **Create** your user-data directory either:\n - By launching Chrome/Chromium externally with `--user-data-dir=/some/path`\n - Or by using the built-in `BrowserProfiler.create_profile()` method\n -…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "Crawl4AI", "Managed Browsers", "BrowserProfiler", "Playwright", "Magic Mode", "identity-based crawling", "persistent profiles", "geolocation", "locale", "timezone" ], "prerequisites": [], "_content_hash": "69b98b5f69113d5987f4e9ffda67084fa7178c8bd95b34f76a729127485ff1fe", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/lazy-loading/", "title": "Lazy Loading - Crawl4AI Documentation (v0.9.x)", "type": "guide", "summary": "This guide explains how to handle lazy-loaded images in Crawl4AI by using wait_for_images, scan_full_page, and scroll_delay settings, and how to combine these with media filters and domain exclusions.", "sections": [ { "heading": "Handling Lazy-Loaded Images", "content": "Many websites now load images **lazily** as you scroll. If you need to ensure they appear in your final crawl (and in `result.media`), consider:\n\n1. **`wait_for_images=True`** – Wait for images to…", "code_blocks": [], "images": [] }, { "heading": "Example: Ensuring Lazy Images Appear", "content": "**Explanation**:\n\n- **`wait_for_images=True`**\n\n The crawler tries to ensure images have finished loading before finalizing the HTML.\n\n- **`scan_full_page=True`**\n\n Tells the crawler to attempt…", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig\nfrom crawl4ai.async_configs import CacheMode\n\nasync def main():\n config = CrawlerRunConfig(\n # Force the…", "filename": "" } ], "images": [] }, { "heading": "Combining with Other Link & Media Filters", "content": "You can still combine **lazy-load** logic with the usual **exclude_external_images**, **exclude_domains**, or link filtration:\n\nThis approach ensures you see **all** images from the main domain while…", "code_blocks": [ { "language": "python", "code": "config = CrawlerRunConfig(\n wait_for_images=True,\n scan_full_page=True,\n scroll_delay=0.5,\n\n # Filter out external images if you only want local ones\n exclude_external_images=True,…", "filename": "" } ], "images": [] }, { "heading": "Tips & Troubleshooting", "content": "1. **Long Pages**\n\n - Setting `scan_full_page=True` on extremely long or infinite-scroll pages can be resource-intensive.\n\n - Consider using [hooks](../../core/page-interaction/) or specialized…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "lazy-loading", "images", "Crawl4AI", "web crawling", "scroll", "wait_for_images", "scan_full_page", "scroll_delay", "media filters" ], "prerequisites": [], "_content_hash": "cc9921dc0dbcb0f3f555e9c7b2e21d23b3f4231ae4ac5cbfa4ee379197cfb710", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/multi-url-crawling/", "title": "Advanced Multi-URL Crawling with Dispatchers", "type": "api", "summary": "This page covers advanced multi-URL crawling using dispatchers in Crawl4AI, including RateLimiter, CrawlerMonitor, MemoryAdaptiveDispatcher, and SemaphoreDispatcher, with usage examples and…", "sections": [ { "heading": "1. Introduction", "content": "> **Heads Up** : Crawl4AI supports advanced dispatchers for **parallel** or **throttled** crawling, providing dynamic rate limiting and memory usage checks. The built-in `arun_many()` function uses…", "code_blocks": [], "images": [] }, { "heading": "2. Core Components", "content": "", "code_blocks": [], "images": [] }, { "heading": "2.1 Rate Limiter", "content": "Here’s the revised and simplified explanation of the **RateLimiter** , focusing on constructor parameters and adhering to your markdown style and mkDocs guidelines.", "code_blocks": [ { "language": "python", "code": "class RateLimiter:\n def __init__(\n # Random delay range between requests\n base_delay: Tuple[float, float] = (1.0, 3.0), \n\n # Maximum backoff delay\n max_delay: float =…", "filename": "" } ], "images": [] }, { "heading": "RateLimiter Constructor Parameters", "content": "The **RateLimiter** is a utility that helps manage the pace of requests to avoid overloading servers or getting blocked due to rate limits. It operates internally to delay requests and handle retries…", "code_blocks": [ { "language": "python", "code": "from crawl4ai import RateLimiter\n\n# Create a RateLimiter with custom settings\nrate_limiter = RateLimiter(\n base_delay=(2.0, 4.0), # Random delay between 2-4 seconds\n max_delay=30.0, #…", "filename": "" } ], "images": [] }, { "heading": "2.2 Crawler Monitor", "content": "The CrawlerMonitor provides real-time visibility into crawling operations:\n\n **Display Modes** :\n\n- **DETAILED** : Shows individual task status, memory usage, and timing\n- **AGGREGATED** : Displays…", "code_blocks": [ { "language": "python", "code": "from crawl4ai import CrawlerMonitor, DisplayMode\nmonitor = CrawlerMonitor(\n # Maximum rows in live display\n max_visible_rows=15, \n\n # DETAILED or AGGREGATED view…", "filename": "" } ], "images": [] }, { "heading": "3. Available Dispatchers", "content": "", "code_blocks": [], "images": [] }, { "heading": "3.1 MemoryAdaptiveDispatcher (Default)", "content": "Automatically manages concurrency based on system memory usage:\n\n **Constructor Parameters:** \n\n1. **`memory_threshold_percent`** (`float`, default: `90.0`)\n\n Specifies the memory usage threshold…", "code_blocks": [ { "language": "python", "code": "from crawl4ai.async_dispatcher import MemoryAdaptiveDispatcher\n\ndispatcher = MemoryAdaptiveDispatcher(\n memory_threshold_percent=90.0, # Pause if memory exceeds this\n check_interval=1.0,…", "filename": "" } ], "images": [] }, { "heading": "3.2 SemaphoreDispatcher", "content": "Provides simple concurrency control with a fixed limit:\n\n **Constructor Parameters:** \n\n1. **`max_session_permit`** (`int`, default: `20`)\n\n The maximum number of concurrent crawling tasks allowed,…", "code_blocks": [ { "language": "python", "code": "from crawl4ai.async_dispatcher import SemaphoreDispatcher\n\ndispatcher = SemaphoreDispatcher(\n max_session_permit=20, # Maximum concurrent tasks\n rate_limiter=RateLimiter( #…", "filename": "" } ], "images": [] }, { "heading": "4. Usage Examples", "content": "", "code_blocks": [], "images": [] }, { "heading": "4.1 Batch Processing (Default)", "content": "**Review:** \n\n- **Purpose:** Executes a batch crawl with all URLs processed together after crawling is complete.\n- **Dispatcher:** Uses `MemoryAdaptiveDispatcher` to manage concurrency and system…", "code_blocks": [ { "language": "python", "code": "async def crawl_batch():\n browser_config = BrowserConfig(headless=True, verbose=False)\n run_config = CrawlerRunConfig(\n cache_mode=CacheMode.BYPASS,\n stream=False # Default: get…", "filename": "" } ], "images": [] }, { "heading": "4.2 Streaming Mode", "content": "**Review:** \n\n- **Purpose:** Enables streaming to process results as soon as they’re available.\n- **Dispatcher:** Uses `MemoryAdaptiveDispatcher` for concurrency and memory management.\n- **Stream:**…", "code_blocks": [ { "language": "python", "code": "async def crawl_streaming():\n browser_config = BrowserConfig(headless=True, verbose=False)\n run_config = CrawlerRunConfig(\n cache_mode=CacheMode.BYPASS,\n stream=True # Enable…", "filename": "" } ], "images": [] }, { "heading": "4.3 Semaphore-based Crawling", "content": "**Review:** \n\n- **Purpose:** Uses `SemaphoreDispatcher` to limit concurrency with a fixed number of slots.\n- **Dispatcher:** Configured with a semaphore to control parallel crawling tasks.\n- **Rate…", "code_blocks": [ { "language": "python", "code": "async def crawl_with_semaphore(urls):\n browser_config = BrowserConfig(headless=True, verbose=False)\n run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)\n\n dispatcher =…", "filename": "" } ], "images": [] }, { "heading": "4.4 Robots.txt Consideration", "content": "**Review:** \n\n- **Purpose:** Ensures compliance with `robots.txt` rules for ethical and legal web crawling.\n- **Configuration:** Set `check_robots_txt=True` to validate each URL against `robots.txt`…", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode\n\nasync def main():\n urls = [\n \"https://example1.com\",\n \"https://example2.com\",…", "filename": "" } ], "images": [] }, { "heading": "5. Dispatch Results", "content": "Each crawl result includes dispatch information:\n\nAccess via `result.dispatch_result`:", "code_blocks": [ { "language": "python", "code": "@dataclass\nclass DispatchResult:\n task_id: str\n memory_usage: float\n peak_memory: float\n start_time: datetime\n end_time: datetime\n error_message: str = \"\"\nCopy", "filename": "" }, { "language": "python", "code": "for result in results:\n if result.success:\n dr = result.dispatch_result\n print(f\"URL: {result.url}\")\n print(f\"Memory: {dr.memory_usage:.1f}MB\")\n print(f\"Duration:…", "filename": "" } ], "images": [] }, { "heading": "6. URL-Specific Configurations", "content": "When crawling diverse content types, you often need different configurations for different URLs. For example:\n- PDFs need specialized extraction\n- Blog pages benefit from content filtering\n- Dynamic…", "code_blocks": [], "images": [] }, { "heading": "6.1 Basic URL Pattern Matching", "content": "**Important** : A `CrawlerRunConfig` without `url_matcher` (or with `url_matcher=None`) matches ALL URLs. This makes it perfect as a default/fallback configuration.\n\nThe `url_matcher` parameter…", "code_blocks": [ { "language": "python", "code": "from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, MatchMode\nfrom crawl4ai.processors.pdf import PDFContentScrapingStrategy\nfrom crawl4ai.extraction_strategy import…", "filename": "" } ], "images": [] }, { "heading": "6.2 Advanced Pattern Matching", "content": "**Important** : A `CrawlerRunConfig` without `url_matcher` (or with `url_matcher=None`) matches ALL URLs. This makes it perfect as a default/fallback configuration.\n\nThe `url_matcher` parameter…", "code_blocks": [], "images": [] }, { "heading": "Glob Patterns (Strings)", "content": "", "code_blocks": [ { "language": "python", "code": "# Simple patterns\n\"*.pdf\" # Any PDF file\n\"*/api/*\" # Any URL with /api/ in path\n\"https://*.example.com/*\" # Subdomain matching\n\"*://example.com/blog/*\" # Any…", "filename": "" } ], "images": [] }, { "heading": "Custom Functions", "content": "", "code_blocks": [ { "language": "python", "code": "# Complex logic with lambdas\nlambda url: url.startswith('https://') and 'secure' in url\nlambda url: len(url) > 50 and url.count('/') > 5\nlambda url: any(domain in url for domain in ['api.', 'data.',…", "filename": "" } ], "images": [] }, { "heading": "Mixed Lists with AND/OR Logic", "content": "", "code_blocks": [ { "language": "python", "code": "# Combine multiple conditions\nCrawlerRunConfig(\n url_matcher=[\n \"https://*\", # Must be HTTPS\n lambda url: 'internal' in url, # Must contain 'internal'…", "filename": "" } ], "images": [] }, { "heading": "6.3 Practical Example: News Site Crawler", "content": "", "code_blocks": [ { "language": "python", "code": "async def crawl_news_site():\n dispatcher = MemoryAdaptiveDispatcher(\n memory_threshold_percent=70.0,\n rate_limiter=RateLimiter(base_delay=(1.0, 2.0))\n )\n\n configs = [\n #…", "filename": "" } ], "images": [] }, { "heading": "6.4 Best Practices", "content": "- **Order Matters** : Configs are evaluated in order - put specific patterns before general ones\n- **Default Config Behavior** :\n - A config without `url_matcher` matches ALL URLs\n - Always include…", "code_blocks": [ { "language": "python", "code": "config = CrawlerRunConfig(url_matcher=\"*.pdf\")\nprint(config.is_match(\"https://example.com/doc.pdf\")) # True\n\ndefault_config = CrawlerRunConfig() # No…", "filename": "" } ], "images": [] }, { "heading": "7. Summary", "content": "1. **Two Dispatcher Types** :\n\n - MemoryAdaptiveDispatcher (default): Dynamic concurrency based on memory\n - SemaphoreDispatcher: Fixed concurrency limit\n\n2. **Optional Components** :\n\n -…", "code_blocks": [], "images": [] } ], "parameters": [ { "name": "base_delay", "type": "Tuple[float, float]", "description": "The range for a random delay (in seconds) between consecutive requests to the same domain. A random delay is chosen between base_delay[0] and base_delay[1] for each request.", "default": "(1.0, 3.0)", "required": false }, { "name": "max_delay", "type": "float", "description": "The maximum allowable delay when rate-limiting errors occur. When servers return rate-limit responses (e.g., 429 or 503), the delay increases exponentially with jitter, capped at this value.", "default": "60.0", "required": false }, { "name": "max_retries", "type": "int", "description": "The maximum number of retries for a request if rate-limiting errors occur. After encountering a rate-limit response, the RateLimiter retries the request up to this number of times.", "default": "3", "required": false }, { "name": "rate_limit_codes", "type": "List[int]", "description": "A list of HTTP status codes that trigger the rate-limiting logic. These status codes indicate the server is overwhelmed or actively limiting requests.", "default": "[429, 503]", "required": false }, { "name": "memory_threshold_percent", "type": "float", "description": "Specifies the memory usage threshold (as a percentage). If system memory usage exceeds this value, the dispatcher pauses crawling to prevent system overload.", "default": "90.0", "required": false }, { "name": "check_interval", "type": "float", "description": "The interval (in seconds) at which the dispatcher checks system memory usage.", "default": "1.0", "required": false }, { "name": "max_session_permit", "type": "int", "description": "The maximum number of concurrent crawling tasks allowed. This ensures resource limits are respected while maintaining concurrency.", "default": "10", "required": false }, { "name": "memory_wait_timeout", "type": "float", "description": "Optional timeout (in seconds). If memory usage exceeds memory_threshold_percent for longer than this duration, a MemoryError is raised.", "default": "600.0", "required": false }, { "name": "rate_limiter", "type": "RateLimiter", "description": "Optional rate-limiting logic to avoid server-side blocking (e.g., for handling 429 or 503 errors).", "default": "None", "required": false }, { "name": "monitor", "type": "CrawlerMonitor", "description": "Optional monitoring for real-time task tracking and performance insights.", "default": "None", "required": false }, { "name": "max_session_permit", "type": "int", "description": "The maximum number of concurrent crawling tasks allowed, irrespective of semaphore slots.", "default": "20", "required": false }, { "name": "rate_limiter", "type": "RateLimiter", "description": "Optional rate-limiting logic to avoid overwhelming servers.", "default": "None", "required": false }, { "name": "monitor", "type": "CrawlerMonitor", "description": "Optional monitoring for tracking task progress and resource usage.", "default": "None", "required": false }, { "name": "max_visible_rows", "type": "int", "description": "Maximum rows in live display.", "default": "15", "required": false }, { "name": "display_mode", "type": "DisplayMode", "description": "DETAILED or AGGREGATED view.", "default": "DisplayMode.DETAILED", "required": false } ], "see_also": [], "tags": [ "Crawl4AI", "dispatcher", "RateLimiter", "MemoryAdaptiveDispatcher", "SemaphoreDispatcher", "CrawlerMonitor", "concurrency", "rate limiting", "web crawling", "async", "Python" ], "prerequisites": [], "_content_hash": "77b33cadfdf25a242e60dbf57a6d410c35c585bfda48406e0eb4c7f441c1725f", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/network-console-capture/", "title": "Network Requests & Console Message Capturing", "type": "guide", "summary": "This page explains how to capture network requests and browser console messages during a crawl using Crawl4AI, including configuration, example usage, data structures, benefits, and use cases.", "sections": [ { "heading": "Configuration", "content": "To enable network and console capturing, use these configuration options:", "code_blocks": [ { "language": "python", "code": "from crawl4ai import AsyncWebCrawler, CrawlerRunConfig\n\n# Enable both network request capture and console message capture\nconfig = CrawlerRunConfig(\n capture_network_requests=True, # Capture all…", "filename": "" } ], "images": [] }, { "heading": "Example Usage", "content": "", "code_blocks": [ { "language": "python", "code": "import asyncio\nimport json\nfrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig\n\nasync def main():\n # Enable both network request capture and console message capture\n config =…", "filename": "" } ], "images": [] }, { "heading": "Captured Data Structure", "content": "The `result.network_requests` contains a list of dictionaries, each representing a network event with these common fields:\n\n| Field | Description |\n| --- | --- |\n| `event_type` | Type of event:…", "code_blocks": [], "images": [] }, { "heading": "Network Requests", "content": "The `result.network_requests` contains a list of dictionaries, each representing a network event with these common fields:\n\n| Field | Description |\n| --- | --- |\n| `event_type` | Type of event:…", "code_blocks": [], "images": [] }, { "heading": "Request Event Fields", "content": "", "code_blocks": [ { "language": "json", "code": "{\n \"event_type\": \"request\",\n \"url\": \"https://example.com/api/data.json\",\n \"method\": \"GET\",\n \"headers\": {\"User-Agent\": \"...\", \"Accept\": \"...\"},\n \"post_data\": \"key=value&otherkey=value\",…", "filename": "" } ], "images": [] }, { "heading": "Response Event Fields", "content": "", "code_blocks": [ { "language": "json", "code": "{\n \"event_type\": \"response\",\n \"url\": \"https://example.com/api/data.json\",\n \"status\": 200,\n \"status_text\": \"OK\",\n \"headers\": {\"Content-Type\": \"application/json\", \"Cache-Control\": \"...\"},…", "filename": "" } ], "images": [] }, { "heading": "Failed Request Event Fields", "content": "", "code_blocks": [ { "language": "json", "code": "{\n \"event_type\": \"request_failed\",\n \"url\": \"https://example.com/missing.png\",\n \"method\": \"GET\",\n \"resource_type\": \"image\",\n \"failure_text\": \"net::ERR_ABORTED 404\",\n \"timestamp\": 1633456789.789\n}", "filename": "" } ], "images": [] }, { "heading": "Console Messages", "content": "The `result.console_messages` contains a list of dictionaries, each representing a console message with these common fields:\n\n| Field | Description |\n| --- | --- |\n| `type` | Message type: `\"log\"`,…", "code_blocks": [], "images": [] }, { "heading": "Console Message Example", "content": "", "code_blocks": [ { "language": "json", "code": "{\n \"type\": \"error\",\n \"text\": \"Uncaught TypeError: Cannot read property 'length' of undefined\",\n \"location\": \"https://example.com/script.js:123:45\",\n \"timestamp\": 1633456790.123\n}", "filename": "" } ], "images": [] }, { "heading": "Key Benefits", "content": "**Full Request Visibility** : Capture all network activity including:\n- Requests (URLs, methods, headers, post data)\n- Responses (status codes, headers, timing)\n- Failed requests (with error…", "code_blocks": [], "images": [] }, { "heading": "Use Cases", "content": "**API Discovery** : Identify hidden endpoints and data flows in single-page applications\n\n**Debugging** : Track down JavaScript errors affecting page functionality\n\n**Security Auditing** : Detect…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "Crawl4AI", "network capture", "console capture", "debugging", "security", "performance", "API discovery" ], "prerequisites": [], "_content_hash": "d9aa7df721a13da71678ee324a44f49924597d1069e446e041912448c9dc5874", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/pdf-parsing/", "title": "PDF Processing Strategies", "type": "api", "summary": "This page describes the PDF processing strategies in Crawl4AI, including PDFCrawlerStrategy and PDFContentScrapingStrategy, which enable crawling and extracting content from PDF files.", "sections": [ { "heading": "Overview", "content": "`PDFCrawlerStrategy` is an implementation of `AsyncCrawlerStrategy` designed specifically for PDF documents. Instead of interpreting the input URL as an HTML webpage, this strategy treats it as a…", "code_blocks": [], "images": [] }, { "heading": "When to Use", "content": "Use `PDFCrawlerStrategy` when you need to:\n- Process PDF files using the `AsyncWebCrawler`.\n- Handle PDFs from both web URLs (e.g., `https://example.com/document.pdf`) and local file paths (e.g.,…", "code_blocks": [], "images": [] }, { "heading": "Key Methods and Their Behavior", "content": "- **`__init__(self, logger: AsyncLogger = None)`** :\n\n- Initializes the strategy.\n- `logger`: An optional `AsyncLogger` instance (from `crawl4ai.async_logger`) for logging purposes.\n\n- **`async…", "code_blocks": [], "images": [] }, { "heading": "Example Usage", "content": "", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig\nfrom crawl4ai.processors.pdf import PDFCrawlerStrategy, PDFContentScrapingStrategy\n\nasync def main():\n # Initialize the PDF…", "filename": "" } ], "images": [] }, { "heading": "Pros and Cons", "content": "**Pros:** \n- Enables `AsyncWebCrawler` to handle PDF sources directly using familiar `arun` calls.\n- Provides a consistent interface for specifying PDF sources (URLs or local paths).\n-…", "code_blocks": [], "images": [] }, { "heading": "Key Configuration Attributes", "content": "When initializing `PDFContentScrapingStrategy`, you can configure its behavior using the following attributes:\n- **`extract_images: bool = False`** : If `True`, the strategy will attempt to…", "code_blocks": [], "images": [] } ], "parameters": [ { "name": "logger", "type": "AsyncLogger", "description": "An optional AsyncLogger instance for logging purposes.", "default": "None", "required": false }, { "name": "url", "type": "str", "description": "The URL pointing to a PDF file.", "default": "", "required": true }, { "name": "save_images_locally", "type": "bool", "description": "If True (and extract_images is also True), extracted images will be saved to disk in the image_save_dir.", "default": "False", "required": false }, { "name": "extract_images", "type": "bool", "description": "If True, the strategy will attempt to extract images from the PDF.", "default": "False", "required": false }, { "name": "image_save_dir", "type": "str", "description": "Specifies the directory where extracted images should be saved if save_images_locally is True.", "default": "None", "required": false }, { "name": "batch_size", "type": "int", "description": "Defines how many PDF pages are processed in a single batch.", "default": "4", "required": false }, { "name": "logger", "type": "AsyncLogger", "description": "An optional AsyncLogger instance for logging.", "default": "None", "required": false }, { "name": "url", "type": "str", "description": "The path or URL to the PDF file.", "default": "", "required": true }, { "name": "html", "type": "str", "description": "Typically an empty string when used with PDFCrawlerStrategy, as the content is a PDF, not HTML.", "default": "", "required": true }, { "name": "url", "type": "str", "description": "The path or URL to the PDF file.", "default": "", "required": true }, { "name": "html", "type": "str", "description": "Typically an empty string when used with PDFCrawlerStrategy, as the content is a PDF, not HTML.", "default": "", "required": true }, { "name": "url", "type": "str", "description": "The URL or path to the PDF file.", "default": "", "required": true } ], "see_also": [], "tags": [ "PDF", "Crawl4AI", "AsyncWebCrawler", "PDFCrawlerStrategy", "PDFContentScrapingStrategy", "scraping", "parsing", "document processing", "image extraction", "metadata extraction" ], "prerequisites": [], "_content_hash": "a9f3d19898fc0ba1d5161b18ee8440150e804d2a1d000f41be537a2779b19624", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/proxy-security/", "title": "Proxy & Security", "type": "guide", "summary": "This guide covers proxy configuration and security features in Crawl4AI, including SSL certificate analysis and proxy rotation strategies.", "sections": [ { "heading": "Understanding Proxy Configuration", "content": "Crawl4AI recommends configuring proxies per request through `CrawlerRunConfig.proxy_config`. This gives you precise control, enables rotation strategies, and keeps examples simple enough to copy,…", "code_blocks": [], "images": [] }, { "heading": "Basic Proxy Setup", "content": "Configure proxies that apply to each crawl operation:\n\nWhy request-level?\n\n`CrawlerRunConfig.proxy_config` keeps each request self-contained, so swapping proxies or rotation strategies is just a…", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, ProxyConfig\n\nrun_config = CrawlerRunConfig(proxy_config=ProxyConfig(server=\"http://proxy.example.com:8080\"))\n#…", "filename": "" } ], "images": [] }, { "heading": "Supported Proxy Formats", "content": "The `ProxyConfig.from_string()` method supports multiple formats:", "code_blocks": [ { "language": "python", "code": "from crawl4ai import ProxyConfig\n\n# HTTP proxy with authentication\nproxy1 = ProxyConfig.from_string(\"http://user:pass@192.168.1.1:8080\")\n\n# HTTPS proxy\nproxy2 =…", "filename": "" } ], "images": [] }, { "heading": "Authenticated Proxies", "content": "For proxies requiring authentication:", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler,BrowserConfig, CrawlerRunConfig, ProxyConfig\n\nrun_config = CrawlerRunConfig(\n proxy_config=ProxyConfig(…", "filename": "" } ], "images": [] }, { "heading": "Environment Variable Configuration", "content": "Load proxies from environment variables for easy configuration:", "code_blocks": [ { "language": "python", "code": "import os\nfrom crawl4ai import ProxyConfig, CrawlerRunConfig\n\n# Set environment variable\nos.environ[\"PROXIES\"] = \"ip1:port1:user1:pass1,ip2:port2:user2:pass2,ip3:port3\"\n\n# Load all proxies\nproxies =…", "filename": "" } ], "images": [] }, { "heading": "Rotating Proxies", "content": "Crawl4AI supports automatic proxy rotation to distribute requests across multiple proxy servers. Rotation is applied per request using a rotation strategy on `CrawlerRunConfig`.", "code_blocks": [], "images": [] }, { "heading": "Proxy Rotation (recommended)", "content": "", "code_blocks": [ { "language": "python", "code": "import asyncio\nimport re\nfrom crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode, ProxyConfig\nfrom crawl4ai.proxy_strategy import RoundRobinProxyStrategy\n\nasync def main():…", "filename": "" } ], "images": [] }, { "heading": "SSL Certificate Analysis", "content": "Combine proxy usage with SSL certificate inspection for enhanced security analysis. SSL certificate fetching is configured per request via `CrawlerRunConfig`.", "code_blocks": [], "images": [] }, { "heading": "Per-Request SSL Certificate Analysis", "content": "", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig\n\nrun_config = CrawlerRunConfig(\n proxy_config={\n \"server\": \"http://proxy.example.com:8080\",…", "filename": "" } ], "images": [] }, { "heading": "Security Best Practices", "content": "", "code_blocks": [], "images": [] }, { "heading": "1. Proxy Rotation for Anonymity", "content": "", "code_blocks": [ { "language": "python", "code": "from crawl4ai import CrawlerRunConfig, ProxyConfig\nfrom crawl4ai.proxy_strategy import RoundRobinProxyStrategy\n\n# Use multiple proxies to avoid IP blocking\nproxies =…", "filename": "" } ], "images": [] }, { "heading": "2. SSL Certificate Verification", "content": "", "code_blocks": [ { "language": "python", "code": "from crawl4ai import CrawlerRunConfig\n\n# Always verify SSL certificates when possible\n# Per-request (affects specific requests)\nrun_config = CrawlerRunConfig(fetch_ssl_certificate=True)", "filename": "" } ], "images": [] }, { "heading": "3. Environment Variable Security", "content": "", "code_blocks": [ { "language": "bash", "code": "# Use environment variables for sensitive proxy credentials\n# Avoid hardcoding usernames/passwords in code\nexport PROXIES=\"ip1:port1:user1:pass1,ip2:port2:user2:pass2\"", "filename": "" } ], "images": [] }, { "heading": "4. SOCKS5 for Enhanced Security", "content": "", "code_blocks": [ { "language": "python", "code": "from crawl4ai import CrawlerRunConfig\n\n# Prefer SOCKS5 proxies for better protocol support\nrun_config = CrawlerRunConfig(proxy_config=\"socks5://proxy.example.com:1080\")", "filename": "" } ], "images": [] }, { "heading": "Migration from Deprecated `proxy` Parameter", "content": "The legacy `proxy` argument on `BrowserConfig` is deprecated. Configure proxies through `CrawlerRunConfig.proxy_config` so each request fully describes its network settings.", "code_blocks": [ { "language": "python", "code": "# Old (deprecated) approach\n# from crawl4ai import BrowserConfig\n# browser_config = BrowserConfig(proxy_config=\"http://proxy.example.com:8080\")\n\n# New (preferred) approach\nfrom crawl4ai import…", "filename": "" } ], "images": [] }, { "heading": "Safe Logging of Proxies", "content": "", "code_blocks": [ { "language": "python", "code": "from crawl4ai import ProxyConfig\n\ndef safe_proxy_repr(proxy: ProxyConfig):\n if getattr(proxy, \"username\", None):\n return f\"{proxy.server} (auth: ****)\"\n return proxy.server", "filename": "" } ], "images": [] }, { "heading": "Troubleshooting", "content": "", "code_blocks": [], "images": [] }, { "heading": "Common Issues", "content": "Proxy connection failed\n\n- Verify the proxy server is reachable from your network.\n- Double-check authentication credentials.\n- Ensure the protocol matches (`http`, `https`, or `socks5`).\n\nSSL…", "code_blocks": [], "images": [] }, { "heading": "See Also", "content": "[Anti-Bot Detection & Fallback](../anti-bot-and-fallback/) — Automatic retry with proxy escalation and fallback functions when anti-bot blocking is detected", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "proxy", "security", "ssl", "certificate", "rotation", "crawl4ai", "authentication", "socks5" ], "prerequisites": [], "_content_hash": "d5518b24e6f706e1dfa5093f264fef073d2fa813a83fe817826a9769bbec22a6", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/session-management/", "title": "Session Management - Crawl4AI Documentation (v0.9.x)", "type": "guide", "summary": "This page explains how to use session management in Crawl4AI to maintain state across multiple requests, enabling sequential crawling, dynamic content handling, and advanced techniques like custom…", "sections": [ { "heading": "Overview", "content": "Session management in Crawl4AI is a powerful feature that allows you to maintain state across multiple requests, making it particularly suitable for handling complex multi-step crawling tasks. It…", "code_blocks": [], "images": [] }, { "heading": "Basic Session Usage", "content": "Use `BrowserConfig` and `CrawlerRunConfig` to maintain state with a `session_id`:", "code_blocks": [ { "language": "python", "code": "from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig\n\nasync with AsyncWebCrawler() as crawler:\n session_id = \"my_session\"\n\n # Define configurations\n config1 =…", "filename": "" } ], "images": [] }, { "heading": "Dynamic Content with Sessions", "content": "Here's an example of crawling GitHub commits across multiple pages while preserving session state:", "code_blocks": [ { "language": "python", "code": "from crawl4ai.async_configs import CrawlerRunConfig\nfrom crawl4ai import JsonCssExtractionStrategy\nfrom crawl4ai.cache_context import CacheMode\n\nasync def crawl_dynamic_content():\n url =…", "filename": "" } ], "images": [] }, { "heading": "Example 1: Basic Session-Based Crawling", "content": "A simple example using session-based crawling:\n\nThis example shows:\n1. Reusing the same `session_id` across multiple requests.\n2. Executing JavaScript to load more content dynamically.\n3. Properly…", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig\nfrom crawl4ai.cache_context import CacheMode\n\nasync def basic_session_crawl():\n async with AsyncWebCrawler() as…", "filename": "" } ], "images": [] }, { "heading": "Advanced Technique 1: Custom Execution Hooks", "content": "> Warning: You might feel confused by the end of the next few examples 😅, so make sure you are comfortable with the order of the parts before you start this.\n\nUse custom hooks to handle complex…", "code_blocks": [ { "language": "python", "code": "async def advanced_session_crawl_with_hooks():\n first_commit = \"\"\n\n async def on_execution_started(page):\n nonlocal first_commit\n try:\n while True:…", "filename": "" } ], "images": [] }, { "heading": "Advanced Technique 2: Integrated JavaScript Execution and Waiting", "content": "Combine JavaScript execution and waiting logic for concise handling of dynamic content:", "code_blocks": [ { "language": "python", "code": "async def integrated_js_and_wait_crawl():\n async with AsyncWebCrawler() as crawler:\n session_id = \"integrated_session\"\n url = \"https://github.com/example/repo/commits/main\"…", "filename": "" } ], "images": [] }, { "heading": "Common Use Cases for Sessions", "content": "1. **Authentication Flows**: Login and interact with secured pages.\n\n2. **Pagination Handling**: Navigate through multiple pages.\n\n3. **Form Submissions**: Fill forms, submit, and process…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "session management", "crawling", "browser automation", "async", "python", "crawl4ai" ], "prerequisites": [], "_content_hash": "fa62a727c57a013bd7f85806480ad509c46a4fd743a6a187c01ceeef8d9e90fa", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/ssl-certificate/", "title": "SSLCertificate Reference", "type": "api", "summary": "Reference for the SSLCertificate class in Crawl4AI, covering how to load, inspect, and export SSL/TLS certificate data, and how to use it with fetch_ssl_certificate=True in CrawlerRunConfig.", "sections": [ { "heading": "1. Overview", "content": "The **`SSLCertificate`** class encapsulates an SSL certificate’s data and allows exporting it in various formats (PEM, DER, JSON, or text). It’s used within **Crawl4AI** whenever you set…", "code_blocks": [ { "language": "python", "code": "class SSLCertificate:\n \"\"\"\n Represents an SSL certificate with methods to export in various formats.\n\n Main Methods:\n - from_url(url, timeout=10)\n - from_file(file_path)\n -…", "filename": "" } ], "images": [] }, { "heading": "Typical Use Case", "content": "- You **enable** certificate fetching in your crawl by:\n- After `arun()`, if `result.ssl_certificate` is present, it’s an instance of **`SSLCertificate`**.\n- You can **read** basic properties…", "code_blocks": [ { "language": "python", "code": "CrawlerRunConfig(fetch_ssl_certificate=True, ...)", "filename": "" } ], "images": [] }, { "heading": "2. Construction & Fetching", "content": "", "code_blocks": [], "images": [] }, { "heading": "2.1 from_url(url, timeout=10)", "content": "Manually load an SSL certificate from a given URL (port 443). Typically used internally, but you can call it directly if you want:", "code_blocks": [ { "language": "python", "code": "cert = SSLCertificate.from_url(\"https://example.com\")\nif cert:\n print(\"Fingerprint:\", cert.fingerprint)", "filename": "" } ], "images": [] }, { "heading": "2.2 from_file(file_path)", "content": "Load from a file containing certificate data in ASN.1 or DER. Rarely needed unless you have local cert files:", "code_blocks": [ { "language": "python", "code": "cert = SSLCertificate.from_file(\"/path/to/cert.der\")", "filename": "" } ], "images": [] }, { "heading": "2.3 from_binary(binary_data)", "content": "Initialize from raw binary. E.g., if you captured it from a socket or another source:", "code_blocks": [ { "language": "python", "code": "cert = SSLCertificate.from_binary(raw_bytes)", "filename": "" } ], "images": [] }, { "heading": "3. Common Properties", "content": "After obtaining a **`SSLCertificate`** instance (e.g. `result.ssl_certificate` from a crawl), you can read:\n\n1. **`issuer`** *(dict)*\n - E.g. `{\"CN\": \"My Root CA\", \"O\": \"...\"}`\n2. **`subject`**…", "code_blocks": [], "images": [] }, { "heading": "4. Export Methods", "content": "Once you have a **`SSLCertificate`** object, you can **export** or **inspect** it:", "code_blocks": [], "images": [] }, { "heading": "4.1 to_json(filepath=None) → Optional[str]", "content": "- Returns a JSON string containing the parsed certificate fields.\n- If `filepath` is provided, saves it to disk instead, returning `None`.\n\n**Usage**:", "code_blocks": [ { "language": "python", "code": "json_data = cert.to_json() # returns JSON string\ncert.to_json(\"certificate.json\") # writes file, returns None", "filename": "" } ], "images": [] }, { "heading": "4.2 to_pem(filepath=None) → Optional[str]", "content": "- Returns a PEM-encoded string (common for web servers).\n- If `filepath` is provided, saves it to disk instead.", "code_blocks": [ { "language": "python", "code": "pem_str = cert.to_pem() # in-memory PEM string\ncert.to_pem(\"/path/to/cert.pem\") # saved to file", "filename": "" } ], "images": [] }, { "heading": "4.3 to_der(filepath=None) → Optional[bytes]", "content": "- Returns the original DER (binary ASN.1) bytes.\n- If `filepath` is specified, writes the bytes there instead.", "code_blocks": [ { "language": "python", "code": "der_bytes = cert.to_der()\ncert.to_der(\"certificate.der\")", "filename": "" } ], "images": [] }, { "heading": "4.4 (Optional) export_as_text()", "content": "- If you see a method like `export_as_text()`, it typically returns an OpenSSL-style textual representation.\n- Not always needed, but can help for debugging or manual inspection.", "code_blocks": [], "images": [] }, { "heading": "5. Example Usage in Crawl4AI", "content": "Below is a minimal sample showing how the crawler obtains an SSL cert from a site, then reads or exports it. The code snippet:", "code_blocks": [ { "language": "python", "code": "import asyncio\nimport os\nfrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode\n\nasync def main():\n tmp_dir = \"tmp\"\n os.makedirs(tmp_dir, exist_ok=True)\n\n config =…", "filename": "" } ], "images": [] }, { "heading": "6. Notes & Best Practices", "content": "1. **Timeout**: `SSLCertificate.from_url` internally uses a default **10s** socket connect and wraps SSL.\n2. **Binary Form**: The certificate is loaded in ASN.1 (DER) form, then re-parsed by…", "code_blocks": [], "images": [] }, { "heading": "Summary", "content": "- **`SSLCertificate`** is a convenience class for capturing and exporting the **TLS certificate** from your crawled site(s).\n- Common usage is in the **`CrawlResult.ssl_certificate`** field,…", "code_blocks": [], "images": [] } ], "parameters": [ { "name": "from_url.url", "type": "str", "description": "The URL to load the SSL certificate from (port 443).", "default": "", "required": true }, { "name": "from_url.timeout", "type": "int", "description": "Socket connect timeout in seconds.", "default": "10", "required": false }, { "name": "from_file.file_path", "type": "str", "description": "Path to a file containing certificate data in ASN.1 or DER.", "default": "", "required": true }, { "name": "from_binary.binary_data", "type": "bytes", "description": "Raw binary certificate data.", "default": "", "required": true }, { "name": "to_json.filepath", "type": "Optional[str]", "description": "If provided, saves the JSON output to disk instead of returning a string.", "default": "None", "required": false }, { "name": "to_pem.filepath", "type": "Optional[str]", "description": "If provided, saves the PEM output to disk instead of returning a string.", "default": "None", "required": false }, { "name": "to_der.filepath", "type": "Optional[str]", "description": "If provided, writes the DER bytes to disk instead of returning bytes.", "default": "None", "required": false } ], "see_also": [], "tags": [ "SSLCertificate", "SSL", "TLS", "certificate", "Crawl4AI", "CrawlerRunConfig", "PEM", "DER", "JSON", "OpenSSL", "AsyncWebCrawler" ], "prerequisites": [], "_content_hash": "678bee01067ce4b13bbc2594cd9976d10d5c334cb2f9968896cc89cc433e5f4c", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/undetected-browser/", "title": "Undetected Browser Mode", "type": "guide", "summary": "This guide covers Crawl4AI's anti-bot features: Stealth Mode and Undetected Browser Mode, including how to use them, when to use each, and best practices for evading bot detection.", "sections": [ { "heading": "Overview", "content": "Crawl4AI offers two powerful anti-bot features to help you access websites with bot detection:\n\n- **Stealth Mode** - Uses playwright-stealth to modify browser fingerprints and behaviors\n-…", "code_blocks": [], "images": [] }, { "heading": "Anti-Bot Features Comparison", "content": "| Feature | Regular Browser | Stealth Mode | Undetected Browser |\n| --- | --- | --- | --- |\n| WebDriver Detection | ❌ | ✅ | ✅ |\n| Navigator Properties | ❌ | ✅ | ✅ |\n| Plugin Emulation | ❌ | ✅ | ✅ |\n|…", "code_blocks": [], "images": [] }, { "heading": "When to Use Each Approach", "content": "", "code_blocks": [], "images": [] }, { "heading": "Use Regular Browser + Stealth Mode When:", "content": "- Sites have basic bot detection (checking navigator.webdriver, plugins, etc.)\n- You need good performance with basic protection\n- Sites check for common automation indicators", "code_blocks": [], "images": [] }, { "heading": "Use Undetected Browser When:", "content": "- Sites employ sophisticated bot detection services (Cloudflare, DataDome, etc.)\n- Stealth mode alone isn't sufficient\n- You're willing to trade some performance for better evasion", "code_blocks": [], "images": [] }, { "heading": "Best Practice: Progressive Enhancement", "content": "- **Start with**: Regular browser + Stealth mode\n- **If blocked**: Switch to Undetected browser\n- **If still blocked**: Combine Undetected browser + Stealth mode", "code_blocks": [], "images": [] }, { "heading": "Stealth Mode", "content": "Stealth mode is the simpler anti-bot solution that works with both regular and undetected browsers:", "code_blocks": [ { "language": "python", "code": "from crawl4ai import AsyncWebCrawler, BrowserConfig\n\n# Enable stealth mode with regular browser\nbrowser_config = BrowserConfig(\n enable_stealth=True, # Simple flag to enable\n headless=False…", "filename": "" } ], "images": [] }, { "heading": "What Stealth Mode Does:", "content": "- Removes `navigator.webdriver` flag\n- Modifies browser fingerprints\n- Emulates realistic plugin behavior\n- Adjusts navigator properties\n- Fixes common automation leaks", "code_blocks": [], "images": [] }, { "heading": "Undetected Browser Mode", "content": "For sites with sophisticated bot detection that stealth mode can't bypass, use the undetected browser adapter:", "code_blocks": [], "images": [] }, { "heading": "Key Features", "content": "- **Drop-in Replacement**: Uses the same API as regular browser mode\n- **Enhanced Stealth**: Built-in patches to evade common detection methods\n- **Browser Adapter Pattern**: Seamlessly switch…", "code_blocks": [], "images": [] }, { "heading": "Quick Start", "content": "", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import (\n AsyncWebCrawler, \n BrowserConfig, \n CrawlerRunConfig,\n UndetectedAdapter\n)\nfrom crawl4ai.async_crawler_strategy import…", "filename": "" } ], "images": [] }, { "heading": "Combining Both Features", "content": "For maximum evasion, combine stealth mode with undetected browser:", "code_blocks": [ { "language": "python", "code": "from crawl4ai import AsyncWebCrawler, BrowserConfig, UndetectedAdapter\nfrom crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy\n\n# Create browser config with stealth…", "filename": "" } ], "images": [] }, { "heading": "Examples", "content": "", "code_blocks": [], "images": [] }, { "heading": "Example 1: Basic Stealth Mode", "content": "", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig\n\nasync def test_stealth_mode():\n # Simple stealth mode configuration\n browser_config = BrowserConfig(…", "filename": "" } ], "images": [] }, { "heading": "Example 2: Undetected Browser Mode", "content": "", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import (\n AsyncWebCrawler,\n BrowserConfig,\n CrawlerRunConfig,\n UndetectedAdapter\n)\nfrom crawl4ai.async_crawler_strategy import…", "filename": "" } ], "images": [] }, { "heading": "Browser Adapter Pattern", "content": "The undetected browser support is implemented using an adapter pattern, allowing seamless switching between different browser implementations:\n\nThe adapter handles:\n- JavaScript execution\n- Console…", "code_blocks": [ { "language": "python", "code": "# Regular browser adapter (default)\nfrom crawl4ai import PlaywrightAdapter\nregular_adapter = PlaywrightAdapter()\n\n# Undetected browser adapter\nfrom crawl4ai import…", "filename": "" } ], "images": [] }, { "heading": "Best Practices", "content": "- **Avoid Headless Mode**: Detection is easier in headless mode\n- **Use Reasonable Delays**: Don't rush through pages\n- **Rotate User Agents**: You can customize user agents\n- **Handle Failures…", "code_blocks": [ { "language": "python", "code": "browser_config = BrowserConfig(headless=False)", "filename": "" }, { "language": "python", "code": "crawler_config = CrawlerRunConfig(\n wait_time=3.0, # Wait 3 seconds after page load\n delay_before_return_html=2.0 # Additional delay\n)", "filename": "" }, { "language": "python", "code": "browser_config = BrowserConfig(\n headers={\"User-Agent\": \"your-user-agent\"}\n)", "filename": "" }, { "language": "python", "code": "if not result.success:\n print(f\"Crawl failed: {result.error_message}\")", "filename": "" } ], "images": [] }, { "heading": "Advanced Usage Tips", "content": "", "code_blocks": [], "images": [] }, { "heading": "Progressive Detection Handling", "content": "", "code_blocks": [ { "language": "python", "code": "async def crawl_with_progressive_evasion(url):\n # Step 1: Try regular browser with stealth\n browser_config = BrowserConfig(\n enable_stealth=True,\n headless=False\n )\n\n async…", "filename": "" } ], "images": [] }, { "heading": "Installation", "content": "The undetected browser dependencies are automatically installed when you run:\n\nThis command installs all necessary browser dependencies for both regular and undetected modes.", "code_blocks": [ { "language": "bash", "code": "crawl4ai-setup", "filename": "" } ], "images": [] }, { "heading": "Limitations", "content": "- **Performance**: Slightly slower than regular mode due to additional patches\n- **Headless Detection**: Some sites can still detect headless mode\n- **Resource Usage**: May use more resources than…", "code_blocks": [], "images": [] }, { "heading": "Troubleshooting", "content": "", "code_blocks": [], "images": [] }, { "heading": "Browser Not Found", "content": "Run the setup command:", "code_blocks": [ { "language": "bash", "code": "crawl4ai-setup", "filename": "" } ], "images": [] }, { "heading": "Detection Still Occurring", "content": "Try combining with other features:", "code_blocks": [ { "language": "python", "code": "crawler_config = CrawlerRunConfig(\n simulate_user=True, # Add user simulation\n magic=True, # Enable magic mode\n wait_time=5.0, # Longer waits\n)", "filename": "" } ], "images": [] }, { "heading": "Performance Issues", "content": "If experiencing slow performance:", "code_blocks": [ { "language": "python", "code": "# Use selective undetected mode only for protected sites\nif is_protected_site(url):\n adapter = UndetectedAdapter()\nelse:\n adapter = PlaywrightAdapter() # Default adapter", "filename": "" } ], "images": [] }, { "heading": "Future Plans", "content": "**Note**: In future versions of Crawl4AI, we may enable stealth mode and undetected browser by default to provide better out-of-the-box success rates. For now, users should explicitly enable these…", "code_blocks": [], "images": [] }, { "heading": "Conclusion", "content": "Crawl4AI provides flexible anti-bot solutions:\n\n- **Start Simple**: Use regular browser + stealth mode for most sites\n- **Escalate if Needed**: Switch to undetected browser for sophisticated…", "code_blocks": [], "images": [] }, { "heading": "See Also", "content": "- [Advanced Features](../advanced-features/) - Overview of all advanced features\n- [Proxy & Security](../proxy-security/) - Using proxies with anti-bot features\n- [Session…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "Crawl4AI", "Stealth Mode", "Undetected Browser", "Anti-bot", "Playwright", "Browser Automation", "Web Scraping" ], "prerequisites": [], "_content_hash": "33163d8cc0605541ef6bd082f38540286b73414c1c0fff6c32d39bc5baca4fef", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/advanced/virtual-scroll/", "title": "Virtual Scroll - Crawl4AI Documentation (v0.9.x)", "type": "guide", "summary": "This page explains Crawl4AI's Virtual Scroll feature for handling virtual scrolling websites, covering configuration, usage examples, comparison with scan_full_page, and performance tips.", "sections": [ { "heading": "Understanding Virtual Scroll", "content": "Modern websites increasingly use virtual scrolling (also called windowed rendering or viewport rendering) to handle large datasets efficiently. This technique only renders visible items in the DOM,…", "code_blocks": [ { "language": "text", "code": "Traditional Scroll: Virtual Scroll:\n┌─────────────┐ ┌─────────────┐\n│ Item 1 │ │ Item 11 │ <- Items 1-10 removed\n│ Item 2 │ │ Item 12 │…", "filename": "" } ], "images": [] }, { "heading": "Basic Usage", "content": "", "code_blocks": [ { "language": "python", "code": "from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, VirtualScrollConfig\n\n# Configure virtual scroll\nvirtual_config = VirtualScrollConfig(\n container_selector=\"#feed\", # CSS selector for…", "filename": "" } ], "images": [] }, { "heading": "Configuration Parameters", "content": "###### VirtualScrollConfig\n\n| Parameter | Type | Default | Description |\n| --- | --- | --- | --- |\n| `container_selector` | `str` | Required | CSS selector for the scrollable container |\n|…", "code_blocks": [], "images": [] }, { "heading": "Real-World Examples", "content": "###### Twitter-like Timeline\n\nTwitter replaces tweets as you scroll.\n\n###### Instagram Grid\n\nInstagram uses virtualized grid for performance.\n\n###### Mixed Content (News Feed)\n\nSome sites mix static and…", "code_blocks": [ { "language": "python", "code": "from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, VirtualScrollConfig, BrowserConfig\n\nasync def crawl_twitter_timeline():\n # Twitter replaces tweets as you scroll\n virtual_config =…", "filename": "" }, { "language": "python", "code": "async def crawl_instagram_grid():\n # Instagram uses virtualized grid for performance\n virtual_config = VirtualScrollConfig(\n container_selector=\"article\", # Main feed container…", "filename": "" }, { "language": "python", "code": "async def crawl_mixed_feed():\n # Featured articles stay, regular articles virtualize\n virtual_config = VirtualScrollConfig(\n container_selector=\".main-feed\",\n scroll_count=25,…", "filename": "" } ], "images": [] }, { "heading": "Virtual Scroll vs scan_full_page", "content": "Both features handle dynamic content, but serve different purposes:\n\n| Feature | Virtual Scroll | scan_full_page |\n| --- | --- | --- |\n| **Purpose** | Capture content that's replaced during scroll |…", "code_blocks": [], "images": [] }, { "heading": "Combining with Extraction", "content": "Virtual Scroll works seamlessly with extraction strategies.", "code_blocks": [ { "language": "python", "code": "from crawl4ai import LLMExtractionStrategy, LLMConfig\n\n# Define extraction schema\nschema = {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\", \n \"properties\": {…", "filename": "" } ], "images": [] }, { "heading": "Performance Tips", "content": "- **Container Selection**: Be specific with selectors. Using the correct container improves performance.\n- **Scroll Count**: Start conservative and increase as needed.\n- **Wait Times**: Adjust based…", "code_blocks": [ { "language": "python", "code": "# Start with fewer scrolls\nvirtual_config = VirtualScrollConfig(\n container_selector=\"#feed\",\n scroll_count=10 # Test with 10, increase if needed\n)", "filename": "" }, { "language": "python", "code": "# Fast sites\nwait_after_scroll=0.2\n\n# Slower sites or heavy content\nwait_after_scroll=1.5", "filename": "" }, { "language": "python", "code": "browser_config = BrowserConfig(headless=False)\nasync with AsyncWebCrawler(config=browser_config) as crawler:\n # Watch the scrolling happen", "filename": "" } ], "images": [] }, { "heading": "How It Works Internally", "content": "- **Detection Phase**: Scrolls and compares HTML to detect behavior\n- **Capture Phase**: For replaced content, stores HTML chunks at each position\n- **Merge Phase**: Combines all chunks, removing…", "code_blocks": [], "images": [] }, { "heading": "Error Handling", "content": "Virtual Scroll handles errors gracefully. If the container isn't found, crawling continues normally without virtual scroll.", "code_blocks": [ { "language": "python", "code": "# If container not found or scrolling fails\nresult = await crawler.arun(url=\"...\", config=config)\n\nif result.success:\n # Virtual scroll worked or wasn't needed\n print(f\"Captured…", "filename": "" } ], "images": [] }, { "heading": "Complete Example", "content": "See our comprehensive example that demonstrates:\n- Twitter-like feeds\n- Instagram grids\n- Traditional infinite scroll\n- Mixed content scenarios\n- Performance comparisons\n\nThe example includes a local…", "code_blocks": [ { "language": "bash", "code": "# Run the examples\ncd docs/examples\npython virtual_scroll_example.py", "filename": "" } ], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "virtual scroll", "windowed rendering", "viewport rendering", "infinite scroll", "Crawl4AI", "web crawling", "dynamic content", "scraping" ], "prerequisites": [], "_content_hash": "a132c3fd838fb9c4e775f090484a5c22217412f2e188f800efcce0d5b4c05225", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/api/adaptive-crawler/", "title": "AdaptiveCrawler", "type": "api", "summary": "The AdaptiveCrawler class implements intelligent web crawling that automatically determines when sufficient information has been gathered to answer a query. It uses a three-layer scoring system to…", "sections": [ { "heading": "Constructor", "content": "###### Parameters\n\n- **crawler** (`AsyncWebCrawler`): The underlying web crawler instance to use for fetching pages\n- **config** (`Optional[AdaptiveConfig]`): Configuration settings for adaptive…", "code_blocks": [ { "language": "python", "code": "AdaptiveCrawler(\n crawler: AsyncWebCrawler,\n config: Optional[AdaptiveConfig] = None\n)", "filename": "" } ], "images": [] }, { "heading": "Primary Method", "content": "###### digest()\n\nThe main method that performs adaptive crawling starting from a URL with a specific query.\n\n###### Parameters\n\n- **start_url** (`str`): The starting URL for crawling\n- **query**…", "code_blocks": [ { "language": "python", "code": "async def digest(\n start_url: str,\n query: str,\n resume_from: Optional[Union[str, Path]] = None\n) -> CrawlState", "filename": "" }, { "language": "python", "code": "async with AsyncWebCrawler() as crawler:\n adaptive = AdaptiveCrawler(crawler)\n state = await adaptive.digest(\n start_url=\"https://docs.python.org\",\n query=\"async context…", "filename": "" } ], "images": [] }, { "heading": "Properties", "content": "###### confidence\n\nCurrent confidence score (0-1) indicating information sufficiency.\n\n###### coverage_stats\n\nDictionary containing detailed coverage statistics.\n\nReturns:\n- **coverage** : Query term…", "code_blocks": [ { "language": "python", "code": "@property\ndef confidence(self) -> float", "filename": "" }, { "language": "python", "code": "@property \ndef coverage_stats(self) -> Dict[str, float]", "filename": "" }, { "language": "python", "code": "@property\ndef is_sufficient(self) -> bool", "filename": "" }, { "language": "python", "code": "@property\ndef state(self) -> CrawlState", "filename": "" } ], "images": [] }, { "heading": "Methods", "content": "###### get_relevant_content()\n\nRetrieve the most relevant content from the knowledge base.\n\n###### Parameters\n\n- **top_k** (`int`): Number of top relevant documents to return (default: 5)\n\n####…", "code_blocks": [ { "language": "python", "code": "def get_relevant_content(\n self,\n top_k: int = 5\n) -> List[Dict[str, Any]]", "filename": "" }, { "language": "python", "code": "def print_stats(\n self,\n detailed: bool = False\n) -> None", "filename": "" }, { "language": "python", "code": "def export_knowledge_base(\n self,\n path: Union[str, Path]\n) -> None", "filename": "" }, { "language": "python", "code": "adaptive.export_knowledge_base(\"my_knowledge.jsonl\")", "filename": "" }, { "language": "python", "code": "async def import_knowledge_base(\n self,\n path: Union[str, Path]\n) -> None", "filename": "" } ], "images": [] }, { "heading": "Configuration", "content": "The `AdaptiveConfig` class controls the behavior of adaptive crawling:\n\n###### Example with Custom Config", "code_blocks": [ { "language": "python", "code": "@dataclass\nclass AdaptiveConfig:\n confidence_threshold: float = 0.8 # Stop when confidence reaches this\n max_pages: int = 50 # Maximum pages to crawl\n top_k_links:…", "filename": "" }, { "language": "python", "code": "config = AdaptiveConfig(\n confidence_threshold=0.7,\n max_pages=20,\n top_k_links=3\n)\n\nadaptive = AdaptiveCrawler(crawler, config=config)", "filename": "" } ], "images": [] }, { "heading": "Complete Example", "content": "", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler, AdaptiveCrawler, AdaptiveConfig\n\nasync def main():\n # Configure adaptive crawling\n config = AdaptiveConfig(…", "filename": "" } ], "images": [] }, { "heading": "See Also", "content": "- [digest() Method Reference](../digest/)\n- [Adaptive Crawling Guide](../../core/adaptive-crawling/)\n- [Advanced Adaptive Strategies](../../advanced/adaptive-strategies/)", "code_blocks": [], "images": [] } ], "parameters": [ { "name": "crawler", "type": "AsyncWebCrawler", "description": "The underlying web crawler instance to use for fetching pages", "default": "", "required": true }, { "name": "config", "type": "Optional[AdaptiveConfig]", "description": "Configuration settings for adaptive crawling behavior. If not provided, uses default settings.", "default": "None", "required": false }, { "name": "start_url", "type": "str", "description": "The starting URL for crawling", "default": "", "required": true }, { "name": "query", "type": "str", "description": "The search query that guides the crawling process", "default": "", "required": true }, { "name": "resume_from", "type": "Optional[Union[str, Path]]", "description": "Path to a saved state file to resume from", "default": "None", "required": false }, { "name": "top_k", "type": "int", "description": "Number of top relevant documents to return", "default": "5", "required": false }, { "name": "detailed", "type": "bool", "description": "If True, shows detailed metrics with colors. If False, shows summary table.", "default": "False", "required": false }, { "name": "path", "type": "Union[str, Path]", "description": "Output file path for JSONL export", "default": "", "required": true }, { "name": "path", "type": "Union[str, Path]", "description": "Path to JSONL file to import", "default": "", "required": true } ], "see_also": [], "tags": [ "crawling", "adaptive", "web scraping", "AsyncWebCrawler", "Crawl4AI", "coverage", "consistency", "saturation" ], "prerequisites": [], "_content_hash": "3f98737cd61f2a399695aae3ef579feced480b34c6e0f3a6df1e4a14d25df467", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/api/arun/", "title": "`arun()` Parameter Guide (New Approach)", "type": "guide", "summary": "A guide to the parameters of the `arun()` method in Crawl4AI, now organized under `CrawlerRunConfig`, covering caching, content processing, navigation, session management, media options, extraction,…", "sections": [ { "heading": "Introduction", "content": "In Crawl4AI's **latest** configuration model, nearly all parameters that once went directly to `arun()` are now part of **`CrawlerRunConfig`** . When calling `arun()`, you provide:\n\nBelow is an…", "code_blocks": [ { "language": "python", "code": "await crawler.arun(\n url=\"https://example.com\", \n config=my_run_config\n)", "filename": "" } ], "images": [] }, { "heading": "1. Core Usage", "content": "**Key Fields**:\n- `verbose=True` logs each crawl step. \n- `cache_mode` decides how to read/write the local crawl cache.", "code_blocks": [ { "language": "python", "code": "from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode\n\nasync def main():\n run_config = CrawlerRunConfig(\n verbose=True, # Detailed logging…", "filename": "" } ], "images": [] }, { "heading": "2. Cache Control", "content": "**`cache_mode`** (default: `CacheMode.ENABLED`)\n\nUse a built-in enum from `CacheMode`:\n\n- `ENABLED`: Normal caching—reads if available, writes if missing.\n- `DISABLED`: No caching—always refetch…", "code_blocks": [ { "language": "python", "code": "run_config = CrawlerRunConfig(\n cache_mode=CacheMode.BYPASS\n)", "filename": "" } ], "images": [] }, { "heading": "3.1 Text Processing", "content": "", "code_blocks": [ { "language": "python", "code": "run_config = CrawlerRunConfig(\n word_count_threshold=10, # Ignore text blocks <10 words\n only_text=False, # If True, tries to remove non-text elements\n keep_data_attributes=False…", "filename": "" } ], "images": [] }, { "heading": "3.2 Content Selection", "content": "", "code_blocks": [ { "language": "python", "code": "run_config = CrawlerRunConfig(\n css_selector=\".main-content\", # Focus on .main-content region only\n excluded_tags=[\"form\", \"nav\"], # Remove entire tag blocks\n remove_forms=True,…", "filename": "" } ], "images": [] }, { "heading": "3.3 Link Handling", "content": "", "code_blocks": [ { "language": "python", "code": "run_config = CrawlerRunConfig(\n exclude_external_links=True, # Remove external links from final content\n exclude_social_media_links=True, # Remove links to known social sites…", "filename": "" } ], "images": [] }, { "heading": "3.4 Media Filtering", "content": "", "code_blocks": [ { "language": "python", "code": "run_config = CrawlerRunConfig(\n exclude_external_images=True # Strip images from other domains\n)", "filename": "" } ], "images": [] }, { "heading": "4.1 Basic Browser Flow", "content": "**Key Fields**:\n- `wait_for`:\n - `\"css:selector\"` or\n - `\"js:() => boolean\"`\n e.g. `js:() => document.querySelectorAll('.item').length > 10`.\n- `mean_delay` & `max_range`: define random delays for…", "code_blocks": [ { "language": "python", "code": "run_config = CrawlerRunConfig(\n wait_for=\"css:.dynamic-content\", # Wait for .dynamic-content\n delay_before_return_html=2.0, # Wait 2s before capturing final HTML\n page_timeout=60000,…", "filename": "" } ], "images": [] }, { "heading": "4.2 JavaScript Execution", "content": "- `js_code` can be a single string or a list of strings.\n- `js_only=True` means “I’m continuing in the same session with new JS steps, no new full navigation.”", "code_blocks": [ { "language": "python", "code": "run_config = CrawlerRunConfig(\n js_code=[\n \"window.scrollTo(0, document.body.scrollHeight);\",\n \"document.querySelector('.load-more')?.click();\"\n ],\n js_only=False\n)", "filename": "" } ], "images": [] }, { "heading": "4.3 Anti-Bot", "content": "- `magic=True` tries multiple stealth features. \n- `simulate_user=True` mimics mouse movements or random delays. \n- `override_navigator=True` fakes some navigator properties (like user agent checks).", "code_blocks": [ { "language": "python", "code": "run_config = CrawlerRunConfig(\n magic=True,\n simulate_user=True,\n override_navigator=True\n)", "filename": "" } ], "images": [] }, { "heading": "5. Session Management", "content": "If re-used in subsequent `arun()` calls, the same tab/page context is continued (helpful for multi-step tasks or stateful browsing).", "code_blocks": [ { "language": "python", "code": "run_config = CrawlerRunConfig(\n session_id=\"my_session123\"\n)", "filename": "" } ], "images": [] }, { "heading": "6. Screenshot, PDF & Media Options", "content": "**Where they appear**:\n- `result.screenshot` → Base64 screenshot string.\n- `result.pdf` → Byte array with PDF data.", "code_blocks": [ { "language": "python", "code": "run_config = CrawlerRunConfig(\n screenshot=True, # Grab a screenshot as base64\n screenshot_wait_for=1.0, # Wait 1s before capturing\n pdf=True, # Also…", "filename": "" } ], "images": [] }, { "heading": "7. Extraction Strategy", "content": "The extracted data will appear in `result.extracted_content`.", "code_blocks": [ { "language": "python", "code": "run_config = CrawlerRunConfig(\n extraction_strategy=my_css_or_llm_strategy\n)", "filename": "" } ], "images": [] }, { "heading": "8. Comprehensive Example", "content": "Below is a snippet combining many parameters:\n\n**What we covered**:\n1. **Crawling** the main content region, ignoring external links. \n2. Running **JavaScript** to click “.show-more”. \n3. **Waiting**…", "code_blocks": [ { "language": "python", "code": "import asyncio\nfrom crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode\nfrom crawl4ai import JsonCssExtractionStrategy\n\nasync def main():\n # Example schema\n schema = {\n \"name\":…", "filename": "" } ], "images": [] }, { "heading": "9. Best Practices", "content": "1. **Use `BrowserConfig` for global browser** settings (headless, user agent). \n2. **Use `CrawlerRunConfig`** to handle the **specific** crawl needs: content filtering, caching, JS, screenshot,…", "code_blocks": [], "images": [] }, { "heading": "10. Conclusion", "content": "All parameters that used to be direct arguments to `arun()` now belong in **`CrawlerRunConfig`** . This approach:\n\n- Makes code **clearer** and **more maintainable**.\n- Minimizes confusion about…", "code_blocks": [], "images": [] } ], "parameters": [], "see_also": [], "tags": [ "Crawl4AI", "arun", "CrawlerRunConfig", "caching", "content processing", "JavaScript", "screenshot", "PDF", "extraction", "session management", "anti-bot", "best practices" ], "prerequisites": [], "_content_hash": "b87bf8b18e7571949d4a839f34c565f4d7e1bf5d3fe619ca5ad0f3a7cd9f1f50", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/api/arun_many/", "title": "arun_many()", "type": "api", "summary": "Reference for the arun_many() function in Crawl4AI, which crawls multiple URLs concurrently or in batches, with support for dispatchers, streaming, and per-URL configurations.", "sections": [ { "heading": "Function Signature", "content": "", "code_blocks": [ { "language": "python", "code": "async def arun_many(\n urls: Union[List[str], List[Any]],\n config: Optional[Union[CrawlerRunConfig, List[CrawlerRunConfig]]] = None,\n dispatcher: Optional[BaseDispatcher] = None,\n ...\n) ->…", "filename": "" } ], "images": [] }, { "heading": "Differences from arun()", "content": "1. **Multiple URLs**:\n- Instead of crawling a single URL, you pass a list of them (strings or tasks).\n- The function returns `RunManyReturn` which contains either a **list** of `CrawlResult` or an…", "code_blocks": [], "images": [] }, { "heading": "Basic Example (Batch Mode)", "content": "", "code_blocks": [ { "language": "python", "code": "# Minimal usage: The default dispatcher will be used\nresults = await crawler.arun_many(\n urls=[\"https://site1.com\", \"https://site2.com\"],\n config=CrawlerRunConfig(stream=False) # Default…", "filename": "" } ], "images": [] }, { "heading": "Streaming Example", "content": "", "code_blocks": [ { "language": "python", "code": "config = CrawlerRunConfig(\n stream=True, # Enable streaming mode\n cache_mode=CacheMode.BYPASS\n)\n\n# Process results as they complete\nasync for result in await crawler.arun_many(…", "filename": "" } ], "images": [] }, { "heading": "With a Custom Dispatcher", "content": "", "code_blocks": [ { "language": "python", "code": "dispatcher = MemoryAdaptiveDispatcher(\n memory_threshold_percent=70.0,\n max_session_permit=10\n)\nresults = await crawler.arun_many(\n urls=[\"https://site1.com\", \"https://site2.com\",…", "filename": "" } ], "images": [] }, { "heading": "URL-Specific Configurations", "content": "Instead of using one config for all URLs, provide a list of configs with `url_matcher` patterns:\n\n**URL Matching Features**:\n- **String patterns**: `\"*.pdf\"`, `\"*/blog/*\"`, `\"*python.org*\"`\n-…", "code_blocks": [ { "language": "python", "code": "from crawl4ai import CrawlerRunConfig, MatchMode\nfrom crawl4ai.processors.pdf import PDFContentScrapingStrategy\nfrom crawl4ai.extraction_strategy import JsonCssExtractionStrategy\nfrom…", "filename": "" } ], "images": [] }, { "heading": "Return Value", "content": "Returns a **`RunManyReturn`** object which contains either a **list** of [`CrawlResult`](../crawl-result/) objects, or an **async generator** if streaming is enabled. You can iterate to check…", "code_blocks": [], "images": [] }, { "heading": "Dispatcher Reference", "content": "- **`MemoryAdaptiveDispatcher`** : Dynamically manages concurrency based on system memory usage.\n- **`SemaphoreDispatcher`** : Fixed concurrency limit, simpler but less adaptive.\n\nFor advanced usage…", "code_blocks": [], "images": [] }, { "heading": "Common Pitfalls", "content": "1. **Large Lists** : If you pass thousands of URLs, be mindful of memory or rate-limits. A dispatcher can help.\n2. **Session Reuse** : If you need specialized logins or persistent contexts, ensure…", "code_blocks": [], "images": [] }, { "heading": "Conclusion", "content": "Use `arun_many()` when you want to **crawl multiple URLs** simultaneously or in controlled parallel tasks. If you need advanced concurrency features (like memory-based adaptive throttling or complex…", "code_blocks": [], "images": [] } ], "parameters": [ { "name": "urls", "type": "Union[List[str], List[Any]]", "description": "A list of URLs (or tasks) to crawl.", "default": "", "required": true }, { "name": "config", "type": "Optional[Union[CrawlerRunConfig, List[CrawlerRunConfig]]]", "description": "Either a single CrawlerRunConfig applying to all URLs, or a list of CrawlerRunConfig objects with url_matcher patterns.", "default": "None", "required": false }, { "name": "dispatcher", "type": "Optional[BaseDispatcher]", "description": "A concurrency controller (e.g. MemoryAdaptiveDispatcher).", "default": "None", "required": false } ], "see_also": [], "tags": [ "Crawl4AI", "arun_many", "crawling", "concurrency", "dispatcher", "streaming", "batch" ], "prerequisites": [], "_content_hash": "c137298e045ce75a16e03f85ab44dd5d058c49a604f776a35fac7467de005044", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/api/async-webcrawler/", "title": "AsyncWebCrawler - Crawl4AI Documentation (v0.9.x)", "type": "reference", "summary": "Extraction fallback content.", "sections": [ { "heading": "AsyncWebCrawler - Crawl4AI Documentation (v0.9.x)", "content": "\n\n\n##### AsyncWebCrawler\n\n\n\n\nThe **`AsyncWebCrawler`** is the core class for asynchronous web crawling in Crawl4AI. You typically create it **once** , optionally customize it with a **`BrowserConfig`** (e.g., headless, user agent), then **run** multiple **`arun()`** calls with different **`CrawlerRunConfig`** objects.\n\n\n\n\n **Recommended usage** :\n\n\n\n\n1.  **Create** a `BrowserConfig` for global browser settings.  \n\n\n\n\n2.  **Instantiate** `AsyncWebCrawler(config=browser_config)`.  \n\n\n\n\n3.  **Use** the crawler in an async context manager (`async with`) or manage start/close manually.  \n\n\n\n\n4.  **Call** `arun(url, config=crawler_run_config)` for each page you want.\n\n\n\n\n\n##### 1. Constructor Overview\n\n\n\n\n\n **Notes** :\n\n\n\n\n\n\n- **Legacy** parameters like `always_bypass_cache` remain for backward compatibility, but prefer to set **caching** in `CrawlerRunConfig`.\n\n\n\n\n\n\n##### 2. Lifecycle: Start/Close or Context Manager\n\n\n\n\n###### 2.1 Context Manager (Recommended)\n\n\n\n\n\nWhen the `async with` block ends, the crawler cleans up (closes the browser, etc.).\n\n\n\n\n###### 2.2 Manual Start & Close\n\n\n\n\n\nUse this style if you have a **long-running** application or need full control of the crawler’s lifecycle.\n\n\n\n\n\n##### 3. Primary Method: `arun()`\n\n\n\n\n\n###### 3.1 New Approach\n\n\n\n\nYou pass a `CrawlerRunConfig` object that sets up everything about a crawl—content filtering, caching, session reuse, JS code, screenshots, etc.\n\n\n\n\n\n###### 3.2 Legacy Parameters Still Accepted\n\n\n\n\nFor **backward** compatibility, `arun()` can still accept direct arguments like `css_selector=...`, `word_count_threshold=...`, etc., but we strongly advise migrating them into a **`CrawlerRunConfig`** .\n\n\n\n\n\n##### 4. Batch Processing: `arun_many()`\n\n\n\n\n\n###### 4.1 Resource-Aware Crawling\n\n\n\n\nThe `arun_many()` method now uses an intelligent dispatcher that:\n\n\n\n\n\n\n- Monitors system memory usage\n\n- Implements adaptive rate limiting\n\n- Provides detailed progress monitoring\n\n- Manages concurrent crawls efficiently\n\n\n\n\n\n###### 4.2 Example Usage\n\n\n\n\nCheck page [Multi-url Crawling](../../advanced/multi-url-crawling/) for a detailed example of how to use `arun_many()`.\n\n\n\n\n\n **Explanation** :\n\n\n\n\n\n\n- We define a **`BrowserConfig`** with Firefox, no headless, and `verbose=True`.\n\n- We define a **`CrawlerRunConfig`** that **bypasses cache** , uses a **CSS** extraction schema, has a `word_count_threshold=15`, etc.\n\n- We pass them to `AsyncWebCrawler(config=...)` and `arun(url=..., config=...)`.\n\n\n\n\n\n\n##### 7. Best Practices & Migration Notes\n\n\n\n\n1.  **Use** `BrowserConfig` for **global** settings about the browser’s environment.  \n2.  **Use** `CrawlerRunConfig` for **per-crawl** logic (caching, content filtering, extraction strategies, wait conditions).  \n3.  **Avoid** legacy parameters like `css_selector` or `word_count_threshold` directly in `arun()`. Instead:\n\n\n\n\n\n4.  **Context Manager** usage is simplest unless you want a persistent crawler across many calls.\n\n\n\n\n\n##### 8. Summary\n\n\n\n\n **AsyncWebCrawler** is your entry point to asynchronous crawling:\n\n\n\n\n\n\n- **Constructor** accepts **`BrowserConfig`** (or defaults).\n\n- **`arun(url, config=CrawlerRunConfig)`** is the main method for single-page crawls.\n\n- **`arun_many(urls, config=CrawlerRunConfig)`** handles concurrency across multiple URLs.\n\n- For advanced lifecycle control, use `start()` and `close()` explicitly.\n\n\n\n\n\n **Migration** : \n\n\n\n\n\n\n- If you used `AsyncWebCrawler(browser_type=\"chromium\", css_selector=\"...\")`, move browser settings to `BrowserConfig(...)` and content/crawl logic to `CrawlerRunConfig(...)`.\n\n\n\n\n\nThis modular approach ensures your code is **clean** , **scalable** , and **easy to maintain** . For any advanced or rarely used parameters, see the [BrowserConfig docs](../parameters/).\n\n\n", "code_blocks": [ { "language": "python", "code": "class AsyncWebCrawler:\n def __init__(\n self,\n crawler_strategy: Optional[AsyncCrawlerStrategy] = None,\n config: Optional[BrowserConfig] = None,\n always_bypass_cache: bool = False, # deprecated\n always_by_pass_cache: Optional[bool] = None, # also deprecated\n base_directory: str = ...,\n thread_safe: bool = False,\n **kwargs,\n ):\n \"\"\"\n Create an AsyncWebCrawler instance.\n\n Args:\n crawler_strategy: \n (Advanced) Provide a custom crawler strategy if needed.\n config: \n A BrowserConfig object specifying how the browser is set up.\n always_bypass_cache: \n (Deprecated) Use CrawlerRunConfig.cache_mode instead.\n base_directory: \n Folder for storing caches/logs (if relevant).\n thread_safe: \n If True, attempts some concurrency safeguards. Usually False.\n **kwargs: \n Additional legacy or debugging parameters.\n \"\"\"\n )\n\n### Typical Initialization\n\n```python\nfrom crawl4ai import AsyncWebCrawler, BrowserConfig\n\nbrowser_cfg = BrowserConfig(\n browser_type=\"chromium\",\n headless=True,\n verbose=True\n)\n\ncrawler = AsyncWebCrawler(config=browser_cfg)", "filename": "" }, { "language": "csharp", "code": "async with AsyncWebCrawler(config=browser_cfg) as crawler:\n result = await crawler.arun(\"https://example.com\")\n # The crawler automatically starts/closes resources", "filename": "" }, { "language": "csharp", "code": "crawler = AsyncWebCrawler(config=browser_cfg)\nawait crawler.start()\n\nresult1 = await crawler.arun(\"https://example.com\")\nresult2 = await crawler.arun(\"https://another.com\")\n\nawait crawler.close()", "filename": "" }, { "language": "python", "code": "async def arun(\n self,\n url: str,\n config: Optional[CrawlerRunConfig] = None,\n # Legacy parameters for backward compatibility...\n) -> RunManyReturn:\n ...", "filename": "" }, { "language": "python", "code": "import asyncio\nfrom crawl4ai import CrawlerRunConfig, CacheMode\n\nrun_cfg = CrawlerRunConfig(\n cache_mode=CacheMode.BYPASS,\n css_selector=\"main.article\",\n word_count_threshold=10,\n screenshot=True\n)\n\nasync with AsyncWebCrawler(config=browser_cfg) as crawler:\n result = await crawler.arun(\"https://example.com/news\", config=run_cfg)\n print(\"Crawled HTML length:\", len(result.cleaned_html))\n if result.screenshot:\n print(\"Screenshot base64 length:\", len(result.screenshot))", "filename": "" }, { "language": "python", "code": "async def arun_many(\n self,\n urls: List[str],\n config: Optional[CrawlerRunConfig] = None,\n # Legacy parameters maintained for backwards compatibility...\n) -> RunManyReturn:\n \"\"\"\n Process multiple URLs with intelligent rate limiting and resource monitoring.\n \"\"\"", "filename": "" }, { "language": "python", "code": "### 4.3 Key Features\n\n1. **Rate Limiting**\n\n - Automatic delay between requests\n - Exponential backoff on rate limit detection\n - Domain-specific rate limiting\n - Configurable retry strategy\n\n2. **Resource Monitoring**\n\n - Memory usage tracking\n - Adaptive concurrency based on system load\n - Automatic pausing when resources are constrained\n\n3. **Progress Monitoring**\n\n - Detailed or aggregated progress display\n - Real-time status updates\n - Memory usage statistics\n\n4. **Error Handling**\n\n - Graceful handling of rate limits\n - Automatic retries with backoff\n - Detailed error reporting\n\n---\n\n## 5. `CrawlResult` Output\n\nEach `arun()` returns a **`CrawlResult`** containing:\n\n- `url`: Final URL (if redirected).\n- `html`: Original HTML.\n- `cleaned_html`: Sanitized HTML.\n- `markdown_v2`: Removed in v0.5. Accessing it raises `AttributeError`; use `markdown`.\n- `extracted_content`: If an extraction strategy was used (JSON for CSS/LLM strategies).\n- `screenshot`, `pdf`: If screenshots/PDF requested.\n- `media`, `links`: Information about discovered images/links.\n- `success`, `error_message`: Status info.\n\nFor details, see [CrawlResult doc](./crawl-result.md).\n\n---\n\n## 6. Quick Example\n\nBelow is an example hooking it all together:\n\n```python\nimport asyncio\nfrom crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode\nfrom crawl4ai import JsonCssExtractionStrategy\nimport json\n\nasync def main():\n # 1. Browser config\n browser_cfg = BrowserConfig(\n browser_type=\"firefox\",\n headless=False,\n verbose=True\n )\n\n # 2. Run config\n schema = {\n \"name\": \"Articles\",\n \"baseSelector\": \"article.post\",\n \"fields\": [\n {\n \"name\": \"title\", \n \"selector\": \"h2\", \n \"type\": \"text\"\n },\n {\n \"name\": \"url\", \n \"selector\": \"a\", \n \"type\": \"attribute\", \n \"attribute\": \"href\"\n }\n ]\n }\n\n run_cfg = CrawlerRunConfig(\n cache_mode=CacheMode.BYPASS,\n extraction_strategy=JsonCssExtractionStrategy(schema),\n word_count_threshold=15,\n remove_overlay_elements=True,\n wait_for=\"css:.post\" # Wait for posts to appear\n )\n\n async with AsyncWebCrawler(config=browser_cfg) as crawler:\n result = await crawler.arun(\n url=\"https://example.com/blog\",\n config=run_cfg\n )\n\n if result.success:\n print(\"Cleaned HTML length:\", len(result.cleaned_html))\n if result.extracted_content:\n articles = json.loads(result.extracted_content)\n print(\"Extracted articles:\", articles[:2])\n else:\n print(\"Error:\", result.error_message)\n\nasyncio.run(main())", "filename": "" }, { "language": "ini", "code": "run_cfg = CrawlerRunConfig(css_selector=\".main-content\", word_count_threshold=20)\nresult = await crawler.arun(url=\"...\", config=run_cfg)", "filename": "" } ], "images": [] } ], "parameters": [], "see_also": [], "tags": [], "prerequisites": [], "_content_hash": "8a6384fe9b45b4de2464b9233f32722d567cbe227ca8851933ca86afc69a2d6e", "error": "", "finish_reason": "", "_canonical": true }, { "url": "https://docs.crawl4ai.com/api/c4a-script-reference/", "title": "C4A-Script API Reference", "type": "reference", "summary": "Complete reference for all C4A-Script commands, syntax, and advanced features, including navigation, waiting, mouse, keyboard, control flow, variables, procedures, and integration with Crawl4AI.", "sections": [ { "heading": "Command Categories", "content": "Complete reference for all C4A-Script commands, syntax, and advanced features.", "code_blocks": [], "images": [] }, { "heading": "🧭 Navigation Commands", "content": "Navigate between pages and manage browser history.", "code_blocks": [], "images": [] }, { "heading": "`GO `", "content": "Navigate to a specific URL.\n\n**Syntax:**\n\n```\nGO \n```\n\n**Parameters:**\n- `url` - Target URL (string)\n\n**Examples:**\n\n```\nGO https://example.com\nGO https://api.example.com/login\nGO…", "code_blocks": [ { "language": "", "code": "GO ", "filename": "" }, { "language": "", "code": "GO https://example.com\nGO https://api.example.com/login\nGO /relative/path", "filename": "" } ], "images": [] }, { "heading": "`RELOAD`", "content": "Refresh the current page.\n\n**Syntax:**\n\n```\nRELOAD\n```\n\n**Examples:**\n\n```\nRELOAD\n```\n\n**Notes:**\n- Equivalent to pressing F5 or clicking browser refresh\n- Waits for page reload to complete\n-…", "code_blocks": [ { "language": "", "code": "RELOAD", "filename": "" }, { "language": "", "code": "RELOAD", "filename": "" } ], "images": [] }, { "heading": "`BACK`", "content": "Navigate back in browser history.\n\n**Syntax:**\n\n```\nBACK\n```\n\n**Examples:**\n\n```\nBACK\n```\n\n**Notes:**\n- Equivalent to clicking browser back button\n- Does nothing if no previous page exists\n- Waits…", "code_blocks": [ { "language": "", "code": "BACK", "filename": "" }, { "language": "", "code": "BACK", "filename": "" } ], "images": [] }, { "heading": "`FORWARD`", "content": "Navigate forward in browser history.\n\n**Syntax:**\n\n```\nFORWARD\n```\n\n**Examples:**\n\n```\nFORWARD\n```\n\n**Notes:**\n- Equivalent to clicking browser forward button\n- Does nothing if no next page exists\n-…", "code_blocks": [ { "language": "", "code": "FORWARD", "filename": "" }, { "language": "", "code": "FORWARD", "filename": "" } ], "images": [] }, { "heading": "⏱️ Wait Commands", "content": "Control timing and synchronization with page elements.", "code_blocks": [], "images": [] }, { "heading": "`WAIT