Web Scraping

from tools.scrape import scrape_page, scrape_pages

# Single page → markdown
res = scrape_page(url="https://example.com")
if "error" not in res:
    print(res["title"], res["status"], len(res["content"]))

# JS-rendered page
res = scrape_page(url="https://spa-site.com", mode="browser", timeout=60)

# Bot-protected page
res = scrape_page(url="https://protected-site.com", mode="stealth", solve_cloudflare=True)

# Batch — split successes from failures
batch = scrape_pages(urls=[...], mode="fast")   # <= 10 URLs
pages = [r for r in batch["results"] if "error" not in r]
failed = [(r["url"], r["error"]) for r in batch["results"] if "error" in r]

# Raw HTML when you need to parse structure yourself
res = scrape_page(url="https://example.com", extraction="html")
from bs4 import BeautifulSoup
soup = BeautifulSoup(res["content"], "html.parser")
titles = [h1.get_text() for h1 in soup.find_all("h1")]

from scrapling.fetchers import AsyncFetcher

page = await AsyncFetcher.get("https://example.com", stealthy_headers=True)
print(page.status)       # 200
print(page.body)         # Raw bytes
print(page.headers)      # Response headers

# CSS selectors (Scrapy-style pseudo-elements)
titles = page.css("h1::text").getall()
links = page.css("a::attr(href)").getall()

# XPath
items = page.xpath("//div[@class='item']/text()").getall()

# BeautifulSoup-style
divs = page.find_all("div", class_="content")

from scrapling.fetchers import DynamicFetcher

page = await DynamicFetcher.async_fetch(
    "https://spa-website.com",
    headless=True,
    network_idle=True,
    disable_resources=True,
    timeout=30000,          # milliseconds here, unlike the MCP tools
    wait_selector=".data-table",
)
rows = page.css("table.data-table tr")
for row in rows:
    cells = row.css("td::text").getall()

from scrapling.fetchers import StealthyFetcher

page = await StealthyFetcher.async_fetch(
    "https://protected-site.com",
    headless=True,
    solve_cloudflare=True,
    network_idle=True,
)

from scrapling.fetchers import FetcherSession

with FetcherSession(impersonate="chrome") as session:
    login_page = session.post("https://site.com/login", data={...})
    dashboard = session.get("https://site.com/dashboard")
    data = dashboard.css(".user-data::text").getall()

from scrapling.spiders import Spider, Request, Response

class PriceScraper(Spider):
    name = "prices"
    start_urls = ["https://example.com/products"]
    concurrent_requests = 5

    async def parse(self, response: Response):
        for product in response.css(".product"):
            yield {
                "name": product.css(".name::text").get(),
                "price": product.css(".price::text").get(),
            }
        next_page = response.css("a.next::attr(href)").get()
        if next_page:
            yield Request(next_page)

spider = PriceScraper()
result = spider.start()
result.items.to_json("<task_name>/data/prices.json")

import html_to_markdown

markdown = html_to_markdown.convert(
    html_string, html_to_markdown.ConversionOptions(extract_metadata=False)
).content

# Article-only extraction (strips nav/ads/boilerplate)
import trafilatura

article = trafilatura.extract(html_string, output_format="markdown", favor_recall=True)