Web Crawling

node -e "
const { chromium } = require('playwright');
(async () => {
  const browser = await chromium.launch();
  const page = await browser.newPage();
  await page.goto('https://example.com');
  const title = await page.title();
  const text = await page.locator('body').innerText();
  console.log(JSON.stringify({ title, text }));
  await browser.close();
})();
"

node -e "
const { chromium } = require('playwright');
(async () => {
  const browser = await chromium.launch();
  const page = await browser.newPage();
  await page.goto('https://example.com');
  await page.screenshot({ path: 'screenshot.png', fullPage: true });
  await browser.close();
})();
"

node -e "
const { chromium } = require('playwright');
(async () => {
  const browser = await chromium.launch();
  const page = await browser.newPage();
  await page.goto('https://example.com/data');
  const rows = await page.locator('table tr').evaluateAll(trs =>
    trs.map(tr => Array.from(tr.querySelectorAll('td,th')).map(c => c.textContent.trim()))
  );
  console.log(JSON.stringify(rows));
  await browser.close();
})();
"

node -e "
const { chromium } = require('playwright');
(async () => {
  const browser = await chromium.launch();
  const page = await browser.newPage();
  await page.goto('https://example.com/spa');
  await page.waitForSelector('.content-loaded');
  const data = await page.locator('.content-loaded').innerText();
  console.log(data);
  await browser.close();
})();
"

node -e "
const puppeteer = require('puppeteer');
(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('https://example.com', { waitUntil: 'networkidle0' });
  await page.pdf({ path: 'output.pdf', format: 'A4' });
  await browser.close();
})();
"

katana -u https://example.com -silent -d 3

katana -u https://example.com -silent -d 2 -jsonl

katana -u https://example.com -headless -d 2 -silent

scrapy fetch --nolog https://example.com | python3 -c "
import sys
from scrapy import Selector
html = sys.stdin.read()
sel = Selector(text=html)
for item in sel.css('h2::text').getall():
    print(item)
"

node -e "
const { PlaywrightCrawler } = require('crawlee');
const crawler = new PlaywrightCrawler({
  async requestHandler({ page, request }) {
    const title = await page.title();
    console.log(JSON.stringify({ url: request.url, title }));
  },
  maxRequestsPerCrawl: 10,
});
crawler.run(['https://example.com']);
"

katana -u https://example.com -silent -d 2 | grep '/blog/' | head -5 | while read url; do
  node -e "
    const { chromium } = require('playwright');
    (async () => {
      const browser = await chromium.launch();
      const page = await browser.newPage();
      await page.goto('$url');
      const title = await page.title();
      const text = await page.locator('article').innerText().catch(() => '');
      console.log(JSON.stringify({ url: '$url', title, text: text.slice(0, 500) }));
      await browser.close();
    })();
  "
done

katana -u https://example.com -silent -d 2 -jsonl > crawl.jsonl
duckdb -c "SELECT endpoint, status_code, COUNT(*) FROM read_json_auto('crawl.jsonl') GROUP BY endpoint, status_code ORDER BY COUNT(*) DESC"