Twitter X Scraping

// challengeSolver.ts — detect challenge
const CHALLENGE_MARKER = 'Verifying your request';
export function isChallengePage(html: string): boolean {
  return html.includes(CHALLENGE_MARKER);
}

// scrapeNitterList.ts — switch modes on detection
if (isChallengePage(html)) {
  useBrowserMode = true;
  const browserHtml = await fetchWithBrowser(url, debug);
  return { html: browserHtml, url };
}

import { chromium } from 'playwright';

const browser = await chromium.launch({
  channel: 'chrome',
  headless: false,
  args: ['--disable-blink-features=AutomationControlled'],
});

const context = await browser.newContext({ locale: 'en-US' });
await context.addInitScript(() => {
  Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
});

const page = await context.newPage();
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 30000 });

// Wait for challenge to auto-resolve
await page.waitForSelector('.timeline-item', { timeout: 30000 });
const html = await page.content();

const maxRetries = 10;
const baseDelayMs = 2000;

for (let attempt = 0; attempt <= maxRetries; attempt++) {
  const res = await axios.get(url, {
    headers: FETCH_HEADERS,
    timeout: 30000,
    validateStatus: (s) => (s >= 200 && s < 400) || s === 429 || s === 502 || s === 503,
  });

  if (res.status === 429 || res.status === 502 || res.status === 503) {
    const ra = res.headers['retry-after'];
    if (ra && /^\d+$/.test(ra)) {
      await sleep(Math.min((parseInt(ra) + 2) * 1000, 5 * 60 * 1000));
      continue;
    }
    const jitter = Math.floor(Math.random() * 250);
    const waitMs = Math.min(baseDelayMs * Math.pow(2, attempt) + jitter, 60_000);
    await sleep(waitMs);
    continue;
  }

  if (res.status === 200) return res.data;
}

const FETCH_HEADERS = {
  'User-Agent': 'Mozilla/5.0 (compatible; MyBot/1.0) Node.js',
  'Accept-Language': 'en-US,en;q=0.9',
  'Accept': 'text/html,application/xhtml+xml',
  'Accept-Encoding': 'gzip, compress, deflate, br',
};

import * as cheerio from 'cheerio';

const $ = cheerio.load(html);

// All tweets on the page
const $items = $('.timeline .timeline-item').filter((_, el) =>
  $(el).find('.tweet-body').length > 0
);

$items.each((_, el) => {
  const $el = $(el);

  // Tweet URL and ID
  const href = $el.find('a.tweet-link').first().attr('href'); // e.g. "/user/status/12345"
  const id = href?.match(/status\/(\d+)/)?.[1];

  // Timestamp — from the `title` attribute of the date link
  const title = $el.find('.tweet-date a').attr('title'); // e.g. "Apr 15, 2026 · 10:00 AM UTC"
  const timestamp = new Date(title?.replace(/\s*[·•]\s*/g, ' ') ?? '');

  // Tweet text
  const text = $el.find('.tweet-content.media-body').text().trim();

  // Author
  const username = $el.find('.tweet-header a.username').attr('title');
  const displayName = $el.find('.tweet-header a.fullname').attr('title');

  // Images
  $el.find('.attachments .attachment.image').each((_, img) => {
    const src = $(img).find('img').attr('src');  // relative URL
    const fullSrc = `https://nitter.net${src}`;
  });

  // Engagement stats
  $el.find('.tweet-stats .tweet-stat').each((_, stat) => {
    const icon = $(stat).find('.icon-container > span').attr('class') ?? '';
    const n = parseInt($(stat).text().replace(/\D/g, '') || '0');
    if (icon.includes('icon-heart')) console.log('likes:', n);
    if (icon.includes('icon-retweet')) console.log('retweets:', n);
  });

  // Is a retweet?
  const retweetedBy = $el.find('.retweet-header').text().trim() || undefined;

  // Quoted tweet
  const $quote = $el.find('.quote').first();
  if ($quote.length) {
    const quoteText = $quote.find('.quote-text').text().trim();
    const quoteUser = $quote.find('.tweet-name-row a.username').text().trim();
  }
});

function findNextCursor($: cheerio.CheerioAPI): string | undefined {
  const href = $('div.show-more a').last().attr('href') ?? '';
  const m = href.match(/[?&]cursor=([^&]+)/);
  return m ? decodeURIComponent(m[1]) : undefined;
}

// In the pagination loop:
let cursor: string | undefined;
while (pagesFetched < maxPages) {
  const url = `https://nitter.net/i/lists/${listId}${cursor ? `?cursor=${encodeURIComponent(cursor)}` : ''}`;
  const html = await fetchPage(url);
  const $ = cheerio.load(html);
  // ... parse tweets ...
  const next = findNextCursor($);
  if (!next) break;
  cursor = next;
}

// After parsing all tweets on a page:
const oldestNonRTMs = Math.min(
  ...pageTweets.filter(t => !t.retweetedBy).map(t => t.timestampMs)
);

if (oldestNonRTMs < startDate.getTime()) {
  console.log('Reached tweets older than start date; stopping.');
  break;
}

import axios from 'axios';
import * as cheerio from 'cheerio';

const LIST_ID = '1585430245762441216';
const BASE = 'https://nitter.net';
const HEADERS = {
  'User-Agent': 'Mozilla/5.0 (compatible; MyBot/1.0) Node.js',
  'Accept': 'text/html,application/xhtml+xml',
};

async function scrapeListPage(cursor?: string) {
  const url = `${BASE}/i/lists/${LIST_ID}${cursor ? `?cursor=${encodeURIComponent(cursor)}` : ''}`;
  const res = await axios.get(url, { headers: HEADERS, timeout: 30000 });
  const $ = cheerio.load(res.data);

  const tweets: { id: string; text: string; timestamp: string }[] = [];

  $('.timeline .timeline-item').filter((_, el) =>
    $(el).find('.tweet-body').length > 0
  ).each((_, el) => {
    const $el = $(el);
    const href = $el.find('a.tweet-link').first().attr('href') ?? '';
    const id = href.match(/status\/(\d+)/)?.[1] ?? href;
    const title = $el.find('.tweet-date a').attr('title') ?? '';
    const timestamp = new Date(title.replace(/\s*[·•]\s*/g, ' ')).toISOString();
    const text = $el.find('.tweet-content.media-body').text().trim();
    tweets.push({ id, text, timestamp });
  });

  const nextHref = $('div.show-more a').last().attr('href') ?? '';
  const nextCursor = nextHref.match(/[?&]cursor=([^&]+)/)?.[1];

  return { tweets, nextCursor: nextCursor ? decodeURIComponent(nextCursor) : undefined };
}

async function main() {
  let cursor: string | undefined;
  let allTweets: { id: string; text: string; timestamp: string }[] = [];

  for (let page = 0; page < 5; page++) {
    const { tweets, nextCursor } = await scrapeListPage(cursor);
    allTweets.push(...tweets);
    console.log(`Page ${page + 1}: +${tweets.length} tweets (${allTweets.length} total)`);
    if (!nextCursor) break;
    cursor = nextCursor;
    await new Promise(r => setTimeout(r, 1000)); // polite delay
  }

  console.log(JSON.stringify(allTweets, null, 2));
}

main();

const ok = await page.waitForSelector('.timeline-item', { timeout: 10000 })
  .then(() => true).catch(() => false);
// Don't trust page.title() — error pages can spoof it

const browser = await chromium.launch({
  headless: false, // REQUIRED — headless gets blocked
  args: ['--disable-blink-features=AutomationControlled'],
});
await context.addInitScript(() => {
  Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
});

// ✅ correct
const nextUrl = `${baseUrl}/${handle}${nextHref}`; // nextHref starts with "?"

// ❌ wrong — this double-encodes or drops the path
const nextUrl = new URL(nextHref, baseUrl).href;

const datePart = title.split('·')[0].trim(); // "Apr 14, 2026 "
const date = new Date(datePart); // reliable