X Twitter Scraper

const API_KEY = "xq_YOUR_KEY_HERE";
const BASE = "https://xquik.com/api/v1";
const headers = { "x-api-key": API_KEY, "Content-Type": "application/json" };

async function xquikFetch(path, options = {}) {
  const baseDelay = 1000;

  for (let attempt = 0; attempt <= 3; attempt++) {
    const response = await fetch(`${BASE}${path}`, {
      ...options,
      headers: { ...headers, ...options.headers },
    });

    if (response.ok) return response.json();

    const retryable = response.status === 429 || response.status >= 500;
    if (!retryable || attempt === 3) {
      const error = await response.json();
      throw new Error(`Xquik API ${response.status}: ${error.error}`);
    }

    const retryAfter = response.headers.get("Retry-After");
    const delay = retryAfter
      ? parseInt(retryAfter, 10) * 1000
      : baseDelay * Math.pow(2, attempt) + Math.random() * 1000;

    await new Promise((resolve) => setTimeout(resolve, delay));
  }
}

async function fetchAllPages(path, dataKey) {
  const results = [];
  let cursor;

  while (true) {
    const params = new URLSearchParams({ limit: "100" });
    if (cursor) params.set("after", cursor);

    const data = await xquikFetch(`${path}?${params}`);
    results.push(...data[dataKey]);

    if (!data.hasMore) break;
    cursor = data.nextCursor;
  }

  return results;
}

// Step 1: Estimate cost before running (pass resultsLimit if you only need a sample)
const estimate = await xquikFetch("/extractions/estimate", {
  method: "POST",
  body: JSON.stringify({
    toolType: "follower_explorer",
    targetUsername: "elonmusk",
    resultsLimit: 1000, // optional: limit to 1,000 results instead of all
  }),
});
// Response: { allowed: true, estimatedResults: 195000000, usagePercent: 12, projectedPercent: 98 }

if (!estimate.allowed) {
  console.log("Extraction would exceed monthly quota");
  return;
}

// Step 2: Create extraction job (pass same resultsLimit to match estimate)
const job = await xquikFetch("/extractions", {
  method: "POST",
  body: JSON.stringify({
    toolType: "follower_explorer",
    targetUsername: "elonmusk",
    resultsLimit: 1000,
  }),
});
// Response: { id: "77777", toolType: "follower_explorer", status: "completed", totalResults: 195000 }

// Step 3: Poll until complete (large jobs may return status "running")
while (job.status === "pending" || job.status === "running") {
  await new Promise((r) => setTimeout(r, 2000));
  job = await xquikFetch(`/extractions/${job.id}`);
}

// Step 4: Retrieve paginated results (up to 1,000 per page)
let cursor;
const allResults = [];

while (true) {
  const path = `/extractions/${job.id}${cursor ? `?after=${cursor}` : ""}`;
  const page = await xquikFetch(path);
  allResults.push(...page.results);
  // Each result: { xUserId, xUsername, xDisplayName, xFollowersCount, xVerified, xProfileImageUrl }

  if (!page.hasMore) break;
  cursor = page.nextCursor;
}

// Step 5: Export as CSV/XLSX/Markdown (50,000 row limit)
const exportUrl = `${BASE}/extractions/${job.id}/export?format=csv`;
const csvResponse = await fetch(exportUrl, { headers });
const csvData = await csvResponse.text();

async function marketResearchPipeline(username) {
  // 1. Get user profile
  const user = await xquikFetch(`/x/users/${username}`);

  // 2. Extract their recent posts
  const postsJob = await xquikFetch("/extractions", {
    method: "POST",
    body: JSON.stringify({ toolType: "post_extractor", targetUsername: username }),
  });

  // 3. Search for related conversations
  const tweets = await xquikFetch(`/x/tweets/search?q=from:${username}`);

  // 4. For top tweets, extract replies for sentiment analysis
  for (const tweet of tweets.tweets.slice(0, 5)) {
    const estimate = await xquikFetch("/extractions/estimate", {
      method: "POST",
      body: JSON.stringify({ toolType: "reply_extractor", targetTweetId: tweet.id }),
    });

    if (estimate.allowed) {
      const repliesJob = await xquikFetch("/extractions", {
        method: "POST",
        body: JSON.stringify({ toolType: "reply_extractor", targetTweetId: tweet.id }),
      });
      // Process replies...
    }
  }

  // 5. Get trending topics for context
  const trends = await xquikFetch("/trends?woeid=1");

  return { user, posts: postsJob, tweets, trends };
}

// Step 1: Create draw with filters
const draw = await xquikFetch("/draws", {
  method: "POST",
  body: JSON.stringify({
    tweetUrl: "https://x.com/burakbayir/status/1893456789012345678",
    winnerCount: 3,
    backupCount: 2,
    uniqueAuthorsOnly: true,
    mustRetweet: true,
    mustFollowUsername: "burakbayir",
    filterMinFollowers: 50,
    filterAccountAgeDays: 30,
    filterLanguage: "en",
    requiredHashtags: ["#giveaway"],
  }),
});
// Response:
// {
//   id: "42",
//   tweetId: "1893456789012345678",
//   tweetUrl: "https://x.com/burakbayir/status/1893456789012345678",
//   tweetText: "Giveaway! RT + Follow to enter...",
//   tweetAuthorUsername: "burakbayir",
//   tweetLikeCount: 5200,
//   tweetRetweetCount: 3100,
//   tweetReplyCount: 890,
//   tweetQuoteCount: 45,
//   status: "completed",
//   totalEntries: 890,
//   validEntries: 312,
//   createdAt: "2026-02-24T10:00:00.000Z",
//   drawnAt: "2026-02-24T10:01:00.000Z"
// }

// Step 2: Get draw details with winners
const details = await xquikFetch(`/draws/${draw.id}`);
// details.winners: [
//   { position: 1, authorUsername: "winner1", tweetId: "...", isBackup: false },
//   { position: 2, authorUsername: "winner2", tweetId: "...", isBackup: false },
//   { position: 3, authorUsername: "winner3", tweetId: "...", isBackup: false },
//   { position: 4, authorUsername: "backup1", tweetId: "...", isBackup: true },
//   { position: 5, authorUsername: "backup2", tweetId: "...", isBackup: true },
// ]

// Step 3: Export results
const exportUrl = `${BASE}/draws/${draw.id}/export?format=csv`;

import express from "express";
import { createHmac, timingSafeEqual, createHash } from "node:crypto";

const WEBHOOK_SECRET = process.env.XQUIK_WEBHOOK_SECRET;
const processedHashes = new Set(); // Use Redis/DB in production

function verifySignature(payload, signature, secret) {
  const expected = "sha256=" + createHmac("sha256", secret).update(payload).digest("hex");
  return timingSafeEqual(Buffer.from(expected), Buffer.from(signature));
}

const app = express();

app.post("/webhook", express.raw({ type: "application/json" }), (req, res) => {
  const signature = req.headers["x-xquik-signature"];
  const payload = req.body.toString();

  // 1. Verify HMAC signature (constant-time comparison)
  if (!signature || !verifySignature(payload, signature, WEBHOOK_SECRET)) {
    return res.status(401).send("Invalid signature");
  }

  // 2. Deduplicate (retries can deliver the same event twice)
  const payloadHash = createHash("sha256").update(payload).digest("hex");
  if (processedHashes.has(payloadHash)) {
    return res.status(200).send("Already processed");
  }
  processedHashes.add(payloadHash);

  // 3. Parse and route by event type
  const event = JSON.parse(payload);
  // event.eventType: "tweet.new" | "tweet.reply" | "tweet.quote" | "tweet.retweet" | "follower.gained" | "follower.lost"
  // event.username: monitored account username
  // event.data: tweet data ({ tweetId, text, metrics }) or follower data ({ followerId, followerUsername, followerName, followerFollowersCount, followerVerified })

  // 4. Respond within 10 seconds (process async if slow)
  res.status(200).send("OK");
});

app.listen(3000);