Clay Rate Limits

// src/clay/rate-limiter.ts — respect Clay's plan-level rate limits
import PQueue from 'p-queue';

interface RateLimiterConfig {
  maxPerHour: number;     // Plan limit (e.g., 400 for Explorer)
  maxPerSecond: number;   // Practical burst limit
  webhookLimit: number;   // 50K per webhook lifetime
}

const PLAN_LIMITS: Record<string, RateLimiterConfig> = {
  explorer: { maxPerHour: 400, maxPerSecond: 2, webhookLimit: 50_000 },
  pro:      { maxPerHour: Infinity, maxPerSecond: 10, webhookLimit: 50_000 },
  enterprise: { maxPerHour: Infinity, maxPerSecond: 20, webhookLimit: 50_000 },
};

class ClayRateLimiter {
  private queue: PQueue;
  private submissionCount = 0;
  private hourlyCount = 0;
  private hourlyResetAt: Date;
  private config: RateLimiterConfig;

  constructor(plan: keyof typeof PLAN_LIMITS) {
    this.config = PLAN_LIMITS[plan];
    this.queue = new PQueue({
      concurrency: 1,
      interval: 1000,
      intervalCap: this.config.maxPerSecond,
    });
    this.hourlyResetAt = new Date(Date.now() + 3600_000);
  }

  async submit(webhookUrl: string, data: Record<string, unknown>): Promise<Response> {
    // Check webhook lifetime limit
    if (this.submissionCount >= this.config.webhookLimit) {
      throw new Error(
        `Webhook submission limit (${this.config.webhookLimit}) reached. Create a new webhook.`
      );
    }

    // Check hourly limit
    if (Date.now() > this.hourlyResetAt.getTime()) {
      this.hourlyCount = 0;
      this.hourlyResetAt = new Date(Date.now() + 3600_000);
    }
    if (this.hourlyCount >= this.config.maxPerHour) {
      const waitMs = this.hourlyResetAt.getTime() - Date.now();
      console.log(`Hourly limit reached. Waiting ${(waitMs / 1000).toFixed(0)}s...`);
      await new Promise(r => setTimeout(r, waitMs));
      this.hourlyCount = 0;
    }

    return this.queue.add(async () => {
      const res = await fetch(webhookUrl, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify(data),
      });

      if (res.status === 429) {
        const retryAfter = parseInt(res.headers.get('Retry-After') || '60');
        console.log(`429 rate limited. Waiting ${retryAfter}s...`);
        await new Promise(r => setTimeout(r, retryAfter * 1000));
        return this.submit(webhookUrl, data); // Retry
      }

      this.submissionCount++;
      this.hourlyCount++;
      return res;
    });
  }

  getStats() {
    return {
      totalSubmissions: this.submissionCount,
      hourlyRemaining: this.config.maxPerHour - this.hourlyCount,
      webhookRemaining: this.config.webhookLimit - this.submissionCount,
    };
  }
}

// src/clay/backoff.ts
async function withClayBackoff<T>(
  operation: () => Promise<T>,
  maxRetries = 5
): Promise<T> {
  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    try {
      return await operation();
    } catch (error: any) {
      if (attempt === maxRetries) throw error;

      // Clay returns 429 for plan-level rate limits
      const status = error.status || error.response?.status;
      if (status !== 429 && (status < 500 || status >= 600)) throw error;

      const baseDelay = 1000 * Math.pow(2, attempt); // 1s, 2s, 4s, 8s, 16s
      const jitter = Math.random() * 500;
      const delay = Math.min(baseDelay + jitter, 60_000); // Max 60s

      console.log(`Clay rate limited (attempt ${attempt + 1}). Retrying in ${(delay / 1000).toFixed(1)}s`);
      await new Promise(r => setTimeout(r, delay));
    }
  }
  throw new Error('Unreachable');
}

// src/clay/webhook-manager.ts
interface WebhookState {
  url: string;
  submissionCount: number;
  createdAt: Date;
}

class WebhookManager {
  private webhooks: Map<string, WebhookState> = new Map();
  private readonly LIMIT = 50_000;
  private readonly WARN_THRESHOLD = 45_000;

  registerWebhook(tableId: string, url: string) {
    this.webhooks.set(tableId, { url, submissionCount: 0, createdAt: new Date() });
  }

  async getWebhookUrl(tableId: string): Promise<string> {
    const state = this.webhooks.get(tableId);
    if (!state) throw new Error(`No webhook registered for table ${tableId}`);

    if (state.submissionCount >= this.LIMIT) {
      throw new Error(
        `Webhook for table ${tableId} exhausted (${this.LIMIT} submissions). ` +
        `Create a new webhook in Clay UI: Table > + Add > Webhooks > Monitor webhook`
      );
    }

    if (state.submissionCount >= this.WARN_THRESHOLD) {
      console.warn(
        `Webhook for ${tableId} at ${state.submissionCount}/${this.LIMIT} submissions. ` +
        `Plan to create a replacement soon.`
      );
    }

    return state.url;
  }

  recordSubmission(tableId: string) {
    const state = this.webhooks.get(tableId);
    if (state) state.submissionCount++;
  }
}