Posthog Observability

set -euo pipefail
# Check if events are flowing (last 24 hours)
curl "https://us.posthog.com/api/projects/$POSTHOG_PROJECT_ID/query/" \
  -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "query": {
      "kind": "HogQLQuery",
      "query": "SELECT toStartOfHour(timestamp) AS hour, count() AS events FROM events WHERE timestamp > now() - interval 24 hour GROUP BY hour ORDER BY hour"
    }
  }' | jq '.results | map({hour: .[0], events: .[1]}) | .[-3:]'

// posthog-instrumented.ts
import { PostHog } from 'posthog-node';

const posthog = new PostHog(process.env.NEXT_PUBLIC_POSTHOG_KEY!, {
  host: 'https://us.i.posthog.com',
  personalApiKey: process.env.POSTHOG_FEATURE_FLAGS_SECURE_API_KEY,
});

// Wrap flag evaluation with timing
async function getFlag(flagKey: string, userId: string): Promise<any> {
  const start = performance.now();
  const value = await posthog.getFeatureFlag(flagKey, userId);
  const durationMs = performance.now() - start;

  // Emit metrics to your monitoring system
  emitHistogram('posthog_flag_eval_duration_ms', durationMs, { flag: flagKey });
  emitCounter('posthog_flag_evals_total', 1, { flag: flagKey, result: String(value) });

  // Alert on slow evaluations (likely means local eval not configured)
  if (durationMs > 200) {
    console.warn(`[PostHog] Slow flag eval: ${flagKey} took ${durationMs.toFixed(0)}ms — inspect secure-key cache and fallback behavior`);
  }

  return value;
}

// Example: emit to Prometheus via prom-client
import { Histogram, Counter, Gauge } from 'prom-client';

const flagDuration = new Histogram({
  name: 'posthog_flag_eval_duration_ms',
  help: 'PostHog feature flag evaluation duration',
  labelNames: ['flag'],
  buckets: [1, 5, 10, 50, 100, 200, 500, 1000],
});

const flagEvals = new Counter({
  name: 'posthog_flag_evals_total',
  help: 'Total PostHog feature flag evaluations',
  labelNames: ['flag', 'result'],
});

function emitHistogram(name: string, value: number, labels: Record<string, string>) {
  flagDuration.observe(labels, value);
}

function emitCounter(name: string, value: number, labels: Record<string, string>) {
  flagEvals.inc(labels, value);
}

// Run on a cron (e.g., every 6 hours)
async function checkEventVolume() {
  const result = await fetch(
    `https://us.posthog.com/api/projects/${process.env.POSTHOG_PROJECT_ID}/query/`,
    {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        Authorization: `Bearer ${process.env.POSTHOG_PERSONAL_API_KEY}`,
      },
      body: JSON.stringify({
        query: {
          kind: 'HogQLQuery',
          query: `
            SELECT
              count() AS events_this_month,
              uniq(distinct_id) AS unique_users,
              count() / dateDiff('day', toStartOfMonth(now()), now()) AS daily_avg
            FROM events
            WHERE timestamp > toStartOfMonth(now())
          `,
        },
      }),
    }
  );

  const data = await result.json();
  const [eventsThisMonth, uniqueUsers, dailyAvg] = data.results[0];
  const projectedMonthly = dailyAvg * 30;
  const approvedMonthlyLimit = Number(process.env.POSTHOG_MONTHLY_EVENT_LIMIT);
  if (!Number.isFinite(approvedMonthlyLimit)) {
    throw new Error('Set POSTHOG_MONTHLY_EVENT_LIMIT from the live billing configuration');
  }

  const metrics = {
    events_this_month: eventsThisMonth,
    unique_users: uniqueUsers,
    daily_average: Math.round(dailyAvg),
    projected_monthly: Math.round(projectedMonthly),
    pct_of_approved_limit: Math.round((projectedMonthly / approvedMonthlyLimit) * 100),
  };

  // Emit gauge metrics
  const volumeGauge = new Gauge({
    name: 'posthog_events_month_total',
    help: 'PostHog events this month',
  });
  volumeGauge.set(eventsThisMonth);

  // Apply the organization's reviewed warning ratio to the live product limit.
  const warningRatio = Number(process.env.POSTHOG_USAGE_WARNING_RATIO ?? '0.8');
  if (projectedMonthly > approvedMonthlyLimit * warningRatio) {
    await sendAlert(`PostHog projected usage is ${Math.round(projectedMonthly / approvedMonthlyLimit * 100)}% of the reviewed limit`);
  }

  return metrics;
}

# prometheus/posthog-alerts.yml
# These are application-emitted metrics. Rename them to match the target service.
groups:
  - name: posthog
    rules:
      - alert: PostHogCaptureDeliveryFailures
        expr: rate(posthog_capture_delivery_failures_total[5m]) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "The application is reporting PostHog delivery failures"

      - alert: PostHogFlagEvalSlow
        expr: increase(posthog_flag_eval_slo_breaches_total[5m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Application-observed PostHog flag evaluation latency exceeded its reviewed threshold"

      - alert: PostHogUsageLimitWarning
        expr: posthog_usage_limit_ratio > posthog_usage_warning_ratio
        labels:
          severity: info
        annotations:
          summary: "Projected PostHog usage crossed the organization's reviewed warning ratio"

      - alert: PostHogCaptureErrors
        expr: rate(posthog_capture_errors_total[5m]) > 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "PostHog capture errors elevated — events may be lost"

// Dashboard panels to track PostHog health
const dashboardQueries = {
  // Events per hour (last 24h)
  eventRate: `
    SELECT toStartOfHour(timestamp) AS hour, count() AS events
    FROM events WHERE timestamp > now() - interval 24 hour
    GROUP BY hour ORDER BY hour
  `,

  // Events by type (last 7 days)
  eventsByType: `
    SELECT event, count() AS total
    FROM events WHERE timestamp > now() - interval 7 day
    GROUP BY event ORDER BY total DESC LIMIT 15
  `,

  // Unique users per day (last 30 days)
  dailyActiveUsers: `
    SELECT toDate(timestamp) AS day, uniq(distinct_id) AS users
    FROM events WHERE timestamp > now() - interval 30 day
    GROUP BY day ORDER BY day
  `,

  // Event ingestion latency estimate
  ingestionFreshness: `
    SELECT max(timestamp) AS latest_event,
           dateDiff('second', max(timestamp), now()) AS seconds_behind
    FROM events
  `,
};