Sentry Observability

import winston from 'winston';
import * as Sentry from '@sentry/node';

class SentryTransport extends winston.Transport {
  log(info: any, callback: () => void) {
    setImmediate(callback);

    if (info.level === 'error' || info.level === 'fatal') {
      const error = info.error instanceof Error
        ? info.error
        : new Error(info.message);

      Sentry.withScope((scope) => {
        scope.setTag('logger', 'winston');
        scope.setContext('log_entry', {
          level: info.level,
          timestamp: info.timestamp,
          service: info.service,
        });
        const eventId = Sentry.captureException(error);
        info.sentry_event_id = eventId;
        info.sentry_url = `https://${process.env.SENTRY_ORG}.sentry.io/issues/?query=${eventId}`;
      });
    }
  }
}

const logger = winston.createLogger({
  defaultMeta: { service: 'api-gateway' },
  transports: [
    new winston.transports.Console({ format: winston.format.json() }),
    new SentryTransport(),
  ],
});

import pino from 'pino';
import * as Sentry from '@sentry/node';

const logger = pino({
  hooks: {
    logMethod(inputArgs, method, level) {
      if (level >= 50) { // 50 = error, 60 = fatal
        const [obj, msg] = typeof inputArgs[0] === 'object'
          ? [inputArgs[0], inputArgs[1]]
          : [{}, inputArgs[0]];

        Sentry.withScope((scope) => {
          scope.setTag('logger', 'pino');
          const eventId = Sentry.captureException(
            obj.err instanceof Error ? obj.err : new Error(String(msg))
          );
          if (typeof inputArgs[0] === 'object') {
            inputArgs[0].sentry_event_id = eventId;
          }
        });
      }
      return method.apply(this, inputArgs);
    },
  },
});

import { randomUUID } from 'crypto';
import * as Sentry from '@sentry/node';

app.use((req, res, next) => {
  const requestId = (req.headers['x-request-id'] as string) || randomUUID();
  req.requestId = requestId;
  res.setHeader('x-request-id', requestId);
  Sentry.setTag('request_id', requestId);
  req.log = logger.child({ requestId, path: req.path });
  next();
});

import * as Sentry from '@sentry/node';

// Counter — track error rates alongside business events
Sentry.metrics.increment('checkout.attempted', 1, {
  tags: { payment_provider: 'stripe', plan: 'enterprise' },
});

Sentry.metrics.increment('checkout.failed', 1, {
  tags: { payment_provider: 'stripe', failure_reason: 'timeout' },
});

// Distribution — track latency with error correlation
Sentry.metrics.distribution('api.response_time', responseTimeMs, {
  tags: { endpoint: '/api/orders', status_code: String(res.statusCode) },
  unit: 'millisecond',
});

// Gauge — track queue depth, connection pool size
Sentry.metrics.gauge('db.pool.active', pool.activeCount, {
  tags: { database: 'primary' },
});

// Set — track unique affected users during incidents
Sentry.metrics.set('incident.affected_users', userId, {
  tags: { incident: 'payment-outage-2026-03' },
});

import tracer from 'dd-trace';
import * as Sentry from '@sentry/node';

// dd-trace MUST be initialized before @sentry/node
Sentry.init({
  dsn: process.env.SENTRY_DSN,
  beforeSend(event) {
    const span = tracer.scope().active();
    if (span) {
      const traceId = span.context().toTraceId();
      event.tags = { ...event.tags, 'dd.trace_id': traceId };
      event.contexts = {
        ...event.contexts,
        datadog: {
          trace_url: `https://app.datadoghq.com/apm/trace/${traceId}`,
          trace_id: traceId,
        },
      };
    }
    return event;
  },
});

import * as Sentry from '@sentry/node';

function setMonitoringContext(req: Request) {
  const traceId = Sentry.getActiveSpan()?.spanContext().traceId;
  const spanId = Sentry.getActiveSpan()?.spanContext().spanId;
  const requestId = req.headers['x-request-id'] as string || crypto.randomUUID();

  // setContext creates a named section in the Sentry event sidebar
  Sentry.setContext('monitoring', {
    traceId,
    spanId,
    requestId,
    grafana_dashboard: `https://grafana.example.com/d/abc123?var-trace_id=${traceId}`,
    kibana_logs: `https://kibana.example.com/app/logs?query=request_id:${requestId}`,
    datadog_trace: traceId
      ? `https://app.datadoghq.com/apm/trace/${traceId}`
      : undefined,
  });

  Sentry.setTag('request_id', requestId);
  Sentry.setTag('trace_id', traceId || 'none');
  Sentry.setTag('deployment', process.env.DEPLOYMENT_ID || 'unknown');
}

app.use((req, res, next) => {
  setMonitoringContext(req);
  next();
});

// Receive Sentry webhook, create Grafana annotation
app.post('/sentry-to-grafana', async (req, res) => {
  const { event } = req.body;
  if (!event) return res.status(200).send('ignored');

  await fetch(`${process.env.GRAFANA_URL}/api/annotations`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      Authorization: `Bearer ${process.env.GRAFANA_API_KEY}`,
    },
    body: JSON.stringify({
      dashboardUID: process.env.GRAFANA_DASHBOARD_UID,
      panelId: 1,
      time: new Date(event.datetime).getTime(),
      tags: ['sentry', event.level, event.project],
      text: `**${event.title}**\nLevel: ${event.level}\nView in Sentry`,
    }),
  });

  res.status(201).json({ status: 'annotation_created' });
});

Issue Alert: "Critical Production Error"
  When: An event is first seen
  If: level is fatal AND environment is production
  Then: PagerDuty (Critical) + #alerts-critical Slack + Grafana annotation
  Frequency: Once per issue

Metric Alert: "Error Rate Spike"
  When: Error count > 50 in 5 minutes
  Then: PagerDuty (High) + #alerts-production Slack + webhook to Grafana
  Resolve: Error count < 5 for 10 minutes

Metric Alert: "Latency Regression"
  When: p95(transaction.duration) for /api/* > 2000ms for 10 minutes
  Then: #alerts-performance Slack + JIRA ticket via webhook
  Resolve: p95 < 1000ms for 15 minutes