Vigil Alert

Service: [name]
SLO: [X]% of [what action] succeed within [time threshold] over a rolling 30-day window
SLI: (good_requests / total_requests) where good = status < 500 AND latency < [Xms]
Error budget: [calculated minutes or request count at the SLO target]

# alerts/[service-name]-slo.yaml
groups:
  - name: [service-name]-slo
    rules:

      # Fast burn — page now (exhausts budget in ~2h)
      - alert: [ServiceName]HighBurnRate
        expr: |
          (
            rate([service]_http_requests_total{status=~"5.."}[1h])
            / rate([service]_http_requests_total[1h])
          ) > (14.4 * [error_budget_ratio])
          and
          (
            rate([service]_http_requests_total{status=~"5.."}[5m])
            / rate([service]_http_requests_total[5m])
          ) > (14.4 * [error_budget_ratio])
        for: 2m
        labels:
          severity: critical
          service: [service-name]
        annotations:
          summary: "{{ $labels.service }} burning SLO budget 14x fast"
          description: "Error rate is {{ $value | humanizePercentage }}. At this rate, the 30-day error budget is exhausted in ~2 hours."
          runbook: "https://docs.internal/runbooks/[service-name]-high-burn-rate"

      # Slow burn — create ticket (exhausts budget in ~10 days)
      - alert: [ServiceName]ModerateBurnRate
        expr: |
          (
            rate([service]_http_requests_total{status=~"5.."}[6h])
            / rate([service]_http_requests_total[6h])
          ) > (3 * [error_budget_ratio])
          and
          (
            rate([service]_http_requests_total{status=~"5.."}[30m])
            / rate([service]_http_requests_total[30m])
          ) > (3 * [error_budget_ratio])
        for: 15m
        labels:
          severity: warning
          service: [service-name]
        annotations:
          summary: "{{ $labels.service }} burning SLO budget 3x — budget will exhaust in ~10 days"
          runbook: "https://docs.internal/runbooks/[service-name]-moderate-burn-rate"

      # Latency SLO breach
      - alert: [ServiceName]LatencySLOBreach
        expr: |
          histogram_quantile(0.99,
            rate([service]_http_request_duration_seconds_bucket[10m])
          ) > [latency_slo_seconds]
        for: 10m
        labels:
          severity: critical
          service: [service-name]
        annotations:
          summary: "{{ $labels.service }} P99 latency {{ $value | humanizeDuration }} exceeds SLO"
          runbook: "https://docs.internal/runbooks/[service-name]-latency-breach"

# datadog_monitors.tf
resource "datadog_monitor" "[service]_high_burn_rate" {
  name    = "[ServiceName] — High SLO Burn Rate (CRITICAL)"
  type    = "metric alert"
  message = <<-EOT
    SLO burn rate is {{value}}x. Budget exhausts in ~2 hours.
    Runbook: https://docs.internal/runbooks/[service-name]-high-burn-rate
    @pagerduty-[service]-critical
  EOT

  query = "sum(last_1h):sum:trace.web.request.errors{service:[service-name]}.as_count() / sum:trace.web.request.hits{service:[service-name]}.as_count() > ${14.4 * error_budget_ratio}"

  thresholds = {
    critical = 14.4 * error_budget_ratio
    warning  = 3 * error_budget_ratio
  }

  notify_no_data    = false
  renotify_interval = 60
  tags              = ["service:[service-name]", "team:engineering", "slo:availability"]
}