Cloud Monitoring

Set up monitoring for our Kubernetes microservices on AWS.
- Use Prometheus and Grafana for metrics and dashboards
- Monitor API latency (p99 < 500ms) and error rate (< 1%)
- Send critical alerts to PagerDuty, warnings to Slack
- Aggregate logs with CloudWatch Logs

global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "alert_rules.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets: ["alertmanager:9093"]

scrape_configs:
  - job_name: "node-exporter"
    static_configs:
      - targets: ["node-exporter:9100"]

  - job_name: "app"
    metrics_path: /metrics
    static_configs:
      - targets: ["app:8080"]

  - job_name: "kubernetes-pods"
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: (.+)
        replacement: $1

groups:
  - name: slo-alerts
    rules:
      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m]))
          /
          sum(rate(http_requests_total[5m])) > 0.01
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Error rate exceeds 1% SLO"
          description: "{{ $labels.job }} error rate is {{ $value | humanizePercentage }}"

      - alert: HighP99Latency
        expr: |
          histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))
          > 0.5
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "P99 latency exceeds 500ms SLO"

      - alert: PodCrashLooping
        expr: increase(kube_pod_container_status_restarts_total[1h]) > 3
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Pod {{ $labels.pod }} is crash looping"

      - alert: HighMemoryUsage
        expr: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes > 0.9
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Node memory usage above 90%"

{
  "AWSTemplateFormatVersion": "2010-09-09",
  "Resources": {
    "ApiDashboard": {
      "Type": "AWS::CloudWatch::Dashboard",
      "Properties": {
        "DashboardName": "api-service-dashboard",
        "DashboardBody": "{\"widgets\":[{\"type\":\"metric\",\"properties\":{\"metrics\":[[\"AWS/ApplicationELB\",\"TargetResponseTime\",\"TargetGroup\",\"my-tg\",{\"stat\":\"p99\"}],[\"AWS/ApplicationELB\",\"HTTPCode_Target_5XX_Count\",\"TargetGroup\",\"my-tg\"]],\"period\":300,\"title\":\"API Latency & Errors\"}},{\"type\":\"metric\",\"properties\":{\"metrics\":[[\"Custom/App\",\"ActiveConnections\"],[\"Custom/App\",\"QueueDepth\"]],\"period\":60,\"title\":\"Application Metrics\"}}]}"
      }
    },
    "HighLatencyAlarm": {
      "Type": "AWS::CloudWatch::Alarm",
      "Properties": {
        "AlarmName": "api-high-latency",
        "MetricName": "TargetResponseTime",
        "Namespace": "AWS/ApplicationELB",
        "Statistic": "p99",
        "Period": 300,
        "EvaluationPeriods": 3,
        "Threshold": 0.5,
        "ComparisonOperator": "GreaterThanThreshold",
        "AlarmActions": ["arn:aws:sns:us-east-1:123456789012:ops-alerts"],
        "Dimensions": [
          {"Name": "TargetGroup", "Value": "my-tg"}
        ]
      }
    },
    "HighErrorRateAlarm": {
      "Type": "AWS::CloudWatch::Alarm",
      "Properties": {
        "AlarmName": "api-high-error-rate",
        "MetricName": "HTTPCode_Target_5XX_Count",
        "Namespace": "AWS/ApplicationELB",
        "Statistic": "Sum",
        "Period": 300,
        "EvaluationPeriods": 2,
        "Threshold": 50,
        "ComparisonOperator": "GreaterThanThreshold",
        "AlarmActions": ["arn:aws:sns:us-east-1:123456789012:ops-alerts"]
      }
    }
  }
}