Observability Testing Patterns

vibium go "$GRAFANA_URL/d/api-latency"
vibium wait load
node .claude/skills/qe-browser/scripts/assert.js --checks '[
  {"kind": "selector_visible", "selector": ".panel-title"},
  {"kind": "no_console_errors"},
  {"kind": "no_failed_requests"},
  {"kind": "element_count", "selector": ".panel", "op": ">=", "count": 4}
]'
node .claude/skills/qe-browser/scripts/visual-diff.js --name "grafana-api-latency"

describe('Dashboard Data Accuracy', () => {
  it('order count on dashboard matches database', async () => {
    // Step 1: Get ground truth from source database
    const dbResult = await db.query(
      "SELECT COUNT(*) as count FROM orders WHERE created_at >= NOW() - INTERVAL '24 HOURS'"
    );
    const dbCount = parseInt(dbResult.rows[0].count);

    // Step 2: Query Elasticsearch (same data source as dashboard)
    const esResult = await esClient.search({
      index: 'orders-*',
      body: {
        query: {
          range: { created_at: { gte: 'now-24h' } }
        },
        size: 0,
        track_total_hits: true
      }
    });
    const esCount = esResult.hits.total.value;

    // Step 3: Compare
    expect(esCount).toBe(dbCount);
  });

  it('revenue metric on dashboard matches transaction totals', async () => {
    const dbRevenue = await db.query(
      "SELECT SUM(total) as revenue FROM orders WHERE status = 'COMPLETED' AND created_at >= NOW() - INTERVAL '24 HOURS'"
    );
    const expectedRevenue = parseFloat(dbRevenue.rows[0].revenue);

    const esResult = await esClient.search({
      index: 'orders-*',
      body: {
        query: {
          bool: {
            must: [
              { term: { status: 'COMPLETED' } },
              { range: { created_at: { gte: 'now-24h' } } }
            ]
          }
        },
        aggs: {
          total_revenue: { sum: { field: 'total' } }
        },
        size: 0
      }
    });
    const dashboardRevenue = esResult.aggregations.total_revenue.value;

    // Allow small floating point tolerance
    expect(Math.abs(dashboardRevenue - expectedRevenue)).toBeLessThan(0.01);
  });

  it('error rate percentage is calculated correctly', async () => {
    const esResult = await esClient.search({
      index: 'logs-*',
      body: {
        query: { range: { '@timestamp': { gte: 'now-1h' } } },
        aggs: {
          total: { value_count: { field: 'status_code' } },
          errors: {
            filter: { range: { status_code: { gte: 500 } } },
            aggs: { count: { value_count: { field: 'status_code' } } }
          }
        },
        size: 0
      }
    });

    const total = esResult.aggregations.total.value;
    const errors = esResult.aggregations.errors.count.value;
    const expectedErrorRate = (errors / total) * 100;

    // Fetch what the dashboard shows via Kibana API
    const dashboardPanel = await kibanaApi.get('/api/saved_objects/visualization/error-rate-gauge');
    const displayedErrorRate = await evaluateKibanaVisualization(dashboardPanel);

    expect(Math.abs(displayedErrorRate - expectedErrorRate)).toBeLessThan(0.1);
  });
});

describe('Elasticsearch Query Validation', () => {
  it('validates date histogram aggregation returns correct buckets', async () => {
    // Insert known test data
    const testDocs = [];
    for (let hour = 0; hour < 24; hour++) {
      const timestamp = new Date();
      timestamp.setHours(hour, 0, 0, 0);
      testDocs.push({
        '@timestamp': timestamp.toISOString(),
        service: 'order-api',
        status_code: hour % 5 === 0 ? 500 : 200,
        response_time: 100 + (hour * 10)
      });
    }

    await esClient.bulk({
      index: 'test-logs',
      body: testDocs.flatMap(doc => [{ index: {} }, doc])
    });
    await esClient.indices.refresh({ index: 'test-logs' });

    // Run the same query the dashboard uses
    const result = await esClient.search({
      index: 'test-logs',
      body: {
        query: { match_all: {} },
        aggs: {
          requests_over_time: {
            date_histogram: { field: '@timestamp', fixed_interval: '1h' },
            aggs: {
              avg_response: { avg: { field: 'response_time' } },
              error_count: {
                filter: { range: { status_code: { gte: 500 } } }
              }
            }
          }
        },
        size: 0
      }
    });

    const buckets = result.aggregations.requests_over_time.buckets;
    expect(buckets.length).toBe(24);

    // Verify specific bucket values
    const errorBuckets = buckets.filter(b => b.error_count.doc_count > 0);
    expect(errorBuckets.length).toBe(5); // Hours 0, 5, 10, 15, 20
  });

  it('validates term aggregation for top services', async () => {
    const result = await esClient.search({
      index: 'logs-*',
      body: {
        query: { range: { '@timestamp': { gte: 'now-1h' } } },
        aggs: {
          top_services: {
            terms: { field: 'service.keyword', size: 10 }
          }
        },
        size: 0
      }
    });

    const services = result.aggregations.top_services.buckets;
    expect(services.length).toBeGreaterThan(0);

    // Each bucket should have reasonable doc counts
    for (const bucket of services) {
      expect(bucket.key).toBeDefined();
      expect(bucket.doc_count).toBeGreaterThan(0);
    }
  });
});

describe('Kibana Dashboard Visual Validation', () => {
  it('validates dashboard panels render without errors', async () => {
    await page.goto(`${kibanaUrl}/app/dashboards#/view/operations-overview`);

    // Wait for all panels to finish loading
    await page.waitForSelector('.embPanel__content', { state: 'visible' });
    await page.waitForFunction(() => {
      const loaders = document.querySelectorAll('.euiLoadingSpinner');
      return loaders.length === 0;
    }, { timeout: 30000 });

    // Check no error icons on any panel
    const errorPanels = await page.locator('.embPanel--error').count();
    expect(errorPanels).toBe(0);

    // Check no "No results found" where data is expected
    const noResultPanels = await page.locator('text="No results found"').count();
    expect(noResultPanels).toBe(0);
  });

  it('validates metric visualization shows correct value', async () => {
    await page.goto(`${kibanaUrl}/app/dashboards#/view/operations-overview`);
    await page.waitForLoadState('networkidle');

    // Get the displayed metric value
    const metricValue = await page.locator('[data-test-subj="metricVis-total-orders"] .mtrVis__value').textContent();
    const displayedCount = parseInt(metricValue.replace(/,/g, ''));

    // Compare with direct ES query
    const esResult = await esClient.count({ index: 'orders-*' });

    expect(displayedCount).toBe(esResult.count);
  });

  it('validates table visualization columns and sorting', async () => {
    await page.goto(`${kibanaUrl}/app/dashboards#/view/operations-overview`);
    await page.waitForLoadState('networkidle');

    // Verify expected columns exist
    const headers = await page.locator('.euiTable th').allTextContents();
    expect(headers).toContain('Service');
    expect(headers).toContain('Error Rate');
    expect(headers).toContain('P95 Latency');

    // Verify sorting works
    await page.click('th:has-text("Error Rate")');
    const firstRow = await page.locator('.euiTable tbody tr:first-child td').allTextContents();
    const secondRow = await page.locator('.euiTable tbody tr:nth-child(2) td').allTextContents();

    const firstErrorRate = parseFloat(firstRow[1]);
    const secondErrorRate = parseFloat(secondRow[1]);
    expect(firstErrorRate).toBeGreaterThanOrEqual(secondErrorRate);
  });
});

describe('Alert Rule Validation', () => {
  it('fires alert when error rate exceeds threshold', async () => {
    // Generate errors to exceed the 5% threshold
    const requests = [];
    for (let i = 0; i < 100; i++) {
      requests.push({
        '@timestamp': new Date().toISOString(),
        service: 'payment-api',
        status_code: i < 10 ? 500 : 200, // 10% error rate > 5% threshold
        response_time: 200
      });
    }

    await esClient.bulk({
      index: 'logs-payment',
      body: requests.flatMap(doc => [{ index: {} }, doc])
    });
    await esClient.indices.refresh({ index: 'logs-payment' });

    // Wait for alert evaluation cycle (typically 1 minute)
    await sleep(90000);

    // Check alert was fired
    const alerts = await alertManager.getActiveAlerts({
      filter: 'alertname="HighErrorRate" AND service="payment-api"'
    });
    expect(alerts.length).toBeGreaterThan(0);
    expect(alerts[0].labels.severity).toBe('critical');
  });

  it('alert auto-resolves when condition clears', async () => {
    // First trigger the alert
    await injectErrors('payment-api', { count: 50, total: 100 });
    await sleep(90000);

    let alerts = await alertManager.getActiveAlerts({ filter: 'alertname="HighErrorRate"' });
    expect(alerts.length).toBeGreaterThan(0);

    // Now inject healthy traffic to bring error rate below threshold
    await injectSuccessRequests('payment-api', { count: 1000 });
    await sleep(90000);

    // Alert should auto-resolve
    alerts = await alertManager.getActiveAlerts({ filter: 'alertname="HighErrorRate"' });
    expect(alerts.length).toBe(0);
  });

  it('alert notification reaches correct channel', async () => {
    // Subscribe to notification channel
    const notifications = [];
    const subscription = pagerDutyMock.onIncident((incident) => {
      notifications.push(incident);
    });

    // Trigger alert condition
    await injectErrors('critical-service', { count: 50, total: 100 });
    await sleep(120000);

    expect(notifications.length).toBeGreaterThan(0);
    expect(notifications[0].service.name).toBe('critical-service');
    expect(notifications[0].urgency).toBe('high');

    subscription.unsubscribe();
  });

  it('alert does not fire for brief transient spikes', async () => {
    // Inject a brief 30-second spike (alert requires 5 minutes sustained)
    await injectErrors('api-service', { count: 20, total: 50, duration: 30000 });
    await sleep(120000);

    const alerts = await alertManager.getActiveAlerts({ filter: 'alertname="HighErrorRate"' });
    expect(alerts.length).toBe(0); // Should NOT fire for transient spike
  });
});

describe('Log Aggregation Completeness', () => {
  it('all microservice logs appear in centralized index', async () => {
    const traceId = uuid();
    const services = ['api-gateway', 'auth-service', 'order-service', 'payment-service', 'notification-service'];

    // Generate a log entry with known traceId in each service
    for (const service of services) {
      await serviceLogEmitter.emit(service, {
        level: 'INFO',
        message: `Completeness test - ${traceId}`,
        traceId,
        timestamp: new Date().toISOString()
      });
    }

    // Wait for log pipeline to process (Filebeat -> Logstash -> Elasticsearch)
    await sleep(15000);

    // Query Elasticsearch for the trace ID
    const result = await esClient.search({
      index: 'logs-*',
      body: {
        query: { term: { 'traceId.keyword': traceId } },
        size: 100
      }
    });

    const foundServices = result.hits.hits.map(h => h._source.service);

    // All services should have their log entry in Elasticsearch
    for (const service of services) {
      expect(foundServices).toContain(service);
    }
    expect(foundServices.length).toBe(services.length);
  });

  it('logs retain correct structure after pipeline processing', async () => {
    const testLog = {
      level: 'ERROR',
      message: 'Payment declined',
      traceId: uuid(),
      userId: 'user-123',
      orderId: 'order-456',
      errorCode: 'INSUFFICIENT_FUNDS',
      timestamp: new Date().toISOString()
    };

    await serviceLogEmitter.emit('payment-service', testLog);
    await sleep(10000);

    const result = await esClient.search({
      index: 'logs-*',
      body: { query: { term: { 'traceId.keyword': testLog.traceId } } }
    });

    expect(result.hits.hits.length).toBe(1);
    const indexed = result.hits.hits[0]._source;

    // Verify all fields survived the pipeline
    expect(indexed.level).toBe('ERROR');
    expect(indexed.message).toBe('Payment declined');
    expect(indexed.userId).toBe('user-123');
    expect(indexed.orderId).toBe('order-456');
    expect(indexed.errorCode).toBe('INSUFFICIENT_FUNDS');
  });

  it('detects log volume drops indicating pipeline issues', async () => {
    // Get baseline log volume for the past hour
    const baseline = await esClient.count({
      index: 'logs-*',
      body: { query: { range: { '@timestamp': { gte: 'now-2h', lt: 'now-1h' } } } }
    });

    const current = await esClient.count({
      index: 'logs-*',
      body: { query: { range: { '@timestamp': { gte: 'now-1h' } } } }
    });

    // Current volume should be at least 50% of baseline (not a sudden drop)
    const ratio = current.count / baseline.count;
    expect(ratio).toBeGreaterThan(0.5);
  });
});

describe('Distributed Trace Validation', () => {
  it('captures complete trace across all services', async () => {
    // Make a request that traverses multiple services
    const response = await httpClient.post('/api/orders', {
      customerId: 'CUST-TRACE',
      items: [{ sku: 'ITEM-1', qty: 1 }]
    });

    const traceId = response.headers['x-trace-id'];
    expect(traceId).toBeDefined();

    // Wait for trace to be indexed
    await sleep(10000);

    // Query Jaeger/APM for the trace
    const trace = await jaegerClient.getTrace(traceId);

    // Verify all expected spans exist
    const spanNames = trace.spans.map(s => s.operationName);
    expect(spanNames).toContain('POST /api/orders');
    expect(spanNames).toContain('auth.validateToken');
    expect(spanNames).toContain('order.create');
    expect(spanNames).toContain('payment.authorize');
    expect(spanNames).toContain('inventory.reserve');
    expect(spanNames).toContain('db.insert orders');

    // Verify parent-child relationships
    const apiSpan = trace.spans.find(s => s.operationName === 'POST /api/orders');
    const authSpan = trace.spans.find(s => s.operationName === 'auth.validateToken');
    expect(authSpan.references[0].refType).toBe('CHILD_OF');
    expect(authSpan.references[0].spanID).toBe(apiSpan.spanID);
  });

  it('traces capture error spans correctly', async () => {
    // Trigger a known error
    const response = await httpClient.post('/api/orders', {
      customerId: 'INVALID-CUSTOMER',
      items: [{ sku: 'ITEM-1', qty: 1 }]
    });

    const traceId = response.headers['x-trace-id'];
    await sleep(10000);

    const trace = await jaegerClient.getTrace(traceId);

    // Find error span
    const errorSpan = trace.spans.find(s => s.tags.some(t => t.key === 'error' && t.value === true));
    expect(errorSpan).toBeDefined();
    expect(errorSpan.logs).toContainEqual(
      expect.objectContaining({
        fields: expect.arrayContaining([
          expect.objectContaining({ key: 'error.message' })
        ])
      })
    );
  });

  it('validates trace sampling rate', async () => {
    const requestCount = 100;
    const traceIds = [];

    for (let i = 0; i < requestCount; i++) {
      const resp = await httpClient.get('/api/health');
      if (resp.headers['x-trace-id']) {
        traceIds.push(resp.headers['x-trace-id']);
      }
    }

    await sleep(15000);

    let tracesFound = 0;
    for (const traceId of traceIds) {
      try {
        await jaegerClient.getTrace(traceId);
        tracesFound++;
      } catch (e) {
        // Trace not sampled
      }
    }

    // With 10% sampling rate, expect roughly 10 traces (allow variance)
    const samplingRate = tracesFound / requestCount;
    expect(samplingRate).toBeGreaterThan(0.05);
    expect(samplingRate).toBeLessThan(0.20);
  });
});