Lokalise Incident Runbook

#!/bin/bash
# incident-diagnostics.sh — Run all three checks in sequence
set -uo pipefail

: "${LOKALISE_API_TOKEN:?Set LOKALISE_API_TOKEN before running diagnostics}"
: "${LOKALISE_PROJECT_ID:?Set LOKALISE_PROJECT_ID before running diagnostics}"

echo "=== 1. Lokalise API Health ==="
API_STATUS=$(curl -sf -o /dev/null -w "%{http_code}" \
  "https://api.lokalise.com/api2/projects/${LOKALISE_PROJECT_ID}" \
  -H "X-Api-Token: ${LOKALISE_API_TOKEN}")

case "$API_STATUS" in
  200) echo "API: HEALTHY (200 OK)" ;;
  401) echo "API: AUTH FAILURE (401) — Token invalid or expired. Rotate immediately." ;;
  403) echo "API: FORBIDDEN (403) — Token lacks permissions for this project." ;;
  404) echo "API: NOT FOUND (404) — Check LOKALISE_PROJECT_ID value." ;;
  429) echo "API: RATE LIMITED (429) — Throttled. Wait 10 seconds and retry." ;;
  5*)  echo "API: LOKALISE OUTAGE (${API_STATUS}) — Check https://status.lokalise.com" ;;
  000) echo "API: UNREACHABLE — DNS/network issue. Check connectivity." ;;
  *)   echo "API: UNEXPECTED (${API_STATUS}) — Investigate further." ;;
esac

echo ""
echo "=== 2. Token Validity ==="
TOKEN_CHECK=$(curl -sf "https://api.lokalise.com/api2/projects/${LOKALISE_PROJECT_ID}" \
  -H "X-Api-Token: ${LOKALISE_API_TOKEN}" 2>/dev/null)

if [[ $? -eq 0 ]]; then
  PROJECT_NAME=$(echo "$TOKEN_CHECK" | jq -r '.project.name')
  TEAM_ID=$(echo "$TOKEN_CHECK" | jq -r '.project.team_id')
  echo "Token: VALID"
  echo "  Project: ${PROJECT_NAME}"
  echo "  Team ID: ${TEAM_ID}"
else
  echo "Token: INVALID or project inaccessible"
  echo "  Action: Get a valid token from your secret manager or Lokalise dashboard"
fi

echo ""
echo "=== 3. Rate Limit Status ==="
RATE_RESPONSE=$(curl -sI "https://api.lokalise.com/api2/projects/${LOKALISE_PROJECT_ID}/keys?limit=1" \
  -H "X-Api-Token: ${LOKALISE_API_TOKEN}" 2>/dev/null)

RATE_LIMIT=$(echo "$RATE_RESPONSE" | grep -i "x-ratelimit-limit" | tr -d '\r' | awk '{print $2}')
RATE_REMAINING=$(echo "$RATE_RESPONSE" | grep -i "x-ratelimit-remaining" | tr -d '\r' | awk '{print $2}')

if [[ -n "$RATE_REMAINING" ]]; then
  echo "Rate limit:     ${RATE_LIMIT:-6} req/sec"
  echo "Remaining:      ${RATE_REMAINING} req/sec"
  if [[ "${RATE_REMAINING}" -eq 0 ]]; then
    echo "STATUS: EXHAUSTED — Wait 1 second for reset"
  else
    echo "STATUS: OK"
  fi
else
  echo "Could not determine rate limit status"
fi

# 1. Check if the key exists in Lokalise
KEY_NAME="homepage.welcome_message"  # Replace with the missing key
curl -sf "https://api.lokalise.com/api2/projects/${LOKALISE_PROJECT_ID}/keys?filter_keys=${KEY_NAME}" \
  -H "X-Api-Token: ${LOKALISE_API_TOKEN}" \
  | jq '.keys[] | {key_name: .key_name.web, translations: [.translations[] | {locale: .language_iso, value: .translation}]}'

# 2. Check if the locale file was included in the deployed build
# (run on the production server or check the build artifact)
ls -la /app/locales/  # Adjust path to your deployed locale directory
cat /app/locales/de.json | jq ".$KEY_NAME" 2>/dev/null || echo "Key not found in deployed file"

# 3. Quick fix: Re-download and redeploy
lokalise2 file download \
  --token "$LOKALISE_API_TOKEN" \
  --project-id "$LOKALISE_PROJECT_ID" \
  --format json \
  --original-filenames=true \
  --directory-prefix="" \
  --export-empty-as=base \
  --unzip-to "src/locales/"
# Then trigger a redeploy

# 1. Compare Lokalise timestamp with deployed file
LOKALISE_UPDATED=$(curl -sf "https://api.lokalise.com/api2/projects/${LOKALISE_PROJECT_ID}" \
  -H "X-Api-Token: ${LOKALISE_API_TOKEN}" \
  | jq -r '.project.statistics.datetime')
echo "Lokalise last updated: $LOKALISE_UPDATED"

# 2. Check when your deployed translations were built
stat src/locales/en.json  # File modification time

# 3. Force cache invalidation if using OTA
# For i18next-http-backend or similar:
# Clear the browser/app cache or increment the version query parameter

# 4. Re-download fresh translations
lokalise2 file download \
  --token "$LOKALISE_API_TOKEN" \
  --project-id "$LOKALISE_PROJECT_ID" \
  --format json \
  --original-filenames=true \
  --directory-prefix="" \
  --unzip-to "src/locales/"

# 1. Confirm on status page
echo "Check: https://status.lokalise.com"
curl -sf "https://status.lokalise.com/api/v2/summary.json" 2>/dev/null \
  | jq '.status.description' || echo "Status page unreachable"

# 2. Enable fallback translations
# Your app should have a fallback mechanism. If not, deploy one immediately:

// Emergency fallback implementation
// Add to your translation loader

import bundledTranslations from './locales/en.json';

async function loadTranslations(locale: string): Promise<Record<string, string>> {
  try {
    // Try loading from Lokalise/CDN/API
    const response = await fetch(`/api/translations/${locale}`, {
      signal: AbortSignal.timeout(3000),  // 3-second timeout
    });
    if (!response.ok) throw new Error(`HTTP ${response.status}`);
    return await response.json();
  } catch (error) {
    console.error(`Translation fetch failed for ${locale}, using fallback:`, error);
    // Fall back to bundled English translations
    return bundledTranslations;
  }
}

# 3. If your app crashes without the API, set the env var to enable static fallback:
export LOKALISE_FALLBACK_ENABLED=true
# Then restart the application

# 4. Monitor for recovery
watch -n 30 'curl -sf -o /dev/null -w "%{http_code}" \
  "https://api.lokalise.com/api2/projects/${LOKALISE_PROJECT_ID}" \
  -H "X-Api-Token: ${LOKALISE_API_TOKEN}"'

# 1. Validate the source file locally
jq empty src/locales/en.json && echo "Valid JSON" || echo "INVALID JSON — fix syntax"

# 2. Check file size (Lokalise limit: 50MB per file)
du -h src/locales/en.json

# 3. Test upload with verbose output
lokalise2 file upload \
  --token "$LOKALISE_API_TOKEN" \
  --project-id "$LOKALISE_PROJECT_ID" \
  --file "src/locales/en.json" \
  --lang-iso "en" \
  --replace-modified \
  --poll \
  --poll-timeout 120s 2>&1

# 4. Common fixes:
# - "Unsupported file format": Check file extension matches --format
# - "Key name too long": Lokalise limit is 1024 chars per key
# - "Too many keys": Split into multiple files if > 10,000 keys per upload
# - 429 error: Wait and retry, or reduce upload frequency

// src/i18n/fallback-loader.ts
import fs from 'fs';
import path from 'path';

const CACHE_DIR = path.resolve(__dirname, '../locales');
const FALLBACK_DIR = path.resolve(__dirname, '../locales-fallback');

/**
 * Copy current translations to fallback directory.
 * Run this as a post-build step: `cp -r src/locales/ src/locales-fallback/`
 */
export function loadWithFallback(locale: string): Record<string, unknown> {
  const primaryPath = path.join(CACHE_DIR, `${locale}.json`);
  const fallbackPath = path.join(FALLBACK_DIR, `${locale}.json`);
  const defaultPath = path.join(FALLBACK_DIR, 'en.json');

  // Try primary (freshly downloaded)
  if (fs.existsSync(primaryPath)) {
    try {
      return JSON.parse(fs.readFileSync(primaryPath, 'utf-8'));
    } catch { /* fall through */ }
  }

  // Try locale-specific fallback
  if (fs.existsSync(fallbackPath)) {
    console.warn(`Using fallback translations for ${locale}`);
    return JSON.parse(fs.readFileSync(fallbackPath, 'utf-8'));
  }

  // Last resort: English fallback
  console.error(`No translations available for ${locale}, falling back to English`);
  return JSON.parse(fs.readFileSync(defaultPath, 'utf-8'));
}

[INCIDENT] Translation service degraded
Status: Investigating
Impact: [Users in DE/FR/ES/JA locales may see English text | All translations unavailable]
Start time: YYYY-MM-DD HH:MM UTC
Cause: [Lokalise API outage | Stale translation cache | Missing keys in deployment]
Next update: 15 minutes

[UPDATE] Translation service — Mitigation in progress
Status: Mitigating
Action taken: [Enabled fallback translations | Redeployed with fresh downloads | Reverted to previous build]
ETA for full resolution: [30 minutes | Next deployment cycle | Pending Lokalise recovery]

[RESOLVED] Translation service restored
Duration: X hours Y minutes
Root cause: [Brief description]
Impact: [Number of affected users/requests]
Follow-up: Postmortem scheduled for YYYY-MM-DD