Graceful Shutdown

let isShuttingDown = false;

function onShutdownSignal(signal: string): void {
  if (isShuttingDown) return; // prevent double-shutdown
  isShuttingDown = true;
  console.log(`Received ${signal}, starting graceful shutdown...`);
  shutdown();
}

process.on("SIGTERM", () => onShutdownSignal("SIGTERM"));
process.on("SIGINT", () => onShutdownSignal("SIGINT"));

async function shutdown(): Promise<void> {
  // 1. Stop accepting new connections
  server.close(() => {
    console.log("Server closed — no new connections accepted");
  });

  // 2. Mark health check as not-ready so load balancers stop routing
  //    (readiness probe returns 503 from this point)
}

const DRAIN_TIMEOUT_MS = 25_000; // must be less than orchestrator's terminationGracePeriodSeconds

async function drainAndExit(): Promise<void> {
  const deadline = setTimeout(() => {
    console.error("Drain timeout reached — forcing exit");
    process.exit(1);
  }, DRAIN_TIMEOUT_MS);
  deadline.unref(); // don't keep the event loop alive just for the timer

  try {
    // Wait for active connections to finish
    await waitForActiveConnections();

    // Flush buffered data (logs, metrics, queues)
    await flushBuffers();

    // Close external resource handles
    await closeResources();

    console.log("Graceful shutdown complete");
    process.exit(0);
  } catch (err) {
    console.error("Error during shutdown:", err);
    process.exit(1);
  }
}

import { createServer, IncomingMessage, ServerResponse } from "node:http";

function handleHealthCheck(req: IncomingMessage, res: ServerResponse): void {
  if (req.url === "/healthz") {
    // Keep liveness distinct from readiness while the listener is available.
    // Drain-rejection middleware must not turn this endpoint into a 503.
    res.writeHead(200).end("ok");
    return;
  }

  if (req.url === "/readyz") {
    // Readiness: 503 during shutdown so the load balancer stops routing.
    if (isShuttingDown) {
      res.writeHead(503).end("shutting down");
    } else {
      res.writeHead(200).end("ready");
    }
    return;
  }
}

let activeConnections = 0;
let drainResolve: (() => void) | null = null;

function trackRequest(res: ServerResponse): void {
  activeConnections++;
  let counted = true;
  function release(): void {
    if (!counted) return;
    counted = false;
    activeConnections--;
    if (isShuttingDown && activeConnections === 0 && drainResolve) {
      drainResolve();
    }
  }
  res.on("finish", release);
  res.on("close", release);
}

function waitForActiveConnections(): Promise<void> {
  if (activeConnections === 0) return Promise.resolve();
  return new Promise((resolve) => {
    drainResolve = resolve;
  });
}

import express from "express";
import { createServer } from "node:http";

const app = express();
const server = createServer(app);
let isShuttingDown = false;
let activeRequests = 0;
let drainResolve: (() => void) | null = null;

// Health endpoints — registered BEFORE the drain-rejection middleware so it
// cannot turn liveness into a 503 while the listener is still available.
app.get("/healthz", (_, res) => res.send("ok"));
app.get("/readyz", (_, res) => {
  res.status(isShuttingDown ? 503 : 200).send(isShuttingDown ? "draining" : "ready");
});

// Track in-flight requests and reject new application work during drain.
app.use((req, res, next) => {
  if (isShuttingDown) {
    res.setHeader("Connection", "close");
    res.status(503).json({ error: "Server is shutting down" });
    return;
  }

  activeRequests++;
  let counted = true;
  function release(): void {
    if (!counted) return;
    counted = false;
    activeRequests--;
    if (isShuttingDown && activeRequests === 0 && drainResolve) {
      drainResolve();
    }
  }
  // Listen for both finish (normal) and close (client abort) so the
  // counter always decrements. The once guard prevents double-decrement
  // when both events fire.
  res.on("finish", release);
  res.on("close", release);
  next();
});

// Application routes
app.get("/api/data", async (req, res) => {
  const data = await fetchData();
  res.json(data);
});

// Graceful shutdown
function shutdown(signal: string): void {
  if (isShuttingDown) return;
  isShuttingDown = true;
  console.log(`${signal} received — draining ${activeRequests} active requests`);

  server.close();

  const forceExit = setTimeout(() => {
    console.error("Forced exit — drain timeout exceeded");
    process.exit(1);
  }, 25_000);
  forceExit.unref();

  if (activeRequests === 0) {
    console.log("No active requests — exiting cleanly");
    process.exit(0);
  }

  drainResolve = () => {
    console.log("All requests drained — exiting cleanly");
    process.exit(0);
  };
}

process.on("SIGTERM", () => shutdown("SIGTERM"));
process.on("SIGINT", () => shutdown("SIGINT"));

server.listen(3000, () => console.log("Server ready on :3000"));

from contextlib import asynccontextmanager
from fastapi import FastAPI


@asynccontextmanager
async def lifespan(app: FastAPI):
    app.state.db_pool = await open_database_pool()
    try:
        yield
    finally:
        # Uvicorn has already completed or timed out its request drain.
        await app.state.db_pool.close()


app = FastAPI(lifespan=lifespan)


@app.get("/healthz")
async def healthz():
    return {"status": "ok"}


@app.get("/readyz")
async def readyz():
    return {"status": "ready"}

uvicorn app:app --timeout-graceful-shutdown 25

import { parentPort } from "node:worker_threads";

let isShuttingDown = false;
let currentJob: { id: string; checkpoint: () => Promise<void> } | null = null;

process.on("SIGTERM", async () => {
  isShuttingDown = true;
  console.log("Worker shutting down — finishing current job");

  if (currentJob) {
    await currentJob.checkpoint();
    console.log(`Job ${currentJob.id} checkpointed`);
  }

  process.exit(0);
});

async function processJobs(queue: JobQueue): Promise<void> {
  while (!isShuttingDown) {
    const job = await queue.poll({ timeout: 5000 });
    if (!job) continue;

    currentJob = job;
    await job.execute();
    await queue.ack(job.id);
    currentJob = null;
  }
}