File Processor

SCENARIO                          STRATEGY
------------------------------------------------------
Small files (<10MB)               Multipart form upload
Large files (>10MB)               Presigned URL (direct-to-storage)
Very large files (>1GB)           Resumable upload (tus protocol)
Multiple small files              Batch multipart
Sensitive files                   Server-side upload (no direct)
User avatars/profile images       Multipart + server processing
Video uploads                     Resumable + async processing
API file attachments              Base64 in JSON (small) or multipart

PRESIGNED URL FLOW (recommended for large files):
  1. Client requests upload URL from API
  2. API generates presigned URL (S3, GCS, Azure)
  3. Client uploads directly to storage
  4. Storage notifies API (webhook or client confirms)
  5. API processes file (resize, scan, etc.)

import multer from 'multer';
import path from 'path';

// Configuration
const upload = multer({
  storage: multer.memoryStorage(),
  limits: {
    fileSize: 10 * 1024 * 1024,    // 10MB max
    files: 5,                       // Max 5 files per request
    fields: 10,                     // Max 10 non-file fields
  },
  fileFilter: (req, file, cb) => {
    const allowedTypes = [
      'image/jpeg', 'image/png', 'image/webp', 'image/gif',
      # ... (condensed) ...
    } catch (error) {
      res.status(422).json({ error: error.message });
    }
  }
);

import { S3Client, PutObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';

const s3 = new S3Client({ region: ENV_CONFIG_VALUE });

// Generate presigned upload URL
app.post('/api/upload/presign', authenticate, async (req, res) => {
  const { filename, contentType, size } = req.body;

  // Validate before generating URL
  if (size > 100 * 1024 * 1024) {
    return res.status(400).json({ error: 'File too large (max 100MB)' });
  }

  # ... (condensed) ...

  await db.uploads.update({ where: { id: upload.id }, data: { status: 'processing' } });

  res.json({ status: 'processing' });
});

import { Server as TusServer } from '@tus/server';
import { S3Store } from '@tus/s3-store';

const tusServer = new TusServer({
  path: '/api/upload/resumable',
  datastore: new S3Store({
    s3ClientConfig: {
      region: ENV_CONFIG_VALUE,
    },
    bucket: ENV_CONFIG_VALUE!,
    partSize: 8 * 1024 * 1024,
  }),
  maxSize: 5 * 1024 * 1024 * 1024, // 5GB max
  onUploadCreate: async (req, res, upload) => {
    # ... (condensed) ...
    return res;
  },
});

app.all('/api/upload/resumable/*', (req, res) => tusServer.handle(req, res));

import { fileTypeFromBuffer } from 'file-type';

async function validateFileContent(file: { buffer: Buffer; mimetype: string }): Promise<void> {
  // Check magic bytes (actual file content)
  const detected = await fileTypeFromBuffer(file.buffer);

  if (!detected) {
    throw new Error('Unable to determine file type');
  }

  const allowedMimeTypes: Record<string, string[]> = {
    'image/jpeg': ['jpg', 'jpeg'],
    'image/png':  ['png'],
    'image/webp': ['webp'],
    # ... (condensed) ...
  const pixels = (metadata.width ?? 0) * (metadata.height ?? 0);
  if (pixels > 100_000_000) {
    throw new Error('Image too large (decompression bomb protection)');
  }
}

import NodeClam from 'clamscan';

const clam = await new NodeClam().init({
  clamdscan: {
    socket: '/var/run/clamav/clamd.ctl',
    timeout: 60000,
    multiscan: true,
  },
  preference: 'clamdscan',
});

async function scanBuffer(buffer: Buffer): Promise<{ clean: boolean; virus?: string }> {
  try {
    const { Readable } = await import('stream');
    # ... (condensed) ...
  }

  // 3. Process and store
  return await storeFile(file);
}

import sharp from 'sharp';

// Generate multiple image sizes
async function processImage(
  input: Buffer,
  options: { maxWidth?: number; maxHeight?: number; quality?: number; format?: 'webp' | 'avif' }
): Promise<ProcessedImage> {
  const { quality = 80, format = 'webp' } = options;

  const sizes = [
    { name: 'thumbnail', width: 200, height: 200, fit: 'cover' as const },
    { name: 'small', width: 400, height: 400, fit: 'inside' as const },
    { name: 'medium', width: 800, height: 800, fit: 'inside' as const },
    { name: 'large', width: 1600, height: 1600, fit: 'inside' as const },
  # ... (condensed) ...
  return sharp(input)
    .rotate()  // Auto-rotate based on EXIF before stripping
    .withMetadata({ orientation: undefined })
    .toBuffer();
}

import PDFDocument from 'pdfkit';

async function generateInvoicePDF(invoice: Invoice): Promise<Buffer> {
  return new Promise((resolve, reject) => {
    const doc = new PDFDocument({ margin: 50 });
    const chunks: Buffer[] = [];

    doc.on('data', (chunk: Buffer) => chunks.push(chunk));
    doc.on('end', () => resolve(Buffer.concat(chunks)));
    doc.on('error', reject);

    // Header
    doc.fontSize(24).text('INVOICE', { align: 'right' });
    doc.fontSize(10).text(`Invoice #${invoice.number}`, { align: 'right' });
    # ... (condensed) ...
    doc.text(formatCurrency(invoice.total), 460, y + 15);

    doc.end();
  });
}

import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';

class FileStorage {
  private s3: S3Client;
  private bucket: string;

  constructor() {
    this.s3 = new S3Client({ region: ENV_CONFIG_VALUE });
    this.bucket = ENV_CONFIG_VALUE!;
  }

  async upload(key: string, buffer: Buffer, contentType: string): Promise<string> {
    await this.s3.send(new PutObjectCommand({
      # ... (condensed) ...
      Bucket: this.bucket,
      Key: key,
    }));
  }
}

KEY STRUCTURE:
  {type}/{user_id}/{year}/{month}/{file_id}{extension}
  Example: avatars/usr_abc123/2025/01/f7a9b3c4.webp

  uploads/{user_id}/{file_id}/{variant}.{format}
  Example: uploads/usr_abc123/f7a9b3c4/thumbnail.webp
           uploads/usr_abc123/f7a9b3c4/medium.webp
           uploads/usr_abc123/f7a9b3c4/original.webp

RULES:
  - NEVER use original filename in storage key (security)
  - Generate unique IDs (UUID, ULID)
  - Include user ID for access control and cleanup
  - Date-based prefix helps with S3 partitioning
  - Keep extensions for CDN content-type detection

class CDNService {
  private readonly cdnDomain = ENV_CONFIG_VALUE;

  getPublicUrl(key: string): string {
    return `[reference URL]
  }

  // Image transformation via CDN (Cloudflare, Imgix, etc.)
  getImageUrl(key: string, transforms: ImageTransform): string {
    const params = new URLSearchParams();
    if (transforms.width) params.set('w', String(transforms.width));
    if (transforms.height) params.set('h', String(transforms.height));
    if (transforms.quality) params.set('q', String(transforms.quality));
    if (transforms.format) params.set('f', transforms.format);
    params.set('fit', transforms.fit ?? 'cover');

    return `[reference URL])}/${key}`;
  }
}

import { Queue, Worker } from 'bullmq';

const processingQueue = new Queue('file-processing');

const worker = new Worker('file-processing', async (job) => {
  const { uploadId, key, contentType } = job.data;

  // 1. Download from storage
  const fileBuffer = await storage.download(key);

  // 2. Virus scan
  await job.updateProgress(10);
  const scanResult = await scanBuffer(fileBuffer);
  if (!scanResult.clean) {
    # ... (condensed) ...
    await alerting.notify(`File processing permanently failed: ${job.data.uploadId}`, {
      error: error.message,
    });
  }
});

// Clean up orphaned uploads (pending > 24 hours)
async function cleanupOrphanedUploads() {
  const cutoff = new Date(Date.now() - 24 * 60 * 60 * 1000);

  const orphans = await db.uploads.findMany({
    where: { status: 'pending', createdAt: { lt: cutoff } },
  });

  for (const upload of orphans) {
    try {
      await storage.remove(upload.key);
    } catch {
      // File may not exist, that is fine
    }
    # ... (condensed) ...

// S3 lifecycle policy (set via AWS console or Terraform)
// - Move to Glacier after 90 days
// - Delete incomplete multipart uploads after 7 days
// - Delete expired presigned upload objects after 1 day