feat(backend): implement graceful shutdown, secrets management, API v…

backend/.env.example

@@ -1,26 +1,53 @@
-PORT=3001
-NODE_ENV=development
+# ==============================================================================
+# Nestera Backend — Environment Variables
+# ==============================================================================
+# Copy this file to .env and fill in the values.
+# NEVER commit the .env file — it is listed in .gitignore.
+#
+# Secret Rotation Procedure:
+#   1. Generate the new secret value.
+#   2. Set the new value in your secret store / hosting env vars.
+#   3. Restart the application (graceful shutdown will drain in-flight requests).
+#   4. For JWT_SECRET: existing tokens signed with the old secret will fail
+#      validation — coordinate a rotation window or use a dual-key strategy.
+# ==============================================================================
+
+# ── Core ──────────────────────────────────────────────────────────────────────
+NODE_ENV=development          # development | production | test
+PORT=3001                     # HTTP listen port
 
-# Stellar Network
-STELLAR_NETWORK=testnet
+# ── Database ──────────────────────────────────────────────────────────────────
+# Provide EITHER DATABASE_URL (takes precedence) OR the DB_* variables.
+DATABASE_URL=                 # e.g. postgres://user:pass@host:5432/nestera
+DB_HOST=localhost
+DB_PORT=5432
+DB_NAME=nestera
+DB_USER=postgres
+DB_PASS=                      # REQUIRED in production — load from secret store
+
+# ── Authentication (REQUIRED) ────────────────────────────────────────────────
+JWT_SECRET=                   # Min 10 chars. NEVER commit this value.
+JWT_EXPIRATION=1h             # Token lifetime (e.g. 1h, 7d)
+
+# ── Stellar / Soroban (REQUIRED) ─────────────────────────────────────────────
+STELLAR_NETWORK=testnet       # testnet | mainnet
 SOROBAN_RPC_URL=https://soroban-testnet.stellar.org
 HORIZON_URL=https://horizon-testnet.stellar.org
-STELLAR_WEBHOOK_SECRET=your_webhook_secret_here
+SOROBAN_RPC_FALLBACK_URLS=https://soroban-testnet.stellar.org
+HORIZON_FALLBACK_URLS=https://horizon-testnet.stellar.org
+CONTRACT_ID=                  # Soroban contract ID
+STELLAR_WEBHOOK_SECRET=       # Min 16 chars. HMAC key for webhook verification.
 STELLAR_EVENT_POLL_INTERVAL=10000
 
-# Soroban Testnet – RPC Fallback URLs (comma-separated, in priority order)
-# Uncomment and populate with backup nodes to enable automatic failover.
-SOROBAN_RPC_FALLBACK_URLS=https://rpc-backup1.stellar.org,https://rpc-backup2.stellar.org
-HORIZON_FALLBACK_URLS=https://horizon-backup1.stellar.org,https://horizon-backup2.stellar.org
-
-# RPC Retry Configuration
+# ── RPC Retry ────────────────────────────────────────────────────────────────
 RPC_MAX_RETRIES=3
 RPC_RETRY_DELAY=1000
 RPC_TIMEOUT=10000
 
-# Contract
-CONTRACT_ID=YOUR_DEPLOYED_CONTRACT_ID
+# ── Redis (optional) ─────────────────────────────────────────────────────────
+REDIS_URL=                    # e.g. redis://localhost:6379
 
+# ── Mail / SMTP (optional) ───────────────────────────────────────────────────
 # ── Database ──────────────────────────────────────────────────────────────────
 # Option A – URL-based (takes precedence; typical for cloud/container deployments)
 DATABASE_URL=postgresql://user:password@localhost:5432/nestera
@@ -70,19 +97,41 @@ REDIS_URL=redis://localhost:6379
 # Mail (SMTP)
 MAIL_HOST=smtp.example.com
 MAIL_PORT=587
-MAIL_USER=your_email@example.com
-MAIL_PASS=your_email_password
+MAIL_USER=
+MAIL_PASS=                    # SMTP password — load from secret store
 MAIL_FROM="Nestera" <noreply@nestera.io>
-# Hospital Integration
-HOSPITAL_1_ENDPOINT=https://api.hospital1.com
-HOSPITAL_2_ENDPOINT=https://api.hospital2.com
-HOSPITAL_3_ENDPOINT=https://api.hospital3.com
+
+# ── KYC Provider (optional) ──────────────────────────────────────────────────
+KYC_PROVIDER_BASE_URL=
+KYC_PROVIDER_API_KEY=         # API key — load from secret store
+KYC_PII_ENCRYPTION_KEY=       # Min 16 chars. AES key for PII at rest.
+
+# ── Database Backup / S3 (optional) ──────────────────────────────────────────
+BACKUP_S3_BUCKET=
+BACKUP_S3_REGION=us-east-1
+BACKUP_AWS_ACCESS_KEY_ID=     # AWS credentials — load from secret store
+BACKUP_AWS_SECRET_ACCESS_KEY= # AWS credentials — load from secret store
+# 64 hex characters = 32-byte AES-256 key. Generate with: openssl rand -hex 32
+BACKUP_ENCRYPTION_KEY=
+BACKUP_RETENTION_DAYS=30
+BACKUP_TMP_DIR=/tmp
+
+# ── Hospital Integration (optional) ──────────────────────────────────────────
+HOSPITAL_1_ENDPOINT=
+HOSPITAL_2_ENDPOINT=
+HOSPITAL_3_ENDPOINT=
 HOSPITAL_MAX_RETRIES=3
 HOSPITAL_RETRY_DELAY=1000
 HOSPITAL_REQUEST_TIMEOUT=10000
 HOSPITAL_CIRCUIT_BREAKER_THRESHOLD=5
 HOSPITAL_CIRCUIT_BREAKER_TIMEOUT=60000
 
+# ── Balance Sync (optional) ──────────────────────────────────────────────────
+BALANCE_CACHE_TTL_SECONDS=300
+BALANCE_POLL_INTERVAL_MS=5000
+BALANCE_RECONNECT_INIT_MS=1000
+BALANCE_RECONNECT_MAX_MS=60000
+BALANCE_METRICS_PERSIST_MS=60000
 # ── Database Backup ───────────────────────────────────────────────────────────
 BACKUP_S3_BUCKET=nestera-db-backups
 BACKUP_S3_REGION=us-east-1

backend/scripts/test-graceful-shutdown.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+#
+# Graceful Shutdown Test Script
+#
+# Starts the application, fires several concurrent requests, sends SIGTERM
+# mid-flight, and verifies that in-flight requests complete while new ones
+# are rejected with 503.
+#
+set -euo pipefail
+
+PORT="${PORT:-3001}"
+BASE_URL="http://localhost:${PORT}/api"
+APP_PID=""
+PASS=0
+FAIL=0
+
+cleanup() {
+  if [ -n "$APP_PID" ] && kill -0 "$APP_PID" 2>/dev/null; then
+    kill -9 "$APP_PID" 2>/dev/null || true
+    wait "$APP_PID" 2>/dev/null || true
+  fi
+}
+trap cleanup EXIT
+
+log()  { echo "[TEST] $*"; }
+pass() { PASS=$((PASS + 1)); log "PASS: $*"; }
+fail() { FAIL=$((FAIL + 1)); log "FAIL: $*"; }
+
+# ---------- 1. Build & start app ----------
+log "Building application..."
+cd "$(dirname "$0")/.."
+npm run build 2>&1 | tail -3
+
+log "Starting application on port ${PORT}..."
+NODE_ENV=development \
+  DB_HOST=localhost DB_PORT=5432 DB_NAME=nestera DB_USER=postgres DB_PASS=postgres \
+  JWT_SECRET=test-jwt-secret-for-shutdown-test \
+  JWT_EXPIRATION=1h \
+  SOROBAN_RPC_URL=https://soroban-testnet.stellar.org \
+  HORIZON_URL=https://horizon-testnet.stellar.org \
+  SOROBAN_RPC_FALLBACK_URLS=https://soroban-testnet.stellar.org \
+  HORIZON_FALLBACK_URLS=https://horizon-testnet.stellar.org \
+  CONTRACT_ID=CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC \
+  STELLAR_WEBHOOK_SECRET=test-webhook-secret-16chars \
+  PORT="${PORT}" \
+  node dist/main.js &
+APP_PID=$!
+
+log "Waiting for app to be ready (PID ${APP_PID})..."
+for i in $(seq 1 30); do
+  if curl -sf "${BASE_URL}/health/live" > /dev/null 2>&1; then
+    break
+  fi
+  if ! kill -0 "$APP_PID" 2>/dev/null; then
+    log "Application exited before becoming ready"
+    exit 1
+  fi
+  sleep 1
+done
+
+if ! curl -sf "${BASE_URL}/health/live" > /dev/null 2>&1; then
+  fail "Application did not become ready within 30 seconds"
+  exit 1
+fi
+pass "Application is ready"
+
+# ---------- 2. Verify normal operation ----------
+HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "${BASE_URL}/health/live")
+if [ "$HTTP_CODE" = "200" ]; then
+  pass "Health endpoint returns 200"
+else
+  fail "Health endpoint returned ${HTTP_CODE}, expected 200"
+fi
+
+# ---------- 3. Send SIGTERM and verify shutdown behavior ----------
+log "Sending SIGTERM to PID ${APP_PID}..."
+kill -TERM "$APP_PID"
+
+sleep 1
+
+# After SIGTERM, new requests should fail (connection refused or 503)
+HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "${BASE_URL}/health/live" 2>/dev/null || echo "000")
+if [ "$HTTP_CODE" = "503" ] || [ "$HTTP_CODE" = "000" ]; then
+  pass "New requests rejected after SIGTERM (HTTP ${HTTP_CODE})"
+else
+  fail "Expected 503 or connection refused after SIGTERM, got ${HTTP_CODE}"
+fi
+
+# ---------- 4. Wait for process to exit ----------
+log "Waiting for process to exit..."
+WAIT_COUNT=0
+while kill -0 "$APP_PID" 2>/dev/null; do
+  WAIT_COUNT=$((WAIT_COUNT + 1))
+  if [ "$WAIT_COUNT" -gt 35 ]; then
+    fail "Process did not exit within 35 seconds"
+    break
+  fi
+  sleep 1
+done
+
+if ! kill -0 "$APP_PID" 2>/dev/null; then
+  wait "$APP_PID" 2>/dev/null
+  EXIT_CODE=$?
+  if [ "$EXIT_CODE" -eq 0 ]; then
+    pass "Process exited cleanly with code 0"
+  else
+    fail "Process exited with code ${EXIT_CODE}"
+  fi
+  APP_PID=""
+fi
+
+# ---------- Summary ----------
+echo ""
+log "============================="
+log "Results: ${PASS} passed, ${FAIL} failed"
+log "============================="
+[ "$FAIL" -eq 0 ]

backend/src/common/common.module.ts

@@ -1,6 +1,14 @@
 import { Global, Module } from '@nestjs/common';
 import { PiiEncryptionService } from './services/pii-encryption.service';
 import { RateLimitMonitorService } from './services/rate-limit-monitor.service';
+import { SecretsConfigService } from './services/secrets-config.service';
+
+@Global()
+@Module({
+  providers: [
+    RateLimitMonitorService,
+    PiiEncryptionService,
+    SecretsConfigService,
 import { IdempotencyService } from './services/idempotency.service';
 import { IdempotencyCleanupService } from './services/idempotency-cleanup.service';
 import { LogSanitizerService } from './services/log-sanitizer.service';
@@ -23,6 +31,7 @@ import { CacheModule } from '../modules/cache/cache.module';
   exports: [
     RateLimitMonitorService,
     PiiEncryptionService,
+    SecretsConfigService,
     IdempotencyService,
     LogSanitizerService,
     CompressionMetricsService,

backend/src/common/interceptors/graceful-shutdown.interceptor.ts

@@ -3,7 +3,10 @@ import {
   NestInterceptor,
   ExecutionContext,
   CallHandler,
+  HttpException,
+  HttpStatus,
 } from '@nestjs/common';
+import { Observable, throwError } from 'rxjs';
 import { Observable, EMPTY } from 'rxjs';
 import { finalize } from 'rxjs/operators';
 import { GracefulShutdownService } from '../services/graceful-shutdown.service';
@@ -13,8 +16,17 @@ export class GracefulShutdownInterceptor implements NestInterceptor {
   constructor(private gracefulShutdown: GracefulShutdownService) {}
 
   intercept(context: ExecutionContext, next: CallHandler): Observable<any> {
-    // Reject new requests during shutdown
     if (this.gracefulShutdown.isShutdown()) {
+      return throwError(
+        () =>
+          new HttpException(
+            {
+              statusCode: HttpStatus.SERVICE_UNAVAILABLE,
+              message: 'Service is shutting down',
+            },
+            HttpStatus.SERVICE_UNAVAILABLE,
+          ),
+      );
       const response = context.switchToHttp().getResponse();
       response.setHeader?.('Connection', 'close');
       response.status(503).json({

backend/src/common/interceptors/request-logging.interceptor.ts

@@ -9,6 +9,31 @@ import {
 import { Observable, throwError } from 'rxjs';
 import { tap, catchError } from 'rxjs/operators';
 import { Request, Response } from 'express';
+import { SecretsConfigService } from '../services/secrets-config.service';
+
+const REDACTED_HEADERS = new Set([
+  'authorization',
+  'cookie',
+  'x-api-key',
+  'x-auth-token',
+]);
+
+function sanitizeHeaders(headers: Record<string, any>): Record<string, string> {
+  const safe: Record<string, string> = {};
+  for (const [key, value] of Object.entries(headers)) {
+    if (REDACTED_HEADERS.has(key.toLowerCase())) {
+      safe[key] = '[REDACTED]';
+    } else if (
+      typeof value === 'string' &&
+      SecretsConfigService.isSensitiveKey(key)
+    ) {
+      safe[key] = '[REDACTED]';
+    } else {
+      safe[key] = String(value);
+    }
+  }
+  return safe;
+}
 import { Logger } from 'nestjs-pino';
 import { LogSanitizerService } from '../services/log-sanitizer.service';
 import { ApmService } from '../../modules/apm/apm.service';
@@ -42,6 +67,7 @@ export class RequestLoggingInterceptor implements NestInterceptor {
     const response = context.switchToHttp().getResponse<Response>();
 
     const correlationId =
+      (request.headers['x-correlation-id'] as string) || uuidv4();
       (request as Request & { correlationId?: string }).correlationId ||
       (request.headers['x-correlation-id'] as string) ||
       'unknown';
@@ -50,6 +76,22 @@ export class RequestLoggingInterceptor implements NestInterceptor {
     const { method, ip } = request;
     const url = this.sanitizer?.sanitizeUrl(request.url) ?? request.url;
 
+    (request as any).correlationId = correlationId;
+    response.setHeader('x-correlation-id', correlationId);
+
+    const { method, url, ip } = request;
+    const userAgent = request.headers['user-agent'];
+
+    this.logger.log(
+      JSON.stringify({
+        type: 'REQUEST',
+        correlationId,
+        method,
+        url,
+        ip,
+        userAgent,
+        timestamp: new Date().toISOString(),
+      }),
     // Skip noisy paths
     const rawPath = request.path ?? request.url;
     if (SKIP_LOG_PATHS.has(rawPath)) {