Achieving 99.95% Uptime: Building Self-Healing Infrastructure for 200+ Microservices

SLA (Service Level Agreement)
  ↓
  Commits to customer (legal, contractual)
  Typically: 99.5%–99.99% depending on service tier
  
SLO (Service Level Objective)
  ↓
  Internal target (what we aim for)
  Typically: 99.95%–99.99% (higher than SLA for buffer)
  
SLI (Service Level Indicator)
  ↓
  Measurement (how we know we're meeting SLO)
  Measured continuously via monitoring

Total minutes/year = 365 days × 24 hours × 60 min = 525,600 minutes

Acceptable downtime = (1 - 99.95%) × 525,600 = 0.0005 × 525,600 = 262.8 minutes

In hours: 262.8 / 60 = 4.38 hours (~4 hours 23 minutes)

Per month: 4.38 / 12 = 0.365 hours (~22 minutes)

Per week: 4.38 / 52 = 0.084 hours (~5 minutes)

User Requests
    ↓
API Gateway
    ├→ Authentication Service
    │  └→ Identity Provider
    ├→ Product Service
    │  ├→ Inventory Service
    │  │  └→ PostgreSQL (primary DB)
    │  │     └→ Redis Cache
    │  └→ Pricing Service
    │     └→ Configuration Service
    ├→ Order Service
    │  ├→ Product Service (see above)
    │  ├→ Payment Service
    │  │  └→ Payment Provider API
    │  ├→ Notification Service
    │  │  └→ Message Queue (Kafka)
    │  └→ Order Database
    │     └→ PostgreSQL (primary DB)
    └→ Recommendation Service
       └→ ML Model Service
          └→ Model Store (S3)

# Query Jaeger or Kiali (Istio dashboard) to get dependency graph
# This shows all services that call each other in production

# Example: Get all services that call "order-service"
upstream_services = jaeger_client.query_services(called_by="order-service")
# Returns: [api-gateway, recommendation-service, notification-service]

# Get all services that order-service calls
downstream_services = jaeger_client.query_services(calls="order-service")
# Returns: [payment-service, product-service, inventory-service, order-db]

Region (us-east-1)
├─ AZ 1 (us-east-1a)
│  ├─ Kubernetes Node 1 (m5.2xlarge)
│  ├─ Kubernetes Node 2 (m5.2xlarge)
│  ├─ RDS Primary Database
│  ├─ ElastiCache Redis Primary
│  └─ NAT Gateway
├─ AZ 2 (us-east-1b)
│  ├─ Kubernetes Node 3 (m5.2xlarge)
│  ├─ Kubernetes Node 4 (m5.2xlarge)
│  ├─ RDS Standby Database (synchronous replication)
│  ├─ ElastiCache Redis Replica
│  └─ NAT Gateway
└─ AZ 3 (us-east-1c)
   ├─ Kubernetes Node 5 (m5.2xlarge)
   ├─ Kubernetes Node 6 (m5.2xlarge)
   └─ (Services spread across AZs)

Primary DB (us-east-1a)  [writes + reads]
    ↓ (async replication)
Standby DB (us-east-1b)  [reads only]
    ↓ (promotion on failure)
Promoted Primary (us-east-1b)  [new writes]

Primary DB 1 (us-east-1a)  [writes + reads]
    ↕ (bidirectional replication)
Primary DB 2 (us-east-1b)  [writes + reads]

PostgreSQL Primary (us-east-1a)
    ↓
PostgreSQL Standby 1 (us-east-1b)  
    ↓
PostgreSQL Standby 2 (us-east-1c)

Patroni (distributed consensus)
├─ Monitors all replicas
├─ Detects primary failure
└─ Promotes best standby automatically

// Good health check implementation
func healthHandler(w http.ResponseWriter, r *http.Request) {
    // Check critical dependencies
    checks := map[string]error{
        "database": healthCheckDB(),
        "cache":    healthCheckRedis(),
        "queue":    healthCheckKafka(),
    }
    
    allHealthy := true
    statusCode := http.StatusOK
    
    for name, err := range checks {
        if err != nil {
            allHealthy = false
            statusCode = http.StatusServiceUnavailable
            log.Printf("Health check failed for %s: %v", name, err)
        }
    }
    
    w.WriteHeader(statusCode)
    json.NewEncoder(w).Encode(map[string]interface{}{
        "healthy": allHealthy,
        "checks":  checks,
    })
}

# Terraform for ALB health checks
resource "aws_lb_target_group" "api_servers" {
  name     = "api-servers-tg"
  port     = 8080
  protocol = "HTTP"
  vpc_id   = aws_vpc.main.id

  health_check {
    healthy_threshold   = 2
    unhealthy_threshold = 3
    timeout             = 5  # seconds
    interval            = 10 # seconds, how often to check
    path                = "/health"
    matcher             = "200"  # expect 200 OK
  }

  deregistration_delay = 30  # graceful shutdown period
}

apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: order-service-circuit-breaker
  namespace: production
spec:
  host: order-service.production.svc.cluster.local
  trafficPolicy:
    connectionPool:
      tcp:
        maxConnections: 100  # max concurrent connections
      http:
        http1MaxPendingRequests: 50
        http2MaxRequests: 100
        maxRequestsPerConnection: 2
    outlierDetection:
      consecutive5xxErrors: 5  # open circuit after 5 consecutive 5xx
      interval: 30s  # check interval
      baseEjectionTime: 30s  # how long to keep service out
      maxEjectionPercent: 50  # max % of hosts that can be ejected
      minRequestVolume: 5  # min requests before ejection

# Client-side fallback
def get_order_details(order_id):
    try:
        return fetch_from_order_service(order_id, timeout=5)
    except CircuitBreakerOpen:
        # Service is failing, return cached data or degraded response
        return get_cached_order_details(order_id) or \
               {"id": order_id, "cached": True, "details_incomplete": True}

// Without bulkheads: If order service has slow database queries,
// its thread pool fills up and blocks unrelated requests

// With bulkheads: Separate pools for different request types
ThreadPool criticalPool = new ThreadPool(50);      // Orders, payments
ThreadPool backgroundPool = new ThreadPool(20);    // Analytics, recommendations
ThreadPool internalPool = new ThreadPool(10);      // Internal admin requests

// Route requests to appropriate pool
if (request.isCritical()) {
    return criticalPool.execute(request);
} else if (request.isBackground()) {
    return backgroundPool.execute(request);
} else {
    return internalPool.execute(request);
}

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: order-service-pdb
  namespace: production
spec:
  minAvailable: 2  # at least 2 order-service pods must be running
  selector:
    matchLabels:
      app: order-service
  unhealthyPodEvictionPolicy: AlwaysAllow
  
  # Alternative: specify maxUnavailable
  # maxUnavailable: 1  # at most 1 replica can be disrupted at a time

apiVersion: v1
kind: Pod
metadata:
  name: order-service-pod
spec:
  containers:
  - name: order-service
    image: order-service:v1.2.3
    
    # Liveness: restart if unhealthy
    livenessProbe:
      httpGet:
        path: /health/live
        port: 8080
      initialDelaySeconds: 30  # wait 30s before first check
      periodSeconds: 10        # check every 10 seconds
      timeoutSeconds: 5        # wait 5s for response
      failureThreshold: 3      # restart after 3 consecutive failures

    # Readiness: remove from load balancer if not ready
    readinessProbe:
      httpGet:
        path: /health/ready
        port: 8080
      initialDelaySeconds: 5   # fast readiness check
      periodSeconds: 5         # check frequently
      timeoutSeconds: 3
      failureThreshold: 2      # remove from load balancer after 2 failures

livenessProbe:
  httpGet:
    path: /health
    port: 8080
  initialDelaySeconds: 10
  periodSeconds: 1  # WAY too aggressive, creates probe traffic storm
  failureThreshold: 1  # WAY too aggressive, restarts on single false positive

apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service
spec:
  replicas: 10
  template:
    spec:
      containers:
      - name: order-service
        image: order-service:v1.2.3
        
        resources:
          requests:
            cpu: "500m"      # 0.5 CPU cores guaranteed
            memory: "512Mi"  # 512 MB guaranteed
          limits:
            cpu: "1000m"     # burst up to 1 CPU
            memory: "1Gi"    # burst up to 1 GB

- Kubernetes thinks pod needs little resources
- Schedules too many pods on single node
- Node becomes CPU/memory constrained
- All pods on that node slow down (noisy neighbor problem)
- Service becomes unreliable due to starvation

- Kubernetes thinks pod needs lots of resources
- Schedules fewer pods per node
- Many nodes underutilized
- Higher infrastructure cost
- Less fault tolerance (fewer replicas fit in cluster)

apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service
spec:
  replicas: 10
  template:
    spec:
      affinity:
        podAntiAffinity:
          # Try hard to spread pods across nodes (preferred)
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - order-service
              topologyKey: kubernetes.io/hostname
          
          # MUST spread pods across AZs (required)
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - order-service
            topologyKey: topology.kubernetes.io/zone
      
      containers:
      - name: order-service
        image: order-service:v1.2.3
        # ... rest of config

apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service
  namespace: production
  labels:
    app: order-service
    team: platform
spec:
  replicas: 10
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 2        # max 2 extra pods during update
      maxUnavailable: 0  # zero pods unavailable during update
  
  selector:
    matchLabels:
      app: order-service
  
  template:
    metadata:
      labels:
        app: order-service
        version: v1.2.3
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8080"
        prometheus.io/path: "/metrics"
    
    spec:
      serviceAccountName: order-service
      securityContext:
        runAsNonRoot: true
        fsReadOnlyRootFilesystem: true
      
      # Pod anti-affinity for high availability
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - order-service
            topologyKey: topology.kubernetes.io/zone
      
      # Init container for migrations (before main container starts)
      initContainers:
      - name: db-migration
        image: order-service-migrate:v1.2.3
        env:
        - name: DATABASE_URL
          valueFrom:
            secretKeyRef:
              name: order-service-secrets
              key: database-url
      
      # Main application container
      containers:
      - name: order-service
        image: order-service:v1.2.3
        imagePullPolicy: IfNotPresent
        
        ports:
        - name: http
          containerPort: 8080
          protocol: TCP
        - name: metrics
          containerPort: 9090
          protocol: TCP
        
        # Environment configuration
        env:
        - name: LOG_LEVEL
          value: "info"
        - name: DATABASE_POOL_SIZE
          value: "20"
        - name: CACHE_TTL
          value: "3600"
        
        # Secrets
        envFrom:
        - secretRef:
            name: order-service-secrets
        - configMapRef:
            name: order-service-config
        
        # Resource requests (for scheduling)
        resources:
          requests:
            cpu: "500m"
            memory: "512Mi"
          limits:
            cpu: "1000m"
            memory: "1Gi"
        
        # Startup probe (for slow-starting apps)
        startupProbe:
          httpGet:
            path: /health/startup
            port: http
          initialDelaySeconds: 0
          periodSeconds: 5
          timeoutSeconds: 3
          failureThreshold: 30  # fail after 150 seconds
        
        # Liveness probe (restart if unhealthy)
        livenessProbe:
          httpGet:
            path: /health/live
            port: http
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
          failureThreshold: 3
        
        # Readiness probe (remove from load balancer if not ready)
        readinessProbe:
          httpGet:
            path: /health/ready
            port: http
          initialDelaySeconds: 10
          periodSeconds: 5
          timeoutSeconds: 3
          failureThreshold: 2
        
        # Graceful shutdown
        lifecycle:
          preStop:
            exec:
              command: ["/bin/sh", "-c", "sleep 15"]  # Allow in-flight requests to complete
        
        # Volume mounts
        volumeMounts:
        - name: config
          mountPath: /etc/order-service/config
          readOnly: true
        - name: tmp
          mountPath: /tmp
        
        # Security
        securityContext:
          runAsNonRoot: true
          runAsUser: 1000
          allowPrivilegeEscalation: false
          readOnlyRootFilesystem: true
      
      # Volumes
      volumes:
      - name: config
        configMap:
          name: order-service-config
      - name: tmp
        emptyDir: {}
      
      # Graceful termination period
      terminationGracePeriodSeconds: 30
      
      # DNS policy
      dnsPolicy: ClusterFirst

---
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: order-service-pdb
  namespace: production
spec:
  minAvailable: 2
  selector:
    matchLabels:
      app: order-service

---
apiVersion: v1
kind: Service
metadata:
  name: order-service
  namespace: production
  labels:
    app: order-service
spec:
  type: ClusterIP
  selector:
    app: order-service
  ports:
  - name: http
    port: 80
    targetPort: http
    protocol: TCP
  - name: metrics
    port: 9090
    targetPort: metrics
    protocol: TCP
  sessionAffinity: None
  sessionAffinityConfig:
    clientIP:
      timeoutSeconds: 10800

PostgreSQL Primary (postgres-0)
├─ Synchronous replica 1 (postgres-1)
├─ Synchronous replica 2 (postgres-2)
└─ Patroni cluster manager
   ├─ Distributed consensus (etcd)
   ├─ Automatic failover
   └─ Replica management

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: postgres
  namespace: databases
spec:
  serviceName: postgres
  replicas: 3
  
  selector:
    matchLabels:
      app: postgres
  
  template:
    metadata:
      labels:
        app: postgres
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - postgres
            topologyKey: topology.kubernetes.io/zone
      
      serviceAccountName: postgres
      
      containers:
      - name: postgres
        image: postgres:15
        ports:
        - name: postgresql
          containerPort: 5432
        
        env:
        - name: POSTGRES_DB
          value: production
        - name: PGDATA
          value: /var/lib/postgresql/data/pgdata
        
        resources:
          requests:
            cpu: "2"
            memory: "4Gi"
          limits:
            cpu: "4"
            memory: "8Gi"
        
        livenessProbe:
          exec:
            command:
            - /bin/sh
            - -c
            - pg_isready -U postgres
          initialDelaySeconds: 30
          periodSeconds: 10
          timeoutSeconds: 5
        
        readinessProbe:
          exec:
            command:
            - /bin/sh
            - -c
            - pg_isready -U postgres
          initialDelaySeconds: 5
          periodSeconds: 5
        
        volumeMounts:
        - name: postgresql-storage
          mountPath: /var/lib/postgresql/data
      
      - name: patroni
        image: patroni:latest
        env:
        - name: ETCD_HOSTS
          value: "etcd-0.etcd.databases:2379,etcd-1.etcd.databases:2379,etcd-2.etcd.databases:2379"
        - name: PATRONI_POSTGRESQL_PARAMETERS
          value: "synchronous_commit=on"  # Synchronous replication
        
        ports:
        - name: patroni
          containerPort: 8008
      
      volumes:
      - name: postgresql-storage
        emptyDir: {}
  
  volumeClaimTemplates:
  - metadata:
      name: postgresql-storage
    spec:
      accessModes:
      - ReadWriteOnce
      storageClassName: fast-ssd
      resources:
        requests:
          storage: 100Gi

Redis Master (redis-master)
├─ Replica 1 (redis-replica-1)
├─ Replica 2 (redis-replica-2)
└─ Sentinel cluster (3 sentinels)
   ├─ Monitors master health
   ├─ Detects failure
   └─ Promotes best replica automatically

apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
  name: redis-sentinel
  namespace: databases
spec:
  repo: https://charts.bitnami.com/bitnami
  chart: redis
  version: "17.x"
  values:
    architecture: "replication"
    replica:
      replicaCount: 2
    sentinel:
      enabled: true
      quorum: 2
      downAfterMilliseconds: 5000
      failoverTimeout: 10000
    persistence:
      enabled: true
      size: "50Gi"
    resources:
      requests:
        cpu: "500m"
        memory: "1Gi"
      limits:
        cpu: "1000m"
        memory: "2Gi"

from redis.sentinel import Sentinel

# Sentinel automatically routes to current master
sentinel = Sentinel([('redis-sentinel-0', 26379),
                     ('redis-sentinel-1', 26379),
                     ('redis-sentinel-2', 26379)])

# Automatically fails over if master goes down
redis_master = sentinel.master_for('mymaster', socket_timeout=0.1)

# Use like normal Redis
redis_master.set('key', 'value')
value = redis_master.get('key')

apiVersion: v1
kind: ConfigMap
metadata:
  name: kafka-config
  namespace: queues
data:
  server.properties: |
    broker.id=0
    listeners=PLAINTEXT://0.0.0.0:9092
    advertised.listeners=PLAINTEXT://kafka-0.kafka.queues.svc.cluster.local:9092
    log.dirs=/var/lib/kafka-logs
    num.network.threads=8
    num.io.threads=8
    socket.send.buffer.bytes=102400
    socket.receive.buffer.bytes=102400
    socket.request.max.bytes=104857600
    
    # Replication
    default.replication.factor=3
    min.insync.replicas=2  # Wait for 2 replicas before acking write
    
    # Retention
    log.retention.hours=168
    log.segment.bytes=1073741824
    
    # ZooKeeper
    zookeeper.connect=zookeeper-0.zookeeper.queues.svc.cluster.local:2181,zookeeper-1.zookeeper.queues.svc.cluster.local:2181,zookeeper-2.zookeeper.queues.svc.cluster.local:2181

---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: kafka
  namespace: queues
spec:
  serviceName: kafka
  replicas: 3
  
  selector:
    matchLabels:
      app: kafka
  
  template:
    metadata:
      labels:
        app: kafka
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - kafka
            topologyKey: topology.kubernetes.io/zone
      
      containers:
      - name: kafka
        image: confluentinc/cp-kafka:7.4.0
        
        ports:
        - name: plaintext
          containerPort: 9092
        
        env:
        - name: KAFKA_BROKER_ID
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: KAFKA_ADVERTISED_LISTENERS
          value: "PLAINTEXT://$(HOSTNAME).kafka.queues.svc.cluster.local:9092"
        
        resources:
          requests:
            cpu: "2"
            memory: "4Gi"
          limits:
            cpu: "4"
            memory: "8Gi"
        
        volumeMounts:
        - name: datadir
          mountPath: /var/lib/kafka-logs
  
  volumeClaimTemplates:
  - metadata:
      name: datadir
    spec:
      accessModes:
      - ReadWriteOnce
      storageClassName: fast-ssd
      resources:
        requests:
          storage: 100Gi

from kafka import KafkaProducer
import json

producer = KafkaProducer(
    bootstrap_servers=['kafka-0.kafka.queues:9092', 
                       'kafka-1.kafka.queues:9092',
                       'kafka-2.kafka.queues:9092'],
    acks='all',  # Wait for all in-sync replicas
    retries=3,
    compression_type='snappy'
)

# Publish with callback
def on_send_success(record_metadata):
    print(f"Sent to {record_metadata.topic} partition {record_metadata.partition}")

def on_send_error(exc):
    print(f"Error: {exc}")

future = producer.send('orders', {'order_id': 123, 'amount': 99.99})
future.add_callback(on_send_success)
future.add_errback(on_send_error)

producer.flush()

Scrape Targets (200+ services)
    ↓
Prometheus Server (time-series DB)
    ├─ Scrapes metrics every 15-30 seconds
    ├─ Stores in local TSDB
    ├─ Retains 15-30 days of data
    └─ Evaluates alerting rules
         ↓
         PagerDuty/Slack (firing alerts)

Remote Storage (optional)
    ↓ (long-term retention)
    S3/GCS (Thanos, Cortex, etc.)

# Leaf Prometheus instances (one per team/product)
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-leaf-config
data:
  prometheus.yml: |
    global:
      scrape_interval: 15s
      external_labels:
        cluster: us-east-1
        team: platform
    
    scrape_configs:
    # Scrape services owned by platform team
    - job_name: 'platform-services'
      kubernetes_sd_configs:
      - role: pod
        namespaces:
          names:
          - platform
      relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_scrape]
        action: keep
        regex: 'true'

---
# Central Prometheus (scrapes from leaf instances)
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-central-config
data:
  prometheus.yml: |
    global:
      scrape_interval: 30s
      external_labels:
        cluster: global
    
    scrape_configs:
    # Scrape from leaf Prometheus instances (federation)
    - job_name: 'leaf-prometheus'
      static_configs:
      - targets:
        - 'prometheus-platform.us-east-1:9090'
        - 'prometheus-products.us-east-1:9090'
        - 'prometheus-data.us-east-1:9090'
        - 'prometheus-infrastructure.us-east-1:9090'

from prometheus_client import Counter, Histogram, Gauge
import time

# RED metrics
request_count = Counter(
    'http_requests_total',
    'Total HTTP requests',
    ['method', 'endpoint', 'status']
)

request_latency = Histogram(
    'http_request_duration_seconds',
    'HTTP request latency',
    ['method', 'endpoint'],
    buckets=[0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0]
)

# Helper decorator
def track_metrics(func):
    def wrapper(*args, **kwargs):
        start = time.time()
        try:
            result = func(*args, **kwargs)
            status = 200
            return result
        except Exception as e:
            status = 500
            raise
        finally:
            duration = time.time() - start
            endpoint = func.__name__
            request_latency.labels(method='POST', endpoint=endpoint).observe(duration)
            request_count.labels(method='POST', endpoint=endpoint, status=status).inc()
    return wrapper

@track_metrics
def create_order(order_data):
    # Business logic
    return process_order(order_data)

# Rate (requests per second)
rate(http_requests_total{job="order-service"}[5m])

# Error rate (errors as % of total)
rate(http_requests_total{job="order-service",status=~"5.."}[5m]) / 
rate(http_requests_total{job="order-service"}[5m])

# Latency (p95)
histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))

# Combined SLO check: availability (error rate < 0.1%)
(
  1 - (
    rate(http_requests_total{status=~"5.."}[5m]) / 
    rate(http_requests_total[5m])
  )
) > 0.999

# CPU utilization
container_cpu_usage_seconds_total
container_cpu_throttling_seconds_total  # CPU throttling (oversubscribed)

# Memory utilization
container_memory_usage_bytes
container_memory_max_usage_bytes

# Disk usage
node_filesystem_avail_bytes  # Available disk space
container_fs_usage_bytes      # Container disk usage

# Network utilization
container_network_receive_bytes_total
container_network_transmit_bytes_total

# Disk I/O
node_disk_reads_completed_total
node_disk_writes_completed_total
node_disk_io_time_seconds_total

# Error metrics
node_disk_io_errs_total
node_network_receive_errs_total

# CPU utilization (should be <70% for safety)
rate(container_cpu_usage_seconds_total[5m]) * 100

# Memory utilization (should be <80%)
container_memory_usage_bytes / container_spec_memory_limit_bytes * 100

# Disk utilization (alert at >85%)
node_filesystem_avail_bytes / node_filesystem_size_bytes * 100

# Network saturation (dropped packets indicate saturation)
rate(node_network_receive_drop_total[5m])

from prometheus_client import Gauge, Counter

# Domain-specific metrics for order service
orders_in_progress = Gauge(
    'orders_in_progress',
    'Number of orders currently being processed'
)

orders_completed_total = Counter(
    'orders_completed_total',
    'Total orders completed',
    ['status']  # 'success', 'cancelled'
)

payment_processing_time = Histogram(
    'payment_processing_seconds',
    'Time spent processing payments'
)

inventory_stock_level = Gauge(
    'inventory_stock_level',
    'Current stock level',
    ['product_id']
)

class OrderService:
    def process_order(self, order_id):
        orders_in_progress.inc()
        
        try:
            # Process payment
            start = time.time()
            payment_result = self.process_payment(order_id)
            payment_processing_time.observe(time.time() - start)
            
            # Update inventory
            self.update_inventory(order_id)
            
            orders_completed_total.labels(status='success').inc()
        except Exception as e:
            orders_completed_total.labels(status='failed').inc()
            raise
        finally:
            orders_in_progress.dec()

# ANTI-PATTERN: Using user ID as metric label
user_latency = Histogram(
    'request_latency_seconds',
    'Request latency per user',
    ['user_id']  # Could be millions of unique values!
)

# This creates millions of time series (one per user)
# Memory and disk explode

# Use bounded labels (few unique values)
request_latency = Histogram(
    'request_latency_seconds',
    'Request latency',
    ['service', 'endpoint', 'status_code'],  # Limited cardinality
    buckets=[...]
)

# For per-user metrics, use a separate index/database
# Query user metrics separately if needed

# Alert when metric cardinality is too high
- alert: PrometheusHighMetricCardinality
  expr: count(count by (__name__) (up) > 10000) > 50
  for: 5m
  annotations:
    summary: "Prometheus instance has {{ $value }} high-cardinality metrics"

Application Logs
    ↓
Filebeat (log shipper)
    ↓
Logstash (log processor/enricher)
    ↓
Elasticsearch (full-text search database)
    ↓
Kibana (visualization/query UI)

2025-12-10T12:34:56Z order-service: Error processing order 12345 for user john@example.com: database connection timeout

{
  "timestamp": "2025-12-10T12:34:56Z",
  "service": "order-service",
  "level": "ERROR",
  "request_id": "req-abc123def456",
  "order_id": 12345,
  "user_id": "user-789",
  "error_type": "database_timeout",
  "error_message": "database connection timeout after 5s",
  "duration_ms": 5234,
  "metadata": {
    "region": "us-east-1",
    "pod": "order-service-pod-5",
    "node": "k8s-node-12"
  }
}

import json
import logging
import sys
from pythonjsonlogger import jsonlogger

# Configure JSON logging
logHandler = logging.StreamHandler(sys.stdout)
formatter = jsonlogger.JsonFormatter(
    '%(timestamp)s %(level)s %(name)s %(message)s %(request_id)s %(order_id)s'
)
logHandler.setFormatter(formatter)

logger = logging.getLogger()
logger.addHandler(logHandler)
logger.setLevel(logging.INFO)

# Log with structured fields
logger.info(
    "Order processed",
    extra={
        'timestamp': '2025-12-10T12:34:56Z',
        'request_id': 'req-abc123',
        'order_id': 12345,
        'user_id': 'user-789',
        'duration_ms': 245,
        'status': 'success'
    }
)

import random

def should_log(request_id):
    """Log 10% of requests in production, 100% in dev."""
    if env == 'dev':
        return True
    # Hash-based sampling for consistent trace context
    hash_val = int(hashlib.md5(request_id.encode()).hexdigest(), 16)
    return (hash_val % 100) < 10  # 10% sample

if should_log(request_id):
    logger.info("Request details", extra=log_data)

# Recent logs (7 days): Hot storage (fast, expensive)
# Warm logs (8-30 days): Warm storage (slower, cheaper)
# Archive (31+ days): S3 (very cheap)

elasticsearch.yml:
  index.lifecycle.name: order-service-logs
  index.lifecycle.rollover_alias: order-service-logs-write

ilm_policy:
  phases:
    hot:
      min_age: 0d
      actions:
        rollover:
          max_primary_shard_size: 50GB
    warm:
      min_age: 7d
      actions:
        set_priority:
          priority: 50
        forcemerge:
          max_num_segments: 1
    cold:
      min_age: 30d
      actions:
        searchable_snapshot:
          snapshot_repository: s3-repository
    delete:
      min_age: 90d
      actions:
        delete: {}

# ANTI-PATTERN: Logs credit card numbers
logger.info(f"Payment: {credit_card_number}")

# GOOD: Log hashed or masked values
logger.info(f"Payment: {credit_card_last_4}")

GET logs-*/_search
{
  "query": {
    "bool": {
      "must": [
        {"match": {"level": "ERROR"}},
        {"range": {"timestamp": {"gte": "now-1h"}}}
      ]
    }
  },
  "aggs": {
    "by_service": {
      "terms": {"field": "service", "size": 50}
    }
  }
}

histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m]))

GET logs-*/_search
{
  "query": {
    "match": {"user_id": "user-123"}
  },
  "sort": [{"timestamp": {"order": "asc"}}]
}

User Request (5000ms total)
├─ API Gateway (100ms)
├─ Auth Service (150ms)
├─ Order Service (800ms)
│  ├─ Validate Order (100ms)
│  ├─ Product Service (250ms)  ← slow!
│  │  └─ Database Query (200ms)
│  ├─ Inventory Service (150ms)
│  └─ Payment Service (300ms)
│     └─ External Payment API (280ms)  ← very slow!
└─ Notification Service (200ms)

from jaeger_client import Config
import logging

def init_tracer(service_name):
    config = Config(
        config={
            'sampler': {
                'type': 'const',  # Sample 100% (can reduce in prod)
                'param': 1,
            },
            'logging': True,
            'local_agent': {
                'reporting_host': 'jaeger-agent.monitoring',
                'reporting_port': 6831,
            },
        },
        service_name=service_name,
        validate=True,
    )
    return config.initialize_tracer()

tracer = init_tracer('order-service')

# Instrument a function
def process_order(order_id):
    with tracer.start_active_span('process_order') as scope:
        span = scope.span
        span.set_tag('order_id', order_id)
        
        # Nested spans
        with tracer.start_active_span('validate_order') as nested:
            validate_order(order_id)
        
        with tracer.start_active_span('call_payment_service') as nested:
            payment_result = call_payment_service(order_id)
            nested.span.set_tag('payment_status', payment_result['status'])
        
        return {'success': True}

# Sample traces intelligently
class ProbabilisticSampler:
    def __init__(self, initial_rate=0.001):
        self.rate = initial_rate
    
    def should_sample(self, trace_id, error_occurred=False):
        if error_occurred:
            return True  # Always sample errors
        
        if is_slow_request(trace_id):
            return True  # Always sample slow requests
        
        # Sample normal requests at low rate
        return random.random() < self.rate
    
    def adjust_rate(self, error_rate):
        # If error rate is high, increase sampling
        if error_rate > 0.01:
            self.rate = min(0.1, self.rate * 2)
        else:
            self.rate = max(0.001, self.rate / 2)

groups:
- name: order-service-alerts
  rules:
  
  # P0: Complete outage
  - alert: OrderServiceDown
    expr: up{job="order-service"} == 0
    for: 1m  # Alert after 1 minute (quick response)
    severity: P0
    annotations:
      summary: "Order service is completely down"
      description: "No order-service instances are responding"
      runbook: "https://wiki.company.com/runbooks/order-service-down"
  
  # P1: High error rate
  - alert: OrderServiceHighErrorRate
    expr: |
      rate(http_requests_total{job="order-service",status=~"5.."}[5m]) /
      rate(http_requests_total{job="order-service"}[5m]) > 0.05
    for: 3m  # Allow transient spikes
    severity: P1
    annotations:
      summary: "Order service error rate >5%"
      description: "Error rate: {{ $value | humanizePercentage }}"
  
  # P2: Elevated latency
  - alert: OrderServiceHighLatency
    expr: |
      histogram_quantile(0.95, rate(http_request_duration_seconds_bucket{job="order-service"}[5m])) > 1.0
    for: 5m
    severity: P2
    annotations:
      summary: "Order service p95 latency >1s"
  
  # P3: Database connection pool exhausted
  - alert: OrderServiceDBPoolExhausted
    expr: |
      pg_stat_activity_count{database="orders",state="active"} /
      pg_settings_max_connections > 0.8
    for: 10m
    severity: P3
    annotations:
      summary: "Order service database connection pool 80% exhausted"
  
  # P4: Informational
  - alert: OrderServiceHighMemory
    expr: |
      container_memory_usage_bytes{pod=~"order-service.*"} /
      container_spec_memory_limit_bytes > 0.7
    for: 15m
    severity: P4
    annotations:
      summary: "Order service memory usage >70%"

Week 1: Alice (primary), Bob (backup)
Week 2: Bob (primary), Charlie (backup)
Week 3: Charlie (primary), Dave (backup)
Week 4: Dave (primary), Alice (backup)

Escalation:
- Alert fires
- Page primary on-call (Alice)
- Alice has 5 minutes to acknowledge
- After 5 min, page manager
- Manager can escalate further to VP of Engineering

- alert: HighMemory
  expr: node_memory_MemAvailable_bytes < 1000000000
  annotations:
    summary: "High memory usage detected"
    # Engineer sees this alert: Now what?

- alert: HighMemoryPressure
  expr: node_memory_MemAvailable_bytes < 1000000000
  for: 10m
  severity: P2
  annotations:
    summary: "Node {{ $labels.node }} memory available < 1 GB"
    description: "This usually indicates a memory leak or misconfiguration"
    runbook: "https://wiki/memory-pressure-runbook"
    dashboards: 
      - "https://grafana/d/memory-details/{{ $labels.node }}"
    context:
      container_count: "{{ query('count(container_memory_usage_bytes{node=\"{{ $labels.node }}\"}') }}"
      top_containers: "List top 3 containers by memory"

# Memory Pressure Runbook

## What's happening?
Node {{ $labels.node }} has less than 1 GB available memory, indicating:
- Pods are using more memory than expected
- Possible memory leak in a container
- Insufficient node capacity for workload

## Quick investigation
1. SSH to node: `ssh {{ $labels.node }}`
2. Check memory usage: `free -h`
3. Check top memory consumers: `docker stats --no-stream`
4. Check for memory leaks: `docker logs <container> | grep OOM`

## Actions to take
- **Short term**: Kill largest container or evict least critical pod
- **Medium term**: Add more nodes or resize existing nodes
- **Long term**: Right-size pod resource requests based on actual usage

## Escalate if
- Multiple nodes under memory pressure
- Unable to free memory

apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service
spec:
  replicas: 3
  template:
    spec:
      # Restart policy for automatic recovery
      restartPolicy: Always  # Always restart failed pods
      
      containers:
      - name: order-service
        image: order-service:v1.2.3
        
        # Readiness probe: remove from load balancer if unhealthy
        readinessProbe:
          httpGet:
            path: /health/ready
            port: 8080
          failureThreshold: 2
          periodSeconds: 5
        
        # Liveness probe: restart if unhealthy
        livenessProbe:
          httpGet:
            path: /health/live
            port: 8080
          failureThreshold: 3
          periodSeconds: 10

# Custom operator: automatically promotes replica if primary fails

from kopf import (
    on, Timer, FieldFunction,
    patch, adopt, index
)
import logging

log = logging.getLogger()

@on.event(
    'postgresql.io', 'v1', 'postgresqlcluster',
    labels={'ha-enabled': 'true'}
)
def ensure_primary_alive(spec, name, namespace, **kwargs):
    """Check if primary is alive; promote replica if not."""
    
    primary_pod = f"{name}-primary"
    primary_namespace = namespace
    
    # Check if primary is healthy
    if not is_primary_healthy(primary_pod, primary_namespace):
        log.info(f"Primary {primary_pod} is unhealthy, promoting replica")
        
        # Find best replica
        replicas = get_replicas(name, namespace)
        best_replica = select_best_replica(replicas)
        
        # Promote replica
        promote_replica(best_replica, primary_namespace)
        
        # Update DNS/load balancer to point to new primary
        update_route(f"{name}-primary", best_replica)
        
        # Old primary becomes replica when it recovers
        demote_to_replica(primary_pod, primary_namespace)
        
        log.info(f"Promoted {best_replica} to primary")

@on.event(
    'postgresql.io', 'v1', 'postgresqlcluster',
    labels={'ha-enabled': 'true'},
    initial=False
)
def monitor_replication_lag(spec, name, namespace, **kwargs):
    """Alert if replication lag gets too high."""
    
    replicas = get_replicas(name, namespace)
    
    for replica in replicas:
        lag = get_replication_lag(replica)
        
        if lag > 10000:  # 10 seconds
            log.warning(f"High replication lag on {replica}: {lag}ms")
            trigger_alert('high_replication_lag', {
                'replica': replica,
                'lag_ms': lag
            })

apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service
  annotations:
    # Custom annotation for rollback operator
    auto-rollback: "true"
    health-check-threshold: "0.05"  # Rollback if error rate >5%
spec:
  replicas: 10
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 2
      maxUnavailable: 0  # Zero unavailability
  
  template:
    metadata:
      labels:
        app: order-service
    spec:
      containers:
      - name: order-service
        image: order-service:v1.2.4
        livenessProbe:
          httpGet:
            path: /health/live
            port: 8080
          failureThreshold: 3

@on.update(
    'apps', 'v1', 'deployment',
    labels={'auto-rollback': 'true'},
    field=FieldFunction('status.conditions[?type=="Progressing"]'),
    old_value={'status': 'True'},  # Was progressing
    new_value={'status': 'False'}   # Now not progressing (stuck)
)
def rollback_on_failure(spec, status, name, namespace, **kwargs):
    """Rollback deployment if it gets stuck."""
    
    # Get current error rate
    error_rate = get_current_error_rate(name)
    threshold = float(spec['annotations'].get('health-check-threshold', '0.05'))
    
    if error_rate > threshold:
        log.warning(f"Deployment {name} has high error rate {error_rate:.2%}")
        
        # Get previous revision
        prev_revision = get_previous_revision(name, namespace)
        
        if prev_revision:
            log.info(f"Rolling back to revision {prev_revision}")
            
            # Rollback
            patch_deployment(name, namespace, {
                'spec': {
                    'template': {
                        'metadata': {
                            'labels': {
                                'revision': prev_revision
                            }
                        }
                    }
                }
            })
            
            trigger_alert('deployment_rollback', {
                'deployment': name,
                'error_rate': error_rate,
                'prev_revision': prev_revision
            })

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: order-service-hpa
  namespace: production
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: order-service
  
  minReplicas: 5   # Minimum instances
  maxReplicas: 50  # Maximum instances
  
  metrics:
  
  # Scale based on CPU utilization
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70  # Scale up at 70% CPU
  
  # Scale based on memory
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 80  # Scale up at 80% memory
  
  # Scale based on custom metric (queue depth)
  - type: Pods
    pods:
      metric:
        name: queue_depth
      target:
        type: AverageValue
        averageValue: "50"  # Scale up when avg queue > 50 items
  
  # Behavior (scaling speed)
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 300  # Wait 5 min before scaling down
      policies:
      - type: Percent
        value: 50  # Scale down by max 50% at a time
        periodSeconds: 60
    scaleUp:
      stabilizationWindowSeconds: 0  # Scale up immediately
      policies:
      - type: Percent
        value: 100  # Scale up by max 100% at a time
        periodSeconds: 30

# Application exposes queue depth metric
from prometheus_client import Gauge

order_queue_depth = Gauge(
    'order_queue_depth',
    'Number of orders waiting in queue'
)

# HPA queries this metric and scales accordingly
# When queue_depth > threshold, HPA adds more pods

apiVersion: chaos-mesh.org/v1alpha1
kind: PodChaos
metadata:
  name: kill-order-service-pod
  namespace: chaos
spec:
  action: pod-failure  # Kill pod
  mode: fixed
  value: 1  # Kill 1 pod
  
  selector:
    namespaces:
    - production
    labelSelectors:
      app: order-service
  
  duration: 5m  # Run for 5 minutes
  scheduler:
    cron: "0 11 * * 1-5"  # Run on weekdays at 11 AM (business hours)

---
# Network latency chaos
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: add-order-service-latency
  namespace: chaos
spec:
  action: delay  # Add network delay
  mode: percentage
  percentage: 50  # Affect 50% of traffic
  
  selector:
    namespaces:
    - production
    labelSelectors:
      app: order-service
  
  delay:
    latency: "500ms"  # Add 500ms latency
    jitter: "100ms"
  
  duration: 10m

---
# Packet loss chaos
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: packet-loss-payment-service
  namespace: chaos
spec:
  action: loss  # Drop packets
  mode: percentage
  percentage: 20  # Lose 20% of packets
  
  selector:
    namespaces:
    - production
    labelSelectors:
      app: payment-service
  
  loss:
    loss: "20%"
  
  duration: 5m

---
# Disk failure chaos
apiVersion: chaos-mesh.org/v1alpha1
kind: IOChaos
metadata:
  name: database-io-error
  namespace: chaos
spec:
  action: latency  # Add I/O latency
  mode: fixed
  value: 1
  
  selector:
    namespaces:
    - databases
    labelSelectors:
      app: postgres
  
  latency: "500ms"
  duration: 3m

9:00 AM - Team briefing, review objectives
9:15 AM - Chaos Mesh starts killing random pods
9:30 AM - First alert fires, team responds
10:00 AM - Escalated to VP of Engineering (test escalation path)
10:30 AM - Chaos stops, assessment begins
11:00 AM - Retrospective: What went wrong? What worked? What to improve?
12:00 PM - Post-mortem writeup and action items

Before:
┌─────────────────────────┐
│   Load Balancer         │
└───────────┬─────────────┘
            │
    ┌───────┴────────┐
    ↓                ↓
┌────────────┐  ┌────────────┐
│  Blue v1.2 │  │  Green v1.2│  (inactive)
│ (active)   │  │            │
└────────────┘  └────────────┘

After deployment (v1.3 ready):
┌─────────────────────────┐
│   Load Balancer         │
└───────────┬─────────────┘
            │
    ┌───────┴────────┐
    ↓                ↓
┌────────────┐  ┌────────────┐
│  Blue v1.2 │  │  Green v1.3│  (new version)
│ (active)   │  │ (validated)│
└────────────┘  └────────────┘

After switch:
    (single atomic switch via load balancer)
┌─────────────────────────┐
│   Load Balancer         │
└───────────┬─────────────┘
            │
    ┌───────┴────────┐
    ↓                ↓
┌────────────┐  ┌────────────┐
│  Blue v1.2 │  │  Green v1.3│  (active)
│            │  │ (rollback  │
│(ready to   │  │ ready)     │
│rollback)   │  │            │
└────────────┘  └────────────┘

If issue found: One switch back to Blue (v1.2)
No gradual shift → no partial failures

# Blue deployment (active)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service-blue
  namespace: production
spec:
  replicas: 10
  selector:
    matchLabels:
      app: order-service
      slot: blue
  template:
    metadata:
      labels:
        app: order-service
        slot: blue
    spec:
      containers:
      - name: order-service
        image: order-service:v1.2.3  # Current version

---
# Green deployment (new version, not receiving traffic)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service-green
  namespace: production
spec:
  replicas: 10
  selector:
    matchLabels:
      app: order-service
      slot: green
  template:
    metadata:
      labels:
        app: order-service
        slot: green
    spec:
      containers:
      - name: order-service
        image: order-service:v1.2.4  # New version

---
# Service routes to blue (active)
apiVersion: v1
kind: Service
metadata:
  name: order-service
  namespace: production
spec:
  selector:
    app: order-service
    slot: blue  # Currently points to blue
  ports:
  - port: 80
    targetPort: 8080

---
# Deployment procedure:
# 1. Deploy green with new version (no traffic yet)
# kubectl apply -f green-deployment.yaml
#
# 2. Test green (internal traffic, synthetic tests)
# kubectl port-forward svc/order-service-green 8080:80
#
# 3. Once validated, switch service to green:
# kubectl patch service order-service -p '{"spec":{"selector":{"slot":"green"}}}'
#
# 4. If issue: switch back to blue
# kubectl patch service order-service -p '{"spec":{"selector":{"slot":"blue"}}}'
#
# 5. Once confident: update blue to new version, switch to blue, retire green

0 min: Blue 100%, Green 0%
5 min: Blue 95%, Green 5%
10 min: Blue 90%, Green 10%
15 min: Blue 75%, Green 25%
20 min: Blue 50%, Green 50%
25 min: Blue 25%, Green 75%
30 min: Blue 0%, Green 100%

During this time, monitor Green's error rate, latency, etc.
If metrics degrade, automatically rollback

apiVersion: flagger.app/v1beta1
kind: Canary
metadata:
  name: order-service
  namespace: production
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: order-service
  
  # Service mesh control
  service:
    port: 80
    targetPort: 8080
  
  analysis:
    interval: 1m  # Check metrics every minute
    threshold: 5  # Max 5 consecutive failed checks before rollback
    maxWeight: 50  # Max 50% traffic to canary
    stepWeight: 10  # Increase traffic by 10% every iteration
    
    # Success criteria
    metrics:
    - name: http-success-rate
      thresholdRange:
        min: 99  # Must stay >99% success
      interval: 1m
    
    - name: http-request-duration
      thresholdRange:
        max: 500  # p99 latency must stay <500ms
      interval: 1m
    
    # Custom metrics
    - name: error_rate
      query: |
        sum(rate(http_requests_total{status=~"5.."}[5m])) /
        sum(rate(http_requests_total[5m]))
      thresholdRange:
        max: 0.05  # Error rate <5%
  
  # Webhook for additional validation
  skipAnalysis: false  # Run analysis before rollout
  
  # Automated rollback
  skipFinalCheck: false
  
  # Alert notification
  alerts:
  - name: PagerDuty
    severity: critical
    provider:
      type: pagerduty
      address: https://pagerduty.example.com

apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: order-service
  namespace: production
spec:
  hosts:
  - order-service
  http:
  # Route 50% to stable (v1.2), 50% to canary (v1.3)
  - match:
    - sourceLabels:
        user-cohort: "treatment"  # Users in A/B test
    route:
    - destination:
        host: order-service
        port:
          number: 80
        subset: v1-3
      weight: 100  # All treatment users get v1.3
  
  # Control group gets stable version
  - route:
    - destination:
        host: order-service
        port:
          number: 80
        subset: v1-2  # Stable version
      weight: 100

---
# Track A/B test metrics
apiVersion: apps/v1
kind: Deployment
metadata:
  name: order-service-v1-3
spec:
  template:
    metadata:
      annotations:
        prometheus.io/path: /metrics
        prometheus.io/port: "9090"
    spec:
      containers:
      - name: order-service
        env:
        # Tag all metrics with experiment ID
        - name: EXPERIMENT_ID
          value: "exp_new_checkout_flow_2025-12"
        - name: EXPERIMENT_GROUP
          value: "treatment"

# Compare conversion rate between control and treatment
# Treatment (new checkout flow)
conversion_rate_treatment = 
  sum(rate(order_completed{experiment_group="treatment"}[1h])) /
  sum(rate(order_initiated{experiment_group="treatment"}[1h]))

# Control (stable)
conversion_rate_control = 
  sum(rate(order_completed{experiment_group="control"}[1h])) /
  sum(rate(order_initiated{experiment_group="control"}[1h]))

# If conversion_rate_treatment > conversion_rate_control:
#   Roll out new flow to all users
# Else:
#   Investigate why new flow underperforms

SEV-1 (Critical)
├─ Complete service outage or severe degradation
├─ Customer-facing impact
├─ Requires immediate response
└─ Page all on-call engineers

SEV-2 (High)
├─ Service degraded but partially functional
├─ Impact limited to subset of customers/features
├─ Requires quick response
└─ Page on-call engineer

SEV-3 (Medium)
├─ Service has issues but workaround exists
├─ Minimal impact to customers
└─ Can wait for next business day if after-hours

SEV-4 (Low)
├─ Informational or very limited impact
└─ Log and address during normal work

# Database Replication Lag Incident

## Alert
- Alert: `DatabaseReplicationLagHigh`
- Threshold: >30 seconds
- Severity: P2

## Quick diagnosis
1. Check current replication lag:
   ```sql
   SELECT now() - pg_last_wal_receive_lsn() AS replication_lag;

### Real Incident Case Study (Anonymized)

**Incident: Order Processing Delayed - Case Study**

**Timeline:**

**Timeline diagram:**

### Blameless Postmortem Process

**Postmortem goal**: Learn from incident, improve systems so it doesn't happen again.

**NOT to assign blame** - blame erodes psychological safety and prevents honest learning.

**Postmortem template:**

```markdown
# Postmortem: Order Processing Outage - 2025-12-10 14:32-14:54

## Summary
- Duration: 22 minutes
- Impact: Order processing delayed, 15% error rate
- Affected customers: ~2,000 active users
- Estimated revenue impact: ~$15,000

## Timeline
- 14:32: Latency alert fires
- 14:37: Error rate exceeds threshold, P1 alert triggered
- 14:42: Root cause identified (connection leak)
- 14:48: Payment service restarted, recovery begins
- 14:54: Full recovery, all metrics normal

## Root Cause Analysis

### Primary cause
Connection pool leak in order-service and payment-service:
```python
# BEFORE (buggy code)
def process_order(order_id):
    connection = db_pool.get_connection()
    try:
        return process_payment(order_id, connection)
    except Exception:
        pass  # BUG: Connection never returned on exception

# AFTER (fixed)
def process_order(order_id):
    connection = db_pool.get_connection()
    try:
        return process_payment(order_id, connection)
    finally:
        db_pool.return_connection(connection)  # Always return

Month 1 (Baseline): 98.2% uptime (19.4 hours downtime)
├─ Incidents: 3 major
├─ MTTR: 42 minutes
└─ MTBF: 6 days

Month 2-3 (Architecture): 98.7% uptime
├─ Multi-AZ deployment completed
├─ Database failover automated
├─ Incidents: 2 major
└─ MTTR: 35 minutes

Month 4-6 (Monitoring): 99.1% uptime
├─ Prometheus + Grafana deployed
├─ Alerting rules refined
├─ Incidents: 1 major, 2 minor
└─ MTTR: 18 minutes

Month 7-9 (Self-Healing): 99.5% uptime
├─ Automated pod restart policies
├─ Custom operators for complex recovery
├─ Incidents: 1 minor, several prevented
└─ MTTR: 8 minutes

Month 10-12 (Chaos & Culture): 99.95% uptime
├─ Chaos engineering practices
├─ Blameless postmortems
├─ Preventive measures
├─ Incidents: 0 major (prevented)
└─ MTTR: 3 minutes (when incidents occur)

global:
  scrape_interval: 15s
  evaluation_interval: 15s
  external_labels:
    cluster: 'us-east-1'

scrape_configs:

  # Kubernetes API server
  - job_name: 'kubernetes-apiservers'
    kubernetes_sd_configs:
    - role: endpoints
    scheme: https
    tls_config:
      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    relabel_configs:
    - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
      action: keep
      regex: default;kubernetes;https

  # All pods (service discovery)
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
    - role: pod
    relabel_configs:
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
      action: keep
      regex: 'true'
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
      action: replace
      target_label: __metrics_path__
      regex: (.+)
    - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
      action: replace
      regex: ([^:]+)(?::\d+)?;(\d+)
      replacement: $1:$2
      target_label: __address__
    - action: labelmap
      regex: __meta_kubernetes_pod_label_(.+)
    - source_labels: [__meta_kubernetes_namespace]
      action: replace
      target_label: kubernetes_namespace
    - source_labels: [__meta_kubernetes_pod_name]
      action: replace
      target_label: kubernetes_pod_name

# Alert rules
rule_files:
  - '/etc/prometheus/alert-rules.yaml'

groups:
- name: kubernetes-alerts
  rules:
  
  # Pod not ready
  - alert: PodNotReady
    expr: min(kube_pod_status_ready{condition="false"}) by (pod, namespace) == 1
    for: 10m
    severity: warning
    annotations:
      summary: "Pod {{ $labels.pod }} not ready"
  
  # High error rate
  - alert: HighErrorRate
    expr: |
      (
        sum(rate(http_requests_total{status=~"5.."}[5m])) by (job)
        /
        sum(rate(http_requests_total[5m])) by (job)
      ) > 0.05
    for: 5m
    severity: critical
    annotations:
      summary: "{{ $labels.job }} error rate > 5%"
      value: "{{ $value | humanizePercentage }}"
  
  # High latency
  - alert: HighLatency
    expr: |
      histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, job)) > 1.0
    for: 5m
    severity: warning
    annotations:
      summary: "{{ $labels.job }} p95 latency > 1s"
      value: "{{ $value }}s"
  
  # Database connection pool exhausted
  - alert: DBConnectionPoolExhausted
    expr: |
      pg_stat_activity{state="active"} / pg_settings_max_connections > 0.8
    for: 5m
    severity: critical
    annotations:
      summary: "Database connection pool {{ $labels.datname }} > 80% full"