Skip to content

deployment_guide.md

Latest commit

 

History

History
726 lines (544 loc) · 15.1 KB

deployment_guide.md

File metadata and controls

726 lines (544 loc) · 15.1 KB
Raw

Deployment Guide

Production deployment guide for Python AMR, covering Docker, environment configuration, scaling strategies, monitoring, and operational best practices.

Table of Contents

Overview

Deployment Models

Python AMR supports three deployment patterns:

  1. Single-Instance - Development and small production (<100 req/min)
  2. Multi-Instance - High availability and scaling (100-10k req/min)
  3. Containerized - Docker/Kubernetes orchestration (enterprise scale)

Architecture

┌─────────────────────────────────────────────────────┐
│                Load Balancer (Optional)             │
│                    nginx/HAProxy                    │
└────────────────────┬────────────────────────────────┘
                     │
        ┌────────────┼────────────┐
        │            │            │
        ▼            ▼            ▼
┌─────────────┬─────────────┬─────────────┐
│   API       │   API       │   API       │
│  Instance 1 │  Instance 2 │  Instance 3 │
└──────┬──────┴──────┬──────┴──────┬──────┘
       │             │             │
       └─────────────┼─────────────┘
                     │
        ┌────────────┼────────────┐
        │            │            │
        ▼            ▼            ▼
┌─────────────┬─────────────┬─────────────┐
│  PostgreSQL │   DuckDB    │  Reference  │
│  (Metadata) │ (Outputs)   │  Data       │
└─────────────┴─────────────┴─────────────┘

Production Requirements

Hardware

Component Minimum Recommended
CPU 4 cores 8+ cores
RAM 8 GB 16+ GB
Disk 50 GB 100+ GB SSD
Network 100 Mbps 1 Gbps

Software

  • OS: Ubuntu 22.04 LTS, RHEL 8+, or Debian 11+
  • Python: 3.14.x (required)
  • Database: PostgreSQL 14+ (recommended) or SQLite 3.35+
  • Reverse Proxy: nginx 1.20+ or HAProxy 2.4+
  • Container Runtime (optional): Docker 20.10+ or containerd 1.6+

Docker Deployment

Dockerfile

Create Dockerfile in project root:

FROM python:3.14-slim

# Install system dependencies
RUN apt-get update && apt-get install -y \
    build-essential \
    libpq-dev \
    && rm -rf /var/lib/apt/lists/*

# Set working directory
WORKDIR /app

# Copy requirements
COPY pyproject.toml uv.lock ./

# Install Python dependencies
RUN pip install --no-cache-dir -e ".[dev]"

# Copy application
COPY src/ ./src/
COPY data/ ./data/
COPY scripts/ ./scripts/

# Create non-root user
RUN useradd -m -u 1000 amr && chown -R amr:amr /app
USER amr

# Expose port
EXPOSE 8000

# Health check
HEALTHCHECK --interval=30s --timeout=5s --retries=3 \
    CMD curl -f http://localhost:8000/health || exit 1

# Run application
CMD ["uvicorn", "amr.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

Docker Compose

Create docker-compose.yml:

version: '3.8'

services:
  api:
    build: .
    ports:
      - "8000:8000"
    environment:
      - AMR_METADATA_DB_URL=postgresql+asyncpg://amr:password@postgres:5432/amr
      - AMR_DUCKDB_PATH=/data/runs_duckdb.db
      - AMR_PERSIST_QUEUE_WORKERS=4
      - AMR_LOG_LEVEL=INFO
    volumes:
      - ./data:/app/data
      - duckdb_data:/data
    depends_on:
      - postgres
    restart: unless-stopped

  postgres:
    image: postgres:16-alpine
    environment:
      - POSTGRES_USER=amr
      - POSTGRES_PASSWORD=password
      - POSTGRES_DB=amr
    volumes:
      - postgres_data:/var/lib/postgresql/data
    ports:
      - "5432:5432"
    restart: unless-stopped

  nginx:
    image: nginx:alpine
    ports:
      - "80:80"
      - "443:443"
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
      - ./ssl:/etc/nginx/ssl:ro
    depends_on:
      - api
    restart: unless-stopped

volumes:
  postgres_data:
  duckdb_data:

Build and Run

# Build image
docker build -t python-amr:latest .

# Run with docker-compose
docker-compose up -d

# View logs
docker-compose logs -f api

# Check health
curl http://localhost:8000/health

Environment Configuration

Environment Variables

Create .env file:

# Database Configuration
AMR_METADATA_DB_URL=postgresql+asyncpg://user:pass@localhost:5432/amr
AMR_DUCKDB_PATH=/var/lib/amr/runs_duckdb.db

# Persistence Queue
AMR_PERSIST_QUEUE_WORKERS=4
AMR_PERSIST_QUEUE_MAXSIZE=4096
AMR_PERSIST_RETRY_MAX_RETRIES=3
AMR_PERSIST_RETRY_BACKOFF_MS=50

# Dead Letter Queue
AMR_DEAD_LETTER_MAX_JSON_BYTES=131072
AMR_DEAD_LETTER_MAX_ERROR_CHARS=4096
AMR_DEAD_LETTER_REDACT_KEYS=password,token,secret,api_key

# Logging
AMR_LOG_LEVEL=INFO
AMR_LOG_FORMAT=json  # json or text

# API Configuration
AMR_API_HOST=0.0.0.0
AMR_API_PORT=8000
AMR_API_WORKERS=4  # For production with gunicorn

# Security
AMR_ALLOWED_ORIGINS=https://dashboard.example.com
AMR_ENABLE_CORS=true

# Reference Data
AMR_DATA_DIR=/var/lib/amr/data
AMR_SNAPSHOTS_DIR=/var/lib/amr/data/snapshots

Load Environment

# Using systemd
sudo cp amr.env /etc/amr/amr.env
sudo chmod 600 /etc/amr/amr.env

# Source in service file
EnvironmentFile=/etc/amr/amr.env

Database Setup

PostgreSQL Setup

# Install PostgreSQL
sudo apt-get install postgresql-16

# Create database and user
sudo -u postgres psql <<EOF
CREATE USER amr WITH PASSWORD 'your_secure_password';
CREATE DATABASE amr OWNER amr;
GRANT ALL PRIVILEGES ON DATABASE amr TO amr;
EOF

# Run migrations
PYTHONPATH=src python -c "
from amr.repositories.run_repository import RunRepository
import asyncio
asyncio.run(RunRepository().migrate())
"

PostgreSQL Tuning

Edit /etc/postgresql/16/main/postgresql.conf:

# Memory
shared_buffers = 256MB
effective_cache_size = 1GB
work_mem = 16MB

# Connections
max_connections = 100

# Performance
checkpoint_completion_target = 0.9
wal_buffers = 16MB

# Logging
log_statement = 'mod'  # Log all modifications
log_duration = on
log_min_duration_statement = 1000  # Log slow queries (>1s)

DuckDB Setup

# Create data directory
sudo mkdir -p /var/lib/amr/data
sudo chown amr:amr /var/lib/amr/data

# Initialize DuckDB
PYTHONPATH=src python -c "
import duckdb
conn = duckdb.connect('/var/lib/amr/data/runs_duckdb.db')
conn.execute('''
    CREATE TABLE IF NOT EXISTS run_outputs (
        run_id INTEGER NOT NULL,
        row_index INTEGER NOT NULL,
        input_value VARCHAR,
        sir_result VARCHAR,
        metadata JSON,
        created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
        PRIMARY KEY (run_id, row_index)
    )
''')
conn.close()
"

Scaling Strategies

Vertical Scaling

CPU Scaling

# Increase async workers
export AMR_PERSIST_QUEUE_WORKERS=8

# Increase API workers (with gunicorn)
export AMR_API_WORKERS=8

Memory Scaling

# Increase queue size
export AMR_PERSIST_QUEUE_MAXSIZE=8192

# DuckDB memory limit
export DUCKDB_MEMORY_LIMIT=8GB

Horizontal Scaling

Multi-Instance with Load Balancer

nginx.conf:

upstream amr_api {
    least_conn;
    server api1.internal:8000 max_fails=3 fail_timeout=30s;
    server api2.internal:8000 max_fails=3 fail_timeout=30s;
    server api3.internal:8000 max_fails=3 fail_timeout=30s;
}

server {
    listen 80;
    server_name api.example.com;

    location / {
        proxy_pass http://amr_api;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_connect_timeout 60s;
        proxy_send_timeout 60s;
        proxy_read_timeout 60s;
    }

    location /health {
        access_log off;
        proxy_pass http://amr_api;
    }
}

Shared PostgreSQL

All instances connect to single PostgreSQL:

# Instance 1
export AMR_METADATA_DB_URL=postgresql+asyncpg://amr:pass@postgres.internal:5432/amr

# Instance 2
export AMR_METADATA_DB_URL=postgresql+asyncpg://amr:pass@postgres.internal:5432/amr

# Instance 3
export AMR_METADATA_DB_URL=postgresql+asyncpg://amr:pass@postgres.internal:5432/amr

DuckDB Limitation

DuckDB is single-writer - use one of these strategies:

  1. Separate DuckDB per instance - Each instance has its own DuckDB file
  2. PostgreSQL for outputs - Switch output storage to PostgreSQL (slower)
  3. Write leader - One instance handles all writes, others read-only

Monitoring & Logging

Prometheus Metrics

Expose metrics endpoint:

# Configured in amr.api.app
@app.get("/metrics")
async def metrics():
    """Prometheus-compatible metrics endpoint."""
    return PlainTextResponse(generate_metrics())

prometheus.yml:

scrape_configs:
  - job_name: 'python-amr'
    static_configs:
      - targets: ['api1.internal:8000', 'api2.internal:8000']
    metrics_path: '/metrics'
    scrape_interval: 30s

Grafana Dashboard

Key metrics to monitor:

  • Request rate - amr_requests_total
  • Response time - amr_request_duration_seconds
  • Queue depth - amr_queue_size
  • Database connections - amr_db_connections_active
  • Error rate - amr_errors_total

Structured Logging

Configure JSON logging:

export AMR_LOG_FORMAT=json
export AMR_LOG_LEVEL=INFO

Example log entry:

{
  "timestamp": "2026-02-15T12:00:00Z",
  "level": "INFO",
  "message": "Request processed",
  "method": "POST",
  "path": "/v1/sir/interpret",
  "status_code": 200,
  "duration_ms": 45.3,
  "request_id": "abc123"
}

Centralized Logging

Fluentd/Fluent Bit configuration:

# fluent-bit.conf
[INPUT]
    Name tail
    Path /var/log/amr/api.log
    Parser json

[OUTPUT]
    Name elasticsearch
    Match *
    Host elasticsearch.internal
    Port 9200
    Index amr-logs

Security Configuration

HTTPS/TLS

nginx SSL configuration:

server {
    listen 443 ssl http2;
    server_name api.example.com;

    ssl_certificate /etc/nginx/ssl/cert.pem;
    ssl_certificate_key /etc/nginx/ssl/key.pem;
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers HIGH:!aNULL:!MD5;
    ssl_prefer_server_ciphers on;

    # HSTS
    add_header Strict-Transport-Security "max-age=31536000" always;

    location / {
        proxy_pass http://amr_api;
    }
}

Firewall

# UFW (Ubuntu)
sudo ufw allow 22/tcp    # SSH
sudo ufw allow 80/tcp    # HTTP
sudo ufw allow 443/tcp   # HTTPS
sudo ufw deny 8000/tcp   # Block direct API access
sudo ufw enable

Database Encryption

Enable PostgreSQL SSL:

# postgresql.conf
ssl = on
ssl_cert_file = '/etc/ssl/certs/server.crt'
ssl_key_file = '/etc/ssl/private/server.key'

Connection with SSL:

export AMR_METADATA_DB_URL=postgresql+asyncpg://user:pass@host:5432/amr?ssl=require

Backup & Recovery

PostgreSQL Backup

# Daily backup script
#!/bin/bash
BACKUP_DIR=/var/backups/amr/postgres
DATE=$(date +%Y%m%d_%H%M%S)

pg_dump -U amr -Fc amr > $BACKUP_DIR/amr_$DATE.dump

# Retention: keep last 7 days
find $BACKUP_DIR -name "*.dump" -mtime +7 -delete

# Cron: 2 AM daily
# 0 2 * * * /usr/local/bin/backup_postgres.sh

DuckDB Backup

# Backup DuckDB file
#!/bin/bash
DUCKDB_PATH=/var/lib/amr/data/runs_duckdb.db
BACKUP_DIR=/var/backups/amr/duckdb
DATE=$(date +%Y%m%d_%H%M%S)

# Stop writes (optional)
systemctl stop amr-api

# Copy file
cp $DUCKDB_PATH $BACKUP_DIR/runs_duckdb_$DATE.db

# Restart
systemctl start amr-api

# Retention
find $BACKUP_DIR -name "*.db" -mtime +30 -delete

Restore Procedure

# PostgreSQL restore
pg_restore -U amr -d amr /var/backups/amr/postgres/amr_20260215.dump

# DuckDB restore
systemctl stop amr-api
cp /var/backups/amr/duckdb/runs_duckdb_20260215.db /var/lib/amr/data/runs_duckdb.db
systemctl start amr-api

Performance Tuning

Gunicorn Configuration

For production, use gunicorn instead of uvicorn directly:

# Install
pip install gunicorn

# Run
gunicorn amr.api.app:app \
    --worker-class uvicorn.workers.UvicornWorker \
    --workers 4 \
    --bind 0.0.0.0:8000 \
    --timeout 60 \
    --access-logfile /var/log/amr/access.log \
    --error-logfile /var/log/amr/error.log

Systemd Service

Create /etc/systemd/system/amr-api.service:

[Unit]
Description=Python AMR API
After=network.target postgresql.service

[Service]
Type=notify
User=amr
Group=amr
WorkingDirectory=/opt/amr
EnvironmentFile=/etc/amr/amr.env
ExecStart=/opt/amr/.venv/bin/gunicorn amr.api.app:app \
    --worker-class uvicorn.workers.UvicornWorker \
    --workers 4 \
    --bind 0.0.0.0:8000 \
    --timeout 60
ExecReload=/bin/kill -s HUP $MAINPID
Restart=on-failure
RestartSec=10

[Install]
WantedBy=multi-user.target

Enable and start:

sudo systemctl daemon-reload
sudo systemctl enable amr-api
sudo systemctl start amr-api
sudo systemctl status amr-api

Troubleshooting

High Memory Usage

# Check process memory
ps aux | grep gunicorn

# Reduce workers
export AMR_API_WORKERS=2
export AMR_PERSIST_QUEUE_WORKERS=2

# Set DuckDB memory limit
export DUCKDB_MEMORY_LIMIT=2GB

Queue Backlog

# Check queue health
curl http://localhost:8000/v1/runs/queue

# Increase workers
export AMR_PERSIST_QUEUE_WORKERS=8

# Increase queue size
export AMR_PERSIST_QUEUE_MAXSIZE=8192

Database Connection Errors

# Check PostgreSQL connections
SELECT count(*) FROM pg_stat_activity;

# Increase pool size
export AMR_DB_POOL_SIZE=20
export AMR_DB_MAX_OVERFLOW=40

# Check connection limit
SELECT setting FROM pg_settings WHERE name = 'max_connections';

Performance Degradation

# Check slow queries
tail -f /var/log/postgresql/postgresql-16-main.log | grep "duration"

# Analyze DuckDB
sqlite3 runs_duckdb.db "PRAGMA analyze"

# Check disk space
df -h /var/lib/amr

Related Documentation

Last Updated: 2026-02-15