diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d7e18518bc..b0c07e6fd6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,6 +92,7 @@ jobs: uses: actions/setup-node@v4 with: node-version: "20" + cache: "npm" - id: get_tag run: | @@ -111,14 +112,14 @@ jobs: # Build browser JS bundle cd packages/karriojs - npm install + npm ci npx gulp build --output "${GITHUB_WORKSPACE}/apps/api/karrio/server/static/karrio/js/karrio.js" cd - - name: Build embeddable elements run: | cd packages/elements - npm install + npm ci npm run build # Copy built elements to Django static directory @@ -163,15 +164,28 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - id: get_tag run: | cat ./apps/api/karrio/server/VERSION echo "tag=$(cat ./apps/api/karrio/server/VERSION)" >> "$GITHUB_ENV" - name: Build karrio dashboard image - run: | - echo 'Building karrio dashboard:${{ env.tag }}...' - ./bin/build-dashboard-image ${{ env.tag }} + uses: docker/build-push-action@v6 + with: + context: . + file: ./docker/dashboard/Dockerfile + push: false + load: true + tags: karrio/dashboard:${{ env.tag }} + build-args: | + VERSION=${{ env.tag }} + NEXT_PUBLIC_DASHBOARD_VERSION=${{ env.tag }} + SOURCE=https://github.com/karrioapi/karrio + cache-from: type=gha,scope=dashboard + cache-to: type=gha,mode=max,scope=dashboard - name: Push karrio dashboard image run: | diff --git a/.github/workflows/insiders-build.yml b/.github/workflows/insiders-build.yml index e4a5b8f088..021bdca73a 100644 --- a/.github/workflows/insiders-build.yml +++ b/.github/workflows/insiders-build.yml @@ -95,6 +95,7 @@ jobs: uses: actions/setup-node@v4 with: node-version: "20" + cache: "npm" - id: get_tag run: | @@ -112,14 +113,14 @@ jobs: --additional-properties=useSingleRequestParameter=true cd packages/karriojs - npm install + npm ci npx gulp build --output "${GITHUB_WORKSPACE}/apps/api/karrio/server/static/karrio/js/karrio.js" cd - - name: Build embeddable elements run: | cd packages/elements - npm install + npm ci npm run build ELEMENTS_STATIC="${GITHUB_WORKSPACE}/apps/api/karrio/server/static/karrio/elements" @@ -156,14 +157,27 @@ jobs: submodules: recursive token: ${{ secrets.GH_PAT }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - id: get_tag run: | cat ./apps/api/karrio/server/VERSION echo "tag=$(cat ./apps/api/karrio/server/VERSION)" >> "$GITHUB_ENV" + - name: Login to GHCR + run: echo ${{ secrets.GH_PAT }} | docker login ghcr.io -u USERNAME --password-stdin + - name: Build insider dashboard image - run: | - echo 'Build and push karrio-insiders dashboard:${{ env.tag }}...' - echo ${{ secrets.GH_PAT }} | docker login ghcr.io -u USERNAME --password-stdin - KARRIO_IMAGE=ghcr.io/karrioapi/dashboard SOURCE=https://github.com/karrioapi/karrio-insiders ./bin/build-dashboard-image ${{ env.tag }} && - docker push ghcr.io/karrioapi/dashboard:${{ env.tag }} || exit 1 + uses: docker/build-push-action@v6 + with: + context: . + file: ./docker/dashboard/Dockerfile + push: true + tags: ghcr.io/karrioapi/dashboard:${{ env.tag }} + build-args: | + VERSION=${{ env.tag }} + NEXT_PUBLIC_DASHBOARD_VERSION=${{ env.tag }} + SOURCE=https://github.com/karrioapi/karrio-insiders + cache-from: type=gha,scope=insiders-dashboard + cache-to: type=gha,mode=max,scope=insiders-dashboard diff --git a/.github/workflows/platform-build.yml b/.github/workflows/platform-build.yml index 7628bd185a..3275549b8f 100644 --- a/.github/workflows/platform-build.yml +++ b/.github/workflows/platform-build.yml @@ -41,6 +41,7 @@ jobs: uses: actions/setup-node@v4 with: node-version: "20" + cache: "npm" - id: get_tag run: | @@ -57,14 +58,14 @@ jobs: --additional-properties=useSingleRequestParameter=true cd packages/karriojs - npm install + npm ci npx gulp build --output "${GITHUB_WORKSPACE}/apps/api/karrio/server/static/karrio/js/karrio.js" cd - - name: Build embeddable elements run: | cd packages/elements - npm install + npm ci npm run build ELEMENTS_STATIC="${GITHUB_WORKSPACE}/apps/api/karrio/server/static/karrio/elements" @@ -126,6 +127,12 @@ jobs: submodules: recursive token: ${{ secrets.GH_PAT }} + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/apps/api/gunicorn-cfg.py b/apps/api/gunicorn-cfg.py index 9d1a2fe7e8..0014fb2e94 100644 --- a/apps/api/gunicorn-cfg.py +++ b/apps/api/gunicorn-cfg.py @@ -1,7 +1,6 @@ -# -*- encoding: utf-8 -*- import decouple -KARRIO_HOST = decouple.config("KARRIO_HTTP_HOST", default="0.0.0.0") +KARRIO_HOST = decouple.config("KARRIO_HTTP_HOST", default="0.0.0.0") # noqa: S104 KARRIO_PORT = decouple.config("KARRIO_HTTP_PORT", default=5002) bind = f"{KARRIO_HOST}:{KARRIO_PORT}" @@ -10,3 +9,12 @@ capture_output = True enable_stdio_inheritance = True workers = decouple.config("KARRIO_WORKERS", default=2, cast=int) + +# NOTE: preload_app is intentionally NOT set here. +# With UvicornWorker (ASGI), preload_app=True causes: +# 1. asyncio.CancelledError in django-health-check (stale event loop from master) +# 2. psycopg "BAD" connections (DB pool forked from master) +# UvicornWorker manages its own lifecycle and ignores gunicorn's post_fork, +# so these issues cannot be fixed via post_fork hooks. +# Cold-start CPU optimization should be addressed at the k8s level +# (resource requests/limits, startup probes) instead. diff --git a/apps/api/karrio/server/__init__.py b/apps/api/karrio/server/__init__.py index 0d1f7edf5d..d55ccad1f5 100644 --- a/apps/api/karrio/server/__init__.py +++ b/apps/api/karrio/server/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/apps/api/karrio/server/__main__.py b/apps/api/karrio/server/__main__.py index 4bb823c45c..e139181b68 100644 --- a/apps/api/karrio/server/__main__.py +++ b/apps/api/karrio/server/__main__.py @@ -1,11 +1,12 @@ #!/usr/bin/env python """Karrio's command-line utility for administrative tasks.""" + import os import sys def main(): - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'karrio.server.settings') + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "karrio.server.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: @@ -17,5 +18,5 @@ def main(): execute_from_command_line(sys.argv) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/apps/api/karrio/server/asgi.py b/apps/api/karrio/server/asgi.py index 41ddbc5218..106a2e83a6 100644 --- a/apps/api/karrio/server/asgi.py +++ b/apps/api/karrio/server/asgi.py @@ -1,17 +1,61 @@ """ ASGI config for karrio.server project. -It exposes the ASGI callable as a module-level variable named ``application``. +Exposes the ASGI callable as a module-level variable named ``application``. + +The callable is a thin async wrapper around Django's ASGI app that +captures the worker's running event loop on the first http/websocket +request. The captured loop is handed to the ``karrio.server.servicebus`` +publisher (if installed) so its async producer can schedule publishes +on the worker's event loop via ``asyncio.run_coroutine_threadsafe``. + +Why first-request capture and not the ASGI lifespan protocol: our +custom ``UvicornWorker`` (karrio/apps/api/karrio/server/workers.py) sets +``"lifespan": "off"`` explicitly because Django's stock ASGI app does +not handle lifespan messages. First-request capture is one global + +a handful of lines, no worker config change needed. For more information on this file, see https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ """ +import asyncio +import logging import os from django.core.asgi import get_asgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'karrio.server.settings') -os.environ.setdefault('DJANGO_ALLOW_ASYNC_UNSAFE', 'true') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "karrio.server.settings") +os.environ.setdefault("DJANGO_ALLOW_ASYNC_UNSAFE", "true") + +logger = logging.getLogger(__name__) + +_django_app = get_asgi_application() +_loop_captured = False + + +async def application(scope, receive, send): + """ASGI application with first-request event-loop capture. + + Non-http/websocket scopes pass straight through to Django. The + capture happens once per worker; subsequent requests are zero-cost + fast-path (a single boolean check). + """ + global _loop_captured + if not _loop_captured and scope.get("type") in ("http", "websocket"): + _loop_captured = True + try: + from karrio.server.servicebus import set_async_loop -application = get_asgi_application() + set_async_loop(asyncio.get_running_loop()) + logger.debug("ASGI: captured event loop for servicebus async publisher") + except ImportError: + # servicebus extension not installed in this deployment; that's + # fine — bridge dual-publish will use the sync path. + pass + except Exception: + logger.warning( + "ASGI: failed to capture event loop for servicebus async publisher", + exc_info=True, + ) + await _django_app(scope, receive, send) diff --git a/apps/api/karrio/server/lib/otel_huey.py b/apps/api/karrio/server/lib/otel_huey.py deleted file mode 100644 index 0baad44df1..0000000000 --- a/apps/api/karrio/server/lib/otel_huey.py +++ /dev/null @@ -1,146 +0,0 @@ -""" -OpenTelemetry instrumentation for Huey task queue. - -This module provides tracing support for Huey tasks, enabling distributed tracing -across API requests and background tasks. -""" -import functools -import logging -from typing import Any, Callable, Dict, Optional - -from opentelemetry import trace, context, propagate -from opentelemetry.trace import Status, StatusCode -from opentelemetry.semconv.trace import SpanAttributes - -logger = logging.getLogger(__name__) - - -class HueyInstrumentor: - """Instrumentation for Huey task queue.""" - - _instance = None - _instrumented = False - - def __new__(cls): - if cls._instance is None: - cls._instance = super().__new__(cls) - return cls._instance - - def instrument(self, huey_instance=None): - """ - Instrument Huey for OpenTelemetry tracing. - - Args: - huey_instance: The Huey instance to instrument. If None, will try to - import from Django settings. - """ - if self._instrumented: - logger.debug("Huey already instrumented") - return - - try: - if huey_instance is None: - from django.conf import settings - huey_instance = settings.HUEY - - # Wrap the task decorator - original_task = huey_instance.task - huey_instance.task = self._wrap_task_decorator(original_task, huey_instance) - - # Wrap periodic tasks - if hasattr(huey_instance, 'periodic_task'): - original_periodic = huey_instance.periodic_task - huey_instance.periodic_task = self._wrap_task_decorator(original_periodic, huey_instance) - - self._instrumented = True - logger.info("Huey instrumented for OpenTelemetry") - - except Exception as e: - logger.warning(f"Failed to instrument Huey: {e}") - - def _wrap_task_decorator(self, original_decorator: Callable, huey_instance) -> Callable: - """Wrap the Huey task decorator to add tracing.""" - - @functools.wraps(original_decorator) - def wrapped_decorator(*args, **kwargs): - decorated = original_decorator(*args, **kwargs) - - def task_wrapper(fn): - task_fn = decorated(fn) - - @functools.wraps(task_fn) - def traced_task(*task_args, **task_kwargs): - tracer = trace.get_tracer(__name__) - - # Extract trace context from task kwargs if present - trace_context = task_kwargs.pop('_otel_context', None) - if trace_context: - ctx = propagate.extract(trace_context) - token = context.attach(ctx) - else: - token = None - - # Start span for the task - task_name = fn.__name__ - with tracer.start_as_current_span( - f"huey.task.{task_name}", - kind=trace.SpanKind.CONSUMER, - ) as span: - try: - # Set span attributes - span.set_attribute("messaging.system", "huey") - span.set_attribute("messaging.destination", task_name) - span.set_attribute("messaging.operation", "process") - span.set_attribute("task.name", task_name) - - # Execute the task - result = task_fn(*task_args, **task_kwargs) - span.set_status(Status(StatusCode.OK)) - return result - - except Exception as e: - span.set_status(Status(StatusCode.ERROR, str(e))) - span.record_exception(e) - raise - finally: - if token: - context.detach(token) - - # Preserve original attributes - traced_task.task = task_fn.task if hasattr(task_fn, 'task') else task_fn - if hasattr(task_fn, '__name__'): - traced_task.__name__ = task_fn.__name__ - if hasattr(task_fn, '__module__'): - traced_task.__module__ = task_fn.__module__ - - return traced_task - - return task_wrapper - - return wrapped_decorator - - -def inject_trace_context(task_kwargs: Dict[str, Any]) -> Dict[str, Any]: - """ - Inject current trace context into task kwargs for propagation. - - This should be called when enqueuing a task to propagate the trace context - to the background worker. - - Args: - task_kwargs: The kwargs to be passed to the task - - Returns: - Updated kwargs with trace context - """ - carrier = {} - propagate.inject(carrier) - if carrier: - task_kwargs['_otel_context'] = carrier - return task_kwargs - - -def instrument_huey(): - """Convenience function to instrument Huey.""" - instrumentor = HueyInstrumentor() - instrumentor.instrument() \ No newline at end of file diff --git a/apps/api/karrio/server/settings/__init__.py b/apps/api/karrio/server/settings/__init__.py index 475f01cdad..9c91e6742d 100644 --- a/apps/api/karrio/server/settings/__init__.py +++ b/apps/api/karrio/server/settings/__init__.py @@ -1,4 +1,5 @@ # type: ignore +# ruff: noqa: F403, F405 __path__ = __import__("pkgutil").extend_path(__path__, __name__) import importlib.util @@ -34,5 +35,13 @@ from karrio.server.settings.admin import * +if importlib.util.find_spec("karrio.server.settings.huey") is not None: + from karrio.server.settings.huey import * + + +if importlib.util.find_spec("karrio.server.settings.servicebus") is not None: + from karrio.server.settings.servicebus import * + + if importlib.util.find_spec("karrio.server.settings.main") is not None: from karrio.server.settings.main import * diff --git a/apps/api/karrio/server/settings/apm.py b/apps/api/karrio/server/settings/apm.py index 0535b9f92c..18f97ddd24 100644 --- a/apps/api/karrio/server/settings/apm.py +++ b/apps/api/karrio/server/settings/apm.py @@ -1,8 +1,15 @@ # type: ignore +# ruff: noqa: F403, F405 +import asyncio +import logging +import time +from contextlib import suppress + +import karrio.server.core.telemetry_scrubbing as telemetry_scrubbing import posthog import sentry_sdk -from sentry_sdk.integrations.django import DjangoIntegration from karrio.server.settings.base import * +from sentry_sdk.integrations.django import DjangoIntegration # Try to import PostHog integration, fallback if not available try: @@ -50,25 +57,36 @@ # 5. Logging Integration: ERROR/CRITICAL logs are sent to Sentry # # Environment Variables: -# SENTRY_DSN - Sentry Data Source Name (required to enable) -# SENTRY_ENVIRONMENT - Environment name (default: from ENV or "production") -# SENTRY_RELEASE - Release version (default: VERSION) -# SENTRY_TRACES_SAMPLE_RATE - Transaction sampling rate 0.0-1.0 (default: 1.0) -# SENTRY_PROFILES_SAMPLE_RATE - Profile sampling rate 0.0-1.0 (default: 1.0) -# SENTRY_SEND_PII - Send personally identifiable information (default: true) -# SENTRY_DEBUG - Enable Sentry debug mode (default: false) +# SENTRY_DSN - Sentry Data Source Name (required to enable) +# SENTRY_ENVIRONMENT - Environment name (default: from ENV or "production") +# SENTRY_RELEASE - Release version (default: VERSION) +# SENTRY_TRACES_SAMPLE_RATE - Transaction sampling rate 0.0-1.0 (default: 1.0) +# SENTRY_PROFILES_SAMPLE_RATE - Profile sampling rate 0.0-1.0 (default: 1.0) +# SENTRY_SEND_PII - Send personally identifiable information (default: false) +# SENTRY_DEBUG - Enable Sentry debug mode (default: false) # SENTRY_TRACE_PROPAGATION_TARGETS - Regex patterns for distributed tracing (default: localhost) # +# Worker Uptime Monitor (set on the worker deployment only): +# SENTRY_WORKER_MONITOR_SLUG - Sentry Cron Monitor slug for the worker heartbeat. +# Create the monitor in Sentry UI (type: Heartbeat, +# check-in margin: 2 min, max runtime: 1 min). +# When set, the worker sends an IN_PROGRESS check-in +# every SENTRY_WORKER_HEARTBEAT_INTERVAL seconds. +# SENTRY_WORKER_HEARTBEAT_INTERVAL - Heartbeat interval in seconds (default: 60) +# # ============================================================================= -sentry_sdk.utils.MAX_STRING_LENGTH = 4096 +# Sentry truncates string values (including HTTP request bodies) to this +# limit. The default (1024) is too short for wawi payloads. 16 KB is +# enough for most shipment requests while avoiding excessive event size. +sentry_sdk.utils.MAX_STRING_LENGTH = 16384 SENTRY_DSN = config("SENTRY_DSN", default=None) SENTRY_ENVIRONMENT = config("SENTRY_ENVIRONMENT", default=config("ENV", default="production")) SENTRY_RELEASE = config("SENTRY_RELEASE", default=config("VERSION", default=None)) # Lower default sample rates for better performance (was 1.0/100%) SENTRY_TRACES_SAMPLE_RATE = config("SENTRY_TRACES_SAMPLE_RATE", default=1.0, cast=float) # 100% - capture all traces SENTRY_PROFILES_SAMPLE_RATE = config("SENTRY_PROFILES_SAMPLE_RATE", default=0.0, cast=float) # Disabled by default -SENTRY_SEND_PII = config("SENTRY_SEND_PII", default=True, cast=bool) +SENTRY_SEND_PII = config("SENTRY_SEND_PII", default=False, cast=bool) SENTRY_DEBUG = config("SENTRY_DEBUG", default=False, cast=bool) # Trace propagation targets for distributed tracing (internal services only) # Comma-separated list of regex patterns; headers are NOT sent to carrier APIs @@ -78,6 +96,22 @@ default=r"localhost", ) +_SENTRY_NOISY_ENDPOINTS = ( + "/health", + "/ready", + "/live", + "/status", + "/_health", + "/favicon.ico", + "/static/", + "/robots.txt", +) + +# Harmless exceptions filtered out before reaching Sentry. +_SENTRY_FILTERED_EXCEPTIONS = ( + asyncio.CancelledError, # ASGI probe disconnects before response completes +) + def _sentry_traces_sampler(sampling_context): """Custom traces sampler that ensures requests with x-request-id are always traced.""" @@ -87,7 +121,7 @@ def _sentry_traces_sampler(sampling_context): return 1.0 # Health checks - never trace path = wsgi_env.get("PATH_INFO", "") - if any(path.startswith(ep) for ep in ["/health", "/ready", "/live", "/status"]): + if any(path.startswith(endpoint) for endpoint in _SENTRY_NOISY_ENDPOINTS): return 0.0 return SENTRY_TRACES_SAMPLE_RATE @@ -100,6 +134,10 @@ def _sentry_before_send(event, hint): - Add custom tags - Filter out certain events """ + exc_info = hint.get("exc_info") + if exc_info and isinstance(exc_info[1], _SENTRY_FILTERED_EXCEPTIONS): + return None + # Scrub sensitive data from request bodies if "request" in event: request_data = event["request"] @@ -114,9 +152,17 @@ def _sentry_before_send(event, hint): # Scrub POST data if "data" in request_data and isinstance(request_data["data"], dict): sensitive_fields = [ - "password", "secret", "token", "api_key", "apikey", - "access_token", "refresh_token", "client_secret", - "account_number", "meter_number", "license_key", + "password", + "secret", + "token", + "api_key", + "apikey", + "access_token", + "refresh_token", + "client_secret", + "account_number", + "meter_number", + "license_key", ] for field in sensitive_fields: for key in list(request_data["data"].keys()): @@ -135,21 +181,59 @@ def _sentry_before_send_transaction(event, hint): """ transaction_name = event.get("transaction", "") - # Filter out health check and monitoring endpoints - noisy_endpoints = [ - "/health", - "/ready", - "/live", - "/_health", - "/favicon.ico", - "/static/", - "/robots.txt", - ] - - for endpoint in noisy_endpoints: + for endpoint in _SENTRY_NOISY_ENDPOINTS: if transaction_name.startswith(endpoint): return None # Drop this transaction + # SHIP2-1185 — surface per-transaction scrub timing in pod logs so we can + # correlate /v1/wawi/* latency against missing http.carrier spans. Bounded + # to the wawi prefix to avoid log flood on other endpoints. + _diag_log = transaction_name.startswith("/v1/wawi/") + _t0 = time.monotonic() if _diag_log else None + _span_count = 0 + _carrier_count = 0 + _scrubbed_bytes = 0 + + try: + for span in event.get("spans", []): + _span_count += 1 + if _diag_log and span.get("op") in ("http.carrier", "http.client"): + _carrier_count += 1 + _data = span.get("data") or {} + for _k in ( + "http.request.body", + "http.response.body", + "request.body", + "response.body", + "http.url", + ): + _v = _data.get(_k) + if isinstance(_v, str): + _scrubbed_bytes += len(_v) + span["data"] = telemetry_scrubbing.scrub_span_data(span.get("data")) + except Exception: + if _diag_log: + logging.getLogger(__name__).warning( + "sentry.before_send_transaction.raised txn=%s spans=%d carrier=%d bytes=%d", + transaction_name, + _span_count, + _carrier_count, + _scrubbed_bytes, + exc_info=True, + ) + return event + + if _diag_log: + _dt_ms = (time.monotonic() - _t0) * 1000 + logging.getLogger(__name__).info( + "sentry.before_send_transaction.ok txn=%s spans=%d carrier=%d bytes=%d dur_ms=%.1f", + transaction_name, + _span_count, + _carrier_count, + _scrubbed_bytes, + _dt_ms, + ) + return event @@ -158,8 +242,8 @@ def _sentry_before_send_transaction(event, hint): integrations = [ DjangoIntegration( transaction_style="url", # Use URL patterns for transaction names - middleware_spans=False, # Disabled for performance (request_id is tagged via set_tag, not spans) - signals_spans=False, # Disabled for performance + middleware_spans=False, # Disabled for performance (request_id is tagged via set_tag, not spans) + signals_spans=False, # Disabled for performance ), ] @@ -170,79 +254,75 @@ def _sentry_before_send_transaction(event, hint): # Try to add Redis integration if Redis is configured try: from sentry_sdk.integrations.redis import RedisIntegration + if config("REDIS_URL", default=None) or config("REDIS_HOST", default=None): integrations.append(RedisIntegration()) except ImportError: pass - # Try to add Huey integration for background tasks - try: + # Try to add Huey integration for background tasks (dev only; not installed in servicebus images) + with suppress(Exception): from sentry_sdk.integrations.huey import HueyIntegration + integrations.append(HueyIntegration()) - except ImportError: - pass # Note: karrio SDK uses urllib (not requests), so RequestsIntegration is not used here. # Carrier HTTP calls are instrumented via _urlopen_with_span() in karrio/core/utils/helpers.py # which wraps urlopen with sentry_sdk.start_span() directly. # Try to add httpx integration for async HTTP clients - try: + with suppress(Exception): from sentry_sdk.integrations.httpx import HttpxIntegration + integrations.append(HttpxIntegration()) - except Exception: - pass # httpx may not be installed # Try to add Strawberry GraphQL integration - try: + with suppress(Exception): from sentry_sdk.integrations.strawberry import StrawberryIntegration + integrations.append(StrawberryIntegration(async_execution=False)) - except Exception: - pass # strawberry integration may not be available # Parse trace propagation targets (comma-separated regex patterns) - trace_targets = [ - pattern.strip() - for pattern in SENTRY_TRACE_PROPAGATION_TARGETS.split(",") - if pattern.strip() - ] + trace_targets = [pattern.strip() for pattern in SENTRY_TRACE_PROPAGATION_TARGETS.split(",") if pattern.strip()] sentry_sdk.init( dsn=SENTRY_DSN, integrations=integrations, - + # In-app frame classification — karrio.* code is "in app", everything else is library code. + # Drives which frames are expanded by default in the Sentry UI. + in_app_include=["karrio"], # Environment and release tracking environment=SENTRY_ENVIRONMENT, release=SENTRY_RELEASE, - # Performance monitoring — use sampler for fine-grained control # Always trace requests with explicit x-request-id; skip health checks traces_sampler=_sentry_traces_sampler, # Only enable profiling if explicitly configured (disabled by default) - **({"profile_session_sample_rate": SENTRY_PROFILES_SAMPLE_RATE, "profile_lifecycle": "trace"} if SENTRY_PROFILES_SAMPLE_RATE > 0 else {}), - + **( + {"profile_session_sample_rate": SENTRY_PROFILES_SAMPLE_RATE, "profile_lifecycle": "trace"} + if SENTRY_PROFILES_SAMPLE_RATE > 0 + else {} + ), # Distributed tracing - propagate trace headers to internal services # Note: Headers are NOT sent to carrier APIs (they wouldn't understand them) trace_propagation_targets=trace_targets, - # Privacy settings send_default_pii=SENTRY_SEND_PII, - # Logging integration enable_logs=True, - # Debug mode debug=SENTRY_DEBUG, - # Event processing hooks before_send=_sentry_before_send, before_send_transaction=_sentry_before_send_transaction, - + # Request body capture — "always" ensures the full POST body is + # attached to Sentry events (needed for debugging wawi payloads). + max_request_body_size="always", # Additional options (reduced for performance) - max_breadcrumbs=50, # Enough for carrier call traces without excessive memory - attach_stacktrace=True, # Attach stack traces to messages - include_source_context=False, # Disabled for performance (avoids file I/O on capture) - include_local_variables=False, # Disabled for performance (avoids capturing all stack locals) + max_breadcrumbs=50, # Enough for carrier call traces without excessive memory + attach_stacktrace=True, # Attach stack traces to messages + include_source_context=True, # Read .py files on capture so frames show inline code context + include_local_variables=False, # Disabled for performance (avoids capturing all stack locals) ) # Set default tags that will be applied to all events @@ -250,11 +330,9 @@ def _sentry_before_send_transaction(event, hint): sentry_sdk.set_tag("framework", "django") import logging + logger = logging.getLogger(__name__) - logger.info( - f"Sentry initialized: env={SENTRY_ENVIRONMENT}, " - f"traces_sample_rate={SENTRY_TRACES_SAMPLE_RATE}" - ) + logger.info(f"Sentry initialized: env={SENTRY_ENVIRONMENT}, traces_sample_rate={SENTRY_TRACES_SAMPLE_RATE}") # OpenTelemetry Configuration @@ -272,26 +350,27 @@ def _sentry_before_send_transaction(event, hint): # Only initialize OpenTelemetry if enabled and endpoint is configured if OTEL_ENABLED and OTEL_EXPORTER_OTLP_ENDPOINT: import logging - from opentelemetry import trace, metrics - from opentelemetry.sdk.trace import TracerProvider - from opentelemetry.sdk.trace.export import BatchSpanProcessor - from opentelemetry.sdk.metrics import MeterProvider - from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader - from opentelemetry.sdk.resources import Resource, SERVICE_NAME, SERVICE_VERSION + + from opentelemetry import metrics, trace from opentelemetry.instrumentation.django import DjangoInstrumentor - from opentelemetry.instrumentation.requests import RequestsInstrumentor from opentelemetry.instrumentation.logging import LoggingInstrumentor - from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor + from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor from opentelemetry.instrumentation.redis import RedisInstrumentor - + from opentelemetry.instrumentation.requests import RequestsInstrumentor + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + from opentelemetry.sdk.resources import SERVICE_NAME, SERVICE_VERSION, Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + # Import appropriate exporter based on protocol if OTEL_EXPORTER_OTLP_PROTOCOL == "grpc": - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter else: # http/protobuf - from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter - + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + # Parse headers if provided headers = {} if OTEL_EXPORTER_OTLP_HEADERS: @@ -299,7 +378,7 @@ def _sentry_before_send_transaction(event, hint): if "=" in header_pair: key, value = header_pair.split("=", 1) headers[key.strip()] = value.strip() - + # Parse resource attributes resource_attributes = { SERVICE_NAME: OTEL_SERVICE_NAME, @@ -307,20 +386,20 @@ def _sentry_before_send_transaction(event, hint): "environment": OTEL_ENVIRONMENT, "deployment.environment": OTEL_ENVIRONMENT, } - + if OTEL_RESOURCE_ATTRIBUTES: for attr_pair in OTEL_RESOURCE_ATTRIBUTES.split(","): if "=" in attr_pair: key, value = attr_pair.split("=", 1) resource_attributes[key.strip()] = value.strip() - + # Create resource resource = Resource(attributes=resource_attributes) - + # Configure Trace Provider trace_provider = TracerProvider(resource=resource) trace.set_tracer_provider(trace_provider) - + # Configure span exporter span_exporter = OTLPSpanExporter( endpoint=OTEL_EXPORTER_OTLP_ENDPOINT, @@ -328,7 +407,7 @@ def _sentry_before_send_transaction(event, hint): ) span_processor = BatchSpanProcessor(span_exporter) trace_provider.add_span_processor(span_processor) - + # Configure Metrics Provider metric_exporter = OTLPMetricExporter( endpoint=OTEL_EXPORTER_OTLP_ENDPOINT, @@ -343,42 +422,31 @@ def _sentry_before_send_transaction(event, hint): metric_readers=[metric_reader], ) metrics.set_meter_provider(meter_provider) - + # Instrument Django DjangoInstrumentor().instrument( is_sql_commentor_enabled=True, # Add trace context to SQL queries request_hook=lambda span, request: span.set_attribute("http.client_ip", request.META.get("REMOTE_ADDR", "")), - response_hook=lambda span, request, response: span.set_attribute("http.response.size", len(response.content) if hasattr(response, 'content') else 0), + response_hook=lambda span, request, response: span.set_attribute( + "http.response.size", len(response.content) if hasattr(response, "content") else 0 + ), ) - + # Instrument other libraries RequestsInstrumentor().instrument() # HTTP client requests LoggingInstrumentor().instrument(set_logging_format=True) # Add trace context to logs - + # Instrument database if PostgreSQL is used if config("DATABASE_ENGINE", default="").endswith("postgresql"): - try: - Psycopg2Instrumentor().instrument() - except Exception: - pass # Psycopg2 might not be installed - + with suppress(Exception): + PsycopgInstrumentor().instrument() + # Instrument Redis if configured if config("REDIS_URL", default=None) or config("REDIS_HOST", default=None): - try: + with suppress(Exception): RedisInstrumentor().instrument() - except Exception: - pass # Redis might not be installed - - # Instrument Huey task queue - try: - from huey.contrib.djhuey import HUEY as huey_instance - from karrio.server.lib.otel_huey import HueyInstrumentor - instrumentor = HueyInstrumentor() - instrumentor.instrument(huey_instance) - except Exception as e: - logger = logging.getLogger(__name__) - logger.warning(f"Failed to instrument Huey: {e}") + # Huey OTel instrumentation is handled by karrio.server.huey.apps.HueyConfig.ready() # Log that OpenTelemetry is enabled logger = logging.getLogger(__name__) @@ -432,7 +500,8 @@ def _sentry_before_send_transaction(event, hint): try: import ddtrace - from ddtrace import config as dd_config, tracer, patch_all + from ddtrace import config as dd_config + from ddtrace import patch_all, tracer # Configure tracer ddtrace.config.service = DD_SERVICE @@ -454,6 +523,7 @@ def _sentry_before_send_transaction(event, hint): # Set global sample rate from ddtrace.sampler import DatadogSampler + tracer.configure(sampler=DatadogSampler(default_sample_rate=DD_TRACE_SAMPLE_RATE)) # Enable log injection @@ -471,18 +541,15 @@ def _sentry_before_send_transaction(event, hint): ) # Patch Huey for background task tracing - try: + with suppress(Exception): from ddtrace import patch + patch(huey=True) - except Exception: - pass # Huey integration may not be available in all ddtrace versions # Enable profiling if configured if DD_PROFILING_ENABLED: - try: + with suppress(ImportError): import ddtrace.profiling.auto # noqa: F401 - except ImportError: - pass # Configure DogStatsD for metrics try: @@ -504,20 +571,19 @@ def _sentry_before_send_transaction(event, hint): pass # datadog package not installed, metrics won't work import logging + logger = logging.getLogger(__name__) logger.info( - f"Datadog APM initialized: service={DD_SERVICE}, env={DD_ENV}, " - f"agent={DD_AGENT_HOST}:{DD_TRACE_AGENT_PORT}" + f"Datadog APM initialized: service={DD_SERVICE}, env={DD_ENV}, agent={DD_AGENT_HOST}:{DD_TRACE_AGENT_PORT}" ) except ImportError: import logging + logger = logging.getLogger(__name__) - logger.warning( - "Datadog tracing enabled but ddtrace package not installed. " - "Install with: pip install ddtrace" - ) + logger.warning("Datadog tracing enabled but ddtrace package not installed. Install with: pip install ddtrace") except Exception as e: import logging + logger = logging.getLogger(__name__) logger.warning(f"Failed to initialize Datadog APM: {e}") diff --git a/apps/api/karrio/server/settings/base.py b/apps/api/karrio/server/settings/base.py index 84377e0e05..4ab373e9f1 100644 --- a/apps/api/karrio/server/settings/base.py +++ b/apps/api/karrio/server/settings/base.py @@ -10,23 +10,23 @@ https://docs.djangoproject.com/en/3.0/ref/settings/ """ +import importlib import os +import sys as _sys +from datetime import timedelta +from pathlib import Path + import decouple -import importlib import dj_database_url - -from pathlib import Path -from datetime import timedelta -from django.urls import reverse_lazy from corsheaders.defaults import default_headers from django.core.management.utils import get_random_secret_key +from django.urls import reverse_lazy from django.utils.translation import gettext_lazy as _ - # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = Path(__file__).resolve().parent.parent.parent -with open(BASE_DIR / "server" / "VERSION", "r") as v: +with open(BASE_DIR / "server" / "VERSION") as v: VERSION = v.read().strip() @@ -35,11 +35,11 @@ if not config("SECRET_KEY", default=None): try: # Note: Using print here intentionally as logging isn't configured yet - print("> fallback .env.sample...") + print("> fallback .env.sample...") # noqa: T201 config = decouple.Config(decouple.RepositoryEnv(".env.sample")) except Exception as e: # Note: Using print here intentionally as logging isn't configured yet - print(f"> error: {e}") + print(f"> error: {e}") # noqa: T201 # Quick-start development settings - unsuitable for production @@ -48,7 +48,7 @@ SECRET_KEY = config("SECRET_KEY", default=get_random_secret_key()) # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = config("DEBUG_MODE", default=True, cast=bool) +DEBUG = config("DEBUG_MODE", default=False, cast=bool) # custom env WORK_DIR = config("WORK_DIR", default="") @@ -88,16 +88,14 @@ SECURE_SSL_REDIRECT = True # Exempt health check endpoint from HTTPS redirect for Kubernetes probes - SECURE_REDIRECT_EXEMPT = [r'^status/$'] + SECURE_REDIRECT_EXEMPT = [r"^status/$"] SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") SESSION_COOKIE_SECURE = True SECURE_HSTS_SECONDS = 1 SECURE_HSTS_INCLUDE_SUBDOMAINS = True CSRF_COOKIE_SECURE = True SECURE_HSTS_PRELOAD = True - CSRF_TRUSTED_ORIGINS = config("CSRF_TRUSTED_ORIGINS", default="https://*").split( - "," - ) + CSRF_TRUSTED_ORIGINS = config("CSRF_TRUSTED_ORIGINS", default="https://*").split(",") # karrio packages settings @@ -171,18 +169,12 @@ KARRIO_APPS = [cfg["app"] for cfg in KARRIO_CONF] KARRIO_URLS = [cfg["urls"] for cfg in KARRIO_CONF if "urls" in cfg] -ALLOW_ADMIN_APPROVED_SIGNUP = config( - "ALLOW_ADMIN_APPROVED_SIGNUP", default=False, cast=bool -) -ALLOW_SIGNUP = ( - config("ALLOW_SIGNUP", default=False, cast=bool) or ALLOW_ADMIN_APPROVED_SIGNUP -) +ALLOW_ADMIN_APPROVED_SIGNUP = config("ALLOW_ADMIN_APPROVED_SIGNUP", default=False, cast=bool) +ALLOW_SIGNUP = config("ALLOW_SIGNUP", default=False, cast=bool) or ALLOW_ADMIN_APPROVED_SIGNUP MULTI_ORGANIZATIONS = ( importlib.util.find_spec("karrio.server.orgs") is not None # type:ignore ) -ALLOW_MULTI_ACCOUNT = config( - "ALLOW_MULTI_ACCOUNT", default=MULTI_ORGANIZATIONS, cast=bool -) +ALLOW_MULTI_ACCOUNT = config("ALLOW_MULTI_ACCOUNT", default=MULTI_ORGANIZATIONS, cast=bool) ADMIN_DASHBOARD = importlib.util.find_spec( # type:ignore "karrio.server.admin" ) is not None and config("ADMIN_DASHBOARD", default=True, cast=bool) @@ -191,7 +183,7 @@ ) APPS_MANAGEMENT = ( importlib.util.find_spec("karrio.server.apps") is not None # type:ignore -) and config("APPS_MANAGEMENT", default=(True if DEBUG else False), cast=bool) +) and config("APPS_MANAGEMENT", default=bool(DEBUG), cast=bool) DOCUMENTS_MANAGEMENT = ( importlib.util.find_spec("karrio.server.documents") is not None # type:ignore ) @@ -208,9 +200,13 @@ "karrio.server.audit" ) is not None and config("AUDIT_LOGGING", default=True, cast=bool) PERSIST_SDK_TRACING = config("PERSIST_SDK_TRACING", default=True, cast=bool) +# Emit the built-in shipment.purchased webhook from the core events signal. +# Default on; set False once the bridge fan-out owns the purchased webhook +# (see modules/bridge consumer) to avoid double-delivery. +BUILT_IN_WEBHOOKS = config("BUILT_IN_WEBHOOKS", default=True, cast=bool) WORKFLOW_MANAGEMENT = ( importlib.util.find_spec("karrio.server.automation") is not None # type:ignore -) and config("WORKFLOW_MANAGEMENT", default=(True if DEBUG else False), cast=bool) +) and config("WORKFLOW_MANAGEMENT", default=bool(DEBUG), cast=bool) SHIPPING_RULES = ( importlib.util.find_spec("karrio.server.automation") is not None # type:ignore ) @@ -219,7 +215,7 @@ ) ADVANCED_ANALYTICS = ( importlib.util.find_spec("karrio.server.analytics") is not None # type:ignore -) and config("ADVANCED_ANALYTICS", default=(True if DEBUG else False), cast=bool) +) and config("ADVANCED_ANALYTICS", default=bool(DEBUG), cast=bool) # Feature flags @@ -288,11 +284,12 @@ *KARRIO_APPS, *BASE_APPS, "rest_framework", + "rest_framework_simplejwt.token_blacklist", "django_email_verification", "rest_framework_tracking", "drf_spectacular", "constance.backends.database", - "huey.contrib.djhuey", + # "huey.contrib.djhuey" — moved to karrio.server.settings.huey (auto-discovered) "corsheaders", "oauth2_provider", *OTP_APPS, @@ -314,6 +311,10 @@ "karrio.server.core.middleware.SessionContext", ] +# Allow large GraphQL mutations (rate sheet upsert sends all zones + services +# + service_rates in one request; UPS sheets can have 5000+ rows). +DATA_UPLOAD_MAX_MEMORY_SIZE = 20 * 1024 * 1024 # 20 MB (Django default: 2.5 MB) + TEMPLATES = [ { @@ -348,21 +349,38 @@ _DB_NAME = config("DATABASE_NAME", default="db") _DB_ENGINE = "sqlite3" if "sqlite3" in _DB_NAME else "postgresql" -CONN_MAX_AGE = config("DATABASE_CONN_MAX_AGE", default=0, cast=int) -DB_ENGINE = "django.db.backends.{}".format( - config("DATABASE_ENGINE", default=_DB_ENGINE) -) +CONN_MAX_AGE = config("DATABASE_CONN_MAX_AGE", default=60, cast=int) +# Validate connection health before reuse; transparently reconnects if the DB +# dropped an idle connection (e.g. Azure PostgreSQL load-balancer idle timeout). +# Requires CONN_MAX_AGE > 0 to have effect. Added in Django 4.1. +CONN_HEALTH_CHECKS = config("DATABASE_CONN_HEALTH_CHECKS", default=True, cast=bool) +DB_ENGINE = "django.db.backends.{}".format(config("DATABASE_ENGINE", default=_DB_ENGINE)) DB_NAME = ( os.path.join(WORK_DIR, f"{_DB_NAME}{'' if '.sqlite3' in _DB_NAME else '.sqlite3'}") if "sqlite3" in DB_ENGINE else _DB_NAME ) +# Connection pool (psycopg 3) — required under ASGI. Django's CONN_MAX_AGE and +# CONN_HEALTH_CHECKS fire on the request_started/request_finished signals on the +# main async coroutine, but sync views run under sync_to_async in a +# ThreadPoolExecutor whose thread-local connection cache is never aged out or +# health-checked. The process-wide pool serves both paths and uses pool-level +# health checks on checkout to fail fast on sockets the network silently dropped. +# See GH #508. +DB_POOL_ENABLED = config("DATABASE_POOL_ENABLED", default="postgresql" in DB_ENGINE, cast=bool) +DB_POOL_MIN_SIZE = config("DATABASE_POOL_MIN_SIZE", default=4, cast=int) +DB_POOL_MAX_SIZE = config("DATABASE_POOL_MAX_SIZE", default=20, cast=int) +DB_POOL_MAX_LIFETIME = config("DATABASE_POOL_MAX_LIFETIME", default=3600, cast=int) +DB_POOL_MAX_IDLE = config("DATABASE_POOL_MAX_IDLE", default=600, cast=int) +DB_POOL_TIMEOUT = config("DATABASE_POOL_TIMEOUT", default=10, cast=int) + DATABASES = { "default": { "NAME": DB_NAME, "ENGINE": DB_ENGINE, "CONN_MAX_AGE": CONN_MAX_AGE, + "CONN_HEALTH_CHECKS": CONN_HEALTH_CHECKS, "PORT": config("DATABASE_PORT", default="5432"), "HOST": config("DATABASE_HOST", default="localhost"), "USER": config("DATABASE_USERNAME", default="postgres"), @@ -376,12 +394,68 @@ TEST_RUNNER = "karrio.server.test_runner.KarrioTestRunner" +# Password hashing. PBKDF2 preferred; Django auto-upgrades legacy hashes to it +# on successful login. MD5 kept only as a trailing fallback so accounts hashed +# while prod ran MD5-only (#926) still verify and migrate forward — drop it once +# they have. +PASSWORD_HASHERS = [ + "django.contrib.auth.hashers.PBKDF2PasswordHasher", + "django.contrib.auth.hashers.PBKDF2SHA1PasswordHasher", + "django.contrib.auth.hashers.Argon2PasswordHasher", + "django.contrib.auth.hashers.BCryptSHA256PasswordHasher", + "django.contrib.auth.hashers.ScryptPasswordHasher", + "django.contrib.auth.hashers.MD5PasswordHasher", # legacy fallback — remove after #926 migration window +] + +# Test suite only: MD5 first for speed. Gated on an exact argv token so it can +# never match a prod process — the old `"karrio" in argv[0]` matched everything +# under /karrio/venv, which was the #926 root cause. +if "test" in _sys.argv: + PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher", *PASSWORD_HASHERS] + if config("DATABASE_URL", default=None): db_from_env = dj_database_url.config( - conn_max_age=config("DATABASE_CONN_MAX_AGE", default=0, cast=int) + conn_max_age=config("DATABASE_CONN_MAX_AGE", default=60, cast=int), + conn_health_checks=config("DATABASE_CONN_HEALTH_CHECKS", default=True, cast=bool), ) DATABASES["default"].update(db_from_env) +# Postgres-only connection hardening (libpq TCP keepalives + psycopg3 pool). +# Keepalives make the OS fail stale sockets fast when the network path silently +# drops TCP (e.g. Azure Private Endpoint ~4 min TCP idle timeout) so psycopg +# reconnects instead of reusing a dead socket. The pool bounds backend fan-out +# per process and health-checks connections on checkout. See GH #508. +if "postgresql" in DATABASES["default"]["ENGINE"]: + _db_options = DATABASES["default"].setdefault("OPTIONS", {}) + _db_options.setdefault("keepalives", 1) + _db_options.setdefault("keepalives_idle", 60) + _db_options.setdefault("keepalives_interval", 10) + _db_options.setdefault("keepalives_count", 3) + + if DB_POOL_ENABLED: + try: + import psycopg_pool # noqa: F401 + + # Django's postgresql backend already passes + # check=ConnectionPool.check_connection when constructing the pool; + # supplying it again here raises TypeError (multiple values for 'check'). + _db_options["pool"] = { + "min_size": DB_POOL_MIN_SIZE, + "max_size": DB_POOL_MAX_SIZE, + "max_lifetime": DB_POOL_MAX_LIFETIME, + "max_idle": DB_POOL_MAX_IDLE, + "timeout": DB_POOL_TIMEOUT, + } + # Django requires persistent-connection cache disabled when a pool + # is configured; the pool owns connection lifetime now. + DATABASES["default"]["CONN_MAX_AGE"] = 0 + DATABASES["default"]["CONN_HEALTH_CHECKS"] = False + except ImportError: + # psycopg3 / psycopg-pool not installed (e.g. psycopg2 fallback). + # Keepalives alone still cover the ASGI stale-socket case, just + # without per-process pool bounds. + pass + # Configure workers database for SQLite storage when Redis is not available if not config("REDIS_URL", default=None) and not config("REDIS_HOST", default=None): _WORKER_DB_DIR = config("WORKER_DB_DIR", default=WORK_DIR) @@ -448,9 +522,7 @@ # https://docs.djangoproject.com/en/3.0/howto/static-files/ STATIC_HOST = config("CDN_STATIC_HOST", default="") if not DEBUG else "" -STATIC_URL = STATIC_HOST + config( - "STATIC_URL", default=f"{BASE_PATH}/static/".replace("//", "/") -) +STATIC_URL = STATIC_HOST + config("STATIC_URL", default=f"{BASE_PATH}/static/".replace("//", "/")) STATIC_ROOT = config("STATIC_ROOT_DIR", default=(BASE_DIR / "server" / "staticfiles")) STATICFILES_DIRS = [ @@ -512,14 +584,10 @@ # JWT config SIMPLE_JWT = { - "ACCESS_TOKEN_LIFETIME": timedelta( - config("JWT_ACCESS_EXPIRY", default="30", cast=int) - ), - "REFRESH_TOKEN_LIFETIME": timedelta( - config("JWT_REFRESH_EXPIRY", default="3", cast=int) - ), + "ACCESS_TOKEN_LIFETIME": timedelta(config("JWT_ACCESS_EXPIRY", default="30", cast=int)), + "REFRESH_TOKEN_LIFETIME": timedelta(config("JWT_REFRESH_EXPIRY", default="3", cast=int)), "ROTATE_REFRESH_TOKENS": False, - "BLACKLIST_AFTER_ROTATION": False, + "BLACKLIST_AFTER_ROTATION": True, "UPDATE_LAST_LOGIN": True, "ALGORITHM": "HS256", "SIGNING_KEY": SECRET_KEY, @@ -541,10 +609,9 @@ # JWT Cookie settings for HTTP-only cookie authentication JWT_AUTH_COOKIE = config("JWT_AUTH_COOKIE", default="karrio_access_token") JWT_REFRESH_COOKIE = config("JWT_REFRESH_COOKIE", default="karrio_refresh_token") -JWT_AUTH_COOKIE_SECURE = config( - "JWT_AUTH_COOKIE_SECURE", default=USE_HTTPS, cast=bool -) +JWT_AUTH_COOKIE_SECURE = config("JWT_AUTH_COOKIE_SECURE", default=USE_HTTPS, cast=bool) JWT_AUTH_COOKIE_SAMESITE = config("JWT_AUTH_COOKIE_SAMESITE", default="Lax") +JWT_AUTH_COOKIE_DOMAIN = config("JWT_AUTH_COOKIE_DOMAIN", default=None) JWT_AUTH_COOKIE_PATH = config("JWT_AUTH_COOKIE_PATH", default="/") # OAuth2 config @@ -582,9 +649,7 @@ "OAUTH2_TOKEN_URL": "/oauth/token/", "OAUTH2_REFRESH_URL": None, "OAUTH2_SCOPES": OAUTH2_PROVIDER["SCOPES"], - "AUTHENTICATION_WHITELIST": [ - _ for _ in AUTHENTICATION_CLASSES if "Session" not in _ - ], + "AUTHENTICATION_WHITELIST": [_ for _ in AUTHENTICATION_CLASSES if "Session" not in _], "POSTPROCESSING_HOOKS": [ "karrio.server.openapi.custom_postprocessing_hook", ], @@ -656,6 +721,9 @@ "handlers": ["console"], "level": "INFO", }, + "loggers": { + "azure.servicebus._pyamqp": {"level": "WARNING"}, + }, } else: # Traditional Django logging configuration @@ -711,11 +779,10 @@ }, } - # Initialize Loguru if enabled if USE_LOGURU: try: - from karrio.server.core.logging import setup_django_loguru, logger + from karrio.server.core.logging import setup_django_loguru setup_django_loguru( level=LOG_LEVEL, @@ -725,6 +792,6 @@ ) except ImportError as e: # Note: Using print here as Loguru failed to load - print(f"Warning: Failed to initialize Loguru: {e}") - print("Falling back to standard Django logging") + print(f"Warning: Failed to initialize Loguru: {e}") # noqa: T201 + print("Falling back to standard Django logging") # noqa: T201 USE_LOGURU = False diff --git a/apps/api/karrio/server/settings/cache.py b/apps/api/karrio/server/settings/cache.py index c593f7b2ab..8809e79d95 100644 --- a/apps/api/karrio/server/settings/cache.py +++ b/apps/api/karrio/server/settings/cache.py @@ -1,11 +1,11 @@ # type: ignore -import sys +# ruff: noqa: F403, F405 import socket +import sys + from decouple import config +from karrio.server.settings.apm import HEALTH_CHECK_CHECKS from karrio.server.settings.base import * -from karrio.server.settings.apm import HEALTH_CHECK_APPS, HEALTH_CHECK_CHECKS -from karrio.server.core.logging import logger - CACHE_TTL = 60 * 15 @@ -30,8 +30,7 @@ # Parse REDIS_URL or construct from individual parameters if REDIS_URL is not None: - from urllib.parse import urlparse, urlunparse - import re + from urllib.parse import urlparse parsed = urlparse(REDIS_URL) @@ -61,18 +60,14 @@ REDIS_DB = config("REDIS_CACHE_DB", default="0") REDIS_AUTH = f"{REDIS_USERNAME}:{REDIS_PASSWORD}@" if REDIS_PASSWORD else "" REDIS_SCHEME = "rediss" if REDIS_SSL else "redis" - REDIS_CONNECTION_URL = ( - f'{REDIS_SCHEME}://{REDIS_AUTH}{REDIS_HOST}:{REDIS_PORT or "6379"}/{REDIS_DB}' - ) + REDIS_CONNECTION_URL = f"{REDIS_SCHEME}://{REDIS_AUTH}{REDIS_HOST}:{REDIS_PORT or '6379'}/{REDIS_DB}" # Configure Django cache if Redis is available and not in worker mode if REDIS_HOST is not None and not SKIP_DEFAULT_CACHE: # Configure connection pool with max_connections to prevent exhaustion # Default: 50 connections per process (2 Gunicorn workers = 100 total) # Azure Redis Basic: 256 max connections total - REDIS_CACHE_MAX_CONNECTIONS = config( - "REDIS_CACHE_MAX_CONNECTIONS", default=50, cast=int - ) + REDIS_CACHE_MAX_CONNECTIONS = config("REDIS_CACHE_MAX_CONNECTIONS", default=50, cast=int) pool_kwargs = {"max_connections": REDIS_CACHE_MAX_CONNECTIONS} if REDIS_SSL: @@ -84,11 +79,7 @@ "LOCATION": REDIS_CONNECTION_URL, "OPTIONS": { "CLIENT_CLASS": "django_redis.client.DefaultClient", - **( - {"CONNECTION_POOL_KWARGS": {"ssl_cert_reqs": None}} - if REDIS_SSL - else {} - ), + **({"CONNECTION_POOL_KWARGS": {"ssl_cert_reqs": None}} if REDIS_SSL else {}), }, "KEY_PREFIX": REDIS_PREFIX, } @@ -103,14 +94,12 @@ import redis.asyncio as aioredis _redis_check_client = aioredis.Redis.from_url(REDIS_CONNECTION_URL) - HEALTH_CHECK_CHECKS.append( - ("health_check.contrib.redis.Redis", {"client": _redis_check_client}) - ) + HEALTH_CHECK_CHECKS.append(("health_check.contrib.redis.Redis", {"client": _redis_check_client})) - print(f"Redis cache connection initialized") + print("Redis cache connection initialized") # noqa: T201 elif SKIP_DEFAULT_CACHE: - print( + print( # noqa: T201 "Skipping default Redis cache configuration (worker mode - only HUEY Redis needed)" ) else: diff --git a/apps/api/karrio/server/settings/constance.py b/apps/api/karrio/server/settings/constance.py index bb2af7e5f9..7f7f7123dd 100644 --- a/apps/api/karrio/server/settings/constance.py +++ b/apps/api/karrio/server/settings/constance.py @@ -1,17 +1,18 @@ """Dynamic configuration editable on runtime powered by django-constance.""" -from decouple import config +import importlib.util + import karrio.references as ref import karrio.server.settings.base as base +from decouple import config from karrio.server.settings.email import ( - EMAIL_USE_TLS, - EMAIL_HOST_USER, - EMAIL_HOST_PASSWORD, + EMAIL_FROM_ADDRESS, EMAIL_HOST, + EMAIL_HOST_PASSWORD, + EMAIL_HOST_USER, EMAIL_PORT, - EMAIL_FROM_ADDRESS, + EMAIL_USE_TLS, ) -import importlib.util CONSTANCE_BACKEND = "constance.backends.database.DatabaseBackend" CONSTANCE_DATABASE_PREFIX = "constance:core:" @@ -19,9 +20,7 @@ DATA_ARCHIVING_SCHEDULE = config("DATA_ARCHIVING_SCHEDULE", default=168, cast=int) GOOGLE_CLOUD_API_KEY = config("GOOGLE_CLOUD_API_KEY", default="") -CANADAPOST_ADDRESS_COMPLETE_API_KEY = config( - "CANADAPOST_ADDRESS_COMPLETE_API_KEY", default="" -) +CANADAPOST_ADDRESS_COMPLETE_API_KEY = config("CANADAPOST_ADDRESS_COMPLETE_API_KEY", default="") # data retention env in days ORDER_DATA_RETENTION = config("ORDER_DATA_RETENTION", default=183, cast=int) @@ -32,11 +31,171 @@ # background tracking max active age TRACKER_MAX_ACTIVE_DAYS = config("TRACKER_MAX_ACTIVE_DAYS", default=90, cast=int) -# registry config -ENABLE_ALL_PLUGINS_BY_DEFAULT = config( - "ENABLE_ALL_PLUGINS_BY_DEFAULT", default=True if base.DEBUG else False, cast=bool +# paperless trade (ETD): the system-wide default commercial-invoice template +# BODY (HTML/Jinja), rendered inline when a merchant enables paperless_trade +# without supplying their own document or per-shipment invoice_template (e.g. the +# post_upload flow on GLS). It is the template *body*, not a slug — so a default +# always resolves without depending on a tenant-owned DocumentTemplate row. The +# render context exposes `shipment`, `line_items`, `carrier`, `orders`, plus +# `units`/`utils`/`lib`. Editable at runtime via Constance; clear it to opt out +# (the post_upload job then records an actionable `skipped` reason — D15). +DEFAULT_COMMERCIAL_INVOICE_TEMPLATE = """ + +
+ +|
+ Exporter / From
+ {{ shipment.shipper.company_name or shipment.shipper.person_name or '-' }}
+ {% if shipment.shipper.company_name and shipment.shipper.person_name %}{{ shipment.shipper.person_name }} {% endif %}
+ {{ shipment.shipper.address_line1 }}{% if shipment.shipper.address_line2 %}, {{ shipment.shipper.address_line2 }}{% endif %}
+ {{ shipment.shipper.postal_code }} {{ shipment.shipper.city }}{% if shipment.shipper.state_code %}, {{ shipment.shipper.state_code }}{% endif %}
+ {{ shipment.shipper.country_code }}
+ {% if shipment.shipper.phone_number %}Tel: {{ shipment.shipper.phone_number }} {% endif %}
+ {% if shipment.shipper.email %}Email: {{ shipment.shipper.email }} {% endif %}
+ {% if shipment.shipper.federal_tax_id %}Tax ID: {{ shipment.shipper.federal_tax_id }} {% endif %}
+ {% if co.vat_registration_number %}VAT No: {{ co.vat_registration_number }} {% endif %}
+ {% if co.eori_number %}EORI No: {{ co.eori_number }} {% endif %}
+ |
+
+ Consignee / Ship To
+ {{ shipment.recipient.company_name or shipment.recipient.person_name or '-' }}
+ {% if shipment.recipient.company_name and shipment.recipient.person_name %}{{ shipment.recipient.person_name }} {% endif %}
+ {{ shipment.recipient.address_line1 }}{% if shipment.recipient.address_line2 %}, {{ shipment.recipient.address_line2 }}{% endif %}
+ {{ shipment.recipient.postal_code }} {{ shipment.recipient.city }}{% if shipment.recipient.state_code %}, {{ shipment.recipient.state_code }}{% endif %}
+ {{ shipment.recipient.country_code }}
+ {% if shipment.recipient.phone_number %}Tel: {{ shipment.recipient.phone_number }} {% endif %}
+ {% if shipment.recipient.email %}Email: {{ shipment.recipient.email }} {% endif %}
+ {% if shipment.recipient.federal_tax_id %}Tax ID / VAT No: {{ shipment.recipient.federal_tax_id }} {% endif %}
+ |
+
| # | +Description of Goods | +HS Code | +Origin | +Qty | +Unit Value | +Total Value | +
|---|---|---|---|---|---|---|
| {{ loop.index }} | +{{ it.description or it.title or '-' }}{% if it.sku %} SKU: {{ it.sku }} {% endif %} |
+ {{ it.hs_code or '-' }} | +{{ it.origin_country or shipment.shipper.country_code or '-' }} | +{{ q|int }} | +{{ "%.2f"|format(u) }} {{ it.value_currency or currency }} | +{{ "%.2f"|format(q * u) }} {{ it.value_currency or currency }} | +
| No commodities declared | ||||||
| Subtotal | {{ "%.2f"|format(ns.sub) }} {{ currency }} |
| Declared Value | {{ "%.2f"|format(c.duty.declared_value|float) }} {{ c.duty.currency or currency }} |
| Total | {{ "%.2f"|format(ns.sub) }} {{ currency }} |
| Currency | {{ currency or '-' }} |
| Total Packages | {{ (shipment.parcels or [])|length or 1 }} |
| Total Weight | {{ "%.2f"|format(ns.wt) }} {{ ns.wu }} |
location_error
+ Parcel format code. The available formats depend on the product/service/country combination.
Allowed values:
B - BoxableN - Non boxableL - LargeXL - Extra LargeNote: For e-PAQ Select (EPAQSCT) and e-PAQ Elite (EPAQELT), format may be omitted unless instructed otherwise.
Product code identifying the shipping product.
Allowed values:
EPAQSTD - e-PAQ StandardEPAQPLS - e-PAQ PlusEPAQSCT - e-PAQ SelectEPAQELT - e-PAQ EliteEPAQGO - e-PAQ GOReturn-specific products (used only with return label functionality):
EPAQRETDOM - e-PAQ Domestic ReturnsEPAQRETINT - e-PAQ International ReturnsRETDOM - Domestic Returns (deprecated)RETINT - International Returns (deprecated)Service code defining the service level for the shipment.
Allowed values:
CUP - Customs unpaid / Customs paid at destinationCPPR - Customs prepaid by retailerCPPS - Customs prepaid by shopperReturn-specific services (used only with return shipments):
RETPP - Prepaid returnRETPAP - Partially paid returnList of additional option codes for the shipment. Multiple options can be combined.
Allowed values:
PM - Printed MatterECO - EconomyPOX - Post office boxPD - Personal DeliverySIG - SignaturePUDO - Pick-up / Drop-offMBX - Mailbox PlusDG - Dangerous GoodsSAT - Saturday DeliveryDA - Direct AccessSGF - Smartgate FlexSGD - Smartgate DirectOPT1 - Option1OPT2 - Option2List of additional option codes for the shipment. Multiple options can be combined.
Allowed values:
PM - Printed MatterECO - EconomyPOX - Post office boxPD - Personal DeliverySIG - SignaturePUDO - Pick-up / Drop-offMBX - Mailbox PlusDG - Dangerous GoodsSAT - Saturday DeliveryDA - Direct AccessSGF - Smartgate FlexSGD - Smartgate DirectOPT1 - Option1OPT2 - Option2Insurance code specifying the coverage level for the shipment.
Allowed values:
EL45 - Enhanced liability up to 45 EUREL150 - Enhanced liability up to 150 EUREL500 - Enhanced liability up to 500 EURNote: For product EPAQPLS (e-PAQ Plus), only EL45 and EL150 are available. For all other products, EL45, EL150, and EL500 are available.
Return label product type defining the return shipping product.
Allowed values:
EPAQRETDOM - e-PAQ Domestic ReturnsEPAQRETINT - e-PAQ International ReturnsRETDOM - Domestic Returns (deprecated)RETINT - International Returns (deprecated)Payment method for the return label, defining who bears the return shipping cost.
Allowed values:
RETPP - Prepaid return (seller/retailer pays the return shipping cost)RETPAP - Partially paid return (receiver/buyer pays the return shipping cost)Parcel format code. The available formats depend on the + product/service/country combination.
Allowed values:
+B - BoxableN - Non boxableL
+ - LargeXL - Extra LargeNote:
+ For e-PAQ Select (EPAQSCT) and e-PAQ Elite (EPAQELT),
+ format may be omitted unless instructed otherwise.
Product code identifying the shipping product.
Allowed + values:
EPAQSTD - e-PAQ StandardEPAQPLS
+ - e-PAQ PlusEPAQSCT - e-PAQ SelectEPAQELT
+ - e-PAQ EliteEPAQGO - e-PAQ GOReturn-specific + products (used only with return label functionality):
EPAQRETDOM
+ - e-PAQ Domestic ReturnsEPAQRETINT - e-PAQ International
+ ReturnsRETDOM - Domestic Returns (deprecated)RETINT
+ - International Returns (deprecated)Service code defining the service level for the shipment.
+Allowed values:
CUP -
+ Customs unpaid / Customs paid at destinationCPPR
+ - Customs prepaid by retailerCPPS - Customs prepaid
+ by shopperReturn-specific services (used only with + return shipments):
RETPP - Prepaid
+ returnRETPAP - Partially paid returnList of additional option codes for the shipment. Multiple + options can be combined.
Allowed values:
PM
+ - Printed MatterECO - EconomyPOX
+ - Post office boxPD - Personal DeliverySIG
+ - SignaturePUDO - Pick-up / Drop-offMBX
+ - Mailbox PlusDG - Dangerous GoodsSAT
+ - Saturday DeliveryDA - Direct AccessSGF
+ - Smartgate FlexSGD - Smartgate DirectOPT1
+ - Option1OPT2 - Option2List of additional option codes for the shipment. Multiple + options can be combined.
Allowed values:
+PM - Printed MatterECO
+ - EconomyPOX - Post office boxPD
+ - Personal DeliverySIG - SignaturePUDO
+ - Pick-up / Drop-offMBX - Mailbox PlusDG
+ - Dangerous GoodsSAT - Saturday DeliveryDA
+ - Direct AccessSGF - Smartgate FlexSGD
+ - Smartgate DirectOPT1 - Option1OPT2
+ - Option2Insurance code specifying the coverage level for the shipment.
+Allowed values:
EL45 -
+ Enhanced liability up to 45 EUREL150 - Enhanced
+ liability up to 150 EUREL500 - Enhanced liability
+ up to 500 EURNote: For product EPAQPLS
+ (e-PAQ Plus), only EL45 and EL150 are available.
+ For all other products, EL45, EL150, and EL500
+ are available.
Return label product type defining the return shipping product.
+Allowed values:
EPAQRETDOM
+ - e-PAQ Domestic ReturnsEPAQRETINT - e-PAQ International
+ ReturnsRETDOM - Domestic Returns (deprecated)RETINT
+ - International Returns (deprecated)Payment method for the return label, defining who bears + the return shipping cost.
Allowed values:
+RETPP - Prepaid return (seller/retailer pays the
+ return shipping cost)RETPAP - Partially paid
+ return (receiver/buyer pays the return shipping cost)