diff --git a/CLAUDE.md b/CLAUDE.md index c4bc08a8..09e2fd19 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,10 +8,6 @@ > > Read those first. The guidance below only adds **repo-specific context** -- it does not override anything in the root. -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - ## Project Overview Ledger Sync is a self-hosted personal finance dashboard that turns Excel bank statements into 24 pages of analytics -- spending breakdowns, investment tracking, tax planning, cash flow visualization, AI-powered chat, and more. Supports multi-currency display with live exchange rates. Monorepo: Python FastAPI backend + React TypeScript frontend. @@ -94,14 +90,14 @@ Layered architecture: ### Frontend (`frontend/src/`) -- **`pages/`** - 25 page components (24 feature pages + `MorePage` for phone grid nav), all lazy-loaded via `React.lazy` for code splitting. Pages import directly (no barrel re-export). **Structure convention**: multi-file pages use kebab-case directories (`bill-calendar/`, `year-in-review/`, `tax-planning/`, `trends-forecasts/`, `comparison/`, `goals/`, `income-expense-flow/`, `settings/`, `subscription-tracker/`) -- each containing `PageName.tsx` + `use.ts` + `types.ts` + `*utils.ts` + `components/` subfolder. Single-file pages use PascalCase (`DashboardPage.tsx`, `BudgetPage.tsx`, etc.). Settings uses `sections/` (instead of `components/`) because "section" is the domain term. `settings/sectionPrimitives.tsx` provides shared Section/FieldLabel/FieldHint primitives. See [`docs/HANDBOOK.md`](docs/HANDBOOK.md) for the full card-by-card walkthrough of every page + setting (formula + data source per metric), or [`docs/PAGES.md`](docs/PAGES.md) for the shorter data-focused catalog. +- **`pages/`** - 25 page components (24 feature pages + `MorePage` for phone grid nav), all lazy-loaded via `React.lazy` for code splitting. Pages import directly (no barrel re-export). **Structure convention**: multi-file pages use kebab-case directories (`bill-calendar/`, `year-in-review/`, `tax-planning/`, `trends-forecasts/`, `comparison/`, `goals/`, `income-expense-flow/`, `settings/`, `subscription-tracker/`, `budget/`) -- each containing `PageName.tsx` + `use.ts` + `types.ts` + `*utils.ts` + `components/` subfolder. Single-file pages use PascalCase (`DashboardPage.tsx`, `TransactionsPage.tsx`, etc.). Settings uses `sections/` (instead of `components/`) because "section" is the domain term. `settings/sectionPrimitives.tsx` provides shared Section/FieldLabel/FieldHint primitives. See [`docs/HANDBOOK.md`](docs/HANDBOOK.md) for the full card-by-card walkthrough of every page + setting (formula + data source per metric), or [`docs/PAGES.md`](docs/PAGES.md) for the shorter data-focused catalog. - **`components/`** - Organized by domain: `analytics/` (chart components including `CategoryBreakdown` for shared category treemaps), `chat/` (ChatWidget, ChatPanel, ChatMessage, useChat hook for AI chatbot), `layout/` (AppLayout, Sidebar with ProfileModal), `shared/` (reusable components like MetricCard, ProgressBar, Sparkline, AnalyticsTimeFilter, ProtectedRoute, ProfileModal, ChunkErrorBoundary, EmptyState, LoadingSkeleton, QuickInsights), `transactions/`, `upload/`, `ui/` (base primitives including ChartContainer, PageContainer, PageHeader, Spinner, DataTable, ConfirmDialog, CollapsibleSection, Money, plus chartDefaults helpers `referenceLine`/`currencyTooltipFormatter`). - **`hooks/`** - Custom React hooks. `useAnalyticsTimeFilter` encapsulates time-filter state (view mode, date range, FY) shared across all analytics pages. `useChartDimensions` provides responsive chart sizing. `hooks/api/` contains TanStack Query hooks for API calls, configured with `staleTime: Infinity` and `gcTime: 1 hour`. - **`services/api/`** - Axios-based API client. Axios interceptor auto-attaches JWT `Authorization` header. `aiConfig.ts` handles AI provider configuration CRUD. - **`store/`** - Zustand stores: `authStore` (JWT tokens with persist middleware), `accountStore`, `budgetStore`, `investmentAccountStore`, `preferencesStore`. - **`types/`** - Shared TypeScript type definitions. `salary.ts` defines `SalaryComponents`, `RsuGrant`, `GrowthAssumptions`, and `ProjectedFYBreakdown` interfaces. - **`constants/`** - Colors, animations, chart configuration tokens. `columns.ts` defines flexible column name mappings (`COLUMN_MAPPINGS`), required columns, and valid transaction types for the client-side file parser. -- **`lib/`** - Utility functions: formatters, date utils, tax calculator, export helpers. `fileParser.ts` handles client-side Excel/CSV parsing (lazy-loads SheetJS, computes SHA-256 hash via `crypto.subtle`, maps columns, validates rows). `projectionCalculator.ts` provides pure functions (`projectFiscalYear`, `projectMultipleYears`, `getRsuVestingsByFY`) for multi-year salary/tax projections with full TDD test coverage in `__tests__/projectionCalculator.test.ts`. `chatAdapters.ts` provides streaming request builders and SSE parsers for OpenAI/Anthropic/Bedrock. `chatContext.ts` builds a compressed financial context prompt from existing V2 analytics endpoints. `tax-config/` holds per-FY tax rules (slabs, surcharge, 87A rebate, standard deduction, cess, professional tax) with `getTaxConfig(fyStartYear)` lookup — adding a new Budget is a one-file edit with newest-first fallback for unknown FYs. +- **`lib/`** - Utility functions: formatters, date utils, tax calculator, export helpers. `fileParser.ts` handles client-side Excel/CSV parsing (lazy-loads SheetJS, computes SHA-256 hash via `crypto.subtle`, maps columns, validates rows). `projectionCalculator.ts` provides pure functions (`projectFiscalYear`, `projectMultipleYears`, `getRsuVestingsByFY`) for multi-year salary/tax projections with full TDD test coverage in `__tests__/projectionCalculator.test.ts`. `chatAdapters.ts` provides streaming request builders and SSE parsers for OpenAI/Anthropic/Bedrock. `chatContext.ts` builds a compressed financial context prompt from existing V2 analytics endpoints. `tax-config/` holds per-FY tax rules (slabs, surcharge, 87A rebate, standard deduction, cess, professional tax) with `getTaxConfig(fyStartYear)` lookup -- adding a new Budget is a one-file edit with newest-first fallback for unknown FYs. ### Key Patterns @@ -115,12 +111,12 @@ Layered architecture: - **Database**: SQLite for development (`./ledger_sync.db`), Neon PostgreSQL 17 in production (Singapore region, free tier, 0.5 GB). Schema managed by Alembic migrations. Database auto-initializes on app startup via `init_db()`. SQLite connections apply performance PRAGMAs (WAL mode, 64MB cache, NORMAL sync). PostgreSQL pool is env-configurable via `LEDGER_SYNC_DB_*` settings (pool_size, max_overflow, pool_recycle_seconds, connect_timeout_seconds, statement_timeout_seconds, idle_transaction_timeout_seconds; defaults sized for Neon free tier: 5/3/300/10/30/60). Compatible with Neon's PgBouncer pooler. - **Database-agnostic SQL**: SQLite uses `strftime()`, PostgreSQL uses `to_char()`. Always use `query_helpers.py` helpers (`fmt_year_month`, `fmt_year`, `fmt_month`, `fmt_date`) instead of `func.strftime()` directly -- raw SQLite SQL will break production. - **DB URL normalization**: `session.py` auto-converts `postgresql://` and `postgresql+psycopg2://` to `postgresql+psycopg://` (psycopg v3 driver). -- **Security**: Rate limiting (slowapi) on `/api/auth/refresh`, OAuth callbacks, `/api/upload`, and `/api/ai/bedrock/chat` — IP-keyed, applied via `@limiter.limit()` with a globally registered 429 handler. Security headers (CSP, HSTS, X-Frame-Options), query timeouts. SheetJS installed from CDN (`cdn.sheetjs.com/xlsx-0.20.3`) to avoid npm registry vulnerabilities. OAuth secrets stored server-side only; frontend never sees provider tokens. AI API keys encrypted at rest with AES-256-GCM (PBKDF2-derived key from JWT secret, per-ciphertext random 128-bit salt). +- **Security**: Rate limiting (slowapi) on `/api/auth/refresh`, OAuth callbacks, `/api/upload`, and `/api/ai/bedrock/chat` -- IP-keyed, applied via `@limiter.limit()` with a globally registered 429 handler. Security headers (CSP, HSTS, X-Frame-Options), query timeouts. SheetJS installed from CDN (`cdn.sheetjs.com/xlsx-0.20.3`) to avoid npm registry vulnerabilities. OAuth secrets stored server-side only; frontend never sees provider tokens. AI API keys encrypted at rest with AES-256-GCM (PBKDF2-derived key from JWT secret, per-ciphertext random 128-bit salt). - **AI Chatbot (app_bedrock + BYOK, tool-calling)**: Two modes stored in `user_preferences.ai_mode`: - **`app_bedrock` (default)** -- new users get a working chatbot with zero setup. Server uses its own Bedrock bearer token (`LEDGER_SYNC_BEDROCK_API_KEY`) and a fixed cheap model (Haiku 4.5). Rate-limited to `LEDGER_SYNC_AI_DAILY_MESSAGE_LIMIT` messages/day (default 10) per user to keep the shared-key cost predictable. Hitting the cap returns a 429 with a "switch to BYOK" pointer. - **`byok`** -- user configures provider (OpenAI/Anthropic/Bedrock), model, and API key in Settings > AI Assistant. Per-user daily/monthly token limits configurable. - **Transport**: OpenAI and Anthropic go browser-direct (both support CORS; Anthropic needs `anthropic-dangerous-direct-browser-access: true`). Bedrock goes through `/api/ai/bedrock/chat` because it needs SigV4 auth and doesn't support CORS. **All three are now non-streaming** -- `boto3.client('bedrock-runtime').converse()` returns a full JSON response, then re-emitted as a plain JSON body from FastAPI. Streaming was dropped when tool-calling was added because parsing `tool_use` events out of SSE was fragile; non-streaming with a 2-5s wait keeps the tool loop simple. - - **Tool calling (v2.5+)**: The bot has 15 read-only tools in `backend/src/ledger_sync/api/ai_tools.py` -- `list_accounts`, `search_transactions`, `get_monthly_summary`, `list_categories`, `get_category_spending`, `get_net_worth`, `list_recurring`, `list_goals`, `list_recent_months`, `get_fy_summary`, `list_budgets`, `get_savings_rate`, `get_top_merchants`, `get_transfer_flows`, `get_investment_holdings`. Same schema works across all three providers. Frontend tool loop in `useChat`: send -> receive `tool_use` -> execute tools in parallel -> append `tool_result` -> resend -> repeat until `end_turn` or 6-round limit. Every tool is user-scoped via `CurrentUser`; the LLM cannot see another user's data. + - **Tool calling (v2.5+)**: The bot has 15 read-only tools registered in `backend/src/ledger_sync/api/ai_tools_impl/` (registry pattern; `ai_tools.py` is the thin router) -- `list_accounts`, `search_transactions`, `get_monthly_summary`, `list_categories`, `get_category_spending`, `get_net_worth`, `list_recurring`, `list_goals`, `list_recent_months`, `get_fy_summary`, `list_budgets`, `get_cash_flow`, `get_tax_summary`, `get_preferences_summary`, `list_anomalies`. Same schema works across all three providers. Frontend tool loop in `useChat`: send -> receive `tool_use` -> execute tools in parallel -> append `tool_result` -> resend -> repeat until `end_turn` or 6-round limit. Every tool is user-scoped via `CurrentUser`; the LLM cannot see another user's data. - **System prompt shrunk to ~10 lines** (preferences, today's date, anti-hallucination nudge). No more "context stuffing" -- the bot reaches for tools instead of making up plausible numbers. - **Usage logging**: every LLM round-trip is logged to `ai_usage_log` (provider, model, input/output tokens, estimated USD, tool rounds). `GET /api/ai/usage` returns today / MTD / all-time rollups. - **PWA**: App is installable on mobile/desktop. Config lives in `frontend/vite.config.ts` (`VitePWA` plugin). Manifest uses relative `start_url`/`scope` so it works under the `/ledger-sync/` GH Pages base path. Service worker precaches the app shell only -- **never** caches `/api/*` (enforced via `navigateFallbackDenylist`) so financial data is always fresh. Icons are generated from `frontend/public/pwa-icon-source.svg` via `pnpm run generate:icons`; `pwa-assets.config.ts` overrides `minimal-2023`'s apple transform to `padding: 0` + transparent background so the gradient paints corner-to-corner and iOS applies its own squircle mask. Do NOT pass `--preset` on the generator CLI -- it overrides the config file. `registerType: 'autoUpdate'` + `clientsClaim` means updates propagate within one app restart. @@ -146,9 +142,7 @@ Three services, all free tier: ### CI Pipeline (`.github/workflows/ci.yml`) -Single job: `uv sync` + `pnpm install` -> lint -> type-check -> test -> build - -Runs on push/PR to main. Python 3.12, Node 22, pnpm 10, uv (latest). +Three jobs on push/PR to main: `frontend` (shared-workflows `node-ci.yml`, working-directory frontend), `backend` (inline: uv sync --all-extras --group dev, ruff check + format --check, mypy, pytest on Python 3.13 -- inline because the shared python-ci swallows failures with `|| true`), and `security` (shared-workflows `security-scan.yml`). ## Code Quality Rules @@ -160,40 +154,7 @@ Runs on push/PR to main. Python 3.12, Node 22, pnpm 10, uv (latest). ## Project Skills -Repo-specific skills live in [`.claude/skills//SKILL.md`](.claude/skills/). Two flavors: - -**Atlas skills** (`user-invocable: false`) — codebase mental models, auto-load by `paths:` scope. They give Claude background knowledge on demand without taking context space upfront. Index in [MEMORY.md](MEMORY.md). - -| Atlas | Auto-loads when... | -| --- | --- | -| `backend-atlas` | Editing any backend Python file | -| `frontend-atlas` | Editing any frontend TS/TSX file | -| `data-flow-atlas` | Always available (no path scope) | -| `domain-atlas` | Editing tax/FY/currency/instrument code | -| `deployment-atlas` | Editing workflows, vercel.json, settings | -| `indian-finance-expert` | Editing tax/investment/savings code — domain-expert reference | - -**Frontend craft skills** (`user-invocable: false`) — practice/best-practice guides grounded in current library docs + this repo's actual fixed bugs. Auto-load by `paths:` when editing frontend code. - -| Craft skill | Auto-loads when... | -| --- | --- | -| `query-states` | Editing pages/hooks/components — loading/empty/error states + undefined-data guards (TanStack Query v5) | -| `accessible-ui` | Editing components/pages — ARIA names, modal semantics, label association, contrast | -| `recharts-viz` | Editing analytics/chart components — ResponsiveContainer, token colors, NaN/empty guards | -| `react-patterns` | Editing any frontend TS/TSX — useEffectEvent, stable keys, stale-state, lazy/Suspense, hooks lint | -| `design-system-ui` | Editing components/pages/CSS — design tokens, mobile-first + 44px touch, h-dvh, banned AI-slop patterns | -| `ui-patterns-reference` | Editing components/pages/CSS — cross-system principles (shadcn/Radix/Material/Primer/Nielsen/Refactoring UI): hierarchy, semantic tokens, tables, forms, heuristics | - -**Task skills** (user-invocable, also auto-trigger on phrasing) — recipes for recurring work. Consolidated to broad workflows rather than one skill per layer. - -| Task skill | Trigger when... | -| --- | --- | -| `add-feature` | **Full-stack feature** — backend endpoint + Pydantic schema + frontend service + hook + page/component | -| `new-ai-tool` | Adding a tool the AI chatbot can call (the only single-file workflow that stays standalone) | -| `new-migration` | Touching anything in `db/_models/` (DB schema change with the empty-downgrade convention) | -| `schema-drift-check` | Pydantic schema or response shape changed (catch silent TS drift before PR) | -| `release-changelog` | Cutting a release / version bump | -| `debug-finance` | Wrong number / missing data / unexpected analytics | +The repo-specific skill set (atlas/craft/task skills under `.claude/skills/`) was untracked and removed from the working tree on 2026-07-03 (commit 594f102, "untrack AI assistant local files"). It no longer exists locally -- recover from git history at `594f102^` if ever needed. Until restored, workspace-level and user-level skills cover this repo. ## New Feature Patterns diff --git a/backend/src/ledger_sync/api/categorization_rules.py b/backend/src/ledger_sync/api/categorization_rules.py new file mode 100644 index 00000000..c8b5dbdf --- /dev/null +++ b/backend/src/ledger_sync/api/categorization_rules.py @@ -0,0 +1,160 @@ +"""Categorization rule API endpoints. + +CRUD for user-defined "note/account contains X -> set category Y" rules, +plus an explicit retroactive apply endpoint. Rules never auto-run on +save -- import-time application happens in the sync engine, and the +user triggers retroactive application via POST /apply. All business +logic lives in ``core/rules.py``; this router only maps models to +responses. +""" + +from fastapi import APIRouter, HTTPException +from sqlalchemy import select +from sqlalchemy.exc import OperationalError + +from ledger_sync.api.deps import CurrentUser, DatabaseSession +from ledger_sync.core import rules as rules_engine +from ledger_sync.core.analytics_engine import AnalyticsEngine +from ledger_sync.db.models import CategorizationRule +from ledger_sync.schemas.categorization_rules import ( + CategorizationRuleCreateRequest, + CategorizationRuleResponse, + CategorizationRuleUpdateRequest, + RulesApplyResponse, +) +from ledger_sync.utils.logging import logger + +router = APIRouter(prefix="/api/categorization-rules", tags=["categorization-rules"]) + + +def _to_rule_response(rule: CategorizationRule) -> CategorizationRuleResponse: + """Convert a CategorizationRule model to a CategorizationRuleResponse.""" + return CategorizationRuleResponse( + id=rule.id, + match_field=rule.match_field, + pattern=rule.pattern, + category=rule.category, + subcategory=rule.subcategory or "", + is_active=rule.is_active, + sort_order=rule.sort_order, + created_at=rule.created_at.isoformat(), + ) + + +@router.get("") +async def list_rules( + current_user: CurrentUser, + db: DatabaseSession, +) -> list[CategorizationRuleResponse]: + """List all of the user's rules (including inactive) in evaluation order.""" + stmt = ( + select(CategorizationRule) + .where(CategorizationRule.user_id == current_user.id) + .order_by(CategorizationRule.sort_order.asc(), CategorizationRule.id.asc()) + ) + rules = db.execute(stmt).scalars().all() + return [_to_rule_response(rule) for rule in rules] + + +@router.post("", status_code=201) +async def create_rule( + payload: CategorizationRuleCreateRequest, + current_user: CurrentUser, + db: DatabaseSession, +) -> CategorizationRuleResponse: + """Create a rule. Does NOT apply it retroactively -- use POST /apply.""" + rule = CategorizationRule( + user_id=current_user.id, + match_field=payload.match_field, + pattern=payload.pattern, + category=payload.category, + subcategory=payload.subcategory, + is_active=payload.is_active, + sort_order=payload.sort_order, + ) + db.add(rule) + db.commit() + db.refresh(rule) + return _to_rule_response(rule) + + +@router.put("/{rule_id}", responses={404: {"description": "Rule not found"}}) +async def update_rule( + rule_id: int, + payload: CategorizationRuleUpdateRequest, + current_user: CurrentUser, + db: DatabaseSession, +) -> CategorizationRuleResponse: + """Fully replace a rule. Does NOT retro-apply -- use POST /apply.""" + stmt = select(CategorizationRule).where( + CategorizationRule.id == rule_id, + CategorizationRule.user_id == current_user.id, + ) + rule = db.execute(stmt).scalar_one_or_none() + if rule is None: + raise HTTPException(status_code=404, detail="Rule not found") + + rule.match_field = payload.match_field + rule.pattern = payload.pattern + rule.category = payload.category + rule.subcategory = payload.subcategory + rule.is_active = payload.is_active + rule.sort_order = payload.sort_order + + db.commit() + db.refresh(rule) + return _to_rule_response(rule) + + +@router.delete("/{rule_id}", status_code=204) +async def delete_rule( + rule_id: int, + current_user: CurrentUser, + db: DatabaseSession, +) -> None: + """Delete a rule. Idempotent: a nonexistent id is also a 204.""" + stmt = select(CategorizationRule).where( + CategorizationRule.id == rule_id, + CategorizationRule.user_id == current_user.id, + ) + rule = db.execute(stmt).scalar_one_or_none() + if rule is not None: + db.delete(rule) + db.commit() + + +@router.post("/apply") +async def apply_rules( + current_user: CurrentUser, + db: DatabaseSession, +) -> RulesApplyResponse: + """Apply all active rules to the user's live non-transfer transactions. + + Updated rows get NEW transaction_id values (category feeds the dedup + hash) and their tags are migrated server-side. Analytics tables bake + in categories, so a full analytics rebuild runs afterwards -- + non-fatally: the apply itself still succeeds if the rebuild fails. + """ + matched, updated = rules_engine.apply_rules_retroactively(db, current_user.id) + + analytics_refreshed = True + try: + analytics = AnalyticsEngine(db, user_id=current_user.id) + analytics.run_full_analytics(source_file="rules_apply") + except (OSError, RuntimeError, ValueError, OperationalError) as exc: + # Don't fail the apply if the analytics rebuild blows up -- the + # category rewrites are already committed; the user can re-run + # POST /api/analytics/v2/refresh. + logger.warning( + "Post-apply analytics refresh failed for user_id=%s: %s", + current_user.id, + exc, + ) + db.rollback() + analytics_refreshed = False + + return RulesApplyResponse( + matched=matched, + updated=updated, + analytics_refreshed=analytics_refreshed, + ) diff --git a/backend/src/ledger_sync/api/main.py b/backend/src/ledger_sync/api/main.py index f86ec3f3..917d427b 100644 --- a/backend/src/ledger_sync/api/main.py +++ b/backend/src/ledger_sync/api/main.py @@ -25,6 +25,7 @@ from ledger_sync.api.analytics_v2 import router as analytics_v2_router from ledger_sync.api.auth import router as auth_router from ledger_sync.api.calculations import router as calculations_router +from ledger_sync.api.categorization_rules import router as categorization_rules_router from ledger_sync.api.exchange_rates import router as exchange_rates_router from ledger_sync.api.meta import router as meta_router from ledger_sync.api.oauth import router as oauth_router @@ -32,6 +33,7 @@ from ledger_sync.api.rate_limit import limiter from ledger_sync.api.rates import router as rates_router from ledger_sync.api.reports import router as reports_router +from ledger_sync.api.saved_views import router as saved_views_router from ledger_sync.api.stock_price import router as stock_price_router from ledger_sync.api.transactions import router as transactions_router from ledger_sync.api.upload import router as upload_router @@ -260,8 +262,10 @@ async def add_timing_header( app.include_router(calculations_router) app.include_router(meta_router) app.include_router(account_classifications_router) +app.include_router(categorization_rules_router) app.include_router(preferences_router) app.include_router(reports_router) +app.include_router(saved_views_router) app.include_router(transactions_router) app.include_router(upload_router) app.include_router(exchange_rates_router) diff --git a/backend/src/ledger_sync/api/saved_views.py b/backend/src/ledger_sync/api/saved_views.py new file mode 100644 index 00000000..c5b95647 --- /dev/null +++ b/backend/src/ledger_sync/api/saved_views.py @@ -0,0 +1,101 @@ +"""Saved filter view API endpoints. + +Named snapshots of the Transactions page filter state. The ``filters`` +payload is an opaque JSON object echoed verbatim -- the backend never +validates its keys, so new frontend filter fields need zero backend +changes. POST upserts by (user, name). +""" + +import json + +from fastapi import APIRouter +from sqlalchemy import select + +from ledger_sync.api.deps import CurrentUser, DatabaseSession +from ledger_sync.db.models import SavedFilterView +from ledger_sync.schemas.saved_views import SavedViewCreateRequest, SavedViewResponse + +router = APIRouter(prefix="/api/saved-views", tags=["saved-views"]) + + +def _to_view_response(view: SavedFilterView) -> SavedViewResponse: + """Convert a SavedFilterView model to a SavedViewResponse.""" + try: + filters = json.loads(view.filters) + if not isinstance(filters, dict): + filters = {} + except (TypeError, ValueError): + filters = {} + return SavedViewResponse( + id=view.id, + name=view.name, + filters=filters, + created_at=view.created_at.isoformat(), + updated_at=view.updated_at.isoformat(), + ) + + +@router.get("") +async def list_saved_views( + current_user: CurrentUser, + db: DatabaseSession, +) -> list[SavedViewResponse]: + """List the user's saved views ordered by name.""" + stmt = ( + select(SavedFilterView) + .where(SavedFilterView.user_id == current_user.id) + .order_by(SavedFilterView.name.asc()) + ) + views = db.execute(stmt).scalars().all() + return [_to_view_response(view) for view in views] + + +@router.post("") +async def save_view( + payload: SavedViewCreateRequest, + current_user: CurrentUser, + db: DatabaseSession, +) -> SavedViewResponse: + """Create or update a saved view -- UPSERT by (user, name). + + If a view with this name already exists for the user, its filters + and updated_at are overwritten and the existing id is returned. + Always 200, never 201/409, so the frontend "Save current view" flow + can blindly POST without checking name collisions. + """ + stmt = select(SavedFilterView).where( + SavedFilterView.user_id == current_user.id, + SavedFilterView.name == payload.name, + ) + view = db.execute(stmt).scalar_one_or_none() + + if view is not None: + view.filters = json.dumps(payload.filters) + else: + view = SavedFilterView( + user_id=current_user.id, + name=payload.name, + filters=json.dumps(payload.filters), + ) + db.add(view) + + db.commit() + db.refresh(view) + return _to_view_response(view) + + +@router.delete("/{view_id}", status_code=204) +async def delete_saved_view( + view_id: int, + current_user: CurrentUser, + db: DatabaseSession, +) -> None: + """Delete a saved view. Idempotent: a nonexistent id is also a 204.""" + stmt = select(SavedFilterView).where( + SavedFilterView.id == view_id, + SavedFilterView.user_id == current_user.id, + ) + view = db.execute(stmt).scalar_one_or_none() + if view is not None: + db.delete(view) + db.commit() diff --git a/backend/src/ledger_sync/api/transactions.py b/backend/src/ledger_sync/api/transactions.py index 1ac01b62..e3404946 100644 --- a/backend/src/ledger_sync/api/transactions.py +++ b/backend/src/ledger_sync/api/transactions.py @@ -8,7 +8,7 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Response from pydantic import BaseModel -from sqlalchemy import func, literal, or_ +from sqlalchemy import delete, exists, func, literal, or_ from sqlalchemy.orm import Query as SAQuery from sqlalchemy.orm import Session @@ -17,13 +17,15 @@ apply_excluded_accounts_filter, excluded_accounts_for, ) -from ledger_sync.db.models import Transaction, TransactionType, User +from ledger_sync.db.models import Transaction, TransactionTag, TransactionType, User from ledger_sync.ingest.hash_id import TransactionHasher from ledger_sync.schemas.transactions import ( + TagFacet, TransactionCreateRequest, TransactionFacetsResponse, TransactionResponse, TransactionsListResponse, + TransactionTagsUpdateRequest, ) _TxQuery = SAQuery[Transaction] @@ -71,6 +73,7 @@ class SearchFilters(BaseModel): max_amount: Annotated[float | None, Query(description="Maximum amount")] = None start_date: Annotated[datetime | None, Query(description=START_DATE_DESC)] = None end_date: Annotated[datetime | None, Query(description=END_DATE_DESC)] = None + tag: Annotated[str | None, Query(max_length=100, description="Filter by exact tag")] = None def _apply_search_filters( @@ -173,7 +176,54 @@ def _apply_sorting( return tx_query.order_by(sort_column.asc()) -def _to_transaction_response(tx: Transaction) -> TransactionResponse: +def _apply_tag_filter(tx_query: _TxQuery, user_id: int, tag: str | None) -> _TxQuery: + """Filter to transactions carrying *tag* via an EXISTS subquery. + + Exact string match, DB-agnostic. No-op when *tag* is unset. + """ + if not tag: + return tx_query + return tx_query.filter( + exists().where( + (TransactionTag.user_id == user_id) + & (TransactionTag.transaction_id == Transaction.transaction_id) + & (TransactionTag.tag == tag) + ) + ) + + +def _tags_for_transactions( + db: Session, + user_id: int, + transaction_ids: list[str], +) -> dict[str, list[str]]: + """Batch-fetch tags for a page of transactions in one query. + + Returns a ``{transaction_id: [tags...]}`` map with each tag list + sorted alphabetically. Missing ids simply have no entry. + """ + if not transaction_ids: + return {} + rows = ( + db.query(TransactionTag.transaction_id, TransactionTag.tag) + .filter( + TransactionTag.user_id == user_id, + TransactionTag.transaction_id.in_(transaction_ids), + ) + .all() + ) + tags_map: dict[str, list[str]] = {} + for txn_id, tag in rows: + tags_map.setdefault(txn_id, []).append(tag) + for tag_list in tags_map.values(): + tag_list.sort() + return tags_map + + +def _to_transaction_response( + tx: Transaction, + tags: list[str] | None = None, +) -> TransactionResponse: """Convert a Transaction model to a TransactionResponse.""" return TransactionResponse( id=tx.transaction_id, @@ -190,6 +240,7 @@ def _to_transaction_response(tx: Transaction) -> TransactionResponse: source_file=tx.source_file, last_seen_at=tx.last_seen_at.isoformat(), is_transfer=tx.type.value == "Transfer", + tags=tags or [], ) @@ -261,8 +312,12 @@ async def get_transactions( # Apply sorting and pagination transactions = query.order_by(Transaction.date.desc()).offset(offset).limit(limit).all() + tags_map = _tags_for_transactions( + db, current_user.id, [tx.transaction_id for tx in transactions] + ) + return TransactionsListResponse( - data=[_to_transaction_response(tx) for tx in transactions], + data=[_to_transaction_response(tx, tags_map.get(tx.transaction_id)) for tx in transactions], total=total, limit=limit, offset=offset, @@ -319,9 +374,28 @@ async def get_transaction_facets( expense = counts.get(TransactionType.EXPENSE, 0) transfer = counts.get(TransactionType.TRANSFER, 0) + # Tag facets: distinct tags with live-transaction counts. Joins to + # transactions so soft-deleted rows drop out, and honours the same + # excluded-accounts preference as the other facets. + tag_query = ( + db.query(TransactionTag.tag, func.count()) + .join(Transaction, Transaction.transaction_id == TransactionTag.transaction_id) + .filter( + TransactionTag.user_id == current_user.id, + Transaction.is_deleted.is_(False), + ) + ) + tag_query = apply_excluded_accounts_filter(tag_query, excluded_accounts_for(current_user)) + tag_rows = tag_query.group_by(TransactionTag.tag).all() + tag_facets = [ + TagFacet(name=name, count=count) + for name, count in sorted(tag_rows, key=lambda r: (-r[1], r[0])) + ] + return TransactionFacetsResponse( categories=sorted(categories, key=lambda s: s.lower()), accounts=sorted(accounts, key=lambda s: s.lower()), + tags=tag_facets, income_count=income, expense_count=expense, transfer_count=transfer, @@ -365,6 +439,7 @@ async def search_transactions( # Apply all search filters tx_query = _apply_search_filters(tx_query, filters) + tx_query = _apply_tag_filter(tx_query, current_user.id, filters.tag) # Get total count before pagination total = tx_query.count() @@ -373,8 +448,15 @@ async def search_transactions( tx_query = _apply_sorting(tx_query, sort_by, sort_order) transactions = tx_query.offset(offset).limit(limit).all() + tags_map = _tags_for_transactions( + db, current_user.id, [tx.transaction_id for tx in transactions] + ) + return { - "data": [_to_transaction_response(tx).model_dump() for tx in transactions], + "data": [ + _to_transaction_response(tx, tags_map.get(tx.transaction_id)).model_dump() + for tx in transactions + ], "total": total, "limit": limit, "offset": offset, @@ -535,3 +617,87 @@ async def create_transaction( db.refresh(transaction) return _to_transaction_response(transaction) + + +# --- Transaction Tags Endpoint --- + + +@router.put( + "/api/transactions/{transaction_id}/tags", + responses={ + 404: {"description": "Transaction not found"}, + 422: {"description": "Validation error"}, + }, +) +async def set_transaction_tags( + transaction_id: str, + payload: TransactionTagsUpdateRequest, + current_user: CurrentUser, + db: DatabaseSession, +) -> dict[str, Any]: + """Replace the full tag list of a transaction. + + Replace-all semantics: an empty list clears every tag. Tags are + trimmed, empties dropped, exact duplicates removed (order preserved). + Tags are case-sensitive and do NOT feed the dedup hash, so setting + them never changes the transaction_id. + + Args: + transaction_id: 64-char transaction id + payload: Full replacement tag list + current_user: Authenticated user + db: Database session + + Returns: + The normalized stored tag list, in stored order + + Raises: + HTTPException: 404 when the transaction doesn't exist for this + user (or is soft-deleted); 422 on tag length/count violations + + """ + transaction = ( + db.query(Transaction) + .filter( + Transaction.transaction_id == transaction_id, + Transaction.user_id == current_user.id, + Transaction.is_deleted.is_(False), + ) + .first() + ) + if transaction is None: + raise HTTPException(status_code=404, detail="Transaction not found") + + # Normalize: trim, drop empties, reject overlong, dedupe preserving order. + tags: list[str] = [] + for raw in payload.tags: + tag = raw.strip() + if not tag: + continue + if len(tag) > 50: + raise HTTPException( + status_code=422, + detail=f"Tag exceeds 50 characters: {tag[:50]}...", + ) + if tag not in tags: + tags.append(tag) + if len(tags) > 10: + raise HTTPException(status_code=422, detail="A transaction can have at most 10 tags") + + db.execute( + delete(TransactionTag).where( + TransactionTag.user_id == current_user.id, + TransactionTag.transaction_id == transaction_id, + ) + ) + for tag in tags: + db.add( + TransactionTag( + user_id=current_user.id, + transaction_id=transaction_id, + tag=tag, + ) + ) + db.commit() + + return {"transaction_id": transaction_id, "tags": tags} diff --git a/backend/src/ledger_sync/core/rules.py b/backend/src/ledger_sync/core/rules.py new file mode 100644 index 00000000..2baa4a12 --- /dev/null +++ b/backend/src/ledger_sync/core/rules.py @@ -0,0 +1,242 @@ +"""Categorization rule engine. + +Rules are user-defined "field contains pattern -> set category" mappings +evaluated case-insensitively, first-match-wins, ordered by +(sort_order asc, id asc). Two application points: + +* Import time (pre-hash): ``SyncEngine._reconcile_and_log`` mutates + normalized rows BEFORE any hashing, so re-uploads stay deterministic + (same raw row + same rules = same hash = dedup skip). +* Retroactive: ``apply_rules_retroactively`` rewrites live non-transfer + rows AND recomputes their transaction_id (category feeds the dedup + hash -- without the rehash, the next full-snapshot upload would + soft-delete the retro-updated rows and re-insert duplicates). Tags + reference transaction_id, so they are migrated in the same pass. + +Transfers are always exempt: their category is synthesized from account +names and feeds transfer-leg pairing. +""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy import delete, select, update +from sqlalchemy.orm import Session + +from ledger_sync.db.models import ( + Anomaly, + CategorizationRule, + Transaction, + TransactionTag, + TransactionType, +) +from ledger_sync.ingest.hash_id import TransactionHasher +from ledger_sync.utils.logging import logger + +# Commit every N updated rows so a large retro-apply stays under the +# Neon 30s statement timeout. +_COMMIT_CHUNK = 500 + + +def load_active_rules(session: Session, user_id: int) -> list[CategorizationRule]: + """Return the user's active rules in deterministic evaluation order.""" + stmt = ( + select(CategorizationRule) + .where( + CategorizationRule.user_id == user_id, + CategorizationRule.is_active.is_(True), + ) + .order_by(CategorizationRule.sort_order.asc(), CategorizationRule.id.asc()) + ) + return list(session.execute(stmt).scalars().all()) + + +def match_rule(rule: CategorizationRule, note: str | None, account: str | None) -> bool: + """Return True when *rule*'s pattern is a case-insensitive substring of its field.""" + needle = rule.pattern.strip().lower() + haystack = (note if rule.match_field == "note" else account) or "" + return needle != "" and needle in haystack.lower() + + +def apply_rules_to_row(rules: list[CategorizationRule], normalized_row: dict[str, Any]) -> bool: + """Apply the first matching rule to a normalized import row, in place. + + Skips transfer rows (their category is synthesized from account names). + On match, sets both ``category`` and ``subcategory`` (None clears it). + + Returns: + True when a rule matched and the row was mutated. + + """ + if normalized_row.get("is_transfer"): + return False + note = normalized_row.get("note") + account = normalized_row.get("account") + for rule in rules: + if match_rule(rule, note, account): + normalized_row["category"] = rule.category + normalized_row["subcategory"] = rule.subcategory + return True + return False + + +def _rehash_with_collision_handling( + hasher: TransactionHasher, + row: Transaction, + rule: CategorizationRule, + user_id: int, + live_ids: set[str], +) -> str: + """Recompute the dedup hash with the rule's category/subcategory. + + Bumps occurrence until the id is unique among the user's live ids + (including ids already assigned earlier in this pass), mirroring + ``Reconciler.reconcile_batch`` Phase 1. + """ + old_id = row.transaction_id + occurrence = 0 + while True: + new_id = hasher.generate_transaction_id( + date=row.date, + amount=row.amount, + account=row.account, + note=row.note, + category=rule.category, + subcategory=rule.subcategory, + tx_type=row.type.value, + user_id=user_id, + occurrence=occurrence, + ) + if new_id == old_id or new_id not in live_ids: + return new_id + occurrence += 1 + + +def _move_transaction_id( + session: Session, + row: Transaction, + rule: CategorizationRule, + user_id: int, + new_id: str, +) -> None: + """Rewrite *row*'s PK to *new_id*, carrying its tags and anomalies along. + + A direct UPDATE of the child FK column would violate the constraint on + Postgres in either order (the new parent id doesn't exist yet / the old + one still has children), so: collect -> delete -> flush the parent PK + change -> re-insert against the new id. Anomalies (nullable FK) are + detached before the PK moves and re-pointed after. + """ + old_id = row.transaction_id + tag_values = [ + tag_row[0] + for tag_row in session.execute( + select(TransactionTag.tag).where( + TransactionTag.user_id == user_id, + TransactionTag.transaction_id == old_id, + ) + ).all() + ] + if tag_values: + session.execute( + delete(TransactionTag).where( + TransactionTag.user_id == user_id, + TransactionTag.transaction_id == old_id, + ) + ) + anomaly_ids = [ + anomaly_row[0] + for anomaly_row in session.execute( + select(Anomaly.id).where( + Anomaly.user_id == user_id, + Anomaly.transaction_id == old_id, + ) + ).all() + ] + if anomaly_ids: + session.execute( + update(Anomaly).where(Anomaly.id.in_(anomaly_ids)).values(transaction_id=None) + ) + row.transaction_id = new_id + row.category = rule.category + row.subcategory = rule.subcategory + session.flush() + for tag in tag_values: + session.add(TransactionTag(user_id=user_id, transaction_id=new_id, tag=tag)) + if anomaly_ids: + session.execute( + update(Anomaly).where(Anomaly.id.in_(anomaly_ids)).values(transaction_id=new_id) + ) + + +def apply_rules_retroactively(session: Session, user_id: int) -> tuple[int, int]: + """Apply all active rules to the user's live non-transfer transactions. + + For each rewritten row the transaction_id is recomputed with the new + category/subcategory (occurrence-collision handling mirrors + ``Reconciler.reconcile_batch`` Phase 1) and any transaction_tags rows + are migrated to the new id. Commits every ``_COMMIT_CHUNK`` updated + rows and once at the end. + + Returns: + Tuple of (matched, updated): matched counts rows where some rule + matched; updated is the subset actually rewritten (rows already + at the target category+subcategory are skipped). + + """ + rules = load_active_rules(session, user_id) + if not rules: + return (0, 0) + + hasher = TransactionHasher() + + stmt = select(Transaction).where( + Transaction.user_id == user_id, + Transaction.is_deleted.is_(False), + Transaction.type != TransactionType.TRANSFER, + ) + rows = list(session.execute(stmt).scalars().all()) + + # Live ids of this user -- used for collision detection when rehashing. + # Updated as we go so ids assigned earlier in this batch also collide. + live_ids: set[str] = {row.transaction_id for row in rows} + + matched = 0 + updated = 0 + pending_since_commit = 0 + + for row in rows: + rule = next((r for r in rules if match_rule(r, row.note, row.account)), None) + if rule is None: + continue + matched += 1 + + if row.category == rule.category and row.subcategory == rule.subcategory: + continue + + old_id = row.transaction_id + new_id = _rehash_with_collision_handling(hasher, row, rule, user_id, live_ids) + + if new_id != old_id: + _move_transaction_id(session, row, rule, user_id, new_id) + live_ids.discard(old_id) + live_ids.add(new_id) + else: + row.category = rule.category + row.subcategory = rule.subcategory + updated += 1 + pending_since_commit += 1 + + if pending_since_commit >= _COMMIT_CHUNK: + session.commit() + pending_since_commit = 0 + + session.commit() + logger.info( + "Retroactive rule apply for user_id=%s: matched=%d updated=%d", + user_id, + matched, + updated, + ) + return (matched, updated) diff --git a/backend/src/ledger_sync/core/sync_engine.py b/backend/src/ledger_sync/core/sync_engine.py index 6adcf09c..c176540d 100644 --- a/backend/src/ledger_sync/core/sync_engine.py +++ b/backend/src/ledger_sync/core/sync_engine.py @@ -7,6 +7,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session +from ledger_sync.core import rules from ledger_sync.core.analytics_engine import AnalyticsEngine from ledger_sync.core.reconciler import Reconciler, ReconciliationStats from ledger_sync.db.models import ImportLog @@ -74,6 +75,16 @@ def _reconcile_and_log( Shared tail of both import paths: splits rows by transfer flag, reconciles each batch, accumulates stats, and writes the ImportLog. """ + # Apply the user's categorization rules BEFORE any hashing: category + # and subcategory feed the SHA-256 transaction_id, so mutating rows + # here keeps re-uploads deterministic (same raw row + same rules = + # same hash = dedup skip). apply_rules_to_row skips transfer rows. + if self.user_id is not None: + active_rules = rules.load_active_rules(self.session, self.user_id) + if active_rules: + for row in normalized_rows: + rules.apply_rules_to_row(active_rules, row) + transactions = [r for r in normalized_rows if not r.get("is_transfer", False)] transfers = [r for r in normalized_rows if r.get("is_transfer", False)] logger.info("Found %d transactions and %d transfers", len(transactions), len(transfers)) diff --git a/backend/src/ledger_sync/db/_models/__init__.py b/backend/src/ledger_sync/db/_models/__init__.py index 0f47df5a..8dbae1da 100644 --- a/backend/src/ledger_sync/db/_models/__init__.py +++ b/backend/src/ledger_sync/db/_models/__init__.py @@ -31,6 +31,11 @@ NetWorthSnapshot, TaxRecord, ) +from ledger_sync.db._models.organization import ( + CategorizationRule, + SavedFilterView, + TransactionTag, +) from ledger_sync.db._models.planning import ( Anomaly, Budget, @@ -56,6 +61,7 @@ "AnomalyType", "AuditLog", "Budget", + "CategorizationRule", "CategoryTrend", "CohortSpending", "ColumnMappingLog", @@ -70,9 +76,11 @@ "NetWorthSnapshot", "RecurrenceFrequency", "RecurringTransaction", + "SavedFilterView", "ScheduledTransaction", "TaxRecord", "Transaction", + "TransactionTag", "TransactionType", "TransferFlow", "User", diff --git a/backend/src/ledger_sync/db/_models/organization.py b/backend/src/ledger_sync/db/_models/organization.py new file mode 100644 index 00000000..23b7ffbc --- /dev/null +++ b/backend/src/ledger_sync/db/_models/organization.py @@ -0,0 +1,164 @@ +"""CategorizationRule, TransactionTag, SavedFilterView models. + +Organization features: user-defined categorization rules applied at +import time (pre-hash) and on demand, free-string transaction tags, +and named saved filter views for the Transactions page. +""" + +from datetime import UTC, datetime +from typing import TYPE_CHECKING + +from sqlalchemy import ( + Boolean, + DateTime, + ForeignKey, + Index, + Integer, + String, + Text, +) +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from ledger_sync.db._models._constants import USER_FK +from ledger_sync.db.base import Base + +if TYPE_CHECKING: + from ledger_sync.db._models.user import User + + +class CategorizationRule(Base): + """User-defined "field contains pattern -> set category" rule. + + Rules are evaluated case-insensitively, first-match-wins, ordered by + (sort_order asc, id asc). ``match_field`` is either ``note`` or + ``account`` -- validated at the API layer, stored as a plain string. + On match, both ``category`` AND ``subcategory`` are applied + (``subcategory=None`` clears it). + """ + + __tablename__ = "categorization_rules" + + id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True) + + # User foreign key - scopes rule to owner + user_id: Mapped[int] = mapped_column( + Integer, ForeignKey(USER_FK, ondelete="CASCADE"), nullable=False + ) + + match_field: Mapped[str] = mapped_column(String(20), nullable=False, default="note") + pattern: Mapped[str] = mapped_column(String(255), nullable=False) + category: Mapped[str] = mapped_column(String(255), nullable=False) + subcategory: Mapped[str | None] = mapped_column(String(255), nullable=True) + is_active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + sort_order: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + + # Metadata + created_at: Mapped[datetime] = mapped_column( + DateTime, nullable=False, default=lambda: datetime.now(UTC) + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime, + nullable=False, + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + ) + + # Relationship back to user + user: Mapped["User"] = relationship("User", back_populates="categorization_rules") + + __table_args__ = (Index("ix_categorization_rules_user", "user_id"),) + + def __repr__(self) -> str: + """Return string representation.""" + return ( + f" {self.category})>" + ) + + +class TransactionTag(Base): + """Free-string tag attached to a transaction. + + One row per (transaction, tag). Tags are case-sensitive exact + strings stored trimmed. No separate lookup table: filtering uses an + EXISTS subquery, facets use GROUP BY tag. + """ + + __tablename__ = "transaction_tags" + + id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True) + + # User foreign key - scopes tag to owner + user_id: Mapped[int] = mapped_column( + Integer, ForeignKey(USER_FK, ondelete="CASCADE"), nullable=False + ) + transaction_id: Mapped[str] = mapped_column( + String(64), + ForeignKey("transactions.transaction_id", ondelete="CASCADE"), + nullable=False, + ) + tag: Mapped[str] = mapped_column(String(100), nullable=False) + + # Metadata + created_at: Mapped[datetime] = mapped_column( + DateTime, nullable=False, default=lambda: datetime.now(UTC) + ) + + # Relationship back to user + user: Mapped["User"] = relationship("User", back_populates="transaction_tags") + + __table_args__ = ( + Index( + "ix_transaction_tags_user_txn_tag", + "user_id", + "transaction_id", + "tag", + unique=True, + ), + Index("ix_transaction_tags_user_tag", "user_id", "tag"), + ) + + def __repr__(self) -> str: + """Return string representation.""" + return f"" + + +class SavedFilterView(Base): + """Named snapshot of the Transactions page filter state. + + ``filters`` is an opaque JSON string (the frontend FilterValues + object verbatim) -- the backend never inspects its keys, so new + frontend filter fields require zero backend changes. One view per + (user, name); POST upserts by name. + """ + + __tablename__ = "saved_filter_views" + + id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True) + + # User foreign key - scopes view to owner + user_id: Mapped[int] = mapped_column( + Integer, ForeignKey(USER_FK, ondelete="CASCADE"), nullable=False + ) + name: Mapped[str] = mapped_column(String(100), nullable=False) + filters: Mapped[str] = mapped_column(Text, nullable=False, default="{}") + + # Metadata + created_at: Mapped[datetime] = mapped_column( + DateTime, nullable=False, default=lambda: datetime.now(UTC) + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime, + nullable=False, + default=lambda: datetime.now(UTC), + onupdate=lambda: datetime.now(UTC), + ) + + # Relationship back to user + user: Mapped["User"] = relationship("User", back_populates="saved_filter_views") + + __table_args__ = (Index("ix_saved_filter_views_user_name", "user_id", "name", unique=True),) + + def __repr__(self) -> str: + """Return string representation.""" + return f"" diff --git a/backend/src/ledger_sync/db/_models/user.py b/backend/src/ledger_sync/db/_models/user.py index 1f1a7f3b..c6a50eea 100644 --- a/backend/src/ledger_sync/db/_models/user.py +++ b/backend/src/ledger_sync/db/_models/user.py @@ -21,6 +21,11 @@ if TYPE_CHECKING: from ledger_sync.db._models.ai_usage import AIUsageLog + from ledger_sync.db._models.organization import ( + CategorizationRule, + SavedFilterView, + TransactionTag, + ) from ledger_sync.db._models.planning import ( Anomaly, Budget, @@ -136,6 +141,27 @@ class User(Base): cascade=CASCADE_ALL_DELETE_ORPHAN, passive_deletes=True, ) + categorization_rules: Mapped["list[CategorizationRule]"] = relationship( + "CategorizationRule", + back_populates="user", + lazy="select", + cascade=CASCADE_ALL_DELETE_ORPHAN, + passive_deletes=True, + ) + transaction_tags: Mapped["list[TransactionTag]"] = relationship( + "TransactionTag", + back_populates="user", + lazy="select", + cascade=CASCADE_ALL_DELETE_ORPHAN, + passive_deletes=True, + ) + saved_filter_views: Mapped["list[SavedFilterView]"] = relationship( + "SavedFilterView", + back_populates="user", + lazy="select", + cascade=CASCADE_ALL_DELETE_ORPHAN, + passive_deletes=True, + ) def __repr__(self) -> str: """Return string representation.""" diff --git a/backend/src/ledger_sync/db/migrations/versions/20260707_1200_add_tags_rules_saved_views.py b/backend/src/ledger_sync/db/migrations/versions/20260707_1200_add_tags_rules_saved_views.py new file mode 100644 index 00000000..704d2a18 --- /dev/null +++ b/backend/src/ledger_sync/db/migrations/versions/20260707_1200_add_tags_rules_saved_views.py @@ -0,0 +1,111 @@ +"""add tags, categorization rules, and saved filter views tables + +Revision ID: tags_rules_views_2026 +Revises: cascade_user_fks_2026 +Create Date: 2026-07-07 12:00:00.000000 + +Adds three user-scoped tables for the organization feature set: + +``categorization_rules`` -- ordered "note/account contains X -> set +category Y" rules. Applied pre-hash at import time and on-demand via +POST /api/categorization-rules/apply. ``match_field`` is a plain +validated string (Pydantic pattern), not a DB enum, to keep this +migration a set of plain CREATEs. + +``transaction_tags`` -- one row per (transaction, tag) association. +Free-string tags, no lookup table: server-side filtering uses an +EXISTS subquery, facets use GROUP BY tag, and PATCH set-tags is a +delete+insert. The (user_id, transaction_id, tag) unique index also +serves the per-transaction lookup; (user_id, tag) serves facets and +the tag filter. + +``saved_filter_views`` -- named snapshots of the frontend filter +state. ``filters`` is an opaque JSON string the backend never +inspects, so new frontend filter fields need zero backend changes. +POST upserts by (user_id, name), enforced by the unique index. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "tags_rules_views_2026" +down_revision: str | None = "cascade_user_fks_2026" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +# All three tables scope rows to a user with the same CASCADE FK target. +_USERS_ID = "users.id" + + +def upgrade() -> None: + op.create_table( + "categorization_rules", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("match_field", sa.String(length=20), nullable=False, server_default="note"), + sa.Column("pattern", sa.String(length=255), nullable=False), + sa.Column("category", sa.String(length=255), nullable=False), + sa.Column("subcategory", sa.String(length=255), nullable=True), + sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("sort_order", sa.Integer(), nullable=False, server_default="0"), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(["user_id"], [_USERS_ID], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_categorization_rules_user", "categorization_rules", ["user_id"], unique=False + ) + + op.create_table( + "transaction_tags", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("transaction_id", sa.String(length=64), nullable=False), + sa.Column("tag", sa.String(length=100), nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(["user_id"], [_USERS_ID], ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["transaction_id"], ["transactions.transaction_id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_transaction_tags_user_txn_tag", + "transaction_tags", + ["user_id", "transaction_id", "tag"], + unique=True, + ) + op.create_index( + "ix_transaction_tags_user_tag", + "transaction_tags", + ["user_id", "tag"], + unique=False, + ) + + op.create_table( + "saved_filter_views", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("user_id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=100), nullable=False), + sa.Column("filters", sa.Text(), nullable=False, server_default="{}"), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(["user_id"], [_USERS_ID], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_saved_filter_views_user_name", + "saved_filter_views", + ["user_id", "name"], + unique=True, + ) + + +def downgrade() -> None: + # Per project convention: rollback via DB backup, not down-migration. + pass diff --git a/backend/src/ledger_sync/schemas/__init__.py b/backend/src/ledger_sync/schemas/__init__.py index 9c168716..733aa823 100644 --- a/backend/src/ledger_sync/schemas/__init__.py +++ b/backend/src/ledger_sync/schemas/__init__.py @@ -15,23 +15,43 @@ UserResponse, UserUpdate, ) +from ledger_sync.schemas.categorization_rules import ( + CategorizationRuleCreateRequest, + CategorizationRuleResponse, + CategorizationRuleUpdateRequest, + RulesApplyResponse, +) +from ledger_sync.schemas.saved_views import ( + SavedViewCreateRequest, + SavedViewResponse, +) from ledger_sync.schemas.transactions import ( HealthResponse, + TagFacet, TransactionResponse, TransactionsListResponse, + TransactionTagsUpdateRequest, UploadResponse, ) __all__ = [ + "CategorizationRuleCreateRequest", + "CategorizationRuleResponse", + "CategorizationRuleUpdateRequest", "HealthResponse", "MessageResponse", "OAuthCallbackRequest", "OAuthProviderConfig", "RefreshTokenRequest", + "RulesApplyResponse", + "SavedViewCreateRequest", + "SavedViewResponse", + "TagFacet", "Token", "TokenData", "TokenPayload", "TransactionResponse", + "TransactionTagsUpdateRequest", "TransactionsListResponse", "UploadResponse", "UserResponse", diff --git a/backend/src/ledger_sync/schemas/categorization_rules.py b/backend/src/ledger_sync/schemas/categorization_rules.py new file mode 100644 index 00000000..4c8a1aae --- /dev/null +++ b/backend/src/ledger_sync/schemas/categorization_rules.py @@ -0,0 +1,61 @@ +"""Pydantic schemas for categorization rule API requests and responses.""" + +from pydantic import BaseModel, Field + + +class CategorizationRuleCreateRequest(BaseModel): + """Request schema for creating a categorization rule.""" + + match_field: str = Field( + "note", + pattern="^(note|account)$", + description="Transaction field the pattern is matched against", + ) + pattern: str = Field( + ..., + min_length=1, + max_length=255, + description="Case-insensitive substring to match", + ) + category: str = Field( + ..., + min_length=1, + max_length=255, + description="Category to set on match", + ) + subcategory: str | None = Field( + None, + max_length=255, + description="Subcategory to set on match (null clears it)", + ) + is_active: bool = Field(True, description="Whether the rule participates in matching") + sort_order: int = Field(0, ge=0, description="Evaluation order (lower runs first)") + + +class CategorizationRuleUpdateRequest(CategorizationRuleCreateRequest): + """Request schema for fully replacing a categorization rule (not a partial PATCH). + + Same fields and validation as create -- PUT is a full replace, so the + contract is identical by design. + """ + + +class CategorizationRuleResponse(BaseModel): + """Single categorization rule response model.""" + + id: int + match_field: str + pattern: str + category: str + subcategory: str + is_active: bool + sort_order: int + created_at: str + + +class RulesApplyResponse(BaseModel): + """Response for the retroactive rule application endpoint.""" + + matched: int + updated: int + analytics_refreshed: bool diff --git a/backend/src/ledger_sync/schemas/saved_views.py b/backend/src/ledger_sync/schemas/saved_views.py new file mode 100644 index 00000000..bad03141 --- /dev/null +++ b/backend/src/ledger_sync/schemas/saved_views.py @@ -0,0 +1,24 @@ +"""Pydantic schemas for saved filter view API requests and responses.""" + +from typing import Any + +from pydantic import BaseModel, Field + + +class SavedViewCreateRequest(BaseModel): + """Request schema for creating (or upserting by name) a saved filter view.""" + + name: str = Field(..., min_length=1, max_length=100, description="View name (unique per user)") + filters: dict[str, Any] = Field( + default_factory=dict, description="Opaque frontend filter state" + ) + + +class SavedViewResponse(BaseModel): + """Single saved filter view response model.""" + + id: int + name: str + filters: dict[str, Any] + created_at: str + updated_at: str diff --git a/backend/src/ledger_sync/schemas/transactions.py b/backend/src/ledger_sync/schemas/transactions.py index 87e49980..214be317 100644 --- a/backend/src/ledger_sync/schemas/transactions.py +++ b/backend/src/ledger_sync/schemas/transactions.py @@ -38,6 +38,7 @@ class TransactionResponse(BaseModel): source_file: str last_seen_at: str is_transfer: bool + tags: list[str] = Field(default_factory=list) class TransactionsListResponse(BaseModel): @@ -50,6 +51,13 @@ class TransactionsListResponse(BaseModel): has_more: bool +class TagFacet(BaseModel): + """Single tag facet entry: tag name plus live-transaction usage count.""" + + name: str + count: int + + class TransactionFacetsResponse(BaseModel): """Lightweight facets for the Transactions page UI. @@ -61,6 +69,7 @@ class TransactionFacetsResponse(BaseModel): categories: list[str] accounts: list[str] + tags: list[TagFacet] = Field(default_factory=list) income_count: int expense_count: int transfer_count: int @@ -87,3 +96,13 @@ class TransactionCreateRequest(BaseModel): note: str | None = Field(None, description="Optional note or description") from_account: str | None = Field(None, description="Source account (for transfers)") to_account: str | None = Field(None, description="Destination account (for transfers)") + + +class TransactionTagsUpdateRequest(BaseModel): + """Request schema for replacing a transaction's tag list. + + Full replacement: an empty list clears all tags. The router + additionally validates each tag trimmed to 1-50 chars. + """ + + tags: list[str] = Field(..., max_length=10, description="Full replacement tag list") diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 45c47875..ad821ca2 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -4,16 +4,65 @@ from decimal import Decimal import pytest +from fastapi.testclient import TestClient from sqlalchemy import create_engine from sqlalchemy.orm import Session, sessionmaker +from sqlalchemy.pool import StaticPool +from ledger_sync.api.deps import get_current_user +from ledger_sync.api.main import app from ledger_sync.db.base import Base -from ledger_sync.db.models import Transaction, TransactionType, User +from ledger_sync.db.models import Transaction, TransactionType, User, UserPreferences +from ledger_sync.db.session import get_session # Fake bcrypt hash for test fixtures — not a real credential TEST_BCRYPT_HASH = "$2b$12$dummy_hash_for_testing_purposes" # noqa: S105 +@pytest.fixture +def two_user_client(): + """HTTP-boundary fixture: TestClient + two seeded users + swappable auth. + + StaticPool + check_same_thread=False shares one in-memory SQLite + connection between the fixture thread and the TestClient request thread. + ``current["user"]`` can be reassigned mid-test to act as the other user + (user-scoping tests). Yields (client, session, user_a, user_b, current). + """ + engine = create_engine( + "sqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + Base.metadata.create_all(engine) + TestSession = sessionmaker(bind=engine) # noqa: N806 + session = TestSession() + + user_a = User(email="a@example.com", is_active=True, is_verified=True, hashed_password="") + user_b = User(email="b@example.com", is_active=True, is_verified=True, hashed_password="") + session.add_all([user_a, user_b]) + session.flush() + session.add(UserPreferences(user_id=user_a.id, essential_categories="[]")) + session.add(UserPreferences(user_id=user_b.id, essential_categories="[]")) + session.commit() + + current = {"user": user_a} + + def override_get_session(): + yield session + + def override_get_current_user(): + return current["user"] + + app.dependency_overrides[get_session] = override_get_session + app.dependency_overrides[get_current_user] = override_get_current_user + + client = TestClient(app) + yield client, session, user_a, user_b, current + + app.dependency_overrides.clear() + session.close() + + @pytest.fixture def test_db_session() -> Session: """Create a test database session.""" diff --git a/backend/tests/integration/test_categorization_rules.py b/backend/tests/integration/test_categorization_rules.py new file mode 100644 index 00000000..fb4053bf --- /dev/null +++ b/backend/tests/integration/test_categorization_rules.py @@ -0,0 +1,346 @@ +"""Integration tests for the /api/categorization-rules endpoints. + +TestClient with dependency overrides for get_session and get_current_user. +SQLite in-memory with StaticPool + check_same_thread=False so the test +session and the request handler thread share one connection (and therefore +the same in-memory database). +""" + +from __future__ import annotations + +from datetime import datetime +from decimal import Decimal +from typing import Any + +import pytest +from sqlalchemy.orm import Session + +from ledger_sync.core.sync_engine import SyncEngine +from ledger_sync.db.models import ( + Transaction, + TransactionTag, + TransactionType, + User, +) +from ledger_sync.ingest.hash_id import TransactionHasher + +RULE_KEYS = { + "id", + "match_field", + "pattern", + "category", + "subcategory", + "is_active", + "sort_order", + "created_at", +} + + +@pytest.fixture +def rules_client(two_user_client): + """Alias for the shared two-user HTTP fixture (see tests/conftest.py).""" + return two_user_client + + +def _rule_payload(**overrides: Any) -> dict[str, Any]: + payload: dict[str, Any] = { + "match_field": "note", + "pattern": "swiggy", + "category": "Food", + "subcategory": "Delivery", + "is_active": True, + "sort_order": 0, + } + payload.update(overrides) + return payload + + +def _seed_txn( + session: Session, + user_id: int, + tx_id: str, + *, + note: str | None = None, + category: str = "Misc", + subcategory: str | None = None, + account: str = "Cash", + amount: float = 250.0, + tx_type: TransactionType = TransactionType.EXPENSE, + date: datetime | None = None, +) -> Transaction: + txn = Transaction( + transaction_id=tx_id.ljust(64, "0")[:64], + user_id=user_id, + date=date or datetime(2026, 6, 1, 12, 0, 0), # noqa: DTZ001 - naive like SQLite storage + amount=Decimal(str(amount)), + currency="INR", + type=tx_type, + account=account, + category=category, + subcategory=subcategory, + note=note, + source_file="test.xlsx", + ) + session.add(txn) + session.commit() + return txn + + +def _recomputed_id(txn: Transaction, user_id: int, occurrence: int = 0) -> str: + return TransactionHasher().generate_transaction_id( + date=txn.date, + amount=txn.amount, + account=txn.account, + note=txn.note, + category=txn.category, + subcategory=txn.subcategory, + tx_type=txn.type.value, + user_id=user_id, + occurrence=occurrence, + ) + + +# --- CRUD --- + + +def test_crud_roundtrip(rules_client) -> None: + client, _, _, _, _ = rules_client + + # Create (subcategory omitted -> stored NULL -> coalesced to ""). + created = client.post("/api/categorization-rules", json=_rule_payload(subcategory=None)) + assert created.status_code == 201, created.json() + rule = created.json() + assert rule["subcategory"] == "" + rule_id = rule["id"] + + # List contains it. + listed = client.get("/api/categorization-rules") + assert listed.status_code == 200 + assert [r["id"] for r in listed.json()] == [rule_id] + assert listed.json()[0]["pattern"] == "swiggy" + + # Full-replace update. + updated = client.put( + f"/api/categorization-rules/{rule_id}", + json=_rule_payload(pattern="zomato", category="Dining", is_active=False, sort_order=3), + ) + assert updated.status_code == 200, updated.json() + body = updated.json() + assert body["pattern"] == "zomato" + assert body["category"] == "Dining" + assert body["subcategory"] == "Delivery" + assert body["is_active"] is False + assert body["sort_order"] == 3 + + # Delete -> 204 empty body, then list is empty. + deleted = client.delete(f"/api/categorization-rules/{rule_id}") + assert deleted.status_code == 204 + assert deleted.content == b"" + assert client.get("/api/categorization-rules").json() == [] + + +def test_put_unknown_rule_returns_404(rules_client) -> None: + client, _, _, _, _ = rules_client + + r = client.put("/api/categorization-rules/99999", json=_rule_payload()) + + assert r.status_code == 404 + assert r.json() == {"detail": "Rule not found"} + + +def test_delete_is_idempotent_204(rules_client) -> None: + client, _, _, _, _ = rules_client + + assert client.delete("/api/categorization-rules/99999").status_code == 204 + + created = client.post("/api/categorization-rules", json=_rule_payload()).json() + assert client.delete(f"/api/categorization-rules/{created['id']}").status_code == 204 + # Second delete of the same id is still 204. + assert client.delete(f"/api/categorization-rules/{created['id']}").status_code == 204 + + +def test_rules_are_user_scoped(rules_client) -> None: + client, _, _, _, current = rules_client + client.post("/api/categorization-rules", json=_rule_payload()) + rule_id = client.get("/api/categorization-rules").json()[0]["id"] + + current["user"] = _switch_user(rules_client, "b") + as_b = client.get("/api/categorization-rules") + + assert as_b.status_code == 200 + assert as_b.json() == [] + # Cross-user PUT is a 404, never a 403 or a mutation. + assert client.put(f"/api/categorization-rules/{rule_id}", json=_rule_payload()).status_code == ( + 404 + ) + + +def _switch_user(rules_client_tuple, which: str) -> User: + _, _, user_a, user_b, _ = rules_client_tuple + return user_b if which == "b" else user_a + + +def test_response_contract(rules_client) -> None: + client, _, _, _, _ = rules_client + + created = client.post("/api/categorization-rules", json=_rule_payload()) + + assert set(created.json().keys()) == RULE_KEYS + listed = client.get("/api/categorization-rules").json() + assert set(listed[0].keys()) == RULE_KEYS + + +# --- POST /apply --- + + +def test_apply_updates_category_and_rehashes_transaction_id(rules_client) -> None: + client, session, user_a, _, _ = rules_client + seeded = _seed_txn(session, user_a.id, "seed1", note="swiggy order", category="Misc") + old_id = seeded.transaction_id + client.post("/api/categorization-rules", json=_rule_payload(subcategory=None)) + + r = client.post("/api/categorization-rules/apply") + + assert r.status_code == 200, r.json() + body = r.json() + assert body["matched"] == 1 + assert body["updated"] == 1 + assert "analytics_refreshed" in body + + session.expire_all() + row = session.query(Transaction).filter(Transaction.user_id == user_a.id).one() + assert row.category == "Food" + assert row.subcategory is None + assert row.transaction_id != old_id + assert row.transaction_id == _recomputed_id(row, user_a.id) + + +def test_apply_skips_transfers_and_already_correct_rows(rules_client) -> None: + client, session, user_a, _, _ = rules_client + transfer = _seed_txn( + session, + user_a.id, + "xfer1", + note="swiggy wallet topup", + category="Transfer: Cash -> Wallet", + tx_type=TransactionType.TRANSFER, + ) + transfer_id = transfer.transaction_id + # Already at the rule's target category+subcategory. + _seed_txn(session, user_a.id, "ok1", note="swiggy order", category="Food", subcategory=None) + client.post("/api/categorization-rules", json=_rule_payload(subcategory=None)) + + r = client.post("/api/categorization-rules/apply") + + body = r.json() + assert body["matched"] == 1 # only the already-correct expense row + assert body["updated"] == 0 + + session.expire_all() + xfer = session.get(Transaction, transfer_id) + assert xfer is not None + assert xfer.category == "Transfer: Cash -> Wallet" + + +def test_apply_migrates_transaction_tags_to_new_id(rules_client) -> None: + client, session, user_a, _, _ = rules_client + seeded = _seed_txn(session, user_a.id, "tagged1", note="swiggy order", category="Misc") + old_id = seeded.transaction_id + session.add(TransactionTag(user_id=user_a.id, transaction_id=old_id, tag="work")) + session.commit() + client.post("/api/categorization-rules", json=_rule_payload(subcategory=None)) + + r = client.post("/api/categorization-rules/apply") + + assert r.json()["updated"] == 1 + session.expire_all() + row = session.query(Transaction).filter(Transaction.user_id == user_a.id).one() + assert row.transaction_id != old_id + tags = session.query(TransactionTag).filter(TransactionTag.user_id == user_a.id).all() + assert [(t.transaction_id, t.tag) for t in tags] == [(row.transaction_id, "work")] + + +def test_apply_handles_occurrence_collision(rules_client) -> None: + client, session, user_a, _, _ = rules_client + # Identical (date, amount, account, note) so both rehash to the same + # base id once the rule sets the same category on both. + _seed_txn(session, user_a.id, "dupA", note="swiggy order", category="MiscA") + _seed_txn(session, user_a.id, "dupB", note="swiggy order", category="MiscB") + client.post("/api/categorization-rules", json=_rule_payload(subcategory=None)) + + r = client.post("/api/categorization-rules/apply") + + body = r.json() + assert body["matched"] == 2 + assert body["updated"] == 2 + session.expire_all() + rows = session.query(Transaction).filter(Transaction.user_id == user_a.id).all() + assert len(rows) == 2 + ids = {row.transaction_id for row in rows} + assert len(ids) == 2 # distinct final ids despite identical hash inputs + assert all(row.category == "Food" for row in rows) + recomputed = {_recomputed_id(rows[0], user_a.id, occurrence=n) for n in (0, 1)} + assert ids == recomputed + + +def test_apply_with_no_active_rules_returns_zero_counts(rules_client) -> None: + client, session, user_a, _, _ = rules_client + _seed_txn(session, user_a.id, "seed1", note="swiggy order", category="Misc") + client.post("/api/categorization-rules", json=_rule_payload(is_active=False)) + + r = client.post("/api/categorization-rules/apply") + + assert r.status_code == 200 + body = r.json() + assert body["matched"] == 0 + assert body["updated"] == 0 + + +def test_apply_only_touches_current_users_transactions(rules_client) -> None: + client, session, _user_a, user_b, _ = rules_client + b_txn = _seed_txn(session, user_b.id, "bseed", note="swiggy order", category="Misc") + b_id = b_txn.transaction_id + client.post("/api/categorization-rules", json=_rule_payload()) # rule belongs to user A + + r = client.post("/api/categorization-rules/apply") + + assert r.json()["matched"] == 0 + session.expire_all() + b_row = session.get(Transaction, b_id) + assert b_row is not None + assert b_row.category == "Misc" + + +# --- Import-time application --- + + +def test_import_applies_rules_pre_hash_and_reupload_is_idempotent(rules_client) -> None: + client, session, user_a, _, _ = rules_client + client.post("/api/categorization-rules", json=_rule_payload(subcategory=None)) + rows = [ + { + "date": "2026-06-01", + "amount": 250.0, + "currency": "INR", + "type": "Expense", + "account": "Cash", + "category": "Misc", + "subcategory": None, + "note": "swiggy order", + } + ] + engine = SyncEngine(session, user_id=user_a.id) + + first = engine.import_rows(rows, file_name="june.xlsx", file_hash="hash-1") + + assert first.inserted == 1 + stored = session.query(Transaction).filter(Transaction.user_id == user_a.id).one() + assert stored.category == "Food" # rule category, not the raw "Misc" + assert stored.transaction_id == _recomputed_id(stored, user_a.id) + + # Re-uploading the identical rows is a no-op: rules ran pre-hash both + # times, so the recomputed ids match and every row dedups. + second = engine.import_rows(rows, file_name="june.xlsx", file_hash="hash-1", force=True) + + assert second.inserted == 0 + assert session.query(Transaction).filter(Transaction.user_id == user_a.id).count() == 1 diff --git a/backend/tests/integration/test_saved_views.py b/backend/tests/integration/test_saved_views.py new file mode 100644 index 00000000..a4b64811 --- /dev/null +++ b/backend/tests/integration/test_saved_views.py @@ -0,0 +1,96 @@ +"""Integration tests for the /api/saved-views endpoints. + +Covers create/list roundtrip (opaque filters echoed verbatim), upsert-by-name +semantics, idempotent delete, user scoping, and the response contract. +""" + +from __future__ import annotations + +import pytest + +VIEW_KEYS = {"id", "name", "filters", "created_at", "updated_at"} + + +@pytest.fixture +def views_client(two_user_client): + """Alias for the shared two-user HTTP fixture (see tests/conftest.py).""" + return two_user_client + + +def test_post_creates_and_get_lists(views_client) -> None: + client, _, _, _, _ = views_client + filters = { + "category": "Food", + "min_amount": 500, + "tag": "work", + "nested": {"sort_by": "amount", "flags": [1, 2, 3]}, + } + + created = client.post("/api/saved-views", json={"name": "Big food spends", "filters": filters}) + + assert created.status_code == 200, created.json() + assert created.json()["filters"] == filters # echoed verbatim, numbers intact + + listed = client.get("/api/saved-views") + assert listed.status_code == 200 + views = listed.json() + assert len(views) == 1 + assert views[0]["name"] == "Big food spends" + assert views[0]["filters"] == filters + + +def test_post_same_name_upserts(views_client) -> None: + client, _, _, _, _ = views_client + first = client.post( + "/api/saved-views", json={"name": "Monthly", "filters": {"category": "Food"}} + ).json() + + second = client.post( + "/api/saved-views", json={"name": "Monthly", "filters": {"category": "Rent"}} + ).json() + + assert second["id"] == first["id"] # same row, not a new one + assert second["filters"] == {"category": "Rent"} # overwritten + assert second["created_at"] == first["created_at"] + assert second["updated_at"] >= first["updated_at"] + assert len(client.get("/api/saved-views").json()) == 1 + + +def test_delete_204_and_idempotent(views_client) -> None: + client, _, _, _, _ = views_client + view = client.post("/api/saved-views", json={"name": "Temp", "filters": {}}).json() + + deleted = client.delete(f"/api/saved-views/{view['id']}") + + assert deleted.status_code == 204 + assert deleted.content == b"" + assert client.get("/api/saved-views").json() == [] + # Deleting again (and any nonexistent id) is still 204. + assert client.delete(f"/api/saved-views/{view['id']}").status_code == 204 + assert client.delete("/api/saved-views/99999").status_code == 204 + + +def test_views_are_user_scoped(views_client) -> None: + client, _, _, user_b, current = views_client + a_view = client.post("/api/saved-views", json={"name": "A's view", "filters": {}}).json() + + current["user"] = user_b + as_b_list = client.get("/api/saved-views") + as_b_delete = client.delete(f"/api/saved-views/{a_view['id']}") + + assert as_b_list.json() == [] + assert as_b_delete.status_code == 204 # no-op, never a 404/403 + + # A's view is intact after B's delete attempt. + current["user"] = views_client[2] + assert [v["id"] for v in client.get("/api/saved-views").json()] == [a_view["id"]] + + +def test_response_contract(views_client) -> None: + client, _, _, _, _ = views_client + + created = client.post("/api/saved-views", json={"name": "Contract", "filters": {"x": 1}}) + + assert set(created.json().keys()) == VIEW_KEYS + listed = client.get("/api/saved-views").json() + assert set(listed[0].keys()) == VIEW_KEYS diff --git a/backend/tests/integration/test_transaction_tags.py b/backend/tests/integration/test_transaction_tags.py new file mode 100644 index 00000000..81937985 --- /dev/null +++ b/backend/tests/integration/test_transaction_tags.py @@ -0,0 +1,201 @@ +"""Integration tests for transaction tags. + +Covers PUT /api/transactions/{id}/tags (replace-all semantics, +normalization, validation, 404 cases), the ``tag`` filter on +GET /api/transactions/search, tags on list/search responses, and the +tag facets on GET /api/transactions/facets. +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from decimal import Decimal + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy.orm import Session + +from ledger_sync.db.models import Transaction, TransactionType, UserPreferences + +FACET_KEYS = { + "categories", + "accounts", + "tags", + "income_count", + "expense_count", + "transfer_count", + "total_count", +} + + +@pytest.fixture +def tags_client(two_user_client): + """Alias for the shared two-user HTTP fixture (see tests/conftest.py).""" + return two_user_client + + +def _seed_txn( + session: Session, + user_id: int, + tx_id: str, + *, + category: str = "Food", + account: str = "Cash", + is_deleted: bool = False, + tx_type: TransactionType = TransactionType.EXPENSE, +) -> str: + transaction_id = tx_id.ljust(64, "0")[:64] + session.add( + Transaction( + transaction_id=transaction_id, + user_id=user_id, + date=datetime(2026, 6, 1, tzinfo=UTC), + amount=Decimal("100.00"), + currency="INR", + type=tx_type, + account=account, + category=category, + source_file="test.xlsx", + is_deleted=is_deleted, + ) + ) + session.commit() + return transaction_id + + +def _put_tags(client: TestClient, txn_id: str, tags: list[str]): + return client.put(f"/api/transactions/{txn_id}/tags", json={"tags": tags}) + + +# --- PUT /api/transactions/{id}/tags --- + + +def test_put_tags_replaces_full_set(tags_client) -> None: + client, session, user_a, _, _ = tags_client + txn_id = _seed_txn(session, user_a.id, "t1") + + first = _put_tags(client, txn_id, ["a", "b"]) + assert first.status_code == 200, first.json() + assert first.json() == {"transaction_id": txn_id, "tags": ["a", "b"]} + + # Full replacement: only 'c' survives. + second = _put_tags(client, txn_id, ["c"]) + assert second.json()["tags"] == ["c"] + + # Empty list clears everything. + cleared = _put_tags(client, txn_id, []) + assert cleared.status_code == 200 + assert cleared.json()["tags"] == [] + listed = client.get("/api/transactions").json() + assert listed["data"][0]["tags"] == [] + + +def test_put_tags_normalizes(tags_client) -> None: + client, session, user_a, _, _ = tags_client + txn_id = _seed_txn(session, user_a.id, "t1") + + r = _put_tags(client, txn_id, [" work ", "", " ", "work", "travel"]) + + assert r.status_code == 200 + # Trimmed, empties dropped, exact duplicates deduped, order preserved. + assert r.json()["tags"] == ["work", "travel"] + + +def test_put_tags_validation(tags_client) -> None: + client, session, user_a, _, _ = tags_client + txn_id = _seed_txn(session, user_a.id, "t1") + + eleven = [f"tag{i}" for i in range(11)] + assert _put_tags(client, txn_id, eleven).status_code == 422 + + assert _put_tags(client, txn_id, ["x" * 51]).status_code == 422 + + +def test_put_tags_unknown_or_foreign_transaction_404(tags_client) -> None: + client, session, user_a, user_b, _ = tags_client + foreign_id = _seed_txn(session, user_b.id, "foreign") + deleted_id = _seed_txn(session, user_a.id, "gone", is_deleted=True) + + missing = _put_tags(client, "0" * 64, ["work"]) + foreign = _put_tags(client, foreign_id, ["work"]) + soft_deleted = _put_tags(client, deleted_id, ["work"]) + + for response in (missing, foreign, soft_deleted): + assert response.status_code == 404 + assert response.json() == {"detail": "Transaction not found"} + + +# --- Search tag filter --- + + +def test_search_tag_filter_exact_match(tags_client) -> None: + client, session, user_a, _, _ = tags_client + tagged = _seed_txn(session, user_a.id, "tagged", category="Food") + _seed_txn(session, user_a.id, "untagged", category="Food") + other_cat = _seed_txn(session, user_a.id, "rent", category="Rent") + _put_tags(client, tagged, ["work"]) + _put_tags(client, other_cat, ["work"]) + + only_work = client.get("/api/transactions/search", params={"tag": "work"}) + assert only_work.status_code == 200 + assert {t["id"] for t in only_work.json()["data"]} == {tagged, other_cat} + + # Composes (AND) with the category filter. + composed = client.get("/api/transactions/search", params={"tag": "work", "category": "Food"}) + assert [t["id"] for t in composed.json()["data"]] == [tagged] + + # Tags are case-sensitive: 'Work' matches nothing. + case_variant = client.get("/api/transactions/search", params={"tag": "Work"}) + assert case_variant.json()["data"] == [] + + +def test_list_and_search_responses_include_sorted_tags(tags_client) -> None: + client, session, user_a, _, _ = tags_client + tagged = _seed_txn(session, user_a.id, "tagged") + untagged = _seed_txn(session, user_a.id, "plain") + _put_tags(client, tagged, ["zebra", "apple"]) + + listed = client.get("/api/transactions").json()["data"] + searched = client.get("/api/transactions/search").json()["data"] + + for rows in (listed, searched): + by_id = {row["id"]: row for row in rows} + assert by_id[tagged]["tags"] == ["apple", "zebra"] # alphabetical + assert by_id[untagged]["tags"] == [] + + +# --- Facets --- + + +def test_facets_include_tag_counts(tags_client) -> None: + client, session, user_a, _, _ = tags_client + session.query(UserPreferences).filter_by(user_id=user_a.id).update( + {"excluded_accounts": json.dumps(["Excluded Wallet"])} + ) + session.commit() + live_1 = _seed_txn(session, user_a.id, "live1") + live_2 = _seed_txn(session, user_a.id, "live2") + excluded = _seed_txn(session, user_a.id, "excl", account="Excluded Wallet") + # Tag everything BEFORE soft-deleting so the PUT 404 guard doesn't trip. + _put_tags(client, live_1, ["work", "travel"]) + _put_tags(client, live_2, ["work"]) + _put_tags(client, excluded, ["work"]) + dead = _seed_txn(session, user_a.id, "dead") + _put_tags(client, dead, ["work"]) + session.query(Transaction).filter(Transaction.transaction_id == dead).update( + {"is_deleted": True} + ) + session.commit() + + r = client.get("/api/transactions/facets") + + assert r.status_code == 200 + body = r.json() + assert set(body.keys()) == FACET_KEYS + # Soft-deleted and excluded-account transactions don't count: + # work -> live_1 + live_2 only; travel -> live_1. Sorted count desc, name asc. + assert body["tags"] == [ + {"name": "work", "count": 2}, + {"name": "travel", "count": 1}, + ] diff --git a/backend/tests/unit/test_rules_engine.py b/backend/tests/unit/test_rules_engine.py new file mode 100644 index 00000000..cf51cbad --- /dev/null +++ b/backend/tests/unit/test_rules_engine.py @@ -0,0 +1,151 @@ +"""Unit tests for the categorization rule engine (core/rules.py). + +Covers match_rule semantics (case-insensitive substring, field selection, +empty patterns), apply_rules_to_row (first-match-wins, transfer exemption, +subcategory clearing), and load_active_rules ordering/filtering. +""" + +from __future__ import annotations + +from typing import Any + +from sqlalchemy.orm import Session + +from ledger_sync.core.rules import apply_rules_to_row, load_active_rules, match_rule +from ledger_sync.db.models import CategorizationRule, User + + +def _rule( + pattern: str, + category: str, + *, + match_field: str = "note", + subcategory: str | None = None, +) -> CategorizationRule: + """Build an unpersisted rule for pure matching tests.""" + return CategorizationRule( + match_field=match_field, + pattern=pattern, + category=category, + subcategory=subcategory, + ) + + +def _row( + note: str | None = None, + account: str = "HDFC", + *, + is_transfer: bool = False, + category: str = "Misc", + subcategory: str | None = "Old Sub", +) -> dict[str, Any]: + return { + "note": note, + "account": account, + "category": category, + "subcategory": subcategory, + "is_transfer": is_transfer, + } + + +def _persist_rule( + session: Session, + user: User, + pattern: str, + category: str, + *, + sort_order: int = 0, + is_active: bool = True, +) -> CategorizationRule: + rule = CategorizationRule( + user_id=user.id, + match_field="note", + pattern=pattern, + category=category, + subcategory=None, + is_active=is_active, + sort_order=sort_order, + ) + session.add(rule) + session.commit() + return rule + + +def test_match_is_case_insensitive_substring() -> None: + rule = _rule("SWIGGY", "Food") + + assert match_rule(rule, "UPI-swiggy-food", None) is True + assert match_rule(rule, "no delivery apps here", None) is False + + +def test_first_match_wins_by_sort_order_then_id(test_db_session: Session, test_user: User) -> None: + # Lower sort_order wins even when inserted later. + later_but_first = _persist_rule(test_db_session, test_user, "swiggy", "Winner", sort_order=0) + _persist_rule(test_db_session, test_user, "swiggy", "Loser", sort_order=1) + + rules = load_active_rules(test_db_session, test_user.id) + row = _row(note="swiggy order") + assert apply_rules_to_row(rules, row) is True + assert row["category"] == later_but_first.category == "Winner" + + # Tie on sort_order: lower id (inserted first) wins. + tie_a = _persist_rule(test_db_session, test_user, "zomato", "TieFirst", sort_order=5) + tie_b = _persist_rule(test_db_session, test_user, "zomato", "TieSecond", sort_order=5) + assert tie_a.id < tie_b.id + + rules = load_active_rules(test_db_session, test_user.id) + row = _row(note="zomato dinner") + apply_rules_to_row(rules, row) + assert row["category"] == "TieFirst" + + +def test_inactive_rules_are_skipped(test_db_session: Session, test_user: User) -> None: + _persist_rule(test_db_session, test_user, "swiggy", "Food", is_active=False) + + rules = load_active_rules(test_db_session, test_user.id) + row = _row(note="swiggy order") + + assert rules == [] + assert apply_rules_to_row(rules, row) is False + assert row["category"] == "Misc" + + +def test_transfer_rows_are_never_mutated() -> None: + rules = [_rule("swiggy", "Food")] + row = _row(note="swiggy order", is_transfer=True) + + assert apply_rules_to_row(rules, row) is False + assert row["category"] == "Misc" + assert row["subcategory"] == "Old Sub" + + +def test_match_sets_category_and_subcategory_including_none() -> None: + # subcategory=None on the rule clears the row's existing subcategory. + rules = [_rule("swiggy", "Food", subcategory=None)] + row = _row(note="swiggy order", subcategory="Old Sub") + + assert apply_rules_to_row(rules, row) is True + assert row["category"] == "Food" + assert row["subcategory"] is None + + # A non-null rule subcategory is set as-is. + rules = [_rule("uber", "Transportation", subcategory="Cabs")] + row = _row(note="uber trip", subcategory=None) + apply_rules_to_row(rules, row) + assert row["category"] == "Transportation" + assert row["subcategory"] == "Cabs" + + +def test_account_match_field_checks_account_not_note() -> None: + rule = _rule("hdfc", "Banking", match_field="account") + + assert match_rule(rule, None, "HDFC Savings") is True + # Pattern present in the note but not the account must NOT match. + assert match_rule(rule, "paid via hdfc upi", "Cash") is False + + +def test_empty_or_whitespace_pattern_never_matches() -> None: + rule = _rule(" ", "Food") + + assert match_rule(rule, "anything at all", "any account") is False + assert match_rule(rule, "", "") is False diff --git a/frontend/src/components/transactions/SavedViewsMenu.tsx b/frontend/src/components/transactions/SavedViewsMenu.tsx new file mode 100644 index 00000000..7b6e6921 --- /dev/null +++ b/frontend/src/components/transactions/SavedViewsMenu.tsx @@ -0,0 +1,132 @@ +import { useState, useRef } from 'react' + +import { Bookmark, Trash2 } from 'lucide-react' +import { toast } from 'sonner' + +import { useSavedViews, useSaveView, useDeleteView } from '@/hooks/api/useSavedViews' +import { useDismissable } from '@/hooks/useDismissable' + +import type { FilterValues } from './TransactionFilters' + +interface SavedViewsMenuProps { + currentFilters: FilterValues + onApply: (filters: FilterValues) => void +} + +/** + * Dropdown in the Transactions filter bar: save the current filter set under + * a name, apply a saved view, or delete one. Saving upserts by name. + */ +export default function SavedViewsMenu({ currentFilters, onApply }: Readonly) { + const [open, setOpen] = useState(false) + const [name, setName] = useState('') + const ref = useRef(null) + + const { data: views } = useSavedViews() + const saveView = useSaveView() + const deleteView = useDeleteView() + + useDismissable(open, ref, () => setOpen(false)) + + const handleSave = () => { + const trimmed = name.trim() + if (!trimmed) return + // FilterValues is an interface (no implicit index signature), so it needs + // a spread into a fresh object to satisfy Record. + saveView.mutate( + { name: trimmed, filters: { ...currentFilters } }, + { + onSuccess: () => { + setName('') + toast.success('View saved') + }, + onError: () => toast.error('Failed to save view'), + }, + ) + } + + const handleDelete = (id: number) => { + deleteView.mutate(id, { + onError: () => toast.error('Failed to delete view'), + }) + } + + return ( +
+ + + {open && ( +
+ {/* Saved view list */} + {views && views.length > 0 ? ( +
    + {views.map((view) => ( +
  • + + +
    • + ))} +
+ ) : ( +

No saved views yet

+ )} + +
+ + {/* Save current view */} +
+ setName(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') handleSave() + }} + placeholder="View name" + maxLength={100} + className="w-full px-3 py-2 bg-[var(--overlay-2)] border border-border rounded-lg text-foreground text-sm focus:border-primary focus:outline-none transition-colors min-h-[44px]" + aria-label="Name for saved view" + /> + +
+
+ )} +
+ ) +} diff --git a/frontend/src/components/transactions/TagChips.tsx b/frontend/src/components/transactions/TagChips.tsx new file mode 100644 index 00000000..cc7de8ad --- /dev/null +++ b/frontend/src/components/transactions/TagChips.tsx @@ -0,0 +1,15 @@ +/** Small rounded tag chips rendered under the category cell and in mobile cards. */ +export default function TagChips({ tags }: Readonly<{ tags: string[] }>) { + return ( +
+ {tags.map((tag) => ( + + {tag} + + ))} +
+ ) +} diff --git a/frontend/src/components/transactions/TagEditor.tsx b/frontend/src/components/transactions/TagEditor.tsx new file mode 100644 index 00000000..5ee1f649 --- /dev/null +++ b/frontend/src/components/transactions/TagEditor.tsx @@ -0,0 +1,152 @@ +import { useState, useRef } from 'react' + +import { Tag, Plus } from 'lucide-react' +import { toast } from 'sonner' + +import { useUpdateTransactionTags } from '@/hooks/api/useTags' +import { useDismissable } from '@/hooks/useDismissable' + +const MAX_TAGS = 10 +const MAX_TAG_LENGTH = 50 + +interface TagEditorProps { + transactionId: string + tags: string[] + availableTags: string[] +} + +/** + * Inline popover for editing a transaction's tags: check/uncheck existing + * tags or add a new one. Apply replaces the full tag list via PUT. + */ +export default function TagEditor({ transactionId, tags, availableTags }: Readonly) { + const [open, setOpen] = useState(false) + const [draft, setDraft] = useState(tags) + const [newTag, setNewTag] = useState('') + const ref = useRef(null) + + const updateTags = useUpdateTransactionTags() + + // Re-seed the draft each time the popover opens (discard stale edits) + const handleOpen = () => { + setDraft(tags) + setNewTag('') + setOpen(true) + } + + // Closing by outside click / Escape discards the draft + useDismissable(open, ref, () => setOpen(false)) + + const toggleTag = (tag: string) => { + setDraft((prev) => (prev.includes(tag) ? prev.filter((t) => t !== tag) : [...prev, tag])) + } + + const handleAddNew = () => { + const trimmed = newTag.trim() + if (!trimmed || trimmed.length > MAX_TAG_LENGTH) return + if (draft.includes(trimmed) || draft.length >= MAX_TAGS) return + setDraft((prev) => [...prev, trimmed]) + setNewTag('') + } + + const handleApply = () => { + updateTags.mutate( + { transactionId, tags: draft }, + { + onSuccess: () => setOpen(false), + onError: () => toast.error('Failed to update tags'), + }, + ) + } + + // All checkable options: known facet tags plus draft-only new ones + const options = [...new Set([...availableTags, ...draft])] + const atCap = draft.length >= MAX_TAGS + + return ( +
+ + + {open && ( +
+ {options.length > 0 ? ( +
    + {options.map((tag) => ( +
  • + +
    • + ))} +
+ ) : ( +

No tags yet

+ )} + + {/* Add new tag */} +
+ setNewTag(e.target.value)} + onKeyDown={(e) => { + if (e.key === 'Enter') handleAddNew() + }} + placeholder="New tag" + maxLength={MAX_TAG_LENGTH} + disabled={atCap} + className="w-full px-3 py-2 bg-[var(--overlay-2)] border border-border rounded-lg text-foreground text-sm focus:border-primary focus:outline-none transition-colors min-h-[44px] disabled:opacity-50" + aria-label="New tag name" + /> + +
+ {atCap && ( +

Maximum of {MAX_TAGS} tags per transaction

+ )} + + {/* Apply / Cancel */} +
+ + +
+
+ )} +
+ ) +} diff --git a/frontend/src/components/transactions/TransactionFilters.tsx b/frontend/src/components/transactions/TransactionFilters.tsx index ea5dd96c..1337b215 100644 --- a/frontend/src/components/transactions/TransactionFilters.tsx +++ b/frontend/src/components/transactions/TransactionFilters.tsx @@ -1,12 +1,21 @@ -import { useState, useCallback, useEffect, useEffectEvent, useRef } from 'react' +import { useState, useCallback, useEffect, useEffectEvent, useRef, type ReactNode } from 'react' import { Search, Filter, X, Calendar } from 'lucide-react' import { motion, AnimatePresence } from 'framer-motion' import { usePreferencesStore } from '@/store/preferencesStore' +import type { TagFacet } from '@/services/api/transactions' interface TransactionFiltersProps { onFilterChange: (filters: FilterValues) => void categories: string[] accounts: string[] + /** + * Seeds the internal filter state on mount. The parent re-seeds by + * remounting this component via a `key` change (saved-view apply). + */ + initialValues?: FilterValues + tagOptions?: TagFacet[] + /** Rendered in the search-bar row between the input and the Filters toggle. */ + savedViewsSlot?: ReactNode } export interface FilterValues { @@ -14,6 +23,7 @@ export interface FilterValues { category?: string account?: string type?: string + tag?: string start_date?: string end_date?: string min_amount?: number @@ -34,9 +44,16 @@ function useDebounce(value: T, delay: number): T { return debouncedValue } -export default function TransactionFilters({ onFilterChange, categories, accounts }: Readonly) { - const [filters, setFilters] = useState({}) - const [searchQuery, setSearchQuery] = useState('') +export default function TransactionFilters({ + onFilterChange, + categories, + accounts, + initialValues, + tagOptions = [], + savedViewsSlot, +}: Readonly) { + const [filters, setFilters] = useState(initialValues ?? {}) + const [searchQuery, setSearchQuery] = useState(initialValues?.query ?? '') const [showAdvanced, setShowAdvanced] = useState(false) const isFirstRender = useRef(true) const currencySymbol = usePreferencesStore((state) => state.displayPreferences.currencySymbol) @@ -107,6 +124,7 @@ export default function TransactionFilters({ onFilterChange, categories, account aria-label="Search transactions" />
+ {savedViewsSlot}