From 7d5ade6b6a5fb87369cb1caa0200c02796c9c14b Mon Sep 17 00:00:00 2001 From: Jonas Miederer Date: Mon, 7 Sep 2026 08:39:06 +0000 Subject: [PATCH] chore: switch to all-SQLite stack (drop Postgres service) --- .devcontainer/Dockerfile.lfx | 2 +- .devcontainer/docker-compose.yml | 41 ++++++++++++++++---------------- .devcontainer/init-db.sql | 7 ------ .devcontainer/setup.sh | 12 ++++++++-- .env.example | 18 +++++++------- .gitignore | 6 +++++ backend/app/db.py | 15 +++++++++++- backend/app/settings.py | 11 +++++---- backend/pyproject.toml | 4 ++-- backend/scripts/smoke_e2e.py | 23 ++++++++++++++---- backend/tests/conftest.py | 23 +++++++++--------- backend/tests/test_models.py | 4 +++- 12 files changed, 104 insertions(+), 62 deletions(-) delete mode 100644 .devcontainer/init-db.sql diff --git a/.devcontainer/Dockerfile.lfx b/.devcontainer/Dockerfile.lfx index 42c9480..00514b0 100644 --- a/.devcontainer/Dockerfile.lfx +++ b/.devcontainer/Dockerfile.lfx @@ -8,7 +8,7 @@ ENV PYTHONUNBUFFERED=1 \ PIP_NO_CACHE_DIR=1 # lfx = flow runtime; requests = convenience for custom-component code. -RUN pip install "lfx==1.12.0" requests +RUN pip install lfx requests RUN mkdir -p /flows VOLUME /flows diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 7c0b6a9..9acbe67 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -19,26 +19,8 @@ services: - "8000:8000" # FastAPI - "3000:3000" # Vite - db: - image: postgres:16-trixie - env_file: - - ../.env - volumes: - - postgres-data:/var/lib/postgresql/data - - ./init-db.sql:/docker-entrypoint-initdb.d/init-db.sql:ro - ports: - - "5432:5432" - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 5s - timeout: 3s - retries: 10 - langflow: - image: langflowai/langflow:1.12.0 - depends_on: - db: - condition: service_healthy + image: langflowai/langflow:latest env_file: - ../.env environment: @@ -49,6 +31,9 @@ services: PORT: "7860" LANGFLOW_AUTO_LOGIN: "false" LANGFLOW_API_KEY_SOURCE: env + # SQLite file inside the langflow-data volume (absolute path required + # by Langflow); overrides any postgres URL in ../.env. + LANGFLOW_DATABASE_URL: sqlite:////app/langflow/langflow.db volumes: - langflow-data:/app/langflow ports: @@ -85,9 +70,23 @@ services: retries: 12 start_period: 30s + # Optional nightly auto-updates of :latest images. Not part of the default + # dev startup (no container updates needed while iterating). + watchtower: + image: nickfedor/watchtower:latest + profiles: ["updates"] + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + WATCHTOWER_LABEL_ENABLE: "true" + WATCHTOWER_SCHEDULE: "0 0 4 * * *" + WATCHTOWER_CLEANUP: "true" + WATCHTOWER_ROLLING_RESTART: "true" + labels: + com.centurylinklabs.watchtower.enable: "true" + volumes: kirche-einfach-venv: kirche-einfach-node-modules: - postgres-data: langflow-data: - lfx-data: + lfx-data: \ No newline at end of file diff --git a/.devcontainer/init-db.sql b/.devcontainer/init-db.sql deleted file mode 100644 index 795450a..0000000 --- a/.devcontainer/init-db.sql +++ /dev/null @@ -1,7 +0,0 @@ --- Runs once on first init of the postgres-data volume. --- The main app database ($POSTGRES_DB) is created by the image itself. --- Langflow (authoring UI) gets its own database on the same instance. -CREATE DATABASE langflow; - --- Isolated database for the pytest suite (migrated via Alembic in conftest). -CREATE DATABASE kirche_einfach_test; diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh index 5b9be3b..1ed3096 100644 --- a/.devcontainer/setup.sh +++ b/.devcontainer/setup.sh @@ -13,7 +13,7 @@ echo "--- Installing Node dependencies ---" cd /workspaces/kirche-einfach/frontend npm install -# Compose services (db, langflow, lfx) must be up; fail loudly otherwise. +# Compose services (langflow, lfx) must be up; fail loudly otherwise. echo "--- Waiting for compose services ---" if [ ! -f /workspaces/kirche-einfach/.env ]; then echo "ERROR: /workspaces/kirche-einfach/.env is missing. Copy .env.example to .env and fill in secrets." >&2 @@ -24,7 +24,7 @@ import socket import sys import time -targets = [("db", 5432), ("langflow", 7860), ("lfx", 6004)] +targets = [("langflow", 7860), ("lfx", 6004)] deadline = time.time() + 180 pending = dict(targets) while pending and time.time() < deadline: @@ -45,6 +45,14 @@ if pending: sys.exit(f"ERROR: compose services not ready: {sorted(pending)}") EOF +# Migrate the SQLite schema and reconcile backend/flows into the DB + LFX. +echo "--- Migrating database ---" +cd /workspaces/kirche-einfach/backend +uv run alembic upgrade head + +echo "--- Syncing canonical flows ---" +uv run python scripts/sync_flows.py --sync || echo "WARN: flow sync failed (check backend/flows and LFX)" + echo "" echo "=== Setup complete ===" echo "Forgejo username: $FORGEJO_USERNAME" diff --git a/.env.example b/.env.example index 05f1325..cb90e32 100644 --- a/.env.example +++ b/.env.example @@ -1,12 +1,14 @@ # Copy to `.env` and fill in values. `.env` is gitignored. FORGEJO_TOKEN= -# --- databases (compose service `db`) --- -POSTGRES_USER=kirche -POSTGRES_PASSWORD= -POSTGRES_DB=kirche_einfach -DATABASE_URL=postgresql+psycopg://kirche:@db:5432/kirche_einfach -TEST_DATABASE_URL=postgresql+psycopg://kirche:@db:5432/kirche_einfach_test +# --- app database (SQLite, default if unset) --- +# Override only if you want a different engine/URL. +# DATABASE_URL=sqlite:///backend/data/kirche.db + +# --- canonical flow definitions --- +# FLOWS_DIR=backend/flows +# Auto-reconcile backend/flows into DB + LFX while the API runs. +# FLOWS_WATCH_ENABLED=true # --- langflow (authoring, dev only) --- LANGFLOW_URL=http://langflow:7860 @@ -15,7 +17,7 @@ LANGFLOW_SUPERUSER_PASSWORD= LANGFLOW_SECRET_KEY= # Single static key; validated against env (LANGFLOW_API_KEY_SOURCE=env), superuser scope. LANGFLOW_API_KEY=sk- -LANGFLOW_DATABASE_URL=postgresql://kirche:@db:5432/langflow +# Langflow stores flows in its own SQLite file when unset. # --- lfx (flow runtime) --- LFX_URL=http://lfx:6004 @@ -25,4 +27,4 @@ LFX_API_KEY=lfx- CALLBACK_BASE_URL=http://app:8000 # --- LLM (optional; tests mock the provider without it) --- -# LLM_API_KEY= +# LLM_API_KEY= \ No newline at end of file diff --git a/.gitignore b/.gitignore index ecebd9c..ba92757 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,12 @@ dist/ .env .env.local +# Runtime SQLite databases +backend/data/ +*.db +*.db-shm +*.db-wal + # IDE .vscode/ .idea/ diff --git a/backend/app/db.py b/backend/app/db.py index 668fede..768c2ea 100644 --- a/backend/app/db.py +++ b/backend/app/db.py @@ -1,6 +1,7 @@ """Database engine, sessions, and FastAPI dependency.""" from collections.abc import Iterator +from pathlib import Path from sqlalchemy import Engine from sqlmodel import Session, SQLModel, create_engine @@ -10,11 +11,23 @@ from .settings import settings _engine: Engine | None = None +def _engine_kwargs(url: str) -> dict: + if url.startswith("sqlite"): + # FastAPI runs sync endpoints in a threadpool; connections are shared + # across threads, so defer the same-thread check. File-based SQLite + # needs an existing parent dir (SQLite won't create it). + return {"connect_args": {"check_same_thread": False}} + return {} + + def get_engine(url: str | None = None) -> Engine: """Process-wide engine singleton (override URL in tests via settings).""" global _engine if _engine is None: - _engine = create_engine(url or settings.database_url) + url = url or settings.database_url + if url.startswith("sqlite:///") and not url.startswith("sqlite:///:memory:"): + Path(url.removeprefix("sqlite:///")).parent.mkdir(parents=True, exist_ok=True) + _engine = create_engine(url, **_engine_kwargs(url)) return _engine diff --git a/backend/app/settings.py b/backend/app/settings.py index 8f4b7f5..f3373df 100644 --- a/backend/app/settings.py +++ b/backend/app/settings.py @@ -13,10 +13,13 @@ class Settings(BaseSettings): version: str = "0.1.0" debug: bool = False - database_url: str = "postgresql+psycopg://kirche:kirche@localhost:5432/kirche_einfach" - test_database_url: str = ( - "postgresql+psycopg://kirche:kirche@localhost:5432/kirche_einfach_test" - ) + # App database (SQLite file). Override via DATABASE_URL for other engines. + database_url: str = str(REPO_ROOT / "backend" / "data" / "kirche.db") + + # Canonical flow definitions (source of truth): one Langflow export per file. + flows_dir: str = str(REPO_ROOT / "backend" / "flows") + # Auto-reconcile backend/flows into the DB + LFX while the API is running. + flows_watch_enabled: bool = True langflow_url: str = "http://localhost:7860" langflow_api_key: str = "" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index d8ffaae..bfec02f 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -7,10 +7,10 @@ dependencies = [ "alembic>=1.14", "fastapi[standard]>=0.115.0", "httpx>=0.28", - "psycopg[binary]>=3.2", "pydantic-settings>=2.0", "sqlmodel>=0.0.24", "uvicorn[standard]>=0.30.0", + "watchfiles>=0.24", ] [build-system] @@ -27,7 +27,7 @@ testpaths = ["tests"] [dependency-groups] dev = [ - "lfx==1.12.0", + "lfx", "pytest>=8", "pytest-asyncio>=0.24", "pytest-cov>=6", diff --git a/backend/scripts/smoke_e2e.py b/backend/scripts/smoke_e2e.py index 90cb3d2..4ce0692 100644 --- a/backend/scripts/smoke_e2e.py +++ b/backend/scripts/smoke_e2e.py @@ -1,7 +1,8 @@ """Manual end-to-end smoke: real LFX + real API, frontend-shaped calls. Usage (from backend/): - TEST_DATABASE_URL=postgresql+psycopg://... uv run python scripts/smoke_e2e.py + uv run python scripts/smoke_e2e.py +Runs the API against a throwaway SQLite file and an ephemeral LFX pass. Exits non-zero on any unexpected status/body. """ @@ -11,7 +12,7 @@ import json import os import subprocess import sys -import time +import tempfile import urllib.request from pathlib import Path @@ -54,17 +55,31 @@ def check(name: str, cond: bool, detail: str = "") -> None: def main() -> None: + tmp = Path(tempfile.mkdtemp(prefix="kirche-smoke-")) flow_dir = Path("/tmp/smoke-e2e-flows") flow_dir.mkdir(exist_ok=True) (flow_dir / "seed.json").write_text( (FIXTURES / "simple_rule.json").read_text() ) + # Migrate a throwaway SQLite schema to head before starting the API. + from alembic import command as alembic_command + from alembic.config import Config + + db_url = f"sqlite:///{tmp / 'smoke.db'}" + cfg = Config() + cfg.set_main_option("script_location", str(BACKEND / "migrations")) + cfg.set_main_option("sqlalchemy.url", db_url) + alembic_command.upgrade(cfg, "head") + env = { **os.environ, + # lfx prefers LANGFLOW_API_KEY over LFX_API_KEY; the repo .env may set + # it. Clear it so the serve key below is the one we send. + "LANGFLOW_API_KEY": "", "LFX_API_KEY": LFX_KEY, - "DATABASE_URL": os.environ["TEST_DATABASE_URL"], + "DATABASE_URL": db_url, + "FLOWS_WATCH_ENABLED": "false", "LFX_URL": f"http://127.0.0.1:{LFX_PORT}", - "LFX_API_KEY": LFX_KEY, "CALLBACK_BASE_URL": f"http://127.0.0.1:{API_PORT}", } lfx = subprocess.Popen( diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 7629df3..16dafb6 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -1,25 +1,22 @@ """Shared pytest fixtures. -Requires TEST_DATABASE_URL (a dedicated Postgres database, e.g. the -`kirche_einfach_test` database from the compose stack). The URL is mapped -onto DATABASE_URL *before* any app module is imported, so settings and the -alembic env both target the test database. Never point this at production. +Tests run on a throwaway SQLite file (no external database required). +`DATABASE_URL` and `FLOWS_DIR` are pointed at a session temp dir *before* any +app module is imported, so settings pick them up at import time. The app +schema is migrated to head once per session via Alembic. """ import os +import tempfile from pathlib import Path import pytest -TEST_DB_URL = os.environ.get("TEST_DATABASE_URL") -if not TEST_DB_URL: - raise RuntimeError( - "TEST_DATABASE_URL is not set. Refusing to run tests without " - "an explicit test database (see .env.example)." - ) +TEST_AREA = Path(tempfile.mkdtemp(prefix="kirche-tests-")) # Must precede all `app.*` imports: Settings is instantiated at import time. -os.environ["DATABASE_URL"] = TEST_DB_URL +os.environ["DATABASE_URL"] = f"sqlite:///{TEST_AREA / 'test.db'}" +os.environ["FLOWS_DIR"] = str(TEST_AREA / "flows") BACKEND_DIR = Path(__file__).resolve().parent.parent @@ -115,6 +112,10 @@ def lfx_server(tmp_path_factory): env = { **os.environ, # lfx serve refuses to start without a serve key; requests need it too. + # get_api_key() prefers LANGFLOW_API_KEY over LFX_API_KEY, and the repo + # .env may put a real Langflow key in the environment — clear it so the + # test key below is the one the client sends. + "LANGFLOW_API_KEY": "", "LFX_API_KEY": "test-lfx-key", } proc = subprocess.Popen( diff --git a/backend/tests/test_models.py b/backend/tests/test_models.py index 40a82de..0f3c650 100644 --- a/backend/tests/test_models.py +++ b/backend/tests/test_models.py @@ -16,7 +16,9 @@ def test_flow_definition_defaults(session): assert d.status == "active" assert d.continuation_flow_id is None assert d.created_at is not None - assert d.created_at.tzinfo is not None + # Timestamps are written as UTC. SQLite does not round-trip tzinfo, so + # after a refresh the value is calendar-time UTC without tzinfo. + assert d.created_at.year >= 2026 def test_flow_definition_name_unique(session):