chore: switch to all-SQLite stack (drop Postgres service)
This commit is contained in:
parent
74831483eb
commit
7d5ade6b6a
12 changed files with 104 additions and 62 deletions
|
|
@ -8,7 +8,7 @@ ENV PYTHONUNBUFFERED=1 \
|
|||
PIP_NO_CACHE_DIR=1
|
||||
|
||||
# lfx = flow runtime; requests = convenience for custom-component code.
|
||||
RUN pip install "lfx==1.12.0" requests
|
||||
RUN pip install lfx requests
|
||||
|
||||
RUN mkdir -p /flows
|
||||
VOLUME /flows
|
||||
|
|
|
|||
|
|
@ -19,26 +19,8 @@ services:
|
|||
- "8000:8000" # FastAPI
|
||||
- "3000:3000" # Vite
|
||||
|
||||
db:
|
||||
image: postgres:16-trixie
|
||||
env_file:
|
||||
- ../.env
|
||||
volumes:
|
||||
- postgres-data:/var/lib/postgresql/data
|
||||
- ./init-db.sql:/docker-entrypoint-initdb.d/init-db.sql:ro
|
||||
ports:
|
||||
- "5432:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U postgres"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
|
||||
langflow:
|
||||
image: langflowai/langflow:1.12.0
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
image: langflowai/langflow:latest
|
||||
env_file:
|
||||
- ../.env
|
||||
environment:
|
||||
|
|
@ -49,6 +31,9 @@ services:
|
|||
PORT: "7860"
|
||||
LANGFLOW_AUTO_LOGIN: "false"
|
||||
LANGFLOW_API_KEY_SOURCE: env
|
||||
# SQLite file inside the langflow-data volume (absolute path required
|
||||
# by Langflow); overrides any postgres URL in ../.env.
|
||||
LANGFLOW_DATABASE_URL: sqlite:////app/langflow/langflow.db
|
||||
volumes:
|
||||
- langflow-data:/app/langflow
|
||||
ports:
|
||||
|
|
@ -85,9 +70,23 @@ services:
|
|||
retries: 12
|
||||
start_period: 30s
|
||||
|
||||
# Optional nightly auto-updates of :latest images. Not part of the default
|
||||
# dev startup (no container updates needed while iterating).
|
||||
watchtower:
|
||||
image: nickfedor/watchtower:latest
|
||||
profiles: ["updates"]
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
environment:
|
||||
WATCHTOWER_LABEL_ENABLE: "true"
|
||||
WATCHTOWER_SCHEDULE: "0 0 4 * * *"
|
||||
WATCHTOWER_CLEANUP: "true"
|
||||
WATCHTOWER_ROLLING_RESTART: "true"
|
||||
labels:
|
||||
com.centurylinklabs.watchtower.enable: "true"
|
||||
|
||||
volumes:
|
||||
kirche-einfach-venv:
|
||||
kirche-einfach-node-modules:
|
||||
postgres-data:
|
||||
langflow-data:
|
||||
lfx-data:
|
||||
lfx-data:
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
-- Runs once on first init of the postgres-data volume.
|
||||
-- The main app database ($POSTGRES_DB) is created by the image itself.
|
||||
-- Langflow (authoring UI) gets its own database on the same instance.
|
||||
CREATE DATABASE langflow;
|
||||
|
||||
-- Isolated database for the pytest suite (migrated via Alembic in conftest).
|
||||
CREATE DATABASE kirche_einfach_test;
|
||||
|
|
@ -13,7 +13,7 @@ echo "--- Installing Node dependencies ---"
|
|||
cd /workspaces/kirche-einfach/frontend
|
||||
npm install
|
||||
|
||||
# Compose services (db, langflow, lfx) must be up; fail loudly otherwise.
|
||||
# Compose services (langflow, lfx) must be up; fail loudly otherwise.
|
||||
echo "--- Waiting for compose services ---"
|
||||
if [ ! -f /workspaces/kirche-einfach/.env ]; then
|
||||
echo "ERROR: /workspaces/kirche-einfach/.env is missing. Copy .env.example to .env and fill in secrets." >&2
|
||||
|
|
@ -24,7 +24,7 @@ import socket
|
|||
import sys
|
||||
import time
|
||||
|
||||
targets = [("db", 5432), ("langflow", 7860), ("lfx", 6004)]
|
||||
targets = [("langflow", 7860), ("lfx", 6004)]
|
||||
deadline = time.time() + 180
|
||||
pending = dict(targets)
|
||||
while pending and time.time() < deadline:
|
||||
|
|
@ -45,6 +45,14 @@ if pending:
|
|||
sys.exit(f"ERROR: compose services not ready: {sorted(pending)}")
|
||||
EOF
|
||||
|
||||
# Migrate the SQLite schema and reconcile backend/flows into the DB + LFX.
|
||||
echo "--- Migrating database ---"
|
||||
cd /workspaces/kirche-einfach/backend
|
||||
uv run alembic upgrade head
|
||||
|
||||
echo "--- Syncing canonical flows ---"
|
||||
uv run python scripts/sync_flows.py --sync || echo "WARN: flow sync failed (check backend/flows and LFX)"
|
||||
|
||||
echo ""
|
||||
echo "=== Setup complete ==="
|
||||
echo "Forgejo username: $FORGEJO_USERNAME"
|
||||
|
|
|
|||
18
.env.example
18
.env.example
|
|
@ -1,12 +1,14 @@
|
|||
# Copy to `.env` and fill in values. `.env` is gitignored.
|
||||
FORGEJO_TOKEN=
|
||||
|
||||
# --- databases (compose service `db`) ---
|
||||
POSTGRES_USER=kirche
|
||||
POSTGRES_PASSWORD=
|
||||
POSTGRES_DB=kirche_einfach
|
||||
DATABASE_URL=postgresql+psycopg://kirche:<password>@db:5432/kirche_einfach
|
||||
TEST_DATABASE_URL=postgresql+psycopg://kirche:<password>@db:5432/kirche_einfach_test
|
||||
# --- app database (SQLite, default if unset) ---
|
||||
# Override only if you want a different engine/URL.
|
||||
# DATABASE_URL=sqlite:///backend/data/kirche.db
|
||||
|
||||
# --- canonical flow definitions ---
|
||||
# FLOWS_DIR=backend/flows
|
||||
# Auto-reconcile backend/flows into DB + LFX while the API runs.
|
||||
# FLOWS_WATCH_ENABLED=true
|
||||
|
||||
# --- langflow (authoring, dev only) ---
|
||||
LANGFLOW_URL=http://langflow:7860
|
||||
|
|
@ -15,7 +17,7 @@ LANGFLOW_SUPERUSER_PASSWORD=
|
|||
LANGFLOW_SECRET_KEY=
|
||||
# Single static key; validated against env (LANGFLOW_API_KEY_SOURCE=env), superuser scope.
|
||||
LANGFLOW_API_KEY=sk-
|
||||
LANGFLOW_DATABASE_URL=postgresql://kirche:<password>@db:5432/langflow
|
||||
# Langflow stores flows in its own SQLite file when unset.
|
||||
|
||||
# --- lfx (flow runtime) ---
|
||||
LFX_URL=http://lfx:6004
|
||||
|
|
@ -25,4 +27,4 @@ LFX_API_KEY=lfx-
|
|||
CALLBACK_BASE_URL=http://app:8000
|
||||
|
||||
# --- LLM (optional; tests mock the provider without it) ---
|
||||
# LLM_API_KEY=
|
||||
# LLM_API_KEY=
|
||||
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -14,6 +14,12 @@ dist/
|
|||
.env
|
||||
.env.local
|
||||
|
||||
# Runtime SQLite databases
|
||||
backend/data/
|
||||
*.db
|
||||
*.db-shm
|
||||
*.db-wal
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""Database engine, sessions, and FastAPI dependency."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import Engine
|
||||
from sqlmodel import Session, SQLModel, create_engine
|
||||
|
|
@ -10,11 +11,23 @@ from .settings import settings
|
|||
_engine: Engine | None = None
|
||||
|
||||
|
||||
def _engine_kwargs(url: str) -> dict:
|
||||
if url.startswith("sqlite"):
|
||||
# FastAPI runs sync endpoints in a threadpool; connections are shared
|
||||
# across threads, so defer the same-thread check. File-based SQLite
|
||||
# needs an existing parent dir (SQLite won't create it).
|
||||
return {"connect_args": {"check_same_thread": False}}
|
||||
return {}
|
||||
|
||||
|
||||
def get_engine(url: str | None = None) -> Engine:
|
||||
"""Process-wide engine singleton (override URL in tests via settings)."""
|
||||
global _engine
|
||||
if _engine is None:
|
||||
_engine = create_engine(url or settings.database_url)
|
||||
url = url or settings.database_url
|
||||
if url.startswith("sqlite:///") and not url.startswith("sqlite:///:memory:"):
|
||||
Path(url.removeprefix("sqlite:///")).parent.mkdir(parents=True, exist_ok=True)
|
||||
_engine = create_engine(url, **_engine_kwargs(url))
|
||||
return _engine
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -13,10 +13,13 @@ class Settings(BaseSettings):
|
|||
version: str = "0.1.0"
|
||||
debug: bool = False
|
||||
|
||||
database_url: str = "postgresql+psycopg://kirche:kirche@localhost:5432/kirche_einfach"
|
||||
test_database_url: str = (
|
||||
"postgresql+psycopg://kirche:kirche@localhost:5432/kirche_einfach_test"
|
||||
)
|
||||
# App database (SQLite file). Override via DATABASE_URL for other engines.
|
||||
database_url: str = str(REPO_ROOT / "backend" / "data" / "kirche.db")
|
||||
|
||||
# Canonical flow definitions (source of truth): one Langflow export per file.
|
||||
flows_dir: str = str(REPO_ROOT / "backend" / "flows")
|
||||
# Auto-reconcile backend/flows into the DB + LFX while the API is running.
|
||||
flows_watch_enabled: bool = True
|
||||
|
||||
langflow_url: str = "http://localhost:7860"
|
||||
langflow_api_key: str = ""
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@ dependencies = [
|
|||
"alembic>=1.14",
|
||||
"fastapi[standard]>=0.115.0",
|
||||
"httpx>=0.28",
|
||||
"psycopg[binary]>=3.2",
|
||||
"pydantic-settings>=2.0",
|
||||
"sqlmodel>=0.0.24",
|
||||
"uvicorn[standard]>=0.30.0",
|
||||
"watchfiles>=0.24",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
|
|
@ -27,7 +27,7 @@ testpaths = ["tests"]
|
|||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"lfx==1.12.0",
|
||||
"lfx",
|
||||
"pytest>=8",
|
||||
"pytest-asyncio>=0.24",
|
||||
"pytest-cov>=6",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
"""Manual end-to-end smoke: real LFX + real API, frontend-shaped calls.
|
||||
|
||||
Usage (from backend/):
|
||||
TEST_DATABASE_URL=postgresql+psycopg://... uv run python scripts/smoke_e2e.py
|
||||
uv run python scripts/smoke_e2e.py
|
||||
Runs the API against a throwaway SQLite file and an ephemeral LFX pass.
|
||||
Exits non-zero on any unexpected status/body.
|
||||
"""
|
||||
|
||||
|
|
@ -11,7 +12,7 @@ import json
|
|||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import tempfile
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
|
@ -54,17 +55,31 @@ def check(name: str, cond: bool, detail: str = "") -> None:
|
|||
|
||||
|
||||
def main() -> None:
|
||||
tmp = Path(tempfile.mkdtemp(prefix="kirche-smoke-"))
|
||||
flow_dir = Path("/tmp/smoke-e2e-flows")
|
||||
flow_dir.mkdir(exist_ok=True)
|
||||
(flow_dir / "seed.json").write_text(
|
||||
(FIXTURES / "simple_rule.json").read_text()
|
||||
)
|
||||
# Migrate a throwaway SQLite schema to head before starting the API.
|
||||
from alembic import command as alembic_command
|
||||
from alembic.config import Config
|
||||
|
||||
db_url = f"sqlite:///{tmp / 'smoke.db'}"
|
||||
cfg = Config()
|
||||
cfg.set_main_option("script_location", str(BACKEND / "migrations"))
|
||||
cfg.set_main_option("sqlalchemy.url", db_url)
|
||||
alembic_command.upgrade(cfg, "head")
|
||||
|
||||
env = {
|
||||
**os.environ,
|
||||
# lfx prefers LANGFLOW_API_KEY over LFX_API_KEY; the repo .env may set
|
||||
# it. Clear it so the serve key below is the one we send.
|
||||
"LANGFLOW_API_KEY": "",
|
||||
"LFX_API_KEY": LFX_KEY,
|
||||
"DATABASE_URL": os.environ["TEST_DATABASE_URL"],
|
||||
"DATABASE_URL": db_url,
|
||||
"FLOWS_WATCH_ENABLED": "false",
|
||||
"LFX_URL": f"http://127.0.0.1:{LFX_PORT}",
|
||||
"LFX_API_KEY": LFX_KEY,
|
||||
"CALLBACK_BASE_URL": f"http://127.0.0.1:{API_PORT}",
|
||||
}
|
||||
lfx = subprocess.Popen(
|
||||
|
|
|
|||
|
|
@ -1,25 +1,22 @@
|
|||
"""Shared pytest fixtures.
|
||||
|
||||
Requires TEST_DATABASE_URL (a dedicated Postgres database, e.g. the
|
||||
`kirche_einfach_test` database from the compose stack). The URL is mapped
|
||||
onto DATABASE_URL *before* any app module is imported, so settings and the
|
||||
alembic env both target the test database. Never point this at production.
|
||||
Tests run on a throwaway SQLite file (no external database required).
|
||||
`DATABASE_URL` and `FLOWS_DIR` are pointed at a session temp dir *before* any
|
||||
app module is imported, so settings pick them up at import time. The app
|
||||
schema is migrated to head once per session via Alembic.
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
TEST_DB_URL = os.environ.get("TEST_DATABASE_URL")
|
||||
if not TEST_DB_URL:
|
||||
raise RuntimeError(
|
||||
"TEST_DATABASE_URL is not set. Refusing to run tests without "
|
||||
"an explicit test database (see .env.example)."
|
||||
)
|
||||
TEST_AREA = Path(tempfile.mkdtemp(prefix="kirche-tests-"))
|
||||
|
||||
# Must precede all `app.*` imports: Settings is instantiated at import time.
|
||||
os.environ["DATABASE_URL"] = TEST_DB_URL
|
||||
os.environ["DATABASE_URL"] = f"sqlite:///{TEST_AREA / 'test.db'}"
|
||||
os.environ["FLOWS_DIR"] = str(TEST_AREA / "flows")
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
|
@ -115,6 +112,10 @@ def lfx_server(tmp_path_factory):
|
|||
env = {
|
||||
**os.environ,
|
||||
# lfx serve refuses to start without a serve key; requests need it too.
|
||||
# get_api_key() prefers LANGFLOW_API_KEY over LFX_API_KEY, and the repo
|
||||
# .env may put a real Langflow key in the environment — clear it so the
|
||||
# test key below is the one the client sends.
|
||||
"LANGFLOW_API_KEY": "",
|
||||
"LFX_API_KEY": "test-lfx-key",
|
||||
}
|
||||
proc = subprocess.Popen(
|
||||
|
|
|
|||
|
|
@ -16,7 +16,9 @@ def test_flow_definition_defaults(session):
|
|||
assert d.status == "active"
|
||||
assert d.continuation_flow_id is None
|
||||
assert d.created_at is not None
|
||||
assert d.created_at.tzinfo is not None
|
||||
# Timestamps are written as UTC. SQLite does not round-trip tzinfo, so
|
||||
# after a refresh the value is calendar-time UTC without tzinfo.
|
||||
assert d.created_at.year >= 2026
|
||||
|
||||
|
||||
def test_flow_definition_name_unique(session):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue