chore: switch to all-SQLite stack (drop Postgres service)
This commit is contained in:
parent
74831483eb
commit
7d5ade6b6a
12 changed files with 104 additions and 62 deletions
|
|
@ -8,7 +8,7 @@ ENV PYTHONUNBUFFERED=1 \
|
||||||
PIP_NO_CACHE_DIR=1
|
PIP_NO_CACHE_DIR=1
|
||||||
|
|
||||||
# lfx = flow runtime; requests = convenience for custom-component code.
|
# lfx = flow runtime; requests = convenience for custom-component code.
|
||||||
RUN pip install "lfx==1.12.0" requests
|
RUN pip install lfx requests
|
||||||
|
|
||||||
RUN mkdir -p /flows
|
RUN mkdir -p /flows
|
||||||
VOLUME /flows
|
VOLUME /flows
|
||||||
|
|
|
||||||
|
|
@ -19,26 +19,8 @@ services:
|
||||||
- "8000:8000" # FastAPI
|
- "8000:8000" # FastAPI
|
||||||
- "3000:3000" # Vite
|
- "3000:3000" # Vite
|
||||||
|
|
||||||
db:
|
|
||||||
image: postgres:16-trixie
|
|
||||||
env_file:
|
|
||||||
- ../.env
|
|
||||||
volumes:
|
|
||||||
- postgres-data:/var/lib/postgresql/data
|
|
||||||
- ./init-db.sql:/docker-entrypoint-initdb.d/init-db.sql:ro
|
|
||||||
ports:
|
|
||||||
- "5432:5432"
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD-SHELL", "pg_isready -U postgres"]
|
|
||||||
interval: 5s
|
|
||||||
timeout: 3s
|
|
||||||
retries: 10
|
|
||||||
|
|
||||||
langflow:
|
langflow:
|
||||||
image: langflowai/langflow:1.12.0
|
image: langflowai/langflow:latest
|
||||||
depends_on:
|
|
||||||
db:
|
|
||||||
condition: service_healthy
|
|
||||||
env_file:
|
env_file:
|
||||||
- ../.env
|
- ../.env
|
||||||
environment:
|
environment:
|
||||||
|
|
@ -49,6 +31,9 @@ services:
|
||||||
PORT: "7860"
|
PORT: "7860"
|
||||||
LANGFLOW_AUTO_LOGIN: "false"
|
LANGFLOW_AUTO_LOGIN: "false"
|
||||||
LANGFLOW_API_KEY_SOURCE: env
|
LANGFLOW_API_KEY_SOURCE: env
|
||||||
|
# SQLite file inside the langflow-data volume (absolute path required
|
||||||
|
# by Langflow); overrides any postgres URL in ../.env.
|
||||||
|
LANGFLOW_DATABASE_URL: sqlite:////app/langflow/langflow.db
|
||||||
volumes:
|
volumes:
|
||||||
- langflow-data:/app/langflow
|
- langflow-data:/app/langflow
|
||||||
ports:
|
ports:
|
||||||
|
|
@ -85,9 +70,23 @@ services:
|
||||||
retries: 12
|
retries: 12
|
||||||
start_period: 30s
|
start_period: 30s
|
||||||
|
|
||||||
|
# Optional nightly auto-updates of :latest images. Not part of the default
|
||||||
|
# dev startup (no container updates needed while iterating).
|
||||||
|
watchtower:
|
||||||
|
image: nickfedor/watchtower:latest
|
||||||
|
profiles: ["updates"]
|
||||||
|
volumes:
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
|
environment:
|
||||||
|
WATCHTOWER_LABEL_ENABLE: "true"
|
||||||
|
WATCHTOWER_SCHEDULE: "0 0 4 * * *"
|
||||||
|
WATCHTOWER_CLEANUP: "true"
|
||||||
|
WATCHTOWER_ROLLING_RESTART: "true"
|
||||||
|
labels:
|
||||||
|
com.centurylinklabs.watchtower.enable: "true"
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
kirche-einfach-venv:
|
kirche-einfach-venv:
|
||||||
kirche-einfach-node-modules:
|
kirche-einfach-node-modules:
|
||||||
postgres-data:
|
|
||||||
langflow-data:
|
langflow-data:
|
||||||
lfx-data:
|
lfx-data:
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
-- Runs once on first init of the postgres-data volume.
|
|
||||||
-- The main app database ($POSTGRES_DB) is created by the image itself.
|
|
||||||
-- Langflow (authoring UI) gets its own database on the same instance.
|
|
||||||
CREATE DATABASE langflow;
|
|
||||||
|
|
||||||
-- Isolated database for the pytest suite (migrated via Alembic in conftest).
|
|
||||||
CREATE DATABASE kirche_einfach_test;
|
|
||||||
|
|
@ -13,7 +13,7 @@ echo "--- Installing Node dependencies ---"
|
||||||
cd /workspaces/kirche-einfach/frontend
|
cd /workspaces/kirche-einfach/frontend
|
||||||
npm install
|
npm install
|
||||||
|
|
||||||
# Compose services (db, langflow, lfx) must be up; fail loudly otherwise.
|
# Compose services (langflow, lfx) must be up; fail loudly otherwise.
|
||||||
echo "--- Waiting for compose services ---"
|
echo "--- Waiting for compose services ---"
|
||||||
if [ ! -f /workspaces/kirche-einfach/.env ]; then
|
if [ ! -f /workspaces/kirche-einfach/.env ]; then
|
||||||
echo "ERROR: /workspaces/kirche-einfach/.env is missing. Copy .env.example to .env and fill in secrets." >&2
|
echo "ERROR: /workspaces/kirche-einfach/.env is missing. Copy .env.example to .env and fill in secrets." >&2
|
||||||
|
|
@ -24,7 +24,7 @@ import socket
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
|
|
||||||
targets = [("db", 5432), ("langflow", 7860), ("lfx", 6004)]
|
targets = [("langflow", 7860), ("lfx", 6004)]
|
||||||
deadline = time.time() + 180
|
deadline = time.time() + 180
|
||||||
pending = dict(targets)
|
pending = dict(targets)
|
||||||
while pending and time.time() < deadline:
|
while pending and time.time() < deadline:
|
||||||
|
|
@ -45,6 +45,14 @@ if pending:
|
||||||
sys.exit(f"ERROR: compose services not ready: {sorted(pending)}")
|
sys.exit(f"ERROR: compose services not ready: {sorted(pending)}")
|
||||||
EOF
|
EOF
|
||||||
|
|
||||||
|
# Migrate the SQLite schema and reconcile backend/flows into the DB + LFX.
|
||||||
|
echo "--- Migrating database ---"
|
||||||
|
cd /workspaces/kirche-einfach/backend
|
||||||
|
uv run alembic upgrade head
|
||||||
|
|
||||||
|
echo "--- Syncing canonical flows ---"
|
||||||
|
uv run python scripts/sync_flows.py --sync || echo "WARN: flow sync failed (check backend/flows and LFX)"
|
||||||
|
|
||||||
echo ""
|
echo ""
|
||||||
echo "=== Setup complete ==="
|
echo "=== Setup complete ==="
|
||||||
echo "Forgejo username: $FORGEJO_USERNAME"
|
echo "Forgejo username: $FORGEJO_USERNAME"
|
||||||
|
|
|
||||||
18
.env.example
18
.env.example
|
|
@ -1,12 +1,14 @@
|
||||||
# Copy to `.env` and fill in values. `.env` is gitignored.
|
# Copy to `.env` and fill in values. `.env` is gitignored.
|
||||||
FORGEJO_TOKEN=
|
FORGEJO_TOKEN=
|
||||||
|
|
||||||
# --- databases (compose service `db`) ---
|
# --- app database (SQLite, default if unset) ---
|
||||||
POSTGRES_USER=kirche
|
# Override only if you want a different engine/URL.
|
||||||
POSTGRES_PASSWORD=
|
# DATABASE_URL=sqlite:///backend/data/kirche.db
|
||||||
POSTGRES_DB=kirche_einfach
|
|
||||||
DATABASE_URL=postgresql+psycopg://kirche:<password>@db:5432/kirche_einfach
|
# --- canonical flow definitions ---
|
||||||
TEST_DATABASE_URL=postgresql+psycopg://kirche:<password>@db:5432/kirche_einfach_test
|
# FLOWS_DIR=backend/flows
|
||||||
|
# Auto-reconcile backend/flows into DB + LFX while the API runs.
|
||||||
|
# FLOWS_WATCH_ENABLED=true
|
||||||
|
|
||||||
# --- langflow (authoring, dev only) ---
|
# --- langflow (authoring, dev only) ---
|
||||||
LANGFLOW_URL=http://langflow:7860
|
LANGFLOW_URL=http://langflow:7860
|
||||||
|
|
@ -15,7 +17,7 @@ LANGFLOW_SUPERUSER_PASSWORD=
|
||||||
LANGFLOW_SECRET_KEY=
|
LANGFLOW_SECRET_KEY=
|
||||||
# Single static key; validated against env (LANGFLOW_API_KEY_SOURCE=env), superuser scope.
|
# Single static key; validated against env (LANGFLOW_API_KEY_SOURCE=env), superuser scope.
|
||||||
LANGFLOW_API_KEY=sk-
|
LANGFLOW_API_KEY=sk-
|
||||||
LANGFLOW_DATABASE_URL=postgresql://kirche:<password>@db:5432/langflow
|
# Langflow stores flows in its own SQLite file when unset.
|
||||||
|
|
||||||
# --- lfx (flow runtime) ---
|
# --- lfx (flow runtime) ---
|
||||||
LFX_URL=http://lfx:6004
|
LFX_URL=http://lfx:6004
|
||||||
|
|
@ -25,4 +27,4 @@ LFX_API_KEY=lfx-
|
||||||
CALLBACK_BASE_URL=http://app:8000
|
CALLBACK_BASE_URL=http://app:8000
|
||||||
|
|
||||||
# --- LLM (optional; tests mock the provider without it) ---
|
# --- LLM (optional; tests mock the provider without it) ---
|
||||||
# LLM_API_KEY=
|
# LLM_API_KEY=
|
||||||
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -14,6 +14,12 @@ dist/
|
||||||
.env
|
.env
|
||||||
.env.local
|
.env.local
|
||||||
|
|
||||||
|
# Runtime SQLite databases
|
||||||
|
backend/data/
|
||||||
|
*.db
|
||||||
|
*.db-shm
|
||||||
|
*.db-wal
|
||||||
|
|
||||||
# IDE
|
# IDE
|
||||||
.vscode/
|
.vscode/
|
||||||
.idea/
|
.idea/
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
"""Database engine, sessions, and FastAPI dependency."""
|
"""Database engine, sessions, and FastAPI dependency."""
|
||||||
|
|
||||||
from collections.abc import Iterator
|
from collections.abc import Iterator
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from sqlalchemy import Engine
|
from sqlalchemy import Engine
|
||||||
from sqlmodel import Session, SQLModel, create_engine
|
from sqlmodel import Session, SQLModel, create_engine
|
||||||
|
|
@ -10,11 +11,23 @@ from .settings import settings
|
||||||
_engine: Engine | None = None
|
_engine: Engine | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _engine_kwargs(url: str) -> dict:
|
||||||
|
if url.startswith("sqlite"):
|
||||||
|
# FastAPI runs sync endpoints in a threadpool; connections are shared
|
||||||
|
# across threads, so defer the same-thread check. File-based SQLite
|
||||||
|
# needs an existing parent dir (SQLite won't create it).
|
||||||
|
return {"connect_args": {"check_same_thread": False}}
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
def get_engine(url: str | None = None) -> Engine:
|
def get_engine(url: str | None = None) -> Engine:
|
||||||
"""Process-wide engine singleton (override URL in tests via settings)."""
|
"""Process-wide engine singleton (override URL in tests via settings)."""
|
||||||
global _engine
|
global _engine
|
||||||
if _engine is None:
|
if _engine is None:
|
||||||
_engine = create_engine(url or settings.database_url)
|
url = url or settings.database_url
|
||||||
|
if url.startswith("sqlite:///") and not url.startswith("sqlite:///:memory:"):
|
||||||
|
Path(url.removeprefix("sqlite:///")).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
_engine = create_engine(url, **_engine_kwargs(url))
|
||||||
return _engine
|
return _engine
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -13,10 +13,13 @@ class Settings(BaseSettings):
|
||||||
version: str = "0.1.0"
|
version: str = "0.1.0"
|
||||||
debug: bool = False
|
debug: bool = False
|
||||||
|
|
||||||
database_url: str = "postgresql+psycopg://kirche:kirche@localhost:5432/kirche_einfach"
|
# App database (SQLite file). Override via DATABASE_URL for other engines.
|
||||||
test_database_url: str = (
|
database_url: str = str(REPO_ROOT / "backend" / "data" / "kirche.db")
|
||||||
"postgresql+psycopg://kirche:kirche@localhost:5432/kirche_einfach_test"
|
|
||||||
)
|
# Canonical flow definitions (source of truth): one Langflow export per file.
|
||||||
|
flows_dir: str = str(REPO_ROOT / "backend" / "flows")
|
||||||
|
# Auto-reconcile backend/flows into the DB + LFX while the API is running.
|
||||||
|
flows_watch_enabled: bool = True
|
||||||
|
|
||||||
langflow_url: str = "http://localhost:7860"
|
langflow_url: str = "http://localhost:7860"
|
||||||
langflow_api_key: str = ""
|
langflow_api_key: str = ""
|
||||||
|
|
|
||||||
|
|
@ -7,10 +7,10 @@ dependencies = [
|
||||||
"alembic>=1.14",
|
"alembic>=1.14",
|
||||||
"fastapi[standard]>=0.115.0",
|
"fastapi[standard]>=0.115.0",
|
||||||
"httpx>=0.28",
|
"httpx>=0.28",
|
||||||
"psycopg[binary]>=3.2",
|
|
||||||
"pydantic-settings>=2.0",
|
"pydantic-settings>=2.0",
|
||||||
"sqlmodel>=0.0.24",
|
"sqlmodel>=0.0.24",
|
||||||
"uvicorn[standard]>=0.30.0",
|
"uvicorn[standard]>=0.30.0",
|
||||||
|
"watchfiles>=0.24",
|
||||||
]
|
]
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
|
|
@ -27,7 +27,7 @@ testpaths = ["tests"]
|
||||||
|
|
||||||
[dependency-groups]
|
[dependency-groups]
|
||||||
dev = [
|
dev = [
|
||||||
"lfx==1.12.0",
|
"lfx",
|
||||||
"pytest>=8",
|
"pytest>=8",
|
||||||
"pytest-asyncio>=0.24",
|
"pytest-asyncio>=0.24",
|
||||||
"pytest-cov>=6",
|
"pytest-cov>=6",
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,8 @@
|
||||||
"""Manual end-to-end smoke: real LFX + real API, frontend-shaped calls.
|
"""Manual end-to-end smoke: real LFX + real API, frontend-shaped calls.
|
||||||
|
|
||||||
Usage (from backend/):
|
Usage (from backend/):
|
||||||
TEST_DATABASE_URL=postgresql+psycopg://... uv run python scripts/smoke_e2e.py
|
uv run python scripts/smoke_e2e.py
|
||||||
|
Runs the API against a throwaway SQLite file and an ephemeral LFX pass.
|
||||||
Exits non-zero on any unexpected status/body.
|
Exits non-zero on any unexpected status/body.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
@ -11,7 +12,7 @@ import json
|
||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
import tempfile
|
||||||
import urllib.request
|
import urllib.request
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
@ -54,17 +55,31 @@ def check(name: str, cond: bool, detail: str = "") -> None:
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
|
tmp = Path(tempfile.mkdtemp(prefix="kirche-smoke-"))
|
||||||
flow_dir = Path("/tmp/smoke-e2e-flows")
|
flow_dir = Path("/tmp/smoke-e2e-flows")
|
||||||
flow_dir.mkdir(exist_ok=True)
|
flow_dir.mkdir(exist_ok=True)
|
||||||
(flow_dir / "seed.json").write_text(
|
(flow_dir / "seed.json").write_text(
|
||||||
(FIXTURES / "simple_rule.json").read_text()
|
(FIXTURES / "simple_rule.json").read_text()
|
||||||
)
|
)
|
||||||
|
# Migrate a throwaway SQLite schema to head before starting the API.
|
||||||
|
from alembic import command as alembic_command
|
||||||
|
from alembic.config import Config
|
||||||
|
|
||||||
|
db_url = f"sqlite:///{tmp / 'smoke.db'}"
|
||||||
|
cfg = Config()
|
||||||
|
cfg.set_main_option("script_location", str(BACKEND / "migrations"))
|
||||||
|
cfg.set_main_option("sqlalchemy.url", db_url)
|
||||||
|
alembic_command.upgrade(cfg, "head")
|
||||||
|
|
||||||
env = {
|
env = {
|
||||||
**os.environ,
|
**os.environ,
|
||||||
|
# lfx prefers LANGFLOW_API_KEY over LFX_API_KEY; the repo .env may set
|
||||||
|
# it. Clear it so the serve key below is the one we send.
|
||||||
|
"LANGFLOW_API_KEY": "",
|
||||||
"LFX_API_KEY": LFX_KEY,
|
"LFX_API_KEY": LFX_KEY,
|
||||||
"DATABASE_URL": os.environ["TEST_DATABASE_URL"],
|
"DATABASE_URL": db_url,
|
||||||
|
"FLOWS_WATCH_ENABLED": "false",
|
||||||
"LFX_URL": f"http://127.0.0.1:{LFX_PORT}",
|
"LFX_URL": f"http://127.0.0.1:{LFX_PORT}",
|
||||||
"LFX_API_KEY": LFX_KEY,
|
|
||||||
"CALLBACK_BASE_URL": f"http://127.0.0.1:{API_PORT}",
|
"CALLBACK_BASE_URL": f"http://127.0.0.1:{API_PORT}",
|
||||||
}
|
}
|
||||||
lfx = subprocess.Popen(
|
lfx = subprocess.Popen(
|
||||||
|
|
|
||||||
|
|
@ -1,25 +1,22 @@
|
||||||
"""Shared pytest fixtures.
|
"""Shared pytest fixtures.
|
||||||
|
|
||||||
Requires TEST_DATABASE_URL (a dedicated Postgres database, e.g. the
|
Tests run on a throwaway SQLite file (no external database required).
|
||||||
`kirche_einfach_test` database from the compose stack). The URL is mapped
|
`DATABASE_URL` and `FLOWS_DIR` are pointed at a session temp dir *before* any
|
||||||
onto DATABASE_URL *before* any app module is imported, so settings and the
|
app module is imported, so settings pick them up at import time. The app
|
||||||
alembic env both target the test database. Never point this at production.
|
schema is migrated to head once per session via Alembic.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
TEST_DB_URL = os.environ.get("TEST_DATABASE_URL")
|
TEST_AREA = Path(tempfile.mkdtemp(prefix="kirche-tests-"))
|
||||||
if not TEST_DB_URL:
|
|
||||||
raise RuntimeError(
|
|
||||||
"TEST_DATABASE_URL is not set. Refusing to run tests without "
|
|
||||||
"an explicit test database (see .env.example)."
|
|
||||||
)
|
|
||||||
|
|
||||||
# Must precede all `app.*` imports: Settings is instantiated at import time.
|
# Must precede all `app.*` imports: Settings is instantiated at import time.
|
||||||
os.environ["DATABASE_URL"] = TEST_DB_URL
|
os.environ["DATABASE_URL"] = f"sqlite:///{TEST_AREA / 'test.db'}"
|
||||||
|
os.environ["FLOWS_DIR"] = str(TEST_AREA / "flows")
|
||||||
|
|
||||||
BACKEND_DIR = Path(__file__).resolve().parent.parent
|
BACKEND_DIR = Path(__file__).resolve().parent.parent
|
||||||
|
|
||||||
|
|
@ -115,6 +112,10 @@ def lfx_server(tmp_path_factory):
|
||||||
env = {
|
env = {
|
||||||
**os.environ,
|
**os.environ,
|
||||||
# lfx serve refuses to start without a serve key; requests need it too.
|
# lfx serve refuses to start without a serve key; requests need it too.
|
||||||
|
# get_api_key() prefers LANGFLOW_API_KEY over LFX_API_KEY, and the repo
|
||||||
|
# .env may put a real Langflow key in the environment — clear it so the
|
||||||
|
# test key below is the one the client sends.
|
||||||
|
"LANGFLOW_API_KEY": "",
|
||||||
"LFX_API_KEY": "test-lfx-key",
|
"LFX_API_KEY": "test-lfx-key",
|
||||||
}
|
}
|
||||||
proc = subprocess.Popen(
|
proc = subprocess.Popen(
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,9 @@ def test_flow_definition_defaults(session):
|
||||||
assert d.status == "active"
|
assert d.status == "active"
|
||||||
assert d.continuation_flow_id is None
|
assert d.continuation_flow_id is None
|
||||||
assert d.created_at is not None
|
assert d.created_at is not None
|
||||||
assert d.created_at.tzinfo is not None
|
# Timestamps are written as UTC. SQLite does not round-trip tzinfo, so
|
||||||
|
# after a refresh the value is calendar-time UTC without tzinfo.
|
||||||
|
assert d.created_at.year >= 2026
|
||||||
|
|
||||||
|
|
||||||
def test_flow_definition_name_unique(session):
|
def test_flow_definition_name_unique(session):
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue