jonas/kirche-einfach
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: canonical flow directory with sync engine (files are source of truth)

Browse source
This commit is contained in:
Jonas Miederer 2026-09-07 08:59:56 +00:00
parent 7d5ade6b6a
commit c65e3ed332
16 changed files with 4167 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

22
backend/app/flows/api.py
View file

@ -4,6 +4,7 @@ from __future__ import annotations
import json
from collections.abc import AsyncIterator, Iterator
from pathlib import Path
from uuid import UUID
import httpx
@ -16,6 +17,7 @@ from app.settings import settings
from . import runner, store
from .deploy import import_from_langflow
from .sync import lfx_id_for, sync_from_directory
from .langflow_client import (
LangflowClient,
LangflowError,
@ -44,6 +46,7 @@ from .schemas import (
FlowVersionRead,
RunRead,
RunRequest,
SyncReport,
)
flows_router = APIRouter(prefix="/api/flows", tags=["flows"])
@ -70,6 +73,7 @@ def _read_definition(d) -> FlowDefinitionRead:
name=d.name,
description=d.description,
status=d.status,
source_path=d.source_path,
continuation_flow_id=d.continuation_flow_id,
created_at=d.created_at,
)
@ -134,6 +138,18 @@ def list_flows(session: Session = Depends(get_db)):
return [_read_definition(d) for d in store.list_definitions(session)]
@flows_router.post("/sync", response_model=SyncReport)
async def sync_flows(
session: Session = Depends(get_db),
lfx: LfxClient = Depends(get_lfx),
):
"""Reconcile backend/flows/*.json into the DB + LFX (add/update/delete)."""
try:
return await sync_from_directory(session, lfx, Path(settings.flows_dir))
except FileNotFoundError as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
@flows_router.get("/{flow_id}", response_model=FlowDefinitionRead)
def get_flow(flow_id: UUID, session: Session = Depends(get_db)):
try:
@ -178,8 +194,12 @@ async def deploy_version(
except store.NotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
try:
# Deterministic LFX id: same (definition, version) always maps to the
# same registration, no matter which path deployed it.
uploaded = await lfx.upload_flow(
version.langflow_json, flow_id=str(version.id), replace=True
version.langflow_json,
flow_id=lfx_id_for(flow_id, version_no),
replace=True,
)
except LfxInvalidFlow as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc

54
backend/app/flows/deploy.py
View file

@ -1,15 +1,15 @@
"""Authoring pipeline: Langflow export -> versioned definition -> LFX deploy."""
"""Authoring pipeline: Langflow export -> canonical file -> version -> LFX deploy."""
from __future__ import annotations
from sqlmodel import Session
from app.models import FlowDefinition, FlowVersion
from app.settings import settings
from . import store
from . import export, store, sync
from .langflow_client import LangflowClient
from .lfx_client import LfxClient
from .schemas import FlowDefinitionCreate, FlowVersionCreate
async def import_from_langflow(
@ -22,39 +22,31 @@ async def import_from_langflow(
author: str = "",
deploy: bool = True,
) -> tuple[FlowDefinition, FlowVersion]:
"""Export a flow from Langflow, store it as a new version, deploy to LFX.
"""Export a flow from Langflow, persist it to backend/flows, then sync.
Definitions are matched by name (explicit `name` wins, else the exported
name); every import creates a new immutable version.
The canonical file is written first so the directory stays the source of
truth; sync_document() derives the definition + version from it.
"""
export = await langflow.export_flow(langflow_flow_id)
flow_name = name or export.get("name") or f"langflow-{langflow_flow_id}"
exported = await langflow.export_flow(langflow_flow_id)
flow_name = name or exported.get("name") or f"langflow-{langflow_flow_id}"
langflow_json = {
"id": export.get("id", langflow_flow_id),
"id": exported.get("id", langflow_flow_id),
"name": flow_name,
"description": export.get("description") or "",
"endpoint_name": export.get("endpoint_name"),
"data": export.get("data", {}),
"tags": export.get("tags", []),
"webhook": export.get("webhook", False),
"is_component": export.get("is_component", False),
"description": exported.get("description") or "",
"endpoint_name": exported.get("endpoint_name"),
"data": exported.get("data", {}),
"tags": exported.get("tags", []),
"webhook": exported.get("webhook", False),
"is_component": exported.get("is_component", False),
}
try:
definition = next(
d for d in store.list_definitions(session) if d.name == flow_name
)
except StopIteration:
definition = store.create_definition(
session, FlowDefinitionCreate(name=flow_name)
)
version = store.create_version(
path = export.write_flow_file(settings.flows_dir, langflow_json, name=flow_name)
definition, version, _action = await sync.sync_document(
session,
definition.id,
FlowVersionCreate(langflow_json=langflow_json, author=author),
lfx,
langflow_json,
source_filename=path.name,
author=author,
deploy=deploy,
)
if deploy:
uploaded = await lfx.upload_flow(
version.langflow_json, flow_id=str(version.id), replace=True
)
version = store.mark_deployed(session, version, uploaded.id)
assert version is not None # sync_document always versions on import
return definition, version

71
backend/app/flows/export.py Normal file
View file

@ -0,0 +1,71 @@
"""Canonical flow files: backend/flows/<name>.json is the source of truth.
One Langflow export per file, committed to git. This module only reads and
writes the directory; reconciliation into the DB lives in sync.py.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import tempfile
from pathlib import Path
FLOWS_GLOB = "*.json"
def slugify(name: str) -> str:
"""Map a flow name to a safe filename stem (lowercase, [a-z0-9-])."""
slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
return slug or "flow"
def canonical_bytes(doc: dict) -> bytes:
"""Deterministic serialization: stable bytes for stable git diffs."""
return (json.dumps(doc, indent=2, ensure_ascii=False, sort_keys=True) + "\n").encode(
"utf-8"
)
def content_hash(doc: dict) -> str:
"""Hash of the parsed document (formatting-independent)."""
return hashlib.sha256(canonical_bytes(doc)).hexdigest()
def flow_path(flows_dir: str | Path, stem: str) -> Path:
return Path(flows_dir) / f"{stem}.json"
def write_flow_file(
flows_dir: str | Path, doc: dict, *, name: str | None = None
) -> Path:
"""Write a flow document to backend/flows atomically (tmp + rename).
The filename derives from `name` (or the document's own name). Returns
the written path.
"""
flows_dir = Path(flows_dir)
flows_dir.mkdir(parents=True, exist_ok=True)
stem = slugify(name or doc.get("name") or "flow")
target = flow_path(flows_dir, stem)
fd, tmp_name = tempfile.mkstemp(
dir=str(flows_dir), prefix=f".{stem}-", suffix=".tmp"
)
try:
with os.fdopen(fd, "wb") as tmp:
tmp.write(canonical_bytes(doc))
os.replace(tmp_name, target)
except BaseException:
try:
os.unlink(tmp_name)
except OSError:
pass
raise
return target
def iter_flow_files(flows_dir: str | Path) -> list[Path]:
"""Sorted list of candidate flow files (top level only)."""
return sorted(Path(flows_dir).glob(FLOWS_GLOB))

15
backend/app/flows/schemas.py
View file

@ -17,6 +17,8 @@ class FlowDefinitionRead(BaseModel):
name: str
description: str
status: str
# backend/flows filename this definition syncs from (None if API-created).
source_path: str | None = None
continuation_flow_id: UUID | None
created_at: datetime
@ -110,3 +112,16 @@ class DecisionRequest(BaseModel):
class DecisionRead(BaseModel):
task: ApprovalRead
continuation_run: RunRead | None
class SyncReport(BaseModel):
"""Result of reconciling backend/flows/*.json into DB + LFX.
Lists hold filename stems; errors maps filename to reason.
"""
added: list[str] = Field(default_factory=list)
updated: list[str] = Field(default_factory=list)
deleted: list[str] = Field(default_factory=list)
skipped: list[str] = Field(default_factory=list)
errors: dict[str, str] = Field(default_factory=dict)

79
backend/app/flows/store.py
View file

@ -56,7 +56,82 @@ def list_definitions(session: Session) -> list[FlowDefinition]:
return list(session.exec(select(FlowDefinition).order_by(FlowDefinition.name)).all())
def _validate_flow_json(flow_json: dict) -> None:
def get_definition_by_source_path(
session: Session, source_path: str
) -> FlowDefinition | None:
"""Find the definition synced from a given backend/flows filename."""
return session.exec(
select(FlowDefinition).where(FlowDefinition.source_path == source_path)
).first()
def set_definition_name(
session: Session, definition: FlowDefinition, name: str
) -> FlowDefinition:
"""Rename a definition and record its source file. Name clashes 409."""
definition.name = name
session.add(definition)
try:
session.commit()
except IntegrityError as exc:
session.rollback()
raise ConflictError(f"flow {name!r} already exists") from exc
session.refresh(definition)
return definition
def next_version_no(session: Session, flow_id: UUID) -> int:
max_no = session.exec(
select(func.max(FlowVersion.version_no)).where(FlowVersion.flow_id == flow_id)
).one()
return (max_no or 0) + 1
def delete_definition_cascade(
session: Session, definition: FlowDefinition
) -> dict[str, int]:
"""Remove a definition and its whole subtree (tasks, runs, versions).
Used when the canonical file disappears: the directory is the source of
truth, so the derived registry follows. Commits once.
"""
version_ids = list(
session.exec(
select(FlowVersion.id).where(FlowVersion.flow_id == definition.id)
).all()
)
task_count = 0
run_count = 0
if version_ids:
tasks = list(
session.exec(
select(ApprovalTask).where(ApprovalTask.version_id.in_(version_ids))
).all()
)
runs = list(
session.exec(
select(FlowRun).where(FlowRun.version_id.in_(version_ids))
).all()
)
task_count, run_count = len(tasks), len(runs)
for row in (*tasks, *runs):
session.delete(row)
for version in session.exec(
select(FlowVersion).where(FlowVersion.flow_id == definition.id)
).all():
session.delete(version)
session.delete(definition)
session.commit()
return {
"definitions": 1,
"versions": len(version_ids),
"runs": run_count,
"tasks": task_count,
}
def validate_flow_json(flow_json: dict) -> None:
"""Reject anything that is not a Langflow-export-shaped document."""
if not isinstance(flow_json, dict):
raise InvalidError("langflow_json must be an object")
data = flow_json.get("data")
@ -70,7 +145,7 @@ def create_version(
session: Session, flow_id: UUID, data: FlowVersionCreate
) -> FlowVersion:
get_definition(session, flow_id)
_validate_flow_json(data.langflow_json)
validate_flow_json(data.langflow_json)
max_no = session.exec(
select(func.max(FlowVersion.version_no)).where(FlowVersion.flow_id == flow_id)
).one()

274
backend/app/flows/sync.py Normal file
View file

@ -0,0 +1,274 @@
"""Reconciliation: backend/flows/*.json (source of truth) -> DB + LFX.
The ONLY place that derives registry state from the canonical directory.
Triggers (CLI, POST /api/flows/sync, embedded watcher) call
sync_from_directory(); single-document callers (e.g. /import) use
sync_document(). Everything is async (the LFX client is async) and guarded
by a module lock so concurrent triggers serialize on SQLite's single writer.
"""
from __future__ import annotations
import asyncio
import json
import logging
import threading
import uuid
from collections.abc import Callable
from pathlib import Path
from uuid import UUID
from sqlmodel import Session
from watchfiles import DefaultFilter, awatch
from app.db import get_engine
from app.models import FlowDefinition, FlowVersion
from app.settings import settings
from . import export, store
from .lfx_client import LfxClient, LfxError
from .schemas import FlowDefinitionCreate, FlowVersionCreate, SyncReport
logger = logging.getLogger(__name__)
# Serializes all writers in this process (sync engine + manual API writes).
sync_lock = threading.Lock()
def lfx_id_for(definition_id: UUID, version_no: int) -> str:
"""Deterministic LFX id (a real UUID: LFX validates the format).
Same (definition, version) always maps to the same id, so re-syncs and
crash recovery overwrite instead of orphaning LFX registrations.
"""
return str(
uuid.uuid5(uuid.NAMESPACE_URL, f"kirche-einfach/{definition_id}/v{version_no}")
)
def _latest_version(session: Session, flow_id: UUID) -> FlowVersion | None:
versions = store.list_versions(session, flow_id)
return versions[-1] if versions else None
async def sync_document(
session: Session,
lfx: LfxClient,
doc: dict,
*,
source_filename: str,
author: str = "",
deploy: bool = True,
) -> tuple[FlowDefinition, FlowVersion | None, str]:
"""Reconcile one parsed document. Returns (definition, version, action).
Actions: "added" (new definition + version), "updated" (new version),
"skipped" (content unchanged). Raises store.ConflictError on name
clashes, store.InvalidError on invalid JSON, LfxError when deploy fails.
With deploy=False a version is recorded without touching LFX.
"""
store.validate_flow_json(doc)
name = doc.get("name") or Path(source_filename).stem
with sync_lock:
definition = store.get_definition_by_source_path(session, source_filename)
if definition is None:
definition = store.create_definition(
session, FlowDefinitionCreate(name=name)
)
definition.source_path = source_filename
session.add(definition)
session.commit()
session.refresh(definition)
action = "added"
else:
action = "updated"
if definition.name != name:
store.set_definition_name(session, definition, name)
latest = _latest_version(session, definition.id)
if (
latest is not None
and export.content_hash(latest.langflow_json)
== export.content_hash(doc)
):
return definition, latest, "skipped"
if not deploy:
version = store.create_version(
session,
definition.id,
FlowVersionCreate(langflow_json=doc, author=author),
)
return definition, version, action
version_no = store.next_version_no(session, definition.id)
lfx_id = lfx_id_for(definition.id, version_no)
uploaded = await lfx.upload_flow(doc, flow_id=lfx_id, replace=True)
version = store.create_version(
session,
definition.id,
FlowVersionCreate(langflow_json=doc, author=author),
)
if version.version_no != version_no: # pragma: no cover - lock prevents this
raise store.ConflictError(
f"concurrent version write for flow {definition.id}"
)
version = store.mark_deployed(session, version, uploaded.id)
if (
latest is not None
and latest.lfx_flow_id
and latest.lfx_flow_id != uploaded.id
):
try:
await lfx.delete_flow(latest.lfx_flow_id)
except LfxError as exc:
logger.warning(
"LFX unregister %s failed (continuing): %s",
latest.lfx_flow_id,
exc,
)
return definition, version, action
async def delete_definition_tree(
session: Session, lfx: LfxClient, definition: FlowDefinition
) -> dict[str, int]:
"""Remove a definition whose canonical file is gone.
LFX registrations go first (best-effort, fail-closed for new runs), the
DB subtree (tasks, runs, versions, definition) after. Deterministic LFX
ids make any crash in between self-healing on the next sync.
"""
for version in store.list_versions(session, definition.id):
if version.lfx_flow_id:
try:
await lfx.delete_flow(version.lfx_flow_id)
except LfxError as exc:
logger.warning(
"LFX unregister %s failed (continuing): %s",
version.lfx_flow_id,
exc,
)
with sync_lock:
return store.delete_definition_cascade(session, definition)
async def sync_from_directory(
session: Session, lfx: LfxClient, flows_dir: str | Path
) -> SyncReport:
"""Reconcile the whole canonical directory. Per-file problems land in
report.errors (never raised); a missing directory raises FileNotFoundError
so misconfiguration fails loudly instead of wiping the registry.
"""
flows_dir = Path(flows_dir)
report = SyncReport()
if not flows_dir.is_dir():
raise FileNotFoundError(f"flows directory missing: {flows_dir}")
files = {path.name: path for path in export.iter_flow_files(flows_dir)}
for filename, path in files.items():
try:
doc = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError) as exc:
report.errors[filename] = f"unreadable: {exc}"
continue
if not isinstance(doc, dict):
report.errors[filename] = "top-level JSON must be an object"
continue
try:
_, _, action = await sync_document(
session, lfx, doc, source_filename=filename
)
except Exception as exc: # noqa: BLE001 - reconciliation is total
report.errors[filename] = str(exc)
continue
getattr(report, action).append(path.stem)
known = {
definition.source_path
for definition in store.list_definitions(session)
if definition.source_path
}
for stale in sorted(known - set(files)):
definition = store.get_definition_by_source_path(session, stale)
if definition is None: # pragma: no cover - defensive
continue
try:
await delete_definition_tree(session, lfx, definition)
except Exception as exc: # noqa: BLE001 - reconciliation is total
report.errors[stale] = str(exc)
continue
report.deleted.append(Path(stale).stem)
return report
def default_session_factory() -> Callable[[], Session]:
return lambda: Session(get_engine())
def default_lfx_factory() -> Callable[[], LfxClient]:
return lambda: LfxClient(
settings.lfx_url, api_key=settings.lfx_api_key or None
)
def run_sync_blocking(
flows_dir: str | Path,
*,
session_factory: Callable[[], Session] | None = None,
lfx_factory: Callable[[], LfxClient] | None = None,
) -> SyncReport:
"""Blocking one-shot sync for worker threads (lifespan, watcher)."""
session = (session_factory or default_session_factory())()
try:
lfx = (lfx_factory or default_lfx_factory())()
return asyncio.run(sync_from_directory(session, lfx, Path(flows_dir)))
finally:
session.close()
class _JsonFilter(DefaultFilter):
def __call__(self, change, path: str) -> bool: # type: ignore[override]
return super().__call__(change, path) and path.endswith(".json")
async def watch_forever(
stop: asyncio.Event,
flows_dir: str | Path,
run_sync: Callable[[], SyncReport] | None = None,
) -> None:
"""Run until `stop` is set: sync on every batch of .json changes.
All failures are logged, never raised — a broken file must not kill the
watcher. `run_sync` is injectable for tests.
"""
flows_dir = Path(flows_dir)
run = run_sync or (lambda: run_sync_blocking(flows_dir))
while not stop.is_set() and not flows_dir.is_dir():
logger.error("flows dir missing, retrying in 30s: %s", flows_dir)
try:
await asyncio.wait_for(stop.wait(), timeout=30)
except asyncio.TimeoutError:
pass
if stop.is_set():
return
logger.info("watching %s", flows_dir)
async for _ in awatch(
flows_dir, watch_filter=_JsonFilter(), debounce=1000, step=100
):
if stop.is_set():
break
try:
report = await asyncio.to_thread(run)
except FileNotFoundError as exc:
logger.error("flow sync failed: %s", exc)
continue
except Exception: # noqa: BLE001 - watcher must survive
logger.exception("flow sync failed")
continue
logger.info(
"flow sync: +%d ~%d -%d skipped=%d errors=%d",
len(report.added),
len(report.updated),
len(report.deleted),
len(report.skipped),
len(report.errors),
)

3
backend/app/models/flow.py
View file

@ -29,6 +29,9 @@ class FlowDefinition(SQLModel, table=True):
name: str = Field(index=True, unique=True, max_length=200)
description: str = Field(default="", max_length=2000)
status: str = Field(default="active", max_length=20) # active | archived
# Filename under backend/flows/ this definition is synced from
# (e.g. "simple_rule.json"). None for definitions created via the API.
source_path: str | None = Field(default=None, max_length=255)
# Flow to run after a human approval on this flow's runs (nullable).
continuation_flow_id: UUID | None = Field(
default=None, foreign_key="flow_definitions.id"