"""Reconciliation: backend/flows/*.json (source of truth) -> DB + LFX. The ONLY place that derives registry state from the canonical directory. Triggers (CLI, POST /api/flows/sync, embedded watcher) call sync_from_directory(); single-document callers (e.g. /import) use sync_document(). Everything is async (the LFX client is async) and guarded by a module lock so concurrent triggers serialize on SQLite's single writer. """ from __future__ import annotations import asyncio import json import logging import threading import uuid from collections.abc import Callable from pathlib import Path from uuid import UUID from sqlmodel import Session from watchfiles import DefaultFilter, awatch from app.db import get_engine from app.models import FlowDefinition, FlowVersion from app.settings import settings from . import export, store from .lfx_client import LfxClient, LfxError from .schemas import FlowDefinitionCreate, FlowVersionCreate, SyncReport logger = logging.getLogger(__name__) # Serializes all writers in this process (sync engine + manual API writes). sync_lock = threading.Lock() def lfx_id_for(definition_id: UUID, version_no: int) -> str: """Deterministic LFX id (a real UUID: LFX validates the format). Same (definition, version) always maps to the same id, so re-syncs and crash recovery overwrite instead of orphaning LFX registrations. """ return str(uuid.uuid5(uuid.NAMESPACE_URL, f"kirche-einfach/{definition_id}/v{version_no}")) def _latest_version(session: Session, flow_id: UUID) -> FlowVersion | None: versions = store.list_versions(session, flow_id) return versions[-1] if versions else None async def sync_document( session: Session, lfx: LfxClient, doc: dict, *, source_filename: str, author: str = "", deploy: bool = True, ) -> tuple[FlowDefinition, FlowVersion | None, str]: """Reconcile one parsed document. Returns (definition, version, action). Actions: "added" (new definition + version), "updated" (new version), "skipped" (content unchanged). Raises store.ConflictError on name clashes, store.InvalidError on invalid JSON, LfxError when deploy fails. With deploy=False a version is recorded without touching LFX. Validation lives in store.create_version (single choke point). """ name = doc.get("name") or Path(source_filename).stem with sync_lock: definition = store.get_definition_by_source_path(session, source_filename) if definition is None: definition = store.create_definition(session, FlowDefinitionCreate(name=name)) definition.source_path = source_filename session.add(definition) session.commit() session.refresh(definition) action = "added" else: action = "updated" if definition.name != name: store.set_definition_name(session, definition, name) latest = _latest_version(session, definition.id) if latest is not None and export.content_hash(latest.langflow_json) == export.content_hash( doc ): return definition, latest, "skipped" if not deploy: version = store.create_version( session, definition.id, FlowVersionCreate(langflow_json=doc, author=author), ) return definition, version, action version_no = store.next_version_no(session, definition.id) lfx_id = lfx_id_for(definition.id, version_no) uploaded = await lfx.upload_flow(doc, flow_id=lfx_id, replace=True) version = store.create_version( session, definition.id, FlowVersionCreate(langflow_json=doc, author=author), version_no=version_no, ) version = store.mark_deployed(session, version, uploaded.id) if latest is not None and latest.lfx_flow_id and latest.lfx_flow_id != uploaded.id: try: await lfx.delete_flow(latest.lfx_flow_id) except LfxError as exc: logger.warning( "LFX unregister %s failed (continuing): %s", latest.lfx_flow_id, exc, ) return definition, version, action async def delete_definition_tree( session: Session, lfx: LfxClient, definition: FlowDefinition ) -> dict[str, int]: """Remove a definition whose canonical file is gone. LFX registrations go first (best-effort, fail-closed for new runs), the DB subtree (tasks, runs, versions, definition) after. Deterministic LFX ids make any crash in between self-healing on the next sync. """ for version in store.list_versions(session, definition.id): if version.lfx_flow_id: try: await lfx.delete_flow(version.lfx_flow_id) except LfxError as exc: logger.warning( "LFX unregister %s failed (continuing): %s", version.lfx_flow_id, exc, ) with sync_lock: return store.delete_definition_cascade(session, definition) async def sync_from_directory( session: Session, lfx: LfxClient, flows_dir: str | Path ) -> SyncReport: """Reconcile the whole canonical directory. Per-file problems land in report.errors (never raised); a missing directory raises FileNotFoundError so misconfiguration fails loudly instead of wiping the registry. """ flows_dir = Path(flows_dir) report = SyncReport() if not flows_dir.is_dir(): raise FileNotFoundError(f"flows directory missing: {flows_dir}") files = {path.name: path for path in export.iter_flow_files(flows_dir)} for filename, path in files.items(): try: doc = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError) as exc: report.errors[filename] = f"unreadable: {exc}" continue if not isinstance(doc, dict): report.errors[filename] = "top-level JSON must be an object" continue try: _, _, action = await sync_document(session, lfx, doc, source_filename=filename) except (store.InvalidError, store.ConflictError, store.NotFoundError, LfxError) as exc: report.errors[filename] = str(exc) continue getattr(report, action).append(path.stem) known = { definition.source_path for definition in store.list_definitions(session) if definition.source_path } for stale in sorted(known - set(files)): definition = store.get_definition_by_source_path(session, stale) if definition is None: # pragma: no cover - defensive continue try: await delete_definition_tree(session, lfx, definition) except store.NotFoundError as exc: report.errors[stale] = str(exc) continue report.deleted.append(Path(stale).stem) return report def format_report(report: SyncReport) -> str: """One-line summary for startup/watcher logs (keeps main.py thin).""" return ( f"+{len(report.added)} ~{len(report.updated)} -{len(report.deleted)} " f"skipped={len(report.skipped)} errors={len(report.errors)}" ) def default_session_factory() -> Callable[[], Session]: return lambda: Session(get_engine()) def default_lfx_factory() -> Callable[[], LfxClient]: return lambda: LfxClient(settings.lfx_url, api_key=settings.lfx_api_key or None) async def sync_full_pass( session: Session, lfx: LfxClient, langflow, flows_dir: str | Path, ) -> SyncReport: """Single orchestration: Langflow pull -> files -> DB + LFX. Shared by POST /api/flows/sync, CLI, and watcher-adjacent paths so the full-pass logic lives once. Pull errors merge with sync errors. """ from .deploy import pull_all_from_langflow flows_dir = Path(flows_dir) pulled_result = await pull_all_from_langflow(langflow, flows_dir) report = await sync_from_directory(session, lfx, flows_dir) report.pulled = pulled_result["pulled"] # type: ignore[typeddict-item] report.errors = {**pulled_result["errors"], **report.errors} # type: ignore[typeddict-item] return report def run_sync_blocking( flows_dir: str | Path, *, session_factory: Callable[[], Session] | None = None, lfx_factory: Callable[[], LfxClient] | None = None, ) -> SyncReport: """Blocking one-shot sync for worker threads (lifespan, watcher).""" session = (session_factory or default_session_factory())() try: lfx = (lfx_factory or default_lfx_factory())() return asyncio.run(sync_from_directory(session, lfx, Path(flows_dir))) finally: session.close() class _JsonFilter(DefaultFilter): def __call__(self, change, path: str) -> bool: # type: ignore[override] if not (super().__call__(change, path) and path.endswith(".json")): return False # The ignore file itself is tiny; re-read per event so no restart # is needed after editing it. return Path(path).stem not in export.ignored_stems(Path(path).parent) async def watch_forever( stop: asyncio.Event, flows_dir: str | Path, run_sync: Callable[[], SyncReport] | None = None, ) -> None: """Run until `stop` is set: sync on every batch of .json changes. All failures are logged, never raised — a broken file must not kill the watcher. `run_sync` is injectable for tests. """ flows_dir = Path(flows_dir) run = run_sync or (lambda: run_sync_blocking(flows_dir)) while not stop.is_set() and not flows_dir.is_dir(): logger.error("flows dir missing, retrying in 30s: %s", flows_dir) try: await asyncio.wait_for(stop.wait(), timeout=30) except TimeoutError: pass if stop.is_set(): return logger.info("watching %s", flows_dir) async for _ in awatch(flows_dir, watch_filter=_JsonFilter(), debounce=1000, step=100): if stop.is_set(): break try: report = await asyncio.to_thread(run) except FileNotFoundError as exc: logger.error("flow sync failed: %s", exc) continue except Exception: logger.exception("flow sync failed") continue logger.info("flow sync: %s", format_report(report))