feat: canonical flow directory with sync engine (files are source of truth)
This commit is contained in:
Jonas Miederer
parent
7d5ade6b6a
commit
c65e3ed332
16 changed files with 4167 additions and 42 deletions
|
|
@ -4,6 +4,7 @@ from __future__ import annotations
|
|||
|
||||
import json
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
import httpx
|
||||
|
|
@ -16,6 +17,7 @@ from app.settings import settings
|
|||
|
||||
from . import runner, store
|
||||
from .deploy import import_from_langflow
|
||||
from .sync import lfx_id_for, sync_from_directory
|
||||
from .langflow_client import (
|
||||
LangflowClient,
|
||||
LangflowError,
|
||||
|
|
@ -44,6 +46,7 @@ from .schemas import (
|
|||
FlowVersionRead,
|
||||
RunRead,
|
||||
RunRequest,
|
||||
SyncReport,
|
||||
)
|
||||
|
||||
flows_router = APIRouter(prefix="/api/flows", tags=["flows"])
|
||||
|
|
@ -70,6 +73,7 @@ def _read_definition(d) -> FlowDefinitionRead:
|
|||
name=d.name,
|
||||
description=d.description,
|
||||
status=d.status,
|
||||
source_path=d.source_path,
|
||||
continuation_flow_id=d.continuation_flow_id,
|
||||
created_at=d.created_at,
|
||||
)
|
||||
|
|
@ -134,6 +138,18 @@ def list_flows(session: Session = Depends(get_db)):
|
|||
return [_read_definition(d) for d in store.list_definitions(session)]
|
||||
|
||||
|
||||
@flows_router.post("/sync", response_model=SyncReport)
|
||||
async def sync_flows(
|
||||
session: Session = Depends(get_db),
|
||||
lfx: LfxClient = Depends(get_lfx),
|
||||
):
|
||||
"""Reconcile backend/flows/*.json into the DB + LFX (add/update/delete)."""
|
||||
try:
|
||||
return await sync_from_directory(session, lfx, Path(settings.flows_dir))
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@flows_router.get("/{flow_id}", response_model=FlowDefinitionRead)
|
||||
def get_flow(flow_id: UUID, session: Session = Depends(get_db)):
|
||||
try:
|
||||
|
|
@ -178,8 +194,12 @@ async def deploy_version(
|
|||
except store.NotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
try:
|
||||
# Deterministic LFX id: same (definition, version) always maps to the
|
||||
# same registration, no matter which path deployed it.
|
||||
uploaded = await lfx.upload_flow(
|
||||
version.langflow_json, flow_id=str(version.id), replace=True
|
||||
version.langflow_json,
|
||||
flow_id=lfx_id_for(flow_id, version_no),
|
||||
replace=True,
|
||||
)
|
||||
except LfxInvalidFlow as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
|
|
|||
|
|
@ -1,15 +1,15 @@
|
|||
"""Authoring pipeline: Langflow export -> versioned definition -> LFX deploy."""
|
||||
"""Authoring pipeline: Langflow export -> canonical file -> version -> LFX deploy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlmodel import Session
|
||||
|
||||
from app.models import FlowDefinition, FlowVersion
|
||||
from app.settings import settings
|
||||
|
||||
from . import store
|
||||
from . import export, store, sync
|
||||
from .langflow_client import LangflowClient
|
||||
from .lfx_client import LfxClient
|
||||
from .schemas import FlowDefinitionCreate, FlowVersionCreate
|
||||
|
||||
|
||||
async def import_from_langflow(
|
||||
|
|
@ -22,39 +22,31 @@ async def import_from_langflow(
|
|||
author: str = "",
|
||||
deploy: bool = True,
|
||||
) -> tuple[FlowDefinition, FlowVersion]:
|
||||
"""Export a flow from Langflow, store it as a new version, deploy to LFX.
|
||||
"""Export a flow from Langflow, persist it to backend/flows, then sync.
|
||||
|
||||
Definitions are matched by name (explicit `name` wins, else the exported
|
||||
name); every import creates a new immutable version.
|
||||
The canonical file is written first so the directory stays the source of
|
||||
truth; sync_document() derives the definition + version from it.
|
||||
"""
|
||||
export = await langflow.export_flow(langflow_flow_id)
|
||||
flow_name = name or export.get("name") or f"langflow-{langflow_flow_id}"
|
||||
exported = await langflow.export_flow(langflow_flow_id)
|
||||
flow_name = name or exported.get("name") or f"langflow-{langflow_flow_id}"
|
||||
langflow_json = {
|
||||
"id": export.get("id", langflow_flow_id),
|
||||
"id": exported.get("id", langflow_flow_id),
|
||||
"name": flow_name,
|
||||
"description": export.get("description") or "",
|
||||
"endpoint_name": export.get("endpoint_name"),
|
||||
"data": export.get("data", {}),
|
||||
"tags": export.get("tags", []),
|
||||
"webhook": export.get("webhook", False),
|
||||
"is_component": export.get("is_component", False),
|
||||
"description": exported.get("description") or "",
|
||||
"endpoint_name": exported.get("endpoint_name"),
|
||||
"data": exported.get("data", {}),
|
||||
"tags": exported.get("tags", []),
|
||||
"webhook": exported.get("webhook", False),
|
||||
"is_component": exported.get("is_component", False),
|
||||
}
|
||||
try:
|
||||
definition = next(
|
||||
d for d in store.list_definitions(session) if d.name == flow_name
|
||||
)
|
||||
except StopIteration:
|
||||
definition = store.create_definition(
|
||||
session, FlowDefinitionCreate(name=flow_name)
|
||||
)
|
||||
version = store.create_version(
|
||||
path = export.write_flow_file(settings.flows_dir, langflow_json, name=flow_name)
|
||||
definition, version, _action = await sync.sync_document(
|
||||
session,
|
||||
definition.id,
|
||||
FlowVersionCreate(langflow_json=langflow_json, author=author),
|
||||
lfx,
|
||||
langflow_json,
|
||||
source_filename=path.name,
|
||||
author=author,
|
||||
deploy=deploy,
|
||||
)
|
||||
if deploy:
|
||||
uploaded = await lfx.upload_flow(
|
||||
version.langflow_json, flow_id=str(version.id), replace=True
|
||||
)
|
||||
version = store.mark_deployed(session, version, uploaded.id)
|
||||
assert version is not None # sync_document always versions on import
|
||||
return definition, version
|
||||
|
|
|
|||
71
backend/app/flows/export.py
Normal file
71
backend/app/flows/export.py
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
"""Canonical flow files: backend/flows/<name>.json is the source of truth.
|
||||
|
||||
One Langflow export per file, committed to git. This module only reads and
|
||||
writes the directory; reconciliation into the DB lives in sync.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
FLOWS_GLOB = "*.json"
|
||||
|
||||
|
||||
def slugify(name: str) -> str:
|
||||
"""Map a flow name to a safe filename stem (lowercase, [a-z0-9-])."""
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
|
||||
return slug or "flow"
|
||||
|
||||
|
||||
def canonical_bytes(doc: dict) -> bytes:
|
||||
"""Deterministic serialization: stable bytes for stable git diffs."""
|
||||
return (json.dumps(doc, indent=2, ensure_ascii=False, sort_keys=True) + "\n").encode(
|
||||
"utf-8"
|
||||
)
|
||||
|
||||
|
||||
def content_hash(doc: dict) -> str:
|
||||
"""Hash of the parsed document (formatting-independent)."""
|
||||
return hashlib.sha256(canonical_bytes(doc)).hexdigest()
|
||||
|
||||
|
||||
def flow_path(flows_dir: str | Path, stem: str) -> Path:
|
||||
return Path(flows_dir) / f"{stem}.json"
|
||||
|
||||
|
||||
def write_flow_file(
|
||||
flows_dir: str | Path, doc: dict, *, name: str | None = None
|
||||
) -> Path:
|
||||
"""Write a flow document to backend/flows atomically (tmp + rename).
|
||||
|
||||
The filename derives from `name` (or the document's own name). Returns
|
||||
the written path.
|
||||
"""
|
||||
flows_dir = Path(flows_dir)
|
||||
flows_dir.mkdir(parents=True, exist_ok=True)
|
||||
stem = slugify(name or doc.get("name") or "flow")
|
||||
target = flow_path(flows_dir, stem)
|
||||
fd, tmp_name = tempfile.mkstemp(
|
||||
dir=str(flows_dir), prefix=f".{stem}-", suffix=".tmp"
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as tmp:
|
||||
tmp.write(canonical_bytes(doc))
|
||||
os.replace(tmp_name, target)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(tmp_name)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
return target
|
||||
|
||||
|
||||
def iter_flow_files(flows_dir: str | Path) -> list[Path]:
|
||||
"""Sorted list of candidate flow files (top level only)."""
|
||||
return sorted(Path(flows_dir).glob(FLOWS_GLOB))
|
||||
|
|
@ -17,6 +17,8 @@ class FlowDefinitionRead(BaseModel):
|
|||
name: str
|
||||
description: str
|
||||
status: str
|
||||
# backend/flows filename this definition syncs from (None if API-created).
|
||||
source_path: str | None = None
|
||||
continuation_flow_id: UUID | None
|
||||
created_at: datetime
|
||||
|
||||
|
|
@ -110,3 +112,16 @@ class DecisionRequest(BaseModel):
|
|||
class DecisionRead(BaseModel):
|
||||
task: ApprovalRead
|
||||
continuation_run: RunRead | None
|
||||
|
||||
|
||||
class SyncReport(BaseModel):
|
||||
"""Result of reconciling backend/flows/*.json into DB + LFX.
|
||||
|
||||
Lists hold filename stems; errors maps filename to reason.
|
||||
"""
|
||||
|
||||
added: list[str] = Field(default_factory=list)
|
||||
updated: list[str] = Field(default_factory=list)
|
||||
deleted: list[str] = Field(default_factory=list)
|
||||
skipped: list[str] = Field(default_factory=list)
|
||||
errors: dict[str, str] = Field(default_factory=dict)
|
||||
|
|
|
|||
|
|
@ -56,7 +56,82 @@ def list_definitions(session: Session) -> list[FlowDefinition]:
|
|||
return list(session.exec(select(FlowDefinition).order_by(FlowDefinition.name)).all())
|
||||
|
||||
|
||||
def _validate_flow_json(flow_json: dict) -> None:
|
||||
def get_definition_by_source_path(
|
||||
session: Session, source_path: str
|
||||
) -> FlowDefinition | None:
|
||||
"""Find the definition synced from a given backend/flows filename."""
|
||||
return session.exec(
|
||||
select(FlowDefinition).where(FlowDefinition.source_path == source_path)
|
||||
).first()
|
||||
|
||||
|
||||
def set_definition_name(
|
||||
session: Session, definition: FlowDefinition, name: str
|
||||
) -> FlowDefinition:
|
||||
"""Rename a definition and record its source file. Name clashes 409."""
|
||||
definition.name = name
|
||||
session.add(definition)
|
||||
try:
|
||||
session.commit()
|
||||
except IntegrityError as exc:
|
||||
session.rollback()
|
||||
raise ConflictError(f"flow {name!r} already exists") from exc
|
||||
session.refresh(definition)
|
||||
return definition
|
||||
|
||||
|
||||
def next_version_no(session: Session, flow_id: UUID) -> int:
|
||||
max_no = session.exec(
|
||||
select(func.max(FlowVersion.version_no)).where(FlowVersion.flow_id == flow_id)
|
||||
).one()
|
||||
return (max_no or 0) + 1
|
||||
|
||||
|
||||
def delete_definition_cascade(
|
||||
session: Session, definition: FlowDefinition
|
||||
) -> dict[str, int]:
|
||||
"""Remove a definition and its whole subtree (tasks, runs, versions).
|
||||
|
||||
Used when the canonical file disappears: the directory is the source of
|
||||
truth, so the derived registry follows. Commits once.
|
||||
"""
|
||||
version_ids = list(
|
||||
session.exec(
|
||||
select(FlowVersion.id).where(FlowVersion.flow_id == definition.id)
|
||||
).all()
|
||||
)
|
||||
task_count = 0
|
||||
run_count = 0
|
||||
if version_ids:
|
||||
tasks = list(
|
||||
session.exec(
|
||||
select(ApprovalTask).where(ApprovalTask.version_id.in_(version_ids))
|
||||
).all()
|
||||
)
|
||||
runs = list(
|
||||
session.exec(
|
||||
select(FlowRun).where(FlowRun.version_id.in_(version_ids))
|
||||
).all()
|
||||
)
|
||||
task_count, run_count = len(tasks), len(runs)
|
||||
for row in (*tasks, *runs):
|
||||
session.delete(row)
|
||||
for version in session.exec(
|
||||
select(FlowVersion).where(FlowVersion.flow_id == definition.id)
|
||||
).all():
|
||||
session.delete(version)
|
||||
session.delete(definition)
|
||||
session.commit()
|
||||
return {
|
||||
"definitions": 1,
|
||||
"versions": len(version_ids),
|
||||
"runs": run_count,
|
||||
"tasks": task_count,
|
||||
}
|
||||
|
||||
|
||||
def validate_flow_json(flow_json: dict) -> None:
|
||||
"""Reject anything that is not a Langflow-export-shaped document."""
|
||||
if not isinstance(flow_json, dict):
|
||||
raise InvalidError("langflow_json must be an object")
|
||||
data = flow_json.get("data")
|
||||
|
|
@ -70,7 +145,7 @@ def create_version(
|
|||
session: Session, flow_id: UUID, data: FlowVersionCreate
|
||||
) -> FlowVersion:
|
||||
get_definition(session, flow_id)
|
||||
_validate_flow_json(data.langflow_json)
|
||||
validate_flow_json(data.langflow_json)
|
||||
max_no = session.exec(
|
||||
select(func.max(FlowVersion.version_no)).where(FlowVersion.flow_id == flow_id)
|
||||
).one()
|
||||
|
|
|
|||
274
backend/app/flows/sync.py
Normal file
274
backend/app/flows/sync.py
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
"""Reconciliation: backend/flows/*.json (source of truth) -> DB + LFX.
|
||||
|
||||
The ONLY place that derives registry state from the canonical directory.
|
||||
Triggers (CLI, POST /api/flows/sync, embedded watcher) call
|
||||
sync_from_directory(); single-document callers (e.g. /import) use
|
||||
sync_document(). Everything is async (the LFX client is async) and guarded
|
||||
by a module lock so concurrent triggers serialize on SQLite's single writer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
from sqlmodel import Session
|
||||
from watchfiles import DefaultFilter, awatch
|
||||
|
||||
from app.db import get_engine
|
||||
from app.models import FlowDefinition, FlowVersion
|
||||
from app.settings import settings
|
||||
|
||||
from . import export, store
|
||||
from .lfx_client import LfxClient, LfxError
|
||||
from .schemas import FlowDefinitionCreate, FlowVersionCreate, SyncReport
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Serializes all writers in this process (sync engine + manual API writes).
|
||||
sync_lock = threading.Lock()
|
||||
|
||||
|
||||
def lfx_id_for(definition_id: UUID, version_no: int) -> str:
|
||||
"""Deterministic LFX id (a real UUID: LFX validates the format).
|
||||
|
||||
Same (definition, version) always maps to the same id, so re-syncs and
|
||||
crash recovery overwrite instead of orphaning LFX registrations.
|
||||
"""
|
||||
return str(
|
||||
uuid.uuid5(uuid.NAMESPACE_URL, f"kirche-einfach/{definition_id}/v{version_no}")
|
||||
)
|
||||
|
||||
|
||||
def _latest_version(session: Session, flow_id: UUID) -> FlowVersion | None:
|
||||
versions = store.list_versions(session, flow_id)
|
||||
return versions[-1] if versions else None
|
||||
|
||||
|
||||
async def sync_document(
|
||||
session: Session,
|
||||
lfx: LfxClient,
|
||||
doc: dict,
|
||||
*,
|
||||
source_filename: str,
|
||||
author: str = "",
|
||||
deploy: bool = True,
|
||||
) -> tuple[FlowDefinition, FlowVersion | None, str]:
|
||||
"""Reconcile one parsed document. Returns (definition, version, action).
|
||||
|
||||
Actions: "added" (new definition + version), "updated" (new version),
|
||||
"skipped" (content unchanged). Raises store.ConflictError on name
|
||||
clashes, store.InvalidError on invalid JSON, LfxError when deploy fails.
|
||||
With deploy=False a version is recorded without touching LFX.
|
||||
"""
|
||||
store.validate_flow_json(doc)
|
||||
name = doc.get("name") or Path(source_filename).stem
|
||||
with sync_lock:
|
||||
definition = store.get_definition_by_source_path(session, source_filename)
|
||||
if definition is None:
|
||||
definition = store.create_definition(
|
||||
session, FlowDefinitionCreate(name=name)
|
||||
)
|
||||
definition.source_path = source_filename
|
||||
session.add(definition)
|
||||
session.commit()
|
||||
session.refresh(definition)
|
||||
action = "added"
|
||||
else:
|
||||
action = "updated"
|
||||
if definition.name != name:
|
||||
store.set_definition_name(session, definition, name)
|
||||
|
||||
latest = _latest_version(session, definition.id)
|
||||
if (
|
||||
latest is not None
|
||||
and export.content_hash(latest.langflow_json)
|
||||
== export.content_hash(doc)
|
||||
):
|
||||
return definition, latest, "skipped"
|
||||
|
||||
if not deploy:
|
||||
version = store.create_version(
|
||||
session,
|
||||
definition.id,
|
||||
FlowVersionCreate(langflow_json=doc, author=author),
|
||||
)
|
||||
return definition, version, action
|
||||
|
||||
version_no = store.next_version_no(session, definition.id)
|
||||
lfx_id = lfx_id_for(definition.id, version_no)
|
||||
uploaded = await lfx.upload_flow(doc, flow_id=lfx_id, replace=True)
|
||||
version = store.create_version(
|
||||
session,
|
||||
definition.id,
|
||||
FlowVersionCreate(langflow_json=doc, author=author),
|
||||
)
|
||||
if version.version_no != version_no: # pragma: no cover - lock prevents this
|
||||
raise store.ConflictError(
|
||||
f"concurrent version write for flow {definition.id}"
|
||||
)
|
||||
version = store.mark_deployed(session, version, uploaded.id)
|
||||
if (
|
||||
latest is not None
|
||||
and latest.lfx_flow_id
|
||||
and latest.lfx_flow_id != uploaded.id
|
||||
):
|
||||
try:
|
||||
await lfx.delete_flow(latest.lfx_flow_id)
|
||||
except LfxError as exc:
|
||||
logger.warning(
|
||||
"LFX unregister %s failed (continuing): %s",
|
||||
latest.lfx_flow_id,
|
||||
exc,
|
||||
)
|
||||
return definition, version, action
|
||||
|
||||
|
||||
async def delete_definition_tree(
|
||||
session: Session, lfx: LfxClient, definition: FlowDefinition
|
||||
) -> dict[str, int]:
|
||||
"""Remove a definition whose canonical file is gone.
|
||||
|
||||
LFX registrations go first (best-effort, fail-closed for new runs), the
|
||||
DB subtree (tasks, runs, versions, definition) after. Deterministic LFX
|
||||
ids make any crash in between self-healing on the next sync.
|
||||
"""
|
||||
for version in store.list_versions(session, definition.id):
|
||||
if version.lfx_flow_id:
|
||||
try:
|
||||
await lfx.delete_flow(version.lfx_flow_id)
|
||||
except LfxError as exc:
|
||||
logger.warning(
|
||||
"LFX unregister %s failed (continuing): %s",
|
||||
version.lfx_flow_id,
|
||||
exc,
|
||||
)
|
||||
with sync_lock:
|
||||
return store.delete_definition_cascade(session, definition)
|
||||
|
||||
|
||||
async def sync_from_directory(
|
||||
session: Session, lfx: LfxClient, flows_dir: str | Path
|
||||
) -> SyncReport:
|
||||
"""Reconcile the whole canonical directory. Per-file problems land in
|
||||
report.errors (never raised); a missing directory raises FileNotFoundError
|
||||
so misconfiguration fails loudly instead of wiping the registry.
|
||||
"""
|
||||
flows_dir = Path(flows_dir)
|
||||
report = SyncReport()
|
||||
if not flows_dir.is_dir():
|
||||
raise FileNotFoundError(f"flows directory missing: {flows_dir}")
|
||||
files = {path.name: path for path in export.iter_flow_files(flows_dir)}
|
||||
for filename, path in files.items():
|
||||
try:
|
||||
doc = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as exc:
|
||||
report.errors[filename] = f"unreadable: {exc}"
|
||||
continue
|
||||
if not isinstance(doc, dict):
|
||||
report.errors[filename] = "top-level JSON must be an object"
|
||||
continue
|
||||
try:
|
||||
_, _, action = await sync_document(
|
||||
session, lfx, doc, source_filename=filename
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - reconciliation is total
|
||||
report.errors[filename] = str(exc)
|
||||
continue
|
||||
getattr(report, action).append(path.stem)
|
||||
known = {
|
||||
definition.source_path
|
||||
for definition in store.list_definitions(session)
|
||||
if definition.source_path
|
||||
}
|
||||
for stale in sorted(known - set(files)):
|
||||
definition = store.get_definition_by_source_path(session, stale)
|
||||
if definition is None: # pragma: no cover - defensive
|
||||
continue
|
||||
try:
|
||||
await delete_definition_tree(session, lfx, definition)
|
||||
except Exception as exc: # noqa: BLE001 - reconciliation is total
|
||||
report.errors[stale] = str(exc)
|
||||
continue
|
||||
report.deleted.append(Path(stale).stem)
|
||||
return report
|
||||
|
||||
|
||||
def default_session_factory() -> Callable[[], Session]:
|
||||
return lambda: Session(get_engine())
|
||||
|
||||
|
||||
def default_lfx_factory() -> Callable[[], LfxClient]:
|
||||
return lambda: LfxClient(
|
||||
settings.lfx_url, api_key=settings.lfx_api_key or None
|
||||
)
|
||||
|
||||
|
||||
def run_sync_blocking(
|
||||
flows_dir: str | Path,
|
||||
*,
|
||||
session_factory: Callable[[], Session] | None = None,
|
||||
lfx_factory: Callable[[], LfxClient] | None = None,
|
||||
) -> SyncReport:
|
||||
"""Blocking one-shot sync for worker threads (lifespan, watcher)."""
|
||||
session = (session_factory or default_session_factory())()
|
||||
try:
|
||||
lfx = (lfx_factory or default_lfx_factory())()
|
||||
return asyncio.run(sync_from_directory(session, lfx, Path(flows_dir)))
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
class _JsonFilter(DefaultFilter):
|
||||
def __call__(self, change, path: str) -> bool: # type: ignore[override]
|
||||
return super().__call__(change, path) and path.endswith(".json")
|
||||
|
||||
|
||||
async def watch_forever(
|
||||
stop: asyncio.Event,
|
||||
flows_dir: str | Path,
|
||||
run_sync: Callable[[], SyncReport] | None = None,
|
||||
) -> None:
|
||||
"""Run until `stop` is set: sync on every batch of .json changes.
|
||||
|
||||
All failures are logged, never raised — a broken file must not kill the
|
||||
watcher. `run_sync` is injectable for tests.
|
||||
"""
|
||||
flows_dir = Path(flows_dir)
|
||||
run = run_sync or (lambda: run_sync_blocking(flows_dir))
|
||||
while not stop.is_set() and not flows_dir.is_dir():
|
||||
logger.error("flows dir missing, retrying in 30s: %s", flows_dir)
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), timeout=30)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
if stop.is_set():
|
||||
return
|
||||
logger.info("watching %s", flows_dir)
|
||||
async for _ in awatch(
|
||||
flows_dir, watch_filter=_JsonFilter(), debounce=1000, step=100
|
||||
):
|
||||
if stop.is_set():
|
||||
break
|
||||
try:
|
||||
report = await asyncio.to_thread(run)
|
||||
except FileNotFoundError as exc:
|
||||
logger.error("flow sync failed: %s", exc)
|
||||
continue
|
||||
except Exception: # noqa: BLE001 - watcher must survive
|
||||
logger.exception("flow sync failed")
|
||||
continue
|
||||
logger.info(
|
||||
"flow sync: +%d ~%d -%d skipped=%d errors=%d",
|
||||
len(report.added),
|
||||
len(report.updated),
|
||||
len(report.deleted),
|
||||
len(report.skipped),
|
||||
len(report.errors),
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue