137 lines
4.7 KiB
Python
137 lines
4.7 KiB
Python
"""Canonical flow files: backend/flows/<name>.json is the source of truth.
|
|
|
|
One Langflow export per file, committed to git. This module only reads and
|
|
writes the directory; reconciliation into the DB lives in sync.py.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
FLOWS_GLOB = "*.json"
|
|
|
|
#: Opt-out list in the flows dir (one stem per line, `#` comments):
|
|
#: listed flows stay in Langflow only — never synced, deployed,
|
|
#: validated, or shown in the frontend.
|
|
IGNORE_FILENAME = ".flowsignore"
|
|
|
|
|
|
def ignored_stems(flows_dir: str | Path) -> set[str]:
|
|
"""Stems excluded from the pipeline (empty when no ignore file)."""
|
|
try:
|
|
text = (Path(flows_dir) / IGNORE_FILENAME).read_text(encoding="utf-8")
|
|
except OSError:
|
|
return set()
|
|
return {
|
|
line.strip()
|
|
for line in text.splitlines()
|
|
if line.strip() and not line.strip().startswith("#")
|
|
}
|
|
|
|
|
|
def filename_stem(name: str) -> str:
|
|
"""Map a flow name to a filename stem, keeping the original name.
|
|
|
|
Only filesystem-reserved characters (``/ \\ : * ? " < > |`` and control
|
|
chars) become ``-``; leading/trailing whitespace and trailing dots are
|
|
stripped (Windows). Case, spaces, underscores and umlauts are preserved:
|
|
``test_flow`` stays ``test_flow``.
|
|
"""
|
|
stem = re.sub(r'[\\/:*?"<>|\x00-\x1f]', "-", name).strip()
|
|
stem = stem.rstrip(".")
|
|
return stem or "flow"
|
|
|
|
|
|
def canonical_bytes(doc: dict) -> bytes:
|
|
"""Deterministic serialization: stable bytes for stable git diffs."""
|
|
return (json.dumps(doc, indent=2, ensure_ascii=False, sort_keys=True) + "\n").encode("utf-8")
|
|
|
|
|
|
def content_hash(doc: dict) -> str:
|
|
"""Hash of the parsed document (formatting-independent)."""
|
|
return hashlib.sha256(canonical_bytes(doc)).hexdigest()
|
|
|
|
|
|
def flow_path(flows_dir: str | Path, stem: str) -> Path:
|
|
return Path(flows_dir) / f"{stem}.json"
|
|
|
|
|
|
def _read_doc(path: Path) -> dict | None:
|
|
try:
|
|
doc = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, ValueError):
|
|
return None
|
|
return doc if isinstance(doc, dict) else None
|
|
|
|
|
|
def find_reuse_path(flows_dir: str | Path, doc: dict) -> tuple[Path | None, bool]:
|
|
"""Existing file to reuse for `doc`: (path, identical).
|
|
|
|
Matches identical content first (any filename), then the same flow name.
|
|
Returns (None, False) when no candidate exists (caller writes a slug file).
|
|
This keeps pulls idempotent: a flow seeded from `simple_rule.json` is
|
|
updated in place instead of duplicated as `simple-rule.json`.
|
|
"""
|
|
flows_dir = Path(flows_dir)
|
|
if not flows_dir.is_dir():
|
|
return None, False
|
|
wanted_hash = content_hash(doc)
|
|
wanted_name = doc.get("name")
|
|
named: Path | None = None
|
|
for path in iter_flow_files(flows_dir):
|
|
existing = _read_doc(path)
|
|
if existing is None:
|
|
continue
|
|
if content_hash(existing) == wanted_hash:
|
|
return path, True
|
|
if named is None and wanted_name and existing.get("name") == wanted_name:
|
|
named = path
|
|
return (named, False) if named is not None else (None, False)
|
|
|
|
|
|
def write_flow_bytes(target: Path, doc: dict) -> Path:
|
|
"""Write a flow document to an explicit path atomically (tmp + rename).
|
|
|
|
The file ends up world-readable (0o644): mkstemp creates 0o600, which
|
|
would lock out the other containers (Langflow/LFX run as different
|
|
users and crash on unreadable seed files).
|
|
"""
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
fd, tmp_name = tempfile.mkstemp(
|
|
dir=str(target.parent), prefix=f".{target.stem}-", suffix=".tmp"
|
|
)
|
|
try:
|
|
with os.fdopen(fd, "wb") as tmp:
|
|
tmp.write(canonical_bytes(doc))
|
|
os.replace(tmp_name, target)
|
|
os.chmod(target, 0o644)
|
|
except BaseException:
|
|
try:
|
|
os.unlink(tmp_name)
|
|
except OSError:
|
|
pass
|
|
raise
|
|
return target
|
|
|
|
|
|
def write_flow_file(flows_dir: str | Path, doc: dict, *, name: str | None = None) -> Path:
|
|
"""Write a flow document to backend/flows atomically (tmp + rename).
|
|
|
|
The filename derives from `name` (or the document's own name). Returns
|
|
the written path.
|
|
"""
|
|
flows_dir = Path(flows_dir)
|
|
flows_dir.mkdir(parents=True, exist_ok=True)
|
|
stem = filename_stem(name or doc.get("name") or "flow")
|
|
return write_flow_bytes(flow_path(flows_dir, stem), doc)
|
|
|
|
|
|
def iter_flow_files(flows_dir: str | Path, *, ignored: set[str] | None = None) -> list[Path]:
|
|
"""Sorted list of candidate flow files (top level only, minus ignored)."""
|
|
skip = ignored if ignored is not None else ignored_stems(flows_dir)
|
|
return sorted(p for p in Path(flows_dir).glob(FLOWS_GLOB) if p.stem not in skip)
|