fix: make Langflow-synced flows visible (lfx volume perms, pull dedup)
This commit is contained in:
parent
1326cece42
commit
95a1c1afb7
12 changed files with 5460 additions and 3859 deletions
|
|
@ -13,6 +13,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||||
Startup-Sync bleiben dateibasiert. Die Verwaltung zeigt eine deutsche
|
Startup-Sync bleiben dateibasiert. Die Verwaltung zeigt eine deutsche
|
||||||
Schritt-Anleitung direkt beim Sync-Button.
|
Schritt-Anleitung direkt beim Sync-Button.
|
||||||
### Fixed
|
### Fixed
|
||||||
|
- Flow-Sync aus Langflow: neue Flows blieben unsichtbar, weil LFX-Uploads
|
||||||
|
mit `Permission denied` scheiterten (named Volume root-owned, LFX läuft
|
||||||
|
rootless). `lfx-entrypoint.sh` chownt `/flows` per gosu vor dem Start;
|
||||||
|
zusätzlich Pull-Dedup (wiederverwendet gleichnamige/identische Dateien
|
||||||
|
statt Bindestrich-Duplikate wie `simple-rule.json`, die jede Sync-Meldung
|
||||||
|
mit Namenskonflikten füllten).
|
||||||
- LFX runtime user gets a home directory (`useradd -m`, `HOME` set):
|
- LFX runtime user gets a home directory (`useradd -m`, `HOME` set):
|
||||||
without one every flow run failed with "Settings service not found".
|
without one every flow run failed with "Settings service not found".
|
||||||
### Security
|
### Security
|
||||||
|
|
|
||||||
|
|
@ -16,11 +16,18 @@ RUN pip install "lfx~=${LFX_VERSION}" requests
|
||||||
# Non-root runtime: uploads persist into the mounted volume. The user needs
|
# Non-root runtime: uploads persist into the mounted volume. The user needs
|
||||||
# a home directory: lfx resolves settings/cache paths from $HOME, and
|
# a home directory: lfx resolves settings/cache paths from $HOME, and
|
||||||
# without one every flow run fails ("Settings service not found").
|
# without one every flow run fails ("Settings service not found").
|
||||||
|
# gosu drops privileges in the entrypoint after chowning /flows: the named
|
||||||
|
# volume arrives root-owned and the server user could otherwise never write
|
||||||
|
# uploads (every deploy would fail with "Permission denied").
|
||||||
ENV HOME=/home/lfx
|
ENV HOME=/home/lfx
|
||||||
RUN useradd -r -m -s /bin/false lfx && mkdir -p /flows && chown lfx /flows
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends gosu \
|
||||||
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
|
&& useradd -r -m -s /bin/false lfx && mkdir -p /flows && chown lfx /flows
|
||||||
|
COPY lfx-entrypoint.sh /usr/local/bin/lfx-entrypoint.sh
|
||||||
VOLUME /flows
|
VOLUME /flows
|
||||||
EXPOSE 6004
|
EXPOSE 6004
|
||||||
USER lfx
|
ENTRYPOINT ["sh", "/usr/local/bin/lfx-entrypoint.sh"]
|
||||||
|
|
||||||
# Serve the /flows dir (uploaded flows persist there via --flow-dir).
|
# Serve the /flows dir (uploaded flows persist there via --flow-dir).
|
||||||
# No positional path: with one, lfx>=1.12.1 exits when the dir holds no
|
# No positional path: with one, lfx>=1.12.1 exits when the dir holds no
|
||||||
|
|
|
||||||
|
|
@ -49,7 +49,12 @@ async def pull_all_from_langflow(
|
||||||
try:
|
try:
|
||||||
exported = await langflow.export_flow(flow_id)
|
exported = await langflow.export_flow(flow_id)
|
||||||
doc = normalize_exported_flow(exported, flow_id)
|
doc = normalize_exported_flow(exported, flow_id)
|
||||||
path = export.write_flow_file(flows_dir, doc, name=doc["name"])
|
reuse, identical = export.find_reuse_path(flows_dir, doc)
|
||||||
|
if identical and reuse is not None:
|
||||||
|
pulled.append(reuse.stem)
|
||||||
|
continue
|
||||||
|
target = reuse or export.flow_path(flows_dir, export.slugify(doc["name"]))
|
||||||
|
path = export.write_flow_bytes(target, doc)
|
||||||
pulled.append(path.stem)
|
pulled.append(path.stem)
|
||||||
except (LangflowError, OSError, ValueError) as exc:
|
except (LangflowError, OSError, ValueError) as exc:
|
||||||
errors[flow_id] = str(exc)
|
errors[flow_id] = str(exc)
|
||||||
|
|
|
||||||
|
|
@ -36,17 +36,45 @@ def flow_path(flows_dir: str | Path, stem: str) -> Path:
|
||||||
return Path(flows_dir) / f"{stem}.json"
|
return Path(flows_dir) / f"{stem}.json"
|
||||||
|
|
||||||
|
|
||||||
def write_flow_file(flows_dir: str | Path, doc: dict, *, name: str | None = None) -> Path:
|
def _read_doc(path: Path) -> dict | None:
|
||||||
"""Write a flow document to backend/flows atomically (tmp + rename).
|
try:
|
||||||
|
doc = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return None
|
||||||
|
return doc if isinstance(doc, dict) else None
|
||||||
|
|
||||||
The filename derives from `name` (or the document's own name). Returns
|
|
||||||
the written path.
|
def find_reuse_path(flows_dir: str | Path, doc: dict) -> tuple[Path | None, bool]:
|
||||||
|
"""Existing file to reuse for `doc`: (path, identical).
|
||||||
|
|
||||||
|
Matches identical content first (any filename), then the same flow name.
|
||||||
|
Returns (None, False) when no candidate exists (caller writes a slug file).
|
||||||
|
This keeps pulls idempotent: a flow seeded from `simple_rule.json` is
|
||||||
|
updated in place instead of duplicated as `simple-rule.json`.
|
||||||
"""
|
"""
|
||||||
flows_dir = Path(flows_dir)
|
flows_dir = Path(flows_dir)
|
||||||
flows_dir.mkdir(parents=True, exist_ok=True)
|
if not flows_dir.is_dir():
|
||||||
stem = slugify(name or doc.get("name") or "flow")
|
return None, False
|
||||||
target = flow_path(flows_dir, stem)
|
wanted_hash = content_hash(doc)
|
||||||
fd, tmp_name = tempfile.mkstemp(dir=str(flows_dir), prefix=f".{stem}-", suffix=".tmp")
|
wanted_name = doc.get("name")
|
||||||
|
named: Path | None = None
|
||||||
|
for path in iter_flow_files(flows_dir):
|
||||||
|
existing = _read_doc(path)
|
||||||
|
if existing is None:
|
||||||
|
continue
|
||||||
|
if content_hash(existing) == wanted_hash:
|
||||||
|
return path, True
|
||||||
|
if named is None and wanted_name and existing.get("name") == wanted_name:
|
||||||
|
named = path
|
||||||
|
return (named, False) if named is not None else (None, False)
|
||||||
|
|
||||||
|
|
||||||
|
def write_flow_bytes(target: Path, doc: dict) -> Path:
|
||||||
|
"""Write a flow document to an explicit path atomically (tmp + rename)."""
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
fd, tmp_name = tempfile.mkstemp(
|
||||||
|
dir=str(target.parent), prefix=f".{target.stem}-", suffix=".tmp"
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
with os.fdopen(fd, "wb") as tmp:
|
with os.fdopen(fd, "wb") as tmp:
|
||||||
tmp.write(canonical_bytes(doc))
|
tmp.write(canonical_bytes(doc))
|
||||||
|
|
@ -60,6 +88,18 @@ def write_flow_file(flows_dir: str | Path, doc: dict, *, name: str | None = None
|
||||||
return target
|
return target
|
||||||
|
|
||||||
|
|
||||||
|
def write_flow_file(flows_dir: str | Path, doc: dict, *, name: str | None = None) -> Path:
|
||||||
|
"""Write a flow document to backend/flows atomically (tmp + rename).
|
||||||
|
|
||||||
|
The filename derives from `name` (or the document's own name). Returns
|
||||||
|
the written path.
|
||||||
|
"""
|
||||||
|
flows_dir = Path(flows_dir)
|
||||||
|
flows_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
stem = slugify(name or doc.get("name") or "flow")
|
||||||
|
return write_flow_bytes(flow_path(flows_dir, stem), doc)
|
||||||
|
|
||||||
|
|
||||||
def iter_flow_files(flows_dir: str | Path) -> list[Path]:
|
def iter_flow_files(flows_dir: str | Path) -> list[Path]:
|
||||||
"""Sorted list of candidate flow files (top level only)."""
|
"""Sorted list of candidate flow files (top level only)."""
|
||||||
return sorted(Path(flows_dir).glob(FLOWS_GLOB))
|
return sorted(Path(flows_dir).glob(FLOWS_GLOB))
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
747
backend/flows/new-flow.json
Normal file
747
backend/flows/new-flow.json
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
747
backend/flows/test-flow.json
Normal file
747
backend/flows/test-flow.json
Normal file
File diff suppressed because one or more lines are too long
|
|
@ -37,3 +37,30 @@ def test_iter_flow_files_only_top_level_json(tmp_path):
|
||||||
sub.mkdir()
|
sub.mkdir()
|
||||||
(sub / "c.json").write_text("{}")
|
(sub / "c.json").write_text("{}")
|
||||||
assert export.iter_flow_files(tmp_path) == [tmp_path / "a.json"]
|
assert export.iter_flow_files(tmp_path) == [tmp_path / "a.json"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_reuse_path_matches_identical_content_first(tmp_path):
|
||||||
|
doc = {"name": "Simple Rule", "data": {"nodes": [], "edges": []}}
|
||||||
|
other = tmp_path / "other-name.json"
|
||||||
|
other.write_text(export.canonical_bytes(doc).decode())
|
||||||
|
path, identical = export.find_reuse_path(tmp_path, dict(doc))
|
||||||
|
assert (path, identical) == (other, True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_reuse_path_matches_same_name(tmp_path):
|
||||||
|
old = {"name": "Simple Rule", "data": {"nodes": [], "edges": []}, "extra": 1}
|
||||||
|
(tmp_path / "simple_rule.json").write_text(json.dumps(old))
|
||||||
|
new = {"name": "Simple Rule", "data": {"nodes": [], "edges": []}}
|
||||||
|
path, identical = export.find_reuse_path(tmp_path, new)
|
||||||
|
assert (path, identical) == (tmp_path / "simple_rule.json", False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_reuse_path_no_match_and_missing_dir(tmp_path):
|
||||||
|
(tmp_path / "a.json").write_text(json.dumps({"name": "Other"}))
|
||||||
|
assert export.find_reuse_path(tmp_path, {"name": "Neu"}) == (None, False)
|
||||||
|
assert export.find_reuse_path(tmp_path / "gibts-nicht", {"name": "Neu"}) == (None, False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_find_reuse_path_skips_unreadable_files(tmp_path):
|
||||||
|
(tmp_path / "kaputt.json").write_text("{kein json")
|
||||||
|
assert export.find_reuse_path(tmp_path, {"name": "Neu"}) == (None, False)
|
||||||
|
|
|
||||||
|
|
@ -176,6 +176,31 @@ async def test_sync_endpoint_pulls_from_langflow_then_syncs(
|
||||||
assert body["skipped"] == ["gehaltserh-hung"]
|
assert body["skipped"] == ["gehaltserh-hung"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_sync_endpoint_pull_reuses_same_named_file(
|
||||||
|
client, lfx_server, tmp_path, monkeypatch, login, override_langflow
|
||||||
|
):
|
||||||
|
"""A pulled flow seeded from simple_rule.json updates it in place
|
||||||
|
instead of duplicating it as simple-rule.json (slug drift)."""
|
||||||
|
await login()
|
||||||
|
monkeypatch.setattr(settings, "flows_dir", str(tmp_path))
|
||||||
|
_seed_dir(tmp_path) # fixture shape, name "simple_rule"
|
||||||
|
langflow_id = "44444444-5555-4666-8777-888888888888"
|
||||||
|
override_langflow(
|
||||||
|
flows=[{"id": langflow_id, "name": "simple_rule"}],
|
||||||
|
docs={langflow_id: _langflow_doc("simple_rule", langflow_id)},
|
||||||
|
)
|
||||||
|
response = await client.post("/api/flows/sync")
|
||||||
|
assert response.status_code == 200, response.text
|
||||||
|
body = response.json()
|
||||||
|
assert body["pulled"] == ["simple_rule"]
|
||||||
|
assert body["added"] == ["simple_rule"]
|
||||||
|
assert body["errors"] == {}
|
||||||
|
assert not (tmp_path / "simple-rule.json").exists()
|
||||||
|
|
||||||
|
response = await client.post("/api/flows/sync")
|
||||||
|
assert response.json()["skipped"] == ["simple_rule"]
|
||||||
|
|
||||||
|
|
||||||
async def test_sync_endpoint_pull_error_continues(
|
async def test_sync_endpoint_pull_error_continues(
|
||||||
client, lfx_server, tmp_path, monkeypatch, login, override_langflow
|
client, lfx_server, tmp_path, monkeypatch, login, override_langflow
|
||||||
):
|
):
|
||||||
|
|
|
||||||
9
lfx-entrypoint.sh
Normal file
9
lfx-entrypoint.sh
Normal file
|
|
@ -0,0 +1,9 @@
|
||||||
|
#!/bin/sh
|
||||||
|
# LFX entrypoint (runs as root, drops privileges for the server).
|
||||||
|
# The named volume arrives root-owned (Docker creates it as root and the
|
||||||
|
# image USER cannot chown it) — without this every flow upload fails with
|
||||||
|
# "Permission denied" and synced flows stay version-less (invisible in UI).
|
||||||
|
set -e
|
||||||
|
|
||||||
|
chown -R lfx:lfx /flows
|
||||||
|
exec gosu lfx "$@"
|
||||||
Loading…
Add table
Add a link
Reference in a new issue