refactor: tighten backend, flows, frontend, tests; push flows to Langflow
This commit is contained in:
Jonas Miederer
parent
6688fb08c6
commit
06ab25fb47
31 changed files with 783 additions and 358 deletions
|
|
@ -12,30 +12,24 @@ from __future__ import annotations
|
|||
import json
|
||||
import logging
|
||||
import secrets
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from collections.abc import AsyncIterator
|
||||
from pathlib import Path
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Query
|
||||
from fastapi.responses import StreamingResponse
|
||||
from sqlmodel import Session
|
||||
|
||||
from app.auth.deps import get_current_user, require_admin
|
||||
from app.db import get_engine, get_session
|
||||
from app.db import get_db
|
||||
from app.models import User
|
||||
from app.settings import settings
|
||||
|
||||
from . import input_schema as form_schema
|
||||
from . import runner, store
|
||||
from .deploy import import_from_langflow, pull_all_from_langflow
|
||||
from .langflow_client import (
|
||||
LangflowClient,
|
||||
LangflowError,
|
||||
LangflowNotFound,
|
||||
LangflowUnavailable,
|
||||
)
|
||||
from .deploy import import_from_langflow
|
||||
from .langflow_client import LangflowClient, LangflowError, LangflowNotFound, LangflowUnavailable
|
||||
from .lfx_client import (
|
||||
LfxClient,
|
||||
LfxConflict,
|
||||
|
|
@ -60,7 +54,7 @@ from .schemas import (
|
|||
RunRequest,
|
||||
SyncReport,
|
||||
)
|
||||
from .sync import lfx_id_for, sync_from_directory
|
||||
from .sync import lfx_id_for
|
||||
|
||||
flows_router = APIRouter(prefix="/api/flows", tags=["flows"])
|
||||
approvals_router = APIRouter(prefix="/api/approvals", tags=["approvals"])
|
||||
|
|
@ -68,10 +62,6 @@ approvals_router = APIRouter(prefix="/api/approvals", tags=["approvals"])
|
|||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_db() -> Iterator[Session]:
|
||||
yield from get_session()
|
||||
|
||||
|
||||
def get_lfx() -> LfxClient:
|
||||
return LfxClient(settings.lfx_url, api_key=settings.lfx_api_key or None)
|
||||
|
||||
|
|
@ -81,60 +71,40 @@ def get_langflow() -> LangflowClient:
|
|||
|
||||
|
||||
def _read_definition(d) -> FlowDefinitionRead:
|
||||
return FlowDefinitionRead(
|
||||
id=d.id,
|
||||
name=d.name,
|
||||
description=d.description,
|
||||
status=d.status,
|
||||
source_path=d.source_path,
|
||||
continuation_flow_id=d.continuation_flow_id,
|
||||
created_at=d.created_at,
|
||||
)
|
||||
return FlowDefinitionRead.model_validate(d, from_attributes=True)
|
||||
|
||||
|
||||
def _read_version(v) -> FlowVersionRead:
|
||||
return FlowVersionRead(
|
||||
id=v.id,
|
||||
flow_id=v.flow_id,
|
||||
version_no=v.version_no,
|
||||
langflow_json=v.langflow_json,
|
||||
lfx_flow_id=v.lfx_flow_id,
|
||||
deployed_at=v.deployed_at,
|
||||
author=v.author,
|
||||
created_at=v.created_at,
|
||||
input_schema=v.input_schema,
|
||||
)
|
||||
return FlowVersionRead.model_validate(v, from_attributes=True)
|
||||
|
||||
|
||||
def _read_run(r) -> RunRead:
|
||||
return RunRead(
|
||||
id=r.id,
|
||||
version_id=r.version_id,
|
||||
tenant_id=r.tenant_id,
|
||||
status=r.status,
|
||||
input=r.input,
|
||||
output=r.output,
|
||||
session_id=r.session_id,
|
||||
created_at=r.created_at,
|
||||
finished_at=r.finished_at,
|
||||
)
|
||||
return RunRead.model_validate(r, from_attributes=True)
|
||||
|
||||
|
||||
def _read_task(t) -> ApprovalRead:
|
||||
return ApprovalRead(
|
||||
id=t.id,
|
||||
run_id=t.run_id,
|
||||
version_id=t.version_id,
|
||||
tenant_id=t.tenant_id,
|
||||
title=t.title,
|
||||
payload=t.payload,
|
||||
continuation_flow_id=t.continuation_flow_id,
|
||||
approver=t.approver,
|
||||
status=t.status,
|
||||
decision_note=t.decision_note,
|
||||
created_at=t.created_at,
|
||||
decided_at=t.decided_at,
|
||||
)
|
||||
return ApprovalRead.model_validate(t, from_attributes=True)
|
||||
|
||||
|
||||
def _lfx_http(exc: LfxError) -> HTTPException:
|
||||
"""Map LFX domain errors to HTTP (single place, was 10x duplicated)."""
|
||||
if isinstance(exc, LfxInvalidFlow):
|
||||
return HTTPException(status_code=422, detail=str(exc))
|
||||
if isinstance(exc, LfxConflict):
|
||||
return HTTPException(status_code=409, detail=str(exc))
|
||||
if isinstance(exc, LfxUnavailable):
|
||||
logger.warning("lfx unavailable: %s", exc)
|
||||
return HTTPException(status_code=503, detail="flow runtime unavailable")
|
||||
logger.warning("lfx error: %s", exc)
|
||||
return HTTPException(status_code=502, detail="flow runtime error")
|
||||
|
||||
|
||||
def _langflow_http(exc: LangflowError) -> HTTPException:
|
||||
if isinstance(exc, LangflowUnavailable):
|
||||
logger.warning("langflow unavailable: %s", exc)
|
||||
return HTTPException(status_code=503, detail="authoring server unavailable")
|
||||
logger.warning("langflow error: %s", exc)
|
||||
return HTTPException(status_code=502, detail="authoring server error")
|
||||
|
||||
|
||||
@flows_router.post("", response_model=FlowDefinitionRead, status_code=201)
|
||||
|
|
@ -164,21 +134,14 @@ async def sync_flows(
|
|||
_admin: Annotated[User, Depends(require_admin)],
|
||||
):
|
||||
"""Manual full pass: Langflow -> backend/flows/*.json -> DB + LFX."""
|
||||
from .sync import sync_full_pass
|
||||
|
||||
try:
|
||||
pulled_result = await pull_all_from_langflow(langflow, Path(settings.flows_dir))
|
||||
except LangflowUnavailable as exc:
|
||||
logger.warning("langflow unavailable: %s", exc)
|
||||
raise HTTPException(status_code=503, detail="authoring server unavailable") from exc
|
||||
return await sync_full_pass(session, lfx, langflow, Path(settings.flows_dir))
|
||||
except LangflowError as exc:
|
||||
logger.warning("langflow error: %s", exc)
|
||||
raise HTTPException(status_code=502, detail="authoring server error") from exc
|
||||
try:
|
||||
report = await sync_from_directory(session, lfx, Path(settings.flows_dir))
|
||||
raise _langflow_http(exc) from exc
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||
report.pulled = pulled_result["pulled"]
|
||||
report.errors = {**pulled_result["errors"], **report.errors}
|
||||
return report
|
||||
|
||||
|
||||
@flows_router.get("/{flow_id}", response_model=FlowDefinitionRead)
|
||||
|
|
@ -236,16 +199,8 @@ async def deploy_version(
|
|||
flow_id=lfx_id_for(flow_id, version_no),
|
||||
replace=True,
|
||||
)
|
||||
except LfxInvalidFlow as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
except LfxConflict as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
except LfxUnavailable as exc:
|
||||
logger.warning("lfx unavailable: %s", exc)
|
||||
raise HTTPException(status_code=503, detail="flow runtime unavailable") from exc
|
||||
except LfxError as exc:
|
||||
logger.warning("lfx error: %s", exc)
|
||||
raise HTTPException(status_code=502, detail="flow runtime error") from exc
|
||||
raise _lfx_http(exc) from exc
|
||||
return _read_version(store.mark_deployed(session, version, uploaded.id))
|
||||
|
||||
|
||||
|
|
@ -290,69 +245,35 @@ async def run_flow(
|
|||
)
|
||||
except LfxNotFound as exc:
|
||||
raise HTTPException(status_code=502, detail=f"LFX drift: {exc}") from exc
|
||||
except LfxUnavailable as exc:
|
||||
logger.warning("lfx unavailable: %s", exc)
|
||||
raise HTTPException(status_code=503, detail="flow runtime unavailable") from exc
|
||||
except LfxError as exc:
|
||||
logger.warning("lfx error: %s", exc)
|
||||
raise HTTPException(status_code=502, detail="flow runtime error") from exc
|
||||
raise _lfx_http(exc) from exc
|
||||
return _read_run(run)
|
||||
|
||||
|
||||
async def _run_stream(
|
||||
session: Session, lfx: LfxClient, version, data: RunRequest, user: User, definition
|
||||
):
|
||||
if not version.lfx_flow_id:
|
||||
raise HTTPException(status_code=422, detail=f"version {version.id} is not deployed to LFX")
|
||||
run = store.create_run(
|
||||
session,
|
||||
version,
|
||||
input=data.input,
|
||||
tenant_id=user.tenant_id,
|
||||
session_id=data.session_id,
|
||||
)
|
||||
envelope = runner.build_envelope(
|
||||
payload=data.input,
|
||||
run_id=run.id,
|
||||
callback_base_url=settings.callback_base_url,
|
||||
continuation_flow_id=definition.continuation_flow_id,
|
||||
callback_token=settings.callback_token,
|
||||
fields=form_schema.schema_field_names(
|
||||
version.input_schema or form_schema.extract_form_schema(version.langflow_json)
|
||||
),
|
||||
)
|
||||
try:
|
||||
run, envelope, lfx_flow_id, sid = await runner.stream_run(
|
||||
session,
|
||||
lfx,
|
||||
version,
|
||||
payload=data.input,
|
||||
tenant_id=user.tenant_id,
|
||||
session_id=data.session_id,
|
||||
callback_base_url=settings.callback_base_url,
|
||||
continuation_flow_id=definition.continuation_flow_id,
|
||||
callback_token=settings.callback_token,
|
||||
)
|
||||
except store.InvalidError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
run_id = run.id
|
||||
lfx_flow_id = version.lfx_flow_id
|
||||
session_id = data.session_id or str(run.id)
|
||||
|
||||
async def event_gen() -> AsyncIterator[str]:
|
||||
yield f"event: run\ndata: {json.dumps({'run_id': str(run_id)})}\n\n"
|
||||
try:
|
||||
async with (
|
||||
httpx.AsyncClient(
|
||||
base_url=lfx.base_url, timeout=lfx.timeout, headers=lfx._headers()
|
||||
) as client,
|
||||
client.stream(
|
||||
"POST",
|
||||
f"/flows/{lfx_flow_id}/stream",
|
||||
json={"input_value": envelope, "session_id": session_id},
|
||||
) as response,
|
||||
):
|
||||
if response.status_code == 404:
|
||||
raise LfxNotFound(f"flow {lfx_flow_id} unknown to LFX")
|
||||
response.raise_for_status()
|
||||
async for chunk in response.aiter_text():
|
||||
yield chunk
|
||||
except (LfxNotFound, httpx.HTTPError) as exc:
|
||||
yield f"event: error\ndata: {json.dumps({'error': str(exc)})}\n\n"
|
||||
# Finalize from a fresh session: the request session may be closed.
|
||||
with Session(get_engine()) as fresh:
|
||||
try:
|
||||
fresh_run = store.get_run(fresh, run_id)
|
||||
status = "pending_approval" if runner._task_exists(fresh, run_id) else "completed"
|
||||
store.set_run_result(fresh, fresh_run, status, {"streamed": True})
|
||||
except store.NotFoundError:
|
||||
status = "failed"
|
||||
async for chunk in runner.stream_lfx_chunks(lfx, lfx_flow_id, envelope, sid):
|
||||
yield chunk
|
||||
status = runner.finalize_streamed_run(run_id)
|
||||
yield f"event: run_finished\ndata: {json.dumps({'run_id': str(run_id), 'status': status})}\n\n"
|
||||
|
||||
return StreamingResponse(event_gen(), media_type="text/event-stream")
|
||||
|
|
@ -379,12 +300,8 @@ async def list_langflow_flows(
|
|||
"""Browse flows on the Langflow authoring server (admin import source)."""
|
||||
try:
|
||||
return await langflow.list_flows()
|
||||
except LangflowUnavailable as exc:
|
||||
logger.warning("langflow unavailable: %s", exc)
|
||||
raise HTTPException(status_code=503, detail="authoring server unavailable") from exc
|
||||
except LangflowError as exc:
|
||||
logger.warning("langflow error: %s", exc)
|
||||
raise HTTPException(status_code=502, detail="authoring server error") from exc
|
||||
raise _langflow_http(exc) from exc
|
||||
|
||||
|
||||
@flows_router.post("/import", response_model=FlowImportRead, status_code=201)
|
||||
|
|
@ -408,26 +325,14 @@ async def import_flow(
|
|||
)
|
||||
except LangflowNotFound as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except LangflowUnavailable as exc:
|
||||
logger.warning("langflow unavailable: %s", exc)
|
||||
raise HTTPException(status_code=503, detail="authoring server unavailable") from exc
|
||||
except LangflowError as exc:
|
||||
logger.warning("langflow error: %s", exc)
|
||||
raise HTTPException(status_code=502, detail="authoring server error") from exc
|
||||
raise _langflow_http(exc) from exc
|
||||
except store.ConflictError as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
except store.InvalidError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
except LfxInvalidFlow as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
except LfxConflict as exc:
|
||||
raise HTTPException(status_code=409, detail=str(exc)) from exc
|
||||
except LfxUnavailable as exc:
|
||||
logger.warning("lfx unavailable: %s", exc)
|
||||
raise HTTPException(status_code=503, detail="flow runtime unavailable") from exc
|
||||
except LfxError as exc:
|
||||
logger.warning("lfx error: %s", exc)
|
||||
raise HTTPException(status_code=502, detail="flow runtime error") from exc
|
||||
raise _lfx_http(exc) from exc
|
||||
return FlowImportRead(definition=_read_definition(definition), version=_read_version(version))
|
||||
|
||||
|
||||
|
|
@ -564,10 +469,8 @@ async def decide_approval(
|
|||
)
|
||||
except LfxNotFound as exc:
|
||||
raise HTTPException(status_code=502, detail=f"LFX drift: {exc}") from exc
|
||||
except LfxUnavailable as exc:
|
||||
raise HTTPException(status_code=503, detail=str(exc)) from exc
|
||||
except LfxError as exc:
|
||||
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
||||
raise _lfx_http(exc) from exc
|
||||
return DecisionRead(
|
||||
task=_read_task(task),
|
||||
continuation_run=_read_run(continuation_run) if continuation_run else None,
|
||||
|
|
|
|||
27
backend/app/flows/base_client.py
Normal file
27
backend/app/flows/base_client.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
"""Shared base for LFX + Langflow async HTTP clients.
|
||||
|
||||
Consolidates base_url stripping, header building, per-request client
|
||||
construction, and HTTPError → domain-error mapping. Subclasses only
|
||||
declare their error types and endpoint calls.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
class BaseApiClient:
|
||||
"""Thin wrapper around httpx.AsyncClient construction."""
|
||||
|
||||
def __init__(self, base_url: str, timeout: float = 60.0, api_key: str | None = None) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout = timeout
|
||||
self.api_key = api_key or ""
|
||||
|
||||
def _headers(self) -> dict[str, str]:
|
||||
return {"x-api-key": self.api_key} if self.api_key else {}
|
||||
|
||||
def _client(self) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(
|
||||
base_url=self.base_url, timeout=self.timeout, headers=self._headers()
|
||||
)
|
||||
|
|
@ -21,7 +21,10 @@ Looks for form nodes — ``CustomInput*`` (``data.type`` or the inner
|
|||
* ``[visible_if ...]`` markers in ``info`` → conditional branches
|
||||
(``allOf`` with ``if``/``then`` per dependent field, arbitrarily nested;
|
||||
see ``_extract_visibility``). The marker is stripped from the shown
|
||||
description.
|
||||
description. Preferred: structured ``visible_if`` key (list of
|
||||
``{field, value}`` or ``{field: value}``) — no quoting mini-language,
|
||||
rename-refactorable, validated at sync time with warnings instead of
|
||||
silent fail-open. Markers stay as deprecated fallback.
|
||||
|
||||
A flow without such a node gets ``{fallback: "textarea"}``: the frontend
|
||||
renders one free-text box instead of a generated form.
|
||||
|
|
@ -177,9 +180,28 @@ def _parse_condition(token: str) -> tuple[str, object] | None:
|
|||
return field, value
|
||||
|
||||
|
||||
def _extract_visibility(info: str) -> tuple[str, list[tuple[str, object]]]:
|
||||
"""(marker-free description, conditions) for a field's ``info`` text."""
|
||||
conditions: list[tuple[str, object]] = []
|
||||
def _extract_visibility(
|
||||
info: str, field: dict | None = None
|
||||
) -> tuple[str, list[tuple[str, object]]]:
|
||||
"""(marker-free description, conditions) for a field's ``info`` text.
|
||||
|
||||
Structured ``visible_if`` on the field wins (list of {field, value} or
|
||||
dict); legacy ``[visible_if ...]`` markers in ``info`` stay as fallback.
|
||||
"""
|
||||
if field is not None:
|
||||
structured = field.get("visible_if")
|
||||
if structured:
|
||||
conditions: list[tuple[str, object]] = []
|
||||
items = structured.items() if isinstance(structured, dict) else structured
|
||||
for item in items or []:
|
||||
if isinstance(item, dict) and "field" in item:
|
||||
conditions.append((item["field"], item.get("value")))
|
||||
elif isinstance(item, (list, tuple)) and len(item) == 2:
|
||||
conditions.append((item[0], item[1]))
|
||||
if conditions:
|
||||
clean = re.sub(r"\s+", " ", (info or "")).strip()
|
||||
return clean, conditions
|
||||
conditions = []
|
||||
|
||||
def _collect(match: re.Match) -> str:
|
||||
for token in _split_conditions(match.group(1)):
|
||||
|
|
@ -263,7 +285,7 @@ def extract_form_schema(langflow_json: dict) -> dict:
|
|||
suffix += 1
|
||||
name = f"{name}_{suffix}"
|
||||
taken.add(name)
|
||||
clean_info, conditions = _extract_visibility(str(field.get("info") or ""))
|
||||
clean_info, conditions = _extract_visibility(str(field.get("info") or ""), field)
|
||||
records.append(
|
||||
{
|
||||
"name": name,
|
||||
|
|
|
|||
|
|
@ -8,6 +8,8 @@ from __future__ import annotations
|
|||
|
||||
import httpx
|
||||
|
||||
from .base_client import BaseApiClient
|
||||
|
||||
|
||||
class LangflowError(Exception):
|
||||
pass
|
||||
|
|
@ -21,23 +23,16 @@ class LangflowUnavailable(LangflowError):
|
|||
pass
|
||||
|
||||
|
||||
class LangflowClient:
|
||||
class LangflowClient(BaseApiClient):
|
||||
"""Thin async wrapper around the Langflow `/api/v1/flows` endpoints."""
|
||||
|
||||
def __init__(self, base_url: str, api_key: str = "", timeout: float = 60.0) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.api_key = api_key
|
||||
self.timeout = timeout
|
||||
|
||||
def _headers(self) -> dict[str, str]:
|
||||
return {"x-api-key": self.api_key} if self.api_key else {}
|
||||
super().__init__(base_url, timeout=timeout, api_key=api_key)
|
||||
|
||||
async def list_flows(self) -> list[dict]:
|
||||
"""List flows (header mode keeps payloads small)."""
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
base_url=self.base_url, timeout=self.timeout, headers=self._headers()
|
||||
) as client:
|
||||
async with self._client() as client:
|
||||
response = await client.get("/api/v1/flows/", params={"header_flows": "true"})
|
||||
if response.status_code == 404:
|
||||
raise LangflowNotFound("flow list not found")
|
||||
|
|
@ -52,9 +47,7 @@ class LangflowClient:
|
|||
async def export_flow(self, langflow_flow_id: str) -> dict:
|
||||
"""Export one flow as Langflow JSON (FlowRead: id/name/description/data)."""
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
base_url=self.base_url, timeout=self.timeout, headers=self._headers()
|
||||
) as client:
|
||||
async with self._client() as client:
|
||||
response = await client.get(f"/api/v1/flows/{langflow_flow_id}")
|
||||
if response.status_code == 404:
|
||||
raise LangflowNotFound(f"flow {langflow_flow_id} not found")
|
||||
|
|
@ -64,3 +57,24 @@ class LangflowClient:
|
|||
raise LangflowError(f"Langflow error {exc.response.status_code}") from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise LangflowUnavailable(f"Langflow unreachable: {exc}") from exc
|
||||
|
||||
async def upsert_flow(self, doc: dict) -> dict:
|
||||
"""Create or update a flow in Langflow by name (authoring UI refresh).
|
||||
|
||||
Used after flow/component tightening so the visual editor shows the
|
||||
latest canonical JSON without a container restart.
|
||||
"""
|
||||
try:
|
||||
async with self._client() as client:
|
||||
existing = await self.list_flows()
|
||||
match = next((f for f in existing if f.get("name") == doc.get("name")), None)
|
||||
if match and match.get("id"):
|
||||
response = await client.patch(f"/api/v1/flows/{match['id']}", json=doc)
|
||||
else:
|
||||
response = await client.post("/api/v1/flows/", json=doc)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except httpx.HTTPStatusError as exc:
|
||||
raise LangflowError(f"Langflow error {exc.response.status_code}") from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise LangflowUnavailable(f"Langflow unreachable: {exc}") from exc
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ from dataclasses import dataclass
|
|||
|
||||
import httpx
|
||||
|
||||
from .base_client import BaseApiClient
|
||||
|
||||
|
||||
class LfxError(Exception):
|
||||
"""Base error for LFX communication problems."""
|
||||
|
|
@ -48,21 +50,11 @@ class LfxRunResult:
|
|||
component: str
|
||||
|
||||
|
||||
class LfxClient:
|
||||
class LfxClient(BaseApiClient):
|
||||
"""Thin async wrapper around the LFX serve HTTP API."""
|
||||
|
||||
def __init__(self, base_url: str, timeout: float = 120.0, api_key: str | None = None) -> None:
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.timeout = timeout
|
||||
self.api_key = api_key
|
||||
|
||||
def _headers(self) -> dict[str, str]:
|
||||
return {"x-api-key": self.api_key} if self.api_key else {}
|
||||
|
||||
def _client(self) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(
|
||||
base_url=self.base_url, timeout=self.timeout, headers=self._headers()
|
||||
)
|
||||
super().__init__(base_url, timeout=timeout, api_key=api_key)
|
||||
|
||||
async def health(self) -> dict:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -3,15 +3,18 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
from uuid import UUID
|
||||
|
||||
from sqlmodel import Session, select
|
||||
import httpx
|
||||
from sqlmodel import Session
|
||||
|
||||
from app.models import ApprovalTask, FlowRun, FlowVersion
|
||||
from app.db import get_engine
|
||||
from app.models import FlowRun, FlowVersion
|
||||
|
||||
from . import input_schema as form_schema
|
||||
from . import store
|
||||
from .lfx_client import LfxClient, LfxRunFailed
|
||||
from .lfx_client import LfxClient, LfxNotFound, LfxRunFailed
|
||||
from .store import InvalidError
|
||||
|
||||
|
||||
|
|
@ -40,12 +43,7 @@ def build_envelope(
|
|||
|
||||
|
||||
def _task_exists(session: Session, run_id: UUID) -> bool:
|
||||
return (
|
||||
session.exec(
|
||||
select(ApprovalTask).where(ApprovalTask.run_id == run_id, ApprovalTask.status == "open")
|
||||
).first()
|
||||
is not None
|
||||
)
|
||||
return store.has_open_task(session, run_id)
|
||||
|
||||
|
||||
async def execute_run(
|
||||
|
|
@ -97,3 +95,73 @@ async def execute_run(
|
|||
if _task_exists(session, run.id):
|
||||
return store.set_run_result(session, run, "pending_approval", output)
|
||||
return store.set_run_result(session, run, "completed", output)
|
||||
|
||||
|
||||
async def stream_run(
|
||||
session: Session,
|
||||
lfx: LfxClient,
|
||||
version: FlowVersion,
|
||||
*,
|
||||
payload: dict,
|
||||
tenant_id: str,
|
||||
session_id: str | None,
|
||||
callback_base_url: str,
|
||||
continuation_flow_id: UUID | None,
|
||||
callback_token: str,
|
||||
) -> tuple[FlowRun, str, str, str]:
|
||||
"""Create a run + envelope for SSE streaming. Returns (run, envelope, lfx_id, sid).
|
||||
|
||||
The actual SSE bytes are produced by the API layer; finalization uses a
|
||||
fresh session (the request session may be closed when the generator ends).
|
||||
"""
|
||||
if not version.lfx_flow_id:
|
||||
raise InvalidError(f"version {version.id} is not deployed to LFX")
|
||||
run = store.create_run(
|
||||
session, version, input=payload, tenant_id=tenant_id, session_id=session_id
|
||||
)
|
||||
cached = version.input_schema or form_schema.extract_form_schema(version.langflow_json)
|
||||
envelope = build_envelope(
|
||||
payload=payload,
|
||||
run_id=run.id,
|
||||
callback_base_url=callback_base_url,
|
||||
continuation_flow_id=continuation_flow_id,
|
||||
callback_token=callback_token,
|
||||
fields=form_schema.schema_field_names(cached),
|
||||
)
|
||||
return run, envelope, version.lfx_flow_id, session_id or str(run.id)
|
||||
|
||||
|
||||
async def stream_lfx_chunks(
|
||||
lfx: LfxClient, lfx_flow_id: str, envelope: str, sid: str
|
||||
) -> AsyncIterator[str]:
|
||||
"""Yield raw SSE chunks from LFX, mapping transport errors to error events."""
|
||||
try:
|
||||
async with (
|
||||
httpx.AsyncClient(
|
||||
base_url=lfx.base_url, timeout=lfx.timeout, headers=lfx._headers()
|
||||
) as client,
|
||||
client.stream(
|
||||
"POST",
|
||||
f"/flows/{lfx_flow_id}/stream",
|
||||
json={"input_value": envelope, "session_id": sid},
|
||||
) as response,
|
||||
):
|
||||
if response.status_code == 404:
|
||||
raise LfxNotFound(f"flow {lfx_flow_id} unknown to LFX")
|
||||
response.raise_for_status()
|
||||
async for chunk in response.aiter_text():
|
||||
yield chunk
|
||||
except (LfxNotFound, httpx.HTTPError) as exc:
|
||||
yield f"event: error\ndata: {json.dumps({'error': str(exc)})}\n\n"
|
||||
|
||||
|
||||
def finalize_streamed_run(run_id: UUID) -> str:
|
||||
"""Mark a streamed run completed/pending_approval from a fresh session."""
|
||||
with Session(get_engine()) as fresh:
|
||||
try:
|
||||
fresh_run = store.get_run(fresh, run_id)
|
||||
status = "pending_approval" if store.has_open_task(fresh, run_id) else "completed"
|
||||
store.set_run_result(fresh, fresh_run, status, {"streamed": True})
|
||||
except store.NotFoundError:
|
||||
status = "failed"
|
||||
return status
|
||||
|
|
|
|||
|
|
@ -132,12 +132,9 @@ def validate_flow_json(flow_json: dict) -> None:
|
|||
def create_version(session: Session, flow_id: UUID, data: FlowVersionCreate) -> FlowVersion:
|
||||
get_definition(session, flow_id)
|
||||
validate_flow_json(data.langflow_json)
|
||||
max_no = session.exec(
|
||||
select(func.max(FlowVersion.version_no)).where(FlowVersion.flow_id == flow_id)
|
||||
).one()
|
||||
version = FlowVersion(
|
||||
flow_id=flow_id,
|
||||
version_no=(max_no or 0) + 1,
|
||||
version_no=next_version_no(session, flow_id),
|
||||
langflow_json=data.langflow_json,
|
||||
author=data.author,
|
||||
# Single choke point: sync, import, and API creation all cache the
|
||||
|
|
@ -229,6 +226,16 @@ def set_run_result(session: Session, run: FlowRun, status: str, output: dict | N
|
|||
return run
|
||||
|
||||
|
||||
def has_open_task(session: Session, run_id: UUID) -> bool:
|
||||
"""True when a run has an open approval task (terminal-state mapping)."""
|
||||
return (
|
||||
session.exec(
|
||||
select(ApprovalTask).where(ApprovalTask.run_id == run_id, ApprovalTask.status == "open")
|
||||
).first()
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
def get_run(session: Session, run_id: UUID) -> FlowRun:
|
||||
run = session.get(FlowRun, run_id)
|
||||
if run is None:
|
||||
|
|
|
|||
|
|
@ -186,6 +186,14 @@ async def sync_from_directory(
|
|||
return report
|
||||
|
||||
|
||||
def format_report(report: SyncReport) -> str:
|
||||
"""One-line summary for startup/watcher logs (keeps main.py thin)."""
|
||||
return (
|
||||
f"+{len(report.added)} ~{len(report.updated)} -{len(report.deleted)} "
|
||||
f"skipped={len(report.skipped)} errors={len(report.errors)}"
|
||||
)
|
||||
|
||||
|
||||
def default_session_factory() -> Callable[[], Session]:
|
||||
return lambda: Session(get_engine())
|
||||
|
||||
|
|
@ -194,6 +202,27 @@ def default_lfx_factory() -> Callable[[], LfxClient]:
|
|||
return lambda: LfxClient(settings.lfx_url, api_key=settings.lfx_api_key or None)
|
||||
|
||||
|
||||
async def sync_full_pass(
|
||||
session: Session,
|
||||
lfx: LfxClient,
|
||||
langflow,
|
||||
flows_dir: str | Path,
|
||||
) -> SyncReport:
|
||||
"""Single orchestration: Langflow pull -> files -> DB + LFX.
|
||||
|
||||
Shared by POST /api/flows/sync, CLI, and watcher-adjacent paths so the
|
||||
full-pass logic lives once. Pull errors merge with sync errors.
|
||||
"""
|
||||
from .deploy import pull_all_from_langflow
|
||||
|
||||
flows_dir = Path(flows_dir)
|
||||
pulled_result = await pull_all_from_langflow(langflow, flows_dir)
|
||||
report = await sync_from_directory(session, lfx, flows_dir)
|
||||
report.pulled = pulled_result["pulled"] # type: ignore[typeddict-item]
|
||||
report.errors = {**pulled_result["errors"], **report.errors} # type: ignore[typeddict-item]
|
||||
return report
|
||||
|
||||
|
||||
def run_sync_blocking(
|
||||
flows_dir: str | Path,
|
||||
*,
|
||||
|
|
@ -250,11 +279,4 @@ async def watch_forever(
|
|||
except Exception:
|
||||
logger.exception("flow sync failed")
|
||||
continue
|
||||
logger.info(
|
||||
"flow sync: +%d ~%d -%d skipped=%d errors=%d",
|
||||
len(report.added),
|
||||
len(report.updated),
|
||||
len(report.deleted),
|
||||
len(report.skipped),
|
||||
len(report.errors),
|
||||
)
|
||||
logger.info("flow sync: %s", format_report(report))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue