mirror of
https://github.com/langgenius/dify.git
synced 2026-11-22 02:59:44 +08:00
2267 lines
89 KiB
Python
2267 lines
89 KiB
Python
import io
|
|
import json
|
|
import logging
|
|
import time
|
|
import uuid
|
|
from collections import Counter
|
|
from collections.abc import Callable, Generator, Mapping, Sequence
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass
|
|
from datetime import timedelta
|
|
from numbers import Real
|
|
from threading import Event, Thread
|
|
from typing import Any
|
|
|
|
from celery import Task, shared_task
|
|
from openpyxl import Workbook
|
|
from openpyxl.cell.cell import Cell
|
|
from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
|
|
from openpyxl.utils import get_column_letter
|
|
from sqlalchemy import select, update
|
|
from sqlalchemy.engine import CursorResult, Result
|
|
from sqlalchemy.orm import Session
|
|
|
|
from clients.agent_backend import (
|
|
AgentBackendRunCancelledInternalEvent,
|
|
AgentBackendRunEventAdapter,
|
|
AgentBackendRunFailedInternalEvent,
|
|
AgentBackendRunSucceededInternalEvent,
|
|
create_agent_backend_run_client,
|
|
)
|
|
from configs import dify_config
|
|
from core.db.session_factory import session_factory
|
|
from core.evaluation.agent.agent_v2_trace_collector import AgentV2TraceCollector
|
|
from core.evaluation.agent.customized_workflow_adapter import (
|
|
AGENT_CUSTOM_EVALUATOR_NODE_ID,
|
|
build_agent_custom_evaluator_node_result,
|
|
)
|
|
from core.evaluation.base_evaluation_instance import BaseEvaluationInstance
|
|
from core.evaluation.entities.config_entity import EvaluationFrameworkEnum
|
|
from core.evaluation.entities.evaluation_entity import (
|
|
AgentEvaluationItem,
|
|
AgentToolDefinition,
|
|
CustomizedMetrics,
|
|
EvaluationCategory,
|
|
EvaluationDatasetInput,
|
|
EvaluationItemResult,
|
|
EvaluationMetric,
|
|
EvaluationRunData,
|
|
NodeInfo,
|
|
)
|
|
from core.evaluation.entities.judgment_entity import JudgmentConfig, JudgmentResult
|
|
from core.evaluation.evaluation_manager import EvaluationManager
|
|
from core.evaluation.exceptions import (
|
|
EvaluationErrorCode,
|
|
EvaluationWorkflowVersionNotFoundError,
|
|
new_public_evaluation_error,
|
|
safe_evaluation_error_text,
|
|
)
|
|
from core.evaluation.judgment.processor import JudgmentProcessor
|
|
from core.evaluation.runners.agent_evaluation_runner import AgentEvaluationRunner
|
|
from core.evaluation.runners.base_evaluation_runner import BaseEvaluationRunner
|
|
from core.evaluation.runners.llm_evaluation_runner import LLMEvaluationRunner
|
|
from core.evaluation.runners.retrieval_evaluation_runner import RetrievalEvaluationRunner
|
|
from extensions.ext_storage import storage
|
|
from extensions.storage.storage_type import StorageType
|
|
from graphon.node_events import NodeRunResult
|
|
from libs.datetime_utils import naive_utc_now
|
|
from models.agent import AgentWorkspaceBinding
|
|
from models.enums import CreatorUserRole
|
|
from models.evaluation import (
|
|
EvaluationRun,
|
|
EvaluationRunItem,
|
|
EvaluationRunItemStatus,
|
|
EvaluationRunStatus,
|
|
EvaluationTargetType,
|
|
)
|
|
from models.model import UploadFile
|
|
from services.agent.evaluation_target_service import AgentEvaluationTargetService
|
|
from services.agent.workspace_service import AgentWorkspaceService
|
|
from services.evaluation_service import EvaluationService
|
|
from tasks.collect_agent_resources_task import enqueue_agent_resource_collection
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_PERSIST_BATCH_SIZE = 10
|
|
_HEARTBEAT_JOIN_TIMEOUT_SECONDS = 5
|
|
|
|
|
|
class EvaluationRunCancelledError(Exception):
|
|
"""Raised when cooperative cancellation stops an evaluation task."""
|
|
|
|
|
|
class EvaluationRunOwnershipLostError(Exception):
|
|
"""Raised when a task invocation no longer owns the evaluation run lease."""
|
|
|
|
|
|
def _affected_rows(result: Result[Any]) -> int:
|
|
"""Require a DML cursor result before relying on its affected-row count."""
|
|
if not isinstance(result, CursorResult):
|
|
raise TypeError("Evaluation lease update did not return a cursor result.")
|
|
return result.rowcount
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class EvaluationRunLease:
|
|
"""Fencing token owned by one Celery invocation of an evaluation run."""
|
|
|
|
run_id: str
|
|
token: str
|
|
|
|
|
|
def _workflow_task_id(evaluation_run_id: str, phase: str, position: int) -> str:
|
|
return EvaluationService.workflow_task_id(evaluation_run_id, phase, position)
|
|
|
|
|
|
def _evaluate_with_customized_workflow(
|
|
evaluation_instance: BaseEvaluationInstance,
|
|
node_run_result_mapping_list: list[dict[str, NodeRunResult]],
|
|
customized_metrics: CustomizedMetrics,
|
|
tenant_id: str,
|
|
cancellation_check: Callable[[], None] | None = None,
|
|
task_id_factory: Callable[[int], str] | None = None,
|
|
) -> list[EvaluationItemResult]:
|
|
"""Run a published evaluator workflow for one tenant and map its outputs to metrics.
|
|
|
|
Precondition: the evaluator App and a published version are configured for
|
|
this tenant. A missing pinned version raises before row execution.
|
|
Postcondition: row failures contain only public diagnostics, results retain
|
|
input positions, and cancellation is checked before and after each row.
|
|
"""
|
|
from core.app.apps.workflow.app_generator import WorkflowAppGenerator
|
|
from core.app.entities.app_invoke_entities import InvokeFrom
|
|
from core.evaluation.runners import get_service_account_for_app
|
|
from models.model import App
|
|
from services.workflow_execution_identity import build_workflow_execution_identity
|
|
from services.workflow_run_agg import WorkflowRunAgg
|
|
from services.workflow_service import WorkflowService
|
|
|
|
workflow_id = customized_metrics.evaluation_workflow_id
|
|
if not workflow_id:
|
|
raise ValueError("customized_metrics must contain 'evaluation_workflow_id' for customized evaluator")
|
|
|
|
with session_factory.create_session() as session:
|
|
app = session.query(App).filter_by(id=workflow_id, tenant_id=tenant_id).first()
|
|
if not app:
|
|
raise ValueError(f"Evaluation workflow app {workflow_id} not found in tenant {tenant_id}")
|
|
service_account = get_service_account_for_app(session, workflow_id, tenant_id)
|
|
workflow_service = WorkflowService()
|
|
if customized_metrics.evaluation_workflow_version_id:
|
|
published_workflow = workflow_service.get_published_workflow_by_id(
|
|
app_model=app,
|
|
workflow_id=customized_metrics.evaluation_workflow_version_id,
|
|
session=session,
|
|
)
|
|
else:
|
|
published_workflow = workflow_service.get_published_workflow(app_model=app, session=session)
|
|
if not published_workflow and customized_metrics.evaluation_workflow_version_id:
|
|
raise EvaluationWorkflowVersionNotFoundError(
|
|
f"Pinned evaluation workflow version {customized_metrics.evaluation_workflow_version_id} no longer exists."
|
|
)
|
|
if not published_workflow:
|
|
raise ValueError(f"No published workflow found for evaluation app {workflow_id}")
|
|
|
|
eval_results: list[EvaluationItemResult] = []
|
|
for idx, node_run_result_mapping in enumerate(node_run_result_mapping_list):
|
|
if cancellation_check is not None:
|
|
cancellation_check()
|
|
try:
|
|
workflow_inputs = evaluation_instance._build_workflow_inputs(
|
|
customized_metrics.input_fields,
|
|
node_run_result_mapping,
|
|
)
|
|
|
|
generator = WorkflowAppGenerator(execution_driver=WorkflowRunAgg.run)
|
|
response: Mapping[str, Any] = generator.generate(
|
|
execution_identity=build_workflow_execution_identity(service_account),
|
|
app_model=app,
|
|
workflow=published_workflow,
|
|
user=service_account,
|
|
args={"inputs": workflow_inputs},
|
|
invoke_from=InvokeFrom.SERVICE_API,
|
|
streaming=False,
|
|
call_depth=0,
|
|
task_id=task_id_factory(idx) if task_id_factory is not None else None,
|
|
)
|
|
|
|
metrics = evaluation_instance._extract_workflow_metrics(
|
|
response,
|
|
workflow_id,
|
|
{field.variable for field in customized_metrics.output_fields},
|
|
)
|
|
if not metrics:
|
|
raise ValueError("Evaluation workflow returned none of the configured metric outputs")
|
|
eval_results.append(EvaluationItemResult(index=idx, metrics=metrics))
|
|
except Exception:
|
|
failure = new_public_evaluation_error(EvaluationErrorCode.CUSTOM_EVALUATOR_FAILED)
|
|
logger.exception(
|
|
"Customized evaluator failed for item %d with workflow %s (error_id=%s)",
|
|
idx,
|
|
workflow_id,
|
|
failure.error_id,
|
|
)
|
|
eval_results.append(EvaluationItemResult(index=idx, error=failure.text))
|
|
if cancellation_check is not None:
|
|
cancellation_check()
|
|
|
|
return eval_results
|
|
|
|
|
|
@shared_task(
|
|
queue="evaluation",
|
|
bind=True,
|
|
acks_late=True,
|
|
reject_on_worker_lost=True,
|
|
max_retries=dify_config.EVALUATION_RUN_MAX_RETRIES,
|
|
)
|
|
def run_evaluation(task: Task, run_data_dict: dict[str, Any]) -> None:
|
|
"""Execute a run with bounded retries and durable row checkpoints.
|
|
|
|
Precondition: the task payload identifies an existing evaluation run.
|
|
Postcondition: dataset-row errors remain terminal item results. Unexpected
|
|
infrastructure failures retry with exponential backoff; terminal run errors
|
|
contain only public diagnostics, while detailed exceptions remain in logs.
|
|
"""
|
|
run_data = EvaluationRunData.model_validate(run_data_dict)
|
|
lease = EvaluationRunLease(run_id=run_data.evaluation_run_id, token=str(uuid.uuid4()))
|
|
with session_factory.create_session() as session:
|
|
try:
|
|
_execute_evaluation(session, run_data, lease)
|
|
except EvaluationRunCancelledError:
|
|
logger.info("Evaluation run %s stopped after cancellation", run_data.evaluation_run_id)
|
|
except EvaluationRunOwnershipLostError:
|
|
logger.info(
|
|
"Evaluation run %s stopped because execution lease %s is no longer current",
|
|
run_data.evaluation_run_id,
|
|
lease.token,
|
|
)
|
|
except EvaluationWorkflowVersionNotFoundError:
|
|
failure = new_public_evaluation_error(EvaluationErrorCode.INVALID_CONFIG)
|
|
logger.exception(
|
|
"Evaluation run %s cannot continue because a pinned workflow version is missing (error_id=%s)",
|
|
run_data.evaluation_run_id,
|
|
failure.error_id,
|
|
)
|
|
_mark_run_failed(session, lease, failure.text)
|
|
except ValueError:
|
|
failure = new_public_evaluation_error(EvaluationErrorCode.INVALID_CONFIG)
|
|
logger.exception(
|
|
"Evaluation run %s failed validation (error_id=%s)", run_data.evaluation_run_id, failure.error_id
|
|
)
|
|
_mark_run_failed(session, lease, failure.text)
|
|
except Exception as exc:
|
|
retry_number = task.request.retries + 1
|
|
max_retries = dify_config.EVALUATION_RUN_MAX_RETRIES
|
|
if task.request.retries >= max_retries:
|
|
failure = new_public_evaluation_error(EvaluationErrorCode.RUN_FAILED)
|
|
logger.exception(
|
|
"Evaluation run %s exhausted its retry budget after %d attempts (error_id=%s)",
|
|
run_data.evaluation_run_id,
|
|
task.request.retries + 1,
|
|
failure.error_id,
|
|
)
|
|
_mark_run_failed(session, lease, failure.text)
|
|
return
|
|
|
|
countdown = dify_config.EVALUATION_RUN_RETRY_DELAY * (2**task.request.retries)
|
|
failure = new_public_evaluation_error(EvaluationErrorCode.RUN_RETRYING)
|
|
logger.warning(
|
|
"Evaluation run %s failed; scheduling retry %d/%d in %d seconds (error_id=%s)",
|
|
run_data.evaluation_run_id,
|
|
retry_number,
|
|
max_retries,
|
|
countdown,
|
|
failure.error_id,
|
|
exc_info=True,
|
|
)
|
|
retry_signal = task.retry(exc=exc, countdown=countdown, throw=False)
|
|
try:
|
|
retry_scheduled = _mark_run_pending_for_retry(
|
|
session,
|
|
lease,
|
|
failure.text,
|
|
retry_number,
|
|
max_retries,
|
|
)
|
|
except Exception:
|
|
logger.exception("Failed to persist retry state for evaluation run %s", run_data.evaluation_run_id)
|
|
retry_scheduled = True
|
|
if not retry_scheduled:
|
|
logger.info("Evaluation run %s became terminal before retry", run_data.evaluation_run_id)
|
|
return
|
|
raise retry_signal from exc
|
|
|
|
|
|
def _renew_evaluation_run_heartbeat(lease: EvaluationRunLease) -> bool:
|
|
"""Renew the lease only while the evaluation remains executable."""
|
|
with session_factory.create_session() as session:
|
|
result = session.execute(
|
|
update(EvaluationRun)
|
|
.where(
|
|
EvaluationRun.id == lease.run_id,
|
|
EvaluationRun.status == EvaluationRunStatus.RUNNING,
|
|
EvaluationRun.execution_token == lease.token,
|
|
)
|
|
.values(heartbeat_at=naive_utc_now())
|
|
)
|
|
session.commit()
|
|
return _affected_rows(result) == 1
|
|
|
|
|
|
def _renew_evaluation_run_heartbeat_while_running(lease: EvaluationRunLease, stop_event: Event) -> None:
|
|
interval = dify_config.EVALUATION_RUN_HEARTBEAT_INTERVAL
|
|
while not stop_event.wait(interval):
|
|
try:
|
|
if not _renew_evaluation_run_heartbeat(lease):
|
|
return
|
|
except Exception:
|
|
logger.exception("Failed to renew heartbeat for evaluation run %s", lease.run_id)
|
|
|
|
|
|
@contextmanager
|
|
def _maintain_evaluation_run_heartbeat(lease: EvaluationRunLease) -> Generator[None]:
|
|
stop_event = Event()
|
|
heartbeat_thread = Thread(
|
|
target=_renew_evaluation_run_heartbeat_while_running,
|
|
args=(lease, stop_event),
|
|
daemon=True,
|
|
name=f"EvaluationRunHeartbeat({lease.run_id})",
|
|
)
|
|
heartbeat_thread.start()
|
|
try:
|
|
yield
|
|
finally:
|
|
stop_event.set()
|
|
heartbeat_thread.join(timeout=_HEARTBEAT_JOIN_TIMEOUT_SECONDS)
|
|
if heartbeat_thread.is_alive():
|
|
logger.warning("Evaluation run %s heartbeat thread did not stop", lease.run_id)
|
|
|
|
|
|
@shared_task(queue="evaluation")
|
|
def reconcile_stale_evaluation_runs() -> int:
|
|
"""Fail running evaluations whose heartbeat lease has expired.
|
|
|
|
The compare-and-set update makes the sweep idempotent and lets a normal
|
|
heartbeat, completion, or cancellation win if it races with the reconciler.
|
|
"""
|
|
cutoff = naive_utc_now() - timedelta(seconds=dify_config.EVALUATION_RUN_LEASE_TIMEOUT)
|
|
stale_runs: list[tuple[str, str, str | None, str, int]] = []
|
|
with session_factory.create_session() as session:
|
|
candidates = list(
|
|
session.scalars(
|
|
select(EvaluationRun)
|
|
.where(
|
|
EvaluationRun.status == EvaluationRunStatus.RUNNING,
|
|
EvaluationRun.heartbeat_at.is_not(None),
|
|
EvaluationRun.heartbeat_at < cutoff,
|
|
)
|
|
.limit(100)
|
|
).all()
|
|
)
|
|
for run in candidates:
|
|
result = session.execute(
|
|
update(EvaluationRun)
|
|
.where(
|
|
EvaluationRun.id == run.id,
|
|
EvaluationRun.status == EvaluationRunStatus.RUNNING,
|
|
EvaluationRun.execution_token == run.execution_token,
|
|
EvaluationRun.heartbeat_at.is_not(None),
|
|
EvaluationRun.heartbeat_at < cutoff,
|
|
)
|
|
.values(
|
|
status=EvaluationRunStatus.FAILED,
|
|
execution_token=None,
|
|
heartbeat_at=None,
|
|
completed_at=naive_utc_now(),
|
|
error="Evaluation run heartbeat lease expired.",
|
|
)
|
|
)
|
|
if _affected_rows(result) == 1:
|
|
stale_runs.append(
|
|
(
|
|
run.tenant_id,
|
|
run.id,
|
|
run.celery_task_id,
|
|
EvaluationTargetType(run.target_type).value,
|
|
run.total_items,
|
|
)
|
|
)
|
|
session.commit()
|
|
|
|
if not stale_runs:
|
|
return 0
|
|
|
|
from celery import current_app as celery_app
|
|
|
|
for tenant_id, run_id, celery_task_id, target_type, total_items in stale_runs:
|
|
if target_type in {EvaluationTargetType.APPS.value, EvaluationTargetType.SNIPPETS.value}:
|
|
EvaluationService._stop_workflow_tasks(run_id, total_items)
|
|
elif target_type == EvaluationTargetType.AGENTS.value:
|
|
EvaluationService._stop_agent_evaluation_tasks(
|
|
tenant_id=tenant_id,
|
|
evaluation_run_id=run_id,
|
|
total_items=total_items,
|
|
)
|
|
if celery_task_id:
|
|
try:
|
|
celery_app.control.revoke(celery_task_id, terminate=False)
|
|
except Exception:
|
|
logger.exception("Failed to revoke stale evaluation task %s", celery_task_id)
|
|
logger.warning("Marked %d stale evaluation runs as failed", len(stale_runs))
|
|
return len(stale_runs)
|
|
|
|
|
|
def _execute_evaluation(session: Session, run_data: EvaluationRunData, lease: EvaluationRunLease) -> None:
|
|
if lease.run_id != run_data.evaluation_run_id:
|
|
raise ValueError("Evaluation run lease does not match the task payload.")
|
|
|
|
evaluation_run = session.scalar(
|
|
select(EvaluationRun).where(EvaluationRun.id == run_data.evaluation_run_id).with_for_update()
|
|
)
|
|
if evaluation_run is None:
|
|
logger.error("EvaluationRun %s not found", run_data.evaluation_run_id)
|
|
return
|
|
if evaluation_run.status in {
|
|
EvaluationRunStatus.COMPLETED,
|
|
EvaluationRunStatus.FAILED,
|
|
EvaluationRunStatus.CANCELLED,
|
|
}:
|
|
session.rollback()
|
|
return
|
|
if evaluation_run.status != EvaluationRunStatus.PENDING or evaluation_run.execution_token is not None:
|
|
logger.info(
|
|
"Skipping duplicate delivery for evaluation run %s; current status=%s owner=%s",
|
|
evaluation_run.id,
|
|
evaluation_run.status,
|
|
evaluation_run.execution_token,
|
|
)
|
|
session.rollback()
|
|
return
|
|
|
|
run_created_by = evaluation_run.created_by
|
|
completed_items, failed_items, checkpointed_items = _get_completed_checkpoint_counts(
|
|
session,
|
|
evaluation_run.id,
|
|
)
|
|
now = naive_utc_now()
|
|
evaluation_run.status = EvaluationRunStatus.RUNNING
|
|
evaluation_run.execution_token = lease.token
|
|
evaluation_run.started_at = evaluation_run.started_at or now
|
|
evaluation_run.heartbeat_at = now
|
|
evaluation_run.completed_at = None
|
|
evaluation_run.completed_items = completed_items
|
|
evaluation_run.failed_items = failed_items
|
|
evaluation_run.progress_rate = max(
|
|
evaluation_run.progress_rate,
|
|
_checkpoint_progress(checkpointed_items, evaluation_run.total_items),
|
|
)
|
|
evaluation_run.error = None
|
|
session.commit()
|
|
|
|
persisted_framework = EvaluationFrameworkEnum(
|
|
evaluation_run.config_snapshot_dict.get("evaluation_framework", EvaluationFrameworkEnum.RAGAS)
|
|
)
|
|
if (
|
|
evaluation_run.tenant_id != run_data.tenant_id
|
|
or EvaluationTargetType(evaluation_run.target_type).value != run_data.target_type
|
|
or evaluation_run.target_id != run_data.target_id
|
|
or evaluation_run.target_workflow_version_id != run_data.target_workflow_version_id
|
|
or evaluation_run.target_agent_config_snapshot_id != run_data.target_agent_config_snapshot_id
|
|
or persisted_framework != run_data.evaluation_framework
|
|
):
|
|
raise ValueError("Evaluation task payload does not match the persisted run.")
|
|
if run_data.target_type == EvaluationTargetType.AGENTS.value and not run_data.target_agent_config_snapshot_id:
|
|
raise ValueError("Evaluation task payload is missing target_agent_config_snapshot_id.")
|
|
if (
|
|
run_data.target_type
|
|
not in {
|
|
EvaluationTargetType.KNOWLEDGE_BASE.value,
|
|
EvaluationTargetType.AGENTS.value,
|
|
}
|
|
and not run_data.target_workflow_version_id
|
|
):
|
|
raise ValueError("Evaluation task payload is missing target_workflow_version_id.")
|
|
|
|
EvaluationService.validate_default_metric_node_types(run_data.default_metrics)
|
|
|
|
evaluation_instance = EvaluationManager.get_evaluation_instance(run_data.evaluation_framework)
|
|
if evaluation_instance is None:
|
|
raise ValueError("Evaluation framework not configured")
|
|
|
|
with _maintain_evaluation_run_heartbeat(lease):
|
|
_execute_running_evaluation(session, run_data, evaluation_instance, run_created_by, lease)
|
|
|
|
|
|
def _execute_running_evaluation(
|
|
session: Session,
|
|
run_data: EvaluationRunData,
|
|
evaluation_instance: BaseEvaluationInstance,
|
|
run_created_by: str,
|
|
lease: EvaluationRunLease,
|
|
) -> None:
|
|
if run_data.target_type == EvaluationTargetType.KNOWLEDGE_BASE.value:
|
|
_resume_retrieval_rows(
|
|
session,
|
|
run_data,
|
|
evaluation_instance,
|
|
run_created_by,
|
|
lease,
|
|
)
|
|
elif run_data.target_type == EvaluationTargetType.AGENTS.value:
|
|
_resume_agent_rows(
|
|
session,
|
|
run_data,
|
|
evaluation_instance,
|
|
run_created_by,
|
|
lease,
|
|
)
|
|
else:
|
|
_resume_workflow_rows(
|
|
session,
|
|
run_data,
|
|
evaluation_instance,
|
|
lease,
|
|
)
|
|
|
|
_assert_execution_lease(session, lease)
|
|
results = _load_completed_checkpoint_results(session, run_data)
|
|
# The cancellation check starts a read transaction. End it before writing
|
|
# the result object to external storage; the final status update below is
|
|
# the database-side compare-and-set that publishes the file atomically.
|
|
session.rollback()
|
|
metrics_summary = _compute_metrics_summary(results, run_data.judgment_config)
|
|
result_xlsx = _generate_result_xlsx(run_data.input_list, results)
|
|
_update_run_progress(session, lease, 0.98)
|
|
session.rollback()
|
|
result_file = _store_result_file(
|
|
run_data.tenant_id,
|
|
run_data.evaluation_run_id,
|
|
result_xlsx,
|
|
session,
|
|
created_by=run_created_by,
|
|
)
|
|
|
|
completed_items = sum(1 for result in results if result.error is None)
|
|
failed_items = len(run_data.input_list) - completed_items
|
|
statement = (
|
|
update(EvaluationRun)
|
|
.where(
|
|
EvaluationRun.id == run_data.evaluation_run_id,
|
|
EvaluationRun.status == EvaluationRunStatus.RUNNING,
|
|
EvaluationRun.execution_token == lease.token,
|
|
)
|
|
.values(
|
|
status=EvaluationRunStatus.COMPLETED,
|
|
execution_token=None,
|
|
heartbeat_at=None,
|
|
completed_at=naive_utc_now(),
|
|
completed_items=completed_items,
|
|
failed_items=failed_items,
|
|
metrics_summary=json.dumps(metrics_summary, ensure_ascii=False),
|
|
result_file_id=result_file.id,
|
|
progress_rate=1.0,
|
|
)
|
|
)
|
|
try:
|
|
result = session.execute(statement)
|
|
except Exception:
|
|
session.rollback()
|
|
_delete_stored_result_file(result_file.key)
|
|
raise
|
|
if _affected_rows(result) != 1:
|
|
session.rollback()
|
|
_delete_stored_result_file(result_file.key)
|
|
_assert_execution_lease(session, lease)
|
|
raise EvaluationRunOwnershipLostError(f"Evaluation run {lease.run_id} completion lease was rejected.")
|
|
session.commit()
|
|
logger.info("Evaluation run %s completed successfully", run_data.evaluation_run_id)
|
|
|
|
|
|
def _resume_retrieval_rows(
|
|
session: Session,
|
|
run_data: EvaluationRunData,
|
|
evaluation_instance: BaseEvaluationInstance,
|
|
run_created_by: str,
|
|
lease: EvaluationRunLease,
|
|
) -> None:
|
|
"""Resume knowledge-base evaluation from completed row checkpoints."""
|
|
completed_indices = _get_completed_checkpoint_indices(session, run_data.evaluation_run_id)
|
|
pending_items = [item for item in run_data.input_list if item.index not in completed_indices]
|
|
if not pending_items:
|
|
return
|
|
|
|
node_run_results, retrieval_errors = EvaluationService.execute_retrieval_test_targets(
|
|
tenant_id=run_data.tenant_id,
|
|
dataset_id=run_data.target_id,
|
|
account_id=run_created_by,
|
|
input_list=pending_items,
|
|
cancellation_check=lambda: _cooperative_checkpoint(session, lease),
|
|
progress_callback=lambda completed, total: _update_run_progress(
|
|
session,
|
|
lease,
|
|
0.05 + 0.45 * (len(completed_indices) + completed) / max(len(run_data.input_list), 1),
|
|
),
|
|
)
|
|
_assert_execution_lease(session, lease)
|
|
for position, item in enumerate(pending_items):
|
|
item_run_data = run_data.model_copy(update={"input_list": [item]})
|
|
_evaluate_retrieval_results(
|
|
session=session,
|
|
run_data=item_run_data,
|
|
evaluation_instance=evaluation_instance,
|
|
node_run_results=[node_run_results[position]],
|
|
retrieval_errors=[retrieval_errors[position]],
|
|
lease=lease,
|
|
report_progress=False,
|
|
)
|
|
|
|
|
|
def _resume_agent_rows(
|
|
session: Session,
|
|
run_data: EvaluationRunData,
|
|
evaluation_instance: BaseEvaluationInstance,
|
|
run_created_by: str,
|
|
lease: EvaluationRunLease,
|
|
) -> None:
|
|
"""Resume each Agent V2 row from its durable backend-run checkpoint."""
|
|
for position, item in enumerate(run_data.input_list):
|
|
checkpoint = session.scalar(
|
|
select(EvaluationRunItem).where(
|
|
EvaluationRunItem.evaluation_run_id == run_data.evaluation_run_id,
|
|
EvaluationRunItem.item_index == item.index,
|
|
)
|
|
)
|
|
if checkpoint is not None and checkpoint.status == EvaluationRunItemStatus.COMPLETED:
|
|
binding_id = checkpoint.agent_workspace_binding_id
|
|
session.rollback()
|
|
if binding_id:
|
|
_retire_agent_binding(run_data.tenant_id, binding_id)
|
|
continue
|
|
|
|
agent_item: AgentEvaluationItem | None = None
|
|
target_error: str | None = None
|
|
if checkpoint is not None and checkpoint.status == EvaluationRunItemStatus.TARGET_COMPLETED:
|
|
target_error = checkpoint.error
|
|
binding_id = checkpoint.agent_workspace_binding_id
|
|
if checkpoint.target_result_json:
|
|
agent_item = AgentEvaluationItem.model_validate_json(checkpoint.target_result_json)
|
|
elif target_error is None:
|
|
target_error = "Checkpointed Agent target result is missing."
|
|
session.rollback()
|
|
if binding_id:
|
|
_retire_agent_binding(run_data.tenant_id, binding_id)
|
|
else:
|
|
session.rollback()
|
|
agent_item, target_error = _execute_agent_target(
|
|
session=session,
|
|
run_data=run_data,
|
|
dataset_item=item,
|
|
account_id=run_created_by,
|
|
lease=lease,
|
|
)
|
|
|
|
_evaluate_agent_row(
|
|
session=session,
|
|
run_data=run_data.model_copy(update={"input_list": [item]}),
|
|
evaluation_instance=evaluation_instance,
|
|
agent_item=agent_item,
|
|
target_error=target_error,
|
|
task_position=position,
|
|
lease=lease,
|
|
)
|
|
_update_run_progress(
|
|
session,
|
|
lease,
|
|
0.05 + 0.9 * (position + 1) / max(len(run_data.input_list), 1),
|
|
)
|
|
|
|
|
|
def _execute_agent_target(
|
|
*,
|
|
session: Session,
|
|
run_data: EvaluationRunData,
|
|
dataset_item: EvaluationDatasetInput,
|
|
account_id: str,
|
|
lease: EvaluationRunLease,
|
|
) -> tuple[AgentEvaluationItem | None, str | None]:
|
|
"""Create or reconnect one isolated Agent V2 backend run.
|
|
|
|
Precondition: the run owns an Agent target snapshot and an active lease.
|
|
Postcondition: completed target checkpoints retain only public errors;
|
|
successful rows return an item, while target failures return a safe error.
|
|
"""
|
|
snapshot_id = run_data.target_agent_config_snapshot_id
|
|
if not snapshot_id:
|
|
raise ValueError("Agent evaluation target snapshot is missing.")
|
|
_cooperative_checkpoint(session, lease)
|
|
target = AgentEvaluationTargetService.resolve_active_snapshot(
|
|
session=session,
|
|
tenant_id=run_data.tenant_id,
|
|
agent_id=run_data.target_id,
|
|
snapshot_id=snapshot_id,
|
|
)
|
|
run_item = session.scalar(
|
|
select(EvaluationRunItem)
|
|
.where(
|
|
EvaluationRunItem.evaluation_run_id == lease.run_id,
|
|
EvaluationRunItem.item_index == dataset_item.index,
|
|
)
|
|
.with_for_update()
|
|
)
|
|
if run_item is None:
|
|
run_item = EvaluationRunItem(
|
|
evaluation_run_id=lease.run_id,
|
|
item_index=dataset_item.index,
|
|
status=EvaluationRunItemStatus.TARGET_RUNNING,
|
|
inputs=json.dumps(dataset_item.inputs, ensure_ascii=False),
|
|
expected_output=dataset_item.serialize_expected_output(),
|
|
)
|
|
session.add(run_item)
|
|
session.flush()
|
|
if run_item.status == EvaluationRunItemStatus.TARGET_COMPLETED:
|
|
persisted_item = (
|
|
AgentEvaluationItem.model_validate_json(run_item.target_result_json)
|
|
if run_item.target_result_json
|
|
else None
|
|
)
|
|
target_error = run_item.error
|
|
session.rollback()
|
|
return persisted_item, target_error
|
|
if run_item.status == EvaluationRunItemStatus.COMPLETED:
|
|
session.rollback()
|
|
return None, "Agent evaluation row was already completed."
|
|
|
|
run_item_id = run_item.id
|
|
checkpoint_binding_id = run_item.agent_workspace_binding_id
|
|
backend_run_id = run_item.agent_backend_run_id
|
|
binding: AgentWorkspaceBinding | None = None
|
|
if checkpoint_binding_id:
|
|
binding = session.scalar(
|
|
select(AgentWorkspaceBinding).where(
|
|
AgentWorkspaceBinding.id == checkpoint_binding_id,
|
|
AgentWorkspaceBinding.tenant_id == run_data.tenant_id,
|
|
)
|
|
)
|
|
if backend_run_id is None:
|
|
binding = AgentEvaluationTargetService.ensure_row_binding(
|
|
session=session,
|
|
target=target,
|
|
tenant_id=run_data.tenant_id,
|
|
run_item_id=run_item_id,
|
|
binding_id=checkpoint_binding_id,
|
|
)
|
|
run_item.agent_workspace_binding_id = binding.id
|
|
run_item.status = EvaluationRunItemStatus.TARGET_RUNNING
|
|
_touch_execution_lease(session, lease)
|
|
session.commit()
|
|
else:
|
|
session.rollback()
|
|
|
|
binding_id = binding.id if binding is not None else checkpoint_binding_id or ""
|
|
backend_binding_ref = binding.backend_binding_ref if binding is not None else "evaluation-replay"
|
|
runtime = AgentEvaluationTargetService.build_runtime_request(
|
|
target=target,
|
|
tenant_id=run_data.tenant_id,
|
|
account_id=account_id,
|
|
run_item_id=run_item_id,
|
|
binding_id=binding_id,
|
|
backend_binding_ref=backend_binding_ref,
|
|
query=str(dataset_item.inputs.get("query", "")),
|
|
idempotency_key=f"evaluation:{run_data.evaluation_run_id}:{run_item_id}",
|
|
)
|
|
available_tools = [
|
|
AgentToolDefinition(
|
|
tool_ref=tool.tool_ref,
|
|
name=tool.name,
|
|
display_name=tool.display_name,
|
|
description=tool.description,
|
|
input_schema=tool.input_schema,
|
|
schema_available=tool.schema_available,
|
|
)
|
|
for tool in runtime.available_tools
|
|
]
|
|
client = create_agent_backend_run_client(
|
|
base_url=dify_config.AGENT_BACKEND_BASE_URL,
|
|
api_token=dify_config.AGENT_BACKEND_API_TOKEN,
|
|
use_fake=dify_config.AGENT_BACKEND_USE_FAKE,
|
|
fake_scenario=dify_config.AGENT_BACKEND_FAKE_SCENARIO,
|
|
stream_read_timeout_seconds=dify_config.AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS,
|
|
stream_max_reconnects=dify_config.AGENT_BACKEND_STREAM_MAX_RECONNECTS,
|
|
)
|
|
if backend_run_id is None:
|
|
create_response = client.create_run(runtime.request)
|
|
backend_run_id = create_response.run_id
|
|
try:
|
|
_persist_agent_backend_run_id(session, lease, dataset_item.index, backend_run_id)
|
|
except Exception:
|
|
try:
|
|
client.cancel_run(backend_run_id)
|
|
except Exception:
|
|
logger.exception("Failed to cancel uncheckpointed Agent evaluation run %s", backend_run_id)
|
|
_retire_agent_binding(run_data.tenant_id, binding_id)
|
|
raise
|
|
|
|
collector = AgentV2TraceCollector(available_tools)
|
|
adapter = AgentBackendRunEventAdapter()
|
|
terminal: (
|
|
AgentBackendRunSucceededInternalEvent
|
|
| AgentBackendRunFailedInternalEvent
|
|
| AgentBackendRunCancelledInternalEvent
|
|
| None
|
|
) = None
|
|
stop_error: EvaluationRunCancelledError | EvaluationRunOwnershipLostError | None = None
|
|
last_lease_check = 0.0
|
|
|
|
def should_stop() -> bool:
|
|
nonlocal last_lease_check, stop_error
|
|
now = time.monotonic()
|
|
if now - last_lease_check < 0.5:
|
|
return False
|
|
last_lease_check = now
|
|
try:
|
|
_cooperative_checkpoint(session, lease)
|
|
except (EvaluationRunCancelledError, EvaluationRunOwnershipLostError) as exc:
|
|
stop_error = exc
|
|
return True
|
|
return False
|
|
|
|
session.rollback()
|
|
for raw_event in client.stream_events(backend_run_id, should_stop=should_stop):
|
|
for event in adapter.adapt(raw_event):
|
|
collector.consume(event)
|
|
if isinstance(
|
|
event,
|
|
AgentBackendRunSucceededInternalEvent
|
|
| AgentBackendRunFailedInternalEvent
|
|
| AgentBackendRunCancelledInternalEvent,
|
|
):
|
|
terminal = event
|
|
break
|
|
if terminal is not None:
|
|
break
|
|
if stop_error is not None:
|
|
raise stop_error
|
|
_cooperative_checkpoint(session, lease)
|
|
if terminal is None:
|
|
raise RuntimeError(f"Agent backend stream for run {backend_run_id} ended without a terminal event.")
|
|
|
|
node_info = _agent_target_node_info(run_data)
|
|
if isinstance(terminal, AgentBackendRunSucceededInternalEvent):
|
|
agent_item = collector.build_item(dataset_item=dataset_item, node_info=node_info, output=terminal.output)
|
|
target_error = None
|
|
elif isinstance(terminal, AgentBackendRunFailedInternalEvent):
|
|
agent_item = None
|
|
failure = new_public_evaluation_error(EvaluationErrorCode.TARGET_FAILED)
|
|
logger.warning(
|
|
"Agent target run %s failed for evaluation row %d (error_id=%s)",
|
|
backend_run_id,
|
|
dataset_item.index,
|
|
failure.error_id,
|
|
)
|
|
target_error = failure.text
|
|
else:
|
|
agent_item = None
|
|
failure = new_public_evaluation_error(EvaluationErrorCode.TARGET_FAILED)
|
|
logger.info(
|
|
"Agent target run %s was cancelled for evaluation row %d (error_id=%s)",
|
|
backend_run_id,
|
|
dataset_item.index,
|
|
failure.error_id,
|
|
)
|
|
target_error = failure.text
|
|
|
|
retired_binding_id = _persist_agent_target_checkpoint(
|
|
session=session,
|
|
lease=lease,
|
|
tenant_id=run_data.tenant_id,
|
|
dataset_item=dataset_item,
|
|
backend_run_id=backend_run_id,
|
|
binding_id=binding_id or None,
|
|
agent_item=agent_item,
|
|
target_error=target_error,
|
|
)
|
|
if retired_binding_id is not None:
|
|
_enqueue_retired_agent_binding(run_data.tenant_id, retired_binding_id)
|
|
return agent_item, target_error
|
|
|
|
|
|
def _persist_agent_backend_run_id(
|
|
session: Session,
|
|
lease: EvaluationRunLease,
|
|
item_index: int,
|
|
backend_run_id: str,
|
|
) -> None:
|
|
_assert_execution_lease(session, lease)
|
|
run_item = session.scalar(
|
|
select(EvaluationRunItem).where(
|
|
EvaluationRunItem.evaluation_run_id == lease.run_id,
|
|
EvaluationRunItem.item_index == item_index,
|
|
)
|
|
)
|
|
if run_item is None:
|
|
raise RuntimeError("Agent evaluation row checkpoint is missing.")
|
|
run_item.agent_backend_run_id = backend_run_id
|
|
run_item.status = EvaluationRunItemStatus.TARGET_RUNNING
|
|
_touch_execution_lease(session, lease)
|
|
session.commit()
|
|
|
|
|
|
def _persist_agent_target_checkpoint(
|
|
*,
|
|
session: Session,
|
|
lease: EvaluationRunLease,
|
|
tenant_id: str,
|
|
dataset_item: EvaluationDatasetInput,
|
|
backend_run_id: str,
|
|
binding_id: str | None,
|
|
agent_item: AgentEvaluationItem | None,
|
|
target_error: str | None,
|
|
) -> str | None:
|
|
"""Checkpoint an Agent target without persisting raw backend error text.
|
|
|
|
Precondition: the lease still owns the row and the backend run is known.
|
|
Postcondition: the target checkpoint is durable with a public error or none.
|
|
"""
|
|
_assert_execution_lease(session, lease)
|
|
run_item = session.scalar(
|
|
select(EvaluationRunItem).where(
|
|
EvaluationRunItem.evaluation_run_id == lease.run_id,
|
|
EvaluationRunItem.item_index == dataset_item.index,
|
|
)
|
|
)
|
|
if run_item is None:
|
|
raise RuntimeError("Agent evaluation row checkpoint is missing.")
|
|
run_item.status = EvaluationRunItemStatus.TARGET_COMPLETED
|
|
run_item.agent_backend_run_id = backend_run_id
|
|
run_item.agent_workspace_binding_id = binding_id
|
|
run_item.inputs = json.dumps(dataset_item.inputs, ensure_ascii=False)
|
|
run_item.expected_output = dataset_item.serialize_expected_output()
|
|
run_item.actual_output = agent_item.output if agent_item is not None else None
|
|
run_item.target_result_json = agent_item.model_dump_json() if agent_item is not None else None
|
|
run_item.error = safe_evaluation_error_text(target_error)
|
|
retired_binding_id = (
|
|
AgentWorkspaceService.retire_binding(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
binding_id=binding_id,
|
|
)
|
|
if binding_id
|
|
else None
|
|
)
|
|
_touch_execution_lease(session, lease)
|
|
session.commit()
|
|
return retired_binding_id
|
|
|
|
|
|
def _enqueue_retired_agent_binding(tenant_id: str, binding_id: str) -> None:
|
|
try:
|
|
enqueue_agent_resource_collection(tenant_id=tenant_id, binding_ids=(binding_id,))
|
|
except Exception:
|
|
logger.exception("Failed to enqueue Agent evaluation binding %s for collection", binding_id)
|
|
|
|
|
|
def _retire_agent_binding(tenant_id: str, binding_id: str) -> None:
|
|
try:
|
|
with session_factory.create_session() as cleanup_session:
|
|
retired_binding_id = AgentWorkspaceService.retire_binding(
|
|
session=cleanup_session,
|
|
tenant_id=tenant_id,
|
|
binding_id=binding_id,
|
|
)
|
|
cleanup_session.commit()
|
|
if retired_binding_id is not None:
|
|
_enqueue_retired_agent_binding(tenant_id, retired_binding_id)
|
|
except Exception:
|
|
logger.exception("Failed to retire Agent evaluation binding %s", binding_id)
|
|
|
|
|
|
def _agent_target_node_info(run_data: EvaluationRunData) -> NodeInfo:
|
|
for metric in run_data.default_metrics:
|
|
if metric.node_info_list:
|
|
return metric.node_info_list[0]
|
|
return NodeInfo(node_id=run_data.target_id, type="agent", title="Agent")
|
|
|
|
|
|
def _evaluate_agent_row(
|
|
*,
|
|
session: Session,
|
|
run_data: EvaluationRunData,
|
|
evaluation_instance: BaseEvaluationInstance,
|
|
agent_item: AgentEvaluationItem | None,
|
|
target_error: str | None,
|
|
task_position: int,
|
|
lease: EvaluationRunLease,
|
|
) -> None:
|
|
"""Score one Agent row and persist a result with public diagnostics.
|
|
|
|
Precondition: run data contains exactly one dataset item under the lease.
|
|
Postcondition: all failures remain row-local and no raw exception is saved.
|
|
"""
|
|
dataset_item = run_data.input_list[0]
|
|
node_info = _agent_target_node_info(run_data)
|
|
if target_error is not None or agent_item is None:
|
|
result = EvaluationItemResult(
|
|
index=dataset_item.index,
|
|
actual_output=agent_item.output if agent_item is not None else None,
|
|
error=safe_evaluation_error_text(target_error)
|
|
or new_public_evaluation_error(EvaluationErrorCode.TARGET_FAILED).text,
|
|
)
|
|
results_by_index = {dataset_item.index: result}
|
|
else:
|
|
results_by_index = {
|
|
dataset_item.index: EvaluationItemResult(
|
|
index=dataset_item.index,
|
|
actual_output=agent_item.output,
|
|
metadata={
|
|
"agent_trace_source": "agent_v2",
|
|
"agent_trace_warnings": agent_item.trajectory.warnings if agent_item.trajectory else [],
|
|
},
|
|
)
|
|
}
|
|
if target_error is None and agent_item is not None and run_data.default_metrics:
|
|
try:
|
|
evaluated = evaluation_instance.evaluate_agent(
|
|
[agent_item],
|
|
[metric.metric for metric in run_data.default_metrics],
|
|
run_data.evaluation_model_provider,
|
|
run_data.evaluation_model,
|
|
run_data.tenant_id,
|
|
)
|
|
if not evaluated:
|
|
raise RuntimeError("Agent evaluator returned no result.")
|
|
result = evaluated[0].model_copy(
|
|
update={
|
|
"index": dataset_item.index,
|
|
"actual_output": agent_item.output,
|
|
"metadata": {
|
|
**evaluated[0].metadata,
|
|
"agent_trace_source": "agent_v2",
|
|
"agent_trace_warnings": agent_item.trajectory.warnings if agent_item.trajectory else [],
|
|
},
|
|
}
|
|
)
|
|
for metric in result.metrics:
|
|
metric.node_info = node_info
|
|
_merge_result(results_by_index, dataset_item.index, result)
|
|
except Exception:
|
|
failure = new_public_evaluation_error(EvaluationErrorCode.METRIC_FAILED)
|
|
logger.exception(
|
|
"Failed Agent V2 metrics for evaluation row %s (error_id=%s)",
|
|
dataset_item.index,
|
|
failure.error_id,
|
|
)
|
|
_merge_result(
|
|
results_by_index,
|
|
dataset_item.index,
|
|
EvaluationItemResult(
|
|
index=dataset_item.index,
|
|
actual_output=agent_item.output,
|
|
error=failure.text,
|
|
),
|
|
)
|
|
|
|
if target_error is None and agent_item is not None and run_data.customized_metrics is not None:
|
|
_clear_workflow_cancellation_signals(session, lease, "evaluator", [task_position])
|
|
try:
|
|
customized_results = _evaluate_with_customized_workflow(
|
|
evaluation_instance=evaluation_instance,
|
|
node_run_result_mapping_list=[
|
|
{AGENT_CUSTOM_EVALUATOR_NODE_ID: build_agent_custom_evaluator_node_result(agent_item)}
|
|
],
|
|
customized_metrics=run_data.customized_metrics,
|
|
tenant_id=run_data.tenant_id,
|
|
cancellation_check=lambda: _cooperative_checkpoint(session, lease),
|
|
task_id_factory=lambda _: _workflow_task_id(
|
|
run_data.evaluation_run_id,
|
|
"evaluator",
|
|
task_position,
|
|
),
|
|
)
|
|
_merge_customized_results(results_by_index, customized_results, [dataset_item])
|
|
except EvaluationWorkflowVersionNotFoundError:
|
|
raise
|
|
except Exception:
|
|
failure = new_public_evaluation_error(EvaluationErrorCode.CUSTOM_EVALUATOR_FAILED)
|
|
logger.exception(
|
|
"Failed customized Agent metrics for evaluation row %s (error_id=%s)",
|
|
dataset_item.index,
|
|
failure.error_id,
|
|
)
|
|
_merge_result(
|
|
results_by_index,
|
|
dataset_item.index,
|
|
EvaluationItemResult(index=dataset_item.index, error=failure.text),
|
|
)
|
|
|
|
results = [results_by_index[dataset_item.index]]
|
|
if run_data.judgment_config:
|
|
results = _apply_judgment(results, run_data.judgment_config)
|
|
results = [_with_overall_score(item) for item in results]
|
|
_persist_results(session, lease, results, [dataset_item])
|
|
|
|
|
|
def _resume_workflow_rows(
|
|
session: Session,
|
|
run_data: EvaluationRunData,
|
|
evaluation_instance: BaseEvaluationInstance,
|
|
lease: EvaluationRunLease,
|
|
) -> None:
|
|
"""Resume target execution and scoring without replaying durable row checkpoints."""
|
|
checkpoint_by_index = {
|
|
checkpoint.item_index: checkpoint
|
|
for checkpoint in session.scalars(
|
|
select(EvaluationRunItem).where(EvaluationRunItem.evaluation_run_id == run_data.evaluation_run_id)
|
|
).all()
|
|
}
|
|
session.rollback()
|
|
|
|
evaluation_inputs: dict[int, tuple[dict[str, NodeRunResult], str | None, str | None]] = {}
|
|
pending_items: list[EvaluationDatasetInput] = []
|
|
pending_positions: list[int] = []
|
|
target_ready_count = 0
|
|
|
|
for position, item in enumerate(run_data.input_list):
|
|
checkpoint = checkpoint_by_index.get(item.index)
|
|
if checkpoint is not None and checkpoint.status == EvaluationRunItemStatus.COMPLETED:
|
|
target_ready_count += 1
|
|
continue
|
|
if checkpoint is not None and checkpoint.status == EvaluationRunItemStatus.TARGET_COMPLETED:
|
|
target_ready_count += 1
|
|
if checkpoint.workflow_run_id:
|
|
checkpoint_node_results = EvaluationService._query_node_run_results(
|
|
session=session,
|
|
tenant_id=run_data.tenant_id,
|
|
app_id=run_data.target_id,
|
|
workflow_run_id=checkpoint.workflow_run_id,
|
|
)
|
|
session.rollback()
|
|
target_error = checkpoint.error
|
|
if not checkpoint_node_results and target_error is None:
|
|
target_error = "Checkpointed target workflow results are no longer available."
|
|
evaluation_inputs[position] = (
|
|
checkpoint_node_results,
|
|
checkpoint.workflow_run_id,
|
|
target_error,
|
|
)
|
|
else:
|
|
evaluation_inputs[position] = ({}, None, "Checkpointed target workflow run ID is missing.")
|
|
continue
|
|
pending_items.append(item)
|
|
pending_positions.append(position)
|
|
|
|
if pending_items:
|
|
_clear_workflow_cancellation_signals(
|
|
session,
|
|
lease,
|
|
"target",
|
|
pending_positions,
|
|
)
|
|
|
|
def _checkpoint_target_result(
|
|
local_position: int,
|
|
node_results: dict[str, NodeRunResult],
|
|
workflow_run_id: str | None,
|
|
error: str | None,
|
|
) -> None:
|
|
if workflow_run_id:
|
|
_persist_target_checkpoint(
|
|
session,
|
|
lease,
|
|
pending_items[local_position],
|
|
workflow_run_id,
|
|
error,
|
|
)
|
|
|
|
node_results_list, workflow_run_ids, target_errors = EvaluationService.execute_targets(
|
|
evaluation_run_id=run_data.evaluation_run_id,
|
|
tenant_id=run_data.tenant_id,
|
|
target_type=run_data.target_type,
|
|
target_id=run_data.target_id,
|
|
target_workflow_version_id=run_data.target_workflow_version_id or "",
|
|
input_list=pending_items,
|
|
cancellation_check=lambda: _cooperative_checkpoint(session, lease),
|
|
progress_callback=lambda completed, total: _update_run_progress(
|
|
session,
|
|
lease,
|
|
0.05 + 0.45 * (target_ready_count + completed) / max(len(run_data.input_list), 1),
|
|
),
|
|
result_callback=_checkpoint_target_result,
|
|
task_positions=pending_positions,
|
|
)
|
|
for local_position, original_position in enumerate(pending_positions):
|
|
evaluation_inputs[original_position] = (
|
|
node_results_list[local_position],
|
|
workflow_run_ids[local_position],
|
|
target_errors[local_position],
|
|
)
|
|
|
|
for position, item in enumerate(run_data.input_list):
|
|
checkpoint = checkpoint_by_index.get(item.index)
|
|
if checkpoint is not None and checkpoint.status == EvaluationRunItemStatus.COMPLETED:
|
|
continue
|
|
_cooperative_checkpoint(session, lease)
|
|
row_node_results, workflow_run_id, target_error = evaluation_inputs[position]
|
|
item_run_data = run_data.model_copy(update={"input_list": [item]})
|
|
if item_run_data.customized_metrics is not None:
|
|
_clear_workflow_cancellation_signals(
|
|
session,
|
|
lease,
|
|
"evaluator",
|
|
[position],
|
|
)
|
|
_execute_evaluation_runner(
|
|
session=session,
|
|
run_data=item_run_data,
|
|
evaluation_instance=evaluation_instance,
|
|
node_run_result_mapping_list=[row_node_results],
|
|
workflow_run_id_map={item.index: workflow_run_id} if workflow_run_id else None,
|
|
target_errors=[target_error],
|
|
task_positions=[position],
|
|
lease=lease,
|
|
report_progress=False,
|
|
)
|
|
|
|
|
|
def _persist_target_checkpoint(
|
|
session: Session,
|
|
lease: EvaluationRunLease,
|
|
item_input: EvaluationDatasetInput,
|
|
workflow_run_id: str,
|
|
target_error: str | None = None,
|
|
) -> None:
|
|
"""Persist target state without exposing raw workflow errors.
|
|
|
|
Precondition: the run is owned by ``lease`` and has a workflow run ID.
|
|
Postcondition: a completed target checkpoint can be resumed, and its error
|
|
is either absent or approved public text.
|
|
"""
|
|
_assert_execution_lease(session, lease)
|
|
run_item = session.scalar(
|
|
select(EvaluationRunItem).where(
|
|
EvaluationRunItem.evaluation_run_id == lease.run_id,
|
|
EvaluationRunItem.item_index == item_input.index,
|
|
)
|
|
)
|
|
if run_item is not None and run_item.status == EvaluationRunItemStatus.COMPLETED:
|
|
session.rollback()
|
|
return
|
|
if run_item is None:
|
|
run_item = EvaluationRunItem(
|
|
evaluation_run_id=lease.run_id,
|
|
item_index=item_input.index,
|
|
)
|
|
session.add(run_item)
|
|
run_item.status = EvaluationRunItemStatus.TARGET_COMPLETED
|
|
run_item.workflow_run_id = workflow_run_id
|
|
run_item.inputs = json.dumps(item_input.inputs, ensure_ascii=False)
|
|
run_item.expected_output = item_input.serialize_expected_output()
|
|
run_item.error = safe_evaluation_error_text(target_error)
|
|
_touch_execution_lease(session, lease)
|
|
session.commit()
|
|
|
|
|
|
def _clear_workflow_cancellation_signals(
|
|
session: Session,
|
|
lease: EvaluationRunLease,
|
|
phase: str,
|
|
positions: Sequence[int],
|
|
) -> None:
|
|
"""Clear signals from an earlier attempt without erasing a concurrent user cancellation."""
|
|
from core.app.apps.execution_coordinator import clear_app_task_cancellation_signals
|
|
|
|
_cooperative_checkpoint(session, lease)
|
|
for position in positions:
|
|
clear_app_task_cancellation_signals(_workflow_task_id(lease.run_id, phase, position))
|
|
_cooperative_checkpoint(session, lease)
|
|
|
|
|
|
def _get_completed_checkpoint_indices(session: Session, evaluation_run_id: str) -> set[int]:
|
|
return set(
|
|
session.scalars(
|
|
select(EvaluationRunItem.item_index).where(
|
|
EvaluationRunItem.evaluation_run_id == evaluation_run_id,
|
|
EvaluationRunItem.status == EvaluationRunItemStatus.COMPLETED,
|
|
)
|
|
).all()
|
|
)
|
|
|
|
|
|
def _get_completed_checkpoint_counts(session: Session, evaluation_run_id: str) -> tuple[int, int, int]:
|
|
rows = session.execute(
|
|
select(EvaluationRunItem.status, EvaluationRunItem.error).where(
|
|
EvaluationRunItem.evaluation_run_id == evaluation_run_id,
|
|
)
|
|
).all()
|
|
completed = sum(1 for status, error in rows if status == EvaluationRunItemStatus.COMPLETED and error is None)
|
|
failed = sum(1 for status, error in rows if status == EvaluationRunItemStatus.COMPLETED and error is not None)
|
|
return completed, failed, completed + failed
|
|
|
|
|
|
def _checkpoint_progress(checkpointed_items: int, total_items: int) -> float:
|
|
if checkpointed_items <= 0 or total_items <= 0:
|
|
return 0.01
|
|
return min(0.95, 0.05 + 0.9 * checkpointed_items / total_items)
|
|
|
|
|
|
def _load_completed_checkpoint_results(
|
|
session: Session,
|
|
run_data: EvaluationRunData,
|
|
) -> list[EvaluationItemResult]:
|
|
checkpoints = list(
|
|
session.scalars(
|
|
select(EvaluationRunItem).where(
|
|
EvaluationRunItem.evaluation_run_id == run_data.evaluation_run_id,
|
|
EvaluationRunItem.status == EvaluationRunItemStatus.COMPLETED,
|
|
)
|
|
).all()
|
|
)
|
|
checkpoint_by_index = {checkpoint.item_index: checkpoint for checkpoint in checkpoints}
|
|
missing_indices = [item.index for item in run_data.input_list if item.index not in checkpoint_by_index]
|
|
if missing_indices:
|
|
raise RuntimeError(f"Evaluation row checkpoints are incomplete: {missing_indices[:10]}")
|
|
|
|
return [_checkpoint_to_result(checkpoint_by_index[item.index]) for item in run_data.input_list]
|
|
|
|
|
|
def _checkpoint_to_result(checkpoint: EvaluationRunItem) -> EvaluationItemResult:
|
|
"""Rebuild a result from a completed checkpoint, masking legacy errors.
|
|
|
|
Precondition: ``checkpoint`` belongs to the requested evaluation run.
|
|
Postcondition: returned error text is safe for subsequent XLSX generation.
|
|
"""
|
|
return EvaluationItemResult(
|
|
index=checkpoint.item_index,
|
|
actual_output=checkpoint.actual_output,
|
|
metrics=[EvaluationMetric.model_validate(metric) for metric in checkpoint.metrics_list],
|
|
metadata=checkpoint.metadata_dict,
|
|
judgment=JudgmentResult.model_validate(checkpoint.judgment_dict),
|
|
overall_score=checkpoint.overall_score,
|
|
error=safe_evaluation_error_text(checkpoint.error),
|
|
)
|
|
|
|
|
|
def _execute_evaluation_runner(
|
|
session: Session,
|
|
run_data: EvaluationRunData,
|
|
evaluation_instance: BaseEvaluationInstance,
|
|
node_run_result_mapping_list: list[dict[str, NodeRunResult]],
|
|
lease: EvaluationRunLease,
|
|
workflow_run_id_map: dict[int, str] | None = None,
|
|
target_errors: Sequence[str | None] | None = None,
|
|
task_positions: Sequence[int] | None = None,
|
|
report_progress: bool = True,
|
|
) -> list[EvaluationItemResult]:
|
|
"""Evaluate configured node/metric pairs and persist one result per row.
|
|
|
|
Precondition: input rows align with target node results and task positions.
|
|
Postcondition: row-level failures carry public diagnostics; their original
|
|
exceptions remain in server logs and do not enter persisted results.
|
|
"""
|
|
results_by_index: dict[int, EvaluationItemResult] = {}
|
|
resolved_task_positions = (
|
|
list(task_positions) if task_positions is not None else list(range(len(run_data.input_list)))
|
|
)
|
|
if len(resolved_task_positions) != len(run_data.input_list):
|
|
raise ValueError("task_positions must align with input_list.")
|
|
|
|
total_steps = sum(len(metric.node_info_list) for metric in run_data.default_metrics) + bool(
|
|
run_data.customized_metrics
|
|
)
|
|
completed_steps = 0
|
|
for default_metric in run_data.default_metrics:
|
|
for node_info in default_metric.node_info_list:
|
|
_cooperative_checkpoint(session, lease)
|
|
node_run_results: list[NodeRunResult] = []
|
|
dataset_items: list[EvaluationDatasetInput] = []
|
|
dataset_indices: list[int] = []
|
|
context_list: list[list[str] | None] = []
|
|
for position, mapping in enumerate(node_run_result_mapping_list):
|
|
node_result = mapping.get(node_info.node_id)
|
|
if node_result is None:
|
|
continue
|
|
node_run_results.append(node_result)
|
|
dataset_items.append(run_data.input_list[position])
|
|
dataset_indices.append(run_data.input_list[position].index)
|
|
context_list.append(
|
|
_collect_retrieval_contexts(
|
|
mapping,
|
|
run_data.llm_context_node_mapping.get(node_info.node_id, []),
|
|
)
|
|
)
|
|
|
|
if not node_run_results:
|
|
continue
|
|
|
|
try:
|
|
runner = _create_runner(EvaluationService.category_for_node_type(node_info.type), evaluation_instance)
|
|
evaluated = runner.evaluate_metrics(
|
|
node_run_result_list=node_run_results,
|
|
default_metric=default_metric,
|
|
model_provider=run_data.evaluation_model_provider,
|
|
model_name=run_data.evaluation_model,
|
|
tenant_id=run_data.tenant_id,
|
|
dataset_items=dataset_items,
|
|
node_info=node_info,
|
|
context_list=context_list,
|
|
)
|
|
_stamp_and_merge(evaluated, dataset_indices, node_info, results_by_index)
|
|
except Exception:
|
|
failure = new_public_evaluation_error(EvaluationErrorCode.METRIC_FAILED)
|
|
logger.exception(
|
|
"Failed metric %s on node %s (error_id=%s)",
|
|
default_metric.metric,
|
|
node_info.node_id,
|
|
failure.error_id,
|
|
)
|
|
for dataset_index in dataset_indices:
|
|
_merge_result(
|
|
results_by_index,
|
|
dataset_index,
|
|
EvaluationItemResult(
|
|
index=dataset_index,
|
|
error=failure.text,
|
|
),
|
|
)
|
|
completed_steps += 1
|
|
if report_progress:
|
|
_update_run_progress(
|
|
session,
|
|
lease,
|
|
0.6 + 0.25 * completed_steps / max(total_steps, 1),
|
|
)
|
|
|
|
if run_data.customized_metrics:
|
|
_cooperative_checkpoint(session, lease)
|
|
successful_positions = [
|
|
position for position, node_results in enumerate(node_run_result_mapping_list) if node_results
|
|
]
|
|
customized_node_results = [node_run_result_mapping_list[position] for position in successful_positions]
|
|
customized_input_items = [run_data.input_list[position] for position in successful_positions]
|
|
try:
|
|
if customized_node_results:
|
|
customized_results = _evaluate_with_customized_workflow(
|
|
evaluation_instance=evaluation_instance,
|
|
node_run_result_mapping_list=customized_node_results,
|
|
customized_metrics=run_data.customized_metrics,
|
|
tenant_id=run_data.tenant_id,
|
|
cancellation_check=lambda: _cooperative_checkpoint(session, lease),
|
|
task_id_factory=lambda position: _workflow_task_id(
|
|
run_data.evaluation_run_id,
|
|
"evaluator",
|
|
resolved_task_positions[successful_positions[position]],
|
|
),
|
|
)
|
|
_merge_customized_results(results_by_index, customized_results, customized_input_items)
|
|
except EvaluationWorkflowVersionNotFoundError:
|
|
raise
|
|
except Exception:
|
|
failure = new_public_evaluation_error(EvaluationErrorCode.CUSTOM_EVALUATOR_FAILED)
|
|
logger.exception(
|
|
"Failed customized metrics for run %s (error_id=%s)",
|
|
run_data.evaluation_run_id,
|
|
failure.error_id,
|
|
)
|
|
for item in customized_input_items:
|
|
_merge_result(
|
|
results_by_index,
|
|
item.index,
|
|
EvaluationItemResult(index=item.index, error=failure.text),
|
|
)
|
|
completed_steps += 1
|
|
if report_progress:
|
|
_update_run_progress(
|
|
session,
|
|
lease,
|
|
0.6 + 0.25 * completed_steps / max(total_steps, 1),
|
|
)
|
|
|
|
results = _finalize_results(
|
|
input_list=run_data.input_list,
|
|
results_by_index=results_by_index,
|
|
missing_errors=_build_missing_result_errors(
|
|
run_data.input_list,
|
|
node_run_result_mapping_list,
|
|
target_errors,
|
|
),
|
|
)
|
|
if run_data.judgment_config:
|
|
results = _apply_judgment(results, run_data.judgment_config)
|
|
results = [_with_overall_score(result) for result in results]
|
|
_persist_results(
|
|
session,
|
|
lease,
|
|
results,
|
|
run_data.input_list,
|
|
workflow_run_id_map,
|
|
)
|
|
return results
|
|
|
|
|
|
def _execute_retrieval_test(
|
|
session: Session,
|
|
account_id: str,
|
|
run_data: EvaluationRunData,
|
|
evaluation_instance: BaseEvaluationInstance,
|
|
lease: EvaluationRunLease,
|
|
report_progress: bool = True,
|
|
) -> list[EvaluationItemResult]:
|
|
node_run_results, retrieval_errors = EvaluationService.execute_retrieval_test_targets(
|
|
tenant_id=run_data.tenant_id,
|
|
dataset_id=run_data.target_id,
|
|
account_id=account_id,
|
|
input_list=run_data.input_list,
|
|
cancellation_check=lambda: _cooperative_checkpoint(session, lease),
|
|
progress_callback=(
|
|
lambda completed, total: _update_run_progress(
|
|
session,
|
|
lease,
|
|
0.05 + 0.45 * completed / max(total, 1),
|
|
)
|
|
)
|
|
if report_progress
|
|
else None,
|
|
)
|
|
_assert_execution_lease(session, lease)
|
|
|
|
return _evaluate_retrieval_results(
|
|
session=session,
|
|
run_data=run_data,
|
|
evaluation_instance=evaluation_instance,
|
|
node_run_results=node_run_results,
|
|
retrieval_errors=retrieval_errors,
|
|
lease=lease,
|
|
report_progress=report_progress,
|
|
)
|
|
|
|
|
|
def _evaluate_retrieval_results(
|
|
session: Session,
|
|
run_data: EvaluationRunData,
|
|
evaluation_instance: BaseEvaluationInstance,
|
|
node_run_results: Sequence[NodeRunResult | None],
|
|
retrieval_errors: Sequence[str | None],
|
|
lease: EvaluationRunLease,
|
|
report_progress: bool = True,
|
|
) -> list[EvaluationItemResult]:
|
|
"""Score retrieval outputs and checkpoint each supplied dataset row.
|
|
|
|
Precondition: retrieval results and errors align with input rows.
|
|
Postcondition: every row is saved with metrics or public failure text.
|
|
"""
|
|
|
|
results_by_index: dict[int, EvaluationItemResult] = {}
|
|
runner = RetrievalEvaluationRunner(evaluation_instance)
|
|
node_info = NodeInfo(
|
|
node_id=run_data.target_id,
|
|
type="knowledge-retrieval",
|
|
title="Knowledge Retrieval",
|
|
)
|
|
successful_positions = [position for position, result in enumerate(node_run_results) if result is not None]
|
|
successful_results = [node_run_results[position] for position in successful_positions]
|
|
typed_successful_results = [result for result in successful_results if result is not None]
|
|
successful_items = [run_data.input_list[position] for position in successful_positions]
|
|
dataset_indices = [item.index for item in successful_items]
|
|
for metric_position, default_metric in enumerate(run_data.default_metrics, start=1):
|
|
_cooperative_checkpoint(session, lease)
|
|
try:
|
|
evaluated = runner.evaluate_metrics(
|
|
node_run_result_list=typed_successful_results,
|
|
default_metric=default_metric,
|
|
model_provider=run_data.evaluation_model_provider,
|
|
model_name=run_data.evaluation_model,
|
|
tenant_id=run_data.tenant_id,
|
|
dataset_items=successful_items,
|
|
node_info=node_info,
|
|
)
|
|
_stamp_and_merge(evaluated, dataset_indices, node_info, results_by_index)
|
|
except Exception:
|
|
failure = new_public_evaluation_error(EvaluationErrorCode.METRIC_FAILED)
|
|
logger.exception(
|
|
"Failed retrieval metric %s for run %s (error_id=%s)",
|
|
default_metric.metric,
|
|
run_data.evaluation_run_id,
|
|
failure.error_id,
|
|
)
|
|
for dataset_index in dataset_indices:
|
|
_merge_result(
|
|
results_by_index,
|
|
dataset_index,
|
|
EvaluationItemResult(
|
|
index=dataset_index,
|
|
error=failure.text,
|
|
),
|
|
)
|
|
if report_progress:
|
|
_update_run_progress(
|
|
session,
|
|
lease,
|
|
0.5 + 0.35 * metric_position / max(len(run_data.default_metrics), 1),
|
|
)
|
|
|
|
results = _finalize_results(
|
|
input_list=run_data.input_list,
|
|
results_by_index=results_by_index,
|
|
missing_errors={
|
|
item.index: safe_evaluation_error_text(
|
|
retrieval_errors[position] or new_public_evaluation_error(EvaluationErrorCode.METRIC_FAILED).text
|
|
)
|
|
for position, item in enumerate(run_data.input_list)
|
|
},
|
|
)
|
|
if run_data.judgment_config:
|
|
results = _apply_judgment(results, run_data.judgment_config)
|
|
results = [_with_overall_score(result) for result in results]
|
|
_persist_results(session, lease, results, run_data.input_list)
|
|
return results
|
|
|
|
|
|
def _stamp_and_merge(
|
|
evaluated: Sequence[EvaluationItemResult],
|
|
dataset_indices: Sequence[int],
|
|
node_info: NodeInfo | None,
|
|
results_by_index: dict[int, EvaluationItemResult],
|
|
) -> None:
|
|
for result in evaluated:
|
|
if result.index < 0 or result.index >= len(dataset_indices):
|
|
logger.warning("Evaluator returned out-of-range result index %s", result.index)
|
|
continue
|
|
dataset_index = dataset_indices[result.index]
|
|
if node_info is not None:
|
|
for metric in result.metrics:
|
|
metric.node_info = node_info
|
|
_merge_result(results_by_index, dataset_index, result)
|
|
|
|
|
|
def _collect_retrieval_contexts(
|
|
node_results: dict[str, NodeRunResult],
|
|
retrieval_node_ids: Sequence[str],
|
|
) -> list[str] | None:
|
|
if not retrieval_node_ids:
|
|
return None
|
|
contexts: list[str] = []
|
|
for node_id in retrieval_node_ids:
|
|
node_result = node_results.get(node_id)
|
|
if node_result is None:
|
|
continue
|
|
raw_results = node_result.outputs.get("result", [])
|
|
if not isinstance(raw_results, list):
|
|
continue
|
|
for result in raw_results:
|
|
if isinstance(result, Mapping) and result.get("content"):
|
|
contexts.append(str(result["content"]))
|
|
return contexts
|
|
|
|
|
|
def _merge_result(
|
|
results_by_index: dict[int, EvaluationItemResult],
|
|
index: int,
|
|
new_result: EvaluationItemResult,
|
|
) -> None:
|
|
existing = results_by_index.get(index)
|
|
if existing is None:
|
|
results_by_index[index] = new_result.model_copy(update={"index": index})
|
|
return
|
|
|
|
errors = [error for error in (existing.error, new_result.error) if error]
|
|
results_by_index[index] = existing.model_copy(
|
|
update={
|
|
"metrics": [*existing.metrics, *new_result.metrics],
|
|
"actual_output": existing.actual_output or new_result.actual_output,
|
|
"metadata": {**existing.metadata, **new_result.metadata},
|
|
"error": "; ".join(dict.fromkeys(errors)) or None,
|
|
}
|
|
)
|
|
|
|
|
|
def _merge_customized_results(
|
|
results_by_index: dict[int, EvaluationItemResult],
|
|
customized_results: Sequence[EvaluationItemResult],
|
|
input_list: Sequence[EvaluationDatasetInput],
|
|
) -> None:
|
|
for result in customized_results:
|
|
dataset_index = input_list[result.index].index if 0 <= result.index < len(input_list) else result.index
|
|
_merge_result(results_by_index, dataset_index, result)
|
|
|
|
|
|
def _build_missing_result_errors(
|
|
input_list: Sequence[EvaluationDatasetInput],
|
|
node_run_result_mapping_list: Sequence[dict[str, NodeRunResult]],
|
|
target_errors: Sequence[str | None] | None = None,
|
|
) -> dict[int, str]:
|
|
"""Build safe fallback errors for input rows without metric results.
|
|
|
|
Precondition: target results and optional errors align with input rows.
|
|
Postcondition: returned values contain only approved public text.
|
|
"""
|
|
errors: dict[int, str] = {}
|
|
for position, (item, node_results) in enumerate(zip(input_list, node_run_result_mapping_list, strict=False)):
|
|
target_error = target_errors[position] if target_errors and position < len(target_errors) else None
|
|
errors[item.index] = (
|
|
safe_evaluation_error_text(target_error)
|
|
if target_error
|
|
else (
|
|
"No evaluation metrics were generated for this row."
|
|
if node_results
|
|
else "Target execution produced no successful node results for this row."
|
|
)
|
|
)
|
|
return errors
|
|
|
|
|
|
def _finalize_results(
|
|
input_list: Sequence[EvaluationDatasetInput],
|
|
results_by_index: dict[int, EvaluationItemResult],
|
|
missing_errors: dict[int, str] | None = None,
|
|
) -> list[EvaluationItemResult]:
|
|
return [
|
|
results_by_index.get(
|
|
item.index,
|
|
EvaluationItemResult(
|
|
index=item.index,
|
|
error=(missing_errors or {}).get(item.index, "No evaluation result was generated for this row."),
|
|
),
|
|
)
|
|
for item in input_list
|
|
]
|
|
|
|
|
|
def _apply_judgment(
|
|
results: Sequence[EvaluationItemResult],
|
|
judgment_config: JudgmentConfig,
|
|
) -> list[EvaluationItemResult]:
|
|
judged: list[EvaluationItemResult] = []
|
|
for result in results:
|
|
if result.error is not None or not result.metrics:
|
|
judged.append(result)
|
|
continue
|
|
metric_values = {
|
|
(metric.node_info.node_id, metric.name): metric.value
|
|
for metric in result.metrics
|
|
if metric.node_info is not None
|
|
}
|
|
judged.append(
|
|
result.model_copy(update={"judgment": JudgmentProcessor.evaluate(metric_values, judgment_config)})
|
|
)
|
|
return judged
|
|
|
|
|
|
def _with_overall_score(result: EvaluationItemResult) -> EvaluationItemResult:
|
|
numeric_values = [
|
|
float(metric.value)
|
|
for metric in result.metrics
|
|
if isinstance(metric.value, Real) and not isinstance(metric.value, bool)
|
|
]
|
|
score = sum(numeric_values) / len(numeric_values) if numeric_values else None
|
|
return result.model_copy(update={"overall_score": score})
|
|
|
|
|
|
def _persist_results(
|
|
session: Session,
|
|
lease: EvaluationRunLease,
|
|
results: Sequence[EvaluationItemResult],
|
|
input_list: Sequence[EvaluationDatasetInput],
|
|
workflow_run_id_map: dict[int, str] | None = None,
|
|
) -> None:
|
|
"""Upsert results in small batches and expose cooperative progress.
|
|
|
|
Precondition: each result index belongs to ``input_list`` and the lease is active.
|
|
Postcondition: checkpoints contain approved public errors, never raw exceptions.
|
|
"""
|
|
result_map = {result.index: result for result in results}
|
|
workflow_run_ids = workflow_run_id_map or {}
|
|
|
|
for position, item_input in enumerate(input_list, start=1):
|
|
_assert_execution_lease(session, lease)
|
|
result = result_map.get(
|
|
item_input.index,
|
|
EvaluationItemResult(index=item_input.index, error="Missing evaluation result."),
|
|
)
|
|
run_item = session.scalar(
|
|
select(EvaluationRunItem).where(
|
|
EvaluationRunItem.evaluation_run_id == lease.run_id,
|
|
EvaluationRunItem.item_index == item_input.index,
|
|
)
|
|
)
|
|
values = {
|
|
"status": EvaluationRunItemStatus.COMPLETED,
|
|
"workflow_run_id": workflow_run_ids.get(
|
|
item_input.index,
|
|
run_item.workflow_run_id if run_item is not None else None,
|
|
),
|
|
"inputs": json.dumps(item_input.inputs, ensure_ascii=False),
|
|
"expected_output": item_input.serialize_expected_output(),
|
|
"actual_output": result.actual_output,
|
|
"metrics": json.dumps(
|
|
[metric.model_dump(mode="json") for metric in result.metrics],
|
|
ensure_ascii=False,
|
|
)
|
|
if result.metrics
|
|
else None,
|
|
"judgment": json.dumps(result.judgment.model_dump(mode="json"), ensure_ascii=False),
|
|
"metadata_json": json.dumps(result.metadata, ensure_ascii=False) if result.metadata else None,
|
|
"error": safe_evaluation_error_text(result.error),
|
|
"overall_score": result.overall_score,
|
|
}
|
|
if run_item is None:
|
|
run_item = EvaluationRunItem(
|
|
evaluation_run_id=lease.run_id,
|
|
item_index=item_input.index,
|
|
**values,
|
|
)
|
|
session.add(run_item)
|
|
else:
|
|
for field_name, value in values.items():
|
|
setattr(run_item, field_name, value)
|
|
|
|
if position % _PERSIST_BATCH_SIZE == 0 or position == len(input_list):
|
|
session.flush()
|
|
completed, failed, checkpointed = _get_completed_checkpoint_counts(session, lease.run_id)
|
|
total_items = session.scalar(select(EvaluationRun.total_items).where(EvaluationRun.id == lease.run_id))
|
|
progress_update = (
|
|
update(EvaluationRun)
|
|
.where(
|
|
EvaluationRun.id == lease.run_id,
|
|
EvaluationRun.status == EvaluationRunStatus.RUNNING,
|
|
EvaluationRun.execution_token == lease.token,
|
|
)
|
|
.values(
|
|
completed_items=completed,
|
|
failed_items=failed,
|
|
progress_rate=_checkpoint_progress(checkpointed, total_items or 0),
|
|
)
|
|
)
|
|
update_result = session.execute(progress_update)
|
|
if _affected_rows(update_result) != 1:
|
|
session.rollback()
|
|
_assert_execution_lease(session, lease)
|
|
raise EvaluationRunOwnershipLostError(f"Evaluation run {lease.run_id} checkpoint lease was rejected.")
|
|
session.commit()
|
|
|
|
|
|
def _assert_execution_lease(session: Session, lease: EvaluationRunLease) -> None:
|
|
"""Reject work from cancelled runs and stale task invocations."""
|
|
session.expire_all()
|
|
row = session.execute(
|
|
select(EvaluationRun.status, EvaluationRun.execution_token).where(EvaluationRun.id == lease.run_id)
|
|
).one_or_none()
|
|
if row is None:
|
|
raise EvaluationRunOwnershipLostError(f"Evaluation run {lease.run_id} no longer exists.")
|
|
|
|
status, execution_token = row
|
|
if status == EvaluationRunStatus.CANCELLED:
|
|
raise EvaluationRunCancelledError()
|
|
if status != EvaluationRunStatus.RUNNING or execution_token != lease.token:
|
|
raise EvaluationRunOwnershipLostError(
|
|
f"Evaluation run {lease.run_id} is owned by another execution or is no longer executable."
|
|
)
|
|
|
|
|
|
def _cooperative_checkpoint(session: Session, lease: EvaluationRunLease) -> None:
|
|
"""Check ownership and close the read transaction before external work."""
|
|
_assert_execution_lease(session, lease)
|
|
session.rollback()
|
|
|
|
|
|
def _touch_execution_lease(session: Session, lease: EvaluationRunLease) -> None:
|
|
"""Fence a pending transaction by locking and renewing its current owner."""
|
|
result = session.execute(
|
|
update(EvaluationRun)
|
|
.where(
|
|
EvaluationRun.id == lease.run_id,
|
|
EvaluationRun.status == EvaluationRunStatus.RUNNING,
|
|
EvaluationRun.execution_token == lease.token,
|
|
)
|
|
.values(heartbeat_at=naive_utc_now())
|
|
)
|
|
if _affected_rows(result) != 1:
|
|
session.rollback()
|
|
_assert_execution_lease(session, lease)
|
|
raise EvaluationRunOwnershipLostError(f"Evaluation run {lease.run_id} lease update was rejected.")
|
|
|
|
|
|
def _update_run_progress(session: Session, lease: EvaluationRunLease, progress_rate: float) -> None:
|
|
result = session.execute(
|
|
update(EvaluationRun)
|
|
.where(
|
|
EvaluationRun.id == lease.run_id,
|
|
EvaluationRun.status == EvaluationRunStatus.RUNNING,
|
|
EvaluationRun.execution_token == lease.token,
|
|
)
|
|
.values(progress_rate=min(max(progress_rate, 0.0), 0.99))
|
|
)
|
|
if _affected_rows(result) != 1:
|
|
session.rollback()
|
|
_assert_execution_lease(session, lease)
|
|
raise EvaluationRunOwnershipLostError(f"Evaluation run {lease.run_id} progress lease was rejected.")
|
|
session.commit()
|
|
|
|
|
|
def _create_runner(
|
|
category: EvaluationCategory,
|
|
evaluation_instance: BaseEvaluationInstance,
|
|
) -> BaseEvaluationRunner:
|
|
if category == EvaluationCategory.LLM:
|
|
return LLMEvaluationRunner(evaluation_instance)
|
|
if category == EvaluationCategory.AGENT:
|
|
return AgentEvaluationRunner(evaluation_instance)
|
|
if category in {EvaluationCategory.RETRIEVAL, EvaluationCategory.KNOWLEDGE_BASE}:
|
|
return RetrievalEvaluationRunner(evaluation_instance)
|
|
raise ValueError(f"Unsupported evaluation category: {category}")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Status / summary / XLSX / storage helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _mark_run_failed(session: Session, lease: EvaluationRunLease, error: str) -> None:
|
|
"""Fail the run only while this invocation still owns it.
|
|
|
|
Precondition: ``lease`` identifies the running invocation; ``error`` may be untrusted.
|
|
Postcondition: a matching run is failed with a public error; other owners are unchanged.
|
|
"""
|
|
try:
|
|
session.rollback()
|
|
run_identity = session.execute(
|
|
select(EvaluationRun.tenant_id, EvaluationRun.target_type, EvaluationRun.total_items).where(
|
|
EvaluationRun.id == lease.run_id
|
|
)
|
|
).one_or_none()
|
|
result = session.execute(
|
|
update(EvaluationRun)
|
|
.where(
|
|
EvaluationRun.id == lease.run_id,
|
|
EvaluationRun.status == EvaluationRunStatus.RUNNING,
|
|
EvaluationRun.execution_token == lease.token,
|
|
)
|
|
.values(
|
|
status=EvaluationRunStatus.FAILED,
|
|
execution_token=None,
|
|
heartbeat_at=None,
|
|
error=safe_evaluation_error_text(error),
|
|
completed_at=naive_utc_now(),
|
|
)
|
|
)
|
|
session.commit()
|
|
if (
|
|
_affected_rows(result) == 1
|
|
and run_identity is not None
|
|
and EvaluationTargetType(run_identity.target_type) == EvaluationTargetType.AGENTS
|
|
):
|
|
EvaluationService._stop_agent_evaluation_tasks(
|
|
tenant_id=run_identity.tenant_id,
|
|
evaluation_run_id=lease.run_id,
|
|
total_items=run_identity.total_items,
|
|
)
|
|
except Exception:
|
|
logger.exception("Failed to mark run %s as failed", lease.run_id)
|
|
|
|
|
|
def _mark_run_pending_for_retry(
|
|
session: Session,
|
|
lease: EvaluationRunLease,
|
|
error: str,
|
|
retry_number: int,
|
|
max_retries: int,
|
|
) -> bool:
|
|
"""Release an owned run for retry while retaining row checkpoints.
|
|
|
|
Precondition: ``error`` is a public diagnostic and ``lease`` identifies the running owner.
|
|
Postcondition: a matching run is pending with a safe error; an ownership mismatch leaves it unchanged.
|
|
"""
|
|
session.rollback()
|
|
result = session.execute(
|
|
update(EvaluationRun)
|
|
.where(
|
|
EvaluationRun.id == lease.run_id,
|
|
EvaluationRun.status == EvaluationRunStatus.RUNNING,
|
|
EvaluationRun.execution_token == lease.token,
|
|
)
|
|
.values(
|
|
status=EvaluationRunStatus.PENDING,
|
|
execution_token=None,
|
|
heartbeat_at=None,
|
|
completed_at=None,
|
|
error=safe_evaluation_error_text(error),
|
|
)
|
|
)
|
|
session.commit()
|
|
released = _affected_rows(result) == 1
|
|
if released:
|
|
logger.info("Evaluation run %s scheduled for retry %d/%d", lease.run_id, retry_number, max_retries)
|
|
return released
|
|
|
|
|
|
def _compute_metrics_summary(
|
|
results: list[EvaluationItemResult],
|
|
judgment_config: JudgmentConfig | None,
|
|
) -> dict[str, Any]:
|
|
"""Aggregate numeric scores by node ID and metric name without modifying results.
|
|
|
|
Titles are display metadata: equal titles never merge different nodes, and
|
|
a changed title never splits one node. Summary labels match XLSX headers.
|
|
"""
|
|
metric_columns = _metric_columns(results)
|
|
metric_values: dict[_MetricIdentity, list[float]] = {}
|
|
for result in results:
|
|
for metric in result.metrics:
|
|
if isinstance(metric.value, Real) and not isinstance(metric.value, bool):
|
|
metric_values.setdefault(_metric_identity(metric), []).append(float(metric.value))
|
|
|
|
summary: dict[str, Any] = {
|
|
"metrics": {
|
|
metric_columns[identity]: {
|
|
"count": len(values),
|
|
"average": sum(values) / len(values),
|
|
"min": min(values),
|
|
"max": max(values),
|
|
}
|
|
for identity, values in metric_values.items()
|
|
}
|
|
}
|
|
|
|
if judgment_config is not None and judgment_config.conditions:
|
|
evaluated_results: list[EvaluationItemResult] = [
|
|
result for result in results if result.error is None and result.metrics
|
|
]
|
|
passed_items = sum(1 for result in evaluated_results if result.judgment.passed)
|
|
evaluated_items = len(evaluated_results)
|
|
summary["_judgment"] = {
|
|
"enabled": True,
|
|
"logical_operator": judgment_config.logical_operator,
|
|
"configured_conditions": len(judgment_config.conditions),
|
|
"evaluated_items": evaluated_items,
|
|
"passed_items": passed_items,
|
|
"failed_items": evaluated_items - passed_items,
|
|
"pass_rate": passed_items / evaluated_items if evaluated_items else 0.0,
|
|
}
|
|
|
|
return summary
|
|
|
|
|
|
def _generate_result_xlsx(
|
|
input_list: list[EvaluationDatasetInput],
|
|
results: list[EvaluationItemResult],
|
|
) -> bytes:
|
|
"""Render dataset rows and results as XLSX bytes.
|
|
|
|
Precondition: structured metric values are JSON-serializable, as they are
|
|
in saved checkpoints. Unsupported values raise during serialization.
|
|
Postcondition: metric identities keep distinct columns; strings are literal
|
|
cells, numeric scores remain numeric, and legacy/raw errors are masked.
|
|
"""
|
|
wb = Workbook()
|
|
ws = wb.active
|
|
if ws is None:
|
|
ws = wb.create_sheet("Evaluation Results")
|
|
ws.title = "Evaluation Results"
|
|
|
|
header_font = Font(bold=True, color="FFFFFF")
|
|
header_fill = PatternFill(start_color="4472C4", end_color="4472C4", fill_type="solid")
|
|
header_alignment = Alignment(horizontal="center", vertical="center")
|
|
thin_border = Border(
|
|
left=Side(style="thin"),
|
|
right=Side(style="thin"),
|
|
top=Side(style="thin"),
|
|
bottom=Side(style="thin"),
|
|
)
|
|
|
|
def write_cell(row: int, column: int, value: Any) -> Cell:
|
|
"""Write an openpyxl-supported value and return its bordered cell.
|
|
|
|
Strings stay literal after serialization; other values retain their
|
|
native cell type. Unsupported values raise from openpyxl.
|
|
"""
|
|
cell = ws.cell(row=row, column=column, value=value)
|
|
if isinstance(value, str):
|
|
cell.data_type = "s"
|
|
cell.border = thin_border
|
|
return cell
|
|
|
|
metric_columns = _metric_columns(results)
|
|
|
|
# Collect all input keys
|
|
input_keys: list[str] = []
|
|
for item in input_list:
|
|
for key in item.inputs:
|
|
if key not in input_keys:
|
|
input_keys.append(key)
|
|
|
|
expected_output_headers: list[str] = []
|
|
for item in input_list:
|
|
for header, _ in item.iter_expected_output_columns():
|
|
if header not in expected_output_headers:
|
|
expected_output_headers.append(header)
|
|
|
|
has_judgment = any(bool(r.judgment.condition_results) for r in results)
|
|
|
|
judgment_headers = ["judgment"] if has_judgment else []
|
|
headers = (
|
|
["index"]
|
|
+ input_keys
|
|
+ expected_output_headers
|
|
+ ["actual_output"]
|
|
+ list(metric_columns.values())
|
|
+ judgment_headers
|
|
+ ["error"]
|
|
)
|
|
|
|
for col_idx, header in enumerate(headers, start=1):
|
|
cell = write_cell(1, col_idx, header)
|
|
cell.font = header_font
|
|
cell.fill = header_fill
|
|
cell.alignment = header_alignment
|
|
|
|
ws.column_dimensions["A"].width = 10
|
|
for col_idx in range(2, len(headers) + 1):
|
|
ws.column_dimensions[get_column_letter(col_idx)].width = 25
|
|
|
|
result_by_index = {r.index: r for r in results}
|
|
|
|
for row_idx, item in enumerate(input_list, start=2):
|
|
row_result = result_by_index.get(item.index)
|
|
|
|
col = 1
|
|
write_cell(row_idx, col, item.index)
|
|
col += 1
|
|
|
|
for key in input_keys:
|
|
val = item.inputs.get(key, "")
|
|
write_cell(row_idx, col, str(val))
|
|
col += 1
|
|
|
|
for header in expected_output_headers:
|
|
write_cell(row_idx, col, _get_expected_output_cell_value(item, header))
|
|
col += 1
|
|
|
|
write_cell(row_idx, col, row_result.actual_output if row_result else "")
|
|
col += 1
|
|
|
|
metric_scores = {_metric_identity(metric): metric.value for metric in row_result.metrics} if row_result else {}
|
|
for identity in metric_columns:
|
|
score = metric_scores.get(identity)
|
|
if isinstance(score, (dict, list)):
|
|
score = json.dumps(score, ensure_ascii=False)
|
|
write_cell(row_idx, col, score if score is not None else "")
|
|
col += 1
|
|
|
|
if has_judgment:
|
|
if row_result and row_result.judgment.condition_results:
|
|
judgment_value = "Pass" if row_result.judgment.passed else "Fail"
|
|
else:
|
|
judgment_value = ""
|
|
write_cell(row_idx, col, judgment_value)
|
|
col += 1
|
|
|
|
write_cell(row_idx, col, safe_evaluation_error_text(row_result.error) if row_result else "")
|
|
|
|
output = io.BytesIO()
|
|
wb.save(output)
|
|
output.seek(0)
|
|
return output.getvalue()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class _MetricIdentity:
|
|
"""Stable report identity; a null node ID denotes a target-scoped metric."""
|
|
|
|
node_id: str | None
|
|
name: str
|
|
|
|
|
|
def _metric_identity(metric: EvaluationMetric) -> _MetricIdentity:
|
|
"""Return the score's node/name identity, independent of mutable display metadata."""
|
|
return _MetricIdentity(metric.node_info.node_id if metric.node_info is not None else None, metric.name)
|
|
|
|
|
|
def _metric_columns(results: Sequence[EvaluationItemResult]) -> dict[_MetricIdentity, str]:
|
|
"""Assign unique display labels to result identities in first-seen order.
|
|
|
|
Preconditions: results contain validated metrics with optional node info.
|
|
Postconditions: each node/metric identity has one unique label; ordinary
|
|
labels retain the first title seen, customized evaluator labels use the
|
|
``Custom Evaluator : `` prefix, and results are unchanged. Duplicate labels
|
|
gain the node ID and, if needed, a suffix.
|
|
"""
|
|
labels: dict[_MetricIdentity, str] = {}
|
|
for result in results:
|
|
for metric in result.metrics:
|
|
node = metric.node_info
|
|
if node is not None and node.type == "customized":
|
|
label = f"Custom Evaluator : {metric.name}"
|
|
else:
|
|
label = f"{node.title or node.node_id} : {metric.name}" if node is not None else metric.name
|
|
labels.setdefault(_metric_identity(metric), label)
|
|
|
|
label_counts = Counter(labels.values())
|
|
reserved_labels = set(labels.values())
|
|
columns: dict[_MetricIdentity, str] = {}
|
|
for identity, label in labels.items():
|
|
column_label = label
|
|
if label_counts[label] > 1:
|
|
scope = identity.node_id if identity.node_id is not None else "unscoped"
|
|
qualified_label = f"{label} [{scope}]"
|
|
column_label = qualified_label
|
|
suffix = 2
|
|
while column_label in reserved_labels:
|
|
column_label = f"{qualified_label} ({suffix})"
|
|
suffix += 1
|
|
reserved_labels.add(column_label)
|
|
columns[identity] = column_label
|
|
return columns
|
|
|
|
|
|
def _get_expected_output_cell_value(item: EvaluationDatasetInput, header: str) -> str:
|
|
"""Return the expected-output cell value for a generated result file header."""
|
|
for column_header, value in item.iter_expected_output_columns():
|
|
if column_header == header:
|
|
return value
|
|
return ""
|
|
|
|
|
|
def _store_result_file(
|
|
tenant_id: str,
|
|
run_id: str,
|
|
xlsx_content: bytes,
|
|
session: Session,
|
|
created_by: str,
|
|
) -> UploadFile:
|
|
"""Save result bytes and stage the matching UploadFile in the current transaction."""
|
|
from libs.uuid_utils import uuidv7
|
|
|
|
filename = f"evaluation-result-{run_id[:8]}.xlsx"
|
|
storage_key = f"evaluation_results/{tenant_id}/{str(uuidv7())}.xlsx"
|
|
storage.save(storage_key, xlsx_content)
|
|
|
|
upload_file = UploadFile(
|
|
tenant_id=tenant_id,
|
|
storage_type=StorageType(dify_config.STORAGE_TYPE),
|
|
key=storage_key,
|
|
name=filename,
|
|
size=len(xlsx_content),
|
|
extension="xlsx",
|
|
mime_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
created_by_role=CreatorUserRole.ACCOUNT,
|
|
created_by=created_by,
|
|
created_at=naive_utc_now(),
|
|
used=True,
|
|
used_by=created_by,
|
|
used_at=naive_utc_now(),
|
|
)
|
|
try:
|
|
session.add(upload_file)
|
|
session.flush()
|
|
except Exception:
|
|
session.rollback()
|
|
_delete_stored_result_file(storage_key)
|
|
raise
|
|
return upload_file
|
|
|
|
|
|
def _delete_stored_result_file(storage_key: str) -> None:
|
|
"""Best-effort compensation when a staged result file cannot be published."""
|
|
try:
|
|
storage.delete(storage_key)
|
|
except Exception:
|
|
logger.exception("Failed to delete unpublished evaluation result file %s", storage_key)
|