mirror of
https://github.com/langgenius/dify.git
synced 2026-11-22 02:59:44 +08:00
126 lines
5.4 KiB
Python
126 lines
5.4 KiB
Python
"""Evaluation failures and the public representation of their diagnostics."""
|
|
|
|
import re
|
|
import uuid
|
|
from dataclasses import dataclass
|
|
from enum import StrEnum
|
|
from typing import overload
|
|
|
|
|
|
class EvaluationWorkflowVersionNotFoundError(Exception):
|
|
"""Raised when a workflow version pinned by an evaluation run no longer exists."""
|
|
|
|
|
|
class EvaluationErrorCode(StrEnum):
|
|
"""Stable categories permitted in evaluation results and API responses."""
|
|
|
|
MODEL_REQUEST_FAILED = "EVAL_MODEL_REQUEST_FAILED"
|
|
METRIC_FAILED = "EVAL_METRIC_FAILED"
|
|
UNSUPPORTED_METRIC = "EVAL_UNSUPPORTED_METRIC"
|
|
CUSTOM_EVALUATOR_FAILED = "EVAL_CUSTOM_EVALUATOR_FAILED"
|
|
TARGET_FAILED = "EVAL_TARGET_FAILED"
|
|
TARGET_DATA_MISSING = "EVAL_TARGET_DATA_MISSING"
|
|
RETRIEVAL_FAILED = "EVAL_RETRIEVAL_FAILED"
|
|
RUN_FAILED = "EVAL_RUN_FAILED"
|
|
RUN_DISPATCH_FAILED = "EVAL_RUN_DISPATCH_FAILED"
|
|
RUN_RETRYING = "EVAL_RUN_RETRYING"
|
|
INVALID_CONFIG = "EVAL_INVALID_CONFIG"
|
|
JUDGMENT_FAILED = "EVAL_JUDGMENT_FAILED"
|
|
|
|
|
|
_PUBLIC_MESSAGES = {
|
|
EvaluationErrorCode.MODEL_REQUEST_FAILED: (
|
|
"Evaluation model request failed. Check the model configuration and retry."
|
|
),
|
|
EvaluationErrorCode.METRIC_FAILED: "Metric evaluation failed. Check the evaluation configuration and retry.",
|
|
EvaluationErrorCode.UNSUPPORTED_METRIC: "The selected metric is not supported by this evaluator.",
|
|
EvaluationErrorCode.CUSTOM_EVALUATOR_FAILED: "Custom evaluator failed. Check its workflow and retry.",
|
|
EvaluationErrorCode.TARGET_FAILED: "Target execution failed. Check the target configuration and retry.",
|
|
EvaluationErrorCode.TARGET_DATA_MISSING: "The target run did not provide data required for this metric.",
|
|
EvaluationErrorCode.RETRIEVAL_FAILED: "Knowledge retrieval failed. Check the knowledge base and retry.",
|
|
EvaluationErrorCode.RUN_FAILED: "Evaluation run failed. Please retry or contact support.",
|
|
EvaluationErrorCode.RUN_DISPATCH_FAILED: "Could not start evaluation. Please retry or contact support.",
|
|
EvaluationErrorCode.RUN_RETRYING: "Evaluation is retrying after a temporary failure.",
|
|
EvaluationErrorCode.INVALID_CONFIG: "Evaluation configuration is invalid. Check it and retry.",
|
|
EvaluationErrorCode.JUDGMENT_FAILED: "Evaluation judgment failed. Check its conditions and retry.",
|
|
}
|
|
_PUBLIC_ERROR_PATTERN = re.compile(r"(?P<code>EVAL_[A-Z_]+): .* \[error_id=(?P<error_id>[0-9a-f]{32})\]")
|
|
_LEGACY_ERROR_MESSAGE = "Evaluation failed. Please retry or contact support."
|
|
_SAFE_STATIC_ERRORS = frozenset(
|
|
{
|
|
"Agent evaluation row was already completed.",
|
|
"Checkpointed Agent target result is missing.",
|
|
"Checkpointed target workflow results are no longer available.",
|
|
"Checkpointed target workflow run ID is missing.",
|
|
"Evaluation run heartbeat lease expired.",
|
|
"Metric 'tool_call_accuracy' requires non-empty expected_tool_calls.",
|
|
"Missing evaluation result.",
|
|
"No evaluation metrics were generated for this row.",
|
|
"No evaluation result was generated for this row.",
|
|
"Target execution produced no successful node results.",
|
|
"Target execution produced no successful node results for this row.",
|
|
"Target execution returned no workflow run ID.",
|
|
"The customized evaluator app was deleted.",
|
|
}
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PublicEvaluationError:
|
|
"""A fixed public message and its server-side diagnostic reference."""
|
|
|
|
code: EvaluationErrorCode
|
|
error_id: str
|
|
|
|
@property
|
|
def text(self) -> str:
|
|
"""Render an approved message and reference without exception data.
|
|
|
|
Precondition: the code is a supported category and the reference is a UUID hex value.
|
|
Postcondition: the result is canonical and safe for API and XLSX output.
|
|
"""
|
|
return f"{self.code.value}: {_PUBLIC_MESSAGES[self.code]} [error_id={self.error_id}]"
|
|
|
|
|
|
def new_public_evaluation_error(code: EvaluationErrorCode) -> PublicEvaluationError:
|
|
"""Create a fresh diagnostic reference for an approved error category.
|
|
|
|
Precondition: ``code`` is a member of ``EvaluationErrorCode``.
|
|
Postcondition: the returned public text contains no exception data and the
|
|
reference can be attached to the corresponding server-side log event.
|
|
"""
|
|
return PublicEvaluationError(code=code, error_id=uuid.uuid4().hex)
|
|
|
|
|
|
@overload
|
|
def safe_evaluation_error_text(error: None) -> None: ...
|
|
|
|
|
|
@overload
|
|
def safe_evaluation_error_text(error: str) -> str: ...
|
|
|
|
|
|
def safe_evaluation_error_text(error: str | None) -> str | None:
|
|
"""Allow only canonical public errors through a result serialization boundary.
|
|
|
|
Precondition: ``error`` is absent or an untrusted persisted/result string.
|
|
Postcondition: unknown, legacy, or mixed raw errors become a fixed generic
|
|
message; known public errors retain their original diagnostic references.
|
|
"""
|
|
if error is None:
|
|
return None
|
|
|
|
parts = error.split("; ")
|
|
for part in parts:
|
|
if part not in _SAFE_STATIC_ERRORS:
|
|
match = _PUBLIC_ERROR_PATTERN.fullmatch(part)
|
|
if match is None:
|
|
return _LEGACY_ERROR_MESSAGE
|
|
try:
|
|
code = EvaluationErrorCode(match.group("code"))
|
|
except ValueError:
|
|
return _LEGACY_ERROR_MESSAGE
|
|
if part != PublicEvaluationError(code=code, error_id=match.group("error_id")).text:
|
|
return _LEGACY_ERROR_MESSAGE
|
|
return error
|