Files
dify/api/controllers/console/evaluation.py

1365 lines
51 KiB
Python

from __future__ import annotations
from collections.abc import Callable, Mapping
from datetime import datetime
from functools import wraps
from typing import Any, ParamSpec, TypeVar
from urllib.parse import quote
from uuid import UUID
from flask import Response, request
from flask_restx import Resource
from pydantic import AliasChoices, BaseModel, Field, RootModel, field_validator
from sqlalchemy.orm import Session
from werkzeug.exceptions import BadRequest, Forbidden, NotFound, TooManyRequests
from configs import dify_config
from constants.languages import get_valid_language, languages
from controllers.common.fields import BinaryFileResponse
from controllers.common.rbac import AgentId, DatasetId, PlainApp, RBACCheck, Workspace, enforce_rbac_checks
from controllers.common.schema import (
query_params_from_model,
register_response_schema_models,
register_schema_models,
)
from controllers.common.session import with_session
from controllers.console import console_ns
from controllers.console.wraps import (
RBACPermission,
account_initialization_required,
edit_permission_required,
evaluation_edition_required,
evaluation_paid_plan_required,
rbac_permission_required,
setup_required,
)
from core.evaluation.entities.config_entity import EvaluationFrameworkEnum, SelectableEvaluationFramework
from core.evaluation.entities.evaluation_entity import (
CustomizedMetrics,
DefaultMetric,
EvaluationCategory,
EvaluationConfigData,
EvaluationMetric,
EvaluationRunRequest,
NodeInfo,
)
from core.evaluation.entities.judgment_entity import JudgmentConfig
from core.evaluation.exceptions import safe_evaluation_error_text
from extensions.ext_application_services import application_services
from extensions.ext_storage import storage
from fields.base import ResponseModel
from fields.member_fields import SimpleAccount
from graphon.file import helpers as file_helpers
from libs.flask_restx_compat import BINARY_RESPONSE_MEDIA_TYPES_VENDOR_KEY
from libs.helper import dump_response, to_timestamp
from libs.login import current_account_with_tenant, login_required
from machinery.context import RequestContext
from models import Account, App
from models.agent import Agent
from models.evaluation import EvaluationConfiguration, EvaluationRun, EvaluationTargetType
from models.model import AppMode
from models.snippet import CustomizedSnippet
from services.app.access import resolve_app_run_access_filter
from services.errors.base import NoPermissionError
from services.errors.evaluation import (
EvaluationDatasetInvalidError,
EvaluationFrameworkNotConfiguredError,
EvaluationMaxConcurrentRunsError,
EvaluationNotFoundError,
)
from services.evaluation_service import EvaluationService
from services.knowledge.dataset_access import AccessibleDataset, DatasetAccessDeniedError, DatasetNotFoundError
from services.workflow_service import WorkflowService
P = ParamSpec("P")
R = TypeVar("R")
EvaluationTarget = App | CustomizedSnippet | Agent
XLSX_MIME_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
class EvaluationEditionResource(Resource):
method_decorators = [evaluation_edition_required]
class EvaluationConfigPayload(EvaluationConfigData):
pass
class EvaluationRunPayload(EvaluationRunRequest):
pass
class EvaluationFrameworkQuery(BaseModel):
evaluation_framework: SelectableEvaluationFramework = EvaluationFrameworkEnum.RAGAS
class EvaluationNodeInfoPayload(BaseModel):
metrics: list[str] = Field(default_factory=list)
class EvaluationRunListQuery(BaseModel):
page: int = Field(default=1, ge=1, le=99999)
page_size: int = Field(default=20, ge=1, le=100)
class EvaluationRunDetailQuery(BaseModel):
page: int = Field(default=1, ge=1, le=99999)
page_size: int = Field(default=50, ge=1, le=100)
class AvailableEvaluationWorkflowQuery(BaseModel):
page: int = Field(default=1, ge=1, le=99999)
limit: int = Field(default=10, ge=1, le=100)
user_id: str | None = None
named_only: bool = False
keyword: str | None = Field(default=None, max_length=255)
class EvaluationConfigResponse(ResponseModel):
evaluation_framework: SelectableEvaluationFramework = EvaluationFrameworkEnum.RAGAS
evaluation_model: str | None = None
evaluation_model_provider: str | None = None
default_metrics: list[DefaultMetric] | None = None
customized_metrics: CustomizedMetrics | None = None
judgment_config: JudgmentConfig | None = None
class EvaluationTemplateColumnResponse(ResponseModel):
name: str
type: str
class EvaluationTemplateColumnsResponse(ResponseModel):
columns: list[EvaluationTemplateColumnResponse]
class EvaluationRunResponse(ResponseModel):
id: str
tenant_id: str
target_type: str
target_id: str
target_workflow_version_id: str | None = None
target_agent_config_snapshot_id: str | None = None
evaluation_config_id: str
status: str
dataset_file_id: str | None = None
result_file_id: str | None = None
total_items: int
completed_items: int
failed_items: int
progress: float
metrics_summary: dict[str, Any] = Field(
default_factory=dict,
validation_alias=AliasChoices("metrics_summary_dict", "metrics_summary"),
)
error: str | None = None
created_by: str
started_at: int | None = None
completed_at: int | None = None
created_at: int | None = None
@field_validator("error", mode="before")
@classmethod
def _safe_error(cls, value: str | None) -> str | None:
"""Keep only approved diagnostics in run API responses.
Precondition: value comes from a persisted run and may contain legacy text.
Postcondition: the response never returns an unapproved raw exception.
"""
return safe_evaluation_error_text(value)
@field_validator("started_at", "completed_at", "created_at", mode="before")
@classmethod
def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
return to_timestamp(value)
class EvaluationRunItemResponse(ResponseModel):
id: str
item_index: int
workflow_run_id: str | None = None
inputs: dict[str, Any] = Field(default_factory=dict, validation_alias=AliasChoices("inputs_dict", "inputs"))
expected_output: str | None = None
actual_output: str | None = None
metrics: list[EvaluationMetric] = Field(
default_factory=list,
validation_alias=AliasChoices("metrics_list", "metrics"),
)
judgment: dict[str, Any] = Field(
default_factory=dict,
validation_alias=AliasChoices("judgment_dict", "judgment"),
)
metadata: dict[str, Any] = Field(
default_factory=dict,
validation_alias=AliasChoices("metadata_dict", "metadata"),
)
error: str | None = None
overall_score: float | None = None
@field_validator("error", mode="before")
@classmethod
def _safe_error(cls, value: str | None) -> str | None:
"""Keep only approved diagnostics in item API responses.
Precondition: value comes from a persisted row and may contain legacy text.
Postcondition: the response never returns an unapproved raw exception.
"""
return safe_evaluation_error_text(value)
@field_validator("judgment", mode="before")
@classmethod
def _safe_judgment_errors(cls, value: dict[str, Any]) -> dict[str, Any]:
"""Mask legacy exception text in nested condition diagnostics.
Precondition: ``value`` is a decoded judgment object from a run item.
Postcondition: condition errors contain only approved public text and
all other judgment fields retain their original values.
"""
conditions = value.get("condition_results")
if not isinstance(conditions, list):
return value
return {
**value,
"condition_results": [
{**condition, "error": safe_evaluation_error_text(condition.get("error"))}
if isinstance(condition, Mapping)
else condition
for condition in conditions
],
}
class EvaluationRunPaginationResponse(ResponseModel):
data: list[EvaluationRunResponse]
total: int
page: int
page_size: int
class EvaluationRunItemPaginationResponse(ResponseModel):
data: list[EvaluationRunItemResponse]
total: int
page: int
page_size: int
class EvaluationRunDetailResponse(ResponseModel):
run: EvaluationRunResponse
items: EvaluationRunItemPaginationResponse
class EvaluationMetricsResponse(ResponseModel):
metrics: dict[str, list[str]] | list[str]
class EvaluationDefaultMetricsResponse(ResponseModel):
default_metrics: list[DefaultMetric]
class EvaluationAvailableMetricsResponse(ResponseModel):
metrics: list[str]
class EvaluationNodeInfoResponse(RootModel[dict[str, list[NodeInfo]]]):
pass
class EvaluationFileResponse(ResponseModel):
id: str
name: str
size: int
extension: str
mime_type: str
created_at: int | None = None
download_url: str
@field_validator("created_at", mode="before")
@classmethod
def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
return to_timestamp(value)
class AvailableEvaluationWorkflowResponse(ResponseModel):
id: str
app_id: str
app_name: str
type: str
kind: str
version: str
marked_name: str
marked_comment: str
hash: str
created_by: SimpleAccount | None = None
created_at: int
updated_by: SimpleAccount | None = None
updated_at: int
@field_validator("created_at", "updated_at", mode="before")
@classmethod
def _normalize_timestamp(cls, value: datetime | int | None) -> int:
timestamp = to_timestamp(value)
if timestamp is None:
raise ValueError("timestamp is required")
return timestamp
class AvailableEvaluationWorkflowPaginationResponse(ResponseModel):
items: list[AvailableEvaluationWorkflowResponse]
page: int
limit: int
has_more: bool
class EvaluationAssociatedTargetResponse(ResponseModel):
target_type: str
target_id: str
target_name: str
class EvaluationAssociatedTargetListResponse(ResponseModel):
items: list[EvaluationAssociatedTargetResponse]
register_schema_models(
console_ns,
EvaluationConfigPayload,
EvaluationRunPayload,
EvaluationFrameworkQuery,
EvaluationNodeInfoPayload,
EvaluationRunListQuery,
EvaluationRunDetailQuery,
AvailableEvaluationWorkflowQuery,
)
register_response_schema_models(
console_ns,
EvaluationConfigResponse,
EvaluationTemplateColumnsResponse,
EvaluationRunResponse,
EvaluationRunPaginationResponse,
EvaluationRunDetailResponse,
EvaluationMetricsResponse,
EvaluationDefaultMetricsResponse,
EvaluationAvailableMetricsResponse,
EvaluationNodeInfoResponse,
EvaluationFileResponse,
AvailableEvaluationWorkflowPaginationResponse,
EvaluationAssociatedTargetListResponse,
BinaryFileResponse,
)
def evaluation_target_required(
*,
app_permission: RBACPermission,
snippet_permission: RBACPermission = RBACPermission.SNIPPETS_CREATE_AND_MODIFY,
agent_permission: RBACPermission = RBACPermission.AGENT_TEST_AND_RUN,
) -> Callable[[Callable[P, R]], Callable[P, R]]:
"""Resolve an app/snippet/Agent target with tenant and dynamic RBAC checks."""
def decorator(view: Callable[P, R]) -> Callable[P, R]:
@wraps(view)
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
session = args[1]
if not isinstance(session, Session):
raise TypeError("Evaluation endpoint requires a database session.")
target_type = str(kwargs.pop("evaluate_target_type"))
target_id = str(kwargs.pop("evaluate_target_id"))
current_user, tenant_id = current_account_with_tenant()
if target_type == EvaluationTargetType.APPS.value:
enforce_rbac_checks(
tenant_id=tenant_id,
account_id=current_user.id,
checks=[RBACCheck(app_permission, PlainApp("evaluate_target_id"))],
path_args={"evaluate_target_id": target_id},
)
elif target_type == EvaluationTargetType.SNIPPETS.value:
enforce_rbac_checks(
tenant_id=tenant_id,
account_id=current_user.id,
checks=[RBACCheck(snippet_permission, Workspace())],
)
elif target_type == EvaluationTargetType.AGENTS.value:
enforce_rbac_checks(
tenant_id=tenant_id,
account_id=current_user.id,
checks=[RBACCheck(agent_permission, AgentId("evaluate_target_id"))],
path_args={"evaluate_target_id": target_id},
)
else:
raise NotFound("Evaluation target not found.")
target: EvaluationTarget | None = EvaluationService.get_console_target(
session, tenant_id, target_type, target_id
)
if target is None:
raise NotFound("Evaluation target not found.")
if isinstance(target, App) and AppMode.value_of(target.mode) != AppMode.WORKFLOW:
raise BadRequest("Only workflow apps support evaluation.")
kwargs["target"] = target
kwargs["target_type"] = target_type
return view(*args, **kwargs)
return wrapper
return decorator
def _get_accessible_dataset(
session: Session,
dataset_id: UUID,
current_user: Account,
tenant_id: str,
) -> AccessibleDataset:
"""Authorize the admitted account and return a tenant-scoped dataset identity.
The existing evaluation transaction stays caller-owned. Knowledge admission
uses its configured application service and raises the native HTTP errors.
"""
try:
ref = application_services().knowledge.datasets.require_accessible(
RequestContext("", None, current_user.id, tenant_id), str(dataset_id)
)
except DatasetNotFoundError as exc:
raise NotFound("Dataset not found.") from exc
except DatasetAccessDeniedError as exc:
raise Forbidden(str(exc)) from exc
return AccessibleDataset(ref.dataset_id, ref.tenant_id)
def _config_response(config: EvaluationConfiguration | None, *, target_type: str) -> dict[str, Any]:
if config is None:
return dump_response(EvaluationConfigResponse, {})
default_metrics = config.default_metrics_list
if target_type != EvaluationTargetType.KNOWLEDGE_BASE.value:
default_metrics = EvaluationService.serialize_console_default_metrics(default_metrics)
return dump_response(
EvaluationConfigResponse,
{
"evaluation_framework": config.evaluation_framework,
"evaluation_model": config.evaluation_model,
"evaluation_model_provider": config.evaluation_model_provider,
"default_metrics": default_metrics,
"customized_metrics": config.customized_metrics_dict,
"judgment_config": EvaluationService.serialize_console_judgment_config(config.judgment_config_dict),
},
)
def _load_evaluation_dataset(
*,
session: Session,
tenant_id: str,
file_id: str,
) -> tuple[bytes, str]:
upload_file = EvaluationService.get_dataset_upload_file(session, tenant_id, file_id)
if upload_file is None:
raise NotFound("Evaluation dataset file not found.")
try:
content = storage.load_once(upload_file.key)
except Exception as exc:
raise BadRequest("Failed to read evaluation dataset file.") from exc
if not content:
raise BadRequest("Evaluation dataset file is empty.")
return content, upload_file.name
def _start_run(
*,
session: Session,
tenant_id: str,
target_type: str,
target_id: str,
account_id: str,
payload: EvaluationRunPayload,
) -> EvaluationRun:
"""Admit a target run for the caller and map service denials to HTTP 403 without dispatching."""
dataset_content, dataset_filename = _load_evaluation_dataset(
session=session,
tenant_id=tenant_id,
file_id=payload.file_id,
)
try:
return EvaluationService.start_evaluation_run(
session=session,
tenant_id=tenant_id,
target_type=target_type,
target_id=target_id,
account_id=account_id,
dataset_file_content=dataset_content,
dataset_filename=dataset_filename,
run_request=payload,
)
except EvaluationFrameworkNotConfiguredError as exc:
raise BadRequest(exc.description) from exc
except EvaluationDatasetInvalidError as exc:
raise BadRequest(exc.description) from exc
except EvaluationNotFoundError as exc:
raise NotFound(exc.description) from exc
except EvaluationMaxConcurrentRunsError as exc:
raise TooManyRequests(exc.description) from exc
except NoPermissionError as exc:
raise Forbidden(str(exc)) from exc
except ValueError as exc:
raise BadRequest(str(exc)) from exc
def _run_detail(
*,
session: Session,
tenant_id: str,
target_type: str,
target_id: str,
run_id: str,
query: EvaluationRunDetailQuery,
) -> dict[str, Any]:
try:
run = EvaluationService.get_evaluation_run_detail(
session,
tenant_id,
target_type,
target_id,
run_id,
)
items, total = EvaluationService.get_evaluation_run_items(
session,
tenant_id,
target_type,
target_id,
run_id,
query.page,
query.page_size,
)
except EvaluationNotFoundError as exc:
raise NotFound(exc.description) from exc
return dump_response(
EvaluationRunDetailResponse,
{
"run": run,
"items": {
"data": items,
"total": total,
"page": query.page,
"page_size": query.page_size,
},
},
)
def _cancel_run(
*,
session: Session,
tenant_id: str,
target_type: str,
target_id: str,
run_id: str,
) -> dict[str, Any]:
try:
run = EvaluationService.cancel_evaluation_run(
session,
tenant_id,
target_type,
target_id,
run_id,
)
except EvaluationNotFoundError as exc:
raise NotFound(exc.description) from exc
except ValueError as exc:
raise BadRequest(str(exc)) from exc
return dump_response(EvaluationRunResponse, run)
def _retry_run(
*,
session: Session,
tenant_id: str,
target_type: str,
target_id: str,
run_id: str,
account_id: str,
) -> dict[str, Any]:
"""Retry a scoped run for the caller; permission denial leaves the failed run unchanged."""
try:
existing_run = EvaluationService.get_evaluation_run_detail(
session,
tenant_id,
target_type,
target_id,
run_id,
)
if not existing_run.dataset_file_id:
raise BadRequest("Evaluation run has no dataset file to retry.")
dataset_content, dataset_filename = _load_evaluation_dataset(
session=session,
tenant_id=tenant_id,
file_id=existing_run.dataset_file_id,
)
run = EvaluationService.retry_evaluation_run(
session=session,
tenant_id=tenant_id,
target_type=target_type,
target_id=target_id,
run_id=run_id,
account_id=account_id,
dataset_file_content=dataset_content,
dataset_filename=dataset_filename,
)
except EvaluationFrameworkNotConfiguredError as exc:
raise BadRequest(exc.description) from exc
except EvaluationDatasetInvalidError as exc:
raise BadRequest(exc.description) from exc
except EvaluationNotFoundError as exc:
raise NotFound(exc.description) from exc
except EvaluationMaxConcurrentRunsError as exc:
raise TooManyRequests(exc.description) from exc
except NoPermissionError as exc:
raise Forbidden(str(exc)) from exc
except ValueError as exc:
raise BadRequest(str(exc)) from exc
return dump_response(EvaluationRunResponse, run)
def _evaluation_file_response(
*,
session: Session,
tenant_id: str,
target_type: str,
target_id: str,
file_id: str,
) -> dict[str, Any]:
try:
upload_file = EvaluationService.get_evaluation_file(
session=session,
tenant_id=tenant_id,
target_type=target_type,
target_id=target_id,
file_id=file_id,
)
except EvaluationNotFoundError as exc:
raise NotFound(exc.description) from exc
return dump_response(
EvaluationFileResponse,
{
"id": upload_file.id,
"name": upload_file.name,
"size": upload_file.size,
"extension": upload_file.extension,
"mime_type": upload_file.mime_type,
"created_at": upload_file.created_at,
"download_url": file_helpers.get_signed_file_url(upload_file_id=upload_file.id, as_attachment=True),
},
)
def _xlsx_response(content: bytes, filename: str) -> Response:
response = Response(
content,
mimetype=XLSX_MIME_TYPE,
)
response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{quote(filename)}"
response.headers["Content-Length"] = str(len(content))
return response
def _evaluation_template_language(account: Account) -> str:
"""Use the current UI locale when valid, then the saved account language."""
cookie_locale = request.cookies.get("locale")
return cookie_locale if cookie_locale in languages else get_valid_language(account.interface_language)
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/dataset-template/download")
class EvaluationDatasetTemplateDownloadApi(EvaluationEditionResource):
@console_ns.expect(console_ns.models[EvaluationConfigPayload.__name__])
@console_ns.doc(
produces=[XLSX_MIME_TYPE],
vendor={BINARY_RESPONSE_MEDIA_TYPES_VENDOR_KEY: [XLSX_MIME_TYPE]},
)
@console_ns.response(
200,
"Evaluation dataset template",
console_ns.models[BinaryFileResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@edit_permission_required
@with_session(write=False)
@evaluation_target_required(
app_permission=RBACPermission.APP_VIEW_LAYOUT,
snippet_permission=RBACPermission.SNIPPETS_CREATE_AND_MODIFY,
)
def post(self, session: Session, target: EvaluationTarget, target_type: str) -> Response:
current_user, _ = current_account_with_tenant()
payload = EvaluationConfigPayload.model_validate(console_ns.payload or {})
try:
content, filename = EvaluationService.generate_dataset_template(
target, target_type, payload, session, language=_evaluation_template_language(current_user)
)
except ValueError as exc:
raise BadRequest(str(exc)) from exc
return _xlsx_response(content, filename)
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation")
class EvaluationDetailApi(EvaluationEditionResource):
@console_ns.response(200, "Evaluation configuration", console_ns.models[EvaluationConfigResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@with_session(write=False)
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
def get(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
_, tenant_id = current_account_with_tenant()
config = EvaluationService.get_evaluation_config(session, tenant_id, target_type, target.id)
return _config_response(config, target_type=target_type)
@console_ns.expect(console_ns.models[EvaluationConfigPayload.__name__])
@console_ns.response(200, "Evaluation configuration saved", console_ns.models[EvaluationConfigResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@edit_permission_required
@with_session
@evaluation_target_required(
app_permission=RBACPermission.APP_EDIT,
snippet_permission=RBACPermission.SNIPPETS_CREATE_AND_MODIFY,
)
def put(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
current_user, tenant_id = current_account_with_tenant()
payload = EvaluationConfigPayload.model_validate(console_ns.payload or {})
try:
config = EvaluationService.save_evaluation_config(
session,
tenant_id,
target_type,
target.id,
current_user.id,
payload,
)
except ValueError as exc:
raise BadRequest(str(exc)) from exc
return _config_response(config, target_type=target_type)
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/template-columns")
class EvaluationTemplateColumnsApi(EvaluationEditionResource):
@console_ns.expect(console_ns.models[EvaluationConfigPayload.__name__])
@console_ns.response(
200,
"Evaluation template columns",
console_ns.models[EvaluationTemplateColumnsResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@with_session(write=False)
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
def post(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
payload = EvaluationConfigPayload.model_validate(console_ns.payload or {})
try:
columns = EvaluationService.get_dataset_column_names(target, target_type, payload, session)
except ValueError as exc:
raise BadRequest(str(exc)) from exc
return dump_response(EvaluationTemplateColumnsResponse, {"columns": columns})
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/logs")
class EvaluationLogsApi(EvaluationEditionResource):
@console_ns.doc(params=query_params_from_model(EvaluationRunListQuery))
@console_ns.response(200, "Evaluation runs", console_ns.models[EvaluationRunPaginationResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@with_session(write=False)
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
def get(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
_, tenant_id = current_account_with_tenant()
query = EvaluationRunListQuery.model_validate(request.args.to_dict(flat=True))
runs, total = EvaluationService.get_evaluation_runs(
session,
tenant_id,
target_type,
target.id,
query.page,
query.page_size,
)
return dump_response(
EvaluationRunPaginationResponse,
{"data": runs, "total": total, "page": query.page, "page_size": query.page_size},
)
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/run")
class EvaluationRunApi(EvaluationEditionResource):
@console_ns.expect(console_ns.models[EvaluationRunPayload.__name__])
@console_ns.response(200, "Evaluation run started", console_ns.models[EvaluationRunResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@edit_permission_required
@with_session
@evaluation_target_required(
app_permission=RBACPermission.APP_TEST_AND_RUN,
snippet_permission=RBACPermission.SNIPPETS_CREATE_AND_MODIFY,
)
def post(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
current_user, tenant_id = current_account_with_tenant()
payload = EvaluationRunPayload.model_validate(console_ns.payload or {})
run = _start_run(
session=session,
tenant_id=tenant_id,
target_type=target_type,
target_id=target.id,
account_id=current_user.id,
payload=payload,
)
return dump_response(EvaluationRunResponse, run)
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/runs/<uuid:run_id>")
class EvaluationRunDetailApi(EvaluationEditionResource):
@console_ns.doc(params=query_params_from_model(EvaluationRunDetailQuery))
@console_ns.response(200, "Evaluation run detail", console_ns.models[EvaluationRunDetailResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@with_session(write=False)
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
def get(
self,
session: Session,
target: EvaluationTarget,
target_type: str,
run_id: UUID,
) -> dict[str, Any]:
_, tenant_id = current_account_with_tenant()
query = EvaluationRunDetailQuery.model_validate(request.args.to_dict(flat=True))
return _run_detail(
session=session,
tenant_id=tenant_id,
target_type=target_type,
target_id=target.id,
run_id=str(run_id),
query=query,
)
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/runs/<uuid:run_id>/cancel")
class EvaluationRunCancelApi(EvaluationEditionResource):
@console_ns.response(200, "Evaluation run cancelled", console_ns.models[EvaluationRunResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@edit_permission_required
@with_session
@evaluation_target_required(
app_permission=RBACPermission.APP_TEST_AND_RUN,
snippet_permission=RBACPermission.SNIPPETS_CREATE_AND_MODIFY,
)
def post(
self,
session: Session,
target: EvaluationTarget,
target_type: str,
run_id: UUID,
) -> dict[str, Any]:
_, tenant_id = current_account_with_tenant()
return _cancel_run(
session=session,
tenant_id=tenant_id,
target_type=target_type,
target_id=target.id,
run_id=str(run_id),
)
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/runs/<uuid:run_id>/retry")
class EvaluationRunRetryApi(EvaluationEditionResource):
@console_ns.response(200, "Evaluation run retry scheduled", console_ns.models[EvaluationRunResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@edit_permission_required
@with_session
@evaluation_target_required(
app_permission=RBACPermission.APP_TEST_AND_RUN,
snippet_permission=RBACPermission.SNIPPETS_CREATE_AND_MODIFY,
)
def post(
self,
session: Session,
target: EvaluationTarget,
target_type: str,
run_id: UUID,
) -> dict[str, Any]:
"""Admit a retry using the current caller and target permissions."""
current_user, tenant_id = current_account_with_tenant()
return _retry_run(
session=session,
tenant_id=tenant_id,
target_type=target_type,
target_id=target.id,
run_id=str(run_id),
account_id=current_user.id,
)
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/metrics")
class EvaluationMetricsApi(EvaluationEditionResource):
@console_ns.doc(params=query_params_from_model(EvaluationFrameworkQuery))
@console_ns.response(200, "Evaluation metrics", console_ns.models[EvaluationMetricsResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@with_session(write=False)
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
def get(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
query = EvaluationFrameworkQuery.model_validate(request.args.to_dict(flat=True))
metrics = {
category.value: EvaluationService.get_console_supported_metrics(category, query.evaluation_framework)
for category in (EvaluationCategory.LLM, EvaluationCategory.RETRIEVAL, EvaluationCategory.AGENT)
}
return dump_response(EvaluationMetricsResponse, {"metrics": metrics})
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/default-metrics")
class EvaluationDefaultMetricsApi(EvaluationEditionResource):
@console_ns.doc(params=query_params_from_model(EvaluationFrameworkQuery))
@console_ns.response(200, "Default metrics", console_ns.models[EvaluationDefaultMetricsResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@with_session(write=False)
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
def get(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
query = EvaluationFrameworkQuery.model_validate(request.args.to_dict(flat=True))
metrics = EvaluationService.get_default_metrics_with_nodes_for_published_target(
target,
target_type,
session,
query.evaluation_framework,
)
return dump_response(
EvaluationDefaultMetricsResponse,
{"default_metrics": EvaluationService.filter_console_default_metrics(metrics)},
)
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/node-info")
class EvaluationNodeInfoApi(EvaluationEditionResource):
@console_ns.expect(console_ns.models[EvaluationNodeInfoPayload.__name__])
@console_ns.response(200, "Evaluation node info", console_ns.models[EvaluationNodeInfoResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@with_session(write=False)
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
def post(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
payload = EvaluationNodeInfoPayload.model_validate(console_ns.payload or {})
nodes = EvaluationService.get_nodes_for_metrics(
target,
target_type,
session,
payload.metrics or None,
)
return EvaluationNodeInfoResponse.model_validate(nodes).model_dump(mode="json")
@console_ns.route("/evaluation/available-metrics")
class EvaluationAvailableMetricsApi(EvaluationEditionResource):
@console_ns.doc(params=query_params_from_model(EvaluationFrameworkQuery))
@console_ns.response(
200,
"Available evaluation metrics",
console_ns.models[EvaluationAvailableMetricsResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
def get(self) -> dict[str, Any]:
query = EvaluationFrameworkQuery.model_validate(request.args.to_dict(flat=True))
return dump_response(
EvaluationAvailableMetricsResponse,
{"metrics": EvaluationService.get_available_metrics(query.evaluation_framework)},
)
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/files/<uuid:file_id>")
class EvaluationFileDownloadApi(EvaluationEditionResource):
@console_ns.response(200, "Evaluation file", console_ns.models[EvaluationFileResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@with_session(write=False)
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
def get(
self,
session: Session,
target: EvaluationTarget,
target_type: str,
file_id: UUID,
) -> dict[str, Any]:
_, tenant_id = current_account_with_tenant()
return _evaluation_file_response(
session=session,
tenant_id=tenant_id,
target_type=target_type,
target_id=target.id,
file_id=str(file_id),
)
@console_ns.route("/workspaces/current/available-evaluation-workflows")
class AvailableEvaluationWorkflowsApi(EvaluationEditionResource):
@console_ns.doc(params=query_params_from_model(AvailableEvaluationWorkflowQuery))
@console_ns.response(
200,
"Available evaluation workflows",
console_ns.models[AvailableEvaluationWorkflowPaginationResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@edit_permission_required
@with_session(write=False)
def get(self, session: Session) -> dict[str, Any]:
"""Return only evaluator Apps this caller may run, with filtered pagination."""
current_user, tenant_id = current_account_with_tenant()
query = AvailableEvaluationWorkflowQuery.model_validate(request.args.to_dict(flat=True))
if query.user_id and query.user_id != current_user.id:
raise Forbidden()
run_access = resolve_app_run_access_filter(tenant_id, current_user.id) if dify_config.RBAC_ENABLED else None
workflows, has_more = WorkflowService().list_published_evaluation_workflows(
session=session,
tenant_id=tenant_id,
page=query.page,
limit=query.limit,
user_id=query.user_id,
named_only=query.named_only,
keyword=query.keyword,
run_access=run_access,
)
app_names = EvaluationService.get_workflow_app_names(
session, tenant_id, {workflow.app_id for workflow in workflows}
)
items = [
{
"id": workflow.id,
"app_id": workflow.app_id,
"app_name": app_names.get(workflow.app_id, ""),
"type": workflow.type.value,
"kind": workflow.resolved_kind.value,
"version": workflow.version,
"marked_name": workflow.marked_name,
"marked_comment": workflow.marked_comment,
"hash": workflow.unique_hash,
"created_by": workflow.get_created_by_account(session=session),
"created_at": workflow.created_at,
"updated_by": workflow.get_updated_by_account(session=session),
"updated_at": workflow.updated_at,
}
for workflow in workflows
]
return dump_response(
AvailableEvaluationWorkflowPaginationResponse,
{"items": items, "page": query.page, "limit": query.limit, "has_more": has_more},
)
@console_ns.route("/workspaces/current/evaluation-workflows/<uuid:workflow_id>/associated-targets")
class EvaluationWorkflowAssociatedTargetsApi(EvaluationEditionResource):
@console_ns.response(
200,
"Associated evaluation targets",
console_ns.models[EvaluationAssociatedTargetListResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@edit_permission_required
@with_session(write=False)
def get(self, session: Session, workflow_id: UUID) -> dict[str, Any]:
_, tenant_id = current_account_with_tenant()
if EvaluationService.get_published_evaluator_workflow(session, tenant_id, str(workflow_id)) is None:
raise NotFound("Evaluation workflow not found.")
configs = EvaluationService.list_targets_by_customized_workflow(session, tenant_id, str(workflow_id))
ids_by_type: dict[str, set[str]] = {}
for config in configs:
ids_by_type.setdefault(config.target_type.value, set()).add(config.target_id)
names = EvaluationService.get_associated_target_names(session, tenant_id, ids_by_type)
items = [
{
"target_type": config.target_type.value,
"target_id": config.target_id,
"target_name": names.get((config.target_type.value, config.target_id), ""),
}
for config in configs
]
return dump_response(EvaluationAssociatedTargetListResponse, {"items": items})
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/template/download")
class DatasetEvaluationTemplateDownloadApi(EvaluationEditionResource):
@console_ns.doc(
produces=[XLSX_MIME_TYPE],
vendor={BINARY_RESPONSE_MEDIA_TYPES_VENDOR_KEY: [XLSX_MIME_TYPE]},
)
@console_ns.response(
200,
"Knowledge base evaluation template",
console_ns.models[BinaryFileResponse.__name__],
)
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
@with_session(write=False)
def post(self, session: Session, dataset_id: UUID) -> Response:
current_user, tenant_id = current_account_with_tenant()
_get_accessible_dataset(session, dataset_id, current_user, tenant_id)
content, filename = EvaluationService.generate_retrieval_dataset_template(
language=_evaluation_template_language(current_user)
)
return _xlsx_response(content, filename)
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation")
class DatasetEvaluationDetailApi(EvaluationEditionResource):
@console_ns.response(200, "Evaluation configuration", console_ns.models[EvaluationConfigResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
@with_session(write=False)
def get(self, session: Session, dataset_id: UUID) -> dict[str, Any]:
current_user, tenant_id = current_account_with_tenant()
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
config = EvaluationService.get_evaluation_config(
session,
tenant_id,
EvaluationTargetType.KNOWLEDGE_BASE.value,
dataset.id,
)
return _config_response(config, target_type=EvaluationTargetType.KNOWLEDGE_BASE.value)
@console_ns.expect(console_ns.models[EvaluationConfigPayload.__name__])
@console_ns.response(200, "Evaluation configuration saved", console_ns.models[EvaluationConfigResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@edit_permission_required
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))
@with_session
def put(self, session: Session, dataset_id: UUID) -> dict[str, Any]:
current_user, tenant_id = current_account_with_tenant()
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
payload = EvaluationConfigPayload.model_validate(console_ns.payload or {})
try:
config = EvaluationService.save_evaluation_config(
session,
tenant_id,
EvaluationTargetType.KNOWLEDGE_BASE.value,
dataset.id,
current_user.id,
payload,
)
except ValueError as exc:
raise BadRequest(str(exc)) from exc
return _config_response(config, target_type=EvaluationTargetType.KNOWLEDGE_BASE.value)
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/run")
class DatasetEvaluationRunApi(EvaluationEditionResource):
@console_ns.expect(console_ns.models[EvaluationRunPayload.__name__])
@console_ns.response(200, "Evaluation run started", console_ns.models[EvaluationRunResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@edit_permission_required
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
@with_session
def post(self, session: Session, dataset_id: UUID) -> dict[str, Any]:
current_user, tenant_id = current_account_with_tenant()
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
payload = EvaluationRunPayload.model_validate(console_ns.payload or {})
run = _start_run(
session=session,
tenant_id=tenant_id,
target_type=EvaluationTargetType.KNOWLEDGE_BASE.value,
target_id=dataset.id,
account_id=current_user.id,
payload=payload,
)
return dump_response(EvaluationRunResponse, run)
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/logs")
class DatasetEvaluationLogsApi(EvaluationEditionResource):
@console_ns.doc(params=query_params_from_model(EvaluationRunListQuery))
@console_ns.response(200, "Evaluation runs", console_ns.models[EvaluationRunPaginationResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
@with_session(write=False)
def get(self, session: Session, dataset_id: UUID) -> dict[str, Any]:
current_user, tenant_id = current_account_with_tenant()
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
query = EvaluationRunListQuery.model_validate(request.args.to_dict(flat=True))
runs, total = EvaluationService.get_evaluation_runs(
session,
tenant_id,
EvaluationTargetType.KNOWLEDGE_BASE.value,
dataset.id,
query.page,
query.page_size,
)
return dump_response(
EvaluationRunPaginationResponse,
{"data": runs, "total": total, "page": query.page, "page_size": query.page_size},
)
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/runs/<uuid:run_id>")
class DatasetEvaluationRunDetailApi(EvaluationEditionResource):
@console_ns.doc(params=query_params_from_model(EvaluationRunDetailQuery))
@console_ns.response(200, "Evaluation run detail", console_ns.models[EvaluationRunDetailResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
@with_session(write=False)
def get(self, session: Session, dataset_id: UUID, run_id: UUID) -> dict[str, Any]:
current_user, tenant_id = current_account_with_tenant()
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
query = EvaluationRunDetailQuery.model_validate(request.args.to_dict(flat=True))
return _run_detail(
session=session,
tenant_id=tenant_id,
target_type=EvaluationTargetType.KNOWLEDGE_BASE.value,
target_id=dataset.id,
run_id=str(run_id),
query=query,
)
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/runs/<uuid:run_id>/cancel")
class DatasetEvaluationRunCancelApi(EvaluationEditionResource):
@console_ns.response(200, "Evaluation run cancelled", console_ns.models[EvaluationRunResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@edit_permission_required
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
@with_session
def post(self, session: Session, dataset_id: UUID, run_id: UUID) -> dict[str, Any]:
current_user, tenant_id = current_account_with_tenant()
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
return _cancel_run(
session=session,
tenant_id=tenant_id,
target_type=EvaluationTargetType.KNOWLEDGE_BASE.value,
target_id=dataset.id,
run_id=str(run_id),
)
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/runs/<uuid:run_id>/retry")
class DatasetEvaluationRunRetryApi(EvaluationEditionResource):
@console_ns.response(200, "Evaluation run retry scheduled", console_ns.models[EvaluationRunResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@edit_permission_required
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
@with_session
def post(self, session: Session, dataset_id: UUID, run_id: UUID) -> dict[str, Any]:
"""Retry a scoped dataset evaluation using the current caller identity."""
current_user, tenant_id = current_account_with_tenant()
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
return _retry_run(
session=session,
tenant_id=tenant_id,
target_type=EvaluationTargetType.KNOWLEDGE_BASE.value,
target_id=dataset.id,
run_id=str(run_id),
account_id=current_user.id,
)
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/metrics")
class DatasetEvaluationMetricsApi(EvaluationEditionResource):
@console_ns.doc(params=query_params_from_model(EvaluationFrameworkQuery))
@console_ns.response(200, "Retrieval evaluation metrics", console_ns.models[EvaluationMetricsResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
@with_session(write=False)
def get(self, session: Session, dataset_id: UUID) -> dict[str, Any]:
current_user, tenant_id = current_account_with_tenant()
_get_accessible_dataset(session, dataset_id, current_user, tenant_id)
query = EvaluationFrameworkQuery.model_validate(request.args.to_dict(flat=True))
return dump_response(
EvaluationMetricsResponse,
{
"metrics": EvaluationService.get_console_supported_metrics(
EvaluationCategory.RETRIEVAL, query.evaluation_framework
)
},
)
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/files/<uuid:file_id>")
class DatasetEvaluationFileDownloadApi(EvaluationEditionResource):
@console_ns.response(200, "Evaluation file", console_ns.models[EvaluationFileResponse.__name__])
@setup_required
@login_required
@account_initialization_required
@evaluation_paid_plan_required
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
@with_session(write=False)
def get(self, session: Session, dataset_id: UUID, file_id: UUID) -> dict[str, Any]:
current_user, tenant_id = current_account_with_tenant()
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
return _evaluation_file_response(
session=session,
tenant_id=tenant_id,
target_type=EvaluationTargetType.KNOWLEDGE_BASE.value,
target_id=dataset.id,
file_id=str(file_id),
)