mirror of
https://github.com/langgenius/dify.git
synced 2026-11-22 19:18:52 +08:00
1365 lines
51 KiB
Python
1365 lines
51 KiB
Python
from __future__ import annotations
|
|
|
|
from collections.abc import Callable, Mapping
|
|
from datetime import datetime
|
|
from functools import wraps
|
|
from typing import Any, ParamSpec, TypeVar
|
|
from urllib.parse import quote
|
|
from uuid import UUID
|
|
|
|
from flask import Response, request
|
|
from flask_restx import Resource
|
|
from pydantic import AliasChoices, BaseModel, Field, RootModel, field_validator
|
|
from sqlalchemy.orm import Session
|
|
from werkzeug.exceptions import BadRequest, Forbidden, NotFound, TooManyRequests
|
|
|
|
from configs import dify_config
|
|
from constants.languages import get_valid_language, languages
|
|
from controllers.common.fields import BinaryFileResponse
|
|
from controllers.common.rbac import AgentId, DatasetId, PlainApp, RBACCheck, Workspace, enforce_rbac_checks
|
|
from controllers.common.schema import (
|
|
query_params_from_model,
|
|
register_response_schema_models,
|
|
register_schema_models,
|
|
)
|
|
from controllers.common.session import with_session
|
|
from controllers.console import console_ns
|
|
from controllers.console.wraps import (
|
|
RBACPermission,
|
|
account_initialization_required,
|
|
edit_permission_required,
|
|
evaluation_edition_required,
|
|
evaluation_paid_plan_required,
|
|
rbac_permission_required,
|
|
setup_required,
|
|
)
|
|
from core.evaluation.entities.config_entity import EvaluationFrameworkEnum, SelectableEvaluationFramework
|
|
from core.evaluation.entities.evaluation_entity import (
|
|
CustomizedMetrics,
|
|
DefaultMetric,
|
|
EvaluationCategory,
|
|
EvaluationConfigData,
|
|
EvaluationMetric,
|
|
EvaluationRunRequest,
|
|
NodeInfo,
|
|
)
|
|
from core.evaluation.entities.judgment_entity import JudgmentConfig
|
|
from core.evaluation.exceptions import safe_evaluation_error_text
|
|
from extensions.ext_application_services import application_services
|
|
from extensions.ext_storage import storage
|
|
from fields.base import ResponseModel
|
|
from fields.member_fields import SimpleAccount
|
|
from graphon.file import helpers as file_helpers
|
|
from libs.flask_restx_compat import BINARY_RESPONSE_MEDIA_TYPES_VENDOR_KEY
|
|
from libs.helper import dump_response, to_timestamp
|
|
from libs.login import current_account_with_tenant, login_required
|
|
from machinery.context import RequestContext
|
|
from models import Account, App
|
|
from models.agent import Agent
|
|
from models.evaluation import EvaluationConfiguration, EvaluationRun, EvaluationTargetType
|
|
from models.model import AppMode
|
|
from models.snippet import CustomizedSnippet
|
|
from services.app.access import resolve_app_run_access_filter
|
|
from services.errors.base import NoPermissionError
|
|
from services.errors.evaluation import (
|
|
EvaluationDatasetInvalidError,
|
|
EvaluationFrameworkNotConfiguredError,
|
|
EvaluationMaxConcurrentRunsError,
|
|
EvaluationNotFoundError,
|
|
)
|
|
from services.evaluation_service import EvaluationService
|
|
from services.knowledge.dataset_access import AccessibleDataset, DatasetAccessDeniedError, DatasetNotFoundError
|
|
from services.workflow_service import WorkflowService
|
|
|
|
P = ParamSpec("P")
|
|
R = TypeVar("R")
|
|
EvaluationTarget = App | CustomizedSnippet | Agent
|
|
XLSX_MIME_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
|
|
|
|
|
|
class EvaluationEditionResource(Resource):
|
|
method_decorators = [evaluation_edition_required]
|
|
|
|
|
|
class EvaluationConfigPayload(EvaluationConfigData):
|
|
pass
|
|
|
|
|
|
class EvaluationRunPayload(EvaluationRunRequest):
|
|
pass
|
|
|
|
|
|
class EvaluationFrameworkQuery(BaseModel):
|
|
evaluation_framework: SelectableEvaluationFramework = EvaluationFrameworkEnum.RAGAS
|
|
|
|
|
|
class EvaluationNodeInfoPayload(BaseModel):
|
|
metrics: list[str] = Field(default_factory=list)
|
|
|
|
|
|
class EvaluationRunListQuery(BaseModel):
|
|
page: int = Field(default=1, ge=1, le=99999)
|
|
page_size: int = Field(default=20, ge=1, le=100)
|
|
|
|
|
|
class EvaluationRunDetailQuery(BaseModel):
|
|
page: int = Field(default=1, ge=1, le=99999)
|
|
page_size: int = Field(default=50, ge=1, le=100)
|
|
|
|
|
|
class AvailableEvaluationWorkflowQuery(BaseModel):
|
|
page: int = Field(default=1, ge=1, le=99999)
|
|
limit: int = Field(default=10, ge=1, le=100)
|
|
user_id: str | None = None
|
|
named_only: bool = False
|
|
keyword: str | None = Field(default=None, max_length=255)
|
|
|
|
|
|
class EvaluationConfigResponse(ResponseModel):
|
|
evaluation_framework: SelectableEvaluationFramework = EvaluationFrameworkEnum.RAGAS
|
|
evaluation_model: str | None = None
|
|
evaluation_model_provider: str | None = None
|
|
default_metrics: list[DefaultMetric] | None = None
|
|
customized_metrics: CustomizedMetrics | None = None
|
|
judgment_config: JudgmentConfig | None = None
|
|
|
|
|
|
class EvaluationTemplateColumnResponse(ResponseModel):
|
|
name: str
|
|
type: str
|
|
|
|
|
|
class EvaluationTemplateColumnsResponse(ResponseModel):
|
|
columns: list[EvaluationTemplateColumnResponse]
|
|
|
|
|
|
class EvaluationRunResponse(ResponseModel):
|
|
id: str
|
|
tenant_id: str
|
|
target_type: str
|
|
target_id: str
|
|
target_workflow_version_id: str | None = None
|
|
target_agent_config_snapshot_id: str | None = None
|
|
evaluation_config_id: str
|
|
status: str
|
|
dataset_file_id: str | None = None
|
|
result_file_id: str | None = None
|
|
total_items: int
|
|
completed_items: int
|
|
failed_items: int
|
|
progress: float
|
|
metrics_summary: dict[str, Any] = Field(
|
|
default_factory=dict,
|
|
validation_alias=AliasChoices("metrics_summary_dict", "metrics_summary"),
|
|
)
|
|
error: str | None = None
|
|
created_by: str
|
|
started_at: int | None = None
|
|
completed_at: int | None = None
|
|
created_at: int | None = None
|
|
|
|
@field_validator("error", mode="before")
|
|
@classmethod
|
|
def _safe_error(cls, value: str | None) -> str | None:
|
|
"""Keep only approved diagnostics in run API responses.
|
|
|
|
Precondition: value comes from a persisted run and may contain legacy text.
|
|
Postcondition: the response never returns an unapproved raw exception.
|
|
"""
|
|
return safe_evaluation_error_text(value)
|
|
|
|
@field_validator("started_at", "completed_at", "created_at", mode="before")
|
|
@classmethod
|
|
def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
|
|
return to_timestamp(value)
|
|
|
|
|
|
class EvaluationRunItemResponse(ResponseModel):
|
|
id: str
|
|
item_index: int
|
|
workflow_run_id: str | None = None
|
|
inputs: dict[str, Any] = Field(default_factory=dict, validation_alias=AliasChoices("inputs_dict", "inputs"))
|
|
expected_output: str | None = None
|
|
actual_output: str | None = None
|
|
metrics: list[EvaluationMetric] = Field(
|
|
default_factory=list,
|
|
validation_alias=AliasChoices("metrics_list", "metrics"),
|
|
)
|
|
judgment: dict[str, Any] = Field(
|
|
default_factory=dict,
|
|
validation_alias=AliasChoices("judgment_dict", "judgment"),
|
|
)
|
|
metadata: dict[str, Any] = Field(
|
|
default_factory=dict,
|
|
validation_alias=AliasChoices("metadata_dict", "metadata"),
|
|
)
|
|
error: str | None = None
|
|
overall_score: float | None = None
|
|
|
|
@field_validator("error", mode="before")
|
|
@classmethod
|
|
def _safe_error(cls, value: str | None) -> str | None:
|
|
"""Keep only approved diagnostics in item API responses.
|
|
|
|
Precondition: value comes from a persisted row and may contain legacy text.
|
|
Postcondition: the response never returns an unapproved raw exception.
|
|
"""
|
|
return safe_evaluation_error_text(value)
|
|
|
|
@field_validator("judgment", mode="before")
|
|
@classmethod
|
|
def _safe_judgment_errors(cls, value: dict[str, Any]) -> dict[str, Any]:
|
|
"""Mask legacy exception text in nested condition diagnostics.
|
|
|
|
Precondition: ``value`` is a decoded judgment object from a run item.
|
|
Postcondition: condition errors contain only approved public text and
|
|
all other judgment fields retain their original values.
|
|
"""
|
|
conditions = value.get("condition_results")
|
|
if not isinstance(conditions, list):
|
|
return value
|
|
return {
|
|
**value,
|
|
"condition_results": [
|
|
{**condition, "error": safe_evaluation_error_text(condition.get("error"))}
|
|
if isinstance(condition, Mapping)
|
|
else condition
|
|
for condition in conditions
|
|
],
|
|
}
|
|
|
|
|
|
class EvaluationRunPaginationResponse(ResponseModel):
|
|
data: list[EvaluationRunResponse]
|
|
total: int
|
|
page: int
|
|
page_size: int
|
|
|
|
|
|
class EvaluationRunItemPaginationResponse(ResponseModel):
|
|
data: list[EvaluationRunItemResponse]
|
|
total: int
|
|
page: int
|
|
page_size: int
|
|
|
|
|
|
class EvaluationRunDetailResponse(ResponseModel):
|
|
run: EvaluationRunResponse
|
|
items: EvaluationRunItemPaginationResponse
|
|
|
|
|
|
class EvaluationMetricsResponse(ResponseModel):
|
|
metrics: dict[str, list[str]] | list[str]
|
|
|
|
|
|
class EvaluationDefaultMetricsResponse(ResponseModel):
|
|
default_metrics: list[DefaultMetric]
|
|
|
|
|
|
class EvaluationAvailableMetricsResponse(ResponseModel):
|
|
metrics: list[str]
|
|
|
|
|
|
class EvaluationNodeInfoResponse(RootModel[dict[str, list[NodeInfo]]]):
|
|
pass
|
|
|
|
|
|
class EvaluationFileResponse(ResponseModel):
|
|
id: str
|
|
name: str
|
|
size: int
|
|
extension: str
|
|
mime_type: str
|
|
created_at: int | None = None
|
|
download_url: str
|
|
|
|
@field_validator("created_at", mode="before")
|
|
@classmethod
|
|
def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
|
|
return to_timestamp(value)
|
|
|
|
|
|
class AvailableEvaluationWorkflowResponse(ResponseModel):
|
|
id: str
|
|
app_id: str
|
|
app_name: str
|
|
type: str
|
|
kind: str
|
|
version: str
|
|
marked_name: str
|
|
marked_comment: str
|
|
hash: str
|
|
created_by: SimpleAccount | None = None
|
|
created_at: int
|
|
updated_by: SimpleAccount | None = None
|
|
updated_at: int
|
|
|
|
@field_validator("created_at", "updated_at", mode="before")
|
|
@classmethod
|
|
def _normalize_timestamp(cls, value: datetime | int | None) -> int:
|
|
timestamp = to_timestamp(value)
|
|
if timestamp is None:
|
|
raise ValueError("timestamp is required")
|
|
return timestamp
|
|
|
|
|
|
class AvailableEvaluationWorkflowPaginationResponse(ResponseModel):
|
|
items: list[AvailableEvaluationWorkflowResponse]
|
|
page: int
|
|
limit: int
|
|
has_more: bool
|
|
|
|
|
|
class EvaluationAssociatedTargetResponse(ResponseModel):
|
|
target_type: str
|
|
target_id: str
|
|
target_name: str
|
|
|
|
|
|
class EvaluationAssociatedTargetListResponse(ResponseModel):
|
|
items: list[EvaluationAssociatedTargetResponse]
|
|
|
|
|
|
register_schema_models(
|
|
console_ns,
|
|
EvaluationConfigPayload,
|
|
EvaluationRunPayload,
|
|
EvaluationFrameworkQuery,
|
|
EvaluationNodeInfoPayload,
|
|
EvaluationRunListQuery,
|
|
EvaluationRunDetailQuery,
|
|
AvailableEvaluationWorkflowQuery,
|
|
)
|
|
register_response_schema_models(
|
|
console_ns,
|
|
EvaluationConfigResponse,
|
|
EvaluationTemplateColumnsResponse,
|
|
EvaluationRunResponse,
|
|
EvaluationRunPaginationResponse,
|
|
EvaluationRunDetailResponse,
|
|
EvaluationMetricsResponse,
|
|
EvaluationDefaultMetricsResponse,
|
|
EvaluationAvailableMetricsResponse,
|
|
EvaluationNodeInfoResponse,
|
|
EvaluationFileResponse,
|
|
AvailableEvaluationWorkflowPaginationResponse,
|
|
EvaluationAssociatedTargetListResponse,
|
|
BinaryFileResponse,
|
|
)
|
|
|
|
|
|
def evaluation_target_required(
|
|
*,
|
|
app_permission: RBACPermission,
|
|
snippet_permission: RBACPermission = RBACPermission.SNIPPETS_CREATE_AND_MODIFY,
|
|
agent_permission: RBACPermission = RBACPermission.AGENT_TEST_AND_RUN,
|
|
) -> Callable[[Callable[P, R]], Callable[P, R]]:
|
|
"""Resolve an app/snippet/Agent target with tenant and dynamic RBAC checks."""
|
|
|
|
def decorator(view: Callable[P, R]) -> Callable[P, R]:
|
|
@wraps(view)
|
|
def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
|
|
session = args[1]
|
|
if not isinstance(session, Session):
|
|
raise TypeError("Evaluation endpoint requires a database session.")
|
|
target_type = str(kwargs.pop("evaluate_target_type"))
|
|
target_id = str(kwargs.pop("evaluate_target_id"))
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
|
|
if target_type == EvaluationTargetType.APPS.value:
|
|
enforce_rbac_checks(
|
|
tenant_id=tenant_id,
|
|
account_id=current_user.id,
|
|
checks=[RBACCheck(app_permission, PlainApp("evaluate_target_id"))],
|
|
path_args={"evaluate_target_id": target_id},
|
|
)
|
|
elif target_type == EvaluationTargetType.SNIPPETS.value:
|
|
enforce_rbac_checks(
|
|
tenant_id=tenant_id,
|
|
account_id=current_user.id,
|
|
checks=[RBACCheck(snippet_permission, Workspace())],
|
|
)
|
|
elif target_type == EvaluationTargetType.AGENTS.value:
|
|
enforce_rbac_checks(
|
|
tenant_id=tenant_id,
|
|
account_id=current_user.id,
|
|
checks=[RBACCheck(agent_permission, AgentId("evaluate_target_id"))],
|
|
path_args={"evaluate_target_id": target_id},
|
|
)
|
|
else:
|
|
raise NotFound("Evaluation target not found.")
|
|
|
|
target: EvaluationTarget | None = EvaluationService.get_console_target(
|
|
session, tenant_id, target_type, target_id
|
|
)
|
|
if target is None:
|
|
raise NotFound("Evaluation target not found.")
|
|
if isinstance(target, App) and AppMode.value_of(target.mode) != AppMode.WORKFLOW:
|
|
raise BadRequest("Only workflow apps support evaluation.")
|
|
|
|
kwargs["target"] = target
|
|
kwargs["target_type"] = target_type
|
|
return view(*args, **kwargs)
|
|
|
|
return wrapper
|
|
|
|
return decorator
|
|
|
|
|
|
def _get_accessible_dataset(
|
|
session: Session,
|
|
dataset_id: UUID,
|
|
current_user: Account,
|
|
tenant_id: str,
|
|
) -> AccessibleDataset:
|
|
"""Authorize the admitted account and return a tenant-scoped dataset identity.
|
|
|
|
The existing evaluation transaction stays caller-owned. Knowledge admission
|
|
uses its configured application service and raises the native HTTP errors.
|
|
"""
|
|
try:
|
|
ref = application_services().knowledge.datasets.require_accessible(
|
|
RequestContext("", None, current_user.id, tenant_id), str(dataset_id)
|
|
)
|
|
except DatasetNotFoundError as exc:
|
|
raise NotFound("Dataset not found.") from exc
|
|
except DatasetAccessDeniedError as exc:
|
|
raise Forbidden(str(exc)) from exc
|
|
return AccessibleDataset(ref.dataset_id, ref.tenant_id)
|
|
|
|
|
|
def _config_response(config: EvaluationConfiguration | None, *, target_type: str) -> dict[str, Any]:
|
|
if config is None:
|
|
return dump_response(EvaluationConfigResponse, {})
|
|
|
|
default_metrics = config.default_metrics_list
|
|
if target_type != EvaluationTargetType.KNOWLEDGE_BASE.value:
|
|
default_metrics = EvaluationService.serialize_console_default_metrics(default_metrics)
|
|
|
|
return dump_response(
|
|
EvaluationConfigResponse,
|
|
{
|
|
"evaluation_framework": config.evaluation_framework,
|
|
"evaluation_model": config.evaluation_model,
|
|
"evaluation_model_provider": config.evaluation_model_provider,
|
|
"default_metrics": default_metrics,
|
|
"customized_metrics": config.customized_metrics_dict,
|
|
"judgment_config": EvaluationService.serialize_console_judgment_config(config.judgment_config_dict),
|
|
},
|
|
)
|
|
|
|
|
|
def _load_evaluation_dataset(
|
|
*,
|
|
session: Session,
|
|
tenant_id: str,
|
|
file_id: str,
|
|
) -> tuple[bytes, str]:
|
|
upload_file = EvaluationService.get_dataset_upload_file(session, tenant_id, file_id)
|
|
if upload_file is None:
|
|
raise NotFound("Evaluation dataset file not found.")
|
|
try:
|
|
content = storage.load_once(upload_file.key)
|
|
except Exception as exc:
|
|
raise BadRequest("Failed to read evaluation dataset file.") from exc
|
|
if not content:
|
|
raise BadRequest("Evaluation dataset file is empty.")
|
|
return content, upload_file.name
|
|
|
|
|
|
def _start_run(
|
|
*,
|
|
session: Session,
|
|
tenant_id: str,
|
|
target_type: str,
|
|
target_id: str,
|
|
account_id: str,
|
|
payload: EvaluationRunPayload,
|
|
) -> EvaluationRun:
|
|
"""Admit a target run for the caller and map service denials to HTTP 403 without dispatching."""
|
|
dataset_content, dataset_filename = _load_evaluation_dataset(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
file_id=payload.file_id,
|
|
)
|
|
try:
|
|
return EvaluationService.start_evaluation_run(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
target_type=target_type,
|
|
target_id=target_id,
|
|
account_id=account_id,
|
|
dataset_file_content=dataset_content,
|
|
dataset_filename=dataset_filename,
|
|
run_request=payload,
|
|
)
|
|
except EvaluationFrameworkNotConfiguredError as exc:
|
|
raise BadRequest(exc.description) from exc
|
|
except EvaluationDatasetInvalidError as exc:
|
|
raise BadRequest(exc.description) from exc
|
|
except EvaluationNotFoundError as exc:
|
|
raise NotFound(exc.description) from exc
|
|
except EvaluationMaxConcurrentRunsError as exc:
|
|
raise TooManyRequests(exc.description) from exc
|
|
except NoPermissionError as exc:
|
|
raise Forbidden(str(exc)) from exc
|
|
except ValueError as exc:
|
|
raise BadRequest(str(exc)) from exc
|
|
|
|
|
|
def _run_detail(
|
|
*,
|
|
session: Session,
|
|
tenant_id: str,
|
|
target_type: str,
|
|
target_id: str,
|
|
run_id: str,
|
|
query: EvaluationRunDetailQuery,
|
|
) -> dict[str, Any]:
|
|
try:
|
|
run = EvaluationService.get_evaluation_run_detail(
|
|
session,
|
|
tenant_id,
|
|
target_type,
|
|
target_id,
|
|
run_id,
|
|
)
|
|
items, total = EvaluationService.get_evaluation_run_items(
|
|
session,
|
|
tenant_id,
|
|
target_type,
|
|
target_id,
|
|
run_id,
|
|
query.page,
|
|
query.page_size,
|
|
)
|
|
except EvaluationNotFoundError as exc:
|
|
raise NotFound(exc.description) from exc
|
|
|
|
return dump_response(
|
|
EvaluationRunDetailResponse,
|
|
{
|
|
"run": run,
|
|
"items": {
|
|
"data": items,
|
|
"total": total,
|
|
"page": query.page,
|
|
"page_size": query.page_size,
|
|
},
|
|
},
|
|
)
|
|
|
|
|
|
def _cancel_run(
|
|
*,
|
|
session: Session,
|
|
tenant_id: str,
|
|
target_type: str,
|
|
target_id: str,
|
|
run_id: str,
|
|
) -> dict[str, Any]:
|
|
try:
|
|
run = EvaluationService.cancel_evaluation_run(
|
|
session,
|
|
tenant_id,
|
|
target_type,
|
|
target_id,
|
|
run_id,
|
|
)
|
|
except EvaluationNotFoundError as exc:
|
|
raise NotFound(exc.description) from exc
|
|
except ValueError as exc:
|
|
raise BadRequest(str(exc)) from exc
|
|
return dump_response(EvaluationRunResponse, run)
|
|
|
|
|
|
def _retry_run(
|
|
*,
|
|
session: Session,
|
|
tenant_id: str,
|
|
target_type: str,
|
|
target_id: str,
|
|
run_id: str,
|
|
account_id: str,
|
|
) -> dict[str, Any]:
|
|
"""Retry a scoped run for the caller; permission denial leaves the failed run unchanged."""
|
|
try:
|
|
existing_run = EvaluationService.get_evaluation_run_detail(
|
|
session,
|
|
tenant_id,
|
|
target_type,
|
|
target_id,
|
|
run_id,
|
|
)
|
|
if not existing_run.dataset_file_id:
|
|
raise BadRequest("Evaluation run has no dataset file to retry.")
|
|
dataset_content, dataset_filename = _load_evaluation_dataset(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
file_id=existing_run.dataset_file_id,
|
|
)
|
|
run = EvaluationService.retry_evaluation_run(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
target_type=target_type,
|
|
target_id=target_id,
|
|
run_id=run_id,
|
|
account_id=account_id,
|
|
dataset_file_content=dataset_content,
|
|
dataset_filename=dataset_filename,
|
|
)
|
|
except EvaluationFrameworkNotConfiguredError as exc:
|
|
raise BadRequest(exc.description) from exc
|
|
except EvaluationDatasetInvalidError as exc:
|
|
raise BadRequest(exc.description) from exc
|
|
except EvaluationNotFoundError as exc:
|
|
raise NotFound(exc.description) from exc
|
|
except EvaluationMaxConcurrentRunsError as exc:
|
|
raise TooManyRequests(exc.description) from exc
|
|
except NoPermissionError as exc:
|
|
raise Forbidden(str(exc)) from exc
|
|
except ValueError as exc:
|
|
raise BadRequest(str(exc)) from exc
|
|
return dump_response(EvaluationRunResponse, run)
|
|
|
|
|
|
def _evaluation_file_response(
|
|
*,
|
|
session: Session,
|
|
tenant_id: str,
|
|
target_type: str,
|
|
target_id: str,
|
|
file_id: str,
|
|
) -> dict[str, Any]:
|
|
try:
|
|
upload_file = EvaluationService.get_evaluation_file(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
target_type=target_type,
|
|
target_id=target_id,
|
|
file_id=file_id,
|
|
)
|
|
except EvaluationNotFoundError as exc:
|
|
raise NotFound(exc.description) from exc
|
|
return dump_response(
|
|
EvaluationFileResponse,
|
|
{
|
|
"id": upload_file.id,
|
|
"name": upload_file.name,
|
|
"size": upload_file.size,
|
|
"extension": upload_file.extension,
|
|
"mime_type": upload_file.mime_type,
|
|
"created_at": upload_file.created_at,
|
|
"download_url": file_helpers.get_signed_file_url(upload_file_id=upload_file.id, as_attachment=True),
|
|
},
|
|
)
|
|
|
|
|
|
def _xlsx_response(content: bytes, filename: str) -> Response:
|
|
response = Response(
|
|
content,
|
|
mimetype=XLSX_MIME_TYPE,
|
|
)
|
|
response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{quote(filename)}"
|
|
response.headers["Content-Length"] = str(len(content))
|
|
return response
|
|
|
|
|
|
def _evaluation_template_language(account: Account) -> str:
|
|
"""Use the current UI locale when valid, then the saved account language."""
|
|
cookie_locale = request.cookies.get("locale")
|
|
return cookie_locale if cookie_locale in languages else get_valid_language(account.interface_language)
|
|
|
|
|
|
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/dataset-template/download")
|
|
class EvaluationDatasetTemplateDownloadApi(EvaluationEditionResource):
|
|
@console_ns.expect(console_ns.models[EvaluationConfigPayload.__name__])
|
|
@console_ns.doc(
|
|
produces=[XLSX_MIME_TYPE],
|
|
vendor={BINARY_RESPONSE_MEDIA_TYPES_VENDOR_KEY: [XLSX_MIME_TYPE]},
|
|
)
|
|
@console_ns.response(
|
|
200,
|
|
"Evaluation dataset template",
|
|
console_ns.models[BinaryFileResponse.__name__],
|
|
)
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@edit_permission_required
|
|
@with_session(write=False)
|
|
@evaluation_target_required(
|
|
app_permission=RBACPermission.APP_VIEW_LAYOUT,
|
|
snippet_permission=RBACPermission.SNIPPETS_CREATE_AND_MODIFY,
|
|
)
|
|
def post(self, session: Session, target: EvaluationTarget, target_type: str) -> Response:
|
|
current_user, _ = current_account_with_tenant()
|
|
payload = EvaluationConfigPayload.model_validate(console_ns.payload or {})
|
|
try:
|
|
content, filename = EvaluationService.generate_dataset_template(
|
|
target, target_type, payload, session, language=_evaluation_template_language(current_user)
|
|
)
|
|
except ValueError as exc:
|
|
raise BadRequest(str(exc)) from exc
|
|
return _xlsx_response(content, filename)
|
|
|
|
|
|
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation")
|
|
class EvaluationDetailApi(EvaluationEditionResource):
|
|
@console_ns.response(200, "Evaluation configuration", console_ns.models[EvaluationConfigResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@with_session(write=False)
|
|
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
|
|
def get(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
|
|
_, tenant_id = current_account_with_tenant()
|
|
config = EvaluationService.get_evaluation_config(session, tenant_id, target_type, target.id)
|
|
return _config_response(config, target_type=target_type)
|
|
|
|
@console_ns.expect(console_ns.models[EvaluationConfigPayload.__name__])
|
|
@console_ns.response(200, "Evaluation configuration saved", console_ns.models[EvaluationConfigResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@edit_permission_required
|
|
@with_session
|
|
@evaluation_target_required(
|
|
app_permission=RBACPermission.APP_EDIT,
|
|
snippet_permission=RBACPermission.SNIPPETS_CREATE_AND_MODIFY,
|
|
)
|
|
def put(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
payload = EvaluationConfigPayload.model_validate(console_ns.payload or {})
|
|
try:
|
|
config = EvaluationService.save_evaluation_config(
|
|
session,
|
|
tenant_id,
|
|
target_type,
|
|
target.id,
|
|
current_user.id,
|
|
payload,
|
|
)
|
|
except ValueError as exc:
|
|
raise BadRequest(str(exc)) from exc
|
|
return _config_response(config, target_type=target_type)
|
|
|
|
|
|
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/template-columns")
|
|
class EvaluationTemplateColumnsApi(EvaluationEditionResource):
|
|
@console_ns.expect(console_ns.models[EvaluationConfigPayload.__name__])
|
|
@console_ns.response(
|
|
200,
|
|
"Evaluation template columns",
|
|
console_ns.models[EvaluationTemplateColumnsResponse.__name__],
|
|
)
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@with_session(write=False)
|
|
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
|
|
def post(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
|
|
payload = EvaluationConfigPayload.model_validate(console_ns.payload or {})
|
|
try:
|
|
columns = EvaluationService.get_dataset_column_names(target, target_type, payload, session)
|
|
except ValueError as exc:
|
|
raise BadRequest(str(exc)) from exc
|
|
return dump_response(EvaluationTemplateColumnsResponse, {"columns": columns})
|
|
|
|
|
|
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/logs")
|
|
class EvaluationLogsApi(EvaluationEditionResource):
|
|
@console_ns.doc(params=query_params_from_model(EvaluationRunListQuery))
|
|
@console_ns.response(200, "Evaluation runs", console_ns.models[EvaluationRunPaginationResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@with_session(write=False)
|
|
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
|
|
def get(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
|
|
_, tenant_id = current_account_with_tenant()
|
|
query = EvaluationRunListQuery.model_validate(request.args.to_dict(flat=True))
|
|
runs, total = EvaluationService.get_evaluation_runs(
|
|
session,
|
|
tenant_id,
|
|
target_type,
|
|
target.id,
|
|
query.page,
|
|
query.page_size,
|
|
)
|
|
return dump_response(
|
|
EvaluationRunPaginationResponse,
|
|
{"data": runs, "total": total, "page": query.page, "page_size": query.page_size},
|
|
)
|
|
|
|
|
|
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/run")
|
|
class EvaluationRunApi(EvaluationEditionResource):
|
|
@console_ns.expect(console_ns.models[EvaluationRunPayload.__name__])
|
|
@console_ns.response(200, "Evaluation run started", console_ns.models[EvaluationRunResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@edit_permission_required
|
|
@with_session
|
|
@evaluation_target_required(
|
|
app_permission=RBACPermission.APP_TEST_AND_RUN,
|
|
snippet_permission=RBACPermission.SNIPPETS_CREATE_AND_MODIFY,
|
|
)
|
|
def post(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
payload = EvaluationRunPayload.model_validate(console_ns.payload or {})
|
|
run = _start_run(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
target_type=target_type,
|
|
target_id=target.id,
|
|
account_id=current_user.id,
|
|
payload=payload,
|
|
)
|
|
return dump_response(EvaluationRunResponse, run)
|
|
|
|
|
|
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/runs/<uuid:run_id>")
|
|
class EvaluationRunDetailApi(EvaluationEditionResource):
|
|
@console_ns.doc(params=query_params_from_model(EvaluationRunDetailQuery))
|
|
@console_ns.response(200, "Evaluation run detail", console_ns.models[EvaluationRunDetailResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@with_session(write=False)
|
|
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
|
|
def get(
|
|
self,
|
|
session: Session,
|
|
target: EvaluationTarget,
|
|
target_type: str,
|
|
run_id: UUID,
|
|
) -> dict[str, Any]:
|
|
_, tenant_id = current_account_with_tenant()
|
|
query = EvaluationRunDetailQuery.model_validate(request.args.to_dict(flat=True))
|
|
return _run_detail(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
target_type=target_type,
|
|
target_id=target.id,
|
|
run_id=str(run_id),
|
|
query=query,
|
|
)
|
|
|
|
|
|
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/runs/<uuid:run_id>/cancel")
|
|
class EvaluationRunCancelApi(EvaluationEditionResource):
|
|
@console_ns.response(200, "Evaluation run cancelled", console_ns.models[EvaluationRunResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@edit_permission_required
|
|
@with_session
|
|
@evaluation_target_required(
|
|
app_permission=RBACPermission.APP_TEST_AND_RUN,
|
|
snippet_permission=RBACPermission.SNIPPETS_CREATE_AND_MODIFY,
|
|
)
|
|
def post(
|
|
self,
|
|
session: Session,
|
|
target: EvaluationTarget,
|
|
target_type: str,
|
|
run_id: UUID,
|
|
) -> dict[str, Any]:
|
|
_, tenant_id = current_account_with_tenant()
|
|
return _cancel_run(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
target_type=target_type,
|
|
target_id=target.id,
|
|
run_id=str(run_id),
|
|
)
|
|
|
|
|
|
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/runs/<uuid:run_id>/retry")
|
|
class EvaluationRunRetryApi(EvaluationEditionResource):
|
|
@console_ns.response(200, "Evaluation run retry scheduled", console_ns.models[EvaluationRunResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@edit_permission_required
|
|
@with_session
|
|
@evaluation_target_required(
|
|
app_permission=RBACPermission.APP_TEST_AND_RUN,
|
|
snippet_permission=RBACPermission.SNIPPETS_CREATE_AND_MODIFY,
|
|
)
|
|
def post(
|
|
self,
|
|
session: Session,
|
|
target: EvaluationTarget,
|
|
target_type: str,
|
|
run_id: UUID,
|
|
) -> dict[str, Any]:
|
|
"""Admit a retry using the current caller and target permissions."""
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
return _retry_run(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
target_type=target_type,
|
|
target_id=target.id,
|
|
run_id=str(run_id),
|
|
account_id=current_user.id,
|
|
)
|
|
|
|
|
|
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/metrics")
|
|
class EvaluationMetricsApi(EvaluationEditionResource):
|
|
@console_ns.doc(params=query_params_from_model(EvaluationFrameworkQuery))
|
|
@console_ns.response(200, "Evaluation metrics", console_ns.models[EvaluationMetricsResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@with_session(write=False)
|
|
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
|
|
def get(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
|
|
query = EvaluationFrameworkQuery.model_validate(request.args.to_dict(flat=True))
|
|
metrics = {
|
|
category.value: EvaluationService.get_console_supported_metrics(category, query.evaluation_framework)
|
|
for category in (EvaluationCategory.LLM, EvaluationCategory.RETRIEVAL, EvaluationCategory.AGENT)
|
|
}
|
|
return dump_response(EvaluationMetricsResponse, {"metrics": metrics})
|
|
|
|
|
|
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/default-metrics")
|
|
class EvaluationDefaultMetricsApi(EvaluationEditionResource):
|
|
@console_ns.doc(params=query_params_from_model(EvaluationFrameworkQuery))
|
|
@console_ns.response(200, "Default metrics", console_ns.models[EvaluationDefaultMetricsResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@with_session(write=False)
|
|
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
|
|
def get(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
|
|
query = EvaluationFrameworkQuery.model_validate(request.args.to_dict(flat=True))
|
|
metrics = EvaluationService.get_default_metrics_with_nodes_for_published_target(
|
|
target,
|
|
target_type,
|
|
session,
|
|
query.evaluation_framework,
|
|
)
|
|
return dump_response(
|
|
EvaluationDefaultMetricsResponse,
|
|
{"default_metrics": EvaluationService.filter_console_default_metrics(metrics)},
|
|
)
|
|
|
|
|
|
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/node-info")
|
|
class EvaluationNodeInfoApi(EvaluationEditionResource):
|
|
@console_ns.expect(console_ns.models[EvaluationNodeInfoPayload.__name__])
|
|
@console_ns.response(200, "Evaluation node info", console_ns.models[EvaluationNodeInfoResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@with_session(write=False)
|
|
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
|
|
def post(self, session: Session, target: EvaluationTarget, target_type: str) -> dict[str, Any]:
|
|
payload = EvaluationNodeInfoPayload.model_validate(console_ns.payload or {})
|
|
nodes = EvaluationService.get_nodes_for_metrics(
|
|
target,
|
|
target_type,
|
|
session,
|
|
payload.metrics or None,
|
|
)
|
|
return EvaluationNodeInfoResponse.model_validate(nodes).model_dump(mode="json")
|
|
|
|
|
|
@console_ns.route("/evaluation/available-metrics")
|
|
class EvaluationAvailableMetricsApi(EvaluationEditionResource):
|
|
@console_ns.doc(params=query_params_from_model(EvaluationFrameworkQuery))
|
|
@console_ns.response(
|
|
200,
|
|
"Available evaluation metrics",
|
|
console_ns.models[EvaluationAvailableMetricsResponse.__name__],
|
|
)
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
def get(self) -> dict[str, Any]:
|
|
query = EvaluationFrameworkQuery.model_validate(request.args.to_dict(flat=True))
|
|
return dump_response(
|
|
EvaluationAvailableMetricsResponse,
|
|
{"metrics": EvaluationService.get_available_metrics(query.evaluation_framework)},
|
|
)
|
|
|
|
|
|
@console_ns.route("/<string:evaluate_target_type>/<uuid:evaluate_target_id>/evaluation/files/<uuid:file_id>")
|
|
class EvaluationFileDownloadApi(EvaluationEditionResource):
|
|
@console_ns.response(200, "Evaluation file", console_ns.models[EvaluationFileResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@with_session(write=False)
|
|
@evaluation_target_required(app_permission=RBACPermission.APP_VIEW_LAYOUT)
|
|
def get(
|
|
self,
|
|
session: Session,
|
|
target: EvaluationTarget,
|
|
target_type: str,
|
|
file_id: UUID,
|
|
) -> dict[str, Any]:
|
|
_, tenant_id = current_account_with_tenant()
|
|
return _evaluation_file_response(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
target_type=target_type,
|
|
target_id=target.id,
|
|
file_id=str(file_id),
|
|
)
|
|
|
|
|
|
@console_ns.route("/workspaces/current/available-evaluation-workflows")
|
|
class AvailableEvaluationWorkflowsApi(EvaluationEditionResource):
|
|
@console_ns.doc(params=query_params_from_model(AvailableEvaluationWorkflowQuery))
|
|
@console_ns.response(
|
|
200,
|
|
"Available evaluation workflows",
|
|
console_ns.models[AvailableEvaluationWorkflowPaginationResponse.__name__],
|
|
)
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@edit_permission_required
|
|
@with_session(write=False)
|
|
def get(self, session: Session) -> dict[str, Any]:
|
|
"""Return only evaluator Apps this caller may run, with filtered pagination."""
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
query = AvailableEvaluationWorkflowQuery.model_validate(request.args.to_dict(flat=True))
|
|
if query.user_id and query.user_id != current_user.id:
|
|
raise Forbidden()
|
|
|
|
run_access = resolve_app_run_access_filter(tenant_id, current_user.id) if dify_config.RBAC_ENABLED else None
|
|
workflows, has_more = WorkflowService().list_published_evaluation_workflows(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
page=query.page,
|
|
limit=query.limit,
|
|
user_id=query.user_id,
|
|
named_only=query.named_only,
|
|
keyword=query.keyword,
|
|
run_access=run_access,
|
|
)
|
|
app_names = EvaluationService.get_workflow_app_names(
|
|
session, tenant_id, {workflow.app_id for workflow in workflows}
|
|
)
|
|
items = [
|
|
{
|
|
"id": workflow.id,
|
|
"app_id": workflow.app_id,
|
|
"app_name": app_names.get(workflow.app_id, ""),
|
|
"type": workflow.type.value,
|
|
"kind": workflow.resolved_kind.value,
|
|
"version": workflow.version,
|
|
"marked_name": workflow.marked_name,
|
|
"marked_comment": workflow.marked_comment,
|
|
"hash": workflow.unique_hash,
|
|
"created_by": workflow.get_created_by_account(session=session),
|
|
"created_at": workflow.created_at,
|
|
"updated_by": workflow.get_updated_by_account(session=session),
|
|
"updated_at": workflow.updated_at,
|
|
}
|
|
for workflow in workflows
|
|
]
|
|
return dump_response(
|
|
AvailableEvaluationWorkflowPaginationResponse,
|
|
{"items": items, "page": query.page, "limit": query.limit, "has_more": has_more},
|
|
)
|
|
|
|
|
|
@console_ns.route("/workspaces/current/evaluation-workflows/<uuid:workflow_id>/associated-targets")
|
|
class EvaluationWorkflowAssociatedTargetsApi(EvaluationEditionResource):
|
|
@console_ns.response(
|
|
200,
|
|
"Associated evaluation targets",
|
|
console_ns.models[EvaluationAssociatedTargetListResponse.__name__],
|
|
)
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@edit_permission_required
|
|
@with_session(write=False)
|
|
def get(self, session: Session, workflow_id: UUID) -> dict[str, Any]:
|
|
_, tenant_id = current_account_with_tenant()
|
|
if EvaluationService.get_published_evaluator_workflow(session, tenant_id, str(workflow_id)) is None:
|
|
raise NotFound("Evaluation workflow not found.")
|
|
configs = EvaluationService.list_targets_by_customized_workflow(session, tenant_id, str(workflow_id))
|
|
|
|
ids_by_type: dict[str, set[str]] = {}
|
|
for config in configs:
|
|
ids_by_type.setdefault(config.target_type.value, set()).add(config.target_id)
|
|
|
|
names = EvaluationService.get_associated_target_names(session, tenant_id, ids_by_type)
|
|
|
|
items = [
|
|
{
|
|
"target_type": config.target_type.value,
|
|
"target_id": config.target_id,
|
|
"target_name": names.get((config.target_type.value, config.target_id), ""),
|
|
}
|
|
for config in configs
|
|
]
|
|
return dump_response(EvaluationAssociatedTargetListResponse, {"items": items})
|
|
|
|
|
|
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/template/download")
|
|
class DatasetEvaluationTemplateDownloadApi(EvaluationEditionResource):
|
|
@console_ns.doc(
|
|
produces=[XLSX_MIME_TYPE],
|
|
vendor={BINARY_RESPONSE_MEDIA_TYPES_VENDOR_KEY: [XLSX_MIME_TYPE]},
|
|
)
|
|
@console_ns.response(
|
|
200,
|
|
"Knowledge base evaluation template",
|
|
console_ns.models[BinaryFileResponse.__name__],
|
|
)
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
|
|
@with_session(write=False)
|
|
def post(self, session: Session, dataset_id: UUID) -> Response:
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
_get_accessible_dataset(session, dataset_id, current_user, tenant_id)
|
|
content, filename = EvaluationService.generate_retrieval_dataset_template(
|
|
language=_evaluation_template_language(current_user)
|
|
)
|
|
return _xlsx_response(content, filename)
|
|
|
|
|
|
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation")
|
|
class DatasetEvaluationDetailApi(EvaluationEditionResource):
|
|
@console_ns.response(200, "Evaluation configuration", console_ns.models[EvaluationConfigResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
|
|
@with_session(write=False)
|
|
def get(self, session: Session, dataset_id: UUID) -> dict[str, Any]:
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
|
|
config = EvaluationService.get_evaluation_config(
|
|
session,
|
|
tenant_id,
|
|
EvaluationTargetType.KNOWLEDGE_BASE.value,
|
|
dataset.id,
|
|
)
|
|
return _config_response(config, target_type=EvaluationTargetType.KNOWLEDGE_BASE.value)
|
|
|
|
@console_ns.expect(console_ns.models[EvaluationConfigPayload.__name__])
|
|
@console_ns.response(200, "Evaluation configuration saved", console_ns.models[EvaluationConfigResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@edit_permission_required
|
|
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_EDIT, DatasetId()))
|
|
@with_session
|
|
def put(self, session: Session, dataset_id: UUID) -> dict[str, Any]:
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
|
|
payload = EvaluationConfigPayload.model_validate(console_ns.payload or {})
|
|
try:
|
|
config = EvaluationService.save_evaluation_config(
|
|
session,
|
|
tenant_id,
|
|
EvaluationTargetType.KNOWLEDGE_BASE.value,
|
|
dataset.id,
|
|
current_user.id,
|
|
payload,
|
|
)
|
|
except ValueError as exc:
|
|
raise BadRequest(str(exc)) from exc
|
|
return _config_response(config, target_type=EvaluationTargetType.KNOWLEDGE_BASE.value)
|
|
|
|
|
|
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/run")
|
|
class DatasetEvaluationRunApi(EvaluationEditionResource):
|
|
@console_ns.expect(console_ns.models[EvaluationRunPayload.__name__])
|
|
@console_ns.response(200, "Evaluation run started", console_ns.models[EvaluationRunResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@edit_permission_required
|
|
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
|
|
@with_session
|
|
def post(self, session: Session, dataset_id: UUID) -> dict[str, Any]:
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
|
|
payload = EvaluationRunPayload.model_validate(console_ns.payload or {})
|
|
run = _start_run(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
target_type=EvaluationTargetType.KNOWLEDGE_BASE.value,
|
|
target_id=dataset.id,
|
|
account_id=current_user.id,
|
|
payload=payload,
|
|
)
|
|
return dump_response(EvaluationRunResponse, run)
|
|
|
|
|
|
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/logs")
|
|
class DatasetEvaluationLogsApi(EvaluationEditionResource):
|
|
@console_ns.doc(params=query_params_from_model(EvaluationRunListQuery))
|
|
@console_ns.response(200, "Evaluation runs", console_ns.models[EvaluationRunPaginationResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
|
|
@with_session(write=False)
|
|
def get(self, session: Session, dataset_id: UUID) -> dict[str, Any]:
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
|
|
query = EvaluationRunListQuery.model_validate(request.args.to_dict(flat=True))
|
|
runs, total = EvaluationService.get_evaluation_runs(
|
|
session,
|
|
tenant_id,
|
|
EvaluationTargetType.KNOWLEDGE_BASE.value,
|
|
dataset.id,
|
|
query.page,
|
|
query.page_size,
|
|
)
|
|
return dump_response(
|
|
EvaluationRunPaginationResponse,
|
|
{"data": runs, "total": total, "page": query.page, "page_size": query.page_size},
|
|
)
|
|
|
|
|
|
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/runs/<uuid:run_id>")
|
|
class DatasetEvaluationRunDetailApi(EvaluationEditionResource):
|
|
@console_ns.doc(params=query_params_from_model(EvaluationRunDetailQuery))
|
|
@console_ns.response(200, "Evaluation run detail", console_ns.models[EvaluationRunDetailResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
|
|
@with_session(write=False)
|
|
def get(self, session: Session, dataset_id: UUID, run_id: UUID) -> dict[str, Any]:
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
|
|
query = EvaluationRunDetailQuery.model_validate(request.args.to_dict(flat=True))
|
|
return _run_detail(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
target_type=EvaluationTargetType.KNOWLEDGE_BASE.value,
|
|
target_id=dataset.id,
|
|
run_id=str(run_id),
|
|
query=query,
|
|
)
|
|
|
|
|
|
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/runs/<uuid:run_id>/cancel")
|
|
class DatasetEvaluationRunCancelApi(EvaluationEditionResource):
|
|
@console_ns.response(200, "Evaluation run cancelled", console_ns.models[EvaluationRunResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@edit_permission_required
|
|
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
|
|
@with_session
|
|
def post(self, session: Session, dataset_id: UUID, run_id: UUID) -> dict[str, Any]:
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
|
|
return _cancel_run(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
target_type=EvaluationTargetType.KNOWLEDGE_BASE.value,
|
|
target_id=dataset.id,
|
|
run_id=str(run_id),
|
|
)
|
|
|
|
|
|
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/runs/<uuid:run_id>/retry")
|
|
class DatasetEvaluationRunRetryApi(EvaluationEditionResource):
|
|
@console_ns.response(200, "Evaluation run retry scheduled", console_ns.models[EvaluationRunResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@edit_permission_required
|
|
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
|
|
@with_session
|
|
def post(self, session: Session, dataset_id: UUID, run_id: UUID) -> dict[str, Any]:
|
|
"""Retry a scoped dataset evaluation using the current caller identity."""
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
|
|
return _retry_run(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
target_type=EvaluationTargetType.KNOWLEDGE_BASE.value,
|
|
target_id=dataset.id,
|
|
run_id=str(run_id),
|
|
account_id=current_user.id,
|
|
)
|
|
|
|
|
|
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/metrics")
|
|
class DatasetEvaluationMetricsApi(EvaluationEditionResource):
|
|
@console_ns.doc(params=query_params_from_model(EvaluationFrameworkQuery))
|
|
@console_ns.response(200, "Retrieval evaluation metrics", console_ns.models[EvaluationMetricsResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
|
|
@with_session(write=False)
|
|
def get(self, session: Session, dataset_id: UUID) -> dict[str, Any]:
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
_get_accessible_dataset(session, dataset_id, current_user, tenant_id)
|
|
query = EvaluationFrameworkQuery.model_validate(request.args.to_dict(flat=True))
|
|
return dump_response(
|
|
EvaluationMetricsResponse,
|
|
{
|
|
"metrics": EvaluationService.get_console_supported_metrics(
|
|
EvaluationCategory.RETRIEVAL, query.evaluation_framework
|
|
)
|
|
},
|
|
)
|
|
|
|
|
|
@console_ns.route("/datasets/<uuid:dataset_id>/evaluation/files/<uuid:file_id>")
|
|
class DatasetEvaluationFileDownloadApi(EvaluationEditionResource):
|
|
@console_ns.response(200, "Evaluation file", console_ns.models[EvaluationFileResponse.__name__])
|
|
@setup_required
|
|
@login_required
|
|
@account_initialization_required
|
|
@evaluation_paid_plan_required
|
|
@rbac_permission_required(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()))
|
|
@with_session(write=False)
|
|
def get(self, session: Session, dataset_id: UUID, file_id: UUID) -> dict[str, Any]:
|
|
current_user, tenant_id = current_account_with_tenant()
|
|
dataset = _get_accessible_dataset(session, dataset_id, current_user, tenant_id)
|
|
return _evaluation_file_response(
|
|
session=session,
|
|
tenant_id=tenant_id,
|
|
target_type=EvaluationTargetType.KNOWLEDGE_BASE.value,
|
|
target_id=dataset.id,
|
|
file_id=str(file_id),
|
|
)
|