mirror of
https://github.com/langgenius/dify.git
synced 2026-11-22 19:18:52 +08:00
Co-authored-by: Stephen Zhou <hi@hyoban.cc> Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: Jyong <jyong@dify.ai>
246 lines
11 KiB
Python
246 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime
|
|
from enum import StrEnum
|
|
from typing import Any
|
|
|
|
import sqlalchemy as sa
|
|
from sqlalchemy import DateTime, Float, Integer, String, Text, func
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
|
|
from core.evaluation.entities.config_entity import EvaluationFrameworkEnum
|
|
from libs.datetime_utils import naive_utc_now
|
|
from libs.uuid_utils import uuidv7
|
|
|
|
from .base import TypeBase
|
|
from .types import EnumText, LongText, StringUUID
|
|
|
|
|
|
class EvaluationRunStatus(StrEnum):
|
|
PENDING = "pending"
|
|
RUNNING = "running"
|
|
COMPLETED = "completed"
|
|
FAILED = "failed"
|
|
CANCELLED = "cancelled"
|
|
|
|
|
|
class EvaluationTargetType(StrEnum):
|
|
APPS = "apps"
|
|
SNIPPETS = "snippets"
|
|
KNOWLEDGE_BASE = "knowledge_base"
|
|
AGENTS = "agents"
|
|
|
|
|
|
class EvaluationRunItemStatus(StrEnum):
|
|
"""Durable checkpoint state for one evaluation dataset row."""
|
|
|
|
TARGET_RUNNING = "target_running"
|
|
TARGET_COMPLETED = "target_completed"
|
|
COMPLETED = "completed"
|
|
|
|
|
|
class EvaluationConfiguration(TypeBase, kw_only=True):
|
|
"""Latest editable evaluation configuration for a tenant-owned target."""
|
|
|
|
__tablename__ = "evaluation_configurations"
|
|
__table_args__ = (
|
|
sa.PrimaryKeyConstraint("id", name="evaluation_configuration_pkey"),
|
|
sa.UniqueConstraint("tenant_id", "target_type", "target_id", name="evaluation_configuration_target_unique"),
|
|
sa.Index("evaluation_configuration_evaluator_app_idx", "tenant_id", "evaluator_app_id"),
|
|
)
|
|
|
|
id: Mapped[str] = mapped_column(
|
|
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7())
|
|
)
|
|
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
|
target_type: Mapped[EvaluationTargetType] = mapped_column(EnumText(EvaluationTargetType, length=32), nullable=False)
|
|
target_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
|
|
|
evaluation_framework: Mapped[EvaluationFrameworkEnum] = mapped_column(
|
|
EnumText(EvaluationFrameworkEnum, length=20),
|
|
nullable=False,
|
|
server_default="ragas",
|
|
default=EvaluationFrameworkEnum.RAGAS,
|
|
)
|
|
evaluation_model_provider: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
|
evaluation_model: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
|
metrics_config: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
|
|
judgment_conditions: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
|
|
evaluator_app_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
|
|
|
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
|
updated_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
|
created_at: Mapped[datetime] = mapped_column(
|
|
DateTime,
|
|
nullable=False,
|
|
insert_default=naive_utc_now,
|
|
default_factory=naive_utc_now,
|
|
server_default=func.current_timestamp(),
|
|
)
|
|
updated_at: Mapped[datetime] = mapped_column(
|
|
DateTime,
|
|
nullable=False,
|
|
server_default=func.current_timestamp(),
|
|
onupdate=func.current_timestamp(),
|
|
insert_default=naive_utc_now,
|
|
default_factory=naive_utc_now,
|
|
)
|
|
|
|
@property
|
|
def metrics_config_dict(self) -> dict[str, Any]:
|
|
return json.loads(self.metrics_config) if self.metrics_config else {}
|
|
|
|
@property
|
|
def default_metrics_list(self) -> list[dict[str, Any]]:
|
|
return self.metrics_config_dict.get("default_metrics", [])
|
|
|
|
@property
|
|
def customized_metrics_dict(self) -> dict[str, Any] | None:
|
|
return self.metrics_config_dict.get("customized_metrics")
|
|
|
|
@property
|
|
def judgment_config_dict(self) -> dict[str, Any] | None:
|
|
if not self.judgment_conditions:
|
|
return None
|
|
value = json.loads(self.judgment_conditions)
|
|
return value or None
|
|
|
|
def __repr__(self) -> str:
|
|
return f"<EvaluationConfiguration(id={self.id}, target={self.target_type}:{self.target_id})>"
|
|
|
|
|
|
class EvaluationRun(TypeBase, kw_only=True):
|
|
"""Immutable configuration snapshot and mutable execution state for one evaluation."""
|
|
|
|
__tablename__ = "evaluation_runs"
|
|
__table_args__ = (
|
|
sa.PrimaryKeyConstraint("id", name="evaluation_run_pkey"),
|
|
sa.Index("evaluation_run_target_created_idx", "tenant_id", "target_type", "target_id", "created_at"),
|
|
sa.Index("evaluation_run_tenant_status_idx", "tenant_id", "status"),
|
|
sa.Index("evaluation_run_status_heartbeat_idx", "status", "heartbeat_at"),
|
|
)
|
|
|
|
id: Mapped[str] = mapped_column(
|
|
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7())
|
|
)
|
|
tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
|
target_type: Mapped[EvaluationTargetType] = mapped_column(EnumText(EvaluationTargetType, length=32), nullable=False)
|
|
target_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
|
target_workflow_version_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
|
target_agent_config_snapshot_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
|
evaluation_config_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
|
config_snapshot: Mapped[str] = mapped_column(LongText, nullable=False)
|
|
|
|
status: Mapped[EvaluationRunStatus] = mapped_column(
|
|
EnumText(EvaluationRunStatus, length=20), nullable=False, default=EvaluationRunStatus.PENDING
|
|
)
|
|
dataset_file_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
|
result_file_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
|
|
|
total_items: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
completed_items: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
failed_items: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
progress_rate: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)
|
|
metrics_summary: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
|
|
error: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
|
|
celery_task_id: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
|
execution_token: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
|
|
|
created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
|
started_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
|
heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
|
completed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True, default=None)
|
|
created_at: Mapped[datetime] = mapped_column(
|
|
DateTime,
|
|
nullable=False,
|
|
insert_default=naive_utc_now,
|
|
default_factory=naive_utc_now,
|
|
server_default=func.current_timestamp(),
|
|
)
|
|
updated_at: Mapped[datetime] = mapped_column(
|
|
DateTime,
|
|
nullable=False,
|
|
server_default=func.current_timestamp(),
|
|
onupdate=func.current_timestamp(),
|
|
insert_default=naive_utc_now,
|
|
default_factory=naive_utc_now,
|
|
)
|
|
|
|
@property
|
|
def progress(self) -> float:
|
|
return self.progress_rate
|
|
|
|
@property
|
|
def metrics_summary_dict(self) -> dict[str, Any]:
|
|
return json.loads(self.metrics_summary) if self.metrics_summary else {}
|
|
|
|
@property
|
|
def config_snapshot_dict(self) -> dict[str, Any]:
|
|
return json.loads(self.config_snapshot)
|
|
|
|
def __repr__(self) -> str:
|
|
return f"<EvaluationRun(id={self.id}, status={self.status})>"
|
|
|
|
|
|
class EvaluationRunItem(TypeBase, kw_only=True):
|
|
"""Per-dataset-row result for an evaluation run."""
|
|
|
|
__tablename__ = "evaluation_run_items"
|
|
__table_args__ = (
|
|
sa.PrimaryKeyConstraint("id", name="evaluation_run_item_pkey"),
|
|
sa.UniqueConstraint("evaluation_run_id", "item_index", name="evaluation_run_item_run_index_unique"),
|
|
sa.Index("evaluation_run_item_workflow_run_idx", "workflow_run_id"),
|
|
sa.Index("evaluation_run_item_agent_backend_run_idx", "agent_backend_run_id"),
|
|
)
|
|
|
|
id: Mapped[str] = mapped_column(
|
|
StringUUID, insert_default=lambda: str(uuidv7()), default_factory=lambda: str(uuidv7())
|
|
)
|
|
evaluation_run_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
|
|
workflow_run_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
|
agent_backend_run_id: Mapped[str | None] = mapped_column(String(255), nullable=True, default=None)
|
|
agent_workspace_binding_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True, default=None)
|
|
item_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
status: Mapped[EvaluationRunItemStatus] = mapped_column(
|
|
EnumText(EvaluationRunItemStatus, length=20),
|
|
nullable=False,
|
|
default=EvaluationRunItemStatus.COMPLETED,
|
|
)
|
|
|
|
inputs: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
|
|
expected_output: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
|
|
actual_output: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
|
|
metrics: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
|
|
judgment: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
|
|
metadata_json: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
|
|
target_result_json: Mapped[str | None] = mapped_column(LongText, nullable=True, default=None)
|
|
error: Mapped[str | None] = mapped_column(Text, nullable=True, default=None)
|
|
overall_score: Mapped[float | None] = mapped_column(Float, nullable=True, default=None)
|
|
created_at: Mapped[datetime] = mapped_column(
|
|
DateTime,
|
|
nullable=False,
|
|
insert_default=naive_utc_now,
|
|
default_factory=naive_utc_now,
|
|
server_default=func.current_timestamp(),
|
|
)
|
|
|
|
@property
|
|
def inputs_dict(self) -> dict[str, Any]:
|
|
return json.loads(self.inputs) if self.inputs else {}
|
|
|
|
@property
|
|
def metrics_list(self) -> list[dict[str, Any]]:
|
|
return json.loads(self.metrics) if self.metrics else []
|
|
|
|
@property
|
|
def judgment_dict(self) -> dict[str, Any]:
|
|
return json.loads(self.judgment) if self.judgment else {}
|
|
|
|
@property
|
|
def metadata_dict(self) -> dict[str, Any]:
|
|
return json.loads(self.metadata_json) if self.metadata_json else {}
|
|
|
|
def __repr__(self) -> str:
|
|
return f"<EvaluationRunItem(id={self.id}, run={self.evaluation_run_id}, index={self.item_index})>"
|