Files
dify/api/controllers/console/datasets/external.py
Jyong 582cab8491 Merge main into stack/knowledge-01-boundaries
Resolve conflicts with the configurable knowledge base description limit,
the blocking workflow run connection fix, read_media tool observations,
debug conversation workspace retirement, Ruff N815 and the strict pyrefly
test conversions.
2026-11-17 17:45:08 +08:00

346 lines
14 KiB
Python

from __future__ import annotations
import logging
from datetime import datetime
from typing import Any, Never
from uuid import UUID
from flask_restx import Resource
from pydantic import AliasChoices, BaseModel, Field, field_serializer, field_validator
from configs import dify_config
from constants import HIDDEN_VALUE
from controllers.common.errors import ConflictError, InternalServerError, NotFoundError
from controllers.common.fields import UsageCountResponse
from controllers.common.rbac import DatasetId, RBACCheck, Workspace
from controllers.common.schema import (
query_params_from_model,
register_response_schema_models,
register_schema_models,
)
from controllers.console import console_ns
from controllers.console.datasets.error import DatasetAccessDeniedRequestError, DatasetNameDuplicateError
from controllers.console.datasets.permission_keys import get_dataset_permission_keys
from controllers.console.flask_admission import console_account_admission
from controllers.console.wraps import (
RBACPermission,
model_validate,
)
from extensions.ext_application_services import application_services
from fields.base import ResponseModel
from fields.dataset_fields import (
DatasetDetailResponse,
)
from fields.hit_testing_fields import (
ExternalHitTestingQueryResponse,
ExternalHitTestingRecordResponse,
ExternalHitTestingResponse,
)
from libs.external_api import error_responses
from libs.helper import dump_response
from machinery.context import RequestContext
from models.account import TenantAccountRole
from services.errors.dataset import DatasetNameDuplicateError as DatasetNameDuplicateFailure
from services.errors.knowledge_retrieval import (
ExternalTemplateConcurrentUpdateError,
ExternalTemplateNotFoundError,
)
from services.knowledge.dataset_access import DatasetAccessDeniedError, DatasetNotFoundError
from services.knowledge.entities.external import ExternalDatasetCreatePayload as ExternalDatasetCreateInput
from services.knowledge.external.application import ExternalHitTestingError
logger = logging.getLogger(__name__)
_EXTERNAL_CONNECT_ROLES = frozenset(
{
TenantAccountRole.OWNER,
TenantAccountRole.ADMIN,
TenantAccountRole.EDITOR,
TenantAccountRole.DATASET_OPERATOR,
}
)
def _raise_external_error(error: Exception) -> Never:
if isinstance(error, ExternalTemplateNotFoundError):
raise NotFoundError(description="API template not found.") from error
if isinstance(error, DatasetNotFoundError):
raise NotFoundError(description=str(error)) from error
if isinstance(error, DatasetAccessDeniedError):
raise DatasetAccessDeniedRequestError(description=str(error)) from error
if isinstance(error, DatasetNameDuplicateFailure):
raise DatasetNameDuplicateError() from error
if isinstance(error, ExternalTemplateConcurrentUpdateError):
raise ConflictError(description=str(error)) from error
if isinstance(error, ExternalHitTestingError):
logger.exception("External hit testing failed")
raise InternalServerError() from error
raise error
class ExternalDatasetCreatePayload(ExternalDatasetCreateInput):
"""Apply the deployment's description limit at the HTTP boundary."""
description: str | None = Field(None, max_length=dify_config.DATASET_DESCRIPTION_MAX_LENGTH)
class ExternalKnowledgeApiPayload(BaseModel):
name: str = Field(..., min_length=1, max_length=40)
settings: dict[str, Any]
class ExternalHitTestingPayload(BaseModel):
query: str
external_retrieval_model: dict[str, Any] | None = None
metadata_filtering_conditions: dict[str, Any] | None = None
class ExternalApiTemplateListQuery(BaseModel):
page: int = Field(default=1, description="Page number")
limit: int = Field(default=20, description="Number of items per page")
keyword: str | None = Field(default=None, description="Search keyword")
class ExternalKnowledgeApiBindingResponse(ResponseModel):
id: str
name: str
class ExternalKnowledgeApiSettingsResponse(ResponseModel):
endpoint: str | None = None
api_key: str | None = None
@field_serializer("api_key")
def _mask_api_key(self, value: str | None) -> str | None:
return HIDDEN_VALUE if value else value
class ExternalKnowledgeApiResponse(ResponseModel):
id: str
tenant_id: str
name: str
description: str
settings: ExternalKnowledgeApiSettingsResponse | None = Field(
validation_alias=AliasChoices("settings_dict", "settings")
)
dataset_bindings: list[ExternalKnowledgeApiBindingResponse]
created_by: str
created_at: str
@field_validator("created_at", mode="before")
@classmethod
def _normalize_created_at(cls, value: datetime | str) -> str:
if isinstance(value, datetime):
return value.isoformat()
return value
class ExternalKnowledgeApiListResponse(ResponseModel):
data: list[ExternalKnowledgeApiResponse]
has_more: bool
limit: int
total: int | None
page: int
register_schema_models(
console_ns,
ExternalKnowledgeApiPayload,
ExternalDatasetCreatePayload,
ExternalHitTestingPayload,
ExternalApiTemplateListQuery,
)
register_response_schema_models(
console_ns,
UsageCountResponse,
DatasetDetailResponse,
ExternalKnowledgeApiBindingResponse,
ExternalKnowledgeApiResponse,
ExternalKnowledgeApiListResponse,
ExternalHitTestingQueryResponse,
ExternalHitTestingRecordResponse,
ExternalHitTestingResponse,
)
@console_ns.route("/datasets/external-knowledge-api")
class ExternalApiTemplateListApi(Resource):
@console_ns.doc("get_external_api_templates")
@console_ns.doc(description="Get external knowledge API templates")
@console_ns.doc(params=query_params_from_model(ExternalApiTemplateListQuery))
@console_ns.response(
200,
"External API templates retrieved successfully",
console_ns.models[ExternalKnowledgeApiListResponse.__name__],
)
@console_account_admission()
@model_validate(ExternalApiTemplateListQuery)
def get(self, req_data: ExternalApiTemplateListQuery, request_context: RequestContext):
result = application_services().knowledge.external.list_templates(
request_context, page=req_data.page, limit=req_data.limit, keyword=req_data.keyword
)
return dump_response(ExternalKnowledgeApiListResponse, result), 200
@error_responses(DatasetNameDuplicateError)
@console_ns.doc("create_external_api_template")
@console_ns.doc(description="Create external knowledge API template")
@console_ns.expect(console_ns.models[ExternalKnowledgeApiPayload.__name__])
@console_ns.response(
201,
"External API template created successfully",
console_ns.models[ExternalKnowledgeApiResponse.__name__],
)
@console_ns.response(403, "Permission denied")
@console_account_admission(
allowed_roles=_EXTERNAL_CONNECT_ROLES,
rbac_checks=(RBACCheck(RBACPermission.DATASET_EXTERNAL_CONNECT, Workspace()),),
)
@model_validate(ExternalKnowledgeApiPayload)
def post(self, req_data: ExternalKnowledgeApiPayload, request_context: RequestContext):
try:
result = application_services().knowledge.external.create_template(
request_context, name=req_data.name, settings=req_data.settings
)
except Exception as error:
_raise_external_error(error)
return dump_response(ExternalKnowledgeApiResponse, result), 201
@console_ns.route("/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>")
class ExternalApiTemplateApi(Resource):
@error_responses(DatasetNameDuplicateError)
@console_ns.doc("get_external_api_template")
@console_ns.doc(description="Get external knowledge API template details")
@console_ns.doc(params={"external_knowledge_api_id": "External knowledge API ID"})
@console_ns.response(
200,
"External API template retrieved successfully",
console_ns.models[ExternalKnowledgeApiResponse.__name__],
)
@console_ns.response(404, "Template not found")
@console_account_admission()
def get(self, request_context: RequestContext, external_knowledge_api_id: UUID):
try:
result = application_services().knowledge.external.get_template(
request_context, template_id=str(external_knowledge_api_id)
)
except Exception as error:
_raise_external_error(error)
return dump_response(ExternalKnowledgeApiResponse, result), 200
@error_responses(DatasetNameDuplicateError)
@console_ns.doc("update_external_api_template")
@console_ns.doc(description="Update external knowledge API template")
@console_ns.doc(params={"external_knowledge_api_id": "External knowledge API ID"})
@console_ns.expect(console_ns.models[ExternalKnowledgeApiPayload.__name__])
@console_ns.response(
200,
"External API template updated successfully",
console_ns.models[ExternalKnowledgeApiResponse.__name__],
)
@console_ns.response(404, "Template not found")
@console_ns.response(409, "Template changed during validation")
@console_account_admission(
allowed_roles=_EXTERNAL_CONNECT_ROLES,
rbac_checks=(RBACCheck(RBACPermission.DATASET_EXTERNAL_CONNECT, Workspace()),),
)
@model_validate(ExternalKnowledgeApiPayload)
def patch(
self, req_data: ExternalKnowledgeApiPayload, request_context: RequestContext, external_knowledge_api_id: UUID
):
try:
result = application_services().knowledge.external.update_template(
request_context,
template_id=str(external_knowledge_api_id),
name=req_data.name,
settings=req_data.settings,
)
except Exception as error:
_raise_external_error(error)
return dump_response(ExternalKnowledgeApiResponse, result), 200
@error_responses(DatasetNameDuplicateError)
@console_ns.response(204, "External knowledge API deleted successfully")
@console_account_admission(
allowed_roles=_EXTERNAL_CONNECT_ROLES,
rbac_checks=(RBACCheck(RBACPermission.DATASET_EXTERNAL_CONNECT, Workspace()),),
)
def delete(self, request_context: RequestContext, external_knowledge_api_id: UUID):
try:
application_services().knowledge.external.delete_template(
request_context, template_id=str(external_knowledge_api_id)
)
except Exception as error:
_raise_external_error(error)
return "", 204
@console_ns.route("/datasets/external-knowledge-api/<uuid:external_knowledge_api_id>/use-check")
class ExternalApiUseCheckApi(Resource):
@console_ns.doc("check_external_api_usage")
@console_ns.doc(description="Check if external knowledge API is being used")
@console_ns.doc(params={"external_knowledge_api_id": "External knowledge API ID"})
@console_ns.response(200, "Usage check completed successfully", console_ns.models[UsageCountResponse.__name__])
@console_account_admission()
def get(self, request_context: RequestContext, external_knowledge_api_id: UUID):
is_using, count = application_services().knowledge.external.template_usage(
request_context, template_id=str(external_knowledge_api_id)
)
return dump_response(UsageCountResponse, {"is_using": is_using, "count": count}), 200
@console_ns.route("/datasets/external")
class ExternalDatasetCreateApi(Resource):
@error_responses(DatasetNameDuplicateError)
@console_ns.doc("create_external_dataset")
@console_ns.doc(description="Create external knowledge dataset")
@console_ns.expect(console_ns.models[ExternalDatasetCreatePayload.__name__])
@console_ns.response(
201, "External dataset created successfully", console_ns.models[DatasetDetailResponse.__name__]
)
@console_ns.response(400, "Invalid parameters")
@console_ns.response(403, "Permission denied")
@console_account_admission(
allowed_roles=_EXTERNAL_CONNECT_ROLES,
rbac_checks=(RBACCheck(RBACPermission.DATASET_EXTERNAL_CONNECT, Workspace()),),
)
@model_validate(ExternalDatasetCreatePayload)
def post(self, req_data: ExternalDatasetCreatePayload, request_context: RequestContext):
try:
result = application_services().knowledge.external.create_dataset(request_context, payload=req_data)
except Exception as error:
_raise_external_error(error)
return dump_response(
DatasetDetailResponse,
{**result, "permission_keys": get_dataset_permission_keys(request_context, result["id"])},
), 201
@console_ns.route("/datasets/<uuid:dataset_id>/external-hit-testing")
class ExternalKnowledgeHitTestingApi(Resource):
@error_responses(DatasetAccessDeniedRequestError)
@console_ns.doc("test_external_knowledge_retrieval")
@console_ns.doc(description="Test external knowledge retrieval for dataset")
@console_ns.doc(params={"dataset_id": "Dataset ID"})
@console_ns.expect(console_ns.models[ExternalHitTestingPayload.__name__])
@console_ns.response(
200,
"External hit testing completed successfully",
console_ns.models[ExternalHitTestingResponse.__name__],
)
@console_ns.response(404, "Dataset not found")
@console_ns.response(400, "Invalid parameters")
@console_account_admission(rbac_checks=(RBACCheck(RBACPermission.DATASET_PIPELINE_TEST, DatasetId()),))
@model_validate(ExternalHitTestingPayload)
def post(self, req_data: ExternalHitTestingPayload, request_context: RequestContext, dataset_id: UUID):
try:
result = application_services().knowledge.external.hit_testing(
request_context,
dataset_id=str(dataset_id),
query=req_data.query,
retrieval_model=req_data.external_retrieval_model,
metadata_filters=req_data.metadata_filtering_conditions,
)
except Exception as error:
_raise_external_error(error)
return dump_response(ExternalHitTestingResponse, result)