# This file was auto-generated by Fern from our API Definition. import typing from json.decoder import JSONDecodeError from ...commons.errors.access_denied_error import ( AccessDeniedError as commons_errors_access_denied_error_AccessDeniedError, ) from ...commons.errors.error import Error from ...commons.errors.method_not_allowed_error import ( MethodNotAllowedError as commons_errors_method_not_allowed_error_MethodNotAllowedError, ) from ...commons.errors.not_found_error import ( NotFoundError as commons_errors_not_found_error_NotFoundError, ) from ...commons.errors.unauthorized_error import ( UnauthorizedError as commons_errors_unauthorized_error_UnauthorizedError, ) from ...core.api_error import ApiError from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper from ...core.http_response import AsyncHttpResponse, HttpResponse from ...core.jsonable_encoder import jsonable_encoder from ...core.pydantic_utilities import parse_obj_as from ...core.request_options import RequestOptions from ...core.serialization import convert_and_respect_annotation_metadata from ..errors.errors.access_denied_error import ( AccessDeniedError as unstable_errors_errors_access_denied_error_AccessDeniedError, ) from ..errors.errors.bad_request_error import BadRequestError from ..errors.errors.conflict_error import ConflictError from ..errors.errors.internal_server_error import InternalServerError from ..errors.errors.method_not_allowed_error import ( MethodNotAllowedError as unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError, ) from ..errors.errors.not_found_error import ( NotFoundError as unstable_errors_errors_not_found_error_NotFoundError, ) from ..errors.errors.too_many_requests_error import TooManyRequestsError from ..errors.errors.unauthorized_error import ( UnauthorizedError as unstable_errors_errors_unauthorized_error_UnauthorizedError, ) from ..errors.errors.unprocessable_content_error import UnprocessableContentError from ..errors.types.public_api_error import PublicApiError from .types.create_evaluator_request import CreateEvaluatorRequest from .types.delete_evaluator_response import DeleteEvaluatorResponse from .types.evaluator import Evaluator from .types.evaluators import Evaluators # this is used as the default value for optional parameters OMIT = typing.cast(typing.Any, ...) class RawEvaluatorsClient: def __init__(self, *, client_wrapper: SyncClientWrapper): self._client_wrapper = client_wrapper def create( self, *, request: CreateEvaluatorRequest, request_options: typing.Optional[RequestOptions] = None, ) -> HttpResponse[Evaluator]: """ Create an evaluator in the authenticated project. Use evaluators to define **how** Langfuse should score data. LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration. Code evaluators define source code and a runtime language. Naming behavior: - If this is a new evaluator name in your project, Langfuse creates version `1`. - If the name already exists in your project, Langfuse creates the next version and returns it. - When a new project version is created, existing evaluation rules in that project automatically move to the newest version for that evaluator name. Recommended workflow: 1. Create the evaluator. 2. Read the returned `variables` array. 3. Read the returned `outputDefinition.dataType` so the client knows whether future scores will be numeric, boolean, or categorical. 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `scope`. Code evaluator validation: - At creation, Langfuse only validates the request shape - The `sourceCode` itself is not executed here. It is first run (preflight-tested against a sample observation) when you link the evaluator to an evaluation rule, so runtime errors in the code surface at evaluation-rule creation, not at evaluator creation. Recovery guidance: - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run with the resolved model configuration. Add a valid explicit `modelConfig`, or configure the project's default evaluation model, then retry the same request. - `400` with `code=invalid_body`: the request shape is malformed. Use the structured `details.issues` array to fix the specific fields and retry. - `400` with `code=invalid_body` on `outputDefinition`: for `type=llm_as_judge`, send `dataType`, `reasoning.description`, and `score.description`. Do not send `version`; it is not part of the public request shape. - If `type` is omitted, Langfuse treats the request as `type=llm_as_judge` for backwards compatibility. New clients should send `type` explicitly. Unstable API note: - This surface may evolve while the underlying evaluation data model is being redesigned. Parameters ---------- request : CreateEvaluatorRequest request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- HttpResponse[Evaluator] """ _response = self._client_wrapper.httpx_client.request( "api/public/unstable/evaluators", method="POST", json=convert_and_respect_annotation_metadata( object_=request, annotation=CreateEvaluatorRequest, direction="write" ), request_options=request_options, omit=OMIT, ) try: if 200 <= _response.status_code < 300: _data = typing.cast( Evaluator, parse_obj_as( type_=Evaluator, # type: ignore object_=_response.json(), ), ) return HttpResponse(response=_response, data=_data) if _response.status_code == 400: raise BadRequestError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise unstable_errors_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise unstable_errors_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 409: raise ConflictError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 422: raise UnprocessableContentError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 429: raise TooManyRequestsError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 500: raise InternalServerError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 400: raise Error( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise commons_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise commons_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise commons_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 404: raise commons_errors_not_found_error_NotFoundError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) _response_json = _response.json() except JSONDecodeError: raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response.text, ) raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response_json, ) def list( self, *, page: typing.Optional[int] = None, limit: typing.Optional[int] = None, request_options: typing.Optional[RequestOptions] = None, ) -> HttpResponse[Evaluators]: """ List the evaluators available to the authenticated project. Important behavior: - This endpoint returns the latest version of each available evaluator. - Results can include evaluators from your project and Langfuse-managed evaluators. - If the same evaluator name exists in both places, both are returned as separate items with different `scope` values. Parameters ---------- page : typing.Optional[int] 1-based page number. Defaults to `1`. limit : typing.Optional[int] Maximum number of items per page. Defaults to `50`. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- HttpResponse[Evaluators] """ _response = self._client_wrapper.httpx_client.request( "api/public/unstable/evaluators", method="GET", params={ "page": page, "limit": limit, }, request_options=request_options, ) try: if 200 <= _response.status_code < 300: _data = typing.cast( Evaluators, parse_obj_as( type_=Evaluators, # type: ignore object_=_response.json(), ), ) return HttpResponse(response=_response, data=_data) if _response.status_code == 400: raise BadRequestError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise unstable_errors_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise unstable_errors_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 429: raise TooManyRequestsError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 500: raise InternalServerError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 400: raise Error( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise commons_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise commons_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise commons_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 404: raise commons_errors_not_found_error_NotFoundError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) _response_json = _response.json() except JSONDecodeError: raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response.text, ) raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response_json, ) def get( self, evaluator_id: str, *, request_options: typing.Optional[RequestOptions] = None, ) -> HttpResponse[Evaluator]: """ Get one evaluator by `id`. Use this endpoint when you want the prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. Parameters ---------- evaluator_id : str Evaluator identifier returned by the evaluator endpoints. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- HttpResponse[Evaluator] """ _response = self._client_wrapper.httpx_client.request( f"api/public/unstable/evaluators/{jsonable_encoder(evaluator_id)}", method="GET", request_options=request_options, ) try: if 200 <= _response.status_code < 300: _data = typing.cast( Evaluator, parse_obj_as( type_=Evaluator, # type: ignore object_=_response.json(), ), ) return HttpResponse(response=_response, data=_data) if _response.status_code == 400: raise BadRequestError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise unstable_errors_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise unstable_errors_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 404: raise unstable_errors_errors_not_found_error_NotFoundError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 429: raise TooManyRequestsError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 500: raise InternalServerError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 400: raise Error( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise commons_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise commons_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise commons_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 404: raise commons_errors_not_found_error_NotFoundError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) _response_json = _response.json() except JSONDecodeError: raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response.text, ) raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response_json, ) def delete( self, evaluator_id: str, *, request_options: typing.Optional[RequestOptions] = None, ) -> HttpResponse[DeleteEvaluatorResponse]: """ Delete an evaluator. Important behavior: - This deletes the evaluator including all of its stored versions; `evaluatorId` may reference any version. - The API returns `409` while evaluation rules still reference the evaluator. Delete those evaluation rules first. - Langfuse-managed evaluators (`scope=managed`) cannot be deleted; the API returns `403`. - Scores already produced by the evaluator are not deleted. Parameters ---------- evaluator_id : str Evaluator identifier returned by the evaluator endpoints. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- HttpResponse[DeleteEvaluatorResponse] """ _response = self._client_wrapper.httpx_client.request( f"api/public/unstable/evaluators/{jsonable_encoder(evaluator_id)}", method="DELETE", request_options=request_options, ) try: if 200 <= _response.status_code < 300: _data = typing.cast( DeleteEvaluatorResponse, parse_obj_as( type_=DeleteEvaluatorResponse, # type: ignore object_=_response.json(), ), ) return HttpResponse(response=_response, data=_data) if _response.status_code == 400: raise BadRequestError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise unstable_errors_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise unstable_errors_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 404: raise unstable_errors_errors_not_found_error_NotFoundError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 409: raise ConflictError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 429: raise TooManyRequestsError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 500: raise InternalServerError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 400: raise Error( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise commons_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise commons_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise commons_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 404: raise commons_errors_not_found_error_NotFoundError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) _response_json = _response.json() except JSONDecodeError: raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response.text, ) raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response_json, ) class AsyncRawEvaluatorsClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): self._client_wrapper = client_wrapper async def create( self, *, request: CreateEvaluatorRequest, request_options: typing.Optional[RequestOptions] = None, ) -> AsyncHttpResponse[Evaluator]: """ Create an evaluator in the authenticated project. Use evaluators to define **how** Langfuse should score data. LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration. Code evaluators define source code and a runtime language. Naming behavior: - If this is a new evaluator name in your project, Langfuse creates version `1`. - If the name already exists in your project, Langfuse creates the next version and returns it. - When a new project version is created, existing evaluation rules in that project automatically move to the newest version for that evaluator name. Recommended workflow: 1. Create the evaluator. 2. Read the returned `variables` array. 3. Read the returned `outputDefinition.dataType` so the client knows whether future scores will be numeric, boolean, or categorical. 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `scope`. Code evaluator validation: - At creation, Langfuse only validates the request shape - The `sourceCode` itself is not executed here. It is first run (preflight-tested against a sample observation) when you link the evaluator to an evaluation rule, so runtime errors in the code surface at evaluation-rule creation, not at evaluator creation. Recovery guidance: - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run with the resolved model configuration. Add a valid explicit `modelConfig`, or configure the project's default evaluation model, then retry the same request. - `400` with `code=invalid_body`: the request shape is malformed. Use the structured `details.issues` array to fix the specific fields and retry. - `400` with `code=invalid_body` on `outputDefinition`: for `type=llm_as_judge`, send `dataType`, `reasoning.description`, and `score.description`. Do not send `version`; it is not part of the public request shape. - If `type` is omitted, Langfuse treats the request as `type=llm_as_judge` for backwards compatibility. New clients should send `type` explicitly. Unstable API note: - This surface may evolve while the underlying evaluation data model is being redesigned. Parameters ---------- request : CreateEvaluatorRequest request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- AsyncHttpResponse[Evaluator] """ _response = await self._client_wrapper.httpx_client.request( "api/public/unstable/evaluators", method="POST", json=convert_and_respect_annotation_metadata( object_=request, annotation=CreateEvaluatorRequest, direction="write" ), request_options=request_options, omit=OMIT, ) try: if 200 <= _response.status_code < 300: _data = typing.cast( Evaluator, parse_obj_as( type_=Evaluator, # type: ignore object_=_response.json(), ), ) return AsyncHttpResponse(response=_response, data=_data) if _response.status_code == 400: raise BadRequestError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise unstable_errors_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise unstable_errors_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 409: raise ConflictError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 422: raise UnprocessableContentError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 429: raise TooManyRequestsError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 500: raise InternalServerError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 400: raise Error( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise commons_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise commons_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise commons_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 404: raise commons_errors_not_found_error_NotFoundError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) _response_json = _response.json() except JSONDecodeError: raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response.text, ) raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response_json, ) async def list( self, *, page: typing.Optional[int] = None, limit: typing.Optional[int] = None, request_options: typing.Optional[RequestOptions] = None, ) -> AsyncHttpResponse[Evaluators]: """ List the evaluators available to the authenticated project. Important behavior: - This endpoint returns the latest version of each available evaluator. - Results can include evaluators from your project and Langfuse-managed evaluators. - If the same evaluator name exists in both places, both are returned as separate items with different `scope` values. Parameters ---------- page : typing.Optional[int] 1-based page number. Defaults to `1`. limit : typing.Optional[int] Maximum number of items per page. Defaults to `50`. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- AsyncHttpResponse[Evaluators] """ _response = await self._client_wrapper.httpx_client.request( "api/public/unstable/evaluators", method="GET", params={ "page": page, "limit": limit, }, request_options=request_options, ) try: if 200 <= _response.status_code < 300: _data = typing.cast( Evaluators, parse_obj_as( type_=Evaluators, # type: ignore object_=_response.json(), ), ) return AsyncHttpResponse(response=_response, data=_data) if _response.status_code == 400: raise BadRequestError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise unstable_errors_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise unstable_errors_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 429: raise TooManyRequestsError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 500: raise InternalServerError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 400: raise Error( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise commons_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise commons_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise commons_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 404: raise commons_errors_not_found_error_NotFoundError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) _response_json = _response.json() except JSONDecodeError: raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response.text, ) raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response_json, ) async def get( self, evaluator_id: str, *, request_options: typing.Optional[RequestOptions] = None, ) -> AsyncHttpResponse[Evaluator]: """ Get one evaluator by `id`. Use this endpoint when you want the prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. Parameters ---------- evaluator_id : str Evaluator identifier returned by the evaluator endpoints. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- AsyncHttpResponse[Evaluator] """ _response = await self._client_wrapper.httpx_client.request( f"api/public/unstable/evaluators/{jsonable_encoder(evaluator_id)}", method="GET", request_options=request_options, ) try: if 200 <= _response.status_code < 300: _data = typing.cast( Evaluator, parse_obj_as( type_=Evaluator, # type: ignore object_=_response.json(), ), ) return AsyncHttpResponse(response=_response, data=_data) if _response.status_code == 400: raise BadRequestError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise unstable_errors_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise unstable_errors_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 404: raise unstable_errors_errors_not_found_error_NotFoundError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 429: raise TooManyRequestsError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 500: raise InternalServerError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 400: raise Error( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise commons_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise commons_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise commons_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 404: raise commons_errors_not_found_error_NotFoundError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) _response_json = _response.json() except JSONDecodeError: raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response.text, ) raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response_json, ) async def delete( self, evaluator_id: str, *, request_options: typing.Optional[RequestOptions] = None, ) -> AsyncHttpResponse[DeleteEvaluatorResponse]: """ Delete an evaluator. Important behavior: - This deletes the evaluator including all of its stored versions; `evaluatorId` may reference any version. - The API returns `409` while evaluation rules still reference the evaluator. Delete those evaluation rules first. - Langfuse-managed evaluators (`scope=managed`) cannot be deleted; the API returns `403`. - Scores already produced by the evaluator are not deleted. Parameters ---------- evaluator_id : str Evaluator identifier returned by the evaluator endpoints. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- AsyncHttpResponse[DeleteEvaluatorResponse] """ _response = await self._client_wrapper.httpx_client.request( f"api/public/unstable/evaluators/{jsonable_encoder(evaluator_id)}", method="DELETE", request_options=request_options, ) try: if 200 <= _response.status_code < 300: _data = typing.cast( DeleteEvaluatorResponse, parse_obj_as( type_=DeleteEvaluatorResponse, # type: ignore object_=_response.json(), ), ) return AsyncHttpResponse(response=_response, data=_data) if _response.status_code == 400: raise BadRequestError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise unstable_errors_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise unstable_errors_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 404: raise unstable_errors_errors_not_found_error_NotFoundError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise unstable_errors_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 409: raise ConflictError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 429: raise TooManyRequestsError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 500: raise InternalServerError( headers=dict(_response.headers), body=typing.cast( PublicApiError, parse_obj_as( type_=PublicApiError, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 400: raise Error( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 401: raise commons_errors_unauthorized_error_UnauthorizedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 403: raise commons_errors_access_denied_error_AccessDeniedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 405: raise commons_errors_method_not_allowed_error_MethodNotAllowedError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) if _response.status_code == 404: raise commons_errors_not_found_error_NotFoundError( headers=dict(_response.headers), body=typing.cast( typing.Any, parse_obj_as( type_=typing.Any, # type: ignore object_=_response.json(), ), ), ) _response_json = _response.json() except JSONDecodeError: raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response.text, ) raise ApiError( status_code=_response.status_code, headers=dict(_response.headers), body=_response_json, )