# This file was auto-generated by Fern from our API Definition. import typing from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper from ...core.request_options import RequestOptions from .raw_client import AsyncRawEvaluatorsClient, RawEvaluatorsClient from .types.create_evaluator_request import CreateEvaluatorRequest from .types.delete_evaluator_response import DeleteEvaluatorResponse from .types.evaluator import Evaluator from .types.evaluators import Evaluators # this is used as the default value for optional parameters OMIT = typing.cast(typing.Any, ...) class EvaluatorsClient: def __init__(self, *, client_wrapper: SyncClientWrapper): self._raw_client = RawEvaluatorsClient(client_wrapper=client_wrapper) @property def with_raw_response(self) -> RawEvaluatorsClient: """ Retrieves a raw implementation of this client that returns raw responses. Returns ------- RawEvaluatorsClient """ return self._raw_client def create( self, *, request: CreateEvaluatorRequest, request_options: typing.Optional[RequestOptions] = None, ) -> Evaluator: """ Create an evaluator in the authenticated project. Use evaluators to define **how** Langfuse should score data. LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration. Code evaluators define source code and a runtime language. Naming behavior: - If this is a new evaluator name in your project, Langfuse creates version `1`. - If the name already exists in your project, Langfuse creates the next version and returns it. - When a new project version is created, existing evaluation rules in that project automatically move to the newest version for that evaluator name. Recommended workflow: 1. Create the evaluator. 2. Read the returned `variables` array. 3. Read the returned `outputDefinition.dataType` so the client knows whether future scores will be numeric, boolean, or categorical. 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `scope`. Code evaluator validation: - At creation, Langfuse only validates the request shape - The `sourceCode` itself is not executed here. It is first run (preflight-tested against a sample observation) when you link the evaluator to an evaluation rule, so runtime errors in the code surface at evaluation-rule creation, not at evaluator creation. Recovery guidance: - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run with the resolved model configuration. Add a valid explicit `modelConfig`, or configure the project's default evaluation model, then retry the same request. - `400` with `code=invalid_body`: the request shape is malformed. Use the structured `details.issues` array to fix the specific fields and retry. - `400` with `code=invalid_body` on `outputDefinition`: for `type=llm_as_judge`, send `dataType`, `reasoning.description`, and `score.description`. Do not send `version`; it is not part of the public request shape. - If `type` is omitted, Langfuse treats the request as `type=llm_as_judge` for backwards compatibility. New clients should send `type` explicitly. Unstable API note: - This surface may evolve while the underlying evaluation data model is being redesigned. Parameters ---------- request : CreateEvaluatorRequest request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- Evaluator Examples -------- from langfuse import LangfuseAPI from langfuse.unstable.commons import ( EvaluatorModelConfig, EvaluatorOutputDataType, EvaluatorOutputDefinition_Numeric, EvaluatorOutputFieldDefinition, ) from langfuse.unstable.evaluators import CreateEvaluatorRequest_LlmAsJudge client = LangfuseAPI( x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", username="YOUR_USERNAME", password="YOUR_PASSWORD", base_url="https://yourhost.com/path/to/api", ) client.unstable.evaluators.create( request=CreateEvaluatorRequest_LlmAsJudge( name="answer-correctness", prompt="You are grading an answer.\n\nInput:\n{{input}}\n\nOutput:\n{{output}}\n\nReturn a score between 0 and 1.\n", output_definition=EvaluatorOutputDefinition_Numeric( data_type=EvaluatorOutputDataType.NUMERIC, reasoning=EvaluatorOutputFieldDefinition( description="Explain why the score was assigned.", ), score=EvaluatorOutputFieldDefinition( description="Correctness score between 0 and 1.", ), ), model_config=EvaluatorModelConfig( provider="openai", model="gpt-4.1-mini", ), ), ) """ _response = self._raw_client.create( request=request, request_options=request_options ) return _response.data def list( self, *, page: typing.Optional[int] = None, limit: typing.Optional[int] = None, request_options: typing.Optional[RequestOptions] = None, ) -> Evaluators: """ List the evaluators available to the authenticated project. Important behavior: - This endpoint returns the latest version of each available evaluator. - Results can include evaluators from your project and Langfuse-managed evaluators. - If the same evaluator name exists in both places, both are returned as separate items with different `scope` values. Parameters ---------- page : typing.Optional[int] 1-based page number. Defaults to `1`. limit : typing.Optional[int] Maximum number of items per page. Defaults to `50`. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- Evaluators Examples -------- from langfuse import LangfuseAPI client = LangfuseAPI( x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", username="YOUR_USERNAME", password="YOUR_PASSWORD", base_url="https://yourhost.com/path/to/api", ) client.unstable.evaluators.list() """ _response = self._raw_client.list( page=page, limit=limit, request_options=request_options ) return _response.data def get( self, evaluator_id: str, *, request_options: typing.Optional[RequestOptions] = None, ) -> Evaluator: """ Get one evaluator by `id`. Use this endpoint when you want the prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. Parameters ---------- evaluator_id : str Evaluator identifier returned by the evaluator endpoints. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- Evaluator Examples -------- from langfuse import LangfuseAPI client = LangfuseAPI( x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", username="YOUR_USERNAME", password="YOUR_PASSWORD", base_url="https://yourhost.com/path/to/api", ) client.unstable.evaluators.get( evaluator_id="evaluatorId", ) """ _response = self._raw_client.get(evaluator_id, request_options=request_options) return _response.data def delete( self, evaluator_id: str, *, request_options: typing.Optional[RequestOptions] = None, ) -> DeleteEvaluatorResponse: """ Delete an evaluator. Important behavior: - This deletes the evaluator including all of its stored versions; `evaluatorId` may reference any version. - The API returns `409` while evaluation rules still reference the evaluator. Delete those evaluation rules first. - Langfuse-managed evaluators (`scope=managed`) cannot be deleted; the API returns `403`. - Scores already produced by the evaluator are not deleted. Parameters ---------- evaluator_id : str Evaluator identifier returned by the evaluator endpoints. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- DeleteEvaluatorResponse Examples -------- from langfuse import LangfuseAPI client = LangfuseAPI( x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", username="YOUR_USERNAME", password="YOUR_PASSWORD", base_url="https://yourhost.com/path/to/api", ) client.unstable.evaluators.delete( evaluator_id="evaluatorId", ) """ _response = self._raw_client.delete( evaluator_id, request_options=request_options ) return _response.data class AsyncEvaluatorsClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): self._raw_client = AsyncRawEvaluatorsClient(client_wrapper=client_wrapper) @property def with_raw_response(self) -> AsyncRawEvaluatorsClient: """ Retrieves a raw implementation of this client that returns raw responses. Returns ------- AsyncRawEvaluatorsClient """ return self._raw_client async def create( self, *, request: CreateEvaluatorRequest, request_options: typing.Optional[RequestOptions] = None, ) -> Evaluator: """ Create an evaluator in the authenticated project. Use evaluators to define **how** Langfuse should score data. LLM-as-a-judge evaluators define a prompt, expected structured output, and optional model configuration. Code evaluators define source code and a runtime language. Naming behavior: - If this is a new evaluator name in your project, Langfuse creates version `1`. - If the name already exists in your project, Langfuse creates the next version and returns it. - When a new project version is created, existing evaluation rules in that project automatically move to the newest version for that evaluator name. Recommended workflow: 1. Create the evaluator. 2. Read the returned `variables` array. 3. Read the returned `outputDefinition.dataType` so the client knows whether future scores will be numeric, boolean, or categorical. 4. Create one or more evaluation rules that reference the returned evaluator family using `name` and `scope`. Code evaluator validation: - At creation, Langfuse only validates the request shape - The `sourceCode` itself is not executed here. It is first run (preflight-tested against a sample observation) when you link the evaluator to an evaluation rule, so runtime errors in the code surface at evaluation-rule creation, not at evaluator creation. Recovery guidance: - `422` with `code=evaluator_preflight_failed`: the evaluator cannot run with the resolved model configuration. Add a valid explicit `modelConfig`, or configure the project's default evaluation model, then retry the same request. - `400` with `code=invalid_body`: the request shape is malformed. Use the structured `details.issues` array to fix the specific fields and retry. - `400` with `code=invalid_body` on `outputDefinition`: for `type=llm_as_judge`, send `dataType`, `reasoning.description`, and `score.description`. Do not send `version`; it is not part of the public request shape. - If `type` is omitted, Langfuse treats the request as `type=llm_as_judge` for backwards compatibility. New clients should send `type` explicitly. Unstable API note: - This surface may evolve while the underlying evaluation data model is being redesigned. Parameters ---------- request : CreateEvaluatorRequest request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- Evaluator Examples -------- import asyncio from langfuse import AsyncLangfuseAPI from langfuse.unstable.commons import ( EvaluatorModelConfig, EvaluatorOutputDataType, EvaluatorOutputDefinition_Numeric, EvaluatorOutputFieldDefinition, ) from langfuse.unstable.evaluators import CreateEvaluatorRequest_LlmAsJudge client = AsyncLangfuseAPI( x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", username="YOUR_USERNAME", password="YOUR_PASSWORD", base_url="https://yourhost.com/path/to/api", ) async def main() -> None: await client.unstable.evaluators.create( request=CreateEvaluatorRequest_LlmAsJudge( name="answer-correctness", prompt="You are grading an answer.\n\nInput:\n{{input}}\n\nOutput:\n{{output}}\n\nReturn a score between 0 and 1.\n", output_definition=EvaluatorOutputDefinition_Numeric( data_type=EvaluatorOutputDataType.NUMERIC, reasoning=EvaluatorOutputFieldDefinition( description="Explain why the score was assigned.", ), score=EvaluatorOutputFieldDefinition( description="Correctness score between 0 and 1.", ), ), model_config=EvaluatorModelConfig( provider="openai", model="gpt-4.1-mini", ), ), ) asyncio.run(main()) """ _response = await self._raw_client.create( request=request, request_options=request_options ) return _response.data async def list( self, *, page: typing.Optional[int] = None, limit: typing.Optional[int] = None, request_options: typing.Optional[RequestOptions] = None, ) -> Evaluators: """ List the evaluators available to the authenticated project. Important behavior: - This endpoint returns the latest version of each available evaluator. - Results can include evaluators from your project and Langfuse-managed evaluators. - If the same evaluator name exists in both places, both are returned as separate items with different `scope` values. Parameters ---------- page : typing.Optional[int] 1-based page number. Defaults to `1`. limit : typing.Optional[int] Maximum number of items per page. Defaults to `50`. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- Evaluators Examples -------- import asyncio from langfuse import AsyncLangfuseAPI client = AsyncLangfuseAPI( x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", username="YOUR_USERNAME", password="YOUR_PASSWORD", base_url="https://yourhost.com/path/to/api", ) async def main() -> None: await client.unstable.evaluators.list() asyncio.run(main()) """ _response = await self._raw_client.list( page=page, limit=limit, request_options=request_options ) return _response.data async def get( self, evaluator_id: str, *, request_options: typing.Optional[RequestOptions] = None, ) -> Evaluator: """ Get one evaluator by `id`. Use this endpoint when you want the prompt, output definition, model configuration, and derived variables for the evaluator you plan to use in an evaluation rule. Parameters ---------- evaluator_id : str Evaluator identifier returned by the evaluator endpoints. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- Evaluator Examples -------- import asyncio from langfuse import AsyncLangfuseAPI client = AsyncLangfuseAPI( x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", username="YOUR_USERNAME", password="YOUR_PASSWORD", base_url="https://yourhost.com/path/to/api", ) async def main() -> None: await client.unstable.evaluators.get( evaluator_id="evaluatorId", ) asyncio.run(main()) """ _response = await self._raw_client.get( evaluator_id, request_options=request_options ) return _response.data async def delete( self, evaluator_id: str, *, request_options: typing.Optional[RequestOptions] = None, ) -> DeleteEvaluatorResponse: """ Delete an evaluator. Important behavior: - This deletes the evaluator including all of its stored versions; `evaluatorId` may reference any version. - The API returns `409` while evaluation rules still reference the evaluator. Delete those evaluation rules first. - Langfuse-managed evaluators (`scope=managed`) cannot be deleted; the API returns `403`. - Scores already produced by the evaluator are not deleted. Parameters ---------- evaluator_id : str Evaluator identifier returned by the evaluator endpoints. request_options : typing.Optional[RequestOptions] Request-specific configuration. Returns ------- DeleteEvaluatorResponse Examples -------- import asyncio from langfuse import AsyncLangfuseAPI client = AsyncLangfuseAPI( x_langfuse_sdk_name="YOUR_X_LANGFUSE_SDK_NAME", x_langfuse_sdk_version="YOUR_X_LANGFUSE_SDK_VERSION", x_langfuse_public_key="YOUR_X_LANGFUSE_PUBLIC_KEY", username="YOUR_USERNAME", password="YOUR_PASSWORD", base_url="https://yourhost.com/path/to/api", ) async def main() -> None: await client.unstable.evaluators.delete( evaluator_id="evaluatorId", ) asyncio.run(main()) """ _response = await self._raw_client.delete( evaluator_id, request_options=request_options ) return _response.data