> ## Documentation Index
> Fetch the complete documentation index at: https://phidatainc.mintlify.site/llms.txt
> Use this file to discover all available pages before exploring further.

# Running Evaluations with AgentOSClient

> Trigger accuracy, performance and reliability evals (with expected tool calls and argument matching) against a remote agent, then list eval runs and fetch one run's details.

Run and manage evaluations using AgentOSClient.

```python run_evals.py theme={null}
"""
Running Evaluations with AgentOSClient

This example demonstrates how to run and manage evaluations
using AgentOSClient.

Prerequisites:
1. Start an AgentOS server with agents
2. Run this script: python 08_run_evals.py
"""

import asyncio

from agno.client import AgentOSClient
from agno.db.schemas.evals import EvalType

# ---------------------------------------------------------------------------
# Create Example
# ---------------------------------------------------------------------------


async def run_accuracy_eval():
    """Run an accuracy evaluation."""
    print("=" * 60)
    print("Running Accuracy Evaluation")
    print("=" * 60)

    client = AgentOSClient(base_url="http://localhost:7777")

    # Get available agents
    config = await client.aget_config()
    if not config.agents:
        print("No agents available")
        return

    agent_id = config.agents[0].id
    print(f"Evaluating agent: {agent_id}")

    # Run accuracy eval
    try:
        eval_result = await client.run_eval(
            agent_id=agent_id,
            eval_type=EvalType.ACCURACY,
            input_text="What is 2 + 2?",
            expected_output="4",
        )

        if eval_result:
            print(f"\nEval ID: {eval_result.id}")
            print(f"Eval Type: {eval_result.eval_type}")
            print(f"Eval Data: {eval_result.eval_data}")
        else:
            print("Evaluation returned no result")

    except Exception as e:
        print(f"Error running eval: {e}")
        if hasattr(e, "response"):
            print(f"Response: {e.response.text}")


async def run_performance_eval():
    """Run a performance evaluation."""
    print("\n" + "=" * 60)
    print("Running Performance Evaluation")
    print("=" * 60)

    client = AgentOSClient(base_url="http://localhost:7777")

    # Get available agents
    config = await client.aget_config()
    if not config.agents:
        print("No agents available")
        return

    agent_id = config.agents[0].id
    print(f"Evaluating agent: {agent_id}")

    # Run performance eval
    try:
        eval_result = await client.run_eval(
            agent_id=agent_id,
            eval_type=EvalType.PERFORMANCE,
            input_text="Hello, how are you?",
            num_iterations=2,  # Run twice to measure performance
        )

        if eval_result:
            print(f"\nEval ID: {eval_result.id}")
            print(f"Eval Type: {eval_result.eval_type}")
            print(f"Performance Data: {eval_result.eval_data}")
        else:
            print("Evaluation returned no result")

    except Exception as e:
        print(f"Error running eval: {e}")
        if hasattr(e, "response"):
            print(f"Response: {e.response.text}")


async def list_eval_runs():
    """List all evaluation runs."""
    print("\n" + "=" * 60)
    print("Listing Evaluation Runs")
    print("=" * 60)

    client = AgentOSClient(base_url="http://localhost:7777")

    try:
        evals = await client.list_eval_runs()
        print(f"\nFound {len(evals.data)} evaluation runs")

        for eval_run in evals.data[:5]:  # Show first 5
            print(f"\n- ID: {eval_run.id}")
            print(f"  Name: {eval_run.name}")
            print(f"  Type: {eval_run.eval_type}")
            print(f"  Agent: {eval_run.agent_id}")

    except Exception as e:
        print(f"Error listing evals: {e}")


async def get_eval_details():
    """Get details of a specific evaluation."""
    print("\n" + "=" * 60)
    print("Getting Evaluation Details")
    print("=" * 60)

    client = AgentOSClient(base_url="http://localhost:7777")

    try:
        # First list evals to get an ID
        evals = await client.list_eval_runs()
        if not evals.data:
            print("No evaluations found")
            return

        eval_id = evals.data[0].id
        print(f"Getting details for eval: {eval_id}")

        eval_run = await client.get_eval_run(eval_id)
        print(f"\nEval ID: {eval_run.id}")
        print(f"Name: {eval_run.name}")
        print(f"Type: {eval_run.eval_type}")
        print(f"Agent ID: {eval_run.agent_id}")
        print(f"Data: {eval_run.eval_data}")

    except Exception as e:
        print(f"Error getting eval: {e}")


async def run_reliability_eval():
    """Run a reliability evaluation with subset matching and argument validation."""
    print("\n" + "=" * 60)
    print("Running Reliability Evaluation")
    print("=" * 60)

    client = AgentOSClient(base_url="http://localhost:7777")

    # Get available agents
    config = await client.aget_config()
    if not config.agents:
        print("No agents available")
        return

    agent_id = config.agents[0].id
    print(f"Evaluating agent: {agent_id}")

    # Run reliability eval with subset matching
    try:
        eval_result = await client.run_eval(
            agent_id=agent_id,
            eval_type=EvalType.RELIABILITY,
            input_text="What is 10 * 5?",
            expected_tool_calls=["multiply"],
            allow_additional_tool_calls=True,
            expected_tool_call_arguments={"multiply": {"a": 10, "b": 5}},
        )

        if eval_result:
            print(f"\nEval ID: {eval_result.id}")
            print(f"Eval Type: {eval_result.eval_type}")
            print(f"Eval Data: {eval_result.eval_data}")
            print(f"Eval Input: {eval_result.eval_input}")
        else:
            print("Evaluation returned no result")

    except Exception as e:
        print(f"Error running eval: {e}")
        if hasattr(e, "response"):
            print(f"Response: {e.response.text}")


async def main():
    await run_accuracy_eval()
    await run_performance_eval()
    await run_reliability_eval()
    await list_eval_runs()
    await get_eval_details()


# ---------------------------------------------------------------------------
# Run Example
# ---------------------------------------------------------------------------

if __name__ == "__main__":
    asyncio.run(main())
```

## Run the Example

<Steps>
  <Snippet file="create-venv-step.mdx" />

  <Step title="Install dependencies">
    ```bash theme={null}
    uv pip install -U "agno[os]" chromadb ddgs openai
    ```
  </Step>

  <Step title="Export your API keys">
    <CodeGroup>
      ```bash Mac/Linux theme={null}
      export OPENAI_API_KEY="your_openai_api_key_here"
      ```

      ```bash Windows theme={null}
      $Env:OPENAI_API_KEY="your_openai_api_key_here"
      ```
    </CodeGroup>
  </Step>

  <Step title="Clone Agno">
    Clone the repository and run the remaining commands from its root:

    ```bash theme={null}
    git clone https://github.com/agno-agi/agno.git
    cd agno
    ```
  </Step>

  <Step title="Start the AgentOS server">
    In another terminal, start the [client example server](/examples/agent-os/client/server) on port 7777:

    ```bash theme={null}
    python cookbook/05_agent_os/client/server.py
    ```
  </Step>

  <Step title="Run the example">
    Run the example from the repository root:

    ```bash theme={null}
    python cookbook/05_agent_os/client/08_run_evals.py
    ```
  </Step>
</Steps>

Full source: [cookbook/05\_agent\_os/client/08\_run\_evals.py](https://github.com/agno-agi/agno/blob/main/cookbook/05_agent_os/client/08_run_evals.py)