Example Eval - OpenAI

from openai import OpenAI
oai = OpenAI()

def openai_agent(case):
    resp = oai.chat.completions.create(
        model="gpt-4o-mini",
        messages=[
            {"role": "system", "content": "You are a helpful support agent."},
            {"role": "user", "content": case.query},
        ],
    )
    return {"output": resp.choices[0].message.content, "metadata": {"model": resp.model}}

report = (
    client.evaluations
    .run(dataset_id="...", subject={"kind": "custom_agent", "displayName": "GPT-4o-mini", "framework": "openai"})
    .execute(openai_agent)
    .finalize()
    .analyze()
)