Evaluation

from azure.ai.evaluation import AzureOpenAIScoreModelGrader

summarization_grader = AzureOpenAIScoreModelGrader(
    model_config=model_config,
    name="summarization_quality",
    prompt="""Rate this news summary on a scale of 1-5.

Article: {{item.article}}
Summary: {{sample.output_text}}

Criteria:
- Captures ALL key facts (who, what, when, where)
- No hallucinated information not in the article
- Concise (under 3 sentences)

Score 1: Missing key facts or hallucinations
Score 3: Captures main point but misses details
Score 5: Perfect summary — all facts, no extras, concise

Return ONLY a number 1-5.""",
    output_type="numeric",
    pass_threshold=3,
)

from azure.ai.evaluation import AzureOpenAIPythonGrader

entity_grader = AzureOpenAIPythonGrader(
    name="entity_extraction_accuracy",
    source="""
import json

def grade(item, sample):
    try:
        extracted = json.loads(sample["output_text"])
        reference = json.loads(item["ground_truth"])
    except (json.JSONDecodeError, KeyError):
        return {"score": 0, "reason": "Invalid JSON output"}

    required_keys = ["people", "organizations", "locations", "dates"]
    missing = [k for k in required_keys if k not in extracted]
    if missing:
        return {"score": 0.5, "reason": f"Missing keys: {missing}"}

    total, matched = 0, 0
    for key in required_keys:
        ref_set = set(str(v).lower() for v in reference.get(key, []))
        ext_set = set(str(v).lower() for v in extracted.get(key, []))
        total += len(ref_set)
        matched += len(ref_set & ext_set)

    score = matched / total if total > 0 else 1.0
    return {"score": score, "reason": f"{matched}/{total} entities matched"}
""",
    pass_threshold=0.7,
)

from azure.ai.evaluation import AzureOpenAIStringCheckGrader

tool_format_grader = AzureOpenAIStringCheckGrader(
    name="tool_call_format",
    input="{{sample.output_text}}",
    operation="like",          # or "eq", "starts_with", "contains"
    reference="function_call",
    pass_threshold=1,
)

classification_grader = AzureOpenAIStringCheckGrader(
    name="classification_accuracy",
    input="{{sample.output_text}}",
    operation="eq",
    reference="{{item.expected_label}}",
    pass_threshold=1,
)

from azure.ai.evaluation import evaluate, F1ScoreEvaluator

result = evaluate(
    data="eval_data.jsonl",
    evaluators={
        "task_grader": my_custom_score_grader,   # primary
        "f1": F1ScoreEvaluator(),                 # token overlap
    },
    output_path="./eval_results.json",
)

for metric, value in result["metrics"].items():
    print(f"{metric}: {value}")