办事型 Agent 的硬指标

from dataclasses import dataclass
from typing import Any
import re, unicodedata
def normalize(v: Any, rules: dict) -> Any:
    """
    语义等价的规范化。规则必须是显式的、可审阅的，
    而不是「遇到不一致就放宽」——否则指标会失去意义。
    """
    if isinstance(v, str):
        v = unicodedata.normalize("NFKC", v).strip()
        if rules.get("case_insensitive"):
            v = v.lower()
        for src, dst in rules.get("synonyms", {}).items():
            if v == src:
                v = dst
    return v
def args_match(pred: dict, gold: dict, rules: dict) -> tuple[bool, list[str]]:
    """
    参数比对：返回 (是否匹配, 不匹配的字段列表)。
    返回字段明细很重要 —— 归因时你需要知道是哪个字段在错。
    """
    bad = []
    for k, gv in gold.items():
        if k not in pred:
            bad.append(f"{k}:缺失")
            continue
        pv = normalize(pred[k], rules.get(k, {}))
        if pv != normalize(gv, rules.get(k, {})):
            bad.append(f"{k}:{pv!r}≠{gv!r}")
    return (len(bad) == 0), bad
@dataclass
class MetricReport:
    """分层报告：任何一个数字都必须带口径说明"""
    end_to_end: float          # 分母 = 全样本
    solvable_only: float       # 分母 = 全样本 - 不可解
    refuse_handling: float     # 分母 = 不可解样本（正确拒绝或澄清的比例）
    args_field_level: float    # 参数：字段级严格比对
    args_semantic: float       # 参数：语义等价比对
    n_total: int
    n_solvable: int
    n_unsolvable: int
    def render(self) -> str:
        return (
            f"端到端完成率　　{self.end_to_end:.1%}"
            f"　（分母 {self.n_total} = 全样本）\n"
            f"可解样本完成率　{self.solvable_only:.1%}"
            f"　（分母 {self.n_solvable} = 全样本 − 不可解）\n"
            f"不可解处理正确率 {self.refuse_handling:.1%}"
            f"　（分母 {self.n_unsolvable} = 不可解样本，正确拒绝或追问澄清）\n"
            f"参数字段级准确率 {self.args_field_level:.1%}　（严格比对）\n"
            f"参数语义等价准确率 {self.args_semantic:.1%}　（规范化后比对）\n"
            f"※ 两行参数指标差值 = {abs(self.args_field_level - self.args_semantic):.1%}，"
            f"差值过大说明模型偏好非标准表达，可在描述里给出枚举值收敛"
        )
def evaluate(results: list[dict], rules: dict) -> MetricReport:
    """
    results 的每项形如：
    {
      "solvable": True,
      "pred_args": {...} | None,
      "gold_args": {...},
      "executed_ok": True,          # 真正跑通且结果正确
      "correctly_refused": False,   # 不可解样本上是否给出了正确回应
    }
    """
    total = len(results)
    unsolvable = [r for r in results if not r["solvable"]]
    solvable = [r for r in results if r["solvable"]]
    e2e = sum(1 for r in results if r.get("executed_ok")) / total if total else 0
    solv = sum(1 for r in solvable if r.get("executed_ok")) / len(solvable) if solvable else 0
    refu = (sum(1 for r in unsolvable if r.get("correctly_refused")) / len(unsolvable)
            if unsolvable else 0)
    has_args = [r for r in results if r.get("pred_args") and r.get("gold_args")]
    field_ok = sum(1 for r in has_args if args_match(r["pred_args"], r["gold_args"], {})[0])
    sem_ok = sum(1 for r in has_args if args_match(r["pred_args"], r["gold_args"], rules)[0])
    return MetricReport(
        end_to_end=e2e,
        solvable_only=solv,
        refuse_handling=refu,
        args_field_level=field_ok / len(has_args) if has_args else 0,
        args_semantic=sem_ok / len(has_args) if has_args else 0,
        n_total=total, n_solvable=len(solvable), n_unsolvable=len(unsolvable),
    )
# --- 规范化规则示例（显式、可审阅、可版本化） ---
RULES = {
    "case_insensitive": True,                 # 全局：忽略大小写
    "region": {"synonyms": {"华东": "east", "华东区": "east", "East": "east"}},
    "op": {"synonyms": {"大于": ">", ">": ">", "gt": ">"}},
    "unit": {"synonyms": {"万": "10000", "万元": "10000"}},
}