bot_ym/iAOP
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 完成 issue #47 Prompt 版本管理 + 幻觉校验中间件（版本库联动 + 评测报告脚本）

Browse Source
This commit is contained in:
yunmei
2026-08-04 23:15:39 +08:00
parent 34c15f7ff8
commit 8dc379c479
7 changed files with 509 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/llm-gateway/tests/test_evaluate_script.py
+55
View File
@@ -0,0 +1,55 @@
# -*- coding: utf-8 -*-
"""评测报告脚本（evaluate_hallucination.py）端到端测试：--demo / --samples / --output。"""
import json
import os
import subprocess
import sys
import tempfile
import unittest
LLM_GW_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SCRIPT = os.path.join(LLM_GW_DIR, "evaluate_hallucination.py")
def _run(args, cwd):
env = dict(os.environ)
env["PYTHONIOENCODING"] = "utf-8" # 避免 Windows 控制台 GBK 编码问题
return subprocess.run(
[sys.executable, SCRIPT] + args,
cwd=cwd, capture_output=True, text=True, encoding="utf-8", env=env,
timeout=60,
)
class EvaluateScriptTest(unittest.TestCase):
def test_demo_to_stdout(self):
proc = _run(["--demo"], cwd=LLM_GW_DIR)
self.assertEqual(proc.returncode, 0, proc.stderr)
self.assertIn("LLM 网关 · 幻觉/事实性校验评测报告", proc.stdout)
self.assertIn("样本总数", proc.stdout)
self.assertIn("支持率", proc.stdout)
self.assertIn("按 Prompt 版本分解", proc.stdout)
def test_samples_json_to_output_file(self):
samples = [
{"answer": "好[来源: X]", "sources": ["X"], "prompt_name": "qa"},
{"answer": "坏[来源: 不存在]", "sources": ["Y"], "prompt_name": "qa"},
]
with tempfile.TemporaryDirectory() as tmp:
samples_path = os.path.join(tmp, "samples.json")
report_path = os.path.join(tmp, "report.md")
with open(samples_path, "w", encoding="utf-8") as fh:
json.dump(samples, fh, ensure_ascii=False)
proc = _run(["--samples", samples_path, "--output", report_path],
cwd=LLM_GW_DIR)
self.assertEqual(proc.returncode, 0, proc.stderr)
self.assertTrue(os.path.exists(report_path))
with open(report_path, "r", encoding="utf-8") as fh:
text = fh.read()
self.assertIn("样本总数：2", text)
self.assertIn("qa@", text) # 与版本库联动：name@version 分解
if __name__ == "__main__":
unittest.main()
core/llm-gateway/tests/test_hallucination.py
+74
View File
@@ -87,10 +87,13 @@ class EvaluateTest(unittest.TestCase):
# 支持 2 条（a/c），human_review 1 条（c）
self.assertEqual(report["supported_rate"], round(2 / 3, 4))
self.assertEqual(report["human_review_rate"], round(1 / 3, 4))
self.assertEqual(len(report["samples"]), 3)
self.assertEqual(report["prompts"], {})
def test_empty_samples(self):
report = self.guard.evaluate([])
self.assertEqual(report["total"], 0)
self.assertEqual(report["samples"], [])
def test_audit_records(self):
self.guard.check("x[来源: A]", sources=["A"])
@@ -99,5 +102,76 @@ class EvaluateTest(unittest.TestCase):
self.assertEqual(records[0]["action"], "pass")
class PromptLinkageTest(unittest.TestCase):
"""Issue #47：与 Prompt 版本库的联动（check / evaluate 按 name@version 分解）。"""
def setUp(self):
from llm_gateway.prompts import PromptRegistry
self.guard = HallucinationGuard()
self.reg = PromptRegistry()
self.reg.update("qa", "问题：{query}", "1.0.0")
self.reg.update("qa", "问题：{query} 请引用SOP", "1.0.1")
self.reg.promote("qa", "1.0.1")
def test_check_records_prompt_version(self):
verdict = self.guard.check(
"按SOP处理。[来源: SOP]", sources=["SOP"],
prompt_name="qa", prompt_version="1.0.0",
)
self.assertEqual(verdict.prompt_name, "qa")
self.assertEqual(verdict.prompt_version, "1.0.0")
records = self.guard.drain_audit()
self.assertEqual(records[0]["prompt_name"], "qa")
self.assertEqual(records[0]["prompt_version"], "1.0.0")
def test_evaluate_breakdown_by_prompt_version(self):
samples = [
{"answer": "好[来源: X]", "sources": ["X"], "prompt_name": "qa", "prompt_version": "1.0.0"},
{"answer": "坏[来源: Y]", "sources": ["Z"], "prompt_name": "qa", "prompt_version": "1.0.0"},
{"answer": "好[来源: X]", "sources": ["X"], "prompt_name": "qa"}, # 缺省取当前默认 1.0.1
]
report = self.guard.evaluate(samples, registry=self.reg)
prompts = report["prompts"]
self.assertIn("qa@1.0.0", prompts)
self.assertIn("qa@1.0.1", prompts)
self.assertEqual(prompts["qa@1.0.0"]["total"], 2)
self.assertEqual(prompts["qa@1.0.0"]["supported"], 1)
self.assertEqual(prompts["qa@1.0.1"]["total"], 1)
# 逐样本记录携带 prompt 标签
self.assertEqual(report["samples"][0]["prompt"], "qa@1.0.0")
def test_evaluate_explicit_version_validated(self):
samples = [
{"answer": "x[来源: A]", "sources": ["A"], "prompt_name": "qa", "prompt_version": "1.0.1"},
]
report = self.guard.evaluate(samples, registry=self.reg)
self.assertIn("qa@1.0.1", report["prompts"])
self.assertNotIn("prompt_error", report["samples"][0])
def test_evaluate_unknown_prompt_does_not_crash(self):
samples = [
{"answer": "x[来源: A]", "sources": ["A"], "prompt_name": "not_exist"},
]
report = self.guard.evaluate(samples, registry=self.reg)
self.assertEqual(report["total"], 1)
self.assertEqual(report["samples"][0]["prompt"], "not_exist@unknown")
self.assertIn("prompt_error", report["samples"][0])
def test_render_report_contains_rates_and_prompt_section(self):
samples = [
{"answer": "好[来源: X]", "sources": ["X"], "prompt_name": "qa", "prompt_version": "1.0.0"},
{"answer": "坏[来源: 不存在]", "sources": ["Z"], "prompt_name": "qa", "prompt_version": "1.0.0"},
]
report = self.guard.evaluate(samples, registry=self.reg)
text = self.guard.render_evaluation_report(report)
self.assertIn("样本总数：2", text)
self.assertIn("支持率", text)
self.assertIn("## 按 Prompt 版本分解", text)
self.assertIn("qa@1.0.0", text)
self.assertIn("## 未通过样本明细", text)
self.assertIn("不存在", text)
if __name__ == "__main__":
unittest.main()