feat: 完成 issue #49 ④ 路由准确率评估脚本（≥96.5%）

2026-08-05 02:45:00 +08:00
parent c5a7a0ee52
commit 437771da34
4 changed files with 319 additions and 6 deletions
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""评测报告脚本（evaluate_routing.py）端到端测试：--demo / --samples / --output / 达标线。"""
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+
+LLM_GW_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+SCRIPT = os.path.join(LLM_GW_DIR, "evaluate_routing.py")
+
+
+def _run(args, cwd):
+    env = dict(os.environ)
+    env["PYTHONIOENCODING"] = "utf-8"   # 避免 Windows 控制台 GBK 编码问题
+    return subprocess.run(
+        [sys.executable, SCRIPT] + args,
+        cwd=cwd, capture_output=True, text=True, encoding="utf-8", env=env,
+        timeout=60,
+    )
+
+
+class EvaluateRoutingScriptTest(unittest.TestCase):
+    def test_demo_to_stdout(self):
+        proc = _run(["--demo"], cwd=LLM_GW_DIR)
+        self.assertEqual(proc.returncode, 0, proc.stderr)
+        self.assertIn("LLM 网关 · 敏感度路由准确率评测报告", proc.stdout)
+        self.assertIn("样本总数", proc.stdout)
+        self.assertIn("路由准确率", proc.stdout)
+        self.assertIn("按预期路由目标分解", proc.stdout)
+        self.assertIn("未通过样本明细", proc.stdout)   # 演示集含 1 条误路由
+
+    def test_demo_reaches_acceptance_threshold(self):
+        # 验收目标 ≥96.5%：内置演示评测集必须达标（stderr 简报含“达标”）
+        proc = _run(["--demo"], cwd=LLM_GW_DIR)
+        self.assertEqual(proc.returncode, 0, proc.stderr)
+        self.assertIn("达标", proc.stderr)
+
+    def test_samples_json_to_output_file(self):
+        samples = [
+            {"query": "炉温当前是多少", "expected": "local"},
+            {"query": "海绵钛是什么", "expected": "cloud"},
+            {"query": "请执行停机操作", "expected": "block"},
+        ]
+        with tempfile.TemporaryDirectory() as tmp:
+            samples_path = os.path.join(tmp, "samples.json")
+            report_path = os.path.join(tmp, "report.md")
+            with open(samples_path, "w", encoding="utf-8") as fh:
+                json.dump(samples, fh, ensure_ascii=False)
+
+            proc = _run(["--samples", samples_path, "--output", report_path],
+                        cwd=LLM_GW_DIR)
+            self.assertEqual(proc.returncode, 0, proc.stderr)
+            self.assertTrue(os.path.exists(report_path))
+            with open(report_path, "r", encoding="utf-8") as fh:
+                text = fh.read()
+            self.assertIn("样本总数：3", text)
+            self.assertIn("路由准确率 100.00%", text)   # 3/3 全对
+
+
+if __name__ == "__main__":
+    unittest.main()