# -*- coding: utf-8 -*- """幻觉/事实性校验（hallucination）单元测试：引用溯源 / 信度阈值 / 评测。""" import os import sys import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import _bootstrap # noqa: F401 from llm_gateway.hallucination import HallucinationGuard # noqa: E402 class CitationCheckTest(unittest.TestCase): def setUp(self): self.guard = HallucinationGuard(default_threshold=0.8) def test_declared_sources_are_verified(self): verdict = self.guard.check( answer="炉温偏高应降温。[来源: 沸腾氯化炉异常处置SOP]", sources=["沸腾氯化炉异常处置SOP", "交接班规范"], ) self.assertTrue(verdict.supported) self.assertEqual(verdict.action, "pass") def test_missing_source_is_unsupported(self): verdict = self.guard.check( answer="应停机。[来源: 不存在的文档]", sources=["沸腾氯化炉异常处置SOP"], ) self.assertFalse(verdict.supported) self.assertEqual(verdict.action, "unsupported") self.assertIn("不存在的文档", verdict.missing_sources) def test_no_citation_is_supported(self): # 无引用声明 = 不判幻觉（引用为强制项由 RAG 模板保证） verdict = self.guard.check(answer="按操作规程执行。", sources=[]) self.assertTrue(verdict.supported) self.assertEqual(verdict.action, "pass") class ConfidenceThresholdTest(unittest.TestCase): def setUp(self): self.guard = HallucinationGuard(default_threshold=0.8) def test_low_confidence_high_stakes_human_review(self): verdict = self.guard.check( answer="建议立即停机。[来源: SOP]", sources=["SOP"], confidence=0.55, high_stakes=True, ) self.assertEqual(verdict.action, "human_review") def test_high_confidence_high_stakes_passes(self): verdict = self.guard.check( answer="建议观察并记录。[来源: SOP]", sources=["SOP"], confidence=0.95, high_stakes=True, ) self.assertEqual(verdict.action, "pass") def test_low_confidence_non_stakes_passes(self): # 非高利害场景不启用阈值 verdict = self.guard.check( answer="一般说明。[来源: 手册]", sources=["手册"], confidence=0.3, high_stakes=False, ) self.assertEqual(verdict.action, "pass") def test_custom_threshold(self): verdict = self.guard.check( answer="处置建议。[来源: 手册]", sources=["手册"], confidence=0.7, high_stakes=True, threshold=0.6, ) self.assertEqual(verdict.action, "pass") class EvaluateTest(unittest.TestCase): def setUp(self): self.guard = HallucinationGuard() def test_evaluate_rates(self): samples = [ {"answer": "a[来源: X]", "sources": ["X"], "confidence": 0.9, "high_stakes": True}, {"answer": "b[来源: Y]", "sources": ["Z"], "confidence": 0.9}, {"answer": "c[来源: X]", "sources": ["X"], "confidence": 0.4, "high_stakes": True}, ] report = self.guard.evaluate(samples) self.assertEqual(report["total"], 3) # 支持 2 条（a/c），human_review 1 条（c） self.assertEqual(report["supported_rate"], round(2 / 3, 4)) self.assertEqual(report["human_review_rate"], round(1 / 3, 4)) self.assertEqual(len(report["samples"]), 3) self.assertEqual(report["prompts"], {}) def test_empty_samples(self): report = self.guard.evaluate([]) self.assertEqual(report["total"], 0) self.assertEqual(report["samples"], []) def test_audit_records(self): self.guard.check("x[来源: A]", sources=["A"]) records = self.guard.drain_audit() self.assertEqual(len(records), 1) self.assertEqual(records[0]["action"], "pass") class PromptLinkageTest(unittest.TestCase): """Issue #47：与 Prompt 版本库的联动（check / evaluate 按 name@version 分解）。""" def setUp(self): from llm_gateway.prompts import PromptRegistry self.guard = HallucinationGuard() self.reg = PromptRegistry() self.reg.update("qa", "问题：{query}", "1.0.0") self.reg.update("qa", "问题：{query} 请引用SOP", "1.0.1") self.reg.promote("qa", "1.0.1") def test_check_records_prompt_version(self): verdict = self.guard.check( "按SOP处理。[来源: SOP]", sources=["SOP"], prompt_name="qa", prompt_version="1.0.0", ) self.assertEqual(verdict.prompt_name, "qa") self.assertEqual(verdict.prompt_version, "1.0.0") records = self.guard.drain_audit() self.assertEqual(records[0]["prompt_name"], "qa") self.assertEqual(records[0]["prompt_version"], "1.0.0") def test_evaluate_breakdown_by_prompt_version(self): samples = [ {"answer": "好[来源: X]", "sources": ["X"], "prompt_name": "qa", "prompt_version": "1.0.0"}, {"answer": "坏[来源: Y]", "sources": ["Z"], "prompt_name": "qa", "prompt_version": "1.0.0"}, {"answer": "好[来源: X]", "sources": ["X"], "prompt_name": "qa"}, # 缺省取当前默认 1.0.1 ] report = self.guard.evaluate(samples, registry=self.reg) prompts = report["prompts"] self.assertIn("qa@1.0.0", prompts) self.assertIn("qa@1.0.1", prompts) self.assertEqual(prompts["qa@1.0.0"]["total"], 2) self.assertEqual(prompts["qa@1.0.0"]["supported"], 1) self.assertEqual(prompts["qa@1.0.1"]["total"], 1) # 逐样本记录携带 prompt 标签 self.assertEqual(report["samples"][0]["prompt"], "qa@1.0.0") def test_evaluate_explicit_version_validated(self): samples = [ {"answer": "x[来源: A]", "sources": ["A"], "prompt_name": "qa", "prompt_version": "1.0.1"}, ] report = self.guard.evaluate(samples, registry=self.reg) self.assertIn("qa@1.0.1", report["prompts"]) self.assertNotIn("prompt_error", report["samples"][0]) def test_evaluate_unknown_prompt_does_not_crash(self): samples = [ {"answer": "x[来源: A]", "sources": ["A"], "prompt_name": "not_exist"}, ] report = self.guard.evaluate(samples, registry=self.reg) self.assertEqual(report["total"], 1) self.assertEqual(report["samples"][0]["prompt"], "not_exist@unknown") self.assertIn("prompt_error", report["samples"][0]) def test_render_report_contains_rates_and_prompt_section(self): samples = [ {"answer": "好[来源: X]", "sources": ["X"], "prompt_name": "qa", "prompt_version": "1.0.0"}, {"answer": "坏[来源: 不存在]", "sources": ["Z"], "prompt_name": "qa", "prompt_version": "1.0.0"}, ] report = self.guard.evaluate(samples, registry=self.reg) text = self.guard.render_evaluation_report(report) self.assertIn("样本总数：2", text) self.assertIn("支持率", text) self.assertIn("## 按 Prompt 版本分解", text) self.assertIn("qa@1.0.0", text) self.assertIn("## 未通过样本明细", text) self.assertIn("不存在", text) if __name__ == "__main__": unittest.main()