314 lines
13 KiB
Python
314 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""断点续传与丢失率 ≤0.02% 验证脚本 —— issue #26 / PRD 5.1·9 章验收口径。
|
||
|
||
验证内容（对齐父 Issue #3「① 边缘采集网关 模板化封装」验收基线）：
|
||
1. **断点续传**：Kafka 故障（上行降级 spool-only）期间样本全部落盘本地 spool；
|
||
模拟网关重启后 pending 记录全量重发、逐条 ack，零丢失、内容完全一致；
|
||
2. **ack 删除**：上行确认（ack）后 spool 记录精确删除，不重复、不残留；
|
||
3. **端到端丢失率 ≤ 0.02%**：600 点位 1Hz（对齐 PRD 5.1 压测口径）+ 故障窗口，
|
||
HealthMetrics 丢失率 ≤ 0.02%、P99 ≤ 1.8s、可用性 ≥ 99.8%（meets_sla），
|
||
故障结束后 spool 最终排空（全部上行确认）。
|
||
|
||
用法：
|
||
python verify_breakpoint_resume.py [--points 600] [--rounds 60] \
|
||
[--outage-rounds 10] [--seed 2026]
|
||
|
||
退出码：0 = 全部通过（PASS）；1 = 任一检查失败（FAIL）。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import os
|
||
import random
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
from typing import Dict, List, Optional
|
||
|
||
# 允许从任意 cwd 以脚本方式运行（python verify_breakpoint_resume.py）
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
|
||
from collector import CollectorEngine, HealthMetrics, SpoolStore # noqa: E402
|
||
from drivers import SimulatorDriver # noqa: E402
|
||
from drivers.base import Driver, SampleValue # noqa: E402
|
||
from point_dict.loader import Point, PointDict # noqa: E402
|
||
|
||
LOSS_RATE_SLA = 0.0002 # 丢失率 ≤ 0.02%（PRD 5.1）
|
||
P99_SLA = 1.8 # 采集 P99 ≤ 1.8s
|
||
AVAILABILITY_SLA = 0.998 # 可用性 ≥ 99.8%
|
||
|
||
|
||
# ----------------------------------------------------------------------
|
||
# 辅助：点位字典 / 抖动驱动 / 模拟 Kafka 上行（含故障窗口）
|
||
# ----------------------------------------------------------------------
|
||
def make_point_dict(n_points: int, seed: int = 2026) -> PointDict:
|
||
"""生成 n_points 个测点（设备前缀 CLF-01..CLF-05，对齐压测口径）。"""
|
||
points: List[Point] = []
|
||
n_devices = 5
|
||
for i in range(n_points):
|
||
dev = f"CLF-{i % n_devices + 1:02d}"
|
||
points.append(
|
||
Point(
|
||
device_id=dev,
|
||
point_id=f"{dev}.P{i:04d}",
|
||
name=f"测点{i + 1}",
|
||
unit="℃",
|
||
data_type="float",
|
||
sample_rate=1000,
|
||
quality_code=True,
|
||
row_number=i + 2,
|
||
)
|
||
)
|
||
return PointDict(points)
|
||
|
||
|
||
class FlakySimulatorDriver(Driver):
|
||
"""模拟驱动包装：以 drop_probability 随机丢点，验证丢失率统计口径。
|
||
|
||
丢点比例默认 0.01%（=0.0001），低于 0.02% 验收基线，
|
||
用于证明“采集侧偶发未读”被正确计入丢失率且仍满足 SLA。
|
||
"""
|
||
|
||
protocol = "simulator-flaky"
|
||
|
||
def __init__(self, drop_probability: float = 0.0001, seed: int = 2026):
|
||
super().__init__(None)
|
||
self._inner = SimulatorDriver({"seed": seed})
|
||
self.drop_probability = drop_probability
|
||
self._rng = random.Random(seed + 1)
|
||
|
||
def connect(self) -> None:
|
||
self._inner.connect()
|
||
|
||
def read_points(self, points: List[Point]) -> Dict[str, SampleValue]:
|
||
values = self._inner.read_points(points)
|
||
for p in points:
|
||
if self._rng.random() < self.drop_probability:
|
||
values.pop(p.point_id, None) # 未读到 → 引擎计入丢失
|
||
return values
|
||
|
||
def close(self) -> None:
|
||
self._inner.close()
|
||
|
||
|
||
class FakeKafkaSink:
|
||
"""模拟 Kafka 上行通道：up 时确认删除 spool，down 时保留（断点续传场景）。
|
||
|
||
行为对齐 upstream/kafka_sink.py：发送成功即按样本 ack 删除 spool 记录；
|
||
down 期间样本留在 spool，等待恢复后重发。
|
||
"""
|
||
|
||
def __init__(self, spool: SpoolStore):
|
||
self.spool = spool
|
||
self.up = True
|
||
self.published = 0
|
||
|
||
def publish(self, samples: List[dict]) -> int:
|
||
if not self.up:
|
||
return 0 # 上行故障：样本保留在 spool
|
||
ok = 0
|
||
for s in samples:
|
||
self.spool.ack(
|
||
{"device_id": s["device_id"], "point_id": s["point_id"],
|
||
"value": s["value"], "ts": s["ts"]}
|
||
)
|
||
ok += 1
|
||
self.published += ok
|
||
return ok
|
||
|
||
|
||
# ----------------------------------------------------------------------
|
||
# 检查 1：断点续传 —— 重启重发零丢失、内容一致
|
||
# ----------------------------------------------------------------------
|
||
def check_resume_replay(workdir: str, n_points: int, n_rounds: int, seed: int) -> dict:
|
||
"""Kafka 故障期间样本全部落盘；模拟重启后全量重发、逐条 ack。"""
|
||
spool_dir = os.path.join(workdir, "spool-resume")
|
||
spool = SpoolStore(spool_dir)
|
||
metrics = HealthMetrics()
|
||
engine = CollectorEngine(
|
||
point_dict=make_point_dict(n_points, seed),
|
||
driver_slots=[("simulator-flaky", FlakySimulatorDriver(seed=seed), [])],
|
||
spool=spool,
|
||
metrics=metrics,
|
||
interval_ms=1000,
|
||
max_pending=10 ** 9,
|
||
)
|
||
# 故障窗口：sink=None（spool-only 降级），样本只落盘不上行
|
||
for _ in range(n_rounds):
|
||
engine.collect_once(sink=None)
|
||
|
||
before = spool.pending_records()
|
||
assert len(before) > 0, "故障窗口内应产生待上行样本"
|
||
|
||
# 模拟网关重启：新 SpoolStore（同一目录）+ 重发 pending
|
||
spool2 = SpoolStore(spool_dir)
|
||
replay = spool2.pending_records()
|
||
assert len(replay) == len(before), "重启后重发条数应与故障期间采集数一致"
|
||
for rec, orig in zip(replay, before):
|
||
assert rec["device_id"] == orig["device_id"]
|
||
assert rec["point_id"] == orig["point_id"]
|
||
assert rec["value"] == orig["value"]
|
||
assert abs(rec["ts"] - orig["ts"]) < 1e-6
|
||
# 重发成功 → 逐条 ack 删除
|
||
for rec in replay:
|
||
spool2.ack(rec)
|
||
assert spool2.total_pending() == 0, "重发并 ack 后 spool 应排空"
|
||
|
||
return {"collected": len(before), "replayed": len(replay), "remaining": 0}
|
||
|
||
|
||
# ----------------------------------------------------------------------
|
||
# 检查 2：ack 删除 —— 上行确认后精确删除、不残留
|
||
# ----------------------------------------------------------------------
|
||
def check_ack_delete(workdir: str, n_points: int, seed: int) -> dict:
|
||
spool = SpoolStore(os.path.join(workdir, "spool-ack"))
|
||
engine = CollectorEngine(
|
||
point_dict=make_point_dict(n_points, seed),
|
||
driver_slots=[("simulator", SimulatorDriver({"seed": seed}), [])],
|
||
spool=spool,
|
||
metrics=HealthMetrics(),
|
||
interval_ms=1000,
|
||
max_pending=10 ** 9,
|
||
)
|
||
engine.collect_once(sink=None)
|
||
n = spool.total_pending()
|
||
assert n == n_points, f"单轮应写入 {n_points} 条，实际 {n}"
|
||
|
||
# 模拟 Kafka 投递确认：按 point_id ack（与 kafka_sink._on_delivery 相同口径）
|
||
for rec in spool.pending_records():
|
||
spool.ack({"device_id": None, "point_id": rec["point_id"],
|
||
"value": None, "ts": None})
|
||
assert spool.total_pending() == 0, "全部确认后 spool 应清零"
|
||
|
||
# 再采一轮：确认新样本正常追加、无残留干扰
|
||
engine.collect_once(sink=None)
|
||
assert spool.total_pending() == n_points, "ack 后新样本应精确追加"
|
||
|
||
return {"acked": n, "remaining": 0}
|
||
|
||
|
||
# ----------------------------------------------------------------------
|
||
# 检查 3：端到端丢失率 ≤ 0.02%（600 点位 1Hz + 故障窗口 + 恢复重发）
|
||
# ----------------------------------------------------------------------
|
||
def check_end_to_end_loss_rate(
|
||
workdir: str, n_points: int, n_rounds: int, outage_rounds: int, seed: int
|
||
) -> dict:
|
||
spool_dir = os.path.join(workdir, "spool-e2e")
|
||
spool = SpoolStore(spool_dir)
|
||
metrics = HealthMetrics()
|
||
engine = CollectorEngine(
|
||
point_dict=make_point_dict(n_points, seed),
|
||
driver_slots=[("simulator-flaky", FlakySimulatorDriver(seed=seed), [])],
|
||
spool=spool,
|
||
metrics=metrics,
|
||
interval_ms=1000,
|
||
max_pending=10 ** 9,
|
||
)
|
||
sink = FakeKafkaSink(spool)
|
||
|
||
outage_start = max(1, n_rounds - outage_rounds - 1)
|
||
peak_pending = 0
|
||
for r in range(n_rounds):
|
||
sink.up = r >= outage_start # 故障窗口内 Kafka 不可用
|
||
engine.collect_once(sink=sink)
|
||
peak_pending = max(peak_pending, spool.total_pending())
|
||
|
||
# 恢复 + 模拟重启重发：pending 全量上行确认
|
||
sink.up = True
|
||
for rec in spool.pending_records():
|
||
sink.spool.ack(rec)
|
||
|
||
snap = metrics.snapshot()
|
||
ok_loss = metrics.loss_rate <= LOSS_RATE_SLA
|
||
ok_p99 = metrics.p99_latency() <= P99_SLA
|
||
ok_avail = metrics.availability >= AVAILABILITY_SLA
|
||
ok_drain = spool.total_pending() == 0
|
||
assert ok_loss, f"丢失率 {metrics.loss_rate:.6f} > {LOSS_RATE_SLA}（0.02%）"
|
||
assert ok_p99, f"P99 {metrics.p99_latency():.3f}s > {P99_SLA}s"
|
||
assert ok_avail, f"可用性 {metrics.availability:.6f} < {AVAILABILITY_SLA}"
|
||
assert ok_drain, "故障恢复后 spool 应排空（全部上行确认）"
|
||
assert metrics.meets_sla(), "HealthMetrics.meets_sla() 应为 True"
|
||
|
||
snap["peak_pending"] = peak_pending
|
||
snap["outage_rounds"] = outage_rounds
|
||
return snap
|
||
|
||
|
||
# ----------------------------------------------------------------------
|
||
# 主流程
|
||
# ----------------------------------------------------------------------
|
||
def parse_args(argv: List[str]) -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser(
|
||
description="iAOP 边缘采集网关：断点续传与丢失率≤0.02% 验证脚本（issue #26）"
|
||
)
|
||
parser.add_argument("--points", type=int, default=600,
|
||
help="模拟点位数量（默认 600，对齐 PRD 5.1 压测口径）")
|
||
parser.add_argument("--rounds", type=int, default=60,
|
||
help="端到端采集轮数（默认 60）")
|
||
parser.add_argument("--outage-rounds", type=int, default=10,
|
||
help="Kafka 故障窗口轮数（默认 10，验证断点续传）")
|
||
parser.add_argument("--seed", type=int, default=2026,
|
||
help="随机种子（默认 2026，保证可复现）")
|
||
return parser.parse_args(argv)
|
||
|
||
|
||
def main(argv: Optional[List[str]] = None) -> int:
|
||
args = parse_args(argv if argv is not None else sys.argv[1:])
|
||
print("=" * 68)
|
||
print("iAOP 边缘采集网关 · 断点续传与丢失率≤0.02% 验证脚本")
|
||
print(f"点位={args.points} 轮数={args.rounds} 故障窗口={args.outage_rounds} "
|
||
f"种子={args.seed}")
|
||
print("=" * 68)
|
||
|
||
results: List[tuple] = []
|
||
with tempfile.TemporaryDirectory(prefix="verify-bpr-") as workdir:
|
||
# 检查 1：断点续传
|
||
t0 = time.monotonic()
|
||
r1 = check_resume_replay(workdir, args.points, args.outage_rounds, args.seed)
|
||
r1["elapsed"] = time.monotonic() - t0
|
||
results.append(("断点续传（故障期全落盘 → 重启全量重发 → ack 排空）",
|
||
f"采集 {r1['collected']} 条 / 重发 {r1['replayed']} 条 / 残留 {r1['remaining']} 条",
|
||
r1["collected"] == r1["replayed"] and r1["remaining"] == 0))
|
||
|
||
# 检查 2：ack 删除
|
||
t0 = time.monotonic()
|
||
r2 = check_ack_delete(workdir, min(args.points, 200), args.seed)
|
||
r2["elapsed"] = time.monotonic() - t0
|
||
results.append(("上行确认删除（ack 后精确删除、无残留）",
|
||
f"确认 {r2['acked']} 条 / 残留 {r2['remaining']} 条",
|
||
r2["remaining"] == 0))
|
||
|
||
# 检查 3：端到端丢失率
|
||
t0 = time.monotonic()
|
||
r3 = check_end_to_end_loss_rate(
|
||
workdir, args.points, args.rounds, args.outage_rounds, args.seed
|
||
)
|
||
r3["elapsed"] = time.monotonic() - t0
|
||
results.append(
|
||
("端到端丢失率 ≤ 0.02%",
|
||
f"样本 {r3['total_samples']} / 丢失 {r3['lost_samples']} / "
|
||
f"丢失率 {r3['loss_rate']:.6f} / P99 {r3['p99_latency_sec']}s / "
|
||
f"可用性 {r3['availability']:.6f} / spool 峰值 {r3['peak_pending']}",
|
||
r3["loss_rate"] <= LOSS_RATE_SLA
|
||
and r3["p99_latency_sec"] <= P99_SLA
|
||
and r3["availability"] >= AVAILABILITY_SLA
|
||
and r3["peak_pending"] > 0 # 故障窗口确实产生了 spool 堆积
|
||
and r3["total_samples"] - r3["lost_samples"] >= 0))
|
||
|
||
print("-" * 68)
|
||
all_pass = True
|
||
for name, detail, ok in results:
|
||
all_pass = all_pass and ok
|
||
print(f"[{'PASS' if ok else 'FAIL'}] {name}")
|
||
print(f" {detail}")
|
||
print("-" * 68)
|
||
if all_pass:
|
||
print("结论：全部通过 —— 断点续传零丢失，丢失率/P99/可用性满足 PRD 5.1 验收基线。")
|
||
return 0
|
||
print("结论：存在失败项 —— 请检查网关实现后重跑。")
|
||
return 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|