feat: 完成 issue #3 边缘采集网关模板化封装
- 点位字典 CSV schema/加载/自动校验（缺失字段/量纲/重复点号，PRD 5.1） - 协议可插拔只读驱动：OPC UA(S7-1200 适配)/S7/Modbus/称重/能源/模拟 - 周期采集引擎：只读+背压保护+健康度指标（丢失率/P99/可用性） - Kafka 流式上行 + 本地 spool 断点续传（丢失率≤0.02% 保障） - 模板配置外置（gateway.yaml + 点位字典 CSV），换行业零改码 - 18 个单元测试全绿；端到端运行 SLA 达标
This commit is contained in:
@@ -0,0 +1,19 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""点位字典（Point Dictionary）模块：CSV schema + 加载 + 自动校验。"""
|
||||
|
||||
from .loader import Point, PointDict, load_point_dict_csv
|
||||
from .schema import CSV_HEADERS, VALID_DATA_TYPES, VALID_UNITS
|
||||
from .validator import ValidationIssue, ValidationReport, validate_point_dict, validate_point_dict_file
|
||||
|
||||
__all__ = [
|
||||
"Point",
|
||||
"PointDict",
|
||||
"load_point_dict_csv",
|
||||
"CSV_HEADERS",
|
||||
"VALID_DATA_TYPES",
|
||||
"VALID_UNITS",
|
||||
"ValidationIssue",
|
||||
"ValidationReport",
|
||||
"validate_point_dict",
|
||||
"validate_point_dict_file",
|
||||
]
|
||||
@@ -0,0 +1,94 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""点位字典 CSV 加载器：CSV → 内存模型（Point 记录列表）。
|
||||
|
||||
复用化工 AI 边缘网关的点位字典机制，改为模板化读取：
|
||||
- 表头必须与 schema.CSV_HEADERS 一致（列顺序不重要，按列名匹配）。
|
||||
- 行为宽松：缺失列/多余列由校验器（validator.py）统一报告，加载器不做丢弃。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from . import schema
|
||||
|
||||
|
||||
@dataclass
|
||||
class Point:
|
||||
"""单条测点记录（对应点位字典 CSV 一行）。"""
|
||||
|
||||
device_id: str
|
||||
point_id: str
|
||||
name: str
|
||||
unit: str
|
||||
data_type: str
|
||||
sample_rate: int
|
||||
quality_code: bool = True
|
||||
opc_node: Optional[str] = None
|
||||
row_number: int = 0 # CSV 行号（从 2 开始，表头为第 1 行），用于报错定位
|
||||
|
||||
@property
|
||||
def topic(self) -> str:
|
||||
"""Kafka 上行 topic（模板化：按设备聚合）。"""
|
||||
return f"{self.device_id}.points"
|
||||
|
||||
|
||||
class PointDict:
|
||||
"""点位字典内存模型。"""
|
||||
|
||||
def __init__(self, points: List[Point]):
|
||||
self.points = points
|
||||
|
||||
def by_point_id(self) -> Dict[str, Point]:
|
||||
return {p.point_id: p for p in self.points}
|
||||
|
||||
def by_device_id(self) -> Dict[str, List[Point]]:
|
||||
grouped: Dict[str, List[Point]] = {}
|
||||
for p in self.points:
|
||||
grouped.setdefault(p.device_id, []).append(p)
|
||||
return grouped
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.points)
|
||||
|
||||
|
||||
def _to_bool(raw: str) -> bool:
|
||||
"""宽松解析布尔列（true/false/1/0/yes/no，大小写不敏感）。"""
|
||||
return raw.strip().lower() in ("1", "true", "yes", "y", "on")
|
||||
|
||||
|
||||
def load_point_dict_csv(path: str) -> PointDict:
|
||||
"""从 CSV 文件加载点位字典。
|
||||
|
||||
Args:
|
||||
path: CSV 文件路径（UTF-8，含表头，表头列名对齐 schema.CSV_HEADERS）。
|
||||
|
||||
Returns:
|
||||
PointDict：点位内存模型。结构/取值问题不在此抛出，
|
||||
统一由 validator.validate_point_dict() 报告（便于聚合展示全部错误）。
|
||||
"""
|
||||
points: List[Point] = []
|
||||
with open(path, "r", encoding="utf-8-sig") as fh:
|
||||
reader = csv.DictReader(fh)
|
||||
fieldnames = list(reader.fieldnames or [])
|
||||
for row_number, row in enumerate(reader, start=2):
|
||||
sample_rate_raw = (row.get("sampleRate") or "").strip()
|
||||
try:
|
||||
sample_rate = int(float(sample_rate_raw)) if sample_rate_raw else 0
|
||||
except ValueError:
|
||||
sample_rate = 0
|
||||
points.append(
|
||||
Point(
|
||||
device_id=(row.get("device_id") or "").strip(),
|
||||
point_id=(row.get("point_id") or "").strip(),
|
||||
name=(row.get("name") or "").strip(),
|
||||
unit=(row.get("unit") or "").strip(),
|
||||
data_type=(row.get("dataType") or "").strip(),
|
||||
sample_rate=sample_rate,
|
||||
quality_code=_to_bool(row["qualityCode"]) if (row.get("qualityCode") or "").strip() else True,
|
||||
opc_node=((row.get("opcNode") or "").strip() or None),
|
||||
row_number=row_number,
|
||||
)
|
||||
)
|
||||
return PointDict(points)
|
||||
@@ -0,0 +1,46 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""点位字典 CSV schema —— 对齐 PRD 5.1「边缘采集网关」字段规范表。
|
||||
|
||||
字段表（PRD 5.1）:
|
||||
device_id string 必填，唯一 设备编号，如 CLF-01
|
||||
point_id string 必填，唯一 测点编号，如 CLF-01.TEMP
|
||||
name string 必填 中文名，如 炉温
|
||||
unit enum 必填 ℃ / kPa / m³/h / % / ...
|
||||
dataType enum 必填 float / int / bool
|
||||
sampleRate int 必填, >0 采集周期(ms)
|
||||
qualityCode bool 默认 true 是否启用质量码
|
||||
opcNode string 选填 OPC UA 节点路径
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
|
||||
# 合法量纲集合（可按行业模板扩展；此处覆盖化工/氯化车间常用量纲）
|
||||
VALID_UNITS: List[str] = [
|
||||
"℃", "kPa", "MPa", "m³/h", "m3/h", "%", "kg", "t", "t/h", "m³", "m3",
|
||||
"A", "V", "Hz", "kW", "kWh", "Pa", "bar", "mm", "L", "L/min", "m/s",
|
||||
]
|
||||
|
||||
# 合法数据类型集合
|
||||
VALID_DATA_TYPES: List[str] = ["float", "int", "bool"]
|
||||
|
||||
# 必填字段
|
||||
REQUIRED_FIELDS: List[str] = ["device_id", "point_id", "name", "unit", "dataType", "sampleRate"]
|
||||
|
||||
# CSV 表头（列顺序固定，便于实施工程师对照 DCS 点表填写）
|
||||
CSV_HEADERS: List[str] = [
|
||||
"device_id", "point_id", "name", "unit", "dataType", "sampleRate",
|
||||
"qualityCode", "opcNode",
|
||||
]
|
||||
|
||||
|
||||
def is_valid_unit(unit: str) -> bool:
|
||||
"""量纲合法性校验（大小写不敏感）。"""
|
||||
if not unit or unit != unit.strip():
|
||||
return False
|
||||
return unit in VALID_UNITS
|
||||
|
||||
|
||||
def is_valid_data_type(dtype: str) -> bool:
|
||||
"""数据类型合法性校验。"""
|
||||
return dtype in VALID_DATA_TYPES
|
||||
@@ -0,0 +1,129 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""点位字典自动校验器。
|
||||
|
||||
校验维度（对齐 issue #3 与 PRD 5.1）：
|
||||
1. 缺失字段：必填列缺失 / 必填值为空；
|
||||
2. 量纲：unit 不在合法量纲集合；
|
||||
3. 重复点号：point_id 重复（同一测点被定义两次）；
|
||||
4. 采样率：sampleRate 必须为正整数；
|
||||
5. 数据类型：dataType 必须为 float/int/bool；
|
||||
6. 表头：CSV 缺少必填列。
|
||||
|
||||
注：同一设备下多个测点行是正常场景（如 CLF-01 的炉温/炉压），
|
||||
因此 device_id 重复不做行级报错。
|
||||
|
||||
返回校验报告（逐条错误 + 汇总），不抛异常，便于配置台一次性展示全部问题。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List
|
||||
|
||||
from . import schema
|
||||
from .loader import Point, PointDict
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationIssue:
|
||||
"""单条校验问题。"""
|
||||
|
||||
code: str # 错误码：missing_field / bad_unit / dup_point / ...
|
||||
row: int # CSV 行号（表头为 1，数据从 2 起；表头级问题 row=1）
|
||||
message: str # 人类可读描述
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationReport:
|
||||
"""校验报告。"""
|
||||
|
||||
issues: List[ValidationIssue] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return not self.issues
|
||||
|
||||
def summary(self) -> str:
|
||||
if self.ok:
|
||||
return "点位字典校验通过"
|
||||
by_code: Dict[str, int] = {}
|
||||
for it in self.issues:
|
||||
by_code[it.code] = by_code.get(it.code, 0) + 1
|
||||
detail = ", ".join(f"{code}×{n}" for code, n in sorted(by_code.items()))
|
||||
return f"点位字典校验失败（共 {len(self.issues)} 条）：{detail}"
|
||||
|
||||
|
||||
def _add_missing_header_issues(report: ValidationReport, headers: List[str]) -> None:
|
||||
missing = [f for f in schema.REQUIRED_FIELDS if f not in headers]
|
||||
for f in missing:
|
||||
report.issues.append(
|
||||
ValidationIssue(code="missing_column", row=1, message=f"CSV 缺少必填列: {f}")
|
||||
)
|
||||
|
||||
|
||||
def validate_point_dict(point_dict: PointDict, headers: List[str]) -> ValidationReport:
|
||||
"""校验点位字典，返回报告。
|
||||
|
||||
Args:
|
||||
point_dict: 加载后的点位字典；
|
||||
headers: CSV 表头（用于检查缺失列）。
|
||||
"""
|
||||
report = ValidationReport()
|
||||
_add_missing_header_issues(report, headers)
|
||||
|
||||
seen_point_ids: Dict[str, int] = {}
|
||||
|
||||
for p in point_dict.points:
|
||||
# 1) 缺失字段
|
||||
for f in schema.REQUIRED_FIELDS:
|
||||
value = getattr(p, {
|
||||
"dataType": "data_type",
|
||||
"sampleRate": "sample_rate",
|
||||
}.get(f, f))
|
||||
if value is None or (isinstance(value, str) and value == ""):
|
||||
report.issues.append(
|
||||
ValidationIssue(code="missing_field", row=p.row_number,
|
||||
message=f"第{p.row_number}行 必填字段缺失: {f}")
|
||||
)
|
||||
# 2) 量纲
|
||||
if p.unit and not schema.is_valid_unit(p.unit):
|
||||
report.issues.append(
|
||||
ValidationIssue(code="bad_unit", row=p.row_number,
|
||||
message=f"第{p.row_number}行 非法量纲: '{p.unit}'（合法值见 schema.VALID_UNITS）")
|
||||
)
|
||||
# 3) 数据类型
|
||||
if p.data_type and not schema.is_valid_data_type(p.data_type):
|
||||
report.issues.append(
|
||||
ValidationIssue(code="bad_data_type", row=p.row_number,
|
||||
message=f"第{p.row_number}行 非法数据类型: '{p.data_type}'（须为 float/int/bool）")
|
||||
)
|
||||
# 4) 采样率
|
||||
if p.sample_rate <= 0:
|
||||
report.issues.append(
|
||||
ValidationIssue(code="bad_sample_rate", row=p.row_number,
|
||||
message=f"第{p.row_number}行 sampleRate 必须为正整数(ms)，当前: {p.sample_rate}")
|
||||
)
|
||||
# 5) 重复点号（同一测点被定义两次）
|
||||
if p.point_id:
|
||||
if p.point_id in seen_point_ids:
|
||||
report.issues.append(
|
||||
ValidationIssue(code="dup_point", row=p.row_number,
|
||||
message=f"第{p.row_number}行 重复测点 point_id: '{p.point_id}'（首次出现于第{seen_point_ids[p.point_id]}行）")
|
||||
)
|
||||
else:
|
||||
seen_point_ids[p.point_id] = p.row_number
|
||||
|
||||
return report
|
||||
|
||||
|
||||
def validate_point_dict_file(path: str) -> ValidationReport:
|
||||
"""便捷入口：加载 + 校验一个 CSV 文件。"""
|
||||
from .loader import load_point_dict_csv
|
||||
|
||||
point_dict = load_point_dict_csv(path)
|
||||
headers: List[str] = []
|
||||
with open(path, "r", encoding="utf-8-sig") as fh:
|
||||
import csv
|
||||
|
||||
reader = csv.DictReader(fh)
|
||||
headers = list(reader.fieldnames or [])
|
||||
return validate_point_dict(point_dict, headers)
|
||||
Reference in New Issue
Block a user