bot_ym/iAOP
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 对齐技术架构补齐传输 mTLS 与结构化 JSON 日志（NFR 9 章）

Browse Source 
架构核对发现 2 处差距，本次补齐：
- Kafka 上行支持 SSL/SASL_SSL 双向 mTLS（8.2 服务间 mTLS 边缘网关↔总线）
- 网关日志支持结构化 JSON 输出（NFR 可维护：统一日志规范）
- 配置示例与 README 验收口径同步更新
This commit is contained in:
yunmei
2026-08-04 15:46:09 +08:00
parent f49c0920d4
commit 098125164d
8 changed files with 481 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/edge-gateway/README.md
+2 -1
View File
@@ -12,7 +12,8 @@
| 采集 P99 延迟 | ≤ 1.8s（600 点位 1Hz） | `collector/metrics.py` 统计，`engine.py` 调度 |
| 丢失率 | ≤ 0.02% | `collector/spool.py` 断点续传 + 背压丢弃计数 |
| 可用性 | ≥ 99.8% | `collector/metrics.py` 轮次成功率 |
| 安全 | 零控制指令下发 | 驱动抽象仅暴露只读读接口（`drivers/base.py`） |
| 安全-网络 | 严格只读、零控制指令；传输 mTLS | 驱动仅只读接口；`upstream/kafka_sink.py` 支持 SSL/SASL_SSL 双向 mTLS |
| 可维护 | 统一日志规范（结构化 JSON） | `main.py --log-json` 输出结构化 JSON 日志 |
## 目录结构
core/edge-gateway/config/gateway.example.yaml
+11
View File
@@ -51,3 +51,14 @@ kafka:
bootstrap_servers: "10.20.0.10:9092"
topic_prefix: "iaop.ti-cl4" # 模板化 topic：{prefix}.{device_id}.points
batch_size: 500
# 传输安全（NFR 9 章：服务间 mTLS，边缘网关↔总线双向认证）。
# 现场部署必须开启；本地联调可保持 PLAINTEXT。
security:
protocol: SSL # PLAINTEXT | SSL | SASL_SSL
ca_location: "/etc/iaop/certs/ca.crt"
cert_location: "/etc/iaop/certs/gateway.crt"
key_location: "/etc/iaop/certs/gateway.key"
# protocol: SASL_SSL 时追加：
# sasl_mechanism: "PLAIN"
# sasl_username: "edge-gateway"
# sasl_password: "..."
core/edge-gateway/main.py
+29 -5
View File
@@ -35,6 +35,8 @@ def parse_args(argv: List[str]) -> argparse.Namespace:
parser.add_argument("--rounds", type=int, default=0,
help="采集轮数上限（0=无限，调试用）")
parser.add_argument("--verbose", action="store_true", help="输出调试日志")
parser.add_argument("--log-json", action="store_true",
help="结构化 JSON 日志（NFR 9 章统一日志规范；生产模式建议开启）")
return parser.parse_args(argv)
@@ -78,7 +80,7 @@ def build_engine(config: dict, point_dict: PointDict):
def build_sink(config: dict, spool) -> object:
"""按模板配置组装 Kafka 上行通道。"""
"""按模板配置组装 Kafka 上行通道（含 mTLS 传输安全，NFR 9 章）。"""
from upstream import KafkaSink
kafka_cfg = config.get("kafka", {})
@@ -87,15 +89,37 @@ def build_sink(config: dict, spool) -> object:
topic_prefix=kafka_cfg.get("topic_prefix", "iaop"),
spool=spool,
batch_size=int(kafka_cfg.get("batch_size", 500)),
security=kafka_cfg.get("security") or {},
)
def main(argv: List[str]) -> int:
args = parse_args(argv)
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
if args.log_json:
# NFR 9 章「可维护」：统一日志规范 —— 结构化 JSON，便于采集到统一日志平台
import json as _json
class JsonFormatter(logging.Formatter):
def format(self, record):
payload = {
"ts": self.formatTime(record, "%Y-%m-%dT%H:%M:%S%z"),
"level": record.levelname,
"logger": record.name,
"msg": record.getMessage(),
}
if record.exc_info:
payload["exc"] = self.formatException(record.exc_info)
return _json.dumps(payload, ensure_ascii=False)
handler = logging.StreamHandler()
handler.setFormatter(JsonFormatter())
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO, handlers=[handler])
else:
logging.basicConfig(
level=logging.DEBUG if args.verbose else logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
# 1) 加载模板配置
with open(args.config, "r", encoding="utf-8") as fh:
core/edge-gateway/upstream/kafka_sink.py
+35 -1
View File
@@ -29,11 +29,24 @@ class KafkaSink:
topic_prefix: str,
spool: SpoolStore,
batch_size: int = 500,
security: Optional[dict] = None,
):
"""
Args:
bootstrap_servers: Kafka 地址列表（逗号分隔）；
topic_prefix: 上行 topic 前缀（模板化 `{prefix}.{device}.points`）；
spool: 本地缓存（断点续传）；
batch_size: 单批上限；
security: 传输安全配置（NFR 9 章：服务间 mTLS）。
约定字段：protocol(PLAINTEXT|SSL|SASL_SSL)、ca_location、
cert_location、key_location、sasl_username、sasl_password。
其中 SSL/SASL_SSL + ca/cert/key = 边缘网关↔总线双向 mTLS。
"""
self.bootstrap_servers = bootstrap_servers
self.topic_prefix = topic_prefix
self.spool = spool
self.batch_size = max(1, batch_size)
self.security = security or {}
self._producer = None
self._degraded = False # True = Kafka 不可用，仅保留 spool
self._lock = threading.Lock()
@@ -42,6 +55,27 @@ class KafkaSink:
self._connect()
# ------------------------------------------------------------------
def _producer_config(self) -> dict:
"""组装 confluent-kafka producer 配置（含 mTLS/SASL 透传）。"""
conf = {"bootstrap.servers": self.bootstrap_servers}
protocol = self.security.get("protocol", "PLAINTEXT").upper()
conf["security.protocol"] = protocol
if protocol in ("SSL", "SASL_SSL"):
for key, kafka_key in (
("ca_location", "ssl.ca.location"),
("cert_location", "ssl.certificate.location"),
("key_location", "ssl.key.location"),
("key_password", "ssl.key.password"),
):
if self.security.get(key):
conf[kafka_key] = self.security[key]
if protocol == "SASL_SSL":
conf["sasl.mechanism"] = self.security.get("sasl_mechanism", "PLAIN")
if self.security.get("sasl_username"):
conf["sasl.username"] = self.security["sasl_username"]
conf["sasl.password"] = self.security.get("sasl_password", "")
return conf
def _connect(self) -> None:
"""尝试连接 Kafka；失败则降级（不阻断采集）。"""
try:
@@ -51,7 +85,7 @@ class KafkaSink:
self._degraded = True
return
try:
self._producer = Producer({"bootstrap.servers": self.bootstrap_servers})
self._producer = Producer(self._producer_config())
except Exception as exc:
logger.warning("Kafka 初始化失败(%s)，降级为 spool-only 模式", exc)
self._degraded = True