feat: 完成 issue #31 ② MinIO 对象存储桶与生命周期策略

2026-08-05 02:13:55 +08:00
parent 169492a6db
commit 2947dae434
5 changed files with 342 additions and 0 deletions
@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+"""MinIO 对象存储桶 + 生命周期策略模板化测试（issue #31）。
+
+覆盖：
+1. 模板配置资产加载（config/minio.template.yaml，含生命周期规则）；
+2. 桶命名：`{template}-{bucket_suffix}`（S3 桶名允许 `-`）；
+3. 对象键：特征快照 / 模型 artifact（与 templating 命名一致）；
+4. 落地命令：`mc mb` 幂等建桶、`mc ilm add` 生命周期规则命令；
+5. 生命周期策略 JSON（S3 LifecycleConfiguration 兼容，MinIO 支持）；
+6. 规则规范化：缺省 id 派生、非法行过滤。
+"""
+import json
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import _bootstrap  # noqa: F401
+
+from data_bus.minio_lifecycle import (  # noqa: E402
+    DEFAULT_CONFIG_PATH,
+    MinioLifecycle,
+)
+
+CONFIG = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+    "config", "minio.template.yaml",
+)
+
+
+class TestConfigLoad(unittest.TestCase):
+    """模板配置资产加载与摘要。"""
+
+    def test_from_template_config(self):
+        life = MinioLifecycle.from_template_config(CONFIG)
+        self.assertEqual(life._tpl.template, "ti-cl4")
+        self.assertEqual(life.bucket(), "ti-cl4-artifacts")
+        self.assertEqual(life.mc_alias, "local")
+        brief = life.brief()
+        self.assertEqual(brief["bucket"], "ti-cl4-artifacts")
+        prefixes = [r["prefix"] for r in brief["lifecycle_rules"]]
+        self.assertIn("features/", prefixes)
+        self.assertIn("models/", prefixes)
+
+    def test_default_config_path_exists(self):
+        self.assertTrue(os.path.isfile(DEFAULT_CONFIG_PATH))
+
+
+class TestBucketNaming(unittest.TestCase):
+    """桶命名与对象键（复用 templating 命名，与 #4 一致）。"""
+
+    def setUp(self):
+        self.life = MinioLifecycle(template="ti-cl4")
+
+    def test_bucket_format(self):
+        self.assertEqual(self.life.bucket(), "ti-cl4-artifacts")
+
+    def test_bucket_suffix_custom(self):
+        self.assertEqual(
+            MinioLifecycle(template="resin", bucket_suffix="store").bucket(),
+            "resin-store")
+
+    def test_snapshot_key(self):
+        # model_id 经 sanitize_sql 清洗（`-` → `_`，与 templating 一致）
+        self.assertEqual(
+            self.life.snapshot_key("quality-forecast", "2026-08-04", 3),
+            "features/quality_forecast/2026-08-04/000003.jsonl")
+
+    def test_model_artifact_key(self):
+        self.assertEqual(
+            self.life.model_artifact_key("quality-forecast", "v1.2"),
+            "models/quality_forecast/v1.2/model.bin")
+
+
+class TestBucketCreateStatements(unittest.TestCase):
+    """mc mb 幂等建桶命令。"""
+
+    def test_create_statement(self):
+        life = MinioLifecycle(template="ti-cl4")
+        self.assertEqual(
+            life.bucket_create_statements(),
+            ["mc mb --ignore-existing local/ti-cl4-artifacts"])
+
+    def test_create_statement_custom_alias(self):
+        life = MinioLifecycle(template="ti-cl4", mc_alias="cluster")
+        self.assertEqual(
+            life.bucket_create_statements(),
+            ["mc mb --ignore-existing cluster/ti-cl4-artifacts"])
+
+
+class TestLifecycleCli(unittest.TestCase):
+    """mc ilm add 生命周期命令（按前缀 + 过期天数）。"""
+
+    def test_cli_statements(self):
+        life = MinioLifecycle(template="ti-cl4", lifecycle=[
+            {"prefix": "features/", "expiration_days": 180},
+            {"prefix": "models/", "expiration_days": 1095},
+        ])
+        cmds = life.lifecycle_cli_statements()
+        self.assertEqual(len(cmds), 2)
+        self.assertIn('mc ilm add --expiry-days 180 --prefix "features/" local/ti-cl4-artifacts', cmds)
+        self.assertIn('mc ilm add --expiry-days 1095 --prefix "models/" local/ti-cl4-artifacts', cmds)
+
+    def test_no_rules_no_commands(self):
+        life = MinioLifecycle(template="ti-cl4", lifecycle=[])
+        self.assertEqual(life.lifecycle_cli_statements(), [])
+
+
+class TestLifecyclePolicyJson(unittest.TestCase):
+    """S3 LifecycleConfiguration 兼容 JSON 策略（MinIO 支持）。"""
+
+    def test_policy_json_structure(self):
+        life = MinioLifecycle(template="ti-cl4", lifecycle=[
+            {"id": "features-expire", "prefix": "features/", "expiration_days": 180},
+        ])
+        policy = json.loads(life.lifecycle_policy_json())
+        rules = policy["Rules"]
+        self.assertEqual(len(rules), 1)
+        rule = rules[0]
+        self.assertEqual(rule["ID"], "features-expire")
+        self.assertEqual(rule["Status"], "Enabled")
+        self.assertEqual(rule["Filter"]["Prefix"], "features/")
+        self.assertEqual(rule["Expiration"]["Days"], 180)
+
+    def test_policy_json_roundtrip(self):
+        life = MinioLifecycle.from_template_config(CONFIG)
+        policy = json.loads(life.lifecycle_policy_json())
+        self.assertEqual(len(policy["Rules"]), 2)
+
+
+class TestRuleNormalization(unittest.TestCase):
+    """规则规范化：缺省 id 派生、非法行过滤。"""
+
+    def test_default_id_from_prefix(self):
+        life = MinioLifecycle(template="ti-cl4", lifecycle=[
+            {"prefix": "features/", "expiration_days": 180},
+        ])
+        self.assertEqual(life.rules[0]["id"], "features/expire")
+
+    def test_invalid_rows_dropped(self):
+        life = MinioLifecycle(template="ti-cl4", lifecycle=[
+            {"prefix": "", "expiration_days": 180},      # 无前缀 → 丢弃
+            {"prefix": "models/", "expiration_days": 0},  # 过期天数非法 → 丢弃
+            {"prefix": "logs/", "expiration_days": 30},   # 合法
+        ])
+        self.assertEqual(len(life.rules), 1)
+        self.assertEqual(life.rules[0]["prefix"], "logs/")
+
+
+if __name__ == "__main__":
+    unittest.main()