feat: 实现自助BI看板功能，支持Superset/Metabase集成

- 新增BI模块(src/bi/)，包含数据模型、服务和控制器 - 支持数据源管理、图表创建、看板配置 - 实现多图表类型：折线图、柱状图、饼图、散点图、面积图、仪表盘、表格 - 提供REST API(/bi/)和前端API(/bi-api/)接口 - 创建响应式前端界面，支持拖拽和实时数据展示 - 默认包含运营总览、设备管理、安全监控看板 - 支持与Superset和Metabase集成 🤖 Generated with [OpenClaw](https://github.com/robocomp/openclaw)
2026-06-15 12:29:34 +08:00
parent 5eae031679
commit 61acfd8f8b
11 changed files with 2689 additions and 0 deletions
@@ -0,0 +1,562 @@
+"""
+BI服务模块
+提供自助BI看板和数据可视化服务
+包括数据集管理、图表创建、看板配置等功能
+"""
+from typing import Dict, List, Optional, Any, Tuple
+from datetime import datetime, timedelta
+import json
+import pandas as pd
+import numpy as np
+from .models import (
+    Chart, ChartType, DataSourceType, Dashboard, 
+    DataSource, Dataset, SupersetIntegration, MetabaseIntegration
+)
+
+class BIService:
+    """BI服务主类"""
+    
+    def __init__(self):
+        # 初始化数据存储
+        self.charts: Dict[str, Chart] = {}
+        self.dashboards: Dict[str, Dashboard] = {}
+        self.data_sources: Dict[str, DataSource] = {}
+        self.datasets: Dict[str, Dataset] = {}
+        self.superset_integration: Optional[SupersetIntegration] = None
+        self.metabase_integration: Optional[MetabaseIntegration] = None
+        
+        # 初始化默认数据源
+        self._init_default_data_sources()
+        self._init_default_charts()
+        self._init_default_dashboards()
+    
+    def _init_default_data_sources(self):
+        """初始化默认数据源"""
+        # 传感器数据源
+        sensor_source = DataSource(
+            id="sensor_data",
+            name="传感器数据",
+            type=DataSourceType.SENSOR_DATA,
+            description="所有IoT传感器实时和历史数据",
+            config={
+                "time_field": "timestamp",
+                "value_field": "value",
+                "device_field": "device_id",
+                "location_field": "location",
+                "type_field": "data_type"
+            },
+            query_template="SELECT * FROM sensor_data WHERE {filters}",
+            columns=[
+                {"name": "id", "type": "integer", "description": "数据记录ID"},
+                {"name": "device_id", "type": "string", "description": "设备ID"},
+                {"name": "data_type", "type": "string", "description": "数据类型"},
+                {"name": "value", "type": "float", "description": "数值"},
+                {"name": "unit", "type": "string", "description": "单位"},
+                {"name": "timestamp", "type": "datetime", "description": "时间戳"},
+                {"name": "location", "type": "string", "description": "位置"},
+                {"name": "quality_score", "type": "float", "description": "质量评分"}
+            ]
+        )
+        self.data_sources[sensor_source.id] = sensor_source
+        
+        # 设备状态数据源
+        device_source = DataSource(
+            id="device_data",
+            name="设备状态",
+            type=DataSourceType.DEVICE_DATA,
+            description="所有设备的运行状态和配置信息",
+            config={
+                "status_field": "status",
+                "type_field": "device_type",
+                "location_field": "location"
+            },
+            query_template="SELECT * FROM device WHERE {filters}",
+            columns=[
+                {"name": "id", "type": "string", "description": "设备ID"},
+                {"name": "name", "type": "string", "description": "设备名称"},
+                {"name": "device_type", "type": "string", "description": "设备类型"},
+                {"name": "location", "type": "string", "description": "位置"},
+                {"name": "status", "type": "string", "description": "状态"},
+                {"name": "install_date", "type": "datetime", "description": "安装日期"},
+                {"name": "metadata", "type": "json", "description": "元数据"}
+            ]
+        )
+        self.data_sources[device_source.id] = device_source
+        
+        # 警报数据源
+        alert_source = DataSource(
+            id="alert_data",
+            name="警报数据",
+            type=DataSourceType.ALERT_DATA,
+            description="系统警报和通知记录",
+            config={
+                "level_field": "level",
+                "type_field": "alert_type",
+                "resolved_field": "resolved"
+            },
+            query_template="SELECT * FROM alert WHERE {filters}",
+            columns=[
+                {"name": "id", "type": "string", "description": "警报ID"},
+                {"name": "device_id", "type": "string", "description": "设备ID"},
+                {"name": "alert_type", "type": "string", "description": "警报类型"},
+                {"name": "level", "type": "string", "description": "警报级别"},
+                {"name": "message", "type": "string", "description": "警报信息"},
+                {"name": "timestamp", "type": "datetime", "description": "发生时间"},
+                {"name": "resolved", "type": "boolean", "description": "是否已解决"},
+                {"name": "resolved_at", "type": "datetime", "description": "解决时间"}
+            ]
+        )
+        self.data_sources[alert_source.id] = alert_source
+        
+        # 系统统计数据源
+        stats_source = DataSource(
+            id="system_stats",
+            name="系统统计",
+            type=DataSourceType.SYSTEM_STATS,
+            description="系统运行性能和使用统计",
+            config={
+                "cpu_field": "cpu_usage_percent",
+                "memory_field": "memory_usage_mb",
+                "records_field": "total_records"
+            },
+            query_template="SELECT * FROM system_stats WHERE {filters}",
+            columns=[
+                {"name": "timestamp", "type": "datetime", "description": "统计时间"},
+                {"name": "total_records", "type": "integer", "description": "总记录数"},
+                {"name": "total_devices", "type": "integer", "description": "设备总数"},
+                {"name": "active_connections", "type": "integer", "description": "活跃连接数"},
+                {"name": "api_requests_count", "type": "integer", "description": "API请求数"},
+                {"name": "alerts_count", "type": "integer", "description": "警报数量"},
+                {"name": "data_quality_score", "type": "float", "description": "数据质量评分"},
+                {"name": "memory_usage_mb", "type": "float", "description": "内存使用量(MB)"},
+                {"name": "cpu_usage_percent", "type": "float", "description": "CPU使用率(%)"}
+            ]
+        )
+        self.data_sources[stats_source.id] = stats_source
+    
+    def _init_default_charts(self):
+        """初始化默认图表"""
+        # 流量趋势图
+        flow_trend = Chart(
+            id="flow_trend",
+            name="流量趋势分析",
+            description="显示各区域流量随时间的变化趋势",
+            chart_type=ChartType.LINE,
+            data_source=DataSourceType.SENSOR_DATA,
+            x_axis="timestamp",
+            y_axis=["value"],
+            group_by=["location"],
+            aggregation="avg",
+            filters={"data_type": "LL"},
+            options={
+                "title": "各区域流量趋势",
+                "yAxis": {"title": "流量 (m³/h)"},
+                "xAxis": {"title": "时间"},
+                "legend": {"show": True},
+                "tooltip": {"trigger": "axis"}
+            },
+            tags=["流量", "趋势", "区域"]
+        )
+        self.charts[flow_trend.id] = flow_trend
+        
+        # 设备状态分布图
+        device_status = Chart(
+            id="device_status_distribution",
+            name="设备状态分布",
+            description="显示不同状态设备的数量分布",
+            chart_type=ChartType.PIE,
+            data_source=DataSourceType.DEVICE_DATA,
+            x_axis="status",
+            y_axis=["count"],
+            aggregation="count",
+            options={
+                "title": "设备状态分布",
+                "legend": {"show": True},
+                "tooltip": {"trigger": "item"}
+            },
+            tags=["设备", "状态", "分布"]
+        )
+        self.charts[device_status.id] = device_status
+        
+        # 警报级别统计图
+        alert_stats = Chart(
+            id="alert_level_stats",
+            name="警报级别统计",
+            description="按级别统计警报数量",
+            chart_type=ChartType.BAR,
+            data_source=DataSourceType.ALERT_DATA,
+            x_axis="level",
+            y_axis=["count"],
+            aggregation="count",
+            filters={"resolved": False},
+            options={
+                "title": "未解决警报按级别统计",
+                "yAxis": {"title": "数量"},
+                "xAxis": {"title": "警报级别"},
+                "legend": {"show": False}
+            },
+            tags=["警报", "级别", "统计"]
+        )
+        self.charts[alert_stats.id] = alert_stats
+        
+        # 系统性能监控图
+        system_performance = Chart(
+            id="system_performance",
+            name="系统性能监控",
+            description="显示系统CPU和内存使用率趋势",
+            chart_type=ChartType.LINE,
+            data_source=DataSourceType.SYSTEM_STATS,
+            x_axis="timestamp",
+            y_axis=["cpu_usage_percent", "memory_usage_mb"],
+            options={
+                "title": "系统性能监控",
+                "yAxis": [{"title": "CPU使用率(%)"}, {"title": "内存使用量(MB)"}],
+                "xAxis": {"title": "时间"},
+                "legend": {"show": True},
+                "tooltip": {"trigger": "axis"}
+            },
+            tags=["系统", "性能", "监控"]
+        )
+        self.charts[system_performance.id] = system_performance
+    
+    def _init_default_dashboards(self):
+        """初始化默认看板"""
+        # 水务运营总览看板
+        overview_dashboard = Dashboard(
+            id="operation_overview",
+            name="水务运营总览",
+            description="水务系统整体运营情况综合看板",
+            charts=["flow_trend", "device_status_distribution", "alert_level_stats", "system_performance"],
+            layout=[
+                {"i": "flow_trend", "x": 0, "y": 0, "w": 12, "h": 8},
+                {"i": "device_status_distribution", "x": 12, "y": 0, "w": 6, "h": 6},
+                {"i": "alert_level_stats", "x": 18, "y": 0, "w": 6, "h": 6},
+                {"i": "system_performance", "x": 0, "y": 8, "w": 24, "h": 8}
+            ],
+            is_public=True,
+            tags=["运营", "总览", "综合"]
+        )
+        self.dashboards[overview_dashboard.id] = overview_dashboard
+        
+        # 设备管理看板
+        device_dashboard = Dashboard(
+            id="device_management",
+            name="设备管理看板",
+            description="设备状态监控和维护管理",
+            charts=["device_status_distribution"],
+            layout=[
+                {"i": "device_status_distribution", "x": 0, "y": 0, "w": 12, "h": 8}
+            ],
+            tags=["设备", "管理", "监控"]
+        )
+        self.dashboards[device_dashboard.id] = device_dashboard
+        
+        # 安全监控看板
+        security_dashboard = Dashboard(
+            id="security_monitoring",
+            name="安全监控看板",
+            description="系统安全和警报监控",
+            charts=["alert_level_stats"],
+            layout=[
+                {"i": "alert_level_stats", "x": 0, "y": 0, "w": 12, "h": 8}
+            ],
+            tags=["安全", "监控", "警报"]
+        )
+        self.dashboards[security_dashboard.id] = security_dashboard
+    
+    def get_chart(self, chart_id: str) -> Optional[Chart]:
+        """获取图表"""
+        return self.charts.get(chart_id)
+    
+    def get_all_charts(self) -> List[Chart]:
+        """获取所有图表"""
+        return list(self.charts.values())
+    
+    def create_chart(self, chart_data: Dict[str, Any]) -> Chart:
+        """创建图表"""
+        chart = Chart.from_dict(chart_data)
+        self.charts[chart.id] = chart
+        return chart
+    
+    def update_chart(self, chart_id: str, chart_data: Dict[str, Any]) -> Optional[Chart]:
+        """更新图表"""
+        if chart_id in self.charts:
+            chart = Chart.from_dict(chart_data)
+            chart.id = chart_id  # 保持ID不变
+            self.charts[chart_id] = chart
+            return chart
+        return None
+    
+    def delete_chart(self, chart_id: str) -> bool:
+        """删除图表"""
+        if chart_id in self.charts:
+            del self.charts[chart_id]
+            # 从所有看板中移除该图表
+            for dashboard in self.dashboards.values():
+                if chart_id in dashboard.charts:
+                    dashboard.charts.remove(chart_id)
+            return True
+        return False
+    
+    def get_dashboard(self, dashboard_id: str) -> Optional[Dashboard]:
+        """获取看板"""
+        return self.dashboards.get(dashboard_id)
+    
+    def get_all_dashboards(self) -> List[Dashboard]:
+        """获取所有看板"""
+        return list(self.dashboards.values())
+    
+    def create_dashboard(self, dashboard_data: Dict[str, Any]) -> Dashboard:
+        """创建看板"""
+        dashboard = Dashboard.from_dict(dashboard_data)
+        self.dashboards[dashboard.id] = dashboard
+        return dashboard
+    
+    def update_dashboard(self, dashboard_id: str, dashboard_data: Dict[str, Any]) -> Optional[Dashboard]:
+        """更新看板"""
+        if dashboard_id in self.dashboards:
+            dashboard = Dashboard.from_dict(dashboard_data)
+            dashboard.id = dashboard_id  # 保持ID不变
+            self.dashboards[dashboard_id] = dashboard
+            return dashboard
+        return None
+    
+    def delete_dashboard(self, dashboard_id: str) -> bool:
+        """删除看板"""
+        if dashboard_id in self.dashboards:
+            del self.dashboards[dashboard_id]
+            return True
+        return False
+    
+    def get_data_source(self, source_id: str) -> Optional[DataSource]:
+        """获取数据源"""
+        return self.data_sources.get(source_id)
+    
+    def get_all_data_sources(self) -> List[DataSource]:
+        """获取所有数据源"""
+        return list(self.data_sources.values())
+    
+    def get_dataset(self, dataset_id: str) -> Optional[Dataset]:
+        """获取数据集"""
+        return self.datasets.get(dataset_id)
+    
+    def get_all_datasets(self) -> List[Dataset]:
+        """获取所有数据集"""
+        return list(self.datasets.values())
+    
+    def execute_chart_data(self, chart_id: str) -> Dict[str, Any]:
+        """执行图表数据查询"""
+        chart = self.get_chart(chart_id)
+        if not chart:
+            return {"error": "Chart not found"}
+        
+        # 这里模拟数据查询，实际应该连接到数据库或数据源
+        data = self._generate_chart_data(chart)
+        
+        return {
+            "chart_id": chart_id,
+            "chart_name": chart.name,
+            "data": data,
+            "columns": chart.options.get("columns", []),
+            "chart_type": chart.chart_type.value,
+            "options": chart.options
+        }
+    
+    def _generate_chart_data(self, chart: Chart) -> List[Dict[str, Any]]:
+        """生成图表数据（模拟）"""
+        # 根据图表类型和数据源生成模拟数据
+        if chart.data_source == DataSourceType.SENSOR_DATA:
+            return self._generate_sensor_data(chart)
+        elif chart.data_source == DataSourceType.DEVICE_DATA:
+            return self._generate_device_data(chart)
+        elif chart.data_source == DataSourceType.ALERT_DATA:
+            return self._generate_alert_data(chart)
+        elif chart.data_source == DataSourceType.SYSTEM_STATS:
+            return self._generate_stats_data(chart)
+        else:
+            return []
+    
+    def _generate_sensor_data(self, chart: Chart) -> List[Dict[str, Any]]:
+        """生成传感器数据"""
+        data = []
+        
+        # 生成时间序列数据
+        base_time = datetime.now() - timedelta(days=7)
+        locations = ["A区", "B区", "C区", "D区"]
+        
+        for i in range(24 * 7):  # 7天，每小时一个点
+            timestamp = base_time + timedelta(hours=i)
+            
+            for location in locations:
+                # 添加一些随机波动
+                base_value = 50 if chart.filters.get("data_type") == "LL" else 1.0
+                value = base_value + np.random.normal(0, 10)
+                
+                data.append({
+                    "timestamp": timestamp.isoformat(),
+                    "location": location,
+                    "value": round(value, 2),
+                    "device_id": f"device_{hash(location) % 10 + 1}",
+                    "data_type": chart.filters.get("data_type", "LL"),
+                    "quality_score": round(np.random.uniform(0.8, 1.0), 2)
+                })
+        
+        # 应用过滤和聚合
+        if chart.group_by:
+            # 简单的分组聚合
+            grouped_data = {}
+            for item in data:
+                key = tuple(item.get(field) for field in chart.group_by)
+                if key not in grouped_data:
+                    grouped_data[key] = []
+                grouped_data[key].append(item)
+            
+            result = []
+            for key, items in grouped_data.items():
+                group_data = {}
+                for i, field in enumerate(chart.group_by):
+                    group_data[field] = key[i]
+                
+                # 聚合计算
+                values = [item["value"] for item in items]
+                if chart.aggregation == "avg":
+                    group_data["value"] = sum(values) / len(values)
+                elif chart.aggregation == "sum":
+                    group_data["value"] = sum(values)
+                elif chart.aggregation == "max":
+                    group_data["value"] = max(values)
+                elif chart.aggregation == "min":
+                    group_data["value"] = min(values)
+                else:
+                    group_data["value"] = sum(values) / len(values)
+                
+                result.append(group_data)
+            
+            return result
+        
+        return data
+    
+    def _generate_device_data(self, chart: Chart) -> List[Dict[str, Any]]:
+        """生成设备数据"""
+        devices = [
+            {"id": "device_1", "name": "流量计-001", "device_type": "流量计", "location": "A区", "status": "active"},
+            {"id": "device_2", "name": "压力计-001", "device_type": "压力计", "location": "A区", "status": "active"},
+            {"id": "device_3", "name": "水位计-001", "device_type": "水位计", "location": "B区", "status": "maintenance"},
+            {"id": "device_4", "name": "浊度计-001", "device_type": "浊度计", "location": "B区", "status": "active"},
+            {"id": "device_5", "name": "pH计-001", "device_type": "pH计", "location": "C区", "status": "inactive"},
+        ]
+        
+        # 按状态分组
+        status_groups = {}
+        for device in devices:
+            status = device["status"]
+            if status not in status_groups:
+                status_groups[status] = []
+            status_groups[status].append(device)
+        
+        # 生成统计数据
+        result = []
+        for status, devices_in_status in status_groups.items():
+            result.append({
+                "status": status,
+                "count": len(devices_in_status),
+                "devices": [d["name"] for d in devices_in_status]
+            })
+        
+        return result
+    
+    def _generate_alert_data(self, chart: Chart) -> List[Dict[str, Any]]:
+        """生成警报数据"""
+        alerts = [
+            {"level": "info", "count": 5, "description": "信息级别警报"},
+            {"level": "warning", "count": 3, "description": "警告级别警报"},
+            {"level": "error", "count": 1, "description": "错误级别警报"},
+            {"level": "critical", "count": 0, "description": "严重级别警报"},
+        ]
+        
+        return alerts
+    
+    def _generate_stats_data(self, chart: Chart) -> List[Dict[str, Any]]:
+        """生成系统统计数据"""
+        data = []
+        base_time = datetime.now() - timedelta(days=1)
+        
+        for i in range(24):  # 24小时数据
+            timestamp = base_time + timedelta(hours=i)
+            
+            data.append({
+                "timestamp": timestamp.isoformat(),
+                "total_records": 1000 + np.random.randint(-100, 100),
+                "total_devices": 25 + np.random.randint(-5, 5),
+                "active_connections": 5 + np.random.randint(-2, 3),
+                "api_requests_count": 150 + np.random.randint(-30, 30),
+                "alerts_count": np.random.randint(0, 5),
+                "data_quality_score": round(np.random.uniform(0.9, 1.0), 2),
+                "memory_usage_mb": 100 + np.random.randint(-20, 20),
+                "cpu_usage_percent": 30 + np.random.randint(-10, 10)
+            })
+        
+        return data
+    
+    def setup_superset_integration(self, integration_data: Dict[str, Any]) -> SupersetIntegration:
+        """设置Superset集成"""
+        integration = SupersetIntegration.from_dict(integration_data)
+        self.superset_integration = integration
+        return integration
+    
+    def setup_metabase_integration(self, integration_data: Dict[str, Any]) -> MetabaseIntegration:
+        """设置Metabase集成"""
+        integration = MetabaseIntegration.from_dict(integration_data)
+        self.metabase_integration = integration
+        return integration
+    
+    def get_chart_data_api(self, chart_id: str) -> Dict[str, Any]:
+        """获取图表数据API接口"""
+        return self.execute_chart_data(chart_id)
+    
+    def get_dashboard_data_api(self, dashboard_id: str) -> Dict[str, Any]:
+        """获取看板数据API接口"""
+        dashboard = self.get_dashboard(dashboard_id)
+        if not dashboard:
+            return {"error": "Dashboard not found"}
+        
+        charts_data = {}
+        for chart_id in dashboard.charts:
+            charts_data[chart_id] = self.get_chart_data_api(chart_id)
+        
+        return {
+            "dashboard_id": dashboard_id,
+            "dashboard_name": dashboard.name,
+            "charts": charts_data,
+            "layout": dashboard.layout
+        }
+    
+    def get_public_dashboards(self) -> List[Dashboard]:
+        """获取公开看板"""
+        return [db for db in self.dashboards.values() if db.is_public]
+    
+    def get_charts_by_tag(self, tag: str) -> List[Chart]:
+        """根据标签获取图表"""
+        return [chart for chart in self.charts.values() if tag in chart.tags]
+    
+    def get_dashboards_by_tag(self, tag: str) -> List[Dashboard]:
+        """根据标签获取看板"""
+        return [dashboard for dashboard in self.dashboards.values() if tag in dashboard.tags]
+    
+    def search_charts(self, keyword: str) -> List[Chart]:
+        """搜索图表"""
+        keyword = keyword.lower()
+        return [
+            chart for chart in self.charts.values()
+            if keyword in chart.name.lower() or keyword in chart.description.lower()
+            or any(keyword in tag.lower() for tag in chart.tags)
+        ]
+    
+    def search_dashboards(self, keyword: str) -> List[Dashboard]:
+        """搜索看板"""
+        keyword = keyword.lower()
+        return [
+            dashboard for dashboard in self.dashboards.values()
+            if keyword in dashboard.name.lower() or keyword in dashboard.description.lower()
+            or any(keyword in tag.lower() for tag in dashboard.tags)
+        ]