""" 数据处理工具模块 提供数据验证、转换、格式化等工具函数 """ import json import csv import pandas as pd from typing import Dict, List, Any, Optional, Union from datetime import datetime, timedelta import hashlib import logging # 配置日志 logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) class DataConverter: """数据转换器""" # 水利行业标准字段映射 FIELD_MAPPING = { "流量": "LL", "压力": "YL", "水位": "SW", "浊度": "ZD", "pH值": "PH", "温度": "WD", "电导率": "DD", "硬度": "YD", # 支持常见的中文字段名 "流量计": "LL", "压力表": "YL", "水位计": "SW", "浊度仪": "ZD", "pH计": "PH", "温度计": "WD", "电导率仪": "DD", "硬度计": "YD" } # 单位转换 UNIT_CONVERSIONS = { # 流量单位转换 (m³/h) "m³/h": 1.0, "L/s": 3.6, # L/s = m³/h / 1000 * 3600 "m³/d": 1/24, # m³/d = m³/h / 24 "L/min": 1/60, # L/min = m³/h / 1000 * 60 # 压力单位转换 (MPa) "MPa": 1.0, "kPa": 0.001, # kPa = MPa / 1000 "bar": 0.1, # bar = MPa * 10 "kgf/cm²": 0.0980665, # kgf/cm² = MPa / 0.0980665 # 水位单位转换 (m) "m": 1.0, "cm": 0.01, # cm = m / 100 "mm": 0.001, # mm = m / 1000 # 浊度单位转换 (NTU) "NTU": 1.0, "FNU": 1.0, # FNU ≈ NTU # pH值单位转换 "pH": 1.0, # 温度单位转换 (°C) "°C": 1.0, "K": 1.0, # 相对差值 "°F": lambda x: (x - 32) / 1.8, # °F to °C # 电导率单位转换 (μS/cm) "μS/cm": 1.0, "mS/cm": 1000, # mS/cm = μS/cm * 1000 "S/m": 10000 # S/m = μS/cm * 100 } @classmethod def normalize_field_name(cls, field_name: str) -> str: """标准化字段名""" if not field_name: return "" field_name = field_name.strip().upper() # 如果已经是标准格式，直接返回 if field_name in cls.FIELD_MAPPING.values(): return field_name # 查映射表 if field_name in cls.FIELD_MAPPING: return cls.FIELD_MAPPING[field_name] # 英文映射 english_mapping = { "flow": "LL", "pressure": "YL", "level": "SW", "turbidity": "ZD", "ph": "PH", "temperature": "WD", "conductivity": "DD", "hardness": "YD" } if field_name.lower() in english_mapping: return english_mapping[field_name.lower()] return field_name @classmethod def convert_unit(cls, value: float, from_unit: str, to_unit: str) -> float: """单位转换""" if from_unit == to_unit: return value if from_unit not in cls.UNIT_CONVERSIONS: raise ValueError(f"不支持的单位: {from_unit}") if to_unit not in cls.UNIT_CONVERSIONS: raise ValueError(f"不支持的目标单位: {to_unit}") from_conv = cls.UNIT_CONVERSIONS[from_unit] to_conv = cls.UNIT_CONVERSIONS[to_unit] if callable(from_conv): value = from_conv(value) if callable(to_conv): return value / to_conv else: return value * (to_conv / from_conv) @classmethod def validate_sensor_data(cls, data: Dict[str, Any]) -> Dict[str, Any]: """验证传感器数据""" errors = [] validated_data = {} # 必需字段验证 required_fields = ["device_id", "data_type", "value"] for field in required_fields: if field not in data: errors.append(f"缺少必需字段: {field}") else: validated_data[field] = data[field] # 数据类型验证和标准化 if "data_type" in validated_data: original_type = validated_data["data_type"] validated_data["data_type"] = cls.normalize_field_name(original_type) if validated_data["data_type"] != original_type: logger.info(f"字段名标准化: {original_type} -> {validated_data['data_type']}") # 数值验证 if "value" in validated_data: try: validated_data["value"] = float(validated_data["value"]) # 检查数值范围 data_type = validated_data.get("data_type", "") if data_type == "LL" and validated_data["value"] < 0: errors.append("流量不能为负数") elif data_type == "YL" and validated_data["value"] < 0: errors.append("压力不能为负数") elif data_type == "SW" and validated_data["value"] < 0: errors.append("水位不能为负数") except (ValueError, TypeError): errors.append(f"无效的数值: {validated_data['value']}") # 地点验证 if "location" not in validated_data or not validated_data["location"]: validated_data["location"] = "未知" # 时间戳处理 if "timestamp" in data: try: if isinstance(data["timestamp"], str): validated_data["timestamp"] = datetime.fromisoformat(data["timestamp"]) else: validated_data["timestamp"] = data["timestamp"] except (ValueError, TypeError): validated_data["timestamp"] = datetime.now() else: validated_data["timestamp"] = datetime.now() return { "valid": len(errors) == 0, "data": validated_data, "errors": errors } class DataFormatter: """数据格式化器""" @staticmethod def format_sensor_data(data: Dict[str, Any], format_type: str = "json") -> str: """格式化传感器数据""" if format_type == "json": return json.dumps(data, ensure_ascii=False, indent=2) elif format_type == "csv": # CSV格式只包含关键字段 csv_fields = ["device_id", "data_type", "value", "location", "timestamp"] csv_data = {k: data.get(k, "") for k in csv_fields} import io output = io.StringIO() writer = csv.DictWriter(output, fieldnames=csv_fields) writer.writeheader() writer.writerow(csv_data) return output.getvalue() else: raise ValueError(f"不支持的格式类型: {format_type}") @staticmethod def format_statistics(stats: Dict[str, Any], format_type: str = "text") -> str: """格式化统计数据""" if format_type == "json": return json.dumps(stats, ensure_ascii=False, indent=2) elif format_type == "text": lines = ["数据统计报告", "=" * 20] lines.append(f"总记录数: {stats.get('total_records', 0)}") if "by_type" in stats: lines.append("\n按数据类型统计:") for data_type, count in stats["by_type"].items(): lines.append(f" {data_type}: {count} 条") if "by_device" in stats: lines.append("\n按设备统计:") for device_id, count in list(stats["by_device"].items())[:10]: # 只显示前10个 lines.append(f" {device_id}: {count} 条") return "\n".join(lines) else: raise ValueError(f"不支持的格式类型: {format_type}") class DataHasher: """数据哈希工具""" @staticmethod def calculate_data_hash(data: Dict[str, Any]) -> str: """计算数据哈希值""" # 将数据转换为字符串 data_str = json.dumps(data, sort_keys=True, ensure_ascii=False) # 计算MD5哈希 hash_md5 = hashlib.md5(data_str.encode()) return hash_md5.hexdigest() @staticmethod def generate_data_id(device_id: str, data_type: str, timestamp: datetime) -> str: """生成数据ID""" # 使用设备ID、数据类型和时间戳生成唯一ID time_str = timestamp.strftime("%Y%m%d_%H%M%S") hash_input = f"{device_id}_{data_type}_{time_str}" hash_md5 = hashlib.md5(hash_input.encode()) return f"{data_type}_{device_id}_{hash_md5.hexdigest()[:8]}" class DataQualityChecker: """数据质量检查器""" @staticmethod def check_data_quality(records: List[Dict[str, Any]]) -> Dict[str, Any]: """检查数据质量""" quality_report = { "total_records": len(records), "valid_records": 0, "invalid_records": 0, "quality_score": 0, "issues": [], "statistics": {} } if not records: quality_report["quality_score"] = 0 return quality_report valid_records = [] for record in records: issues = [] # 检查必需字段 required_fields = ["device_id", "data_type", "value"] for field in required_fields: if field not in record or not record[field]: issues.append(f"缺少必需字段: {field}") # 检查数据类型 if "data_type" in record and record["data_type"]: valid_types = ["LL", "YL", "SW", "ZD", "PH", "WD", "DD", "YD"] if record["data_type"] not in valid_types: issues.append(f"无效的数据类型: {record['data_type']}") # 检查数值范围 if "value" in record and record["value"]: try: value = float(record["value"]) data_type = record.get("data_type", "") if data_type == "LL" and value < 0: issues.append("流量不能为负数") elif data_type == "YL" and value < 0: issues.append("压力不能为负数") elif data_type == "SW" and value < 0: issues.append("水位不能为负数") # 检查异常值 if data_type == "LL" and value > 10000: issues.append("流量值异常大") elif data_type == "YL" and value > 10: issues.append("压力值异常大") except (ValueError, TypeError): issues.append("无效的数值格式") if not issues: valid_records.append(record) quality_report["valid_records"] += 1 else: quality_report["invalid_records"] += 1 quality_report["issues"].extend(issues) # 计算质量分数 quality_report["quality_score"] = quality_report["valid_records"] / len(records) # 统计信息 if records: quality_report["statistics"] = { "completeness": quality_report["valid_records"] / len(records), "uniqueness": len(set(r.get("device_id", "") for r in valid_records)) / len(valid_records) if valid_records else 0, "timeliness": quality_report.calculate_timeliness(records) } return quality_report @staticmethod def calculate_timeliness(records: List[Dict[str, Any]]) -> float: """计算数据及时性（24小时内的数据比例）""" if not records: return 0 now = datetime.now() recent_count = 0 for record in records: timestamp = record.get("timestamp") if timestamp: try: if isinstance(timestamp, str): timestamp = datetime.fromisoformat(timestamp) time_diff = now - timestamp if time_diff <= timedelta(hours=24): recent_count += 1 except: pass return recent_count / len(records) class DataExporter: """数据导出工具""" @staticmethod def export_to_csv(records: List[Dict[str, Any]], file_path: str) -> bool: """导出为CSV文件""" try: if not records: return False # 获取所有字段 all_fields = set() for record in records: all_fields.update(record.keys()) # 排序字段 field_order = ["device_id", "data_type", "value", "location", "timestamp"] for field in all_fields: if field not in field_order: field_order.append(field) with open(file_path, 'w', newline='', encoding='utf-8') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=field_order) writer.writeheader() writer.writerows(records) return True except Exception as e: logger.error(f"导出CSV失败: {str(e)}") return False @staticmethod def export_to_json(records: List[Dict[str, Any]], file_path: str) -> bool: """导出为JSON文件""" try: with open(file_path, 'w', encoding='utf-8') as jsonfile: json.dump(records, jsonfile, ensure_ascii=False, indent=2, default=str) return True except Exception as e: logger.error(f"导出JSON失败: {str(e)}") return False # 全局工具实例 data_converter = DataConverter() data_formatter = DataFormatter() data_hasher = DataHasher() quality_checker = DataQualityChecker() data_exporter = DataExporter()