bot_dev1
5eae031679
- 实现REST API服务器，支持IoT数据、手动录入和批量导入接口 - 实现WebSocket服务器，支持实时数据推送和连接管理 - 实现批量导入模块，支持CSV、Excel、JSON多种格式 - 实现数据处理工具，包含字段映射和单位转换功能 - 实现数据模型定义和数据验证机制 - 创建主程序入口和配置文件 - 添加详细的使用文档和API说明
409 lines
14 KiB
Python
409 lines
14 KiB
Python
"""
|
|
数据处理工具模块
|
|
提供数据验证、转换、格式化等工具函数
|
|
"""
|
|
import json
|
|
import csv
|
|
import pandas as pd
|
|
from typing import Dict, List, Any, Optional, Union
|
|
from datetime import datetime, timedelta
|
|
import hashlib
|
|
import logging
|
|
|
|
# 配置日志
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class DataConverter:
|
|
"""数据转换器"""
|
|
|
|
# 水利行业标准字段映射
|
|
FIELD_MAPPING = {
|
|
"流量": "LL",
|
|
"压力": "YL",
|
|
"水位": "SW",
|
|
"浊度": "ZD",
|
|
"pH值": "PH",
|
|
"温度": "WD",
|
|
"电导率": "DD",
|
|
"硬度": "YD",
|
|
# 支持常见的中文字段名
|
|
"流量计": "LL",
|
|
"压力表": "YL",
|
|
"水位计": "SW",
|
|
"浊度仪": "ZD",
|
|
"pH计": "PH",
|
|
"温度计": "WD",
|
|
"电导率仪": "DD",
|
|
"硬度计": "YD"
|
|
}
|
|
|
|
# 单位转换
|
|
UNIT_CONVERSIONS = {
|
|
# 流量单位转换 (m³/h)
|
|
"m³/h": 1.0,
|
|
"L/s": 3.6, # L/s = m³/h / 1000 * 3600
|
|
"m³/d": 1/24, # m³/d = m³/h / 24
|
|
"L/min": 1/60, # L/min = m³/h / 1000 * 60
|
|
|
|
# 压力单位转换 (MPa)
|
|
"MPa": 1.0,
|
|
"kPa": 0.001, # kPa = MPa / 1000
|
|
"bar": 0.1, # bar = MPa * 10
|
|
"kgf/cm²": 0.0980665, # kgf/cm² = MPa / 0.0980665
|
|
|
|
# 水位单位转换 (m)
|
|
"m": 1.0,
|
|
"cm": 0.01, # cm = m / 100
|
|
"mm": 0.001, # mm = m / 1000
|
|
|
|
# 浊度单位转换 (NTU)
|
|
"NTU": 1.0,
|
|
"FNU": 1.0, # FNU ≈ NTU
|
|
|
|
# pH值单位转换
|
|
"pH": 1.0,
|
|
|
|
# 温度单位转换 (°C)
|
|
"°C": 1.0,
|
|
"K": 1.0, # 相对差值
|
|
"°F": lambda x: (x - 32) / 1.8, # °F to °C
|
|
|
|
# 电导率单位转换 (μS/cm)
|
|
"μS/cm": 1.0,
|
|
"mS/cm": 1000, # mS/cm = μS/cm * 1000
|
|
"S/m": 10000 # S/m = μS/cm * 100
|
|
}
|
|
|
|
@classmethod
|
|
def normalize_field_name(cls, field_name: str) -> str:
|
|
"""标准化字段名"""
|
|
if not field_name:
|
|
return ""
|
|
|
|
field_name = field_name.strip().upper()
|
|
|
|
# 如果已经是标准格式，直接返回
|
|
if field_name in cls.FIELD_MAPPING.values():
|
|
return field_name
|
|
|
|
# 查映射表
|
|
if field_name in cls.FIELD_MAPPING:
|
|
return cls.FIELD_MAPPING[field_name]
|
|
|
|
# 英文映射
|
|
english_mapping = {
|
|
"flow": "LL",
|
|
"pressure": "YL",
|
|
"level": "SW",
|
|
"turbidity": "ZD",
|
|
"ph": "PH",
|
|
"temperature": "WD",
|
|
"conductivity": "DD",
|
|
"hardness": "YD"
|
|
}
|
|
|
|
if field_name.lower() in english_mapping:
|
|
return english_mapping[field_name.lower()]
|
|
|
|
return field_name
|
|
|
|
@classmethod
|
|
def convert_unit(cls, value: float, from_unit: str, to_unit: str) -> float:
|
|
"""单位转换"""
|
|
if from_unit == to_unit:
|
|
return value
|
|
|
|
if from_unit not in cls.UNIT_CONVERSIONS:
|
|
raise ValueError(f"不支持的单位: {from_unit}")
|
|
|
|
if to_unit not in cls.UNIT_CONVERSIONS:
|
|
raise ValueError(f"不支持的目标单位: {to_unit}")
|
|
|
|
from_conv = cls.UNIT_CONVERSIONS[from_unit]
|
|
to_conv = cls.UNIT_CONVERSIONS[to_unit]
|
|
|
|
if callable(from_conv):
|
|
value = from_conv(value)
|
|
|
|
if callable(to_conv):
|
|
return value / to_conv
|
|
else:
|
|
return value * (to_conv / from_conv)
|
|
|
|
@classmethod
|
|
def validate_sensor_data(cls, data: Dict[str, Any]) -> Dict[str, Any]:
|
|
"""验证传感器数据"""
|
|
errors = []
|
|
validated_data = {}
|
|
|
|
# 必需字段验证
|
|
required_fields = ["device_id", "data_type", "value"]
|
|
for field in required_fields:
|
|
if field not in data:
|
|
errors.append(f"缺少必需字段: {field}")
|
|
else:
|
|
validated_data[field] = data[field]
|
|
|
|
# 数据类型验证和标准化
|
|
if "data_type" in validated_data:
|
|
original_type = validated_data["data_type"]
|
|
validated_data["data_type"] = cls.normalize_field_name(original_type)
|
|
|
|
if validated_data["data_type"] != original_type:
|
|
logger.info(f"字段名标准化: {original_type} -> {validated_data['data_type']}")
|
|
|
|
# 数值验证
|
|
if "value" in validated_data:
|
|
try:
|
|
validated_data["value"] = float(validated_data["value"])
|
|
# 检查数值范围
|
|
data_type = validated_data.get("data_type", "")
|
|
if data_type == "LL" and validated_data["value"] < 0:
|
|
errors.append("流量不能为负数")
|
|
elif data_type == "YL" and validated_data["value"] < 0:
|
|
errors.append("压力不能为负数")
|
|
elif data_type == "SW" and validated_data["value"] < 0:
|
|
errors.append("水位不能为负数")
|
|
except (ValueError, TypeError):
|
|
errors.append(f"无效的数值: {validated_data['value']}")
|
|
|
|
# 地点验证
|
|
if "location" not in validated_data or not validated_data["location"]:
|
|
validated_data["location"] = "未知"
|
|
|
|
# 时间戳处理
|
|
if "timestamp" in data:
|
|
try:
|
|
if isinstance(data["timestamp"], str):
|
|
validated_data["timestamp"] = datetime.fromisoformat(data["timestamp"])
|
|
else:
|
|
validated_data["timestamp"] = data["timestamp"]
|
|
except (ValueError, TypeError):
|
|
validated_data["timestamp"] = datetime.now()
|
|
else:
|
|
validated_data["timestamp"] = datetime.now()
|
|
|
|
return {
|
|
"valid": len(errors) == 0,
|
|
"data": validated_data,
|
|
"errors": errors
|
|
}
|
|
|
|
class DataFormatter:
|
|
"""数据格式化器"""
|
|
|
|
@staticmethod
|
|
def format_sensor_data(data: Dict[str, Any], format_type: str = "json") -> str:
|
|
"""格式化传感器数据"""
|
|
if format_type == "json":
|
|
return json.dumps(data, ensure_ascii=False, indent=2)
|
|
elif format_type == "csv":
|
|
# CSV格式只包含关键字段
|
|
csv_fields = ["device_id", "data_type", "value", "location", "timestamp"]
|
|
csv_data = {k: data.get(k, "") for k in csv_fields}
|
|
import io
|
|
output = io.StringIO()
|
|
writer = csv.DictWriter(output, fieldnames=csv_fields)
|
|
writer.writeheader()
|
|
writer.writerow(csv_data)
|
|
return output.getvalue()
|
|
else:
|
|
raise ValueError(f"不支持的格式类型: {format_type}")
|
|
|
|
@staticmethod
|
|
def format_statistics(stats: Dict[str, Any], format_type: str = "text") -> str:
|
|
"""格式化统计数据"""
|
|
if format_type == "json":
|
|
return json.dumps(stats, ensure_ascii=False, indent=2)
|
|
elif format_type == "text":
|
|
lines = ["数据统计报告", "=" * 20]
|
|
lines.append(f"总记录数: {stats.get('total_records', 0)}")
|
|
|
|
if "by_type" in stats:
|
|
lines.append("\n按数据类型统计:")
|
|
for data_type, count in stats["by_type"].items():
|
|
lines.append(f" {data_type}: {count} 条")
|
|
|
|
if "by_device" in stats:
|
|
lines.append("\n按设备统计:")
|
|
for device_id, count in list(stats["by_device"].items())[:10]: # 只显示前10个
|
|
lines.append(f" {device_id}: {count} 条")
|
|
|
|
return "\n".join(lines)
|
|
else:
|
|
raise ValueError(f"不支持的格式类型: {format_type}")
|
|
|
|
class DataHasher:
|
|
"""数据哈希工具"""
|
|
|
|
@staticmethod
|
|
def calculate_data_hash(data: Dict[str, Any]) -> str:
|
|
"""计算数据哈希值"""
|
|
# 将数据转换为字符串
|
|
data_str = json.dumps(data, sort_keys=True, ensure_ascii=False)
|
|
|
|
# 计算MD5哈希
|
|
hash_md5 = hashlib.md5(data_str.encode())
|
|
return hash_md5.hexdigest()
|
|
|
|
@staticmethod
|
|
def generate_data_id(device_id: str, data_type: str, timestamp: datetime) -> str:
|
|
"""生成数据ID"""
|
|
# 使用设备ID、数据类型和时间戳生成唯一ID
|
|
time_str = timestamp.strftime("%Y%m%d_%H%M%S")
|
|
hash_input = f"{device_id}_{data_type}_{time_str}"
|
|
hash_md5 = hashlib.md5(hash_input.encode())
|
|
return f"{data_type}_{device_id}_{hash_md5.hexdigest()[:8]}"
|
|
|
|
class DataQualityChecker:
|
|
"""数据质量检查器"""
|
|
|
|
@staticmethod
|
|
def check_data_quality(records: List[Dict[str, Any]]) -> Dict[str, Any]:
|
|
"""检查数据质量"""
|
|
quality_report = {
|
|
"total_records": len(records),
|
|
"valid_records": 0,
|
|
"invalid_records": 0,
|
|
"quality_score": 0,
|
|
"issues": [],
|
|
"statistics": {}
|
|
}
|
|
|
|
if not records:
|
|
quality_report["quality_score"] = 0
|
|
return quality_report
|
|
|
|
valid_records = []
|
|
|
|
for record in records:
|
|
issues = []
|
|
|
|
# 检查必需字段
|
|
required_fields = ["device_id", "data_type", "value"]
|
|
for field in required_fields:
|
|
if field not in record or not record[field]:
|
|
issues.append(f"缺少必需字段: {field}")
|
|
|
|
# 检查数据类型
|
|
if "data_type" in record and record["data_type"]:
|
|
valid_types = ["LL", "YL", "SW", "ZD", "PH", "WD", "DD", "YD"]
|
|
if record["data_type"] not in valid_types:
|
|
issues.append(f"无效的数据类型: {record['data_type']}")
|
|
|
|
# 检查数值范围
|
|
if "value" in record and record["value"]:
|
|
try:
|
|
value = float(record["value"])
|
|
data_type = record.get("data_type", "")
|
|
|
|
if data_type == "LL" and value < 0:
|
|
issues.append("流量不能为负数")
|
|
elif data_type == "YL" and value < 0:
|
|
issues.append("压力不能为负数")
|
|
elif data_type == "SW" and value < 0:
|
|
issues.append("水位不能为负数")
|
|
|
|
# 检查异常值
|
|
if data_type == "LL" and value > 10000:
|
|
issues.append("流量值异常大")
|
|
elif data_type == "YL" and value > 10:
|
|
issues.append("压力值异常大")
|
|
|
|
except (ValueError, TypeError):
|
|
issues.append("无效的数值格式")
|
|
|
|
if not issues:
|
|
valid_records.append(record)
|
|
quality_report["valid_records"] += 1
|
|
else:
|
|
quality_report["invalid_records"] += 1
|
|
quality_report["issues"].extend(issues)
|
|
|
|
# 计算质量分数
|
|
quality_report["quality_score"] = quality_report["valid_records"] / len(records)
|
|
|
|
# 统计信息
|
|
if records:
|
|
quality_report["statistics"] = {
|
|
"completeness": quality_report["valid_records"] / len(records),
|
|
"uniqueness": len(set(r.get("device_id", "") for r in valid_records)) / len(valid_records) if valid_records else 0,
|
|
"timeliness": quality_report.calculate_timeliness(records)
|
|
}
|
|
|
|
return quality_report
|
|
|
|
@staticmethod
|
|
def calculate_timeliness(records: List[Dict[str, Any]]) -> float:
|
|
"""计算数据及时性（24小时内的数据比例）"""
|
|
if not records:
|
|
return 0
|
|
|
|
now = datetime.now()
|
|
recent_count = 0
|
|
|
|
for record in records:
|
|
timestamp = record.get("timestamp")
|
|
if timestamp:
|
|
try:
|
|
if isinstance(timestamp, str):
|
|
timestamp = datetime.fromisoformat(timestamp)
|
|
|
|
time_diff = now - timestamp
|
|
if time_diff <= timedelta(hours=24):
|
|
recent_count += 1
|
|
except:
|
|
pass
|
|
|
|
return recent_count / len(records)
|
|
|
|
class DataExporter:
|
|
"""数据导出工具"""
|
|
|
|
@staticmethod
|
|
def export_to_csv(records: List[Dict[str, Any]], file_path: str) -> bool:
|
|
"""导出为CSV文件"""
|
|
try:
|
|
if not records:
|
|
return False
|
|
|
|
# 获取所有字段
|
|
all_fields = set()
|
|
for record in records:
|
|
all_fields.update(record.keys())
|
|
|
|
# 排序字段
|
|
field_order = ["device_id", "data_type", "value", "location", "timestamp"]
|
|
for field in all_fields:
|
|
if field not in field_order:
|
|
field_order.append(field)
|
|
|
|
with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
|
|
writer = csv.DictWriter(csvfile, fieldnames=field_order)
|
|
writer.writeheader()
|
|
writer.writerows(records)
|
|
|
|
return True
|
|
|
|
except Exception as e:
|
|
logger.error(f"导出CSV失败: {str(e)}")
|
|
return False
|
|
|
|
@staticmethod
|
|
def export_to_json(records: List[Dict[str, Any]], file_path: str) -> bool:
|
|
"""导出为JSON文件"""
|
|
try:
|
|
with open(file_path, 'w', encoding='utf-8') as jsonfile:
|
|
json.dump(records, jsonfile, ensure_ascii=False, indent=2, default=str)
|
|
return True
|
|
except Exception as e:
|
|
logger.error(f"导出JSON失败: {str(e)}")
|
|
return False
|
|
|
|
# 全局工具实例
|
|
data_converter = DataConverter()
|
|
data_formatter = DataFormatter()
|
|
data_hasher = DataHasher()
|
|
quality_checker = DataQualityChecker()
|
|
data_exporter = DataExporter() |