实现数据接入层功能（REST API + WebSocket + 批量导入）
- 实现REST API服务器，支持IoT数据、手动录入和批量导入接口 - 实现WebSocket服务器，支持实时数据推送和连接管理 - 实现批量导入模块，支持CSV、Excel、JSON多种格式 - 实现数据处理工具，包含字段映射和单位转换功能 - 实现数据模型定义和数据验证机制 - 创建主程序入口和配置文件 - 添加详细的使用文档和API说明
This commit is contained in:
@@ -0,0 +1,409 @@
|
||||
"""
|
||||
数据处理工具模块
|
||||
提供数据验证、转换、格式化等工具函数
|
||||
"""
|
||||
import json
|
||||
import csv
|
||||
import pandas as pd
|
||||
from typing import Dict, List, Any, Optional, Union
|
||||
from datetime import datetime, timedelta
|
||||
import hashlib
|
||||
import logging
|
||||
|
||||
# 配置日志
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class DataConverter:
|
||||
"""数据转换器"""
|
||||
|
||||
# 水利行业标准字段映射
|
||||
FIELD_MAPPING = {
|
||||
"流量": "LL",
|
||||
"压力": "YL",
|
||||
"水位": "SW",
|
||||
"浊度": "ZD",
|
||||
"pH值": "PH",
|
||||
"温度": "WD",
|
||||
"电导率": "DD",
|
||||
"硬度": "YD",
|
||||
# 支持常见的中文字段名
|
||||
"流量计": "LL",
|
||||
"压力表": "YL",
|
||||
"水位计": "SW",
|
||||
"浊度仪": "ZD",
|
||||
"pH计": "PH",
|
||||
"温度计": "WD",
|
||||
"电导率仪": "DD",
|
||||
"硬度计": "YD"
|
||||
}
|
||||
|
||||
# 单位转换
|
||||
UNIT_CONVERSIONS = {
|
||||
# 流量单位转换 (m³/h)
|
||||
"m³/h": 1.0,
|
||||
"L/s": 3.6, # L/s = m³/h / 1000 * 3600
|
||||
"m³/d": 1/24, # m³/d = m³/h / 24
|
||||
"L/min": 1/60, # L/min = m³/h / 1000 * 60
|
||||
|
||||
# 压力单位转换 (MPa)
|
||||
"MPa": 1.0,
|
||||
"kPa": 0.001, # kPa = MPa / 1000
|
||||
"bar": 0.1, # bar = MPa * 10
|
||||
"kgf/cm²": 0.0980665, # kgf/cm² = MPa / 0.0980665
|
||||
|
||||
# 水位单位转换 (m)
|
||||
"m": 1.0,
|
||||
"cm": 0.01, # cm = m / 100
|
||||
"mm": 0.001, # mm = m / 1000
|
||||
|
||||
# 浊度单位转换 (NTU)
|
||||
"NTU": 1.0,
|
||||
"FNU": 1.0, # FNU ≈ NTU
|
||||
|
||||
# pH值单位转换
|
||||
"pH": 1.0,
|
||||
|
||||
# 温度单位转换 (°C)
|
||||
"°C": 1.0,
|
||||
"K": 1.0, # 相对差值
|
||||
"°F": lambda x: (x - 32) / 1.8, # °F to °C
|
||||
|
||||
# 电导率单位转换 (μS/cm)
|
||||
"μS/cm": 1.0,
|
||||
"mS/cm": 1000, # mS/cm = μS/cm * 1000
|
||||
"S/m": 10000 # S/m = μS/cm * 100
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def normalize_field_name(cls, field_name: str) -> str:
|
||||
"""标准化字段名"""
|
||||
if not field_name:
|
||||
return ""
|
||||
|
||||
field_name = field_name.strip().upper()
|
||||
|
||||
# 如果已经是标准格式，直接返回
|
||||
if field_name in cls.FIELD_MAPPING.values():
|
||||
return field_name
|
||||
|
||||
# 查映射表
|
||||
if field_name in cls.FIELD_MAPPING:
|
||||
return cls.FIELD_MAPPING[field_name]
|
||||
|
||||
# 英文映射
|
||||
english_mapping = {
|
||||
"flow": "LL",
|
||||
"pressure": "YL",
|
||||
"level": "SW",
|
||||
"turbidity": "ZD",
|
||||
"ph": "PH",
|
||||
"temperature": "WD",
|
||||
"conductivity": "DD",
|
||||
"hardness": "YD"
|
||||
}
|
||||
|
||||
if field_name.lower() in english_mapping:
|
||||
return english_mapping[field_name.lower()]
|
||||
|
||||
return field_name
|
||||
|
||||
@classmethod
|
||||
def convert_unit(cls, value: float, from_unit: str, to_unit: str) -> float:
|
||||
"""单位转换"""
|
||||
if from_unit == to_unit:
|
||||
return value
|
||||
|
||||
if from_unit not in cls.UNIT_CONVERSIONS:
|
||||
raise ValueError(f"不支持的单位: {from_unit}")
|
||||
|
||||
if to_unit not in cls.UNIT_CONVERSIONS:
|
||||
raise ValueError(f"不支持的目标单位: {to_unit}")
|
||||
|
||||
from_conv = cls.UNIT_CONVERSIONS[from_unit]
|
||||
to_conv = cls.UNIT_CONVERSIONS[to_unit]
|
||||
|
||||
if callable(from_conv):
|
||||
value = from_conv(value)
|
||||
|
||||
if callable(to_conv):
|
||||
return value / to_conv
|
||||
else:
|
||||
return value * (to_conv / from_conv)
|
||||
|
||||
@classmethod
|
||||
def validate_sensor_data(cls, data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""验证传感器数据"""
|
||||
errors = []
|
||||
validated_data = {}
|
||||
|
||||
# 必需字段验证
|
||||
required_fields = ["device_id", "data_type", "value"]
|
||||
for field in required_fields:
|
||||
if field not in data:
|
||||
errors.append(f"缺少必需字段: {field}")
|
||||
else:
|
||||
validated_data[field] = data[field]
|
||||
|
||||
# 数据类型验证和标准化
|
||||
if "data_type" in validated_data:
|
||||
original_type = validated_data["data_type"]
|
||||
validated_data["data_type"] = cls.normalize_field_name(original_type)
|
||||
|
||||
if validated_data["data_type"] != original_type:
|
||||
logger.info(f"字段名标准化: {original_type} -> {validated_data['data_type']}")
|
||||
|
||||
# 数值验证
|
||||
if "value" in validated_data:
|
||||
try:
|
||||
validated_data["value"] = float(validated_data["value"])
|
||||
# 检查数值范围
|
||||
data_type = validated_data.get("data_type", "")
|
||||
if data_type == "LL" and validated_data["value"] < 0:
|
||||
errors.append("流量不能为负数")
|
||||
elif data_type == "YL" and validated_data["value"] < 0:
|
||||
errors.append("压力不能为负数")
|
||||
elif data_type == "SW" and validated_data["value"] < 0:
|
||||
errors.append("水位不能为负数")
|
||||
except (ValueError, TypeError):
|
||||
errors.append(f"无效的数值: {validated_data['value']}")
|
||||
|
||||
# 地点验证
|
||||
if "location" not in validated_data or not validated_data["location"]:
|
||||
validated_data["location"] = "未知"
|
||||
|
||||
# 时间戳处理
|
||||
if "timestamp" in data:
|
||||
try:
|
||||
if isinstance(data["timestamp"], str):
|
||||
validated_data["timestamp"] = datetime.fromisoformat(data["timestamp"])
|
||||
else:
|
||||
validated_data["timestamp"] = data["timestamp"]
|
||||
except (ValueError, TypeError):
|
||||
validated_data["timestamp"] = datetime.now()
|
||||
else:
|
||||
validated_data["timestamp"] = datetime.now()
|
||||
|
||||
return {
|
||||
"valid": len(errors) == 0,
|
||||
"data": validated_data,
|
||||
"errors": errors
|
||||
}
|
||||
|
||||
class DataFormatter:
|
||||
"""数据格式化器"""
|
||||
|
||||
@staticmethod
|
||||
def format_sensor_data(data: Dict[str, Any], format_type: str = "json") -> str:
|
||||
"""格式化传感器数据"""
|
||||
if format_type == "json":
|
||||
return json.dumps(data, ensure_ascii=False, indent=2)
|
||||
elif format_type == "csv":
|
||||
# CSV格式只包含关键字段
|
||||
csv_fields = ["device_id", "data_type", "value", "location", "timestamp"]
|
||||
csv_data = {k: data.get(k, "") for k in csv_fields}
|
||||
import io
|
||||
output = io.StringIO()
|
||||
writer = csv.DictWriter(output, fieldnames=csv_fields)
|
||||
writer.writeheader()
|
||||
writer.writerow(csv_data)
|
||||
return output.getvalue()
|
||||
else:
|
||||
raise ValueError(f"不支持的格式类型: {format_type}")
|
||||
|
||||
@staticmethod
|
||||
def format_statistics(stats: Dict[str, Any], format_type: str = "text") -> str:
|
||||
"""格式化统计数据"""
|
||||
if format_type == "json":
|
||||
return json.dumps(stats, ensure_ascii=False, indent=2)
|
||||
elif format_type == "text":
|
||||
lines = ["数据统计报告", "=" * 20]
|
||||
lines.append(f"总记录数: {stats.get('total_records', 0)}")
|
||||
|
||||
if "by_type" in stats:
|
||||
lines.append("\n按数据类型统计:")
|
||||
for data_type, count in stats["by_type"].items():
|
||||
lines.append(f" {data_type}: {count} 条")
|
||||
|
||||
if "by_device" in stats:
|
||||
lines.append("\n按设备统计:")
|
||||
for device_id, count in list(stats["by_device"].items())[:10]: # 只显示前10个
|
||||
lines.append(f" {device_id}: {count} 条")
|
||||
|
||||
return "\n".join(lines)
|
||||
else:
|
||||
raise ValueError(f"不支持的格式类型: {format_type}")
|
||||
|
||||
class DataHasher:
|
||||
"""数据哈希工具"""
|
||||
|
||||
@staticmethod
|
||||
def calculate_data_hash(data: Dict[str, Any]) -> str:
|
||||
"""计算数据哈希值"""
|
||||
# 将数据转换为字符串
|
||||
data_str = json.dumps(data, sort_keys=True, ensure_ascii=False)
|
||||
|
||||
# 计算MD5哈希
|
||||
hash_md5 = hashlib.md5(data_str.encode())
|
||||
return hash_md5.hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def generate_data_id(device_id: str, data_type: str, timestamp: datetime) -> str:
|
||||
"""生成数据ID"""
|
||||
# 使用设备ID、数据类型和时间戳生成唯一ID
|
||||
time_str = timestamp.strftime("%Y%m%d_%H%M%S")
|
||||
hash_input = f"{device_id}_{data_type}_{time_str}"
|
||||
hash_md5 = hashlib.md5(hash_input.encode())
|
||||
return f"{data_type}_{device_id}_{hash_md5.hexdigest()[:8]}"
|
||||
|
||||
class DataQualityChecker:
|
||||
"""数据质量检查器"""
|
||||
|
||||
@staticmethod
|
||||
def check_data_quality(records: List[Dict[str, Any]]) -> Dict[str, Any]:
|
||||
"""检查数据质量"""
|
||||
quality_report = {
|
||||
"total_records": len(records),
|
||||
"valid_records": 0,
|
||||
"invalid_records": 0,
|
||||
"quality_score": 0,
|
||||
"issues": [],
|
||||
"statistics": {}
|
||||
}
|
||||
|
||||
if not records:
|
||||
quality_report["quality_score"] = 0
|
||||
return quality_report
|
||||
|
||||
valid_records = []
|
||||
|
||||
for record in records:
|
||||
issues = []
|
||||
|
||||
# 检查必需字段
|
||||
required_fields = ["device_id", "data_type", "value"]
|
||||
for field in required_fields:
|
||||
if field not in record or not record[field]:
|
||||
issues.append(f"缺少必需字段: {field}")
|
||||
|
||||
# 检查数据类型
|
||||
if "data_type" in record and record["data_type"]:
|
||||
valid_types = ["LL", "YL", "SW", "ZD", "PH", "WD", "DD", "YD"]
|
||||
if record["data_type"] not in valid_types:
|
||||
issues.append(f"无效的数据类型: {record['data_type']}")
|
||||
|
||||
# 检查数值范围
|
||||
if "value" in record and record["value"]:
|
||||
try:
|
||||
value = float(record["value"])
|
||||
data_type = record.get("data_type", "")
|
||||
|
||||
if data_type == "LL" and value < 0:
|
||||
issues.append("流量不能为负数")
|
||||
elif data_type == "YL" and value < 0:
|
||||
issues.append("压力不能为负数")
|
||||
elif data_type == "SW" and value < 0:
|
||||
issues.append("水位不能为负数")
|
||||
|
||||
# 检查异常值
|
||||
if data_type == "LL" and value > 10000:
|
||||
issues.append("流量值异常大")
|
||||
elif data_type == "YL" and value > 10:
|
||||
issues.append("压力值异常大")
|
||||
|
||||
except (ValueError, TypeError):
|
||||
issues.append("无效的数值格式")
|
||||
|
||||
if not issues:
|
||||
valid_records.append(record)
|
||||
quality_report["valid_records"] += 1
|
||||
else:
|
||||
quality_report["invalid_records"] += 1
|
||||
quality_report["issues"].extend(issues)
|
||||
|
||||
# 计算质量分数
|
||||
quality_report["quality_score"] = quality_report["valid_records"] / len(records)
|
||||
|
||||
# 统计信息
|
||||
if records:
|
||||
quality_report["statistics"] = {
|
||||
"completeness": quality_report["valid_records"] / len(records),
|
||||
"uniqueness": len(set(r.get("device_id", "") for r in valid_records)) / len(valid_records) if valid_records else 0,
|
||||
"timeliness": quality_report.calculate_timeliness(records)
|
||||
}
|
||||
|
||||
return quality_report
|
||||
|
||||
@staticmethod
|
||||
def calculate_timeliness(records: List[Dict[str, Any]]) -> float:
|
||||
"""计算数据及时性（24小时内的数据比例）"""
|
||||
if not records:
|
||||
return 0
|
||||
|
||||
now = datetime.now()
|
||||
recent_count = 0
|
||||
|
||||
for record in records:
|
||||
timestamp = record.get("timestamp")
|
||||
if timestamp:
|
||||
try:
|
||||
if isinstance(timestamp, str):
|
||||
timestamp = datetime.fromisoformat(timestamp)
|
||||
|
||||
time_diff = now - timestamp
|
||||
if time_diff <= timedelta(hours=24):
|
||||
recent_count += 1
|
||||
except:
|
||||
pass
|
||||
|
||||
return recent_count / len(records)
|
||||
|
||||
class DataExporter:
|
||||
"""数据导出工具"""
|
||||
|
||||
@staticmethod
|
||||
def export_to_csv(records: List[Dict[str, Any]], file_path: str) -> bool:
|
||||
"""导出为CSV文件"""
|
||||
try:
|
||||
if not records:
|
||||
return False
|
||||
|
||||
# 获取所有字段
|
||||
all_fields = set()
|
||||
for record in records:
|
||||
all_fields.update(record.keys())
|
||||
|
||||
# 排序字段
|
||||
field_order = ["device_id", "data_type", "value", "location", "timestamp"]
|
||||
for field in all_fields:
|
||||
if field not in field_order:
|
||||
field_order.append(field)
|
||||
|
||||
with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
|
||||
writer = csv.DictWriter(csvfile, fieldnames=field_order)
|
||||
writer.writeheader()
|
||||
writer.writerows(records)
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"导出CSV失败: {str(e)}")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def export_to_json(records: List[Dict[str, Any]], file_path: str) -> bool:
|
||||
"""导出为JSON文件"""
|
||||
try:
|
||||
with open(file_path, 'w', encoding='utf-8') as jsonfile:
|
||||
json.dump(records, jsonfile, ensure_ascii=False, indent=2, default=str)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"导出JSON失败: {str(e)}")
|
||||
return False
|
||||
|
||||
# 全局工具实例
|
||||
data_converter = DataConverter()
|
||||
data_formatter = DataFormatter()
|
||||
data_hasher = DataHasher()
|
||||
quality_checker = DataQualityChecker()
|
||||
data_exporter = DataExporter()
|
||||
Reference in New Issue
Block a user