RD-Agent 教程 - 第五章:问题解决与最佳实践¶
📋 本章概览¶
第五章将系统性地介绍在使用 RD-Agent 过程中可能遇到的常见问题及其解决方案,同时提供生产环境下的最佳实践指导。
🎯 章节目标¶
- ✅ 掌握常见问题诊断:快速定位和解决典型错误
- ✅ 理解系统架构:深入了解RD-Agent内部工作机制
- ✅ 优化性能配置:提升系统运行效率和稳定性
- ✅ 建立监控体系:实现全面的系统监控和告警
- ✅ 制定运维流程:构建可靠的生产环境运维体系
📁 本章结构¶
tutorial_chapter5/
├── tutorial_chapter5_problem_solving.md # 本文件 - 问题解决指南
├── examples/ # 代码示例目录
│ ├── diagnostic_tools.py # 诊断工具脚本
│ ├── performance_monitor.py # 性能监控工具
│ ├── health_checker.py # 健康检查脚本
│ ├── log_analyzer.py # 日志分析工具
│ └── troubleshooting_guide.py # 故障排除指南
└── configs/ # 配置文件示例
├── production_config.yaml # 生产环境配置
├── monitoring_config.yaml # 监控配置
└── logging_config.yaml # 日志配置
🚨 第一部分:常见问题诊断与解决¶
1.1 安装与环境问题¶
问题1:Python版本兼容性错误¶
症状描述:
ERROR: RD-Agent requires Python 3.10 or higher
Current Python version: 3.9.7
解决方案:
# 方案A:使用pyenv管理Python版本
curl https://pyenv.run | bash
pyenv install 3.11.0
pyenv global 3.11.0
# 方案B:使用conda创建新环境
conda create -n rdagent python=3.11
conda activate rdagent
# 方案C:使用虚拟环境
python3.11 -m venv rdagent_env
source rdagent_env/bin/activate # Linux/macOS
# rdagent_env\Scripts\activate # Windows
预防措施:
- 在项目README中明确Python版本要求
- 使用.python-version文件锁定Python版本
- CI/CD中增加Python版本检查
问题2:依赖包冲突¶
症状描述:
ERROR: pip's dependency resolver does not currently consider all packages
Conflicting packages: tensorflow 2.10.0 vs torch 1.13.0
解决方案:
# 1. 清理现有环境
pip freeze > requirements_backup.txt
pip uninstall -r requirements_backup.txt -y
# 2. 分步安装核心依赖
pip install rdagent==0.1.0
pip install torch==1.13.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
pip install tensorflow==2.12.0
# 3. 使用conda解决复杂依赖
conda install pytorch tensorflow scikit-learn -c pytorch -c conda-forge
高级技巧:
# dependency_resolver.py - 智能依赖解析工具
import subprocess
import pkg_resources
from typing import List, Dict
class DependencyResolver:
def __init__(self):
self.conflicts = []
self.recommendations = {}
def check_compatibility(self, packages: List[str]) -> Dict:
"""检查包兼容性"""
results = {"compatible": [], "conflicts": [], "missing": []}
for package in packages:
try:
pkg_resources.require(package)
results["compatible"].append(package)
except pkg_resources.DistributionNotFound:
results["missing"].append(package)
except pkg_resources.VersionConflict as e:
results["conflicts"].append(str(e))
return results
def suggest_resolution(self, conflicts: List[str]) -> Dict:
"""建议解决方案"""
suggestions = {}
for conflict in conflicts:
if "tensorflow" in conflict and "torch" in conflict:
suggestions[conflict] = "使用CPU版本或分离环境"
elif "numpy" in conflict:
suggestions[conflict] = "升级到兼容的numpy版本"
return suggestions
# 使用示例
resolver = DependencyResolver()
required_packages = ["rdagent", "torch>=1.10", "tensorflow>=2.8"]
results = resolver.check_compatibility(required_packages)
print(f"兼容包: {results['compatible']}")
print(f"冲突包: {results['conflicts']}")
问题3:Docker权限问题¶
症状描述:
docker: permission denied while trying to connect to the Docker daemon socket
解决方案:
# Linux解决方案
sudo usermod -aG docker $USER
newgrp docker
# macOS解决方案
# 确保Docker Desktop正在运行
open -a Docker
# 验证Docker权限
docker run hello-world
1.2 API配置问题¶
问题4:OpenAI API密钥无效¶
症状描述:
openai.error.AuthenticationError: Invalid API Key provided
解决方案:
# api_validator.py - API密钥验证工具
import openai
import os
from typing import Optional, Dict
class APIValidator:
def __init__(self):
self.validation_results = {}
def validate_openai_key(self, api_key: Optional[str] = None) -> Dict:
"""验证OpenAI API密钥"""
if not api_key:
api_key = os.getenv("OPENAI_API_KEY")
if not api_key:
return {"valid": False, "error": "API密钥未设置"}
try:
openai.api_key = api_key
# 使用最小成本的API调用进行验证
response = openai.Model.list()
return {"valid": True, "models_count": len(response.data)}
except openai.error.AuthenticationError:
return {"valid": False, "error": "API密钥无效"}
except openai.error.RateLimitError:
return {"valid": True, "warning": "达到速率限制,但密钥有效"}
except Exception as e:
return {"valid": False, "error": f"未知错误: {str(e)}"}
def validate_all_apis(self) -> Dict:
"""验证所有API配置"""
results = {}
# OpenAI
results["openai"] = self.validate_openai_key()
# Azure OpenAI
azure_key = os.getenv("AZURE_OPENAI_API_KEY")
azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")
if azure_key and azure_endpoint:
results["azure_openai"] = self.validate_azure_openai(azure_key, azure_endpoint)
# Kaggle
kaggle_username = os.getenv("KAGGLE_USERNAME")
kaggle_key = os.getenv("KAGGLE_KEY")
if kaggle_username and kaggle_key:
results["kaggle"] = self.validate_kaggle_api(kaggle_username, kaggle_key)
return results
def validate_azure_openai(self, api_key: str, endpoint: str) -> Dict:
"""验证Azure OpenAI配置"""
try:
import requests
headers = {
"api-key": api_key,
"Content-Type": "application/json"
}
url = f"{endpoint}/openai/deployments?api-version=2023-05-15"
response = requests.get(url, headers=headers, timeout=10)
if response.status_code == 200:
return {"valid": True, "deployments": len(response.json().get("data", []))}
else:
return {"valid": False, "error": f"HTTP {response.status_code}"}
except Exception as e:
return {"valid": False, "error": str(e)}
def validate_kaggle_api(self, username: str, key: str) -> Dict:
"""验证Kaggle API配置"""
try:
from kaggle.api.kaggle_api_extended import KaggleApi
api = KaggleApi()
api.authenticate()
# 尝试获取用户信息
user_info = api.get_user()
return {"valid": True, "username": user_info}
except Exception as e:
return {"valid": False, "error": str(e)}
# 使用示例
validator = APIValidator()
results = validator.validate_all_apis()
for service, result in results.items():
if result["valid"]:
print(f"✅ {service}: 配置正确")
else:
print(f"❌ {service}: {result['error']}")
问题5:API速率限制¶
症状描述:
openai.error.RateLimitError: Rate limit reached for requests
解决方案:
# rate_limiter.py - 智能速率限制器
import time
import asyncio
from typing import Callable, Any
from functools import wraps
import logging
class RateLimiter:
def __init__(self, calls_per_minute: int = 60):
self.calls_per_minute = calls_per_minute
self.calls = []
self.lock = asyncio.Lock()
async def acquire(self):
"""获取调用许可"""
async with self.lock:
now = time.time()
# 清理一分钟前的调用记录
self.calls = [call_time for call_time in self.calls if now - call_time < 60]
if len(self.calls) >= self.calls_per_minute:
# 计算需要等待的时间
wait_time = 60 - (now - self.calls[0])
if wait_time > 0:
logging.info(f"速率限制:等待 {wait_time:.2f} 秒")
await asyncio.sleep(wait_time)
return await self.acquire()
self.calls.append(now)
def rate_limited(calls_per_minute: int = 60):
"""速率限制装饰器"""
limiter = RateLimiter(calls_per_minute)
def decorator(func: Callable) -> Callable:
@wraps(func)
async def wrapper(*args, **kwargs):
await limiter.acquire()
return await func(*args, **kwargs)
return wrapper
return decorator
# 使用示例
@rate_limited(calls_per_minute=20) # 每分钟最多20次调用
async def call_openai_api(prompt: str):
"""调用OpenAI API"""
try:
response = await openai.ChatCompletion.acreate(
model="gpt-3.5-turbo",
messages=[{"role": "user", "content": prompt}],
max_tokens=100
)
return response.choices[0].message.content
except Exception as e:
logging.error(f"API调用失败: {e}")
raise
# 批量处理示例
async def process_multiple_requests(prompts: list):
"""批量处理多个请求"""
tasks = [call_openai_api(prompt) for prompt in prompts]
results = await asyncio.gather(*tasks, return_exceptions=True)
successful = [r for r in results if not isinstance(r, Exception)]
failed = [r for r in results if isinstance(r, Exception)]
print(f"成功: {len(successful)}, 失败: {len(failed)}")
return successful, failed
1.3 数据处理问题¶
问题6:内存不足错误¶
症状描述:
MemoryError: Unable to allocate 8.00 GiB for an array
解决方案:
# memory_optimizer.py - 内存优化工具
import pandas as pd
import numpy as np
import gc
import psutil
from typing import Iterator, Optional
import logging
class MemoryOptimizer:
def __init__(self):
self.logger = logging.getLogger(__name__)
def get_memory_usage(self) -> Dict[str, float]:
"""获取当前内存使用情况"""
process = psutil.Process()
memory_info = process.memory_info()
return {
"rss_gb": memory_info.rss / 1024**3, # 物理内存
"vms_gb": memory_info.vms / 1024**3, # 虚拟内存
"percent": process.memory_percent() # 内存使用百分比
}
def optimize_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
"""优化DataFrame内存使用"""
initial_memory = df.memory_usage(deep=True).sum() / 1024**2
# 优化数值类型
for col in df.select_dtypes(include=[np.number]).columns:
col_type = df[col].dtype
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
df[col] = df[col].astype(np.int8)
elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
df[col] = df[col].astype(np.int16)
elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
df[col] = df[col].astype(np.int32)
elif str(col_type)[:5] == 'float':
if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
df[col] = df[col].astype(np.float32)
# 优化字符串类型
for col in df.select_dtypes(include=['object']).columns:
if df[col].nunique() / len(df) < 0.5: # 如果唯一值比例小于50%,转换为category
df[col] = df[col].astype('category')
final_memory = df.memory_usage(deep=True).sum() / 1024**2
reduction = (1 - final_memory/initial_memory) * 100
self.logger.info(f"内存优化完成: {initial_memory:.2f}MB -> {final_memory:.2f}MB "
f"(减少 {reduction:.1f}%)")
return df
def chunked_processing(self, file_path: str, chunk_size: int = 10000) -> Iterator[pd.DataFrame]:
"""分块处理大文件"""
self.logger.info(f"开始分块处理文件: {file_path}")
try:
chunk_iterator = pd.read_csv(file_path, chunksize=chunk_size)
for i, chunk in enumerate(chunk_iterator):
# 优化每个chunk的内存使用
optimized_chunk = self.optimize_dataframe(chunk)
# 监控内存使用
memory_usage = self.get_memory_usage()
if memory_usage["percent"] > 80:
self.logger.warning(f"内存使用率过高: {memory_usage['percent']:.1f}%")
gc.collect() # 强制垃圾回收
yield optimized_chunk
except Exception as e:
self.logger.error(f"文件处理失败: {e}")
raise
# 使用示例
optimizer = MemoryOptimizer()
# 处理大文件
def process_large_dataset(file_path: str):
results = []
for chunk in optimizer.chunked_processing(file_path, chunk_size=5000):
# 对每个chunk进行处理
processed_chunk = chunk.groupby('category').agg({
'value': ['mean', 'sum', 'count']
})
results.append(processed_chunk)
# 定期清理内存
if len(results) % 10 == 0:
gc.collect()
# 合并结果
final_result = pd.concat(results, ignore_index=True)
return optimizer.optimize_dataframe(final_result)
问题7:数据格式不兼容¶
症状描述:
ValueError: Input contains NaN, infinity or a value too large for dtype('float64')
解决方案:
# data_validator.py - 数据验证和清理工具
import pandas as pd
import numpy as np
from typing import Dict, List, Tuple, Optional
import logging
class DataValidator:
def __init__(self):
self.logger = logging.getLogger(__name__)
self.issues = []
def validate_dataset(self, df: pd.DataFrame) -> Dict[str, any]:
"""全面验证数据集"""
validation_report = {
"shape": df.shape,
"memory_usage_mb": df.memory_usage(deep=True).sum() / 1024**2,
"missing_values": {},
"infinite_values": {},
"duplicate_rows": 0,
"data_types": {},
"outliers": {},
"recommendations": []
}
# 检查缺失值
missing_counts = df.isnull().sum()
validation_report["missing_values"] = {
col: {"count": count, "percentage": count/len(df)*100}
for col, count in missing_counts.items() if count > 0
}
# 检查无穷值
numeric_cols = df.select_dtypes(include=[np.number]).columns
for col in numeric_cols:
inf_count = np.isinf(df[col]).sum()
if inf_count > 0:
validation_report["infinite_values"][col] = inf_count
# 检查重复行
validation_report["duplicate_rows"] = df.duplicated().sum()
# 检查数据类型
validation_report["data_types"] = df.dtypes.to_dict()
# 检查异常值(使用IQR方法)
for col in numeric_cols:
Q1 = df[col].quantile(0.25)
Q3 = df[col].quantile(0.75)
IQR = Q3 - Q1
lower_bound = Q1 - 1.5 * IQR
upper_bound = Q3 + 1.5 * IQR
outliers = df[(df[col] < lower_bound) | (df[col] > upper_bound)]
if len(outliers) > 0:
validation_report["outliers"][col] = {
"count": len(outliers),
"percentage": len(outliers)/len(df)*100,
"bounds": [lower_bound, upper_bound]
}
# 生成建议
self._generate_recommendations(validation_report)
return validation_report
def clean_dataset(self, df: pd.DataFrame, strategy: Dict[str, str] = None) -> pd.DataFrame:
"""清理数据集"""
df_cleaned = df.copy()
if strategy is None:
strategy = {
"missing_values": "drop", # drop, fill_mean, fill_median, fill_mode
"infinite_values": "replace", # replace, drop
"duplicates": "drop", # drop, keep_first
"outliers": "keep" # keep, cap, remove
}
# 处理无穷值
if strategy.get("infinite_values") == "replace":
numeric_cols = df_cleaned.select_dtypes(include=[np.number]).columns
for col in numeric_cols:
# 用列的最大/最小有限值替换无穷值
finite_values = df_cleaned[col][np.isfinite(df_cleaned[col])]
if len(finite_values) > 0:
max_finite = finite_values.max()
min_finite = finite_values.min()
df_cleaned[col] = df_cleaned[col].replace([np.inf], max_finite)
df_cleaned[col] = df_cleaned[col].replace([-np.inf], min_finite)
# 处理缺失值
missing_strategy = strategy.get("missing_values", "drop")
if missing_strategy == "drop":
df_cleaned = df_cleaned.dropna()
elif missing_strategy == "fill_mean":
numeric_cols = df_cleaned.select_dtypes(include=[np.number]).columns
df_cleaned[numeric_cols] = df_cleaned[numeric_cols].fillna(
df_cleaned[numeric_cols].mean()
)
elif missing_strategy == "fill_median":
numeric_cols = df_cleaned.select_dtypes(include=[np.number]).columns
df_cleaned[numeric_cols] = df_cleaned[numeric_cols].fillna(
df_cleaned[numeric_cols].median()
)
# 处理重复值
if strategy.get("duplicates") == "drop":
df_cleaned = df_cleaned.drop_duplicates()
# 处理异常值
outlier_strategy = strategy.get("outliers", "keep")
if outlier_strategy != "keep":
numeric_cols = df_cleaned.select_dtypes(include=[np.number]).columns
for col in numeric_cols:
Q1 = df_cleaned[col].quantile(0.25)
Q3 = df_cleaned[col].quantile(0.75)
IQR = Q3 - Q1
lower_bound = Q1 - 1.5 * IQR
upper_bound = Q3 + 1.5 * IQR
if outlier_strategy == "cap":
df_cleaned[col] = df_cleaned[col].clip(lower_bound, upper_bound)
elif outlier_strategy == "remove":
df_cleaned = df_cleaned[
(df_cleaned[col] >= lower_bound) &
(df_cleaned[col] <= upper_bound)
]
self.logger.info(f"数据清理完成: {df.shape} -> {df_cleaned.shape}")
return df_cleaned
def _generate_recommendations(self, report: Dict) -> None:
"""生成数据清理建议"""
recommendations = []
# 缺失值建议
if report["missing_values"]:
total_missing = sum(info["count"] for info in report["missing_values"].values())
recommendations.append(f"发现 {total_missing} 个缺失值,建议进行处理")
# 无穷值建议
if report["infinite_values"]:
recommendations.append("发现无穷值,建议替换或删除")
# 重复值建议
if report["duplicate_rows"] > 0:
recommendations.append(f"发现 {report['duplicate_rows']} 个重复行,建议删除")
# 内存使用建议
if report["memory_usage_mb"] > 1000:
recommendations.append("数据集较大,建议进行内存优化")
report["recommendations"] = recommendations
# 使用示例
validator = DataValidator()
# 验证数据
df = pd.read_csv("data.csv")
report = validator.validate_dataset(df)
print("数据验证报告:")
for key, value in report.items():
if key != "recommendations":
print(f"{key}: {value}")
print("\n建议:")
for rec in report["recommendations"]:
print(f"- {rec}")
# 清理数据
cleaning_strategy = {
"missing_values": "fill_median",
"infinite_values": "replace",
"duplicates": "drop",
"outliers": "cap"
}
cleaned_df = validator.clean_dataset(df, cleaning_strategy)
1.4 模型训练问题¶
问题8:训练过程中断¶
症状描述:
RuntimeError: CUDA out of memory. Tried to allocate 2.00 GiB
解决方案:
# training_manager.py - 训练管理器
import torch
import pickle
import os
import time
import logging
from typing import Dict, Optional, Callable
from datetime import datetime
class TrainingManager:
def __init__(self, checkpoint_dir: str = "./checkpoints"):
self.checkpoint_dir = checkpoint_dir
self.logger = logging.getLogger(__name__)
os.makedirs(checkpoint_dir, exist_ok=True)
# GPU内存管理
if torch.cuda.is_available():
torch.cuda.empty_cache()
self.device = torch.device("cuda")
self.logger.info(f"使用GPU: {torch.cuda.get_device_name()}")
else:
self.device = torch.device("cpu")
self.logger.info("使用CPU训练")
def save_checkpoint(self, model, optimizer, epoch: int, loss: float,
metadata: Dict = None) -> str:
"""保存训练检查点"""
checkpoint = {
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'loss': loss,
'timestamp': datetime.now().isoformat(),
'metadata': metadata or {}
}
checkpoint_path = os.path.join(
self.checkpoint_dir,
f"checkpoint_epoch_{epoch}.pth"
)
torch.save(checkpoint, checkpoint_path)
self.logger.info(f"检查点已保存: {checkpoint_path}")
return checkpoint_path
def load_checkpoint(self, model, optimizer, checkpoint_path: str) -> Dict:
"""加载训练检查点"""
if not os.path.exists(checkpoint_path):
raise FileNotFoundError(f"检查点文件不存在: {checkpoint_path}")
checkpoint = torch.load(checkpoint_path, map_location=self.device)
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
self.logger.info(f"检查点已加载: epoch {checkpoint['epoch']}, loss {checkpoint['loss']}")
return checkpoint
def train_with_recovery(self, model, train_loader, optimizer, criterion,
num_epochs: int, save_every: int = 10,
resume_from: Optional[str] = None) -> Dict:
"""带恢复功能的训练"""
start_epoch = 0
training_history = {"losses": [], "epochs": []}
# 恢复训练
if resume_from:
try:
checkpoint = self.load_checkpoint(model, optimizer, resume_from)
start_epoch = checkpoint['epoch'] + 1
training_history = checkpoint.get('metadata', {}).get('history', training_history)
self.logger.info(f"从epoch {start_epoch}恢复训练")
except Exception as e:
self.logger.error(f"恢复训练失败: {e}")
return {"error": str(e)}
model.to(self.device)
try:
for epoch in range(start_epoch, num_epochs):
epoch_loss = 0.0
model.train()
for batch_idx, (data, target) in enumerate(train_loader):
try:
# 内存管理
if torch.cuda.is_available():
torch.cuda.empty_cache()
data, target = data.to(self.device), target.to(self.device)
optimizer.zero_grad()
output = model(data)
loss = criterion(output, target)
loss.backward()
optimizer.step()
epoch_loss += loss.item()
# 监控GPU内存
if torch.cuda.is_available() and batch_idx % 100 == 0:
memory_allocated = torch.cuda.memory_allocated() / 1024**3
memory_reserved = torch.cuda.memory_reserved() / 1024**3
if memory_allocated > 0.8 * memory_reserved:
self.logger.warning(f"GPU内存使用率过高: {memory_allocated:.2f}GB")
except RuntimeError as e:
if "out of memory" in str(e):
self.logger.error("GPU内存不足,尝试清理内存")
if torch.cuda.is_available():
torch.cuda.empty_cache()
# 可以选择跳过这个batch或者减小batch size
continue
else:
raise e
avg_loss = epoch_loss / len(train_loader)
training_history["losses"].append(avg_loss)
training_history["epochs"].append(epoch)
self.logger.info(f"Epoch {epoch}/{num_epochs}, Loss: {avg_loss:.6f}")
# 定期保存检查点
if epoch % save_every == 0:
self.save_checkpoint(
model, optimizer, epoch, avg_loss,
metadata={"history": training_history}
)
except KeyboardInterrupt:
self.logger.info("训练被用户中断")
self.save_checkpoint(
model, optimizer, epoch, avg_loss,
metadata={"history": training_history, "interrupted": True}
)
except Exception as e:
self.logger.error(f"训练过程中发生错误: {e}")
self.save_checkpoint(
model, optimizer, epoch, avg_loss,
metadata={"history": training_history, "error": str(e)}
)
raise e
return training_history
def optimize_memory_usage(self, model) -> None:
"""优化内存使用"""
# 启用混合精度训练
if torch.cuda.is_available():
from torch.cuda.amp import autocast, GradScaler
self.use_amp = True
self.scaler = GradScaler()
# 启用梯度检查点(节省内存,增加计算时间)
if hasattr(model, 'gradient_checkpointing'):
model.gradient_checkpointing = True
# 优化内存分配策略
if torch.cuda.is_available():
torch.backends.cudnn.benchmark = True
torch.backends.cudnn.deterministic = False
# 使用示例
trainer = TrainingManager(checkpoint_dir="./checkpoints")
# 训练模型
training_history = trainer.train_with_recovery(
model=my_model,
train_loader=train_loader,
optimizer=optimizer,
criterion=nn.CrossEntropyLoss(),
num_epochs=100,
save_every=5,
resume_from="./checkpoints/checkpoint_epoch_20.pth" # 可选:从检查点恢复
)
🔧 第二部分:性能优化与监控¶
2.1 系统性能监控¶
监控工具实现¶
# performance_monitor.py - 综合性能监控系统
import psutil
import GPUtil
import time
import threading
import logging
from typing import Dict, List, Optional, Callable
from datetime import datetime, timedelta
import json
import matplotlib.pyplot as plt
from collections import deque
import numpy as np
class PerformanceMonitor:
def __init__(self, log_interval: int = 60, history_size: int = 1000):
self.log_interval = log_interval
self.history_size = history_size
self.logger = logging.getLogger(__name__)
# 历史数据存储
self.cpu_history = deque(maxlen=history_size)
self.memory_history = deque(maxlen=history_size)
self.gpu_history = deque(maxlen=history_size)
self.disk_history = deque(maxlen=history_size)
self.timestamp_history = deque(maxlen=history_size)
# 监控状态
self.monitoring = False
self.alert_thresholds = {
"cpu_percent": 85.0,
"memory_percent": 90.0,
"gpu_memory_percent": 95.0,
"disk_percent": 95.0
}
self.alert_callbacks = []
def get_system_metrics(self) -> Dict:
"""获取系统指标"""
metrics = {
"timestamp": datetime.now().isoformat(),
"cpu": self._get_cpu_metrics(),
"memory": self._get_memory_metrics(),
"disk": self._get_disk_metrics(),
"gpu": self._get_gpu_metrics(),
"network": self._get_network_metrics()
}
return metrics
def _get_cpu_metrics(self) -> Dict:
"""获取CPU指标"""
return {
"percent": psutil.cpu_percent(interval=1),
"count_logical": psutil.cpu_count(logical=True),
"count_physical": psutil.cpu_count(logical=False),
"frequency": psutil.cpu_freq()._asdict() if psutil.cpu_freq() else None,
"load_average": psutil.getloadavg() if hasattr(psutil, 'getloadavg') else None
}
def _get_memory_metrics(self) -> Dict:
"""获取内存指标"""
virtual_memory = psutil.virtual_memory()
swap_memory = psutil.swap_memory()
return {
"virtual": {
"total_gb": virtual_memory.total / 1024**3,
"available_gb": virtual_memory.available / 1024**3,
"used_gb": virtual_memory.used / 1024**3,
"percent": virtual_memory.percent
},
"swap": {
"total_gb": swap_memory.total / 1024**3,
"used_gb": swap_memory.used / 1024**3,
"percent": swap_memory.percent
}
}
def _get_disk_metrics(self) -> Dict:
"""获取磁盘指标"""
disk_usage = psutil.disk_usage('/')
disk_io = psutil.disk_io_counters()
return {
"usage": {
"total_gb": disk_usage.total / 1024**3,
"used_gb": disk_usage.used / 1024**3,
"free_gb": disk_usage.free / 1024**3,
"percent": (disk_usage.used / disk_usage.total) * 100
},
"io": {
"read_mb": disk_io.read_bytes / 1024**2 if disk_io else 0,
"write_mb": disk_io.write_bytes / 1024**2 if disk_io else 0
}
}
def _get_gpu_metrics(self) -> List[Dict]:
"""获取GPU指标"""
try:
gpus = GPUtil.getGPUs()
gpu_metrics = []
for gpu in gpus:
gpu_metrics.append({
"id": gpu.id,
"name": gpu.name,
"memory_total_gb": gpu.memoryTotal / 1024,
"memory_used_gb": gpu.memoryUsed / 1024,
"memory_percent": (gpu.memoryUsed / gpu.memoryTotal) * 100,
"gpu_percent": gpu.load * 100,
"temperature": gpu.temperature
})
return gpu_metrics
except Exception as e:
self.logger.warning(f"无法获取GPU信息: {e}")
return []
def _get_network_metrics(self) -> Dict:
"""获取网络指标"""
try:
net_io = psutil.net_io_counters()
return {
"bytes_sent_mb": net_io.bytes_sent / 1024**2,
"bytes_recv_mb": net_io.bytes_recv / 1024**2,
"packets_sent": net_io.packets_sent,
"packets_recv": net_io.packets_recv
}
except Exception as e:
self.logger.warning(f"无法获取网络信息: {e}")
return {}
def add_alert_callback(self, callback: Callable[[str, Dict], None]):
"""添加告警回调函数"""
self.alert_callbacks.append(callback)
def check_alerts(self, metrics: Dict) -> List[str]:
"""检查告警条件"""
alerts = []
# CPU告警
if metrics["cpu"]["percent"] > self.alert_thresholds["cpu_percent"]:
alerts.append(f"CPU使用率过高: {metrics['cpu']['percent']:.1f}%")
# 内存告警
memory_percent = metrics["memory"]["virtual"]["percent"]
if memory_percent > self.alert_thresholds["memory_percent"]:
alerts.append(f"内存使用率过高: {memory_percent:.1f}%")
# GPU告警
for gpu in metrics.get("gpu", []):
if gpu["memory_percent"] > self.alert_thresholds["gpu_memory_percent"]:
alerts.append(f"GPU {gpu['id']} 内存使用率过高: {gpu['memory_percent']:.1f}%")
# 磁盘告警
disk_percent = metrics["disk"]["usage"]["percent"]
if disk_percent > self.alert_thresholds["disk_percent"]:
alerts.append(f"磁盘使用率过高: {disk_percent:.1f}%")
return alerts
def start_monitoring(self):
"""开始监控"""
if self.monitoring:
self.logger.warning("监控已在运行")
return
self.monitoring = True
self.logger.info("开始性能监控")
def monitor_loop():
while self.monitoring:
try:
metrics = self.get_system_metrics()
# 存储历史数据
self.timestamp_history.append(datetime.now())
self.cpu_history.append(metrics["cpu"]["percent"])
self.memory_history.append(metrics["memory"]["virtual"]["percent"])
if metrics["gpu"]:
avg_gpu_memory = np.mean([gpu["memory_percent"] for gpu in metrics["gpu"]])
self.gpu_history.append(avg_gpu_memory)
self.disk_history.append(metrics["disk"]["usage"]["percent"])
# 检查告警
alerts = self.check_alerts(metrics)
for alert in alerts:
self.logger.warning(f"性能告警: {alert}")
for callback in self.alert_callbacks:
callback(alert, metrics)
time.sleep(self.log_interval)
except Exception as e:
self.logger.error(f"监控过程中发生错误: {e}")
time.sleep(self.log_interval)
self.monitor_thread = threading.Thread(target=monitor_loop, daemon=True)
self.monitor_thread.start()
def stop_monitoring(self):
"""停止监控"""
self.monitoring = False
self.logger.info("停止性能监控")
def generate_report(self, hours: int = 24) -> Dict:
"""生成性能报告"""
if not self.timestamp_history:
return {"error": "没有监控数据"}
# 计算统计信息
cutoff_time = datetime.now() - timedelta(hours=hours)
# 找到时间范围内的数据
valid_indices = [
i for i, ts in enumerate(self.timestamp_history)
if ts >= cutoff_time
]
if not valid_indices:
return {"error": f"过去{hours}小时内没有数据"}
start_idx = valid_indices[0]
cpu_data = list(self.cpu_history)[start_idx:]
memory_data = list(self.memory_history)[start_idx:]
report = {
"time_range": f"过去{hours}小时",
"data_points": len(cpu_data),
"cpu": {
"average": np.mean(cpu_data),
"max": np.max(cpu_data),
"min": np.min(cpu_data),
"std": np.std(cpu_data)
},
"memory": {
"average": np.mean(memory_data),
"max": np.max(memory_data),
"min": np.min(memory_data),
"std": np.std(memory_data)
}
}
return report
def plot_metrics(self, hours: int = 24, save_path: Optional[str] = None):
"""绘制性能指标图表"""
if not self.timestamp_history:
self.logger.error("没有监控数据可绘制")
return
cutoff_time = datetime.now() - timedelta(hours=hours)
# 筛选数据
valid_data = []
for i, ts in enumerate(self.timestamp_history):
if ts >= cutoff_time:
valid_data.append({
"timestamp": ts,
"cpu": self.cpu_history[i],
"memory": self.memory_history[i],
"disk": self.disk_history[i]
})
if not valid_data:
self.logger.error(f"过去{hours}小时内没有数据")
return
timestamps = [d["timestamp"] for d in valid_data]
cpu_data = [d["cpu"] for d in valid_data]
memory_data = [d["memory"] for d in valid_data]
disk_data = [d["disk"] for d in valid_data]
# 创建图表
fig, axes = plt.subplots(3, 1, figsize=(12, 10))
# CPU使用率
axes[0].plot(timestamps, cpu_data, label="CPU使用率", color="blue")
axes[0].axhline(y=self.alert_thresholds["cpu_percent"], color="red", linestyle="--", alpha=0.5)
axes[0].set_ylabel("CPU %")
axes[0].set_title(f"系统性能监控 - 过去{hours}小时")
axes[0].legend()
axes[0].grid(True, alpha=0.3)
# 内存使用率
axes[1].plot(timestamps, memory_data, label="内存使用率", color="green")
axes[1].axhline(y=self.alert_thresholds["memory_percent"], color="red", linestyle="--", alpha=0.5)
axes[1].set_ylabel("内存 %")
axes[1].legend()
axes[1].grid(True, alpha=0.3)
# 磁盘使用率
axes[2].plot(timestamps, disk_data, label="磁盘使用率", color="orange")
axes[2].axhline(y=self.alert_thresholds["disk_percent"], color="red", linestyle="--", alpha=0.5)
axes[2].set_ylabel("磁盘 %")
axes[2].set_xlabel("时间")
axes[2].legend()
axes[2].grid(True, alpha=0.3)
plt.tight_layout()
if save_path:
plt.savefig(save_path, dpi=300, bbox_inches="tight")
self.logger.info(f"性能图表已保存: {save_path}")
else:
plt.show()
# 使用示例
def alert_handler(alert_message: str, metrics: Dict):
"""告警处理函数"""
print(f"🚨 ALERT: {alert_message}")
# 可以在这里实现发送邮件、短信等功能
# 启动监控
monitor = PerformanceMonitor(log_interval=30) # 每30秒记录一次
monitor.add_alert_callback(alert_handler)
monitor.start_monitoring()
# 运行一段时间后生成报告
time.sleep(300) # 等待5分钟
report = monitor.generate_report(hours=1)
print("性能报告:", json.dumps(report, indent=2, ensure_ascii=False))
# 绘制图表
monitor.plot_metrics(hours=1, save_path="performance_report.png")
# 停止监控
monitor.stop_monitoring()
2.2 模型性能优化¶
# model_optimizer.py - 模型性能优化工具
import torch
import torch.nn as nn
import torch.quantization as quantization
from torch.cuda.amp import autocast, GradScaler
import onnx
import onnxruntime as ort
import time
import numpy as np
from typing import Dict, List, Tuple, Optional
import logging
class ModelOptimizer:
def __init__(self):
self.logger = logging.getLogger(__name__)
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
def benchmark_model(self, model, input_shape: Tuple, num_runs: int = 100) -> Dict:
"""基准测试模型性能"""
model.eval()
model.to(self.device)
# 创建随机输入
dummy_input = torch.randn(input_shape).to(self.device)
# 预热
with torch.no_grad():
for _ in range(10):
_ = model(dummy_input)
# 同步GPU
if torch.cuda.is_available():
torch.cuda.synchronize()
# 基准测试
times = []
with torch.no_grad():
for _ in range(num_runs):
start_time = time.time()
_ = model(dummy_input)
if torch.cuda.is_available():
torch.cuda.synchronize()
end_time = time.time()
times.append(end_time - start_time)
# 计算内存使用
model_size = sum(p.numel() * p.element_size() for p in model.parameters()) / 1024**2
results = {
"average_inference_time": np.mean(times),
"min_inference_time": np.min(times),
"max_inference_time": np.max(times),
"std_inference_time": np.std(times),
"throughput_samples_per_second": input_shape[0] / np.mean(times),
"model_size_mb": model_size,
"device": str(self.device)
}
# GPU内存信息
if torch.cuda.is_available():
results["gpu_memory_allocated_mb"] = torch.cuda.memory_allocated() / 1024**2
results["gpu_memory_reserved_mb"] = torch.cuda.memory_reserved() / 1024**2
return results
def quantize_model(self, model, input_shape: Tuple, method: str = "dynamic") -> nn.Module:
"""量化模型以减少内存和提升速度"""
model.eval()
if method == "dynamic":
# 动态量化
quantized_model = quantization.quantize_dynamic(
model, {nn.Linear, nn.Conv2d}, dtype=torch.qint8
)
self.logger.info("应用动态量化")
elif method == "static":
# 静态量化(需要校准数据)
model.qconfig = quantization.get_default_qconfig('fbgemm')
model_fp32_fused = quantization.fuse_modules(model, [['conv', 'bn', 'relu']])
model_fp32_prepared = quantization.prepare(model_fp32_fused)
# 这里需要用实际数据进行校准
dummy_input = torch.randn(input_shape)
model_fp32_prepared(dummy_input)
quantized_model = quantization.convert(model_fp32_prepared)
self.logger.info("应用静态量化")
else:
raise ValueError(f"不支持的量化方法: {method}")
return quantized_model
def convert_to_onnx(self, model, input_shape: Tuple, output_path: str,
optimize: bool = True) -> str:
"""转换模型为ONNX格式"""
model.eval()
dummy_input = torch.randn(input_shape)
# 导出ONNX
torch.onnx.export(
model,
dummy_input,
output_path,
export_params=True,
opset_version=11,
do_constant_folding=True,
input_names=['input'],
output_names=['output'],
dynamic_axes={
'input': {0: 'batch_size'},
'output': {0: 'batch_size'}
}
)
if optimize:
# 优化ONNX模型
self._optimize_onnx_model(output_path)
self.logger.info(f"模型已导出为ONNX: {output_path}")
return output_path
def _optimize_onnx_model(self, model_path: str):
"""优化ONNX模型"""
try:
import onnxoptimizer
# 加载模型
model = onnx.load(model_path)
# 应用优化
optimized_model = onnxoptimizer.optimize(model)
# 保存优化后的模型
onnx.save(optimized_model, model_path)
self.logger.info("ONNX模型优化完成")
except ImportError:
self.logger.warning("onnxoptimizer未安装,跳过ONNX优化")
def benchmark_onnx_model(self, model_path: str, input_shape: Tuple,
num_runs: int = 100) -> Dict:
"""基准测试ONNX模型"""
# 创建推理会话
session = ort.InferenceSession(model_path)
# 创建随机输入
dummy_input = np.random.randn(*input_shape).astype(np.float32)
# 预热
for _ in range(10):
_ = session.run(None, {'input': dummy_input})
# 基准测试
times = []
for _ in range(num_runs):
start_time = time.time()
_ = session.run(None, {'input': dummy_input})
end_time = time.time()
times.append(end_time - start_time)
results = {
"average_inference_time": np.mean(times),
"min_inference_time": np.min(times),
"max_inference_time": np.max(times),
"std_inference_time": np.std(times),
"throughput_samples_per_second": input_shape[0] / np.mean(times),
"providers": session.get_providers()
}
return results
def enable_mixed_precision(self, model, optimizer) -> Tuple[nn.Module, GradScaler]:
"""启用混合精度训练"""
if not torch.cuda.is_available():
self.logger.warning("CUDA不可用,无法启用混合精度")
return model, None
scaler = GradScaler()
self.logger.info("混合精度训练已启用")
return model, scaler
def optimize_dataloader(self, dataloader, num_workers: int = None,
pin_memory: bool = None) -> Dict:
"""优化数据加载器"""
if num_workers is None:
num_workers = min(4, torch.get_num_threads())
if pin_memory is None:
pin_memory = torch.cuda.is_available()
optimization_suggestions = {
"num_workers": num_workers,
"pin_memory": pin_memory,
"persistent_workers": True if num_workers > 0 else False,
"prefetch_factor": 2 if num_workers > 0 else 2
}
self.logger.info(f"数据加载器优化建议: {optimization_suggestions}")
return optimization_suggestions
def profile_model(self, model, input_shape: Tuple, output_dir: str = "./profiling"):
"""分析模型性能瓶颈"""
import os
os.makedirs(output_dir, exist_ok=True)
model.eval()
model.to(self.device)
dummy_input = torch.randn(input_shape).to(self.device)
# PyTorch Profiler
with torch.profiler.profile(
activities=[torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA],
schedule=torch.profiler.schedule(wait=1, warmup=1, active=3, repeat=2),
on_trace_ready=torch.profiler.tensorboard_trace_handler(output_dir),
record_shapes=True,
profile_memory=True,
with_stack=True
) as prof:
for _ in range(10):
with torch.no_grad():
_ = model(dummy_input)
prof.step()
# 保存性能分析结果
prof.export_chrome_trace(os.path.join(output_dir, "trace.json"))
# 打印关键统计信息
print("Top 10 CPU operations:")
print(prof.key_averages().table(sort_by="cpu_time_total", row_limit=10))
if torch.cuda.is_available():
print("\nTop 10 CUDA operations:")
print(prof.key_averages().table(sort_by="cuda_time_total", row_limit=10))
self.logger.info(f"性能分析结果已保存到: {output_dir}")
# 使用示例
class SimpleModel(nn.Module):
def __init__(self, input_size=784, hidden_size=256, num_classes=10):
super().__init__()
self.fc1 = nn.Linear(input_size, hidden_size)
self.relu = nn.ReLU()
self.fc2 = nn.Linear(hidden_size, hidden_size)
self.fc3 = nn.Linear(hidden_size, num_classes)
def forward(self, x):
x = x.view(x.size(0), -1)
x = self.relu(self.fc1(x))
x = self.relu(self.fc2(x))
x = self.fc3(x)
return x
# 优化示例
optimizer = ModelOptimizer()
model = SimpleModel()
input_shape = (32, 1, 28, 28) # batch_size, channels, height, width
# 1. 基准测试原始模型
print("原始模型性能:")
original_results = optimizer.benchmark_model(model, input_shape)
for key, value in original_results.items():
print(f" {key}: {value}")
# 2. 量化模型
quantized_model = optimizer.quantize_model(model, input_shape, method="dynamic")
print("\n量化模型性能:")
quantized_results = optimizer.benchmark_model(quantized_model, input_shape)
for key, value in quantized_results.items():
print(f" {key}: {value}")
# 3. 转换为ONNX并测试
onnx_path = "model.onnx"
optimizer.convert_to_onnx(model, input_shape, onnx_path)
print("\nONNX模型性能:")
onnx_results = optimizer.benchmark_onnx_model(onnx_path, input_shape)
for key, value in onnx_results.items():
print(f" {key}: {value}")
# 4. 性能分析
optimizer.profile_model(model, input_shape)
# 性能对比
print("\n性能对比:")
print(f"原始模型推理时间: {original_results['average_inference_time']:.4f}s")
print(f"量化模型推理时间: {quantized_results['average_inference_time']:.4f}s")
print(f"ONNX模型推理时间: {onnx_results['average_inference_time']:.4f}s")
speedup_quantized = original_results['average_inference_time'] / quantized_results['average_inference_time']
speedup_onnx = original_results['average_inference_time'] / onnx_results['average_inference_time']
print(f"量化加速比: {speedup_quantized:.2f}x")
print(f"ONNX加速比: {speedup_onnx:.2f}x")
📊 第三部分:日志分析与调试¶
3.1 智能日志分析系统¶
# log_analyzer.py - 智能日志分析工具
import re
import json
import pandas as pd
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Tuple
from collections import Counter, defaultdict
import matplotlib.pyplot as plt
import seaborn as sns
import logging
class LogAnalyzer:
def __init__(self):
self.log_patterns = {
"error": r"ERROR|error|Error|FAILED|failed|Failed",
"warning": r"WARNING|warning|Warning|WARN|warn",
"info": r"INFO|info|Info",
"debug": r"DEBUG|debug|Debug",
"timestamp": r"\d{4}-\d{2}-\d{2}[\s\T]\d{2}:\d{2}:\d{2}",
"ip_address": r"\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b",
"memory_usage": r"memory.*?(\d+\.?\d*)\s*(MB|GB|KB)",
"execution_time": r"(?:took|elapsed|duration).*?(\d+\.?\d*)\s*(s|ms|seconds|milliseconds)",
"api_call": r"API|api|endpoint|request|response"
}
self.error_categories = {
"memory": ["memory", "oom", "out of memory", "allocation"],
"network": ["connection", "timeout", "network", "socket"],
"authentication": ["auth", "login", "token", "permission", "unauthorized"],
"validation": ["validation", "invalid", "format", "schema"],
"performance": ["slow", "timeout", "performance", "bottleneck"],
"dependency": ["import", "module", "package", "dependency", "not found"]
}
def parse_log_file(self, file_path: str, encoding: str = "utf-8") -> List[Dict]:
"""解析日志文件"""
log_entries = []
try:
with open(file_path, 'r', encoding=encoding) as file:
for line_num, line in enumerate(file, 1):
entry = self._parse_log_line(line.strip(), line_num)
if entry:
log_entries.append(entry)
except Exception as e:
logging.error(f"读取日志文件失败: {e}")
raise
return log_entries
def _parse_log_line(self, line: str, line_num: int) -> Optional[Dict]:
"""解析单行日志"""
if not line:
return None
entry = {
"line_number": line_num,
"raw_line": line,
"timestamp": None,
"level": "UNKNOWN",
"message": line,
"patterns_matched": []
}
# 提取时间戳
timestamp_match = re.search(self.log_patterns["timestamp"], line)
if timestamp_match:
try:
entry["timestamp"] = datetime.fromisoformat(
timestamp_match.group().replace(" ", "T")
)
except ValueError:
pass
# 确定日志级别
for level in ["error", "warning", "info", "debug"]:
if re.search(self.log_patterns[level], line, re.IGNORECASE):
entry["level"] = level.upper()
entry["patterns_matched"].append(level)
break
# 匹配其他模式
for pattern_name, pattern in self.log_patterns.items():
if pattern_name not in ["error", "warning", "info", "debug", "timestamp"]:
if re.search(pattern, line, re.IGNORECASE):
entry["patterns_matched"].append(pattern_name)
# 提取数值信息
entry.update(self._extract_metrics(line))
return entry
def _extract_metrics(self, line: str) -> Dict:
"""从日志行中提取指标"""
metrics = {}
# 内存使用
memory_match = re.search(self.log_patterns["memory_usage"], line, re.IGNORECASE)
if memory_match:
value = float(memory_match.group(1))
unit = memory_match.group(2).upper()
# 转换为MB
if unit == "GB":
value *= 1024
elif unit == "KB":
value /= 1024
metrics["memory_usage_mb"] = value
# 执行时间
time_match = re.search(self.log_patterns["execution_time"], line, re.IGNORECASE)
if time_match:
value = float(time_match.group(1))
unit = time_match.group(2).lower()
# 转换为秒
if "ms" in unit or "millisecond" in unit:
value /= 1000
metrics["execution_time_seconds"] = value
return metrics
def analyze_logs(self, log_entries: List[Dict]) -> Dict:
"""分析日志数据"""
if not log_entries:
return {"error": "没有日志数据"}
analysis = {
"summary": self._generate_summary(log_entries),
"error_analysis": self._analyze_errors(log_entries),
"performance_analysis": self._analyze_performance(log_entries),
"timeline_analysis": self._analyze_timeline(log_entries),
"pattern_analysis": self._analyze_patterns(log_entries)
}
return analysis
def _generate_summary(self, log_entries: List[Dict]) -> Dict:
"""生成日志摘要"""
total_entries = len(log_entries)
level_counts = Counter(entry["level"] for entry in log_entries)
# 时间范围
timestamps = [entry["timestamp"] for entry in log_entries if entry["timestamp"]]
time_range = None
if timestamps:
time_range = {
"start": min(timestamps).isoformat(),
"end": max(timestamps).isoformat(),
"duration_hours": (max(timestamps) - min(timestamps)).total_seconds() / 3600
}
return {
"total_entries": total_entries,
"level_distribution": dict(level_counts),
"time_range": time_range,
"entries_with_timestamp": len(timestamps),
"error_rate": level_counts.get("ERROR", 0) / total_entries * 100
}
def _analyze_errors(self, log_entries: List[Dict]) -> Dict:
"""分析错误信息"""
error_entries = [entry for entry in log_entries if entry["level"] == "ERROR"]
if not error_entries:
return {"message": "没有错误日志"}
# 错误分类
categorized_errors = defaultdict(list)
for entry in error_entries:
message = entry["message"].lower()
category = "other"
for cat_name, keywords in self.error_categories.items():
if any(keyword in message for keyword in keywords):
category = cat_name
break
categorized_errors[category].append(entry)
# 错误频率
error_messages = [entry["message"] for entry in error_entries]
message_counts = Counter(error_messages)
# 错误时间分布
error_times = [entry["timestamp"] for entry in error_entries if entry["timestamp"]]
time_distribution = {}
if error_times:
time_distribution = self._get_hourly_distribution(error_times)
return {
"total_errors": len(error_entries),
"error_categories": {cat: len(errors) for cat, errors in categorized_errors.items()},
"top_error_messages": dict(message_counts.most_common(10)),
"time_distribution": time_distribution,
"recent_errors": [
{"timestamp": entry.get("timestamp"), "message": entry["message"]}
for entry in error_entries[-5:]
]
}
def _analyze_performance(self, log_entries: List[Dict]) -> Dict:
"""分析性能指标"""
# 提取性能相关数据
memory_data = [entry["memory_usage_mb"] for entry in log_entries
if "memory_usage_mb" in entry]
execution_times = [entry["execution_time_seconds"] for entry in log_entries
if "execution_time_seconds" in entry]
performance_analysis = {}
if memory_data:
performance_analysis["memory"] = {
"average_mb": sum(memory_data) / len(memory_data),
"max_mb": max(memory_data),
"min_mb": min(memory_data),
"samples": len(memory_data)
}
if execution_times:
performance_analysis["execution_time"] = {
"average_seconds": sum(execution_times) / len(execution_times),
"max_seconds": max(execution_times),
"min_seconds": min(execution_times),
"samples": len(execution_times)
}
# 识别慢查询/操作
slow_threshold = sum(execution_times) / len(execution_times) * 2
slow_operations = [entry for entry in log_entries
if entry.get("execution_time_seconds", 0) > slow_threshold]
performance_analysis["slow_operations"] = {
"threshold_seconds": slow_threshold,
"count": len(slow_operations),
"examples": [entry["message"] for entry in slow_operations[:5]]
}
return performance_analysis
def _analyze_timeline(self, log_entries: List[Dict]) -> Dict:
"""分析时间线"""
timestamped_entries = [entry for entry in log_entries if entry["timestamp"]]
if not timestamped_entries:
return {"message": "没有时间戳信息"}
# 按小时统计
hourly_stats = defaultdict(lambda: {"total": 0, "errors": 0, "warnings": 0})
for entry in timestamped_entries:
hour_key = entry["timestamp"].strftime("%Y-%m-%d %H:00")
hourly_stats[hour_key]["total"] += 1
if entry["level"] == "ERROR":
hourly_stats[hour_key]["errors"] += 1
elif entry["level"] == "WARNING":
hourly_stats[hour_key]["warnings"] += 1
# 识别异常时间段
error_rates = []
for hour_data in hourly_stats.values():
if hour_data["total"] > 0:
error_rate = hour_data["errors"] / hour_data["total"]
error_rates.append(error_rate)
avg_error_rate = sum(error_rates) / len(error_rates) if error_rates else 0
anomalous_hours = []
for hour, data in hourly_stats.items():
if data["total"] > 0:
error_rate = data["errors"] / data["total"]
if error_rate > avg_error_rate * 2: # 错误率是平均值的2倍以上
anomalous_hours.append({
"hour": hour,
"error_rate": error_rate,
"total_logs": data["total"],
"errors": data["errors"]
})
return {
"hourly_statistics": dict(hourly_stats),
"average_error_rate": avg_error_rate,
"anomalous_periods": sorted(anomalous_hours,
key=lambda x: x["error_rate"], reverse=True)
}
def _analyze_patterns(self, log_entries: List[Dict]) -> Dict:
"""分析日志模式"""
pattern_counts = defaultdict(int)
for entry in log_entries:
for pattern in entry["patterns_matched"]:
pattern_counts[pattern] += 1
# API调用分析
api_entries = [entry for entry in log_entries
if "api_call" in entry["patterns_matched"]]
return {
"pattern_frequency": dict(pattern_counts),
"api_activity": {
"total_api_logs": len(api_entries),
"percentage": len(api_entries) / len(log_entries) * 100 if log_entries else 0
}
}
def _get_hourly_distribution(self, timestamps: List[datetime]) -> Dict:
"""获取按小时的分布"""
hour_counts = defaultdict(int)
for ts in timestamps:
hour_counts[ts.hour] += 1
return dict(hour_counts)
def generate_report(self, analysis: Dict, output_path: str = None) -> str:
"""生成分析报告"""
report_lines = []
report_lines.append("# 日志分析报告")
report_lines.append(f"生成时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
report_lines.append("")
# 摘要部分
summary = analysis.get("summary", {})
report_lines.append("## 📊 日志摘要")
report_lines.append(f"- 总日志条数: {summary.get('total_entries', 0)}")
report_lines.append(f"- 错误率: {summary.get('error_rate', 0):.2f}%")
level_dist = summary.get("level_distribution", {})
for level, count in level_dist.items():
report_lines.append(f"- {level}: {count}")
time_range = summary.get("time_range")
if time_range:
report_lines.append(f"- 时间范围: {time_range['start']} 至 {time_range['end']}")
report_lines.append(f"- 持续时间: {time_range['duration_hours']:.2f} 小时")
report_lines.append("")
# 错误分析
error_analysis = analysis.get("error_analysis", {})
if "total_errors" in error_analysis:
report_lines.append("## 🚨 错误分析")
report_lines.append(f"- 总错误数: {error_analysis['total_errors']}")
error_categories = error_analysis.get("error_categories", {})
report_lines.append("- 错误分类:")
for category, count in error_categories.items():
report_lines.append(f" - {category}: {count}")
top_errors = error_analysis.get("top_error_messages", {})
if top_errors:
report_lines.append("- 频繁错误信息:")
for message, count in list(top_errors.items())[:5]:
report_lines.append(f" - {message[:100]}... ({count}次)")
report_lines.append("")
# 性能分析
perf_analysis = analysis.get("performance_analysis", {})
if perf_analysis:
report_lines.append("## ⚡ 性能分析")
memory_stats = perf_analysis.get("memory")
if memory_stats:
report_lines.append("- 内存使用:")
report_lines.append(f" - 平均: {memory_stats['average_mb']:.2f} MB")
report_lines.append(f" - 最大: {memory_stats['max_mb']:.2f} MB")
report_lines.append(f" - 样本数: {memory_stats['samples']}")
exec_stats = perf_analysis.get("execution_time")
if exec_stats:
report_lines.append("- 执行时间:")
report_lines.append(f" - 平均: {exec_stats['average_seconds']:.3f} 秒")
report_lines.append(f" - 最大: {exec_stats['max_seconds']:.3f} 秒")
report_lines.append(f" - 样本数: {exec_stats['samples']}")
slow_ops = perf_analysis.get("slow_operations")
if slow_ops:
report_lines.append(f"- 慢操作: {slow_ops['count']} 个 (阈值: {slow_ops['threshold_seconds']:.3f}s)")
report_lines.append("")
# 异常时间段
timeline = analysis.get("timeline_analysis", {})
anomalous_periods = timeline.get("anomalous_periods", [])
if anomalous_periods:
report_lines.append("## ⏰ 异常时间段")
for period in anomalous_periods[:5]:
report_lines.append(f"- {period['hour']}: 错误率 {period['error_rate']:.2f} "
f"({period['errors']}/{period['total_logs']})")
report_lines.append("")
# 建议
report_lines.append("## 💡 优化建议")
suggestions = self._generate_suggestions(analysis)
for suggestion in suggestions:
report_lines.append(f"- {suggestion}")
report_content = "\n".join(report_lines)
if output_path:
with open(output_path, 'w', encoding='utf-8') as f:
f.write(report_content)
logging.info(f"报告已保存到: {output_path}")
return report_content
def _generate_suggestions(self, analysis: Dict) -> List[str]:
"""生成优化建议"""
suggestions = []
# 基于错误率的建议
summary = analysis.get("summary", {})
error_rate = summary.get("error_rate", 0)
if error_rate > 10:
suggestions.append("错误率过高(>10%),建议检查系统稳定性")
elif error_rate > 5:
suggestions.append("错误率较高(>5%),建议关注错误原因")
# 基于错误分类的建议
error_analysis = analysis.get("error_analysis", {})
error_categories = error_analysis.get("error_categories", {})
if error_categories.get("memory", 0) > 0:
suggestions.append("检测到内存相关错误,建议优化内存使用或增加内存")
if error_categories.get("network", 0) > 0:
suggestions.append("检测到网络相关错误,建议检查网络连接和超时设置")
if error_categories.get("performance", 0) > 0:
suggestions.append("检测到性能问题,建议进行性能调优")
# 基于性能分析的建议
perf_analysis = analysis.get("performance_analysis", {})
slow_ops = perf_analysis.get("slow_operations")
if slow_ops and slow_ops.get("count", 0) > 0:
suggestions.append(f"发现{slow_ops['count']}个慢操作,建议优化算法或增加缓存")
# 基于时间线分析的建议
timeline = analysis.get("timeline_analysis", {})
anomalous_periods = timeline.get("anomalous_periods", [])
if len(anomalous_periods) > 3:
suggestions.append("多个时间段出现异常,建议建立监控告警")
if not suggestions:
suggestions.append("日志状态良好,建议继续保持监控")
return suggestions
def visualize_analysis(self, analysis: Dict, output_dir: str = "./log_analysis"):
"""可视化分析结果"""
import os
os.makedirs(output_dir, exist_ok=True)
# 设置中文字体
plt.rcParams['font.sans-serif'] = ['SimHei', 'DejaVu Sans']
plt.rcParams['axes.unicode_minus'] = False
# 1. 日志级别分布饼图
summary = analysis.get("summary", {})
level_dist = summary.get("level_distribution", {})
if level_dist:
plt.figure(figsize=(10, 6))
# 饼图
plt.subplot(1, 2, 1)
plt.pie(level_dist.values(), labels=level_dist.keys(), autopct='%1.1f%%')
plt.title("日志级别分布")
# 柱状图
plt.subplot(1, 2, 2)
plt.bar(level_dist.keys(), level_dist.values())
plt.title("日志级别统计")
plt.ylabel("数量")
plt.tight_layout()
plt.savefig(os.path.join(output_dir, "log_level_distribution.png"), dpi=300)
plt.close()
# 2. 错误分类分析
error_analysis = analysis.get("error_analysis", {})
error_categories = error_analysis.get("error_categories", {})
if error_categories:
plt.figure(figsize=(12, 6))
categories = list(error_categories.keys())
counts = list(error_categories.values())
plt.bar(categories, counts)
plt.title("错误分类统计")
plt.xlabel("错误类别")
plt.ylabel("数量")
plt.xticks(rotation=45)
plt.tight_layout()
plt.savefig(os.path.join(output_dir, "error_categories.png"), dpi=300)
plt.close()
# 3. 时间线分析
timeline = analysis.get("timeline_analysis", {})
hourly_stats = timeline.get("hourly_statistics", {})
if hourly_stats:
# 准备数据
hours = sorted(hourly_stats.keys())
total_counts = [hourly_stats[hour]["total"] for hour in hours]
error_counts = [hourly_stats[hour]["errors"] for hour in hours]
plt.figure(figsize=(15, 8))
# 总日志数时间线
plt.subplot(2, 1, 1)
plt.plot(range(len(hours)), total_counts, marker='o', linewidth=2)
plt.title("日志数量时间线")
plt.ylabel("日志数量")
plt.grid(True, alpha=0.3)
# 错误数时间线
plt.subplot(2, 1, 2)
plt.plot(range(len(hours)), error_counts, marker='o', color='red', linewidth=2)
plt.title("错误数量时间线")
plt.xlabel("时间")
plt.ylabel("错误数量")
plt.grid(True, alpha=0.3)
# 设置x轴标签(每隔几个小时显示一次)
step = max(1, len(hours) // 10)
plt.xticks(range(0, len(hours), step),
[hours[i].split()[1] for i in range(0, len(hours), step)],
rotation=45)
plt.tight_layout()
plt.savefig(os.path.join(output_dir, "timeline_analysis.png"), dpi=300)
plt.close()
logging.info(f"可视化图表已保存到: {output_dir}")
# 使用示例
def main():
analyzer = LogAnalyzer()
# 解析日志文件
log_entries = analyzer.parse_log_file("application.log")
print(f"解析了 {len(log_entries)} 条日志")
# 分析日志
analysis = analyzer.analyze_logs(log_entries)
# 生成报告
report = analyzer.generate_report(analysis, "log_analysis_report.md")
print("分析报告:")
print(report[:1000]) # 显示报告前1000字符
# 生成可视化图表
analyzer.visualize_analysis(analysis)
# 打印关键信息
print("\n关键发现:")
summary = analysis.get("summary", {})
print(f"- 总日志数: {summary.get('total_entries', 0)}")
print(f"- 错误率: {summary.get('error_rate', 0):.2f}%")
error_analysis = analysis.get("error_analysis", {})
if error_analysis.get("total_errors", 0) > 0:
print(f"- 错误总数: {error_analysis['total_errors']}")
top_errors = error_analysis.get("top_error_messages", {})
if top_errors:
print("- 最频繁的错误:")
for message, count in list(top_errors.items())[:3]:
print(f" {message[:50]}... ({count}次)")
if __name__ == "__main__":
main()
继续下一部分内容...