367 lines
13 KiB
Python
367 lines
13 KiB
Python
"""
|
||
模型比较器模块 - 用于比较不同猫叫声意图分类模型的性能
|
||
|
||
该模块提供了比较DAG-HMM、深度学习、SVM和随机森林等不同分类方法的功能，
|
||
帮助用户选择最适合其数据集的模型。
|
||
"""
|
||
|
||
import os
|
||
import numpy as np
|
||
import json
|
||
import matplotlib.pyplot as plt
|
||
from typing import Dict, Any, List, Optional, Tuple
|
||
import time
|
||
from datetime import datetime
|
||
|
||
from src.cat_intent_classifier_v2 import CatIntentClassifier
|
||
from src.dag_hmm_classifier import DAGHMMClassifier
|
||
|
||
class ModelComparator:
|
||
"""模型比较器类，用于比较不同猫叫声意图分类模型的性能"""
|
||
|
||
def __init__(self, results_dir: str = "./comparison_results"):
|
||
"""
|
||
初始化模型比较器
|
||
|
||
参数:
|
||
results_dir: 结果保存目录
|
||
"""
|
||
self.results_dir = results_dir
|
||
os.makedirs(results_dir, exist_ok=True)
|
||
|
||
# 支持的模型类型
|
||
self.model_types = {
|
||
"dag_hmm": {
|
||
"name": "DAG-HMM",
|
||
"class": DAGHMMClassifier,
|
||
"params": {"n_states": 5, "n_mix": 3}
|
||
},
|
||
"dl": {
|
||
"name": "深度学习",
|
||
"class": CatIntentClassifier,
|
||
"params": {}
|
||
}
|
||
}
|
||
|
||
def compare_models(self, features: List[np.ndarray], labels: List[str],
|
||
model_types: List[str] = None, test_size: float = 0.2,
|
||
cat_name: Optional[str] = None) -> Dict[str, Any]:
|
||
"""
|
||
比较不同模型的性能
|
||
|
||
参数:
|
||
features: 特征序列列表
|
||
labels: 标签列表
|
||
model_types: 要比较的模型类型列表，默认为所有支持的模型
|
||
test_size: 测试集比例
|
||
cat_name: 猫咪名称，默认为None（通用模型）
|
||
|
||
返回:
|
||
results: 比较结果
|
||
"""
|
||
if model_types is None:
|
||
model_types = list(self.model_types.keys())
|
||
|
||
# 验证模型类型
|
||
for model_type in model_types:
|
||
if model_type not in self.model_types:
|
||
raise ValueError(f"不支持的模型类型: {model_type}")
|
||
|
||
# 划分训练集和测试集
|
||
from sklearn.model_selection import train_test_split
|
||
_, test_features, _, test_labels = train_test_split(
|
||
features, labels, test_size=test_size, random_state=42, stratify=labels
|
||
)
|
||
train_features, train_labels = features, labels
|
||
print(f"训练集大小: {len(train_features)}, 测试集大小: {len(test_features)}")
|
||
|
||
# 比较结果
|
||
results = {
|
||
"models": {},
|
||
"best_model": None,
|
||
"comparison_time": datetime.now().isoformat(),
|
||
"dataset_info": {
|
||
"total_samples": len(features),
|
||
"train_samples": len(train_features),
|
||
"test_samples": len(test_features),
|
||
"classes": sorted(list(set(labels))),
|
||
"class_distribution": {label: labels.count(label) for label in set(labels)}
|
||
}
|
||
}
|
||
|
||
# 训练和评估每个模型
|
||
for model_type in model_types:
|
||
model_info = self.model_types[model_type]
|
||
model_name = model_info["name"]
|
||
model_class = model_info["class"]
|
||
model_params = model_info["params"]
|
||
|
||
print(f"\n开始训练和评估 {model_name} 模型...")
|
||
|
||
try:
|
||
# 创建模型
|
||
model = model_class(**model_params)
|
||
|
||
# 记录训练开始时间
|
||
train_start_time = time.time()
|
||
|
||
# 训练模型
|
||
train_metrics = model.train(train_features, train_labels)
|
||
|
||
# 记录训练结束时间
|
||
train_end_time = time.time()
|
||
train_time = train_end_time - train_start_time
|
||
|
||
# 记录评估开始时间
|
||
eval_start_time = time.time()
|
||
|
||
# 评估模型
|
||
eval_metrics = model.evaluate(test_features, test_labels)
|
||
|
||
# 记录评估结束时间
|
||
eval_end_time = time.time()
|
||
eval_time = eval_end_time - eval_start_time
|
||
|
||
# 保存模型
|
||
model_dir = os.path.join(self.results_dir, "models")
|
||
os.makedirs(model_dir, exist_ok=True)
|
||
model_paths = model.save_model(model_dir, cat_name)
|
||
|
||
# 记录结果
|
||
results["models"][model_type] = {
|
||
"name": model_name,
|
||
"train_metrics": train_metrics,
|
||
"eval_metrics": eval_metrics,
|
||
"train_time": train_time,
|
||
"eval_time": eval_time,
|
||
"model_paths": model_paths
|
||
}
|
||
|
||
print(f"{model_name} 模型训练完成，评估指标: {eval_metrics}")
|
||
|
||
except Exception as e:
|
||
print(f"{model_name} 模型训练或评估失败: {e}")
|
||
results["models"][model_type] = {
|
||
"name": model_name,
|
||
"error": str(e)
|
||
}
|
||
|
||
# 确定最佳模型
|
||
best_model = None
|
||
best_accuracy = -1
|
||
|
||
for model_type, model_result in results["models"].items():
|
||
if "eval_metrics" in model_result and "accuracy" in model_result["eval_metrics"]:
|
||
accuracy = model_result["eval_metrics"]["accuracy"]
|
||
if accuracy > best_accuracy:
|
||
best_accuracy = accuracy
|
||
best_model = model_type
|
||
|
||
results["best_model"] = best_model
|
||
|
||
# 保存比较结果
|
||
result_path = os.path.join(
|
||
self.results_dir,
|
||
f"comparison_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
||
)
|
||
|
||
with open(result_path, 'w') as f:
|
||
# 将numpy值转换为Python原生类型
|
||
def convert_numpy(obj):
|
||
if isinstance(obj, np.integer):
|
||
return int(obj)
|
||
elif isinstance(obj, np.floating):
|
||
return float(obj)
|
||
elif isinstance(obj, np.ndarray):
|
||
return obj.tolist()
|
||
return obj
|
||
|
||
json_results = {k: convert_numpy(v) for k, v in results.items()}
|
||
json.dump(json_results, f, indent=2)
|
||
|
||
print(f"\n比较结果已保存到: {result_path}")
|
||
|
||
# 可视化比较结果
|
||
self.visualize_comparison(results)
|
||
|
||
return results
|
||
|
||
def visualize_comparison(self, results: Dict[str, Any]) -> str:
|
||
"""
|
||
可视化比较结果
|
||
|
||
参数:
|
||
results: 比较结果
|
||
|
||
返回:
|
||
plot_path: 图表保存路径
|
||
"""
|
||
# 准备数据
|
||
model_names = []
|
||
accuracies = []
|
||
precisions = []
|
||
recalls = []
|
||
f1_scores = []
|
||
train_times = []
|
||
|
||
for model_type, model_result in results["models"].items():
|
||
if "eval_metrics" in model_result:
|
||
model_names.append(model_result["name"])
|
||
|
||
metrics = model_result["eval_metrics"]
|
||
accuracies.append(metrics.get("accuracy", 0))
|
||
precisions.append(metrics.get("precision", 0))
|
||
recalls.append(metrics.get("recall", 0))
|
||
f1_scores.append(metrics.get("f1", 0))
|
||
|
||
train_times.append(model_result.get("train_time", 0))
|
||
|
||
# 创建图表
|
||
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 7))
|
||
|
||
# 性能指标图
|
||
x = np.arange(len(model_names))
|
||
width = 0.2
|
||
|
||
ax1.bar(x - width*1.5, accuracies, width, label='准确率')
|
||
ax1.bar(x - width/2, precisions, width, label='精确率')
|
||
ax1.bar(x + width/2, recalls, width, label='召回率')
|
||
ax1.bar(x + width*1.5, f1_scores, width, label='F1分数')
|
||
|
||
ax1.set_ylabel('得分')
|
||
ax1.set_title('模型性能比较')
|
||
ax1.set_xticks(x)
|
||
ax1.set_xticklabels(model_names)
|
||
ax1.legend()
|
||
ax1.set_ylim(0, 1.1)
|
||
|
||
# 为每个柱子添加数值标签
|
||
for i, v in enumerate(accuracies):
|
||
ax1.text(i - width*1.5, v + 0.02, f'{v:.2f}', ha='center', va='bottom', fontsize=8)
|
||
for i, v in enumerate(precisions):
|
||
ax1.text(i - width/2, v + 0.02, f'{v:.2f}', ha='center', va='bottom', fontsize=8)
|
||
for i, v in enumerate(recalls):
|
||
ax1.text(i + width/2, v + 0.02, f'{v:.2f}', ha='center', va='bottom', fontsize=8)
|
||
for i, v in enumerate(f1_scores):
|
||
ax1.text(i + width*1.5, v + 0.02, f'{v:.2f}', ha='center', va='bottom', fontsize=8)
|
||
|
||
# 训练时间图
|
||
ax2.bar(model_names, train_times, color='skyblue')
|
||
ax2.set_ylabel('时间 (秒)')
|
||
ax2.set_title('模型训练时间比较')
|
||
|
||
# 为每个柱子添加数值标签
|
||
for i, v in enumerate(train_times):
|
||
ax2.text(i, v + 0.1, f'{v:.1f}s', ha='center', va='bottom')
|
||
|
||
# 标记最佳模型
|
||
best_model = results.get("best_model")
|
||
if best_model and best_model in results["models"]:
|
||
best_model_name = results["models"][best_model]["name"]
|
||
best_index = model_names.index(best_model_name)
|
||
|
||
ax1.get_xticklabels()[best_index].set_color('red')
|
||
ax1.get_xticklabels()[best_index].set_weight('bold')
|
||
|
||
ax2.get_xticklabels()[best_index].set_color('red')
|
||
ax2.get_xticklabels()[best_index].set_weight('bold')
|
||
|
||
# 添加总标题
|
||
plt.suptitle('猫叫声意图分类模型比较', fontsize=16)
|
||
|
||
# 保存图表
|
||
plot_path = os.path.join(
|
||
self.results_dir,
|
||
f"comparison_plot_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
|
||
)
|
||
plt.tight_layout()
|
||
plt.subplots_adjust(top=0.9)
|
||
plt.savefig(plot_path, dpi=300)
|
||
plt.close()
|
||
|
||
print(f"比较图表已保存到: {plot_path}")
|
||
|
||
return plot_path
|
||
|
||
def load_best_model(self, comparison_result_path: str, cat_name: Optional[str] = None) -> Any:
|
||
"""
|
||
加载比较结果中的最佳模型
|
||
|
||
参数:
|
||
comparison_result_path: 比较结果文件路径
|
||
cat_name: 猫咪名称，默认为None（通用模型）
|
||
|
||
返回:
|
||
model: 加载的模型
|
||
"""
|
||
# 加载比较结果
|
||
with open(comparison_result_path, 'r') as f:
|
||
results = json.load(f)
|
||
|
||
# 获取最佳模型类型
|
||
best_model_type = results.get("best_model")
|
||
if not best_model_type:
|
||
raise ValueError("比较结果中没有最佳模型")
|
||
|
||
# 获取最佳模型信息
|
||
best_model_info = results["models"].get(best_model_type)
|
||
if not best_model_info or "model_paths" not in best_model_info:
|
||
raise ValueError(f"无法获取最佳模型 {best_model_type} 的路径信息")
|
||
|
||
# 获取模型类
|
||
model_class = self.model_types[best_model_type]["class"]
|
||
model_params = self.model_types[best_model_type]["params"]
|
||
|
||
# 创建模型
|
||
model = model_class(**model_params)
|
||
|
||
# 确定模型目录
|
||
model_dir = os.path.dirname(best_model_info["model_paths"]["model"])
|
||
|
||
# 加载模型
|
||
model.load_model(model_dir, cat_name)
|
||
|
||
return model
|
||
|
||
|
||
# 示例用法
|
||
if __name__ == "__main__":
|
||
# 创建一些模拟数据
|
||
np.random.seed(42)
|
||
n_samples = 50
|
||
n_features = 1024
|
||
n_timesteps = 10
|
||
|
||
# 生成特征序列
|
||
features = []
|
||
labels = []
|
||
|
||
for i in range(n_samples):
|
||
# 生成一个随机特征序列
|
||
feature = np.random.randn(n_timesteps, n_features)
|
||
features.append(feature)
|
||
|
||
# 生成标签
|
||
if i < n_samples / 3:
|
||
labels.append("快乐")
|
||
elif i < 2 * n_samples / 3:
|
||
labels.append("愤怒")
|
||
else:
|
||
labels.append("饥饿")
|
||
|
||
# 创建比较器
|
||
comparator = ModelComparator()
|
||
|
||
# 比较模型
|
||
results = comparator.compare_models(features, labels)
|
||
|
||
# 加载最佳模型
|
||
best_model = comparator.load_best_model(
|
||
os.path.join(comparator.results_dir,
|
||
f"comparison_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json")
|
||
)
|
||
|
||
# 使用最佳模型进行预测
|
||
prediction = best_model.predict(features[0])
|
||
print(f"最佳模型预测结果: {prediction}")
|