跳转至

RD-Agent 教程 - 第四章:Kaggle 竞赛自动化案例

🏆 案例概述

本案例将以 "House Prices: Advanced Regression Techniques" 竞赛为例,展示如何使用 RD-Agent 构建端到端的 Kaggle 竞赛自动化解决方案。从数据分析到模型优化,再到最终提交,实现竞赛全流程自动化。

🎯 项目目标

竞赛表现目标: - 🥇 进入排行榜前20% - 📊 RMSE分数 < 0.12 - 🔄 模型鲁棒性评分 > 85% - ⚡ 自动化程度 > 90%

技术实现目标: - ✅ 掌握RD-Agent数据科学工作流 - ✅ 学会自动化特征工程和选择 - ✅ 理解模型集成和超参数优化 - ✅ 构建可复用的竞赛框架

📋 实施计划

阶段 任务 预期时间 关键指标
第一阶段 竞赛理解和数据探索 30分钟 数据理解完整度 > 90%
第二阶段 自动化特征工程 60分钟 有效特征数量 > 100
第三阶段 模型训练和优化 90分钟 验证RMSE < 0.13
第四阶段 模型集成和调优 60分钟 集成提升 > 5%
第五阶段 提交和排名分析 30分钟 公榜排名确认

🏠 竞赛分析:房价预测

1. 竞赛理解

# 竞赛基本信息
competition_analysis = {
    "competition_name": "house-prices-advanced-regression-techniques",
    "problem_type": "regression",
    "target_variable": "SalePrice",
    "evaluation_metric": "RMSE",

    "data_characteristics": {
        "train_samples": 1460,
        "test_samples": 1459,
        "features": 79,
        "feature_types": {
            "numerical": 36,
            "categorical": 43
        },
        "missing_values": "moderate",
        "target_distribution": "right_skewed"
    },

    "key_challenges": [
        "高维稀疏特征",
        "缺失值处理",
        "特征工程重要性高",
        "需要防止过拟合",
        "异常值检测"
    ]
}

2. 解决方案架构

graph LR
    A[原始数据] --> B[数据清洗]
    B --> C[缺失值处理]
    C --> D[特征工程]
    D --> E[特征选择]
    E --> F[模型训练]
    F --> G[模型融合]
    G --> H[后处理]
    H --> I[提交结果]

    D --> D1[数值特征]
    D --> D2[类别特征]
    D --> D3[交互特征]
    D --> D4[目标编码]

    F --> F1[XGBoost]
    F --> F2[LightGBM]
    F --> F3[CatBoost]
    F --> F4[Neural Network]

3. 数据探索分析

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats

class HousePricesEDA:
    """房价数据探索性分析"""

    def __init__(self, train_path, test_path):
        self.train_df = pd.read_csv(train_path)
        self.test_df = pd.read_csv(test_path)
        self.target = 'SalePrice'

    def analyze_target_distribution(self):
        """分析目标变量分布"""
        print("🏠 分析房价分布...")

        fig, axes = plt.subplots(2, 2, figsize=(15, 10))

        # 原始分布
        self.train_df[self.target].hist(bins=50, ax=axes[0,0])
        axes[0,0].set_title('原始房价分布')
        axes[0,0].set_xlabel('SalePrice')

        # 对数变换分布
        log_prices = np.log1p(self.train_df[self.target])
        log_prices.hist(bins=50, ax=axes[0,1])
        axes[0,1].set_title('对数变换后分布')
        axes[0,1].set_xlabel('log(SalePrice)')

        # Q-Q图
        stats.probplot(self.train_df[self.target], dist="norm", plot=axes[1,0])
        axes[1,0].set_title('Q-Q图:原始数据')

        stats.probplot(log_prices, dist="norm", plot=axes[1,1])
        axes[1,1].set_title('Q-Q图:对数变换')

        plt.tight_layout()
        plt.show()

        # 统计信息
        print(f"价格统计信息:")
        print(f"  平均值: ${self.train_df[self.target].mean():,.0f}")
        print(f"  中位数: ${self.train_df[self.target].median():,.0f}")
        print(f"  标准差: ${self.train_df[self.target].std():,.0f}")
        print(f"  偏度: {self.train_df[self.target].skew():.3f}")
        print(f"  峰度: {self.train_df[self.target].kurtosis():.3f}")

    def analyze_missing_values(self):
        """分析缺失值模式"""
        print("🔍 分析缺失值模式...")

        # 合并训练和测试数据
        all_data = pd.concat([self.train_df, self.test_df], ignore_index=True)

        # 计算缺失值比例
        missing_ratio = all_data.isnull().sum() / len(all_data)
        missing_features = missing_ratio[missing_ratio > 0].sort_values(ascending=False)

        if len(missing_features) > 0:
            # 可视化缺失值
            plt.figure(figsize=(12, 8))
            missing_features.head(20).plot(kind='bar')
            plt.title('Top 20 缺失值特征')
            plt.ylabel('缺失值比例')
            plt.xticks(rotation=45)
            plt.tight_layout()
            plt.show()

            print(f"有缺失值的特征数量: {len(missing_features)}")
            print(f"缺失值最多的5个特征:")
            for feature, ratio in missing_features.head().items():
                print(f"  {feature}: {ratio:.1%}")

        return missing_features

    def analyze_feature_correlation(self):
        """分析特征相关性"""
        print("📊 分析特征与目标变量的相关性...")

        # 数值特征相关性
        numeric_features = self.train_df.select_dtypes(include=[np.number]).columns
        correlations = self.train_df[numeric_features].corr()[self.target].abs().sort_values(ascending=False)

        # 绘制相关性热图
        plt.figure(figsize=(10, 12))
        correlations[1:21].plot(kind='barh')  # 排除自身相关性
        plt.title('特征与房价相关性 (Top 20)')
        plt.xlabel('绝对相关系数')
        plt.tight_layout()
        plt.show()

        print(f"与房价相关性最高的10个特征:")
        for i, (feature, corr) in enumerate(correlations[1:11].items(), 1):
            print(f"  {i:2d}. {feature:<20}: {corr:.3f}")

        return correlations

    def analyze_categorical_features(self):
        """分析分类特征"""
        print("🏷️  分析分类特征...")

        categorical_features = self.train_df.select_dtypes(include=['object']).columns

        # 计算每个分类特征的唯一值数量
        unique_counts = {}
        for col in categorical_features:
            unique_counts[col] = self.train_df[col].nunique()

        # 分析高基数分类特征
        high_cardinality = {k: v for k, v in unique_counts.items() if v > 10}

        print(f"分类特征总数: {len(categorical_features)}")
        print(f"高基数特征 (>10个类别): {len(high_cardinality)}")

        # 显示几个重要的分类特征与房价的关系
        important_categorical = ['Neighborhood', 'MSZoning', 'SaleType', 'SaleCondition']

        fig, axes = plt.subplots(2, 2, figsize=(16, 12))
        axes = axes.ravel()

        for i, feature in enumerate(important_categorical[:4]):
            if feature in self.train_df.columns:
                # 计算每个类别的平均房价
                avg_price = self.train_df.groupby(feature)[self.target].mean().sort_values(ascending=False)

                avg_price.plot(kind='bar', ax=axes[i])
                axes[i].set_title(f'{feature} vs 平均房价')
                axes[i].set_ylabel('平均房价')
                axes[i].tick_params(axis='x', rotation=45)

        plt.tight_layout()
        plt.show()

        return unique_counts

🔧 自动化特征工程

1. 特征工程流水线

class AutomatedFeatureEngineering:
    """自动化特征工程"""

    def __init__(self):
        self.feature_generators = [
            NumericalFeatureGenerator(),
            CategoricalFeatureGenerator(),
            InteractionFeatureGenerator(),
            TemporalFeatureGenerator(),
            TextFeatureGenerator()
        ]
        self.feature_importance_scores = {}

    def generate_features(self, train_data, test_data):
        """自动生成特征"""
        print("🔧 开始自动化特征工程...")

        all_features = []

        # 基础特征处理
        processed_train, processed_test = self._preprocess_data(train_data, test_data)

        # 生成各类特征
        for generator in self.feature_generators:
            print(f"  🏭 生成 {generator.__class__.__name__} 特征...")

            new_features_train, new_features_test = generator.generate(
                processed_train, processed_test
            )

            # 特征重要性评估
            if 'SalePrice' in processed_train.columns:
                importance_scores = self._evaluate_feature_importance(
                    new_features_train, processed_train['SalePrice']
                )
            else:
                importance_scores = {}

            all_features.append({
                'train': new_features_train,
                'test': new_features_test,
                'importance': importance_scores,
                'generator': generator.__class__.__name__
            })

            print(f"    ✅ 生成了 {new_features_train.shape[1]} 个特征")

        # 特征选择和组合
        final_features = self._select_and_combine_features(all_features, processed_train, processed_test)

        return final_features

    def _preprocess_data(self, train_data, test_data):
        """数据预处理"""
        print("  🧹 数据预处理...")

        # 合并数据进行一致性处理
        train_size = len(train_data)
        all_data = pd.concat([train_data, test_data], ignore_index=True)

        # 缺失值处理
        all_data = self._handle_missing_values(all_data)

        # 异常值处理
        all_data = self._handle_outliers(all_data, train_size)

        # 分离训练和测试数据
        processed_train = all_data[:train_size].copy()
        processed_test = all_data[train_size:].copy()

        return processed_train, processed_test

    def _handle_missing_values(self, data):
        """处理缺失值"""
        # 数值特征:用中位数填充
        numeric_features = data.select_dtypes(include=[np.number]).columns
        for col in numeric_features:
            if data[col].isnull().any():
                data[col].fillna(data[col].median(), inplace=True)

        # 分类特征:用众数填充
        categorical_features = data.select_dtypes(include=['object']).columns
        for col in categorical_features:
            if data[col].isnull().any():
                data[col].fillna(data[col].mode()[0] if not data[col].mode().empty else 'Unknown', inplace=True)

        return data

    def _handle_outliers(self, data, train_size):
        """处理异常值(仅针对训练数据)"""
        if 'SalePrice' in data.columns:
            # 使用IQR方法检测异常值
            Q1 = data['SalePrice'][:train_size].quantile(0.25)
            Q3 = data['SalePrice'][:train_size].quantile(0.75)
            IQR = Q3 - Q1

            lower_bound = Q1 - 1.5 * IQR
            upper_bound = Q3 + 1.5 * IQR

            # 标记异常值但不删除,而是用边界值替换
            outlier_mask = (data['SalePrice'] < lower_bound) | (data['SalePrice'] > upper_bound)
            outlier_count = outlier_mask[:train_size].sum()

            if outlier_count > 0:
                print(f"    检测到 {outlier_count} 个房价异常值")
                # 用合理范围内的值替换
                data.loc[data['SalePrice'] < lower_bound, 'SalePrice'] = lower_bound
                data.loc[data['SalePrice'] > upper_bound, 'SalePrice'] = upper_bound

        return data

class NumericalFeatureGenerator:
    """数值特征生成器"""

    def generate(self, train_data, test_data):
        """生成数值特征"""
        numerical_cols = train_data.select_dtypes(include=[np.number]).columns
        numerical_cols = [col for col in numerical_cols if col not in ['Id', 'SalePrice']]

        new_features_train = pd.DataFrame(index=train_data.index)
        new_features_test = pd.DataFrame(index=test_data.index)

        for col in numerical_cols:
            if col in train_data.columns and col in test_data.columns:
                # 1. 对数变换
                train_log = np.log1p(train_data[col])
                test_log = np.log1p(test_data[col])
                new_features_train[f'{col}_log'] = train_log
                new_features_test[f'{col}_log'] = test_log

                # 2. 平方根变换
                train_sqrt = np.sqrt(train_data[col])
                test_sqrt = np.sqrt(test_data[col])
                new_features_train[f'{col}_sqrt'] = train_sqrt
                new_features_test[f'{col}_sqrt'] = test_sqrt

                # 3. 平方变换
                train_sq = train_data[col] ** 2
                test_sq = test_data[col] ** 2
                new_features_train[f'{col}_sq'] = train_sq
                new_features_test[f'{col}_sq'] = test_sq

                # 4. 分箱特征
                if train_data[col].nunique() > 10:  # 只对连续变量分箱
                    try:
                        train_binned, bins = pd.cut(train_data[col], bins=5, retbins=True, duplicates='drop')
                        test_binned = pd.cut(test_data[col], bins=bins)

                        # 转换为数值编码
                        new_features_train[f'{col}_binned'] = train_binned.cat.codes
                        new_features_test[f'{col}_binned'] = test_binned.cat.codes
                    except:
                        pass  # 如果分箱失败,跳过

        return new_features_train, new_features_test

class CategoricalFeatureGenerator:
    """分类特征生成器"""

    def generate(self, train_data, test_data):
        """生成分类特征"""
        categorical_cols = train_data.select_dtypes(include=['object']).columns

        new_features_train = pd.DataFrame(index=train_data.index)
        new_features_test = pd.DataFrame(index=test_data.index)

        for col in categorical_cols:
            # 1. 标签编码
            from sklearn.preprocessing import LabelEncoder
            le = LabelEncoder()

            # 合并所有唯一值进行编码
            all_values = pd.concat([train_data[col], test_data[col]]).unique()
            le.fit(all_values)

            new_features_train[f'{col}_label'] = le.transform(train_data[col])
            new_features_test[f'{col}_label'] = le.transform(test_data[col])

            # 2. 频率编码
            value_counts = train_data[col].value_counts()
            new_features_train[f'{col}_freq'] = train_data[col].map(value_counts)
            new_features_test[f'{col}_freq'] = test_data[col].map(value_counts).fillna(0)

            # 3. 目标编码 (仅使用训练数据)
            if 'SalePrice' in train_data.columns:
                target_mean = train_data.groupby(col)['SalePrice'].mean()
                global_mean = train_data['SalePrice'].mean()

                new_features_train[f'{col}_target'] = train_data[col].map(target_mean)
                new_features_test[f'{col}_target'] = test_data[col].map(target_mean).fillna(global_mean)

        return new_features_train, new_features_test

class InteractionFeatureGenerator:
    """交互特征生成器"""

    def generate(self, train_data, test_data, max_interactions=50):
        """生成交互特征"""
        # 选择重要的数值特征进行交互
        important_numerical = [
            'GrLivArea', 'TotalBsmtSF', 'GarageCars', 'GarageArea',
            'YearBuilt', 'YearRemodAdd', 'MasVnrArea', 'BsmtFinSF1',
            'LotArea', 'OverallQual', 'OverallCond'
        ]

        # 确保特征存在
        available_features = [col for col in important_numerical
                            if col in train_data.columns and col in test_data.columns]

        interaction_features_train = pd.DataFrame(index=train_data.index)
        interaction_features_test = pd.DataFrame(index=test_data.index)

        # 两两交互
        from itertools import combinations
        interaction_count = 0

        for col1, col2 in combinations(available_features, 2):
            if interaction_count >= max_interactions:
                break

            # 乘积特征
            interaction_features_train[f'{col1}_x_{col2}'] = (
                train_data[col1] * train_data[col2]
            )
            interaction_features_test[f'{col1}_x_{col2}'] = (
                test_data[col1] * test_data[col2]
            )

            # 比值特征
            interaction_features_train[f'{col1}_div_{col2}'] = (
                train_data[col1] / (train_data[col2] + 1)
            )
            interaction_features_test[f'{col1}_div_{col2}'] = (
                test_data[col1] / (test_data[col2] + 1)
            )

            interaction_count += 2

        # 专门的房屋特征组合
        if all(col in train_data.columns for col in ['GrLivArea', 'TotalBsmtSF']):
            # 总居住面积
            interaction_features_train['TotalLivingArea'] = (
                train_data['GrLivArea'] + train_data['TotalBsmtSF']
            )
            interaction_features_test['TotalLivingArea'] = (
                test_data['GrLivArea'] + test_data['TotalBsmtSF']
            )

        if all(col in train_data.columns for col in ['OverallQual', 'GrLivArea']):
            # 质量加权面积
            interaction_features_train['QualityArea'] = (
                train_data['OverallQual'] * train_data['GrLivArea']
            )
            interaction_features_test['QualityArea'] = (
                test_data['OverallQual'] * test_data['GrLivArea']
            )

        return interaction_features_train, interaction_features_test

🤖 RD-Agent 自动化配置

1. 配置文件

# kaggle_house_prices_config.yaml
data_science:
  scenario: "kaggle_competition"
  competition_id: "house-prices-advanced-regression-techniques"
  problem_type: "regression"
  target_column: "SalePrice"

  data:
    train_path: "train.csv"
    test_path: "test.csv"
    id_column: "Id"

  preprocessing:
    missing_value_strategy: "advanced"
    outlier_detection: true
    outlier_method: "iqr"
    feature_scaling: "robust"
    target_transformation: "log"

  feature_engineering:
    automated_features: true
    interaction_terms: true
    polynomial_features: 2
    target_encoding: true
    frequency_encoding: true
    max_features: 500

  model_selection:
    algorithms:
      - "xgboost"
      - "lightgbm"
      - "catboost"
      - "random_forest"
      - "neural_network"
    ensemble_method: "stacking"
    cv_strategy: "kfold"
    cv_folds: 5

  optimization:
    max_iterations: 20
    population_size: 30
    hyperparameter_tuning: "bayesian"
    feature_selection: "genetic"
    early_stopping: true

  evaluation:
    metrics: ["rmse", "mae", "r2"]
    validation_strategy: "holdout"
    test_size: 0.2

  llm:
    chat_model: "gpt-4o-mini"
    api_key: "your_openai_api_key"
    temperature: 0.1
    max_tokens: 4000

2. 运行 RD-Agent 自动化

# 设置 Kaggle API
export KAGGLE_USERNAME="your_username"
export KAGGLE_KEY="your_api_key"

# 下载竞赛数据
kaggle competitions download -c house-prices-advanced-regression-techniques

# 运行 RD-Agent 数据科学流水线
rdagent data_science --config kaggle_house_prices_config.yaml

# 启动 Web UI 监控
rdagent ui --port 19899 --data_science True

🧠 模型集成和优化

1. 自适应模型集成

import xgboost as xgb
import lightgbm as lgb
import catboost as cb
from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor
from sklearn.linear_model import ElasticNet, Ridge
from sklearn.neural_network import MLPRegressor
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
import warnings
warnings.filterwarnings('ignore')

class AdaptiveEnsemble:
    """自适应模型集成"""

    def __init__(self):
        self.base_models = {
            'xgboost': xgb.XGBRegressor(
                n_estimators=1000,
                learning_rate=0.1,
                max_depth=6,
                subsample=0.8,
                colsample_bytree=0.8,
                random_state=42
            ),
            'lightgbm': lgb.LGBMRegressor(
                n_estimators=1000,
                learning_rate=0.1,
                max_depth=6,
                subsample=0.8,
                colsample_bytree=0.8,
                random_state=42,
                verbose=-1
            ),
            'catboost': cb.CatBoostRegressor(
                iterations=1000,
                learning_rate=0.1,
                depth=6,
                random_state=42,
                verbose=False
            ),
            'random_forest': RandomForestRegressor(
                n_estimators=200,
                max_depth=15,
                random_state=42,
                n_jobs=-1
            ),
            'extra_trees': ExtraTreesRegressor(
                n_estimators=200,
                max_depth=15,
                random_state=42,
                n_jobs=-1
            ),
            'elastic_net': ElasticNet(
                alpha=0.1,
                l1_ratio=0.5,
                random_state=42
            )
        }

        self.meta_model = Ridge(alpha=1.0)
        self.model_weights = {}
        self.oof_predictions = {}

    def train_ensemble(self, X_train, y_train, X_test=None):
        """训练集成模型"""
        print("🤖 训练自适应集成模型...")

        # 使用交叉验证生成 Out-of-Fold 预测
        self.oof_predictions = self._generate_oof_predictions(X_train, y_train)

        # 训练元模型
        meta_features = np.column_stack(list(self.oof_predictions.values()))
        self.meta_model.fit(meta_features, y_train)

        # 在测试集上预测
        if X_test is not None:
            test_predictions = self._generate_test_predictions(X_train, y_train, X_test)
            return test_predictions

        return None

    def _generate_oof_predictions(self, X_train, y_train, cv_folds=5):
        """生成 Out-of-Fold 预测"""
        kf = KFold(n_splits=cv_folds, shuffle=True, random_state=42)
        oof_predictions = {}

        for name in self.base_models.keys():
            oof_predictions[name] = np.zeros(len(X_train))

        for fold, (train_idx, valid_idx) in enumerate(kf.split(X_train)):
            print(f"  🔄 训练第 {fold + 1}/{cv_folds} 折...")

            X_fold_train = X_train.iloc[train_idx]
            X_fold_valid = X_train.iloc[valid_idx]
            y_fold_train = y_train.iloc[train_idx]
            y_fold_valid = y_train.iloc[valid_idx]

            fold_scores = {}

            for name, model in self.base_models.items():
                # 训练模型
                if name in ['xgboost', 'lightgbm']:
                    model.fit(
                        X_fold_train, y_fold_train,
                        eval_set=[(X_fold_valid, y_fold_valid)],
                        early_stopping_rounds=50,
                        verbose=False
                    )
                else:
                    model.fit(X_fold_train, y_fold_train)

                # 预测验证集
                fold_pred = model.predict(X_fold_valid)
                oof_predictions[name][valid_idx] = fold_pred

                # 计算折内性能
                fold_rmse = np.sqrt(mean_squared_error(y_fold_valid, fold_pred))
                fold_scores[name] = fold_rmse

            # 输出折内性能
            print(f"    折 {fold + 1} RMSE: " +
                  " | ".join([f"{name}: {score:.4f}" for name, score in fold_scores.items()]))

        # 计算整体 OOF 性能
        print(f"\n  📊 整体 OOF 性能:")
        for name, predictions in oof_predictions.items():
            oof_rmse = np.sqrt(mean_squared_error(y_train, predictions))
            print(f"    {name:15}: {oof_rmse:.4f}")

        return oof_predictions

    def _generate_test_predictions(self, X_train, y_train, X_test):
        """生成测试集预测"""
        print("  🔮 生成测试集预测...")

        test_predictions = {}

        for name, model in self.base_models.items():
            # 在全量训练数据上重新训练
            if name in ['xgboost', 'lightgbm']:
                # 使用一部分数据作为验证集进行早停
                split_idx = int(0.8 * len(X_train))
                model.fit(
                    X_train.iloc[:split_idx], y_train.iloc[:split_idx],
                    eval_set=[(X_train.iloc[split_idx:], y_train.iloc[split_idx:])],
                    early_stopping_rounds=50,
                    verbose=False
                )
            else:
                model.fit(X_train, y_train)

            # 预测测试集
            test_pred = model.predict(X_test)
            test_predictions[name] = test_pred

        # 使用元模型进行最终预测
        test_meta_features = np.column_stack(list(test_predictions.values()))
        final_predictions = self.meta_model.predict(test_meta_features)

        return final_predictions, test_predictions

    def analyze_model_importance(self):
        """分析模型重要性"""
        if not self.oof_predictions:
            print("❌ 需要先训练模型")
            return

        # 计算每个模型的权重(基于元模型系数)
        meta_coefs = self.meta_model.coef_
        model_names = list(self.oof_predictions.keys())

        importance_df = pd.DataFrame({
            'Model': model_names,
            'Weight': meta_coefs,
            'Abs_Weight': np.abs(meta_coefs)
        }).sort_values('Abs_Weight', ascending=False)

        print("\n📊 模型重要性分析:")
        print(importance_df.to_string(index=False))

        # 可视化
        plt.figure(figsize=(10, 6))
        plt.bar(importance_df['Model'], importance_df['Weight'])
        plt.title('模型在集成中的权重')
        plt.ylabel('权重')
        plt.xticks(rotation=45)
        plt.tight_layout()
        plt.show()

        return importance_df

2. 超参数优化

from optuna import create_study, Trial
import optuna

class HyperparameterOptimizer:
    """超参数优化器"""

    def __init__(self, X_train, y_train, cv_folds=5):
        self.X_train = X_train
        self.y_train = y_train
        self.cv_folds = cv_folds
        self.best_params = {}

    def optimize_xgboost(self, n_trials=100):
        """优化XGBoost参数"""
        print("🔧 优化 XGBoost 超参数...")

        def objective(trial: Trial):
            params = {
                'n_estimators': trial.suggest_int('n_estimators', 100, 2000),
                'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),
                'max_depth': trial.suggest_int('max_depth', 3, 10),
                'subsample': trial.suggest_float('subsample', 0.6, 1.0),
                'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),
                'reg_alpha': trial.suggest_float('reg_alpha', 0, 10),
                'reg_lambda': trial.suggest_float('reg_lambda', 0, 10),
                'random_state': 42
            }

            model = xgb.XGBRegressor(**params)
            scores = self._cross_validate(model)
            return np.mean(scores)

        study = create_study(direction='minimize')
        study.optimize(objective, n_trials=n_trials, show_progress_bar=True)

        self.best_params['xgboost'] = study.best_params
        print(f"  最优参数: {study.best_params}")
        print(f"  最优得分: {study.best_value:.4f}")

        return study.best_params

    def optimize_lightgbm(self, n_trials=100):
        """优化LightGBM参数"""
        print("🔧 优化 LightGBM 超参数...")

        def objective(trial: Trial):
            params = {
                'n_estimators': trial.suggest_int('n_estimators', 100, 2000),
                'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3),
                'max_depth': trial.suggest_int('max_depth', 3, 10),
                'subsample': trial.suggest_float('subsample', 0.6, 1.0),
                'colsample_bytree': trial.suggest_float('colsample_bytree', 0.6, 1.0),
                'reg_alpha': trial.suggest_float('reg_alpha', 0, 10),
                'reg_lambda': trial.suggest_float('reg_lambda', 0, 10),
                'num_leaves': trial.suggest_int('num_leaves', 10, 300),
                'random_state': 42,
                'verbose': -1
            }

            model = lgb.LGBMRegressor(**params)
            scores = self._cross_validate(model)
            return np.mean(scores)

        study = create_study(direction='minimize')
        study.optimize(objective, n_trials=n_trials, show_progress_bar=True)

        self.best_params['lightgbm'] = study.best_params
        print(f"  最优参数: {study.best_params}")
        print(f"  最优得分: {study.best_value:.4f}")

        return study.best_params

    def _cross_validate(self, model):
        """交叉验证评估模型"""
        kf = KFold(n_splits=self.cv_folds, shuffle=True, random_state=42)
        scores = []

        for train_idx, valid_idx in kf.split(self.X_train):
            X_fold_train = self.X_train.iloc[train_idx]
            X_fold_valid = self.X_train.iloc[valid_idx]
            y_fold_train = self.y_train.iloc[train_idx]
            y_fold_valid = self.y_train.iloc[valid_idx]

            model.fit(X_fold_train, y_fold_train)
            pred = model.predict(X_fold_valid)
            rmse = np.sqrt(mean_squared_error(y_fold_valid, pred))
            scores.append(rmse)

        return scores

    def optimize_all_models(self):
        """优化所有模型的超参数"""
        print("🎯 开始全模型超参数优化...")

        # 优化主要模型
        self.optimize_xgboost(n_trials=50)
        self.optimize_lightgbm(n_trials=50)

        print("\n✅ 超参数优化完成")
        return self.best_params

📝 提交准备和后处理

1. 结果后处理

class KaggleSubmissionPipeline:
    """Kaggle 提交流水线"""

    def __init__(self, competition_id):
        self.competition_id = competition_id
        self.submission_history = []

    def prepare_submission(self, test_predictions, test_ids, target_column='SalePrice'):
        """准备提交文件"""
        print("📝 准备提交文件...")

        # 后处理预测结果
        processed_predictions = self._post_process_predictions(test_predictions)

        # 创建提交数据框
        submission_df = pd.DataFrame({
            'Id': test_ids,
            target_column: processed_predictions
        })

        # 验证提交格式
        self._validate_submission_format(submission_df)

        # 生成提交文件名
        from datetime import datetime
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        filename = f"submission_{timestamp}.csv"

        # 保存文件
        submission_df.to_csv(filename, index=False)

        print(f"✅ 提交文件已生成: {filename}")
        print(f"   预测统计:")
        print(f"   - 样本数量: {len(processed_predictions)}")
        print(f"   - 最小值: ${processed_predictions.min():,.0f}")
        print(f"   - 最大值: ${processed_predictions.max():,.0f}")
        print(f"   - 平均值: ${processed_predictions.mean():,.0f}")
        print(f"   - 中位数: ${np.median(processed_predictions):,.0f}")

        # 记录提交历史
        self.submission_history.append({
            'filename': filename,
            'timestamp': timestamp,
            'predictions_stats': {
                'mean': processed_predictions.mean(),
                'std': processed_predictions.std(),
                'min': processed_predictions.min(),
                'max': processed_predictions.max()
            }
        })

        return filename, submission_df

    def _post_process_predictions(self, predictions):
        """后处理预测结果"""
        processed = predictions.copy()

        # 1. 反对数变换(如果之前做了对数变换)
        if np.all(processed > 0) and np.all(processed < 20):  # 看起来像是对数值
            processed = np.expm1(processed)

        # 2. 异常值处理
        q1, q3 = np.percentile(processed, [25, 75])
        iqr = q3 - q1
        lower_bound = max(q1 - 1.5 * iqr, 10000)  # 房价至少1万
        upper_bound = min(q3 + 1.5 * iqr, 1000000)  # 房价最多100万

        processed = np.clip(processed, lower_bound, upper_bound)

        # 3. 确保合理的房价范围
        processed = np.maximum(processed, 30000)  # 最低3万
        processed = np.minimum(processed, 800000)  # 最高80万

        # 4. 微调:向中位数回归一点点
        median_price = np.median(processed)
        processed = processed * 0.99 + median_price * 0.01

        return processed

    def _validate_submission_format(self, submission_df):
        """验证提交格式"""
        required_columns = ['Id', 'SalePrice']

        # 检查必需列
        for col in required_columns:
            if col not in submission_df.columns:
                raise ValueError(f"缺少必需列: {col}")

        # 检查数据类型
        if not pd.api.types.is_numeric_dtype(submission_df['Id']):
            raise ValueError("Id列必须是数值类型")

        if not pd.api.types.is_numeric_dtype(submission_df['SalePrice']):
            raise ValueError("SalePrice列必须是数值类型")

        # 检查缺失值
        if submission_df.isnull().any().any():
            raise ValueError("提交文件不能包含缺失值")

        # 检查重复ID
        if submission_df['Id'].duplicated().any():
            raise ValueError("Id列不能有重复值")

        print("  ✅ 提交格式验证通过")

    def submit_to_kaggle(self, filename, message="RD-Agent automated submission"):
        """提交到Kaggle(需要kaggle命令行工具)"""
        import subprocess

        try:
            cmd = f"kaggle competitions submit -c {self.competition_id} -f {filename} -m \"{message}\""
            result = subprocess.run(cmd, shell=True, capture_output=True, text=True)

            if result.returncode == 0:
                print(f"✅ 成功提交到Kaggle: {filename}")
                print(f"   提交信息: {message}")
            else:
                print(f"❌ 提交失败: {result.stderr}")

        except Exception as e:
            print(f"❌ 提交过程出错: {e}")

2. 完整运行流程

def main():
    """主执行流程"""
    print("🏆 RD-Agent Kaggle 竞赛自动化解决方案")
    print("=" * 60)

    # 1. 设置环境和下载数据
    print("\n📥 第一步:数据准备")

    # 确保数据文件存在
    if not os.path.exists('train.csv') or not os.path.exists('test.csv'):
        print("正在下载竞赛数据...")
        os.system("kaggle competitions download -c house-prices-advanced-regression-techniques")
        os.system("unzip -o house-prices-advanced-regression-techniques.zip")

    # 2. 数据探索
    print("\n🔍 第二步:数据探索分析")
    eda = HousePricesEDA('train.csv', 'test.csv')

    eda.analyze_target_distribution()
    missing_features = eda.analyze_missing_values()
    correlations = eda.analyze_feature_correlation()
    categorical_stats = eda.analyze_categorical_features()

    # 3. 特征工程
    print("\n🔧 第三步:自动化特征工程")
    train_df = pd.read_csv('train.csv')
    test_df = pd.read_csv('test.csv')

    feature_engineer = AutomatedFeatureEngineering()
    feature_data = feature_engineer.generate_features(train_df, test_df)

    X_train = feature_data['train'].drop(['Id', 'SalePrice'], axis=1, errors='ignore')
    y_train = np.log1p(train_df['SalePrice'])  # 对数变换目标变量
    X_test = feature_data['test'].drop(['Id'], axis=1, errors='ignore')
    test_ids = test_df['Id']

    print(f"  ✅ 最终特征数量: {X_train.shape[1]}")
    print(f"  ✅ 训练样本数量: {X_train.shape[0]}")

    # 4. 模型训练和集成
    print("\n🤖 第四步:模型集成训练")
    ensemble = AdaptiveEnsemble()

    # 训练集成模型
    test_predictions, individual_predictions = ensemble.train_ensemble(X_train, y_train, X_test)

    # 分析模型重要性
    importance_df = ensemble.analyze_model_importance()

    # 5. 超参数优化(可选)
    print("\n🎯 第五步:超参数优化")
    optimizer = HyperparameterOptimizer(X_train, y_train)
    best_params = optimizer.optimize_all_models()

    # 6. 生成提交文件
    print("\n📝 第六步:生成提交文件")
    submission_pipeline = KaggleSubmissionPipeline("house-prices-advanced-regression-techniques")

    filename, submission_df = submission_pipeline.prepare_submission(
        test_predictions, test_ids
    )

    # 7. 性能分析
    print("\n📊 第七步:性能分析")

    # 交叉验证分数
    cv_scores = []
    kf = KFold(n_splits=5, shuffle=True, random_state=42)

    for train_idx, valid_idx in kf.split(X_train):
        X_fold_train = X_train.iloc[train_idx]
        X_fold_valid = X_train.iloc[valid_idx]
        y_fold_train = y_train.iloc[train_idx]
        y_fold_valid = y_train.iloc[valid_idx]

        # 使用集成模型预测(简化)
        temp_ensemble = AdaptiveEnsemble()
        temp_ensemble.train_ensemble(X_fold_train, y_fold_train)

        # 这里简化处理,实际应该用训练好的模型预测
        fold_rmse = 0.12  # 示例值
        cv_scores.append(fold_rmse)

    print(f"  交叉验证 RMSE: {np.mean(cv_scores):.4f} (+/- {np.std(cv_scores)*2:.4f})")

    # 8. 可选:自动提交
    auto_submit = input("\n是否自动提交到Kaggle? (y/n): ").lower().strip()
    if auto_submit == 'y':
        submission_pipeline.submit_to_kaggle(
            filename,
            "RD-Agent automated ensemble submission"
        )

    print("\n🎉 Kaggle 竞赛解决方案完成!")
    print(f"📁 提交文件: {filename}")
    print("💡 建议下一步:")
    print("   1. 分析特征重要性,进一步优化特征工程")
    print("   2. 尝试更多模型和集成策略")
    print("   3. 进行更细致的超参数调优")
    print("   4. 分析预测结果,寻找改进空间")

if __name__ == "__main__":
    main()

3. 命令行运行

# 设置 Kaggle API
export KAGGLE_USERNAME="your_username"
export KAGGLE_KEY="your_api_key"

# 运行完整解决方案
python /path/to/tutorial_examples/chapter4/kaggle_competition_case.py

# 或者使用 RD-Agent 命令行
rdagent data_science --config kaggle_house_prices_config.yaml

📊 案例总结

学习收获

通过本案例,你应该掌握了:

✅ 端到端Kaggle竞赛流程:从数据探索到最终提交 ✅ 自动化特征工程:数值、分类、交互特征的自动生成 ✅ 模型集成技术:Stacking、Blending等集成方法 ✅ 超参数优化:贝叶斯优化等高效调参技术 ✅ 提交策略优化:后处理、格式验证、自动提交

扩展方向

  • 🧬 高级特征工程:基于领域知识的特征构造
  • 🤖 深度学习模型:Neural Networks、TabNet等
  • 🔍 特征选择优化:遗传算法、递归特征消除
  • 📈 集成策略进阶:Dynamic Ensemble、Bayesian Model Averaging
  • 🎯 多竞赛适配:构建通用竞赛解决方案框架

下一步行动

  1. 参加更多竞赛:将学到的技术应用到其他竞赛
  2. 深入特定技术:专精某个感兴趣的技术方向
  3. 分享和交流:在Kaggle社区分享你的解决方案
  4. 产业应用:将竞赛技术应用到实际业务问题
  5. 团队合作:组建团队参与更复杂的竞赛

🎯 准备好挑战下一个案例了吗?

⬅️ 返回案例概览 ➡️ 进入学术研究案例