"""
数据管理完整示例

演示如何使用自定义数据、数据下载和数据验证
"""

from AlgorithmImports import *
from datetime import timedelta, datetime

# ========== 自定义数据类型 ==========

class AlternativeData(PythonData):
    """
    自定义数据类型示例

    支持从CSV文件导入自定义数据
    """

    def GetSource(self, config, date, isLiveMode):
        """
        定义数据源

        Args:
            config: SubscriptionDataConfig
            date: DateTime
            isLiveMode: bool
        Returns:
            SubscriptionDataSource
        """
        return SubscriptionDataSource(
            "data/alternative_data.csv",  # 数据文件路径
            SubscriptionTransportMedium.LocalFile,
            FileFormat.Csv
        )

    def Reader(self, config, line, date, isLiveMode):
        """
        解析CSV行

        CSV格式：
        date,symbol,value,volume
        2023-01-01,AAPL,150.25,1000000

        Args:
            config: SubscriptionDataConfig
            line: str - CSV行
            date: DateTime
            isLiveMode: bool
        Returns:
            AlternativeData or None
        """
        if not line.strip():
            return None

        # 跳过标题行
        if line.startswith("date"):
            return None

        parts = line.split(',')
        if len(parts) < 4:
            return None

        # 解析数据
        data = AlternativeData()
        data.Symbol = config.Symbol  # 使用配置的Symbol
        data.Time = datetime.strptime(parts[0], "%Y-%m-%d")
        data.EndTime = data.Time + timedelta(days=1)
        data.Value = float(parts[2])

        return data


class SentimentData(PythonData):
    """
    市场情绪数据示例

    数据格式：
    date,sentiment_score,social_volume
    2023-01-01,0.65,10000

    sentiment_score: 0-1，0极度悲观，1极度乐观
    """

    def GetSource(self, config, date, isLiveMode):
        return SubscriptionDataSource(
            "data/sentiment.csv",
            SubscriptionTransportMedium.LocalFile,
            FileFormat.Csv
        )

    def Reader(self, config, line, date, isLiveMode):
        if not line.strip() or line.startswith("date"):
            return None

        parts = line.split(',')
        if len(parts) < 3:
            return None

        data = SentimentData()
        data.Symbol = config.Symbol
        data.Time = datetime.strptime(parts[0], "%Y-%m-%d")
        data.EndTime = data.Time + timedelta(days=1)
        data.Sentiment = float(parts[1])
        data.Volume = int(parts[2])

        return data


# ========== 自定义数据策略示例 ==========

class CustomDataSourceStrategy(QCAlgorithm):
    """
    使用自定义数据源的策略

    演示：
    1. 添加标准证券
    2. 添加自定义数据
    3. 数据质量验证
    4. 基于自定义数据的交易逻辑
    """

    def Initialize(self):
        """初始化"""
        # 基本设置
        self.SetStartDate(2020, 1, 1)
        self.SetEndDate(2023, 12, 31)
        self.SetCash(100000)

        # 添加标准证券
        self.spy = self.AddEquity("SPY", Resolution.Daily).Symbol
        self.aapl = self.AddEquity("AAPL", Resolution.Daily).Symbol

        # 设置数据归一化
        self.Securities[self.spy].SetDataNormalizationMode(
            DataNormalizationMode.Adjusted
        )
        self.Securities[self.aapl].SetDataNormalizationMode(
            DataNormalizationMode.Adjusted
        )

        # 添加自定义情绪数据（如果有数据文件）
        # self.sentiment = self.AddData(SentimentData, "SENTIMENT", Resolution.Daily).Symbol

        # 创建指标
        self.spy_sma = self.SMA(self.spy, 20)
        self.aapl_sma = self.SMA(self.aapl, 20)

        # RSI指标
        self.spy_rsi = self.RSI(self.spy, 14)
        self.aapl_rsi = self.RSI(self.aapl, 14)

        self.SetWarmUp(TimeSpan.FromDays(60))

        # 数据质量检查
        self.data_issues = []

        # 调度：每周检查数据质量
        self.Schedule.On(
            self.DateRules.Every(DayOfWeek.Friday),
            self.TimeRules.AfterMarketClose(self.spy, 30),
            self.WeeklyDataCheck
        )

        self.Debug("Custom Data Source Strategy Initialized")

    def OnData(self, data):
        """主数据处理"""
        # 验证数据质量
        if not self.ValidateDataQuality(data):
            return

        # 等待指标准备
        if not self.spy_sma.IsReady:
            return

        # 获取市场状态
        market_state = self.GetMarketState(data)

        # 根据市场状态调整策略
        if market_state == "bullish":
            self.BullishStrategy()
        elif market_state == "bearish":
            self.BearishStrategy()
        else:
            self.NeutralStrategy()

    def ValidateDataQuality(self, data):
        """
        验证数据质量

        Returns:
            bool: 数据是否有效
        """
        issues_found = []

        # 检查SPY数据
        if data.ContainsKey(self.spy):
            spy_data = data[self.spy]

            # 检查价格
            if spy_data.Price <= 0:
                issues_found.append(f"Invalid SPY price: {spy_data.Price}")

            # 检查成交量
            if spy_data.Volume < 0:
                issues_found.append(f"Invalid SPY volume: {spy_data.Volume}")

        # 检查AAPL数据
        if data.ContainsKey(self.aapl):
            aapl_data = data[self.aapl]

            if aapl_data.Price <= 0:
                issues_found.append(f"Invalid AAPL price: {aapl_data.Price}")

        # 记录问题
        if issues_found:
            for issue in issues_found:
                self.Log(f"Data Issue: {issue}")
            self.data_issues.extend(issues_found)
            return False

        return True

    def GetMarketState(self, data):
        """
        判断市场状态

        Returns:
            str: "bullish", "bearish", 或 "neutral"
        """
        spy_rsi = self.spy_rsi.Current.Value
        spy_ma_trend = self.Securities[self.spy].Price > self.spy_sma.Current.Value

        # RSI和趋势判断：RSI>50 且价格在均线上方视为多头
        if spy_rsi > 50 and spy_ma_trend:
            return "bullish"
        elif spy_rsi < 50:
            return "bearish"
        else:
            return "neutral"

    def BullishStrategy(self):
        """牛市策略：积极买入"""
        if not self.Portfolio[self.spy].IsLong:
            self.SetHoldings(self.spy, 1)
            self.Debug(f"[BULLISH] Buying SPY")

    def BearishStrategy(self):
        """熊市策略：减仓或清仓"""
        if self.Portfolio[self.spy].IsLong:
            self.Liquidate(self.spy)
            self.Debug(f"[BEARISH] Sold SPY")

    def NeutralStrategy(self):
        """中性策略：观望"""
        self.Debug(f"[NEUTRAL] No action")

    def WeeklyDataCheck(self):
        """每周数据检查"""
        self.Debug("\n" + "="*60)
        self.Debug("WEEKLY DATA QUALITY CHECK")
        self.Debug("="*60)

        # 检查最近30天的数据
        history = self.History(
            [self.spy, self.aapl],
            TimeSpan.FromDays(30),
            Resolution.Daily
        )

        if history.empty:
            self.Error("No history data available")
            return

        # 检查每个证券
        for symbol in [self.spy, self.aapl]:
            if symbol in history.index:
                symbol_history = history.loc[symbol]

                # 检查缺失数据
                expected_days = 30
                actual_days = len(symbol_history)
                missing_days = expected_days - actual_days

                # 检查数据连续性
                if len(symbol_history) > 1:
                    date_diff = symbol_history.index.to_series().diff()
                    max_gap = date_diff.max()

                    self.Debug(f"\n{symbol.Value}:")
                    self.Debug(f"  Expected days: {expected_days}")
                    self.Debug(f"  Actual days: {actual_days}")
                    self.Debug(f"  Missing: {missing_days}")
                    self.Debug(f"  Max gap: {max_gap.days} days")

                    if missing_days > 5:
                        self.Log(f"  -> {symbol.Value} has {missing_days} missing days!")
                    if max_gap.days > 7:
                        self.Log(f"  -> {symbol.Value} has {max_gap.days} days gap!")

        self.Debug("="*60)

    def OnEndOfAlgorithm(self):
        """算法结束总结"""
        self.Debug("\n" + "="*60)
        self.Debug("DATA QUALITY SUMMARY")
        self.Debug("="*60)
        self.Debug(f"Total data issues: {len(self.data_issues)}")

        if self.data_issues:
            self.Debug("\nIssues found:")
            for i, issue in enumerate(self.data_issues[:10], 1):
                self.Debug(f"  {i}. {issue}")

        self.Debug("="*60)


class DataPreProcessingStrategy(QCAlgorithm):
    """
    数据预处理策略

    演示：
    1. 数据清洗
    2. 异常值处理
    3. 数据验证
    """

    def Initialize(self):
        self.SetStartDate(2020, 1, 1)
        self.SetEndDate(2023, 12, 31)
        self.SetCash(100000)

        self.spy = self.AddEquity("SPY", Resolution.Daily).Symbol

        # 指标
        self.sma = self.SMA(self.spy, 20)
        self.std = self.STD(self.spy, 20)

        # 异常值检测参数
        self.outlier_threshold = 3  # 3个标准差

        # 记录异常值
        self.outliers_detected = []

        self.SetWarmUp(TimeSpan.FromDays(60))

        # 每月检查数据
        self.Schedule.On(
            self.DateRules.MonthStart(),
            self.TimeRules.AfterMarketOpen(self.spy, 30),
            self.MonthlyDataAnalysis
        )

    def OnData(self, data):
        """处理数据并检测异常"""
        if not self.sma.IsReady:
            return

        # 检查当前价格是否为异常值
        if data.ContainsKey(self.spy):
            price = data[self.spy].Price
            mean = self.sma.Current.Value
            std = self.std.Current.Value

            # Z-score检测
            if std > 0:
                zscore = (price - mean) / std

                if abs(zscore) > self.outlier_threshold:
                    self.outliers_detected.append({
                        'time': self.Time,
                        'price': price,
                        'zscore': zscore
                    })

                    self.Log(f"Outlier detected: Z={zscore:.2f}, Price=${price:.2f}")

                    # 可选：跳过异常值不交易
                    # return

        # 正常交易逻辑
        if self.sma.Current.Value > self.Securities[self.spy].Price * 1.01:
            if not self.Portfolio[self.spy].IsLong:
                self.SetHoldings(self.spy, 1)
        else:
            if self.Portfolio[self.spy].IsLong:
                self.Liquidate(self.spy)

    def MonthlyDataAnalysis(self):
        """每月数据分析"""
        self.Debug(f"\n[Monthly Analysis] {self.Time.date()}")

        # 获取月度数据
        start = self.Time - timedelta(days=30)
        history = self.History(self.spy, start, self.Time, Resolution.Daily)

        if not history.empty:
            closes = history['close']

            # 计算统计
            monthly_return = (closes.iloc[-1] / closes.iloc[0] - 1) * 100
            monthly_volatility = closes.pct_change().std() * np.sqrt(252) * 100

            # 检测本月的异常值
            returns = closes.pct_change().dropna()
            mean_return = returns.mean()
            std_return = returns.std()

            outliers = returns[abs((returns - mean_return) / std_return) > 3]

            self.Debug(f"  Monthly Return: {monthly_return:.2f}%")
            self.Debug(f"  Volatility: {monthly_volatility:.2f}%")
            self.Debug(f"  Outliers detected: {len(outliers)}")

    def OnEndOfAlgorithm(self):
        """结束总结"""
        self.Debug("\n" + "="*60)
        self.Debug("OUTLIER DETECTION SUMMARY")
        self.Debug("="*60)
        self.Debug(f"Total outliers: {len(self.outliers_detected)}")

        if self.outliers_detected:
            # 统计最异常的值
            sorted_outliers = sorted(self.outliers_detected,
                                   key=lambda x: abs(x['zscore']),
                                   reverse=True)

            self.Debug("\nTop 10 outliers:")
            for i, outlier in enumerate(sorted_outliers[:10], 1):
                self.Debug(f"  {i}. {outlier['time']} | "
                          f"Price: ${outlier['price']:.2f} | "
                          f"Z-score: {outlier['zscore']:.2f}")

        self.Debug("="*60)


class MultiSourceDataStrategy(QCAlgorithm):
    """
    多数据源策略

    演示同时使用多个数据源
    """

    def Initialize(self):
        self.SetStartDate(2020, 1, 1)
        self.SetEndDate(2023, 12, 31)
        self.SetCash(100000)

        # 主数据源：股票
        self.spy = self.AddEquity("SPY", Resolution.Daily).Symbol

        # 辅助数据源：VIX（波动率指数）
        self.vix = self.AddEquity("VIX", Resolution.Daily).Symbol

        # 辅助数据源：利率（TLT - 20年期美债）
        self.tlt = self.AddEquity("TLT", Resolution.Daily).Symbol

        # 指标
        self.spy_sma = self.SMA(self.spy, 50)
        self.vix_sma = self.SMA(self.vix, 20)
        self.tlt_sma = self.SMA(self.tlt, 50)

        self.SetWarmUp(TimeSpan.FromDays(60))

        # 数据源状态
        self.data_sources_status = {
            'SPY': True,
            'VIX': True,
            'TLT': True
        }

    def OnData(self, data):
        """使用多个数据源进行决策"""
        # 检查所有数据源
        all_available = True

        for symbol, name in [(self.spy, 'SPY'),
                             (self.vix, 'VIX'),
                             (self.tlt, 'TLT')]:
            if not data.ContainsKey(symbol) or data[symbol] is None:
                if self.data_sources_status[name]:
                    self.Log(f"{name} data not available at {self.Time}")
                    self.data_sources_status[name] = False
                all_available = False
            else:
                if not self.data_sources_status[name]:
                    self.Debug(f"{name} data restored at {self.Time}")
                    self.data_sources_status[name] = True

        # 如果不是所有数据都可用，谨慎交易
        if not all_available:
            return

        # 等待指标准备
        if not self.spy_sma.IsReady:
            return

        # 多数据源分析
        spy_price = self.Securities[self.spy].Price
        spy_ma = self.spy_sma.Current.Value
        vix_level = self.Securities[self.vix].Price
        tlt_trend = self.Securities[self.tlt].Price > self.tlt_sma.Current.Value

        # 综合判断
        # 条件1：价格趋势向上
        price_trend_up = spy_price > spy_ma

        # 条件2：低波动率（VIX < 20）
        low_volatility = vix_level < 20

        # 条件3：利率趋势向上（TLT涨表示利率跌，利好股票）
        # 但这里我们简化为TLT上涨表示风险偏好上升
        risk_on = tlt_trend

        # 综合信号
        bullish_signals = sum([price_trend_up, low_volatility])
        bearish_signals = sum([not price_trend_up, not low_volatility])

        if bullish_signals >= 2:
            if not self.Portfolio[self.spy].IsLong:
                self.SetHoldings(self.spy, 1)
                self.Debug(f"[BUY] PriceUp:{price_trend_up} LowVol:{low_volatility} "
                          f"VIX:{vix_level:.2f}")

        elif bearish_signals >= 2:
            if self.Portfolio[self.spy].IsLong:
                self.Liquidate(self.spy)
                self.Debug(f"[SELL] PriceUp:{price_trend_up} LowVol:{low_volatility} "
                          f"VIX:{vix_level:.2f}")

    def OnEndOfAlgorithm(self):
        """数据源统计"""
        self.Debug("\n" + "="*60)
        self.Debug("DATA SOURCES SUMMARY")
        self.Debug("="*60)

        for name, status in self.data_sources_status.items():
            self.Debug(f"{name}: {'Available' if status else 'Issues Detected'}")

        self.Debug("="*60)
