Skip to content

title: Chapter 7: Data and Historical Queries

Chapter 7: Data and Historical Queries

This chapter takes a detailed look at data access in Lean, including historical data requests, Universe selection, and custom data.

7.1 Data Type Overview

7.1.1 Basic Data Types

Type Python Class Description
TradeBar TradeBar OHLCV data
QuoteBar QuoteBar Bid/ask quote data
Tick Tick Tick-by-tick trade data
Slice Slice All data at a point in time

7.1.2 Data Structures

# TradeBar 结构
trade_bar = {
    'Time': datetime,      # 时间
    'Open': decimal,       # 开盘价
    'High': decimal,       # 最高价
    'Low': decimal,        # 最低价
    'Close': decimal,      # 收盘价
    'Volume': int,         # 成交量
}

# QuoteBar 结构
quote_bar = {
    'Time': datetime,
    'Bid': {
        'Open': decimal,
        'High': decimal,
        'Low': decimal,
        'Close': decimal,
    },
    'Ask': {
        'Open': decimal,
        'High': decimal,
        'Low': decimal,
        'Close': decimal,
    },
}

7.2 Historical Data Requests

7.2.1 Basic Usage

def Initialize(self):
    self.spy = self.AddEquity("SPY", Resolution.Daily).Symbol

def OnData(self, data):
    # 获取 SPY 过去30天的日线数据
    history = self.History(self.spy, 30, Resolution.Daily)

    # history 是 DataFrame(行索引是时间,列名为小写字段,如 'open'/'close')
    for index, row in history.iterrows():
        self.Debug(f"{index}: Close={row['close']}")

    # 或者只迭代某一列(注意:Python 的 History 返回 pandas DataFrame,
    # 不能像 C# 那样按 TradeBar 对象直接迭代 bar.Time / bar.Close)
    for time, close in history['close'].items():
        self.Debug(f"{time}: Close={close}")

7.2.2 Multi-Security Historical Data

def OnData(self, data):
    symbols = [self.spy, self.aapl, self.msft]

    # 获取多个证券的历史数据
    history = self.History(symbols, 100, Resolution.Daily)

    # 按 Symbol 访问
    for symbol in symbols:
        if symbol in history.index:
            symbol_history = history.loc[symbol]
            self.Debug(f"{symbol}: {len(symbol_history)} bars")

7.2.3 Time-Range Historical Data

def OnData(self, data):
    # 方式1:使用 timedelta
    history = self.History(
        self.spy,
        TimeSpan.FromDays(30),
        Resolution.Daily
    )

    # 方式2:指定起止日期
    start = datetime(2023, 1, 1)
    end = datetime(2023, 12, 31)

    history = self.History(
        self.spy,
        start,
        end,
        Resolution.Daily
    )

7.2.4 Historical Data at Different Resolutions

def OnData(self, data):
    # 获取分钟级数据
    minute_history = self.History(self.spy, 100, Resolution.Minute)

    # 获取小时级数据
    hour_history = self.History(self.spy, 100, Resolution.Hour)

    # 获取日线数据
    daily_history = self.History(self.spy, 100, Resolution.Daily)

7.2.5 Handling History Return Values

def OnData(self, data):
    # History 返回 pandas DataFrame(如果启用 pandas)
    history = self.History(self.spy, 30, Resolution.Daily)

    # 检查是否有数据
    if history.empty:
        return

    # 获取收盘价序列
    closes = history['close']

    # 计算简单统计
    avg_close = closes.mean()
    max_close = closes.max()
    min_close = closes.min()

    self.Debug(f"Average: {avg_close:.2f}, Max: {max_close:.2f}, Min: {min_close:.2f}")

7.3 Universe Selection

7.3.1 Manual Universe

def Initialize(self):
    # 手动指定股票列表
    symbols = ["AAPL", "MSFT", "GOOGL", "AMZN", "TSLA"]

    self.SetUniverseSelection(
        ManualUniverseSelectionModel(
            [self.AddEquity(s, Resolution.Daily).Symbol for s in symbols]
        )
    )

7.3.2 Coarse Universe

def Initialize(self):
    # 使用粗粒度筛选
    # 注意:Lean 中没有 CoarseFundamentalUniverse(...) 这种可实例化的筛选入口,
    # 惯用写法是直接调用 self.AddUniverse 并传入粗筛选函数
    self.AddUniverse(self.CoarseSelectionFunction)

def CoarseSelectionFunction(self, coarse):
    """
    粗筛选函数

    Args:
        coarse: List[CoarseFundamental] - 所有股票的粗粒度数据
    """
    # 筛选条件:
    # 1. 有价格数据
    # 2. 价格 > 10 美元
    # 3. 成交量 > 100 万

    filtered = [c for c in coarse
                if c.HasFundamentalData and
                c.Price > 10 and
                c.Volume > 1000000]

    # 按美元交易量排序,取前 100
    sorted_by_dollar_volume = sorted(
        filtered,
        key=lambda c: c.DollarVolume,
        reverse=True
    )

    # 返回前 100 只的 Symbol
    return [c.Symbol for c in sorted_by_dollar_volume[:100]]

7.3.3 Fine Universe

def Initialize(self):
    # 使用精细粒度筛选
    # 惯用写法:self.AddUniverse(粗筛选函数, 精筛选函数),
    # 粗筛选先把候选股票筛出来,精筛选再基于基本面数据做二次过滤
    self.AddUniverse(self.CoarseSelectionFunction, self.FineSelectionFunction)

def CoarseSelectionFunction(self, coarse):
    """粗筛选:按市值和交易量"""
    filtered = [c for c in coarse
                if c.HasFundamentalData and c.Price > 10]

    sorted_by_volume = sorted(
        filtered,
        key=lambda c: c.DollarVolume,
        reverse=True
    )

    return [c.Symbol for c in sorted_by_volume[:500]]

def FineSelectionFunction(self, fine):
    """
    精筛选:使用基本面数据

    Args:
        fine: List[FineFundamental] - 筛选后股票的基本面数据
    """
    # 筛选条件:
    # 1. 市盈率 < 20
    # 2. P/B < 3
    # 3. ROE > 0.1

    filtered = [f for f in fine
                if f.ValuationRatios.PERatio < 20 and
                f.ValuationRatios.PBRatio < 3 and
                f.OperationRatios.ROE.Value > 0.1]

    # 按 ROE 排序,取前 10
    sorted_by_roe = sorted(
        filtered,
        key=lambda f: f.OperationRatios.ROE.Value,
        reverse=True
    )

    return [f.Symbol for f in sorted_by_roe[:10]]

7.3.4 Option Universes

def Initialize(self):
    # 添加期权
    option = self.AddOption("SPY", Resolution.Minute)

    # 设置过滤器
    option.SetFilter(
        -2,              # 行权价:平价 ±2 档
        +2,
        0,               # 最短到期:今天
        30               # 最长到期:30 天
    )

    # 或使用自定义过滤器
    option.SetFilter(self.UniverseFilter)

def UniverseFilter(self, universe):
    """
    自定义期权链过滤器

    Args:
        universe: OptionFilterUniverse - 期权过滤 Universe,
                  通过链式方法组合过滤条件,最后原样返回

    Returns:
        OptionFilterUniverse
    """
    # 注意:OptionFilterUniverse 没有 .Filter(...) 方法,也没有可赋值的
    # IncludeWeeklys 属性;常用的链式方法有
    # StandardsOnly() / WeeklysOnly() / Expiration(min, max) /
    # Strikes(min, max) / Contracts(...) / FrontMonth() 等
    return (universe
            .StandardsOnly()                                    # 只保留标准(月度)合约,排除周期权
            .Expiration(timedelta(days=7), timedelta(days=30))  # 只留 7-30 天到期的合约
            .Strikes(-5, 5))                                    # 只留平价上下各 5 档行权价

7.3.5 Futures Universes

def Initialize(self):
    # 添加期货
    future = self.AddFuture("ES", Resolution.Minute)

    # 设置过滤器(最短/最长到期时间,两个 TimeSpan 参数)
    future.SetFilter(
        timedelta(0),         # 最短到期时间:从今天开始
        timedelta(days=30)    # 最长到期时间:30 天内
    )

    # 或使用简化的前端合约选择
    future.SetFilter(lambda u: u.FrontMonth())

7.4 Custom Data

7.4.1 Creating a Custom Data Type

class CustomData(PythonData):
    """
    自定义数据类型

    支持导入 CSV 格式的自定义数据
    """

    def GetSource(self, config, date, isLiveMode):
        """
        定义数据源

        Args:
            config: SubscriptionDataConfig
            date: DateTime
            isLiveMode: bool
        """
        # 返回数据源
        return SubscriptionDataSource(
            "data/custom_data.csv",  # 数据文件路径
            SubscriptionTransportMedium.LocalFile,
            FileFormat.Csv
        )

    def Reader(self, config, line, date, isLiveMode):
        """
        解析数据行

        Args:
            config: SubscriptionDataConfig
            line: str - CSV 行
            date: DateTime
            isLiveMode: bool
        """
        if not line.strip():
            return None

        # 解析 CSV
        parts = line.split(',')

        # 假设格式:date,symbol,value(date 形如 2023-01-31)
        # 注意:Python 没有 datetime.Parse,解析字符串日期用 datetime.strptime
        custom_data = CustomData()
        custom_data.Symbol = config.Symbol
        custom_data.Time = datetime.strptime(parts[0], "%Y-%m-%d")
        custom_data.Value = float(parts[2])
        custom_data.EndTime = custom_data.Time

        return custom_data

7.4.2 Using Custom Data

def Initialize(self):
    # 添加自定义数据
    self.AddData(CustomData, "CUSTOM", Resolution.Daily)

def OnData(self, data):
    # 访问自定义数据
    if data.ContainsKey("CUSTOM"):
        custom_value = data["CUSTOM"].Value
        self.Debug(f"Custom data: {custom_value}")

7.5 Data Access Patterns

7.5.1 Real-Time Data Access

def OnData(self, data):
    # 方式1:通过 Securities
    spy_price = self.Securities["SPY"].Price
    spy_volume = self.Securities["SPY"].Volume

    # 方式2:通过 Slice
    if data.ContainsKey("SPY"):
        bar = data["SPY"]
        self.Debug(f"Close: {bar.Close}, Volume: {bar.Volume}")

7.5.2 Historical Data Access Patterns

def OnData(self, data):
    # 模式1:获取固定数量
    recent_bars = self.History(self.spy, 20, Resolution.Daily)

    # 模式2:获取时间范围
    month_bars = self.History(
        self.spy,
        TimeSpan.FromDays(30),
        Resolution.Daily
    )

    # 模式3:获取特定时间段
    specific_bars = self.History(
        self.spy,
        datetime(2023, 1, 1),
        datetime(2023, 12, 31),
        Resolution.Daily
    )

7.6 Complete Example: Dynamic Stock Selection Strategy

from AlgorithmImports import *

class DynamicUniverseStrategy(QCAlgorithm):
    """
    动态 Universe 选择策略

    每月根据基本面数据重新选择股票
    """

    def Initialize(self):
        self.SetStartDate(2020, 1, 1)
        self.SetEndDate(2023, 12, 31)
        self.SetCash(100000)

        # 设置 Universe
        # 惯用写法:self.AddUniverse(粗筛选函数, 精筛选函数)
        self.AddUniverse(self.CoarseSelection, self.FineSelection)

        # 设置每月调仓
        self.Schedule.On(
            self.DateRules.MonthStart(),
            self.TimeRules.AfterMarketOpen("SPY", 30),
            self.Rebalance
        )

        self.selected_symbols = []

    def CoarseSelection(self, coarse):
        """粗筛选"""
        # 基本条件
        filtered = [c for c in coarse
                   if c.HasFundamentalData and
                   c.Price > 10 and
                   c.Volume > 1000000]

        # 按交易量排序
        sorted_by_volume = sorted(
            filtered,
            key=lambda c: c.DollarVolume,
            reverse=True
        )

        return [c.Symbol for c in sorted_by_volume[:500]]

    def FineSelection(self, fine):
        """精筛选"""
        # 基本面筛选
        filtered = [f for f in fine
                   if f.ValuationRatios.PERatio > 0 and
                   f.ValuationRatios.PERatio < 25 and
                   f.OperationRatios.ROE.Value > 0.1]

        # 按 ROE 排序
        sorted_by_roe = sorted(
            filtered,
            key=lambda f: f.OperationRatios.ROE.Value,
            reverse=True
        )

        self.selected_symbols = [f.Symbol for f in sorted_by_roe[:20]]
        return self.selected_symbols

    def Rebalance(self):
        """调仓"""
        # 平仓不在新列表中的
        for holding in self.Portfolio.Values:
            if holding.Invested and holding.Symbol not in self.selected_symbols:
                self.Liquidate(holding.Symbol)

        # 等权买入新选中的
        for symbol in self.selected_symbols:
            self.SetHoldings(symbol, 1.0 / len(self.selected_symbols))