/// <summary>
/// Gets the historical data of an bar indicator and convert it into pandas.DataFrame
/// </summary>
/// <param name="indicator">Bar indicator</param>
/// <param name="history">Historical data used to calculate the indicator</param>
/// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param>
/// <returns>pandas.DataFrame containing the historical data of <param name="indicator"></returns>
private PyObject Indicator <T>(IndicatorBase <T> indicator, IEnumerable <Slice> history, Func <IBaseData, T> selector = null)
where T : IBaseData
{
// Reset the indicator
indicator.Reset();
// Create a dictionary of the properties
var name = indicator.GetType().Name;
var properties = indicator.GetType().GetProperties()
.Where(x => x.PropertyType.IsGenericType)
.ToDictionary(x => x.Name, y => new List <IndicatorDataPoint>());
properties.Add(name, new List <IndicatorDataPoint>());
indicator.Updated += (s, e) =>
{
if (!indicator.IsReady)
{
return;
}
foreach (var kvp in properties)
{
var dataPoint = kvp.Key == name ? e : GetPropertyValue(s, kvp.Key + ".Current");
kvp.Value.Add((IndicatorDataPoint)dataPoint);
}
};
selector = selector ?? (x => (T)x);
history.PushThrough(bar => indicator.Update(selector(bar)));
return(PandasConverter.GetIndicatorDataFrame(properties));
}
private double GetExpectedValue(List <Slice> history)
{
var code = @"
import numpy as np
import pandas as pd
import math
from BlackLittermanOptimizationPortfolioConstructionModel import BlackLittermanOptimizationPortfolioConstructionModel as blopcm
def GetDeterminantFromHistory(history):
returns = dict()
history = history.lastprice.unstack(0)
for symbol, df in history.items():
symbolData = blopcm.BlackLittermanSymbolData(symbol, 1, 5)
for time, close in df.dropna().items():
symbolData.Update(time, close)
returns[symbol] = symbolData.Return
df = pd.DataFrame(returns)
if df.isna().sum().sum() > 0:
df[df.columns[-1]] = df[df.columns[-1]].bfill()
df = df.dropna()
return np.linalg.det(df)";
using (Py.GIL())
{
dynamic GetDeterminantFromHistory = PyModule.FromString("GetDeterminantFromHistory", code)
.GetAttr("GetDeterminantFromHistory");
dynamic df = new PandasConverter().GetDataFrame(history);
return(GetDeterminantFromHistory(df));
}
}
public void HandlesLeanDataReaderOutputForSpotMarkets(string securityType, string market, string resolution, string ticker, string fileName, int rowsInfile, double sumValue)
{
using (Py.GIL())
{
// Arrange
var dataFolder = "../../../Data";
var filepath = LeanDataReaderTests.GenerateFilepathForTesting(dataFolder, securityType, market, resolution, ticker, fileName);
var leanDataReader = new LeanDataReader(filepath);
var data = leanDataReader.Parse();
var converter = new PandasConverter();
// Act
dynamic df = converter.GetDataFrame(data);
// Assert
Assert.AreEqual(rowsInfile, df.shape[0].AsManagedObject(typeof(int)));
int columnsNumber = df.shape[1].AsManagedObject(typeof(int));
if (columnsNumber == 3 || columnsNumber == 6)
{
Assert.AreEqual(sumValue, df.get("lastprice").sum().AsManagedObject(typeof(double)), 1e-4);
}
else
{
Assert.AreEqual(sumValue, df.get("close").sum().AsManagedObject(typeof(double)), 1e-4);
}
}
}
public void HandlesLeanDataReaderOutputForOptionAndFutures(string composedFilePath, Symbol symbol, int rowsInfile, double sumValue)
{
using (Py.GIL())
{
// Arrange
var leanDataReader = new LeanDataReader(composedFilePath);
var data = leanDataReader.Parse();
var converter = new PandasConverter();
// Act
dynamic df = converter.GetDataFrame(data);
// Assert
Assert.AreEqual(rowsInfile, df.shape[0].AsManagedObject(typeof(int)));
int columnsNumber = df.shape[1].AsManagedObject(typeof(int));
if (columnsNumber == 3 || columnsNumber == 6)
{
Assert.AreEqual(sumValue, df.get("lastprice").sum().AsManagedObject(typeof(double)), 1e-4);
}
else if (columnsNumber == 1)
{
Assert.AreEqual(sumValue, df.get("openinterest").sum().AsManagedObject(typeof(double)), 1e-4);
}
else
{
Assert.AreEqual(sumValue, df.get("close").sum().AsManagedObject(typeof(double)), 1e-4);
}
}
}
public void HandlesQuoteTicks()
{
var converter = new PandasConverter();
var symbol = Symbols.EURUSD;
var rawBars = Enumerable
.Range(0, 10)
.Select(i => new Tick(DateTime.UtcNow.AddMilliseconds(100 * i), symbol, 0.99m, 1.01m))
.ToArray();
// GetDataFrame with argument of type IEnumerable<QuoteBar>
dynamic dataFrame = converter.GetDataFrame(rawBars);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
Assert.IsTrue(subDataFrame.get("askprice") != null);
Assert.IsTrue(subDataFrame.get("exchange") == null);
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].lastprice.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].LastPrice, value);
}
}
// GetDataFrame with argument of type IEnumerable<QuoteBar>
var history = GetHistory(symbol, Resolution.Tick, rawBars);
dataFrame = converter.GetDataFrame(history);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
Assert.IsTrue(subDataFrame.get("askprice") != null);
Assert.IsTrue(subDataFrame.get("exchange") == null);
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].askprice.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].AskPrice, value);
}
}
}
/// <summary>
/// <see cref = "QuantBook" /> constructor.
/// Provides access to data for quantitative analysis
/// </summary>
public QuantBook() : base()
{
try
{
using (Py.GIL())
{
_pandas = Py.Import("pandas");
}
_converter = new PandasConverter(_pandas);
// Create new instance of QCAlgorithm we are going to wrap
_algorithm = new QCAlgorithm();
// By default, set start date to end data which is yesterday
SetStartDate(_algorithm.EndDate);
// Initialize History Provider
var composer = new Composer();
var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer);
_dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider);
var mapFileProvider = algorithmHandlers.MapFileProvider;
_algorithm.HistoryProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider"));
_algorithm.HistoryProvider.Initialize(null, algorithmHandlers.DataProvider, _dataCacheProvider, mapFileProvider, algorithmHandlers.FactorFileProvider, null);
}
catch (Exception exception)
{
throw new Exception("QuantBook.Main(): " + exception);
}
}
private dynamic GetTestDataFrame(Symbol symbol)
{
var converter = new PandasConverter();
var rawBars = Enumerable
.Range(0, 1)
.Select(i => new Tick(symbol, $"1440{i:D2}00,167{i:D2}00,1{i:D2},T,T,0", new DateTime(2013, 10, 7)))
.ToArray();
return(converter.GetDataFrame(rawBars));
}
public void HandlesTradeBars()
{
var converter = new PandasConverter();
var symbol = Symbols.SPY;
var rawBars = Enumerable
.Range(0, 10)
.Select(i => new TradeBar(DateTime.UtcNow.AddMinutes(i), symbol, i + 101m, i + 102m, i + 100m, i + 101m, 0m))
.ToArray();
// GetDataFrame with argument of type IEnumerable<TradeBar>
dynamic dataFrame = converter.GetDataFrame(rawBars);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var close = subDataFrame.loc[index].close.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Close, close);
}
}
// GetDataFrame with argument of type IEnumerable<TradeBar>
var history = GetHistory(symbol, Resolution.Minute, rawBars);
dataFrame = converter.GetDataFrame(history);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var close = subDataFrame.loc[index].close.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Close, close);
}
}
}
public void HandlesOpenInterestTicks([ValueSource(nameof(ResolutionCases))] Resolution resolution, [ValueSource(nameof(SymbolCases))] Symbol symbol)
{
// Arrange
var converter = new PandasConverter();
var tickType = TickType.OpenInterest;
var dataType = LeanData.GetDataType(resolution, tickType);
var subcriptionDataConfig = new SubscriptionDataConfig(dataType, symbol, resolution,
TimeZones.Chicago, TimeZones.Chicago,
tickType: tickType, fillForward: false,
extendedHours: true, isInternalFeed: true);
var openinterest = new List <OpenInterest>();
for (int i = 0; i < 10; i++)
{
var line = $"{1000 * i},{11 * i}";
var openInterestTicks = new OpenInterest(subcriptionDataConfig, symbol, line, new DateTime(2017, 10, 10));
openinterest.Add(openInterestTicks);
}
// Act
dynamic dataFrame = converter.GetDataFrame(openinterest);
//Assert
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol.Value];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
Assert.IsTrue(subDataFrame.get("openinterest") != null);
var count = subDataFrame.shape[0].AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].openinterest.AsManagedObject(typeof(decimal));
Assert.AreEqual(openinterest[i].Value, value);
}
}
}
/// <summary>
/// Gets the historical data of an indicator and convert it into pandas.DataFrame
/// </summary>
/// <param name="indicator">Indicator</param>
/// <param name="history">Historical data used to calculate the indicator</param>
/// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param>
/// <returns>pandas.DataFrame containing the historical data of <param name="indicator"></returns>
private PyObject Indicator(IndicatorBase <IndicatorDataPoint> indicator, IEnumerable <IBaseData> history, Func <IBaseData, decimal> selector = null)
{
// Reset the indicator
indicator.Reset();
// Create a dictionary of the properties
var name = indicator.GetType().Name;
var properties = indicator.GetType().GetProperties()
.Where(x => x.PropertyType.IsGenericType)
.ToDictionary(x => x.Name, y => new List <IndicatorDataPoint>());
properties.Add(name, new List <IndicatorDataPoint>());
indicator.Updated += (s, e) =>
{
if (!indicator.IsReady)
{
return;
}
foreach (var kvp in properties)
{
var dataPoint = kvp.Key == name ? e : GetPropertyValue(s, kvp.Key + ".Current");
kvp.Value.Add((IndicatorDataPoint)dataPoint);
}
};
selector = selector ?? (x => x.Value);
foreach (var bar in history)
{
var value = selector(bar);
indicator.Update(bar.EndTime, value);
}
return(PandasConverter.GetIndicatorDataFrame(properties));
}
public void HandlesEmptyEnumerable()
{
var converter = new PandasConverter();
var rawBars = Enumerable.Empty <TradeBar>().ToArray();
// GetDataFrame with argument of type IEnumerable<TradeBar>
dynamic dataFrame = converter.GetDataFrame(rawBars);
using (Py.GIL())
{
Assert.IsTrue(dataFrame.empty.AsManagedObject(typeof(bool)));
}
// GetDataFrame with argument of type IEnumerable<TradeBar>
var history = GetHistory(Symbols.SPY, Resolution.Minute, rawBars);
dataFrame = converter.GetDataFrame(history);
using (Py.GIL())
{
Assert.IsTrue(dataFrame.empty.AsManagedObject(typeof(bool)));
}
}
public void HandlesCustomDataBars(Type type, string format)
{
var converter = new PandasConverter();
var symbol = Symbols.LTCUSD;
var config = GetSubscriptionDataConfig <Quandl>(symbol, Resolution.Daily);
var custom = Activator.CreateInstance(type) as BaseData;
if (type == typeof(Quandl))
{
custom.Reader(config, "date,open,high,low,close,transactions", DateTime.UtcNow, false);
}
var rawBars = Enumerable
.Range(0, 10)
.Select(i =>
{
var line = $"{DateTime.UtcNow.AddDays(i).ToString(format)},{i + 101},{i + 102},{i + 100},{i + 101},{i + 101}";
return(custom.Reader(config, line, DateTime.UtcNow.AddDays(i), false));
})
.ToArray();
// GetDataFrame with argument of type IEnumerable<BaseData>
dynamic dataFrame = converter.GetDataFrame(rawBars);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
var transactions = subDataFrame.loc[index].transactions.AsManagedObject(typeof(decimal));
var expected = (rawBars[i] as DynamicData)?.GetProperty("transactions");
expected = expected ?? type.GetProperty("Transactions")?.GetValue(rawBars[i]);
Assert.AreEqual(expected, transactions);
}
}
// GetDataFrame with argument of type IEnumerable<BaseData>
var history = GetHistory(symbol, Resolution.Daily, rawBars);
dataFrame = converter.GetDataFrame(history);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(10, count);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
var transactions = subDataFrame.loc[index].transactions.AsManagedObject(typeof(decimal));
var expected = (rawBars[i] as DynamicData)?.GetProperty("transactions");
expected = expected ?? type.GetProperty("Transactions")?.GetValue(rawBars[i]);
Assert.AreEqual(expected, transactions);
}
}
}
/// <summary>
/// Sets pandas converter
/// </summary>
public void SetPandasConverter()
{
PandasConverter = new PandasConverter();
}
/// <summary>
/// Create a new instance of <see cref="FutureHistory"/>.
/// </summary>
/// <param name="data"></param>
public FutureHistory(IEnumerable <Slice> data)
{
_data = data;
_converter = new PandasConverter();
_dataframe = _converter.GetDataFrame(_data);
}
public void HandlesCustomDataBars()
{
var converter = new PandasConverter();
var symbol = Symbols.LTCUSD;
var config = GetSubscriptionDataConfig <Quandl>(symbol, Resolution.Daily);
var quandl = new Quandl();
quandl.Reader(config, "date,open,high,low,close,settle", DateTime.UtcNow, false);
var rawBars = Enumerable
.Range(0, 10)
.Select(i =>
{
var line = $"{DateTime.UtcNow.AddDays(i).ToString("yyyy-MM-dd")},{i + 101},{i + 102},{i + 100},{i + 101},{i + 101}";
return(quandl.Reader(config, line, DateTime.UtcNow.AddDays(i), false));
})
.ToArray();
// GetDataFrame with argument of type IEnumerable<BaseData>
dynamic dataFrame = converter.GetDataFrame(rawBars);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
var settle = subDataFrame.loc[index].settle.AsManagedObject(typeof(decimal));
Assert.AreEqual(((DynamicData)rawBars[i]).GetProperty("settle"), settle);
}
}
// GetDataFrame with argument of type IEnumerable<BaseData>
var history = GetHistory(symbol, Resolution.Daily, rawBars);
dataFrame = converter.GetDataFrame(history);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
var settle = subDataFrame.loc[index].settle.AsManagedObject(typeof(decimal));
Assert.AreEqual(((DynamicData)rawBars[i]).GetProperty("settle"), settle);
}
}
}
/// <summary>
/// Gets the historical data for the specified symbol over the request span. The symbol must exist in the Securities collection.
/// </summary>
/// <typeparam name="T">The data type of the symbol</typeparam>
/// <param name="symbol">The symbol to retrieve historical data for</param>
/// <param name="span">The span over which to retrieve recent historical data</param>
/// <param name="resolution">The resolution to request</param>
/// <returns>An enumerable of slice containing the requested historical data</returns>
public new PyObject History<T>(Symbol symbol, TimeSpan span, Resolution? resolution = null)
where T : IBaseData
{
return PandasConverter.GetDataFrame(History<T>(symbol, Time - span, Time, resolution).Memoize());
}
public void HandlesCustomDataBarsInheritsFromTradeBar(Type type, string propertyName)
{
var converter = new PandasConverter();
var symbol = Symbols.LTCUSD;
var config = GetSubscriptionDataConfig <Quandl>(symbol, Resolution.Daily);
dynamic custom = Activator.CreateInstance(type);
var rawBars = Enumerable
.Range(0, 10)
.Select(i =>
{
var line = $"{DateTime.UtcNow.AddDays(i).ToStringInvariant("yyyyMMdd HH:mm")},{i + 101},{i + 102},{i + 100},{i + 101},{i + 101}";
return(custom.Reader(config, line, DateTime.UtcNow.AddDays(i), false) as BaseData);
})
.ToArray();
// GetDataFrame with argument of type IEnumerable<BaseData>
dynamic dataFrame = converter.GetDataFrame(rawBars);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
var transactions = subDataFrame.loc[index][propertyName.ToLowerInvariant()].AsManagedObject(typeof(decimal));
var expected = type.GetProperty(propertyName)?.GetValue(rawBars[i]);
Assert.AreEqual(expected, transactions);
}
}
// GetDataFrame with argument of type IEnumerable<BaseData>
var history = GetHistory(symbol, Resolution.Daily, rawBars);
dataFrame = converter.GetDataFrame(history);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(10, count);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
}
}
}
