/// <summary> /// Gets the historical data of an bar indicator and convert it into pandas.DataFrame /// </summary> /// <param name="indicator">Bar indicator</param> /// <param name="history">Historical data used to calculate the indicator</param> /// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param> /// <returns>pandas.DataFrame containing the historical data of <param name="indicator"></returns> private PyObject Indicator <T>(IndicatorBase <T> indicator, IEnumerable <Slice> history, Func <IBaseData, T> selector = null) where T : IBaseData { // Reset the indicator indicator.Reset(); // Create a dictionary of the properties var name = indicator.GetType().Name; var properties = indicator.GetType().GetProperties() .Where(x => x.PropertyType.IsGenericType) .ToDictionary(x => x.Name, y => new List <IndicatorDataPoint>()); properties.Add(name, new List <IndicatorDataPoint>()); indicator.Updated += (s, e) => { if (!indicator.IsReady) { return; } foreach (var kvp in properties) { var dataPoint = kvp.Key == name ? e : GetPropertyValue(s, kvp.Key + ".Current"); kvp.Value.Add((IndicatorDataPoint)dataPoint); } }; selector = selector ?? (x => (T)x); history.PushThrough(bar => indicator.Update(selector(bar))); return(PandasConverter.GetIndicatorDataFrame(properties)); }
private double GetExpectedValue(List <Slice> history) { var code = @" import numpy as np import pandas as pd import math from BlackLittermanOptimizationPortfolioConstructionModel import BlackLittermanOptimizationPortfolioConstructionModel as blopcm def GetDeterminantFromHistory(history): returns = dict() history = history.lastprice.unstack(0) for symbol, df in history.items(): symbolData = blopcm.BlackLittermanSymbolData(symbol, 1, 5) for time, close in df.dropna().items(): symbolData.Update(time, close) returns[symbol] = symbolData.Return df = pd.DataFrame(returns) if df.isna().sum().sum() > 0: df[df.columns[-1]] = df[df.columns[-1]].bfill() df = df.dropna() return np.linalg.det(df)"; using (Py.GIL()) { dynamic GetDeterminantFromHistory = PyModule.FromString("GetDeterminantFromHistory", code) .GetAttr("GetDeterminantFromHistory"); dynamic df = new PandasConverter().GetDataFrame(history); return(GetDeterminantFromHistory(df)); } }
public void HandlesLeanDataReaderOutputForSpotMarkets(string securityType, string market, string resolution, string ticker, string fileName, int rowsInfile, double sumValue) { using (Py.GIL()) { // Arrange var dataFolder = "../../../Data"; var filepath = LeanDataReaderTests.GenerateFilepathForTesting(dataFolder, securityType, market, resolution, ticker, fileName); var leanDataReader = new LeanDataReader(filepath); var data = leanDataReader.Parse(); var converter = new PandasConverter(); // Act dynamic df = converter.GetDataFrame(data); // Assert Assert.AreEqual(rowsInfile, df.shape[0].AsManagedObject(typeof(int))); int columnsNumber = df.shape[1].AsManagedObject(typeof(int)); if (columnsNumber == 3 || columnsNumber == 6) { Assert.AreEqual(sumValue, df.get("lastprice").sum().AsManagedObject(typeof(double)), 1e-4); } else { Assert.AreEqual(sumValue, df.get("close").sum().AsManagedObject(typeof(double)), 1e-4); } } }
public void HandlesLeanDataReaderOutputForOptionAndFutures(string composedFilePath, Symbol symbol, int rowsInfile, double sumValue) { using (Py.GIL()) { // Arrange var leanDataReader = new LeanDataReader(composedFilePath); var data = leanDataReader.Parse(); var converter = new PandasConverter(); // Act dynamic df = converter.GetDataFrame(data); // Assert Assert.AreEqual(rowsInfile, df.shape[0].AsManagedObject(typeof(int))); int columnsNumber = df.shape[1].AsManagedObject(typeof(int)); if (columnsNumber == 3 || columnsNumber == 6) { Assert.AreEqual(sumValue, df.get("lastprice").sum().AsManagedObject(typeof(double)), 1e-4); } else if (columnsNumber == 1) { Assert.AreEqual(sumValue, df.get("openinterest").sum().AsManagedObject(typeof(double)), 1e-4); } else { Assert.AreEqual(sumValue, df.get("close").sum().AsManagedObject(typeof(double)), 1e-4); } } }
public void HandlesQuoteTicks() { var converter = new PandasConverter(); var symbol = Symbols.EURUSD; var rawBars = Enumerable .Range(0, 10) .Select(i => new Tick(DateTime.UtcNow.AddMilliseconds(100 * i), symbol, 0.99m, 1.01m)) .ToArray(); // GetDataFrame with argument of type IEnumerable<QuoteBar> dynamic dataFrame = converter.GetDataFrame(rawBars); using (Py.GIL()) { Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool))); var subDataFrame = dataFrame.loc[symbol]; Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool))); Assert.IsTrue(subDataFrame.get("askprice") != null); Assert.IsTrue(subDataFrame.get("exchange") == null); var count = subDataFrame.__len__().AsManagedObject(typeof(int)); Assert.AreEqual(count, 10); for (var i = 0; i < count; i++) { var index = subDataFrame.index[i]; var value = subDataFrame.loc[index].lastprice.AsManagedObject(typeof(decimal)); Assert.AreEqual(rawBars[i].LastPrice, value); } } // GetDataFrame with argument of type IEnumerable<QuoteBar> var history = GetHistory(symbol, Resolution.Tick, rawBars); dataFrame = converter.GetDataFrame(history); using (Py.GIL()) { Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool))); var subDataFrame = dataFrame.loc[symbol]; Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool))); Assert.IsTrue(subDataFrame.get("askprice") != null); Assert.IsTrue(subDataFrame.get("exchange") == null); var count = subDataFrame.__len__().AsManagedObject(typeof(int)); Assert.AreEqual(count, 10); for (var i = 0; i < count; i++) { var index = subDataFrame.index[i]; var value = subDataFrame.loc[index].askprice.AsManagedObject(typeof(decimal)); Assert.AreEqual(rawBars[i].AskPrice, value); } } }
/// <summary> /// <see cref = "QuantBook" /> constructor. /// Provides access to data for quantitative analysis /// </summary> public QuantBook() : base() { try { using (Py.GIL()) { _pandas = Py.Import("pandas"); } _converter = new PandasConverter(_pandas); // Create new instance of QCAlgorithm we are going to wrap _algorithm = new QCAlgorithm(); // By default, set start date to end data which is yesterday SetStartDate(_algorithm.EndDate); // Initialize History Provider var composer = new Composer(); var algorithmHandlers = LeanEngineAlgorithmHandlers.FromConfiguration(composer); _dataCacheProvider = new ZipDataCacheProvider(algorithmHandlers.DataProvider); var mapFileProvider = algorithmHandlers.MapFileProvider; _algorithm.HistoryProvider = composer.GetExportedValueByTypeName <IHistoryProvider>(Config.Get("history-provider", "SubscriptionDataReaderHistoryProvider")); _algorithm.HistoryProvider.Initialize(null, algorithmHandlers.DataProvider, _dataCacheProvider, mapFileProvider, algorithmHandlers.FactorFileProvider, null); } catch (Exception exception) { throw new Exception("QuantBook.Main(): " + exception); } }
private dynamic GetTestDataFrame(Symbol symbol) { var converter = new PandasConverter(); var rawBars = Enumerable .Range(0, 1) .Select(i => new Tick(symbol, $"1440{i:D2}00,167{i:D2}00,1{i:D2},T,T,0", new DateTime(2013, 10, 7))) .ToArray(); return(converter.GetDataFrame(rawBars)); }
public void HandlesTradeBars() { var converter = new PandasConverter(); var symbol = Symbols.SPY; var rawBars = Enumerable .Range(0, 10) .Select(i => new TradeBar(DateTime.UtcNow.AddMinutes(i), symbol, i + 101m, i + 102m, i + 100m, i + 101m, 0m)) .ToArray(); // GetDataFrame with argument of type IEnumerable<TradeBar> dynamic dataFrame = converter.GetDataFrame(rawBars); using (Py.GIL()) { Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool))); var subDataFrame = dataFrame.loc[symbol]; Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool))); var count = subDataFrame.__len__().AsManagedObject(typeof(int)); Assert.AreEqual(count, 10); for (var i = 0; i < count; i++) { var index = subDataFrame.index[i]; var close = subDataFrame.loc[index].close.AsManagedObject(typeof(decimal)); Assert.AreEqual(rawBars[i].Close, close); } } // GetDataFrame with argument of type IEnumerable<TradeBar> var history = GetHistory(symbol, Resolution.Minute, rawBars); dataFrame = converter.GetDataFrame(history); using (Py.GIL()) { Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool))); var subDataFrame = dataFrame.loc[symbol]; Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool))); var count = subDataFrame.__len__().AsManagedObject(typeof(int)); Assert.AreEqual(count, 10); for (var i = 0; i < count; i++) { var index = subDataFrame.index[i]; var close = subDataFrame.loc[index].close.AsManagedObject(typeof(decimal)); Assert.AreEqual(rawBars[i].Close, close); } } }
public void HandlesOpenInterestTicks([ValueSource(nameof(ResolutionCases))] Resolution resolution, [ValueSource(nameof(SymbolCases))] Symbol symbol) { // Arrange var converter = new PandasConverter(); var tickType = TickType.OpenInterest; var dataType = LeanData.GetDataType(resolution, tickType); var subcriptionDataConfig = new SubscriptionDataConfig(dataType, symbol, resolution, TimeZones.Chicago, TimeZones.Chicago, tickType: tickType, fillForward: false, extendedHours: true, isInternalFeed: true); var openinterest = new List <OpenInterest>(); for (int i = 0; i < 10; i++) { var line = $"{1000 * i},{11 * i}"; var openInterestTicks = new OpenInterest(subcriptionDataConfig, symbol, line, new DateTime(2017, 10, 10)); openinterest.Add(openInterestTicks); } // Act dynamic dataFrame = converter.GetDataFrame(openinterest); //Assert using (Py.GIL()) { Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool))); var subDataFrame = dataFrame.loc[symbol.Value]; Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool))); Assert.IsTrue(subDataFrame.get("openinterest") != null); var count = subDataFrame.shape[0].AsManagedObject(typeof(int)); Assert.AreEqual(count, 10); for (var i = 0; i < count; i++) { var index = subDataFrame.index[i]; var value = subDataFrame.loc[index].openinterest.AsManagedObject(typeof(decimal)); Assert.AreEqual(openinterest[i].Value, value); } } }
/// <summary> /// Gets the historical data of an indicator and convert it into pandas.DataFrame /// </summary> /// <param name="indicator">Indicator</param> /// <param name="history">Historical data used to calculate the indicator</param> /// <param name="selector">Selects a value from the BaseData to send into the indicator, if null defaults to the Value property of BaseData (x => x.Value)</param> /// <returns>pandas.DataFrame containing the historical data of <param name="indicator"></returns> private PyObject Indicator(IndicatorBase <IndicatorDataPoint> indicator, IEnumerable <IBaseData> history, Func <IBaseData, decimal> selector = null) { // Reset the indicator indicator.Reset(); // Create a dictionary of the properties var name = indicator.GetType().Name; var properties = indicator.GetType().GetProperties() .Where(x => x.PropertyType.IsGenericType) .ToDictionary(x => x.Name, y => new List <IndicatorDataPoint>()); properties.Add(name, new List <IndicatorDataPoint>()); indicator.Updated += (s, e) => { if (!indicator.IsReady) { return; } foreach (var kvp in properties) { var dataPoint = kvp.Key == name ? e : GetPropertyValue(s, kvp.Key + ".Current"); kvp.Value.Add((IndicatorDataPoint)dataPoint); } }; selector = selector ?? (x => x.Value); foreach (var bar in history) { var value = selector(bar); indicator.Update(bar.EndTime, value); } return(PandasConverter.GetIndicatorDataFrame(properties)); }
public void HandlesEmptyEnumerable() { var converter = new PandasConverter(); var rawBars = Enumerable.Empty <TradeBar>().ToArray(); // GetDataFrame with argument of type IEnumerable<TradeBar> dynamic dataFrame = converter.GetDataFrame(rawBars); using (Py.GIL()) { Assert.IsTrue(dataFrame.empty.AsManagedObject(typeof(bool))); } // GetDataFrame with argument of type IEnumerable<TradeBar> var history = GetHistory(Symbols.SPY, Resolution.Minute, rawBars); dataFrame = converter.GetDataFrame(history); using (Py.GIL()) { Assert.IsTrue(dataFrame.empty.AsManagedObject(typeof(bool))); } }
public void HandlesCustomDataBars(Type type, string format) { var converter = new PandasConverter(); var symbol = Symbols.LTCUSD; var config = GetSubscriptionDataConfig <Quandl>(symbol, Resolution.Daily); var custom = Activator.CreateInstance(type) as BaseData; if (type == typeof(Quandl)) { custom.Reader(config, "date,open,high,low,close,transactions", DateTime.UtcNow, false); } var rawBars = Enumerable .Range(0, 10) .Select(i => { var line = $"{DateTime.UtcNow.AddDays(i).ToString(format)},{i + 101},{i + 102},{i + 100},{i + 101},{i + 101}"; return(custom.Reader(config, line, DateTime.UtcNow.AddDays(i), false)); }) .ToArray(); // GetDataFrame with argument of type IEnumerable<BaseData> dynamic dataFrame = converter.GetDataFrame(rawBars); using (Py.GIL()) { Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool))); var subDataFrame = dataFrame.loc[symbol]; Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool))); var count = subDataFrame.__len__().AsManagedObject(typeof(int)); Assert.AreEqual(count, 10); for (var i = 0; i < count; i++) { var index = subDataFrame.index[i]; var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal)); Assert.AreEqual(rawBars[i].Value, value); var transactions = subDataFrame.loc[index].transactions.AsManagedObject(typeof(decimal)); var expected = (rawBars[i] as DynamicData)?.GetProperty("transactions"); expected = expected ?? type.GetProperty("Transactions")?.GetValue(rawBars[i]); Assert.AreEqual(expected, transactions); } } // GetDataFrame with argument of type IEnumerable<BaseData> var history = GetHistory(symbol, Resolution.Daily, rawBars); dataFrame = converter.GetDataFrame(history); using (Py.GIL()) { Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool))); var subDataFrame = dataFrame.loc[symbol]; Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool))); var count = subDataFrame.__len__().AsManagedObject(typeof(int)); Assert.AreEqual(10, count); for (var i = 0; i < count; i++) { var index = subDataFrame.index[i]; var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal)); Assert.AreEqual(rawBars[i].Value, value); var transactions = subDataFrame.loc[index].transactions.AsManagedObject(typeof(decimal)); var expected = (rawBars[i] as DynamicData)?.GetProperty("transactions"); expected = expected ?? type.GetProperty("Transactions")?.GetValue(rawBars[i]); Assert.AreEqual(expected, transactions); } } }
/// <summary> /// Sets pandas converter /// </summary> public void SetPandasConverter() { PandasConverter = new PandasConverter(); }
/// <summary> /// Create a new instance of <see cref="FutureHistory"/>. /// </summary> /// <param name="data"></param> public FutureHistory(IEnumerable <Slice> data) { _data = data; _converter = new PandasConverter(); _dataframe = _converter.GetDataFrame(_data); }
public void HandlesCustomDataBars() { var converter = new PandasConverter(); var symbol = Symbols.LTCUSD; var config = GetSubscriptionDataConfig <Quandl>(symbol, Resolution.Daily); var quandl = new Quandl(); quandl.Reader(config, "date,open,high,low,close,settle", DateTime.UtcNow, false); var rawBars = Enumerable .Range(0, 10) .Select(i => { var line = $"{DateTime.UtcNow.AddDays(i).ToString("yyyy-MM-dd")},{i + 101},{i + 102},{i + 100},{i + 101},{i + 101}"; return(quandl.Reader(config, line, DateTime.UtcNow.AddDays(i), false)); }) .ToArray(); // GetDataFrame with argument of type IEnumerable<BaseData> dynamic dataFrame = converter.GetDataFrame(rawBars); using (Py.GIL()) { Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool))); var subDataFrame = dataFrame.loc[symbol]; Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool))); var count = subDataFrame.__len__().AsManagedObject(typeof(int)); Assert.AreEqual(count, 10); for (var i = 0; i < count; i++) { var index = subDataFrame.index[i]; var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal)); Assert.AreEqual(rawBars[i].Value, value); var settle = subDataFrame.loc[index].settle.AsManagedObject(typeof(decimal)); Assert.AreEqual(((DynamicData)rawBars[i]).GetProperty("settle"), settle); } } // GetDataFrame with argument of type IEnumerable<BaseData> var history = GetHistory(symbol, Resolution.Daily, rawBars); dataFrame = converter.GetDataFrame(history); using (Py.GIL()) { Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool))); var subDataFrame = dataFrame.loc[symbol]; Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool))); var count = subDataFrame.__len__().AsManagedObject(typeof(int)); Assert.AreEqual(count, 10); for (var i = 0; i < count; i++) { var index = subDataFrame.index[i]; var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal)); Assert.AreEqual(rawBars[i].Value, value); var settle = subDataFrame.loc[index].settle.AsManagedObject(typeof(decimal)); Assert.AreEqual(((DynamicData)rawBars[i]).GetProperty("settle"), settle); } } }
/// <summary> /// Gets the historical data for the specified symbol over the request span. The symbol must exist in the Securities collection. /// </summary> /// <typeparam name="T">The data type of the symbol</typeparam> /// <param name="symbol">The symbol to retrieve historical data for</param> /// <param name="span">The span over which to retrieve recent historical data</param> /// <param name="resolution">The resolution to request</param> /// <returns>An enumerable of slice containing the requested historical data</returns> public new PyObject History<T>(Symbol symbol, TimeSpan span, Resolution? resolution = null) where T : IBaseData { return PandasConverter.GetDataFrame(History<T>(symbol, Time - span, Time, resolution).Memoize()); }
public void HandlesCustomDataBarsInheritsFromTradeBar(Type type, string propertyName) { var converter = new PandasConverter(); var symbol = Symbols.LTCUSD; var config = GetSubscriptionDataConfig <Quandl>(symbol, Resolution.Daily); dynamic custom = Activator.CreateInstance(type); var rawBars = Enumerable .Range(0, 10) .Select(i => { var line = $"{DateTime.UtcNow.AddDays(i).ToStringInvariant("yyyyMMdd HH:mm")},{i + 101},{i + 102},{i + 100},{i + 101},{i + 101}"; return(custom.Reader(config, line, DateTime.UtcNow.AddDays(i), false) as BaseData); }) .ToArray(); // GetDataFrame with argument of type IEnumerable<BaseData> dynamic dataFrame = converter.GetDataFrame(rawBars); using (Py.GIL()) { Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool))); var subDataFrame = dataFrame.loc[symbol]; Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool))); var count = subDataFrame.__len__().AsManagedObject(typeof(int)); Assert.AreEqual(count, 10); for (var i = 0; i < count; i++) { var index = subDataFrame.index[i]; var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal)); Assert.AreEqual(rawBars[i].Value, value); var transactions = subDataFrame.loc[index][propertyName.ToLowerInvariant()].AsManagedObject(typeof(decimal)); var expected = type.GetProperty(propertyName)?.GetValue(rawBars[i]); Assert.AreEqual(expected, transactions); } } // GetDataFrame with argument of type IEnumerable<BaseData> var history = GetHistory(symbol, Resolution.Daily, rawBars); dataFrame = converter.GetDataFrame(history); using (Py.GIL()) { Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool))); var subDataFrame = dataFrame.loc[symbol]; Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool))); var count = subDataFrame.__len__().AsManagedObject(typeof(int)); Assert.AreEqual(10, count); for (var i = 0; i < count; i++) { var index = subDataFrame.index[i]; var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal)); Assert.AreEqual(rawBars[i].Value, value); } } }