public void HandlesQuoteTicks()
{
var converter = new PandasConverter();
var symbol = Symbols.EURUSD;
var rawBars = Enumerable
.Range(0, 10)
.Select(i => new Tick(DateTime.UtcNow.AddMilliseconds(100 * i), symbol, 0.99m, 1.01m))
.ToArray();
// GetDataFrame with argument of type IEnumerable<QuoteBar>
dynamic dataFrame = converter.GetDataFrame(rawBars);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
Assert.IsTrue(subDataFrame.get("askprice") != null);
Assert.IsTrue(subDataFrame.get("exchange") == null);
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].lastprice.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].LastPrice, value);
}
}
// GetDataFrame with argument of type IEnumerable<QuoteBar>
var history = GetHistory(symbol, Resolution.Tick, rawBars);
dataFrame = converter.GetDataFrame(history);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
Assert.IsTrue(subDataFrame.get("askprice") != null);
Assert.IsTrue(subDataFrame.get("exchange") == null);
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].askprice.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].AskPrice, value);
}
}
}
public void HandlesTradeBars()
{
var converter = new PandasConverter();
var symbol = Symbols.SPY;
var rawBars = Enumerable
.Range(0, 10)
.Select(i => new TradeBar(DateTime.UtcNow.AddMinutes(i), symbol, i + 101m, i + 102m, i + 100m, i + 101m, 0m))
.ToArray();
// GetDataFrame with argument of type IEnumerable<TradeBar>
dynamic dataFrame = converter.GetDataFrame(rawBars);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var close = subDataFrame.loc[index].close.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Close, close);
}
}
// GetDataFrame with argument of type IEnumerable<TradeBar>
var history = GetHistory(symbol, Resolution.Minute, rawBars);
dataFrame = converter.GetDataFrame(history);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var close = subDataFrame.loc[index].close.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Close, close);
}
}
}
/// <summary>
/// Gets the historical data for the specified symbol. The exact number of bars will be returned.
/// The symbol must exist in the Securities collection.
/// </summary>
/// <param name="tickers">The symbols to retrieve historical data for</param>
/// <param name="periods">The number of bars to request</param>
/// <param name="resolution">The resolution to request</param>
/// <returns>A python dictionary with pandas DataFrame containing the requested historical data</returns>
public PyObject History(PyObject tickers, int periods, Resolution?resolution = null)
{
var symbols = GetSymbolsFromPyObject(tickers);
if (symbols == null)
{
return(null);
}
return(_converter.GetDataFrame(History(symbols, periods, resolution)));
}
public void HandlesLeanDataReaderOutputForOptionAndFutures(string composedFilePath, Symbol symbol, int rowsInfile, double sumValue)
{
using (Py.GIL())
{
// Arrange
var leanDataReader = new LeanDataReader(composedFilePath);
var data = leanDataReader.Parse();
var converter = new PandasConverter();
// Act
dynamic df = converter.GetDataFrame(data);
// Assert
Assert.AreEqual(rowsInfile, df.shape[0].AsManagedObject(typeof(int)));
int columnsNumber = df.shape[1].AsManagedObject(typeof(int));
if (columnsNumber == 3 || columnsNumber == 6)
{
Assert.AreEqual(sumValue, df.get("lastprice").sum().AsManagedObject(typeof(double)), 1e-4);
}
else if (columnsNumber == 1)
{
Assert.AreEqual(sumValue, df.get("openinterest").sum().AsManagedObject(typeof(double)), 1e-4);
}
else
{
Assert.AreEqual(sumValue, df.get("close").sum().AsManagedObject(typeof(double)), 1e-4);
}
}
}
public void HandlesLeanDataReaderOutputForSpotMarkets(string securityType, string market, string resolution, string ticker, string fileName, int rowsInfile, double sumValue)
{
using (Py.GIL())
{
// Arrange
var dataFolder = "../../../Data";
var filepath = LeanDataReaderTests.GenerateFilepathForTesting(dataFolder, securityType, market, resolution, ticker, fileName);
var leanDataReader = new LeanDataReader(filepath);
var data = leanDataReader.Parse();
var converter = new PandasConverter();
// Act
dynamic df = converter.GetDataFrame(data);
// Assert
Assert.AreEqual(rowsInfile, df.shape[0].AsManagedObject(typeof(int)));
int columnsNumber = df.shape[1].AsManagedObject(typeof(int));
if (columnsNumber == 3 || columnsNumber == 6)
{
Assert.AreEqual(sumValue, df.get("lastprice").sum().AsManagedObject(typeof(double)), 1e-4);
}
else
{
Assert.AreEqual(sumValue, df.get("close").sum().AsManagedObject(typeof(double)), 1e-4);
}
}
}
private dynamic GetTestDataFrame(Symbol symbol)
{
var converter = new PandasConverter();
var rawBars = Enumerable
.Range(0, 1)
.Select(i => new Tick(symbol, $"1440{i:D2}00,167{i:D2}00,1{i:D2},T,T,0", new DateTime(2013, 10, 7)))
.ToArray();
return(converter.GetDataFrame(rawBars));
}
public void HandlesEmptyEnumerable()
{
var converter = new PandasConverter();
var rawBars = Enumerable.Empty <TradeBar>().ToArray();
// GetDataFrame with argument of type IEnumerable<TradeBar>
dynamic dataFrame = converter.GetDataFrame(rawBars);
using (Py.GIL())
{
Assert.IsTrue(dataFrame.empty.AsManagedObject(typeof(bool)));
}
// GetDataFrame with argument of type IEnumerable<TradeBar>
var history = GetHistory(Symbols.SPY, Resolution.Minute, rawBars);
dataFrame = converter.GetDataFrame(history);
using (Py.GIL())
{
Assert.IsTrue(dataFrame.empty.AsManagedObject(typeof(bool)));
}
}
public void HandlesOpenInterestTicks([ValueSource(nameof(ResolutionCases))] Resolution resolution, [ValueSource(nameof(SymbolCases))] Symbol symbol)
{
// Arrange
var converter = new PandasConverter();
var tickType = TickType.OpenInterest;
var dataType = LeanData.GetDataType(resolution, tickType);
var subcriptionDataConfig = new SubscriptionDataConfig(dataType, symbol, resolution,
TimeZones.Chicago, TimeZones.Chicago,
tickType: tickType, fillForward: false,
extendedHours: true, isInternalFeed: true);
var openinterest = new List <OpenInterest>();
for (int i = 0; i < 10; i++)
{
var line = $"{1000 * i},{11 * i}";
var openInterestTicks = new OpenInterest(subcriptionDataConfig, symbol, line, new DateTime(2017, 10, 10));
openinterest.Add(openInterestTicks);
}
// Act
dynamic dataFrame = converter.GetDataFrame(openinterest);
//Assert
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol.Value];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
Assert.IsTrue(subDataFrame.get("openinterest") != null);
var count = subDataFrame.shape[0].AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].openinterest.AsManagedObject(typeof(decimal));
Assert.AreEqual(openinterest[i].Value, value);
}
}
}
/// <summary>
/// Create a new instance of <see cref="FutureHistory"/>.
/// </summary>
/// <param name="data"></param>
public FutureHistory(IEnumerable <Slice> data)
{
_data = data;
_converter = new PandasConverter();
_dataframe = _converter.GetDataFrame(_data);
}
public void HandlesCustomDataBars()
{
var converter = new PandasConverter();
var symbol = Symbols.LTCUSD;
var config = GetSubscriptionDataConfig <Quandl>(symbol, Resolution.Daily);
var quandl = new Quandl();
quandl.Reader(config, "date,open,high,low,close,settle", DateTime.UtcNow, false);
var rawBars = Enumerable
.Range(0, 10)
.Select(i =>
{
var line = $"{DateTime.UtcNow.AddDays(i).ToString("yyyy-MM-dd")},{i + 101},{i + 102},{i + 100},{i + 101},{i + 101}";
return(quandl.Reader(config, line, DateTime.UtcNow.AddDays(i), false));
})
.ToArray();
// GetDataFrame with argument of type IEnumerable<BaseData>
dynamic dataFrame = converter.GetDataFrame(rawBars);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
var settle = subDataFrame.loc[index].settle.AsManagedObject(typeof(decimal));
Assert.AreEqual(((DynamicData)rawBars[i]).GetProperty("settle"), settle);
}
}
// GetDataFrame with argument of type IEnumerable<BaseData>
var history = GetHistory(symbol, Resolution.Daily, rawBars);
dataFrame = converter.GetDataFrame(history);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
var settle = subDataFrame.loc[index].settle.AsManagedObject(typeof(decimal));
Assert.AreEqual(((DynamicData)rawBars[i]).GetProperty("settle"), settle);
}
}
}
public void HandlesCustomDataBars(Type type, string format)
{
var converter = new PandasConverter();
var symbol = Symbols.LTCUSD;
var config = GetSubscriptionDataConfig <Quandl>(symbol, Resolution.Daily);
var custom = Activator.CreateInstance(type) as BaseData;
if (type == typeof(Quandl))
{
custom.Reader(config, "date,open,high,low,close,transactions", DateTime.UtcNow, false);
}
var rawBars = Enumerable
.Range(0, 10)
.Select(i =>
{
var line = $"{DateTime.UtcNow.AddDays(i).ToString(format)},{i + 101},{i + 102},{i + 100},{i + 101},{i + 101}";
return(custom.Reader(config, line, DateTime.UtcNow.AddDays(i), false));
})
.ToArray();
// GetDataFrame with argument of type IEnumerable<BaseData>
dynamic dataFrame = converter.GetDataFrame(rawBars);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
var transactions = subDataFrame.loc[index].transactions.AsManagedObject(typeof(decimal));
var expected = (rawBars[i] as DynamicData)?.GetProperty("transactions");
expected = expected ?? type.GetProperty("Transactions")?.GetValue(rawBars[i]);
Assert.AreEqual(expected, transactions);
}
}
// GetDataFrame with argument of type IEnumerable<BaseData>
var history = GetHistory(symbol, Resolution.Daily, rawBars);
dataFrame = converter.GetDataFrame(history);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(10, count);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
var transactions = subDataFrame.loc[index].transactions.AsManagedObject(typeof(decimal));
var expected = (rawBars[i] as DynamicData)?.GetProperty("transactions");
expected = expected ?? type.GetProperty("Transactions")?.GetValue(rawBars[i]);
Assert.AreEqual(expected, transactions);
}
}
}
/// <summary>
/// Gets the historical data for the specified symbols between the specified dates. The symbols must exist in the Securities collection.
/// </summary>
/// <param name="span">The span over which to retrieve recent historical data</param>
/// <param name="resolution">The resolution to request</param>
/// <returns>A pandas.DataFrame containing the requested historical data</returns>
public PyObject History(TimeSpan span, Resolution?resolution = null)
{
return(_converter.GetDataFrame(_algorithm.History(_algorithm.Securities.Keys, _algorithm.Time - span, _algorithm.Time, resolution).Memoize()));
}
public void HandlesCustomDataBarsInheritsFromTradeBar(Type type, string propertyName)
{
var converter = new PandasConverter();
var symbol = Symbols.LTCUSD;
var config = GetSubscriptionDataConfig <Quandl>(symbol, Resolution.Daily);
dynamic custom = Activator.CreateInstance(type);
var rawBars = Enumerable
.Range(0, 10)
.Select(i =>
{
var line = $"{DateTime.UtcNow.AddDays(i).ToStringInvariant("yyyyMMdd HH:mm")},{i + 101},{i + 102},{i + 100},{i + 101},{i + 101}";
return(custom.Reader(config, line, DateTime.UtcNow.AddDays(i), false) as BaseData);
})
.ToArray();
// GetDataFrame with argument of type IEnumerable<BaseData>
dynamic dataFrame = converter.GetDataFrame(rawBars);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(count, 10);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
var transactions = subDataFrame.loc[index][propertyName.ToLowerInvariant()].AsManagedObject(typeof(decimal));
var expected = type.GetProperty(propertyName)?.GetValue(rawBars[i]);
Assert.AreEqual(expected, transactions);
}
}
// GetDataFrame with argument of type IEnumerable<BaseData>
var history = GetHistory(symbol, Resolution.Daily, rawBars);
dataFrame = converter.GetDataFrame(history);
using (Py.GIL())
{
Assert.IsFalse(dataFrame.empty.AsManagedObject(typeof(bool)));
var subDataFrame = dataFrame.loc[symbol];
Assert.IsFalse(subDataFrame.empty.AsManagedObject(typeof(bool)));
var count = subDataFrame.__len__().AsManagedObject(typeof(int));
Assert.AreEqual(10, count);
for (var i = 0; i < count; i++)
{
var index = subDataFrame.index[i];
var value = subDataFrame.loc[index].value.AsManagedObject(typeof(decimal));
Assert.AreEqual(rawBars[i].Value, value);
}
}
}
/// <summary>
/// Gets the historical data for the specified symbol over the request span. The symbol must exist in the Securities collection.
/// </summary>
/// <typeparam name="T">The data type of the symbol</typeparam>
/// <param name="symbol">The symbol to retrieve historical data for</param>
/// <param name="span">The span over which to retrieve recent historical data</param>
/// <param name="resolution">The resolution to request</param>
/// <returns>An enumerable of slice containing the requested historical data</returns>
public new PyObject History<T>(Symbol symbol, TimeSpan span, Resolution? resolution = null)
where T : IBaseData
{
return PandasConverter.GetDataFrame(History<T>(symbol, Time - span, Time, resolution).Memoize());
}
