private string LoadFutureData(Symbol future, TickType tickType, Resolution res)
{
var dataType = LeanData.GetDataType(res, tickType);
var config = new SubscriptionDataConfig(dataType, future, res,
TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, tickType);
var date = _fromDate;
var sb = new StringBuilder();
while (date <= _toDate)
{
var leanDataReader = new LeanDataReader(config, future, res, date, _dataDirectory);
foreach (var bar in leanDataReader.Parse())
{
//write base data type back to string
sb.AppendLine(LeanData.GenerateLine(bar, SecurityType.Future, res));
}
date = date.AddDays(1);
}
var csv = sb.ToString();
return(csv);
}
public void ReadsEntireZipFileEntries_Trade()
{
var baseFuture = Symbol.CreateFuture(Futures.Indices.SP500EMini, Market.CME, SecurityIdentifier.DefaultDate);
var filePath = LeanData.GenerateZipFilePath(Globals.DataFolder, baseFuture, new DateTime(2013, 10, 06), Resolution.Minute, TickType.Trade);
var leanDataReader = new LeanDataReader(filePath);
var data = leanDataReader.Parse()
.ToList()
.GroupBy(baseData => baseData.Symbol)
.Select(grp => grp.ToList())
.OrderBy(list => list[0].Symbol)
.ToList();
Assert.AreEqual(2, data.Count);
foreach (var dataForSymbol in data)
{
Assert.IsTrue(dataForSymbol[0] is TradeBar);
Assert.IsFalse(dataForSymbol[0].Symbol.IsCanonical());
Assert.AreEqual(Futures.Indices.SP500EMini, dataForSymbol[0].Symbol.ID.Symbol);
}
Assert.AreEqual(118, data[0].Count);
Assert.AreEqual(10, data[1].Count);
}
public void HandlesLeanDataReaderOutputForOptionAndFutures(string composedFilePath, Symbol symbol, int rowsInfile, double sumValue)
{
using (Py.GIL())
{
// Arrange
var leanDataReader = new LeanDataReader(composedFilePath);
var data = leanDataReader.Parse();
var converter = new PandasConverter();
// Act
dynamic df = converter.GetDataFrame(data);
// Assert
Assert.AreEqual(rowsInfile, df.shape[0].AsManagedObject(typeof(int)));
int columnsNumber = df.shape[1].AsManagedObject(typeof(int));
if (columnsNumber == 3 || columnsNumber == 6)
{
Assert.AreEqual(sumValue, df.get("lastprice").sum().AsManagedObject(typeof(double)), 1e-4);
}
else if (columnsNumber == 1)
{
Assert.AreEqual(sumValue, df.get("openinterest").sum().AsManagedObject(typeof(double)), 1e-4);
}
else
{
Assert.AreEqual(sumValue, df.get("close").sum().AsManagedObject(typeof(double)), 1e-4);
}
}
}
public void HandlesLeanDataReaderOutputForSpotMarkets(string securityType, string market, string resolution, string ticker, string fileName, int rowsInfile, double sumValue)
{
using (Py.GIL())
{
// Arrange
var dataFolder = "../../../Data";
var filepath = LeanDataReaderTests.GenerateFilepathForTesting(dataFolder, securityType, market, resolution, ticker, fileName);
var leanDataReader = new LeanDataReader(filepath);
var data = leanDataReader.Parse();
var converter = new PandasConverter();
// Act
dynamic df = converter.GetDataFrame(data);
// Assert
Assert.AreEqual(rowsInfile, df.shape[0].AsManagedObject(typeof(int)));
int columnsNumber = df.shape[1].AsManagedObject(typeof(int));
if (columnsNumber == 3 || columnsNumber == 6)
{
Assert.AreEqual(sumValue, df.get("lastprice").sum().AsManagedObject(typeof(double)), 1e-4);
}
else
{
Assert.AreEqual(sumValue, df.get("close").sum().AsManagedObject(typeof(double)), 1e-4);
}
}
}
public void LoadsEquity_Daily_SingleEntryZip()
{
var dataPath = LeanData.GenerateZipFilePath(Globals.DataFolder, Symbols.AAPL, DateTime.UtcNow, Resolution.Daily, TickType.Trade);
var leanDataReader = new LeanDataReader(dataPath);
var data = leanDataReader.Parse().ToList();
Assert.AreEqual(5849, data.Count);
Assert.IsTrue(data.All(baseData => baseData.Symbol == Symbols.AAPL && baseData is TradeBar));
}
public void ReadLeanFutureAndOptionDataFromFilePath(string composedFilePath, Symbol symbol, int rowsInfile, double sumValue)
{
// Act
var ldr = new LeanDataReader(composedFilePath);
var data = ldr.Parse().ToList();
// Assert
Assert.True(symbol.Equals(data.First().Symbol));
Assert.AreEqual(rowsInfile, data.Count);
Assert.AreEqual(sumValue, data.Sum(c => c.Value));
}
public void CanDownloadAndSave(SecurityType securityType, TickType tickType, Resolution resolution)
{
var symbol = Symbols.GetBySecurityType(securityType);
var startTimeUtc = GetRepoDataDates(securityType, resolution);
// Override for this case because symbol from Symbols does not have data included
if (securityType == SecurityType.Option)
{
symbol = Symbols.CreateOptionSymbol("GOOG", OptionRight.Call, 770, new DateTime(2015, 12, 24));
startTimeUtc = new DateTime(2015, 12, 23);
}
// EndTime based on start, only do 1 day for anything less than hour because we compare datafiles below
// and minute and finer resolutions store by day
var endTimeUtc = startTimeUtc + TimeSpan.FromDays(resolution >= Resolution.Hour ? 15 : 1);
// Create our writer and LocalHistory brokerage to "download" from
var writer = new LeanDataWriter(_dataDirectory, resolution, securityType, tickType);
var brokerage = new LocalHistoryBrokerage();
var symbols = new List <Symbol>()
{
symbol
};
// "Download" and write to file
writer.DownloadAndSave(brokerage, symbols, startTimeUtc, endTimeUtc);
// Verify the file exists where we expect
var filePath = LeanData.GenerateZipFilePath(_dataDirectory, symbol, startTimeUtc, resolution, tickType);
Assert.IsTrue(File.Exists(filePath));
// Read the file and data
var reader = new LeanDataReader(filePath);
var dataFromFile = reader.Parse().ToList();
// Ensure its not empty and it is actually for this symbol
Assert.IsNotEmpty(dataFromFile);
Assert.IsTrue(dataFromFile.All(x => x.Symbol == symbol));
// Get history directly ourselves and compare with the data in the file
var history = GetHistory(brokerage, resolution, securityType, symbol, tickType, startTimeUtc, endTimeUtc);
CollectionAssert.AreEqual(history.Select(x => x.Time), dataFromFile.Select(x => x.Time));
brokerage.Dispose();
}
public void ReadLeanSpotMarketsSecuritiesDataFromFilePath(string securityType, string market, string resolution, string ticker, string fileName, int rowsInfile, double sumValue)
{
// Arrange
var filepath = GenerateFilepathForTesting(_dataDirectory, securityType, market, resolution, ticker, fileName);
SecurityType securityTypeEnum;
Enum.TryParse(securityType, true, out securityTypeEnum);
var symbol = Symbol.Create(ticker, securityTypeEnum, market);
// Act
var ldr = new LeanDataReader(filepath);
var data = ldr.Parse().ToList();
// Assert
Assert.True(symbol.Equals(data.First().Symbol));
Assert.AreEqual(rowsInfile, data.Count);
Assert.AreEqual(sumValue, data.Sum(c => c.Value));
}
private void ConvertMinuteFuturesData(Symbol canonical, TickType tickType, Resolution outputResolution, Resolution inputResolution = Resolution.Minute)
{
var timeSpans = new Dictionary <Resolution, TimeSpan>()
{
{ Resolution.Daily, TimeSpan.FromHours(24) },
{ Resolution.Hour, TimeSpan.FromHours(1) },
};
var timeSpan = timeSpans[outputResolution];
var tickTypeConsolidatorMap = new Dictionary <TickType, Func <IDataConsolidator> >()
{
{ TickType.Quote, () => new QuoteBarConsolidator(timeSpan) },
{ TickType.OpenInterest, () => new OpenInterestConsolidator(timeSpan) },
{ TickType.Trade, () => new TradeBarConsolidator(timeSpan) }
};
var consolidators = new Dictionary <string, IDataConsolidator>();
var configs = new Dictionary <string, SubscriptionDataConfig>();
var outputFiles = new Dictionary <string, StringBuilder>();
var futures = new Dictionary <string, Symbol>();
var date = _fromDate;
while (date <= _toDate)
{
var futureChain = LoadFutureChain(canonical, date, tickType, inputResolution);
foreach (var future in futureChain)
{
if (!futures.ContainsKey(future.Value))
{
futures[future.Value] = future;
var config = new SubscriptionDataConfig(LeanData.GetDataType(outputResolution, tickType),
future, inputResolution, TimeZones.NewYork, TimeZones.NewYork,
false, false, false, false, tickType);
configs[future.Value] = config;
consolidators[future.Value] = tickTypeConsolidatorMap[tickType].Invoke();
var sb = new StringBuilder();
outputFiles[future.Value] = sb;
consolidators[future.Value].DataConsolidated += (sender, bar) =>
{
sb.Append(LeanData.GenerateLine(bar, SecurityType.Future, outputResolution) + Environment.NewLine);
};
}
var leanDataReader = new LeanDataReader(configs[future.Value], future, inputResolution, date, _dataDirectory);
var consolidator = consolidators[future.Value];
foreach (var bar in leanDataReader.Parse())
{
consolidator.Update(bar);
}
}
date = date.AddDays(1);
}
//write all results
foreach (var consolidator in consolidators.Values)
{
consolidator.Scan(date);
}
var zip = LeanData.GenerateRelativeZipFilePath(canonical, _fromDate, outputResolution, tickType);
var zipPath = Path.Combine(_dataDirectory, zip);
var fi = new FileInfo(zipPath);
if (!fi.Directory.Exists)
{
fi.Directory.Create();
}
foreach (var future in futures.Values)
{
var zipEntry = LeanData.GenerateZipEntryName(future, _fromDate, outputResolution, tickType);
var sb = outputFiles[future.Value];
//Uncomment to write zip files
//QuantConnect.Compression.ZipCreateAppendData(zipPath, zipEntry, sb.ToString());
Assert.IsTrue(sb.Length > 0);
}
}