/// <summary>
/// Write this file to disk.
/// </summary>
/// <param name="filePath">The full path to the new file</param>
/// <param name="data">The data to write as a string</param>
/// <param name="date">The date the data represents</param>
private void WriteFile(string filePath, IEnumerable <string> data, DateTime date)
{
var tempFilePath = filePath + ".tmp";
if (File.Exists(filePath) && !_appendToZips)
{
File.Delete(filePath);
Log.Trace("LeanDataWriter.Write(): Existing deleted: " + filePath);
}
// Create the directory if it doesnt exist
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
if (_appendToZips)
{
var entryName = LeanData.GenerateZipEntryName(_symbol, date, _resolution, _tickType);
Compression.ZipCreateAppendData(filePath, entryName, string.Join(Environment.NewLine, data), true);
Log.Trace("LeanDataWriter.Write(): Appended: " + filePath);
}
else
{
// Write out this data string to a zip file
Compression.ZipData(tempFilePath, LeanData.GenerateZipEntryName(_symbol, date, _resolution, _tickType), data);
// Move temp file to the final destination with the appropriate name
File.Move(tempFilePath, filePath);
Log.Trace("LeanDataWriter.Write(): Created: " + filePath);
}
}
/// <summary>
/// The LeanDataReader constructor
/// </summary>
/// <param name="config">The <see cref="SubscriptionDataConfig"/></param>
/// <param name="symbol">The <see cref="Symbol"/> that will be read</param>
/// <param name="resolution">The <see cref="Resolution"/> that will be read</param>
/// <param name="date">The <see cref="DateTime"/> that will be read</param>
/// <param name="dataFolder">The root data folder</param>
public LeanDataReader(SubscriptionDataConfig config, Symbol symbol, Resolution resolution, DateTime date, string dataFolder)
{
_date = date;
_zipPath = LeanData.GenerateZipFilePath(dataFolder, symbol, date, resolution, config.TickType);
_zipentry = LeanData.GenerateZipEntryName(symbol, date, resolution, config.TickType);
_config = config;
}
/// <summary>
/// Write this file to disk with the given data.
/// </summary>
/// <param name="filePath">The full path to the new file</param>
/// <param name="data">The data to write as a list of dates and strings</param>
/// <remarks>The reason we have the data as IEnumerable(DateTime, string) is to support
/// a generic write that works for all resolutions. In order to merge in hour/daily case I need the
/// date of the data to correctly merge the two. In order to support writing ticks I need to allow
/// two data points to have the same time. Thus I cannot use a single list of just strings nor
/// a sorted dictionary of DateTimes and strings. </remarks>
private void WriteFile(string filePath, List <TimedLine> data)
{
if (data == null || data.Count == 0)
{
return;
}
var date = data[0].Time;
// Generate this csv entry name
var entryName = LeanData.GenerateZipEntryName(_symbol, date, _resolution, _tickType);
// Check disk once for this file ahead of time, reuse where possible
var fileExists = File.Exists(filePath);
// If our file doesn't exist its possible the directory doesn't exist, make sure at least the directory exists
if (!fileExists)
{
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
}
// Handle merging of files
// Only merge on files with hour/daily resolution, that exist, and can be loaded
string finalData = null;
if (_writePolicy == WritePolicy.Append)
{
var streamWriter = new ZipStreamWriter(filePath, entryName);
foreach (var tuple in data)
{
streamWriter.WriteLine(tuple.Line);
}
streamWriter.DisposeSafely();
}
else if (_writePolicy == WritePolicy.Merge && fileExists && TryLoadFile(filePath, entryName, date, out var rows))
{
// Preform merge on loaded rows
foreach (var timedLine in data)
{
rows[timedLine.Time] = timedLine.Line;
}
// Final merged data product
finalData = string.Join("\n", rows.Values);
}
else
{
// Otherwise just extract the data from the given list.
finalData = string.Join("\n", data.Select(x => x.Line));
}
if (finalData != null)
{
var bytes = Encoding.UTF8.GetBytes(finalData);
_dataCacheProvider.Store($"{filePath}#{entryName}", bytes);
}
Log.Debug($"LeanDataWriter.Write(): Appended: {filePath} @ {entryName}");
}
private void SaveMinuteOrSecondOrTick(
List <Symbol> symbols,
DateTime startTimeUtc,
DateTime endTimeUtc,
Symbol canonicalSymbol,
IReadOnlyDictionary <Symbol, List <IGrouping <DateTime, BaseData> > > historyBySymbol)
{
var date = startTimeUtc;
while (date <= endTimeUtc)
{
var zipFileName = Path.Combine(
Globals.DataFolder,
LeanData.GenerateRelativeZipFilePath(canonicalSymbol, date, _resolution, _tickType));
var folder = Path.GetDirectoryName(zipFileName);
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
if (File.Exists(zipFileName))
{
File.Delete(zipFileName);
}
using (var zip = new ZipFile(zipFileName))
{
foreach (var symbol in symbols)
{
var zipEntryName = LeanData.GenerateZipEntryName(symbol, date, _resolution, _tickType);
foreach (var group in historyBySymbol[symbol])
{
if (group.Key == date)
{
var sb = new StringBuilder();
foreach (var row in group)
{
var line = LeanData.GenerateLine(row, _securityType, _resolution);
sb.AppendLine(line);
}
zip.AddEntry(zipEntryName, sb.ToString());
break;
}
}
}
if (zip.Count > 0)
{
zip.Save();
}
}
date = date.AddDays(1);
}
}
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
{
if (isLiveMode)
{
return(new SubscriptionDataSource(string.Empty, SubscriptionTransportMedium.LocalFile));
}
var source = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, config.Resolution, config.TickType);
if (config.SecurityType == SecurityType.Option)
{
source += "#" + LeanData.GenerateZipEntryName(config.Symbol, date, config.Resolution, config.TickType);
}
return(new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.Csv));
}
/// <summary>
/// Write this file to disk
/// </summary>
private void WriteFile(string fileName, string data, DateTime time)
{
data = data.TrimEnd();
if (File.Exists(fileName))
{
File.Delete(fileName);
Log.Trace("LeanDataWriter.Write(): Existing deleted: " + fileName);
}
// Create the directory if it doesnt exist
Directory.CreateDirectory(Path.GetDirectoryName(fileName));
// Write out this data string to a zip file
Compression.Zip(data, fileName, LeanData.GenerateZipEntryName(_symbol.Value, _securityType, time, _resolution, _dataType));
Log.Trace("LeanDataWriter.Write(): Created: " + fileName);
}
public void OptionZipFilePathWithUnderlyingFuture(string futureOptionTicker, string expectedFutureOptionTicker)
{
var underlying = Symbol.CreateFuture(futureOptionTicker, Market.CME, new DateTime(2021, 3, 19));
var optionSymbol = Symbol.CreateOption(
underlying,
Market.CME,
OptionStyle.American,
OptionRight.Put,
4200m,
new DateTime(2021, 3, 18));
var optionZipFilePath = LeanData.GenerateZipFilePath(Globals.DataFolder, optionSymbol, new DateTime(2020, 9, 22), Resolution.Minute, TickType.Quote)
.Replace(Path.DirectorySeparatorChar, '/');
var optionEntryFilePath = LeanData.GenerateZipEntryName(optionSymbol, new DateTime(2020, 9, 22), Resolution.Minute, TickType.Quote);
Assert.AreEqual($"../../../Data/futureoption/cme/minute/{expectedFutureOptionTicker.ToLowerInvariant()}/{underlying.ID.Date:yyyyMMdd}/20200922_quote_american.zip", optionZipFilePath);
Assert.AreEqual($"20200922_{expectedFutureOptionTicker.ToLowerInvariant()}_minute_quote_american_put_42000000_{optionSymbol.ID.Date:yyyyMMdd}.csv", optionEntryFilePath);
}
public void OptionZipFilePathWithUnderlyingEquity()
{
var underlying = Symbol.Create("SPY", SecurityType.Equity, QuantConnect.Market.USA);
var optionSymbol = Symbol.CreateOption(
underlying,
Market.USA,
OptionStyle.American,
OptionRight.Put,
4200m,
new DateTime(2020, 12, 31));
var optionZipFilePath = LeanData.GenerateZipFilePath(Globals.DataFolder, optionSymbol, new DateTime(2020, 9, 22), Resolution.Minute, TickType.Quote)
.Replace(Path.DirectorySeparatorChar, '/');
var optionEntryFilePath = LeanData.GenerateZipEntryName(optionSymbol, new DateTime(2020, 9, 22), Resolution.Minute, TickType.Quote);
Assert.AreEqual("../../../Data/option/usa/minute/spy/20200922_quote_american.zip", optionZipFilePath);
Assert.AreEqual("20200922_spy_minute_quote_american_put_42000000_20201231.csv", optionEntryFilePath);
}
/// <summary>
/// Creates the <see cref="TextWriter"/> that writes data to csv files
/// </summary>
private Writer CreateTextWriter(BaseData data)
{
var entry = LeanData.GenerateZipEntryName(data.Symbol, data.Time.Date, _resolution, _tickType);
var relativePath = LeanData.GenerateRelativeZipFilePath(data.Symbol, data.Time.Date, _resolution, _tickType)
.Replace(".zip", string.Empty);
var path = Path.Combine(Path.Combine(_dataDirectory, relativePath), entry);
var directory = new FileInfo(path).Directory.FullName;
if (!Directory.Exists(directory))
{
// lock before checking again
lock (DirectoryCreateSync) if (!Directory.Exists(directory))
{
Directory.CreateDirectory(directory);
}
}
return(new Writer(path, new StreamWriter(path)));
}
public void FutureOptionSingleZipFileContainingMultipleFuturesOptionsContracts(OptionRight right, int strike, int year, int month, int day)
{
var underlying = Symbol.CreateFuture("GC", Market.COMEX, new DateTime(2020, 4, 28));
var expiry = new DateTime(year, month, day);
var optionSymbol = Symbol.CreateOption(
underlying,
Market.COMEX,
OptionStyle.American,
right,
(decimal)strike,
expiry);
var optionZipFilePath = LeanData.GenerateZipFilePath(Globals.DataFolder, optionSymbol, new DateTime(2020, 1, 5), Resolution.Minute, TickType.Quote)
.Replace(Path.DirectorySeparatorChar, '/');
var optionEntryFilePath = LeanData.GenerateZipEntryName(optionSymbol, new DateTime(2020, 1, 5), Resolution.Minute, TickType.Quote);
Assert.AreEqual("../../../Data/futureoption/comex/minute/og/20200428/20200105_quote_american.zip", optionZipFilePath);
Assert.AreEqual($"20200105_og_minute_quote_american_{right.ToLower()}_{strike}0000_{expiry:yyyyMMdd}.csv", optionEntryFilePath);
}
/// <summary>
/// Write this file to disk.
/// </summary>
/// <param name="filePath">The full path to the new file</param>
/// <param name="data">The data to write as a string</param>
/// <param name="date">The date the data represents</param>
private void WriteFile(string filePath, string data, DateTime date)
{
var tempFilePath = filePath + ".tmp";
data = data.TrimEnd();
if (File.Exists(filePath))
{
File.Delete(filePath);
Log.Trace("LeanDataWriter.Write(): Existing deleted: " + filePath);
}
// Create the directory if it doesnt exist
Directory.CreateDirectory(Path.GetDirectoryName(filePath));
// Write out this data string to a zip file
Compression.Zip(data, tempFilePath, LeanData.GenerateZipEntryName(_symbol.Value, _securityType, date, _resolution, _dataType));
// Move temp file to the final destination with the appropriate name
File.Move(tempFilePath, filePath);
Log.Trace("LeanDataWriter.Write(): Created: " + filePath);
}
public void GenerateZipEntryName(LeanDataTestParameters parameters)
{
var entry = LeanData.GenerateZipEntryName(parameters.Symbol, parameters.Date, parameters.Resolution, parameters.TickType);
Assert.AreEqual(parameters.ExpectedZipEntryName, entry);
}
private void SaveDailyOrHour(
List <Symbol> symbols,
Symbol canonicalSymbol,
IReadOnlyDictionary <Symbol, List <BaseData> > historyBySymbol)
{
var zipFileName = Path.Combine(
_dataDirectory,
LeanData.GenerateRelativeZipFilePath(canonicalSymbol, DateTime.MinValue, _resolution, _tickType));
var folder = Path.GetDirectoryName(zipFileName);
if (!Directory.Exists(folder))
{
Directory.CreateDirectory(folder);
}
using (var zip = new ZipFile(zipFileName))
{
foreach (var symbol in symbols)
{
// Load new data rows into a SortedDictionary for easy merge/update
var newRows = new SortedDictionary <DateTime, string>(historyBySymbol[symbol]
.ToDictionary(x => x.Time, x => LeanData.GenerateLine(x, _securityType, _resolution)));
var rows = new SortedDictionary <DateTime, string>();
var zipEntryName = LeanData.GenerateZipEntryName(symbol, DateTime.MinValue, _resolution, _tickType);
if (zip.ContainsEntry(zipEntryName))
{
// If file exists, we load existing data and perform merge
using (var stream = new MemoryStream())
{
zip[zipEntryName].Extract(stream);
stream.Seek(0, SeekOrigin.Begin);
using (var reader = new StreamReader(stream))
{
string line;
while ((line = reader.ReadLine()) != null)
{
var time = Parse.DateTimeExact(line.Substring(0, DateFormat.TwelveCharacter.Length), DateFormat.TwelveCharacter);
rows[time] = line;
}
}
}
foreach (var kvp in newRows)
{
rows[kvp.Key] = kvp.Value;
}
}
else
{
// No existing file, just use the new data
rows = newRows;
}
// Loop through the SortedDictionary and write to zip entry
var sb = new StringBuilder();
foreach (var kvp in rows)
{
// Build the line and append it to the file
sb.AppendLine(kvp.Value);
}
// Write the zip entry
if (sb.Length > 0)
{
if (zip.ContainsEntry(zipEntryName))
{
zip.RemoveEntry(zipEntryName);
}
zip.AddEntry(zipEntryName, sb.ToString());
}
}
if (zip.Count > 0)
{
zip.Save();
}
}
}
private void ConvertMinuteFuturesData(Symbol canonical, TickType tickType, Resolution outputResolution, Resolution inputResolution = Resolution.Minute)
{
var timeSpans = new Dictionary <Resolution, TimeSpan>()
{
{ Resolution.Daily, TimeSpan.FromHours(24) },
{ Resolution.Hour, TimeSpan.FromHours(1) },
};
var timeSpan = timeSpans[outputResolution];
var tickTypeConsolidatorMap = new Dictionary <TickType, Func <IDataConsolidator> >()
{
{ TickType.Quote, () => new QuoteBarConsolidator(timeSpan) },
{ TickType.OpenInterest, () => new OpenInterestConsolidator(timeSpan) },
{ TickType.Trade, () => new TradeBarConsolidator(timeSpan) }
};
var consolidators = new Dictionary <string, IDataConsolidator>();
var configs = new Dictionary <string, SubscriptionDataConfig>();
var outputFiles = new Dictionary <string, StringBuilder>();
var futures = new Dictionary <string, Symbol>();
var date = _fromDate;
while (date <= _toDate)
{
var futureChain = LoadFutureChain(canonical, date, tickType, inputResolution);
foreach (var future in futureChain)
{
if (!futures.ContainsKey(future.Value))
{
futures[future.Value] = future;
var config = new SubscriptionDataConfig(LeanData.GetDataType(outputResolution, tickType),
future, inputResolution, TimeZones.NewYork, TimeZones.NewYork,
false, false, false, false, tickType);
configs[future.Value] = config;
consolidators[future.Value] = tickTypeConsolidatorMap[tickType].Invoke();
var sb = new StringBuilder();
outputFiles[future.Value] = sb;
consolidators[future.Value].DataConsolidated += (sender, bar) =>
{
sb.Append(LeanData.GenerateLine(bar, SecurityType.Future, outputResolution) + Environment.NewLine);
};
}
var leanDataReader = new LeanDataReader(configs[future.Value], future, inputResolution, date, _dataDirectory);
var consolidator = consolidators[future.Value];
foreach (var bar in leanDataReader.Parse())
{
consolidator.Update(bar);
}
}
date = date.AddDays(1);
}
//write all results
foreach (var consolidator in consolidators.Values)
{
consolidator.Scan(date);
}
var zip = LeanData.GenerateRelativeZipFilePath(canonical, _fromDate, outputResolution, tickType);
var zipPath = Path.Combine(_dataDirectory, zip);
var fi = new FileInfo(zipPath);
if (!fi.Directory.Exists)
{
fi.Directory.Create();
}
foreach (var future in futures.Values)
{
var zipEntry = LeanData.GenerateZipEntryName(future, _fromDate, outputResolution, tickType);
var sb = outputFiles[future.Value];
//Uncomment to write zip files
//QuantConnect.Compression.ZipCreateAppendData(zipPath, zipEntry, sb.ToString());
Assert.IsTrue(sb.Length > 0);
}
}