/// <summary> /// Write this file to disk. /// </summary> /// <param name="filePath">The full path to the new file</param> /// <param name="data">The data to write as a string</param> /// <param name="date">The date the data represents</param> private void WriteFile(string filePath, IEnumerable <string> data, DateTime date) { var tempFilePath = filePath + ".tmp"; if (File.Exists(filePath) && !_appendToZips) { File.Delete(filePath); Log.Trace("LeanDataWriter.Write(): Existing deleted: " + filePath); } // Create the directory if it doesnt exist Directory.CreateDirectory(Path.GetDirectoryName(filePath)); if (_appendToZips) { var entryName = LeanData.GenerateZipEntryName(_symbol, date, _resolution, _tickType); Compression.ZipCreateAppendData(filePath, entryName, string.Join(Environment.NewLine, data), true); Log.Trace("LeanDataWriter.Write(): Appended: " + filePath); } else { // Write out this data string to a zip file Compression.ZipData(tempFilePath, LeanData.GenerateZipEntryName(_symbol, date, _resolution, _tickType), data); // Move temp file to the final destination with the appropriate name File.Move(tempFilePath, filePath); Log.Trace("LeanDataWriter.Write(): Created: " + filePath); } }
/// <summary> /// The LeanDataReader constructor /// </summary> /// <param name="config">The <see cref="SubscriptionDataConfig"/></param> /// <param name="symbol">The <see cref="Symbol"/> that will be read</param> /// <param name="resolution">The <see cref="Resolution"/> that will be read</param> /// <param name="date">The <see cref="DateTime"/> that will be read</param> /// <param name="dataFolder">The root data folder</param> public LeanDataReader(SubscriptionDataConfig config, Symbol symbol, Resolution resolution, DateTime date, string dataFolder) { _date = date; _zipPath = LeanData.GenerateZipFilePath(dataFolder, symbol, date, resolution, config.TickType); _zipentry = LeanData.GenerateZipEntryName(symbol, date, resolution, config.TickType); _config = config; }
/// <summary> /// Write this file to disk with the given data. /// </summary> /// <param name="filePath">The full path to the new file</param> /// <param name="data">The data to write as a list of dates and strings</param> /// <remarks>The reason we have the data as IEnumerable(DateTime, string) is to support /// a generic write that works for all resolutions. In order to merge in hour/daily case I need the /// date of the data to correctly merge the two. In order to support writing ticks I need to allow /// two data points to have the same time. Thus I cannot use a single list of just strings nor /// a sorted dictionary of DateTimes and strings. </remarks> private void WriteFile(string filePath, List <TimedLine> data) { if (data == null || data.Count == 0) { return; } var date = data[0].Time; // Generate this csv entry name var entryName = LeanData.GenerateZipEntryName(_symbol, date, _resolution, _tickType); // Check disk once for this file ahead of time, reuse where possible var fileExists = File.Exists(filePath); // If our file doesn't exist its possible the directory doesn't exist, make sure at least the directory exists if (!fileExists) { Directory.CreateDirectory(Path.GetDirectoryName(filePath)); } // Handle merging of files // Only merge on files with hour/daily resolution, that exist, and can be loaded string finalData = null; if (_writePolicy == WritePolicy.Append) { var streamWriter = new ZipStreamWriter(filePath, entryName); foreach (var tuple in data) { streamWriter.WriteLine(tuple.Line); } streamWriter.DisposeSafely(); } else if (_writePolicy == WritePolicy.Merge && fileExists && TryLoadFile(filePath, entryName, date, out var rows)) { // Preform merge on loaded rows foreach (var timedLine in data) { rows[timedLine.Time] = timedLine.Line; } // Final merged data product finalData = string.Join("\n", rows.Values); } else { // Otherwise just extract the data from the given list. finalData = string.Join("\n", data.Select(x => x.Line)); } if (finalData != null) { var bytes = Encoding.UTF8.GetBytes(finalData); _dataCacheProvider.Store($"{filePath}#{entryName}", bytes); } Log.Debug($"LeanDataWriter.Write(): Appended: {filePath} @ {entryName}"); }
private void SaveMinuteOrSecondOrTick( List <Symbol> symbols, DateTime startTimeUtc, DateTime endTimeUtc, Symbol canonicalSymbol, IReadOnlyDictionary <Symbol, List <IGrouping <DateTime, BaseData> > > historyBySymbol) { var date = startTimeUtc; while (date <= endTimeUtc) { var zipFileName = Path.Combine( Globals.DataFolder, LeanData.GenerateRelativeZipFilePath(canonicalSymbol, date, _resolution, _tickType)); var folder = Path.GetDirectoryName(zipFileName); if (!Directory.Exists(folder)) { Directory.CreateDirectory(folder); } if (File.Exists(zipFileName)) { File.Delete(zipFileName); } using (var zip = new ZipFile(zipFileName)) { foreach (var symbol in symbols) { var zipEntryName = LeanData.GenerateZipEntryName(symbol, date, _resolution, _tickType); foreach (var group in historyBySymbol[symbol]) { if (group.Key == date) { var sb = new StringBuilder(); foreach (var row in group) { var line = LeanData.GenerateLine(row, _securityType, _resolution); sb.AppendLine(line); } zip.AddEntry(zipEntryName, sb.ToString()); break; } } } if (zip.Count > 0) { zip.Save(); } } date = date.AddDays(1); } }
public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode) { if (isLiveMode) { return(new SubscriptionDataSource(string.Empty, SubscriptionTransportMedium.LocalFile)); } var source = LeanData.GenerateZipFilePath(Globals.DataFolder, config.Symbol, date, config.Resolution, config.TickType); if (config.SecurityType == SecurityType.Option) { source += "#" + LeanData.GenerateZipEntryName(config.Symbol, date, config.Resolution, config.TickType); } return(new SubscriptionDataSource(source, SubscriptionTransportMedium.LocalFile, FileFormat.Csv)); }
/// <summary> /// Write this file to disk /// </summary> private void WriteFile(string fileName, string data, DateTime time) { data = data.TrimEnd(); if (File.Exists(fileName)) { File.Delete(fileName); Log.Trace("LeanDataWriter.Write(): Existing deleted: " + fileName); } // Create the directory if it doesnt exist Directory.CreateDirectory(Path.GetDirectoryName(fileName)); // Write out this data string to a zip file Compression.Zip(data, fileName, LeanData.GenerateZipEntryName(_symbol.Value, _securityType, time, _resolution, _dataType)); Log.Trace("LeanDataWriter.Write(): Created: " + fileName); }
public void OptionZipFilePathWithUnderlyingFuture(string futureOptionTicker, string expectedFutureOptionTicker) { var underlying = Symbol.CreateFuture(futureOptionTicker, Market.CME, new DateTime(2021, 3, 19)); var optionSymbol = Symbol.CreateOption( underlying, Market.CME, OptionStyle.American, OptionRight.Put, 4200m, new DateTime(2021, 3, 18)); var optionZipFilePath = LeanData.GenerateZipFilePath(Globals.DataFolder, optionSymbol, new DateTime(2020, 9, 22), Resolution.Minute, TickType.Quote) .Replace(Path.DirectorySeparatorChar, '/'); var optionEntryFilePath = LeanData.GenerateZipEntryName(optionSymbol, new DateTime(2020, 9, 22), Resolution.Minute, TickType.Quote); Assert.AreEqual($"../../../Data/futureoption/cme/minute/{expectedFutureOptionTicker.ToLowerInvariant()}/{underlying.ID.Date:yyyyMMdd}/20200922_quote_american.zip", optionZipFilePath); Assert.AreEqual($"20200922_{expectedFutureOptionTicker.ToLowerInvariant()}_minute_quote_american_put_42000000_{optionSymbol.ID.Date:yyyyMMdd}.csv", optionEntryFilePath); }
public void OptionZipFilePathWithUnderlyingEquity() { var underlying = Symbol.Create("SPY", SecurityType.Equity, QuantConnect.Market.USA); var optionSymbol = Symbol.CreateOption( underlying, Market.USA, OptionStyle.American, OptionRight.Put, 4200m, new DateTime(2020, 12, 31)); var optionZipFilePath = LeanData.GenerateZipFilePath(Globals.DataFolder, optionSymbol, new DateTime(2020, 9, 22), Resolution.Minute, TickType.Quote) .Replace(Path.DirectorySeparatorChar, '/'); var optionEntryFilePath = LeanData.GenerateZipEntryName(optionSymbol, new DateTime(2020, 9, 22), Resolution.Minute, TickType.Quote); Assert.AreEqual("../../../Data/option/usa/minute/spy/20200922_quote_american.zip", optionZipFilePath); Assert.AreEqual("20200922_spy_minute_quote_american_put_42000000_20201231.csv", optionEntryFilePath); }
/// <summary> /// Creates the <see cref="TextWriter"/> that writes data to csv files /// </summary> private Writer CreateTextWriter(BaseData data) { var entry = LeanData.GenerateZipEntryName(data.Symbol, data.Time.Date, _resolution, _tickType); var relativePath = LeanData.GenerateRelativeZipFilePath(data.Symbol, data.Time.Date, _resolution, _tickType) .Replace(".zip", string.Empty); var path = Path.Combine(Path.Combine(_dataDirectory, relativePath), entry); var directory = new FileInfo(path).Directory.FullName; if (!Directory.Exists(directory)) { // lock before checking again lock (DirectoryCreateSync) if (!Directory.Exists(directory)) { Directory.CreateDirectory(directory); } } return(new Writer(path, new StreamWriter(path))); }
public void FutureOptionSingleZipFileContainingMultipleFuturesOptionsContracts(OptionRight right, int strike, int year, int month, int day) { var underlying = Symbol.CreateFuture("GC", Market.COMEX, new DateTime(2020, 4, 28)); var expiry = new DateTime(year, month, day); var optionSymbol = Symbol.CreateOption( underlying, Market.COMEX, OptionStyle.American, right, (decimal)strike, expiry); var optionZipFilePath = LeanData.GenerateZipFilePath(Globals.DataFolder, optionSymbol, new DateTime(2020, 1, 5), Resolution.Minute, TickType.Quote) .Replace(Path.DirectorySeparatorChar, '/'); var optionEntryFilePath = LeanData.GenerateZipEntryName(optionSymbol, new DateTime(2020, 1, 5), Resolution.Minute, TickType.Quote); Assert.AreEqual("../../../Data/futureoption/comex/minute/og/20200428/20200105_quote_american.zip", optionZipFilePath); Assert.AreEqual($"20200105_og_minute_quote_american_{right.ToLower()}_{strike}0000_{expiry:yyyyMMdd}.csv", optionEntryFilePath); }
/// <summary> /// Write this file to disk. /// </summary> /// <param name="filePath">The full path to the new file</param> /// <param name="data">The data to write as a string</param> /// <param name="date">The date the data represents</param> private void WriteFile(string filePath, string data, DateTime date) { var tempFilePath = filePath + ".tmp"; data = data.TrimEnd(); if (File.Exists(filePath)) { File.Delete(filePath); Log.Trace("LeanDataWriter.Write(): Existing deleted: " + filePath); } // Create the directory if it doesnt exist Directory.CreateDirectory(Path.GetDirectoryName(filePath)); // Write out this data string to a zip file Compression.Zip(data, tempFilePath, LeanData.GenerateZipEntryName(_symbol.Value, _securityType, date, _resolution, _dataType)); // Move temp file to the final destination with the appropriate name File.Move(tempFilePath, filePath); Log.Trace("LeanDataWriter.Write(): Created: " + filePath); }
public void GenerateZipEntryName(LeanDataTestParameters parameters) { var entry = LeanData.GenerateZipEntryName(parameters.Symbol, parameters.Date, parameters.Resolution, parameters.TickType); Assert.AreEqual(parameters.ExpectedZipEntryName, entry); }
private void SaveDailyOrHour( List <Symbol> symbols, Symbol canonicalSymbol, IReadOnlyDictionary <Symbol, List <BaseData> > historyBySymbol) { var zipFileName = Path.Combine( _dataDirectory, LeanData.GenerateRelativeZipFilePath(canonicalSymbol, DateTime.MinValue, _resolution, _tickType)); var folder = Path.GetDirectoryName(zipFileName); if (!Directory.Exists(folder)) { Directory.CreateDirectory(folder); } using (var zip = new ZipFile(zipFileName)) { foreach (var symbol in symbols) { // Load new data rows into a SortedDictionary for easy merge/update var newRows = new SortedDictionary <DateTime, string>(historyBySymbol[symbol] .ToDictionary(x => x.Time, x => LeanData.GenerateLine(x, _securityType, _resolution))); var rows = new SortedDictionary <DateTime, string>(); var zipEntryName = LeanData.GenerateZipEntryName(symbol, DateTime.MinValue, _resolution, _tickType); if (zip.ContainsEntry(zipEntryName)) { // If file exists, we load existing data and perform merge using (var stream = new MemoryStream()) { zip[zipEntryName].Extract(stream); stream.Seek(0, SeekOrigin.Begin); using (var reader = new StreamReader(stream)) { string line; while ((line = reader.ReadLine()) != null) { var time = Parse.DateTimeExact(line.Substring(0, DateFormat.TwelveCharacter.Length), DateFormat.TwelveCharacter); rows[time] = line; } } } foreach (var kvp in newRows) { rows[kvp.Key] = kvp.Value; } } else { // No existing file, just use the new data rows = newRows; } // Loop through the SortedDictionary and write to zip entry var sb = new StringBuilder(); foreach (var kvp in rows) { // Build the line and append it to the file sb.AppendLine(kvp.Value); } // Write the zip entry if (sb.Length > 0) { if (zip.ContainsEntry(zipEntryName)) { zip.RemoveEntry(zipEntryName); } zip.AddEntry(zipEntryName, sb.ToString()); } } if (zip.Count > 0) { zip.Save(); } } }
private void ConvertMinuteFuturesData(Symbol canonical, TickType tickType, Resolution outputResolution, Resolution inputResolution = Resolution.Minute) { var timeSpans = new Dictionary <Resolution, TimeSpan>() { { Resolution.Daily, TimeSpan.FromHours(24) }, { Resolution.Hour, TimeSpan.FromHours(1) }, }; var timeSpan = timeSpans[outputResolution]; var tickTypeConsolidatorMap = new Dictionary <TickType, Func <IDataConsolidator> >() { { TickType.Quote, () => new QuoteBarConsolidator(timeSpan) }, { TickType.OpenInterest, () => new OpenInterestConsolidator(timeSpan) }, { TickType.Trade, () => new TradeBarConsolidator(timeSpan) } }; var consolidators = new Dictionary <string, IDataConsolidator>(); var configs = new Dictionary <string, SubscriptionDataConfig>(); var outputFiles = new Dictionary <string, StringBuilder>(); var futures = new Dictionary <string, Symbol>(); var date = _fromDate; while (date <= _toDate) { var futureChain = LoadFutureChain(canonical, date, tickType, inputResolution); foreach (var future in futureChain) { if (!futures.ContainsKey(future.Value)) { futures[future.Value] = future; var config = new SubscriptionDataConfig(LeanData.GetDataType(outputResolution, tickType), future, inputResolution, TimeZones.NewYork, TimeZones.NewYork, false, false, false, false, tickType); configs[future.Value] = config; consolidators[future.Value] = tickTypeConsolidatorMap[tickType].Invoke(); var sb = new StringBuilder(); outputFiles[future.Value] = sb; consolidators[future.Value].DataConsolidated += (sender, bar) => { sb.Append(LeanData.GenerateLine(bar, SecurityType.Future, outputResolution) + Environment.NewLine); }; } var leanDataReader = new LeanDataReader(configs[future.Value], future, inputResolution, date, _dataDirectory); var consolidator = consolidators[future.Value]; foreach (var bar in leanDataReader.Parse()) { consolidator.Update(bar); } } date = date.AddDays(1); } //write all results foreach (var consolidator in consolidators.Values) { consolidator.Scan(date); } var zip = LeanData.GenerateRelativeZipFilePath(canonical, _fromDate, outputResolution, tickType); var zipPath = Path.Combine(_dataDirectory, zip); var fi = new FileInfo(zipPath); if (!fi.Directory.Exists) { fi.Directory.Create(); } foreach (var future in futures.Values) { var zipEntry = LeanData.GenerateZipEntryName(future, _fromDate, outputResolution, tickType); var sb = outputFiles[future.Value]; //Uncomment to write zip files //QuantConnect.Compression.ZipCreateAppendData(zipPath, zipEntry, sb.ToString()); Assert.IsTrue(sb.Length > 0); } }