/// <summary> /// Write the in memory queues to the disk. /// </summary> /// <param name="frontierTime">Current foremost tick time</param> /// <param name="finalFlush">Indicates is this is the final push to disk at the end of the data</param> public void FlushBuffer(DateTime frontierTime, bool finalFlush) { //Force the consolidation if time has past the bar _consolidator.Scan(frontierTime); // If this is the final packet dump it to the queue if (finalFlush) { if (_consolidator.WorkingData != null) { _streamWriter.WriteLine(LeanData.GenerateLine(_consolidator.WorkingData, SecurityType.Future, Resolution)); } _streamWriter.Flush(); _streamWriter.Close(); _streamWriter = null; Interlocked.Add(ref _curFileCount, -1); if (_curFileCount % 1000 == 0) { Log.Trace("Closed some files: {0}", _curFileCount); } } }
/// <summary> /// Create a new AlgoSeekFuturesProcessor for enquing consolidated bars and flushing them to disk /// </summary> /// <param name="symbol">Symbol for the processor</param> /// <param name="date">Reference date for the processor</param> /// <param name="tickType">TradeBar or QuoteBar to generate</param> /// <param name="resolution">Resolution to consolidate</param> /// <param name="dataDirectory">Data directory for LEAN</param> public AlgoSeekFuturesProcessor(Symbol symbol, DateTime date, TickType tickType, Resolution resolution, string dataDirectory) { _symbol = Safe(symbol); _tickType = tickType; _referenceDate = date; _resolution = resolution; _dataDirectory = dataDirectory; // Setup the consolidator for the requested resolution if (resolution == Resolution.Tick) { _consolidator = new IdentityDataConsolidator <Tick>(); } else { switch (tickType) { case TickType.Trade: _consolidator = new TickConsolidator(resolution.ToTimeSpan()); break; case TickType.Quote: _consolidator = new TickQuoteBarConsolidator(resolution.ToTimeSpan()); break; case TickType.OpenInterest: _consolidator = new OpenInterestConsolidator(resolution.ToTimeSpan()); break; } } var path = ZipPath.Replace(".zip", string.Empty); Directory.CreateDirectory(path); var file = Path.Combine(path, EntryPath); try { _streamWriter = new LazyStreamWriter(file); } catch (Exception err) { // we are unable to open new file - it is already opened due to bug in algoseek data Log.Error("File: {0} Err: {1} Source: {2} Stack: {3}", file, err.Message, err.Source, err.StackTrace); var newRandomizedName = (file + "-" + Math.Abs(file.GetHashCode()).ToString()).Replace(".csv", string.Empty) + ".csv"; // we store the information under different (randomized) name Log.Trace("Changing name from {0} to {1}", file, newRandomizedName); _streamWriter = new LazyStreamWriter(newRandomizedName); } // On consolidating the bars put the bar into a queue in memory to be written to disk later. _consolidator.DataConsolidated += (sender, consolidated) => { _streamWriter.WriteLine(LeanData.GenerateLine(consolidated, SecurityType.Future, Resolution)); }; Interlocked.Add(ref _curFileCount, 1); if (_curFileCount % 1000 == 0) { Log.Trace("Opened more files: {0}", _curFileCount); } }