/// <summary>
/// Write the in memory queues to the disk.
/// </summary>
/// <param name="frontierTime">Current foremost tick time</param>
/// <param name="finalFlush">Indicates is this is the final push to disk at the end of the data</param>
public void FlushBuffer(DateTime frontierTime, bool finalFlush)
{
//Force the consolidation if time has past the bar
_consolidator.Scan(frontierTime);
// If this is the final packet dump it to the queue
if (finalFlush)
{
if (_consolidator.WorkingData != null)
{
_streamWriter.WriteLine(LeanData.GenerateLine(_consolidator.WorkingData, SecurityType.Future, Resolution));
}
_streamWriter.Flush();
_streamWriter.Close();
_streamWriter = null;
Interlocked.Add(ref _curFileCount, -1);
if (_curFileCount % 1000 == 0)
{
Log.Trace("Closed some files: {0}", _curFileCount);
}
}
}
/// <summary>
/// Create a new AlgoSeekFuturesProcessor for enquing consolidated bars and flushing them to disk
/// </summary>
/// <param name="symbol">Symbol for the processor</param>
/// <param name="date">Reference date for the processor</param>
/// <param name="tickType">TradeBar or QuoteBar to generate</param>
/// <param name="resolution">Resolution to consolidate</param>
/// <param name="dataDirectory">Data directory for LEAN</param>
public AlgoSeekFuturesProcessor(Symbol symbol, DateTime date, TickType tickType, Resolution resolution, string dataDirectory)
{
_symbol = Safe(symbol);
_tickType = tickType;
_referenceDate = date;
_resolution = resolution;
_dataDirectory = dataDirectory;
// Setup the consolidator for the requested resolution
if (resolution == Resolution.Tick)
{
_consolidator = new IdentityDataConsolidator <Tick>();
}
else
{
switch (tickType)
{
case TickType.Trade:
_consolidator = new TickConsolidator(resolution.ToTimeSpan());
break;
case TickType.Quote:
_consolidator = new TickQuoteBarConsolidator(resolution.ToTimeSpan());
break;
case TickType.OpenInterest:
_consolidator = new OpenInterestConsolidator(resolution.ToTimeSpan());
break;
}
}
var path = ZipPath.Replace(".zip", string.Empty);
Directory.CreateDirectory(path);
var file = Path.Combine(path, EntryPath);
try
{
_streamWriter = new LazyStreamWriter(file);
}
catch (Exception err)
{
// we are unable to open new file - it is already opened due to bug in algoseek data
Log.Error("File: {0} Err: {1} Source: {2} Stack: {3}", file, err.Message, err.Source, err.StackTrace);
var newRandomizedName = (file + "-" + Math.Abs(file.GetHashCode()).ToString()).Replace(".csv", string.Empty) + ".csv";
// we store the information under different (randomized) name
Log.Trace("Changing name from {0} to {1}", file, newRandomizedName);
_streamWriter = new LazyStreamWriter(newRandomizedName);
}
// On consolidating the bars put the bar into a queue in memory to be written to disk later.
_consolidator.DataConsolidated += (sender, consolidated) =>
{
_streamWriter.WriteLine(LeanData.GenerateLine(consolidated, SecurityType.Future, Resolution));
};
Interlocked.Add(ref _curFileCount, 1);
if (_curFileCount % 1000 == 0)
{
Log.Trace("Opened more files: {0}", _curFileCount);
}
}
