/// <summary>
/// Processes the entry.
/// </summary>
/// <param name="coinapiDataReader">The coinapi data reader.</param>
/// <param name="file">The file.</param>
private void ProcessEntry(CoinApiDataReader coinapiDataReader, FileInfo file)
{
var entryData = coinapiDataReader.GetCoinApiEntryData(file, _processingDate);
if (!SupportedMarkets.Contains(entryData.Symbol.ID.Market))
{
// only convert data for supported exchanges
return;
}
// materialize the enumerable into a list, since we need to enumerate over it twice
var ticks = coinapiDataReader.ProcessCoinApiEntry(entryData, file).OrderBy(t => t.Time).ToList();
var writer = new LeanDataWriter(Resolution.Tick, entryData.Symbol, _destinationFolder.FullName, entryData.TickType);
writer.Write(ticks);
Log.Trace($"CoinApiDataConverter(): Starting consolidation for {entryData.Symbol.Value} {entryData.TickType}");
var consolidators = new List <TickAggregator>();
if (entryData.TickType == TickType.Trade)
{
consolidators.AddRange(new[]
{
new TradeTickAggregator(Resolution.Second),
new TradeTickAggregator(Resolution.Minute)
});
}
else
{
consolidators.AddRange(new[]
{
new QuoteTickAggregator(Resolution.Second),
new QuoteTickAggregator(Resolution.Minute)
});
}
foreach (var tick in ticks)
{
if (tick.Suspicious)
{
// When CoinAPI loses connectivity to the exchange, they indicate
// it in the data by providing a value of `-1` for bid/ask price.
// We will keep it in tick data, but will remove it from consolidated data.
continue;
}
foreach (var consolidator in consolidators)
{
consolidator.Update(tick);
}
}
foreach (var consolidator in consolidators)
{
writer = new LeanDataWriter(consolidator.Resolution, entryData.Symbol, _destinationFolder.FullName, entryData.TickType);
writer.Write(consolidator.Flush());
}
}
/// <summary>
/// Processes the entry.
/// </summary>
/// <param name="coinapiDataReader">The coinapi data reader.</param>
/// <param name="file">The file.</param>
private void ProcessEntry(CoinApiDataReader coinapiDataReader, FileInfo file)
{
var entryData = coinapiDataReader.GetCoinApiEntryData(file, _processingDate, _market);
if (!SupportedMarkets.Contains(entryData.Symbol.ID.Market))
{
// only convert data for supported exchanges
return;
}
// materialize the enumerable into a list, since we need to enumerate over it twice
var ticks = coinapiDataReader.ProcessCoinApiEntry(entryData, file).ToList();
var writer = new LeanDataWriter(Resolution.Tick, entryData.Symbol, _destinationFolder.FullName, entryData.TickType);
writer.Write(ticks);
Log.Trace($"CoinApiDataConverter(): Starting consolidation for {entryData.Symbol.Value} {entryData.TickType}");
var consolidators = new List <TickAggregator>();
if (entryData.TickType == TickType.Trade)
{
consolidators.AddRange(new[]
{
new TradeTickAggregator(Resolution.Second),
new TradeTickAggregator(Resolution.Minute)
});
}
else
{
consolidators.AddRange(new[]
{
new QuoteTickAggregator(Resolution.Second),
new QuoteTickAggregator(Resolution.Minute)
});
}
foreach (var tick in ticks)
{
foreach (var consolidator in consolidators)
{
consolidator.Update(tick);
}
}
foreach (var consolidator in consolidators)
{
writer = new LeanDataWriter(consolidator.Resolution, entryData.Symbol, _destinationFolder.FullName, entryData.TickType);
writer.Write(consolidator.Flush());
}
}
/// <summary>
/// Processes the entry.
/// </summary>
/// <param name="coinapiDataReader">The coinapi data reader.</param>
/// <param name="file">The file.</param>
private void ProcessEntry(CoinApiDataReader coinapiDataReader, FileInfo file)
{
var entryData = coinapiDataReader.GetCoinApiEntryData(file, _processingDate);
if (!SupportedMarkets.Contains(entryData.Symbol.ID.Market))
{
// only convert data for supported exchanges
return;
}
var tickData = coinapiDataReader.ProcessCoinApiEntry(entryData, file);
// in some cases the first data points from '_processingDate' get's included in the previous date file
// so we will ready previous date data and drop most of it just to save these midnight ticks
var yesterdayDate = _processingDate.AddDays(-1);
var yesterdaysFile = new FileInfo(file.FullName.Replace(
_processingDate.ToStringInvariant(DateFormat.EightCharacter),
yesterdayDate.ToStringInvariant(DateFormat.EightCharacter)));
if (yesterdaysFile.Exists)
{
var yesterdaysEntryData = coinapiDataReader.GetCoinApiEntryData(yesterdaysFile, yesterdayDate);
tickData = tickData.Concat(coinapiDataReader.ProcessCoinApiEntry(yesterdaysEntryData, yesterdaysFile));
}
else
{
Log.Error($"CoinApiDataConverter(): yesterdays data file not found '{yesterdaysFile.FullName}'");
}
// materialize the enumerable into a list, since we need to enumerate over it twice
var ticks = tickData.Where(tick => tick.Time.Date == _processingDate)
.OrderBy(t => t.Time)
.ToList();
var writer = new LeanDataWriter(Resolution.Tick, entryData.Symbol, _destinationFolder.FullName, entryData.TickType);
writer.Write(ticks);
Log.Trace($"CoinApiDataConverter(): Starting consolidation for {entryData.Symbol.Value} {entryData.TickType}");
var consolidators = new List <TickAggregator>();
if (entryData.TickType == TickType.Trade)
{
consolidators.AddRange(new[]
{
new TradeTickAggregator(Resolution.Second),
new TradeTickAggregator(Resolution.Minute)
});
}
else
{
consolidators.AddRange(new[]
{
new QuoteTickAggregator(Resolution.Second),
new QuoteTickAggregator(Resolution.Minute)
});
}
foreach (var tick in ticks)
{
if (tick.Suspicious)
{
// When CoinAPI loses connectivity to the exchange, they indicate
// it in the data by providing a value of `-1` for bid/ask price.
// We will keep it in tick data, but will remove it from consolidated data.
continue;
}
foreach (var consolidator in consolidators)
{
consolidator.Update(tick);
}
}
foreach (var consolidator in consolidators)
{
writer = new LeanDataWriter(consolidator.Resolution, entryData.Symbol, _destinationFolder.FullName, entryData.TickType);
writer.Write(consolidator.Flush());
}
}