/// <summary> /// Processes the entry. /// </summary> /// <param name="coinapiDataReader">The coinapi data reader.</param> /// <param name="file">The file.</param> private void ProcessEntry(CoinApiDataReader coinapiDataReader, FileInfo file) { var entryData = coinapiDataReader.GetCoinApiEntryData(file, _processingDate); if (!SupportedMarkets.Contains(entryData.Symbol.ID.Market)) { // only convert data for supported exchanges return; } // materialize the enumerable into a list, since we need to enumerate over it twice var ticks = coinapiDataReader.ProcessCoinApiEntry(entryData, file).OrderBy(t => t.Time).ToList(); var writer = new LeanDataWriter(Resolution.Tick, entryData.Symbol, _destinationFolder.FullName, entryData.TickType); writer.Write(ticks); Log.Trace($"CoinApiDataConverter(): Starting consolidation for {entryData.Symbol.Value} {entryData.TickType}"); var consolidators = new List <TickAggregator>(); if (entryData.TickType == TickType.Trade) { consolidators.AddRange(new[] { new TradeTickAggregator(Resolution.Second), new TradeTickAggregator(Resolution.Minute) }); } else { consolidators.AddRange(new[] { new QuoteTickAggregator(Resolution.Second), new QuoteTickAggregator(Resolution.Minute) }); } foreach (var tick in ticks) { if (tick.Suspicious) { // When CoinAPI loses connectivity to the exchange, they indicate // it in the data by providing a value of `-1` for bid/ask price. // We will keep it in tick data, but will remove it from consolidated data. continue; } foreach (var consolidator in consolidators) { consolidator.Update(tick); } } foreach (var consolidator in consolidators) { writer = new LeanDataWriter(consolidator.Resolution, entryData.Symbol, _destinationFolder.FullName, entryData.TickType); writer.Write(consolidator.Flush()); } }
/// <summary> /// Processes the entry. /// </summary> /// <param name="coinapiDataReader">The coinapi data reader.</param> /// <param name="file">The file.</param> private void ProcessEntry(CoinApiDataReader coinapiDataReader, FileInfo file) { var entryData = coinapiDataReader.GetCoinApiEntryData(file, _processingDate, _market); if (!SupportedMarkets.Contains(entryData.Symbol.ID.Market)) { // only convert data for supported exchanges return; } // materialize the enumerable into a list, since we need to enumerate over it twice var ticks = coinapiDataReader.ProcessCoinApiEntry(entryData, file).ToList(); var writer = new LeanDataWriter(Resolution.Tick, entryData.Symbol, _destinationFolder.FullName, entryData.TickType); writer.Write(ticks); Log.Trace($"CoinApiDataConverter(): Starting consolidation for {entryData.Symbol.Value} {entryData.TickType}"); var consolidators = new List <TickAggregator>(); if (entryData.TickType == TickType.Trade) { consolidators.AddRange(new[] { new TradeTickAggregator(Resolution.Second), new TradeTickAggregator(Resolution.Minute) }); } else { consolidators.AddRange(new[] { new QuoteTickAggregator(Resolution.Second), new QuoteTickAggregator(Resolution.Minute) }); } foreach (var tick in ticks) { foreach (var consolidator in consolidators) { consolidator.Update(tick); } } foreach (var consolidator in consolidators) { writer = new LeanDataWriter(consolidator.Resolution, entryData.Symbol, _destinationFolder.FullName, entryData.TickType); writer.Write(consolidator.Flush()); } }
/// <summary> /// Processes the entry. /// </summary> /// <param name="coinapiDataReader">The coinapi data reader.</param> /// <param name="file">The file.</param> private void ProcessEntry(CoinApiDataReader coinapiDataReader, FileInfo file) { var entryData = coinapiDataReader.GetCoinApiEntryData(file, _processingDate); if (!SupportedMarkets.Contains(entryData.Symbol.ID.Market)) { // only convert data for supported exchanges return; } var tickData = coinapiDataReader.ProcessCoinApiEntry(entryData, file); // in some cases the first data points from '_processingDate' get's included in the previous date file // so we will ready previous date data and drop most of it just to save these midnight ticks var yesterdayDate = _processingDate.AddDays(-1); var yesterdaysFile = new FileInfo(file.FullName.Replace( _processingDate.ToStringInvariant(DateFormat.EightCharacter), yesterdayDate.ToStringInvariant(DateFormat.EightCharacter))); if (yesterdaysFile.Exists) { var yesterdaysEntryData = coinapiDataReader.GetCoinApiEntryData(yesterdaysFile, yesterdayDate); tickData = tickData.Concat(coinapiDataReader.ProcessCoinApiEntry(yesterdaysEntryData, yesterdaysFile)); } else { Log.Error($"CoinApiDataConverter(): yesterdays data file not found '{yesterdaysFile.FullName}'"); } // materialize the enumerable into a list, since we need to enumerate over it twice var ticks = tickData.Where(tick => tick.Time.Date == _processingDate) .OrderBy(t => t.Time) .ToList(); var writer = new LeanDataWriter(Resolution.Tick, entryData.Symbol, _destinationFolder.FullName, entryData.TickType); writer.Write(ticks); Log.Trace($"CoinApiDataConverter(): Starting consolidation for {entryData.Symbol.Value} {entryData.TickType}"); var consolidators = new List <TickAggregator>(); if (entryData.TickType == TickType.Trade) { consolidators.AddRange(new[] { new TradeTickAggregator(Resolution.Second), new TradeTickAggregator(Resolution.Minute) }); } else { consolidators.AddRange(new[] { new QuoteTickAggregator(Resolution.Second), new QuoteTickAggregator(Resolution.Minute) }); } foreach (var tick in ticks) { if (tick.Suspicious) { // When CoinAPI loses connectivity to the exchange, they indicate // it in the data by providing a value of `-1` for bid/ask price. // We will keep it in tick data, but will remove it from consolidated data. continue; } foreach (var consolidator in consolidators) { consolidator.Update(tick); } } foreach (var consolidator in consolidators) { writer = new LeanDataWriter(consolidator.Resolution, entryData.Symbol, _destinationFolder.FullName, entryData.TickType); writer.Write(consolidator.Flush()); } }