private async Task ProcessMessageAsync(CandlesUpdatedEvent message) { if (message.Candles == null || !message.Candles.Any()) { return; } var intervalToSkip = new List <CandleTimeInterval> { CandleTimeInterval.Unspecified, CandleTimeInterval.Sec, CandleTimeInterval.Minute }; var candlesToStore = message.Candles.Where(x => !string.IsNullOrEmpty(x.AssetPairId) && x.IsLatestChange && !intervalToSkip.Contains(x.TimeInterval) && x.PriceType != CandlePriceType.Unspecified) .ToList(); var candles = _mapper.Map <List <CandleEntity> >(candlesToStore); await _candlesWriter.BulkInsertOrReplaceAsync(candles); Task.Run(async() => { await _candlesWriter.CleanAndKeepMaxPartitions(0); }); }
private Task PublishV2Async(IEnumerable <CandleUpdateResult> updates) { var @event = new CandlesUpdatedEvent { ContractVersion = Contract.Constants.ContractVersion, UpdateTimestamp = DateTime.UtcNow, Candles = updates .Select(c => new CandleUpdate { IsLatestChange = c.IsLatestChange, ChangeTimestamp = c.Candle.LatestChangeTimestamp, AssetPairId = c.Candle.AssetPairId, PriceType = c.Candle.PriceType, TimeInterval = c.Candle.TimeInterval, CandleTimestamp = c.Candle.Timestamp, Open = c.Candle.Open, Close = c.Candle.Close, Low = c.Candle.Low, High = c.Candle.High, TradingVolume = c.Candle.TradingVolume, TradingOppositeVolume = c.Candle.TradingOppositeVolume, IsLatestCandle = true, LastTradePrice = 0 }) .ToArray() }; lock (_publisher) { // HACK: Actually ProduceAsync is not async, so lock works well return(_publisher.ProduceAsync(@event)); } }
public void ProcessCandles(CandlesUpdatedEvent updatedCandles, MarketType market) { foreach (var candle in updatedCandles.Candles.Where(c => c.IsLatestCandle)) { ProcessCandleAsync(candle, market); } }
private static bool IsValid(CandlesUpdatedEvent candleEvent) { return(candleEvent.Candles.All(c => !string.IsNullOrWhiteSpace(c.AssetPairId) && c.AssetPairId.Length <= _maxStringFieldsLength && c.High >= c.Low && c.CandleTimestamp <= c.ChangeTimestamp)); }
public void ProcessCandles(CandlesUpdatedEvent candleEvent) { foreach (var candle in candleEvent.Candles) { if (candle.TimeInterval != CandleTimeInterval.Minute) { continue; } if (candle.PriceType != CandlePriceType.Ask && candle.PriceType != CandlePriceType.Bid) { continue; } var candlestick = new OutCandlestick { AssetPairId = candle.AssetPairId, IsAsk = candle.PriceType == CandlePriceType.Ask, High = (decimal)candle.High, Low = (decimal)candle.Low, Open = (decimal)candle.Open, Close = (decimal)candle.Close, Start = DateTimeConverter.Convert(candle.CandleTimestamp), Finish = DateTimeConverter.Convert(candle.ChangeTimestamp), }; string key = GetKey(candle); var start = candle.CandleTimestamp; if (_candlesDict.ContainsKey(key)) { var datesDict = _candlesDict[key]; if (datesDict.ContainsKey(start)) { datesDict[start] = Merge(datesDict[start], candlestick); } else { datesDict.Add(start, candlestick); } } else { var datesDict = new Dictionary <DateTime, OutCandlestick> { { start, candlestick }, }; _candlesDict.Add(key, datesDict); } } }
private async Task ProcessCandlesUpdatedEventAsync(CandlesUpdatedEvent candlesUpdate) { try { if (_cacheInitalizationService.InitializationState != CacheInitializationState.Idle) { await Task.Delay(5000); throw new InvalidOperationException("Initialization in progress"); } var validationErrors = ValidateQuote(candlesUpdate); if (validationErrors.Any()) { var message = string.Join("\r\n", validationErrors); _log.Warning(nameof(ProcessCandlesUpdatedEventAsync), message, context: candlesUpdate.ToJson()); return; } var candles = candlesUpdate.Candles .Where(candleUpdate => _candlesChecker.CanHandleAssetPair(candleUpdate.AssetPairId)) .Select(candleUpdate => Candle.Create( priceType: candleUpdate.PriceType, assetPair: candleUpdate.AssetPairId, timeInterval: candleUpdate.TimeInterval, timestamp: candleUpdate.CandleTimestamp, open: candleUpdate.Open, close: candleUpdate.Close, low: candleUpdate.Low, high: candleUpdate.High, tradingVolume: candleUpdate.TradingVolume, tradingOppositeVolume: candleUpdate.TradingOppositeVolume, lastTradePrice: candleUpdate.LastTradePrice, lastUpdateTimestamp: candleUpdate.ChangeTimestamp)) .ToArray(); await _candlesManager.ProcessCandlesAsync(candles); } catch (Exception) { _log.Warning(nameof(ProcessCandlesUpdatedEventAsync), "Failed to process candle", context: candlesUpdate.ToJson()); throw; } }
private async Task ProcessCandlesUpdatedEventAsync(CandlesUpdatedEvent candlesUpdate) { try { var validationErrors = ValidateQuote(candlesUpdate); if (validationErrors.Any()) { var message = string.Join("\r\n", validationErrors); await _log.WriteWarningAsync(nameof(CandlesSubscriber), nameof(CandlesUpdatedEvent), candlesUpdate.ToJson(), message); return; } var candles = candlesUpdate.Candles .Where(candleUpdate => Constants.StoredIntervals.Contains(candleUpdate.TimeInterval) && _candlesChecker.CanHandleAssetPair(candleUpdate.AssetPairId)) .Select(candleUpdate => Candle.Create( priceType: candleUpdate.PriceType, assetPair: candleUpdate.AssetPairId, timeInterval: candleUpdate.TimeInterval, timestamp: candleUpdate.CandleTimestamp, open: candleUpdate.Open, close: candleUpdate.Close, low: candleUpdate.Low, high: candleUpdate.High, tradingVolume: candleUpdate.TradingVolume, tradingOppositeVolume: candleUpdate.TradingOppositeVolume, lastTradePrice: candleUpdate.LastTradePrice, lastUpdateTimestamp: candleUpdate.ChangeTimestamp)) .ToArray(); await _candlesManager.ProcessCandlesAsync(candles); } catch (Exception) { await _log.WriteWarningAsync(nameof(CandlesSubscriber), nameof(ProcessCandlesUpdatedEventAsync), candlesUpdate.ToJson(), "Failed to process candle"); throw; } }
private Task ProcessCandleAsync(CandlesUpdatedEvent updatedCandles) { try { var validationErrors = ValidateCandle(updatedCandles); if (validationErrors.Any()) { var message = string.Join("\r\n", validationErrors); _log.WriteWarning(nameof(ProcessCandleAsync), updatedCandles, message); return(Task.CompletedTask); } _candlesManager.ProcessCandles(updatedCandles, _marketType); } catch (Exception) { _log.WriteWarning(nameof(ProcessCandleAsync), updatedCandles, "Failed to process candle"); throw; } return(Task.CompletedTask); }
private static IReadOnlyCollection <string> ValidateCandle(CandlesUpdatedEvent updatedCandles) { var errors = new List <string>(); if (updatedCandles == null) { errors.Add($"'{nameof(updatedCandles)}' is null."); return(errors); } if (updatedCandles.ContractVersion == null) { errors.Add("Contract version is not specified"); return(errors); } if (updatedCandles.ContractVersion.Major != Constants.ContractVersion.Major && // Version 2 and 3 are still supported updatedCandles.ContractVersion.Major != 2 && updatedCandles.ContractVersion.Major != 3) { errors.Add("Unsupported contract version"); return(errors); } if (updatedCandles.Candles == null || !updatedCandles.Candles.Any()) { errors.Add("Candles is empty"); return(errors); } for (var i = 0; i < updatedCandles.Candles.Count; ++i) { var candle = updatedCandles.Candles[i]; if (string.IsNullOrWhiteSpace(candle.AssetPairId)) { errors.Add($"Empty 'AssetPair' in the candle {i}"); } if (candle.CandleTimestamp.Kind != DateTimeKind.Utc) { errors.Add($"Invalid 'CandleTimestamp' Kind (UTC is required) in the candle {i}"); } if (candle.TimeInterval == CandleTimeInterval.Unspecified) { errors.Add($"Invalid 'TimeInterval' in the candle {i}"); } if (candle.PriceType == CandlePriceType.Unspecified) { errors.Add($"Invalid 'PriceType' in the candle {i}"); } } return(errors); }
private static IReadOnlyCollection<string> ValidateQuote(CandlesUpdatedEvent message) { var errors = new List<string>(); if (message == null) { errors.Add("message is null."); return errors; } if (message.ContractVersion == null) { errors.Add("Contract version is not specified"); return errors; } if (message.ContractVersion.Major != CandlesProducer.Contract.Constants.ContractVersion.Major && // Version 2 and 3 is still supported message.ContractVersion.Major != 2 && message.ContractVersion.Major != 3) { errors.Add("Unsupported contract version"); return errors; } if (message.Candles == null || !message.Candles.Any()) { errors.Add("Candles is empty"); return errors; } for (var i = 0; i < message.Candles.Count; ++i) { var candle = message.Candles[i]; if (string.IsNullOrWhiteSpace(candle.AssetPairId)) { errors.Add($"Empty '{nameof(candle.AssetPairId)}' in the candle {i}"); } if (candle.CandleTimestamp.Kind != DateTimeKind.Utc) { errors.Add($"Invalid '{candle.CandleTimestamp}' kind (UTC is required) in the candle {i}"); } if (candle.TimeInterval == CandleTimeInterval.Unspecified) { errors.Add($"Invalid 'TimeInterval' in the candle {i}"); } if (candle.PriceType == CandlePriceType.Unspecified) { errors.Add($"Invalid 'PriceType' in the candle {i}"); } } return errors; }