private async Task ProcessMessageAsync(CandlesUpdatedEvent message)
{
if (message.Candles == null || !message.Candles.Any())
{
return;
}
var intervalToSkip = new List <CandleTimeInterval>
{
CandleTimeInterval.Unspecified,
CandleTimeInterval.Sec,
CandleTimeInterval.Minute
};
var candlesToStore = message.Candles.Where(x =>
!string.IsNullOrEmpty(x.AssetPairId) && x.IsLatestChange &&
!intervalToSkip.Contains(x.TimeInterval) && x.PriceType != CandlePriceType.Unspecified)
.ToList();
var candles = _mapper.Map <List <CandleEntity> >(candlesToStore);
await _candlesWriter.BulkInsertOrReplaceAsync(candles);
Task.Run(async() =>
{
await _candlesWriter.CleanAndKeepMaxPartitions(0);
});
}
private Task PublishV2Async(IEnumerable <CandleUpdateResult> updates)
{
var @event = new CandlesUpdatedEvent
{
ContractVersion = Contract.Constants.ContractVersion,
UpdateTimestamp = DateTime.UtcNow,
Candles = updates
.Select(c => new CandleUpdate
{
IsLatestChange = c.IsLatestChange,
ChangeTimestamp = c.Candle.LatestChangeTimestamp,
AssetPairId = c.Candle.AssetPairId,
PriceType = c.Candle.PriceType,
TimeInterval = c.Candle.TimeInterval,
CandleTimestamp = c.Candle.Timestamp,
Open = c.Candle.Open,
Close = c.Candle.Close,
Low = c.Candle.Low,
High = c.Candle.High,
TradingVolume = c.Candle.TradingVolume,
TradingOppositeVolume = c.Candle.TradingOppositeVolume,
IsLatestCandle = true,
LastTradePrice = 0
})
.ToArray()
};
lock (_publisher)
{
// HACK: Actually ProduceAsync is not async, so lock works well
return(_publisher.ProduceAsync(@event));
}
}
public void ProcessCandles(CandlesUpdatedEvent updatedCandles, MarketType market)
{
foreach (var candle in updatedCandles.Candles.Where(c => c.IsLatestCandle))
{
ProcessCandleAsync(candle, market);
}
}
private static bool IsValid(CandlesUpdatedEvent candleEvent)
{
return(candleEvent.Candles.All(c =>
!string.IsNullOrWhiteSpace(c.AssetPairId) && c.AssetPairId.Length <= _maxStringFieldsLength &&
c.High >= c.Low &&
c.CandleTimestamp <= c.ChangeTimestamp));
}
public void ProcessCandles(CandlesUpdatedEvent candleEvent)
{
foreach (var candle in candleEvent.Candles)
{
if (candle.TimeInterval != CandleTimeInterval.Minute)
{
continue;
}
if (candle.PriceType != CandlePriceType.Ask && candle.PriceType != CandlePriceType.Bid)
{
continue;
}
var candlestick = new OutCandlestick
{
AssetPairId = candle.AssetPairId,
IsAsk = candle.PriceType == CandlePriceType.Ask,
High = (decimal)candle.High,
Low = (decimal)candle.Low,
Open = (decimal)candle.Open,
Close = (decimal)candle.Close,
Start = DateTimeConverter.Convert(candle.CandleTimestamp),
Finish = DateTimeConverter.Convert(candle.ChangeTimestamp),
};
string key = GetKey(candle);
var start = candle.CandleTimestamp;
if (_candlesDict.ContainsKey(key))
{
var datesDict = _candlesDict[key];
if (datesDict.ContainsKey(start))
{
datesDict[start] = Merge(datesDict[start], candlestick);
}
else
{
datesDict.Add(start, candlestick);
}
}
else
{
var datesDict = new Dictionary <DateTime, OutCandlestick>
{
{ start, candlestick },
};
_candlesDict.Add(key, datesDict);
}
}
}
private async Task ProcessCandlesUpdatedEventAsync(CandlesUpdatedEvent candlesUpdate)
{
try
{
if (_cacheInitalizationService.InitializationState != CacheInitializationState.Idle)
{
await Task.Delay(5000);
throw new InvalidOperationException("Initialization in progress");
}
var validationErrors = ValidateQuote(candlesUpdate);
if (validationErrors.Any())
{
var message = string.Join("\r\n", validationErrors);
_log.Warning(nameof(ProcessCandlesUpdatedEventAsync), message, context: candlesUpdate.ToJson());
return;
}
var candles = candlesUpdate.Candles
.Where(candleUpdate =>
_candlesChecker.CanHandleAssetPair(candleUpdate.AssetPairId))
.Select(candleUpdate => Candle.Create(
priceType: candleUpdate.PriceType,
assetPair: candleUpdate.AssetPairId,
timeInterval: candleUpdate.TimeInterval,
timestamp: candleUpdate.CandleTimestamp,
open: candleUpdate.Open,
close: candleUpdate.Close,
low: candleUpdate.Low,
high: candleUpdate.High,
tradingVolume: candleUpdate.TradingVolume,
tradingOppositeVolume: candleUpdate.TradingOppositeVolume,
lastTradePrice: candleUpdate.LastTradePrice,
lastUpdateTimestamp: candleUpdate.ChangeTimestamp))
.ToArray();
await _candlesManager.ProcessCandlesAsync(candles);
}
catch (Exception)
{
_log.Warning(nameof(ProcessCandlesUpdatedEventAsync), "Failed to process candle", context: candlesUpdate.ToJson());
throw;
}
}
private async Task ProcessCandlesUpdatedEventAsync(CandlesUpdatedEvent candlesUpdate)
{
try
{
var validationErrors = ValidateQuote(candlesUpdate);
if (validationErrors.Any())
{
var message = string.Join("\r\n", validationErrors);
await _log.WriteWarningAsync(nameof(CandlesSubscriber), nameof(CandlesUpdatedEvent), candlesUpdate.ToJson(), message);
return;
}
var candles = candlesUpdate.Candles
.Where(candleUpdate =>
Constants.StoredIntervals.Contains(candleUpdate.TimeInterval) &&
_candlesChecker.CanHandleAssetPair(candleUpdate.AssetPairId))
.Select(candleUpdate => Candle.Create(
priceType: candleUpdate.PriceType,
assetPair: candleUpdate.AssetPairId,
timeInterval: candleUpdate.TimeInterval,
timestamp: candleUpdate.CandleTimestamp,
open: candleUpdate.Open,
close: candleUpdate.Close,
low: candleUpdate.Low,
high: candleUpdate.High,
tradingVolume: candleUpdate.TradingVolume,
tradingOppositeVolume: candleUpdate.TradingOppositeVolume,
lastTradePrice: candleUpdate.LastTradePrice,
lastUpdateTimestamp: candleUpdate.ChangeTimestamp))
.ToArray();
await _candlesManager.ProcessCandlesAsync(candles);
}
catch (Exception)
{
await _log.WriteWarningAsync(nameof(CandlesSubscriber), nameof(ProcessCandlesUpdatedEventAsync), candlesUpdate.ToJson(), "Failed to process candle");
throw;
}
}
private Task ProcessCandleAsync(CandlesUpdatedEvent updatedCandles)
{
try
{
var validationErrors = ValidateCandle(updatedCandles);
if (validationErrors.Any())
{
var message = string.Join("\r\n", validationErrors);
_log.WriteWarning(nameof(ProcessCandleAsync), updatedCandles, message);
return(Task.CompletedTask);
}
_candlesManager.ProcessCandles(updatedCandles, _marketType);
}
catch (Exception)
{
_log.WriteWarning(nameof(ProcessCandleAsync), updatedCandles, "Failed to process candle");
throw;
}
return(Task.CompletedTask);
}
private static IReadOnlyCollection <string> ValidateCandle(CandlesUpdatedEvent updatedCandles)
{
var errors = new List <string>();
if (updatedCandles == null)
{
errors.Add($"'{nameof(updatedCandles)}' is null.");
return(errors);
}
if (updatedCandles.ContractVersion == null)
{
errors.Add("Contract version is not specified");
return(errors);
}
if (updatedCandles.ContractVersion.Major != Constants.ContractVersion.Major &&
// Version 2 and 3 are still supported
updatedCandles.ContractVersion.Major != 2 &&
updatedCandles.ContractVersion.Major != 3)
{
errors.Add("Unsupported contract version");
return(errors);
}
if (updatedCandles.Candles == null || !updatedCandles.Candles.Any())
{
errors.Add("Candles is empty");
return(errors);
}
for (var i = 0; i < updatedCandles.Candles.Count; ++i)
{
var candle = updatedCandles.Candles[i];
if (string.IsNullOrWhiteSpace(candle.AssetPairId))
{
errors.Add($"Empty 'AssetPair' in the candle {i}");
}
if (candle.CandleTimestamp.Kind != DateTimeKind.Utc)
{
errors.Add($"Invalid 'CandleTimestamp' Kind (UTC is required) in the candle {i}");
}
if (candle.TimeInterval == CandleTimeInterval.Unspecified)
{
errors.Add($"Invalid 'TimeInterval' in the candle {i}");
}
if (candle.PriceType == CandlePriceType.Unspecified)
{
errors.Add($"Invalid 'PriceType' in the candle {i}");
}
}
return(errors);
}
private static IReadOnlyCollection<string> ValidateQuote(CandlesUpdatedEvent message)
{
var errors = new List<string>();
if (message == null)
{
errors.Add("message is null.");
return errors;
}
if (message.ContractVersion == null)
{
errors.Add("Contract version is not specified");
return errors;
}
if (message.ContractVersion.Major != CandlesProducer.Contract.Constants.ContractVersion.Major &&
// Version 2 and 3 is still supported
message.ContractVersion.Major != 2 &&
message.ContractVersion.Major != 3)
{
errors.Add("Unsupported contract version");
return errors;
}
if (message.Candles == null || !message.Candles.Any())
{
errors.Add("Candles is empty");
return errors;
}
for (var i = 0; i < message.Candles.Count; ++i)
{
var candle = message.Candles[i];
if (string.IsNullOrWhiteSpace(candle.AssetPairId))
{
errors.Add($"Empty '{nameof(candle.AssetPairId)}' in the candle {i}");
}
if (candle.CandleTimestamp.Kind != DateTimeKind.Utc)
{
errors.Add($"Invalid '{candle.CandleTimestamp}' kind (UTC is required) in the candle {i}");
}
if (candle.TimeInterval == CandleTimeInterval.Unspecified)
{
errors.Add($"Invalid 'TimeInterval' in the candle {i}");
}
if (candle.PriceType == CandlePriceType.Unspecified)
{
errors.Add($"Invalid 'PriceType' in the candle {i}");
}
}
return errors;
}
