public async Task FlushCandlesIfAny()
{
var dataWritingTasks = new List <Task>();
foreach (var interval in Candles.Constants.DbStoredIntervals)
{
// If there still remain any active candles which have not been added to persistent queue till this moment.
if (_activeCandles.TryGetValue(interval, out var unsavedActiveCandle) &&
_persistenceCandleQueue[interval].All(c => c.Timestamp != unsavedActiveCandle.Timestamp))
{
// But we save only candles which are fully completed by the _upperDateLimit moment.
// I.e., if we have _upperDateLimit = 2018.03.05 16:00:15, we should store only the:
// - second candles with timestamp not later than 16:00:14;
// - minute candles not later than 15:59:00;
// - hour candles not later than 15:00:00;
// - day candle not later than 2018.03.04;
// - week candles not later than 2018.02.26;
// - and, finally, month candles not later than 2018.02.
if (_upperDateLimit == null ||
unsavedActiveCandle.Timestamp.AddIntervalTicks(1, unsavedActiveCandle.TimeInterval) <= _upperDateLimit)
{
_persistenceCandleQueue[interval].Add(unsavedActiveCandle);
}
}
// And now, we save the resulting candles to storage (if any).
if (!_persistenceCandleQueue[interval].Any())
{
continue;
}
dataWritingTasks.Add(_historyRepo.ReplaceCandlesAsync(_persistenceCandleQueue[interval]));
_healthService[_assetPairId].SummarySavedCandles +=
_persistenceCandleQueue[interval].Count;
_persistenceCandleQueue[interval].Clear(); // Protection against multiple calls
}
await Task.WhenAll(dataWritingTasks);
_healthService.Health.PersistenceQueueSize = 0;
}
private async Task FixMidPricesAsync(string assetPairId, int assetPairAccuracy, bool analyzeOnly)
{
try
{
_log.Info($"Starting mid candle prices fix for {assetPairId}...");
Health.Message = "Getting candles to fix...";
var candles = await GetFixedCandlesAsync(assetPairId, assetPairAccuracy);
if (!candles.Any())
{
_log.Info($"There are no candles to fix for {assetPairId}. Skipping.");
return;
}
if (analyzeOnly)
{
_log.Info($"Mid candle prices fix for {assetPairId} finished: analyze only. Candles to fix: {Health.CorruptedCandlesCount}");
return;
}
//sort by interval
var candlesByInterval = candles
.GroupBy(c => (int)c.TimeInterval)
.ToSortedDictionary(g => g.Key);
Health.Message = "Fixing candles from Sec to bigger intervals...";
//replace candles from Sec to bigger intervals
foreach (var candleBatch in candlesByInterval)
{
var candlesToReplace = candleBatch.Value.ToList();
await _candlesHistoryRepository.ReplaceCandlesAsync(candlesToReplace);
Health.FixedCandlesCount += candlesToReplace.Count;
}
_log.Info($"Mid candle prices fix for {assetPairId} finished.");
}
catch (Exception ex)
{
Health.Errors.Add($"{assetPairId}: {ex.Message}");
_log.Error(nameof(FixMidPricesAsync), ex);
}
}
public async Task <(int deletedCandlesCount, int replacedCandlesCount)> FixExtremeCandlesAsync(IReadOnlyList <ICandle> extremeCandles, CandlePriceType priceType)
{
// Okay, the incorrect candles are listed. Now we need to group it by time intervals and iterate from the
// smallest interval to the biggest, back. But now we will delete the smallest candles at all, and then
// recalculate (and replace in storage) the bigger candles.
int deletedCountSummary = 0, replacedCountSummary = 0;
var candlesByInterval = extremeCandles
.GroupBy(c => (int)c.TimeInterval)
.ToSortedDictionary(g => g.Key);
if (candlesByInterval.Count != Constants.DbStoredIntervals.Length)
{
throw new ArgumentException($"Something is wrong: the amount of (unique) time intervals in extreme candles list is not equal to stored intervals array length. " +
$"Filtration for {priceType} is impossible.");
}
foreach (var candleBatch in candlesByInterval)
{
var interval = (CandleTimeInterval)candleBatch.Key;
// The Second time interval is the smallest, so, we simply delete such a candles from storage and go next.
if (interval == CandleTimeInterval.Sec)
{
var deletedCountQuant =
await _candlesHistoryRepository.DeleteCandlesAsync(candleBatch.Value.ToList());
deletedCountSummary += deletedCountQuant;
continue;
}
var smallerInterval = GetSmallerInterval(interval);
// My dear friend, who will read or refactor this code in future, please, excuse me for such a terrible
// loop cascade. Currently, I just can't imagine how to make it shorter and more comfortable for reading.
// And from the other side, there is no such a necessity to invent an ideal bicycle here.
// ReSharper disable once PossibleNullReferenceException
var currentCandlesToReplace = new List <ICandle>();
var currentCandlesToDelete = new List <ICandle>();
foreach (var candle in candleBatch.Value.AsEnumerable())
{
var dateFrom = candle.Timestamp;
var dateTo = candle.Timestamp.AddIntervalTicks(1, interval);
var smallerCandles = await _candlesHistoryRepository.GetCandlesAsync(candle.AssetPairId,
smallerInterval, priceType, dateFrom, dateTo);
// Trying to reconstruct the candle from the corresponfing smaller interval candles, if any.
ICandle updatedCandle = null;
foreach (var smallerCandle in smallerCandles)
{
if (updatedCandle == null)
{
updatedCandle = smallerCandle
.RebaseToInterval(interval);
}
else
{
updatedCandle = updatedCandle
.ExtendBy(smallerCandle
.RebaseToInterval(interval));
}
}
// If the candle is not empty (e.g., there are some smaller interval candles for its construction),
// we should update it in storage. Otherwise, it is to be deleted from there.
if (updatedCandle != null)
{
currentCandlesToReplace.Add(updatedCandle);
}
else
{
currentCandlesToDelete.Add(candle);
}
}
if (currentCandlesToDelete.Count > 0)
{
var deletedCountQuant = await _candlesHistoryRepository.DeleteCandlesAsync(currentCandlesToDelete);
deletedCountSummary += deletedCountQuant;
}
if (currentCandlesToReplace.Count > 0)
{
var replacedCountQuant = await _candlesHistoryRepository.ReplaceCandlesAsync(currentCandlesToReplace);
replacedCountSummary += replacedCountQuant;
}
}
return(deletedCandlesCount : deletedCountSummary, replacedCandlesCount : replacedCountSummary);
}