private Candle(
string assetPairId,
CandlePriceType priceType,
CandleTimeInterval timeInterval,
DateTime timestamp,
DateTime latestChangeTimestamp,
DateTime openTimestamp,
double open,
double close,
double low,
double high,
double tradingVolume,
double tradingOppositeVolume)
{
AssetPairId = assetPairId;
PriceType = priceType;
TimeInterval = timeInterval;
Timestamp = timestamp;
LatestChangeTimestamp = latestChangeTimestamp;
OpenTimestamp = openTimestamp;
Open = open;
Close = close;
Low = low;
High = high;
TradingVolume = tradingVolume;
TradingOppositeVolume = tradingOppositeVolume;
}
private static ICandle DeserializeCandle(byte[] value, string assetPairId, CandlePriceType priceType, CandleTimeInterval timeInterval)
{
// value is:
// 0 .. TimestampFormat.Length - 1 bytes: timestamp as yyyyMMddHHmmss in ASCII
// TimestampFormat.Length .. end bytes: serialized RedistCachedCandle
var timestampLength = TimestampFormat.Length;
var timestampString = Encoding.ASCII.GetString(value, 0, timestampLength);
var timestamp = DateTime.ParseExact(timestampString, TimestampFormat, CultureInfo.InvariantCulture);
using (var stream = new MemoryStream(value, timestampLength, value.Length - timestampLength, writable: false))
{
var cachedCandle = MessagePack.MessagePackSerializer.Deserialize <RedisCachedCandle>(stream);
return(Candle.Create(
assetPairId,
priceType,
timeInterval,
timestamp,
cachedCandle.Open,
cachedCandle.Close,
cachedCandle.High,
cachedCandle.Low,
cachedCandle.TradingVolume,
cachedCandle.TradingOppositVolume,
cachedCandle.LastTradePrice,
cachedCandle.LastUpdateTimestamp));
}
}
public Task TruncateCacheAsync(string assetId, CandlePriceType priceType, CandleTimeInterval timeInterval,
int storedCandlesCountLimit, SlotType slotType)
{
var key = GetKey(_market, assetId, priceType, timeInterval, slotType);
return(_database.SortedSetRemoveRangeByRankAsync(key, 0, -storedCandlesCountLimit - 1, CommandFlags.FireAndForget));
}
public IReadOnlyList <ICandle> FillGapUpTo(AssetPair assetPair, CandlePriceType priceType, DateTime dateTime, ICandle endCandle)
{
var key = GetKey(assetPair.Id, priceType);
_lastCandles.TryGetValue(key, out var lastCandle);
if (lastCandle == null)
{
return(new List <ICandle>());
}
var lastCandleHeight = lastCandle.High - lastCandle.Low;
var endCandleHeight = endCandle?.High - endCandle?.Low ?? lastCandleHeight;
var spread = CalculateSpread(lastCandleHeight, endCandleHeight);
var result = GenerateCandles(
assetPair,
priceType,
lastCandle.Timestamp,
dateTime,
lastCandle.Close,
endCandle?.Open ?? lastCandle.Open,
spread)
.ToList();
// Remember the last candle, if any
if (result.Any())
{
_lastCandles[key] = Candle.Copy(result.Last());
}
return(result);
}
private Candle(
string assetPair,
CandlePriceType priceType,
CandleTimeInterval timeInterval,
DateTime timestamp,
double open,
double close,
double high,
double low,
double tradingVolume,
double tradingOppositeVolume,
double lastTradePrice,
DateTime lastUpdateTimestamp)
{
AssetPairId = assetPair;
PriceType = priceType;
TimeInterval = timeInterval;
Timestamp = timestamp;
Open = open;
Close = close;
High = high;
Low = low;
TradingVolume = tradingVolume;
TradingOppositeVolume = tradingOppositeVolume;
LastTradePrice = lastTradePrice;
LastUpdateTimestamp = lastUpdateTimestamp;
}
public static Candle Create(
string assetPair,
CandlePriceType priceType,
CandleTimeInterval timeInterval,
DateTime timestamp,
double open,
double close,
double high,
double low,
double tradingVolume,
double tradingOppositeVolume,
double lastTradePrice,
DateTime lastUpdateTimestamp)
{
return(new Candle(
assetPair,
priceType,
timeInterval,
timestamp.TruncateTo(timeInterval),
open,
close,
high,
low,
tradingVolume,
tradingOppositeVolume,
lastTradePrice,
lastUpdateTimestamp));
}
public async Task <IEnumerable <ICandle> > GetLastCandlesAsync(CandlePriceType priceType, CandleTimeInterval interval, DateTime to, int number)
{
var whereClause =
"WHERE PriceType=@priceTypeVar AND TimeInterval=@intervalVar AND Timestamp <= @toVar";
using (var conn = new SqlConnection(_connectionString))
{
try
{
var objects = await conn.QueryAsync <SqlCandleHistoryItem>($"SELECT TOP {number} * FROM {_tableName} {whereClause} ORDER BY Timestamp DESC",
new { priceTypeVar = priceType, intervalVar = interval, toVar = to }, null, commandTimeout : ReadCommandTimeout);
return(objects.OrderBy(x => x.Timestamp));
}
catch (Exception ex)
{
_log?.WriteErrorAsync(nameof(SqlCandlesHistoryRepository), nameof(GetLastCandlesAsync),
new
{
message = "Failed to get an candle list",
priceType,
interval,
to,
number,
_tableName
}.ToJson(), ex);
return(Enumerable.Empty <ICandle>());
}
}
}
public static Candle CreateQuotingCandle(
string assetPair,
DateTime timestamp,
double price,
CandlePriceType priceType,
CandleTimeInterval timeInterval)
{
if (priceType != CandlePriceType.Ask &&
priceType != CandlePriceType.Bid &&
priceType != CandlePriceType.Mid)
{
throw new ArgumentOutOfRangeException(nameof(priceType), priceType, "Price type should be Ask, Bid or Mid for the quoting candle");
}
var intervalTimestamp = timestamp.TruncateTo(timeInterval);
return new Candle
(
assetPair,
priceType,
timeInterval,
intervalTimestamp,
timestamp,
timestamp,
price,
price,
price,
price,
0,
0
);
}
public async Task <IEnumerable <ICandle> > GetLastCandlesAsync(string assetPairId, CandleTimeInterval interval,
CandlePriceType priceType, DateTime to, int number)
{
var repo = GetRepo(assetPairId);
return(await repo.GetLastCandlesAsync(priceType, interval, to, number));
}
/// <summary>
/// Assumed that all candles have the same AssetPair, PriceType, and Timeinterval
/// </summary>
public async Task InsertOrMergeAsync(IEnumerable <ICandle> candles, CandlePriceType priceType)
{
var partitionKey = CandleHistoryEntity.GeneratePartitionKey(priceType);
// Despite of AzureTableStorage already split requests to chunks,
// splits to the chunks here to reduse cost of operation timeout
var candleByRowsChunks = candles
.GroupBy(candle => CandleHistoryEntity.GenerateRowKey(candle.Timestamp, _timeInterval))
.Batch(100);
foreach (var candleByRowsChunk in candleByRowsChunks)
{
// If we can't store the candles, we can't do anything else, so just retries until success
await Policy
.Handle <Exception>()
.WaitAndRetryForeverAsync(
retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
(exception, timeSpan) =>
{
var context = $"{_assetPairId}-{priceType}-{_timeInterval}";
return(_log.WriteErrorAsync("Persist candle rows chunk with retries", context, exception));
})
.ExecuteAsync(() => SaveCandlesBatchAsync(candleByRowsChunk, partitionKey));
}
}
private decimal GetLastNonZeroPrice(string assetPair, CandlePriceType priceType)
{
var key = GetKey(assetPair, priceType);
_lastNonZeroPrices.TryGetValue(key, out var price);
return(price);
}
/// <summary>
/// Finds out the oldest stored candle's timestamp (if any).
/// </summary>
/// <param name="assetPairId"></param>
/// <param name="priceType"></param>
/// <param name="interval"></param>
/// <returns>The oldest candle or null./></returns>
/// <exception cref="InvalidOperationException">If the specified asset pair is not currently supported by storage.</exception>
public async Task <ICandle> TryGetOldestCandleAsync(string assetPairId, CandlePriceType priceType,
CandleTimeInterval interval)
{
CheckupAssetPairOrFail(assetPairId);
var firstCandle = await _candlesHistoryRepository.TryGetFirstCandleAsync(assetPairId, interval, priceType);
return(firstCandle); // The risk of the null is minimal but not excluded.
}
public async Task <DateTime?> GetStartDateAsync(string assetPair, CandlePriceType priceType)
{
var oldestFeedHistory = await _feedHistoryRepository.GetTopRecordAsync(assetPair, priceType);
return(oldestFeedHistory
?.Candles
.First()
.ToCandle(assetPair, priceType, oldestFeedHistory.DateTime).Timestamp);
}
/// <summary>
/// Initializes a new instance of the GetCandlesHistoryBatchRequest
/// class.
/// </summary>
/// <param name="priceType">Possible values include: 'Unspecified',
/// 'Bid', 'Ask', 'Mid', 'Trades'</param>
/// <param name="timeInterval">Possible values include: 'Unspecified',
/// 'Sec', 'Minute', 'Min5', 'Min15', 'Min30', 'Hour', 'Hour4',
/// 'Hour6', 'Hour12', 'Day', 'Week', 'Month'</param>
/// <param name="fromMoment">Inclusive from moment</param>
/// <param name="toMoment">Exclusive to moment. If equals to the
/// Lykke.Service.CandlesHistory.Models.CandlesHistory.GetCandlesHistoryBatchRequest.FromMoment,
/// then exactly candle for exactly this moment will be
/// returned</param>
public GetCandlesHistoryBatchRequest(CandlePriceType priceType, CandleTimeInterval timeInterval, System.DateTime fromMoment, System.DateTime toMoment, IList <string> assetPairs = default(IList <string>))
{
AssetPairs = assetPairs;
PriceType = priceType;
TimeInterval = timeInterval;
FromMoment = fromMoment;
ToMoment = toMoment;
CustomInit();
}
public async Task <ICandle> TryGetFirstCandleAsync(CandlePriceType priceType, CandleTimeInterval timeInterval)
{
var candleEntity = await _tableStorage.GetTopRecordAsync(CandleHistoryEntity.GeneratePartitionKey(priceType));
return(candleEntity
?.Candles
.First()
.ToCandle(_assetPairId, priceType, candleEntity.DateTime, timeInterval));
}
public async Task <ICandle> TryGetFirstCandleAsync(CandlePriceType priceType, CandleTimeInterval timeInterval)
{
using (var conn = new SqlConnection(_connectionString))
{
var candle = await conn.QueryFirstOrDefaultAsync <SqlCandleHistoryItem>(
$"SELECT TOP(1) * FROM {_tableName} WHERE PriceType=@priceTypeVar AND TimeInterval=@intervalVar ",
new { priceTypeVar = priceType, intervalVar = timeInterval });
return(candle);
}
}
// ReSharper disable once UnusedMember.Global
public static async Task <IReadOnlyDictionary <string, CandlesHistoryResponseModel> > TryGetCandlesHistoryBatchAsync(
this ICandleshistoryservice service,
IList <string> assetPairs,
CandlePriceType priceType,
CandleTimeInterval timeInterval,
DateTime fromMoment,
DateTime toMoment,
CancellationToken cancellationToken = default(CancellationToken))
{
var result = await service.GetCandlesHistoryBatchOrErrorAsync(new GetCandlesHistoryBatchRequest(priceType, timeInterval, fromMoment, toMoment, assetPairs), cancellationToken);
return(result as IReadOnlyDictionary <string, CandlesHistoryResponseModel>);
}
public async Task <IActionResult> GetCandlesHistory(string assetPairId, CandlePriceType priceType, CandleTimeInterval timeInterval, DateTime fromMoment, DateTime toMoment)
{
fromMoment = fromMoment.ToUniversalTime();
toMoment = toMoment.ToUniversalTime();
if (string.IsNullOrWhiteSpace(assetPairId))
{
return(BadRequest(ErrorResponse.Create(nameof(assetPairId), "Asset pair is required")));
}
if (priceType == CandlePriceType.Unspecified)
{
return(BadRequest(ErrorResponse.Create(nameof(timeInterval), $"Price type should not be {CandlePriceType.Unspecified}")));
}
if (timeInterval == CandleTimeInterval.Unspecified)
{
return(BadRequest(ErrorResponse.Create(nameof(timeInterval), $"Time interval should not be {CandleTimeInterval.Unspecified}")));
}
if (fromMoment > toMoment)
{
return(BadRequest(ErrorResponse.Create("From date should be early or equal than To date")));
}
if (!_candleHistoryAssetConnections.ContainsKey(assetPairId))
{
return(BadRequest(ErrorResponse.Create(nameof(assetPairId), "Asset pair is not configured")));
}
if (await _assetPairsManager.TryGetAssetPairAsync(assetPairId) == null)
{
return(BadRequest(ErrorResponse.Create(nameof(assetPairId), "Asset pair not found")));
}
var activeSlot = await _candlesManager.GetActiveSlotAsync();
var candles = await _candlesManager.GetCandlesAsync(assetPairId, priceType, timeInterval, fromMoment, toMoment, activeSlot);
// May return much less candles than it was requested or even an empty set of data for now the service looks
// only through the cache (no persistent data is used).
return(Ok(new CandlesHistoryResponseModel
{
History = candles.Select(c => new CandlesHistoryResponseModel.Candle
{
DateTime = c.Timestamp,
Open = c.Open,
Close = c.Close,
High = c.High,
Low = c.Low,
TradingVolume = c.TradingVolume,
TradingOppositeVolume = c.TradingOppositeVolume,
LastTradePrice = c.LastTradePrice
})
}));
}
public async Task <IEnumerable <ICandle> > GetCandlesAsync(CandlePriceType priceType, CandleTimeInterval interval, DateTime from, DateTime to)
{
if (priceType == CandlePriceType.Unspecified)
{
throw new ArgumentException(nameof(priceType));
}
var query = GetTableQuery(priceType, interval, from, to);
var entities = await _tableStorage.WhereAsync(query);
var candles = entities
.SelectMany(e => e.Candles.Select(ci => ci.ToCandle(_assetPairId, e.PriceType, e.DateTime, interval)));
return(candles.Where(c => c.Timestamp >= from && c.Timestamp < to));
}
public void GetCandlesHistoryValidateParametersNullAssetPairIdTest()
{
CandlePriceType priceType = CandlePriceType.Ask;
CandleTimeInterval timeInterval = CandleTimeInterval.Hour;
DateTime fromMoment = DateTime.Now.AddHours(-12).ToUniversalTime();
DateTime toMoment = DateTime.Now.ToUniversalTime();
MarketType marketType = MarketType.Mt;
Step("Make GET /api/candlesHistory without assetPairId and validate response", () =>
{
var response = apiV2.CandlesHistory.GetCandlesHistory(marketType, null, priceType, timeInterval, fromMoment, toMoment);
Assert.That(response.StatusCode, Is.EqualTo(HttpStatusCode.BadRequest));
});
}
public void GetCandlesHistoryPositiveSpotTest()
{
var marketType = MarketType.Spot;
var assetPairId = "BTCUSD";
CandlePriceType priceType = CandlePriceType.Ask;
CandleTimeInterval timeInterval = CandleTimeInterval.Hour;
DateTime fromMoment = DateTime.Now.AddHours(-12).ToUniversalTime();
DateTime toMoment = DateTime.Now.ToUniversalTime();
Step($"Make GET /api/candlesHistory with parameters: marketType: {marketType}, assetPairId: {assetPairId}, priceType: {priceType}, timeInterval: {timeInterval}, fromMoment: {fromMoment}, toMoment: {toMoment}", () =>
{
var response = apiV2.CandlesHistory.GetCandlesHistory(marketType, assetPairId, priceType, timeInterval, fromMoment, toMoment);
Assert.That(response.StatusCode, Is.EqualTo(HttpStatusCode.OK));
Assert.That(response.GetResponseObject(), Is.Not.Null);
});
}
public MigrationCandleMergeResult Merge(string assetPair, CandlePriceType priceType, CandleTimeInterval timeInterval, DateTime timestamp, double open, double close, double low, double high)
{
var key = GetKey(assetPair, timeInterval, priceType);
Candle oldCandle = null;
var newCandle = _candles.AddOrUpdate(key,
addValueFactory: k => Candle.Create(
assetPair: assetPair,
priceType: priceType,
timeInterval: timeInterval,
timestamp: timestamp,
open: open,
close: close,
high: high,
low: low,
tradingVolume: 0,
tradingOppositeVolume: 0,
lastTradePrice: 0,
lastUpdateTimestamp: timestamp),
updateValueFactory: (k, old) =>
{
oldCandle = old;
// Start new candle?
var intervalTimestamp = timestamp.TruncateTo(timeInterval);
if (oldCandle.Timestamp != intervalTimestamp)
{
return(Candle.Create(
assetPair: assetPair,
priceType: priceType,
timeInterval: timeInterval,
timestamp: intervalTimestamp,
open: open,
close: close,
high: high,
low: low,
tradingVolume: 0,
tradingOppositeVolume: 0,
lastTradePrice: 0,
lastUpdateTimestamp: timestamp));
}
return(oldCandle.Update(close, low, high, 0, 0, 0, timestamp));
});
return(new MigrationCandleMergeResult(newCandle, !newCandle.Equals(oldCandle)));
}
public async Task InitializeAsync(
string assetPairId,
CandlePriceType priceType,
CandleTimeInterval timeInterval,
IReadOnlyCollection <ICandle> candles,
SlotType slotType)
{
foreach (var candle in candles)
{
if (candle.AssetPairId != assetPairId)
{
throw new ArgumentException($"Candle {candle.ToJson()} has invalid AssetPriceId", nameof(candles));
}
if (candle.PriceType != priceType)
{
throw new ArgumentException($"Candle {candle.ToJson()} has invalid PriceType", nameof(candles));
}
if (candle.TimeInterval != timeInterval)
{
throw new ArgumentException($"Candle {candle.ToJson()} has invalid TimeInterval", nameof(candles));
}
}
// Since we have introduced a semaphore waiting in cache initialization service, we do not
// need any additional concurrent-safe actions here. Yes, it's better to wait a semaphore
// somewhere else but here 'cause otherwise we'll get a fully synchronous cache operation.
var key = GetKey(_market, assetPairId, priceType, timeInterval, slotType);
var tasks = new List <Task>
{
_database.KeyDeleteAsync(key)
};
foreach (var candlesBatch in candles.Batch(400))
{
var entites = candlesBatch
.Select(candle => new SortedSetEntry(SerializeCandle(candle), 0))
.ToArray();
tasks.Add(_database.SortedSetAddAsync(key, entites));
}
_database.WaitAll(tasks.ToArray());
await Task.WhenAll(tasks);
}
private async Task <decimal?> GetAvg(string assetPairId, CandlePriceType priceType, DateTime now)
{
var candlesHistory = await _candlesHistoryService.GetCandlesHistoryAsync(assetPairId, priceType,
CandleTimeInterval.Min5, now.AddHours(-12), now);
if (!candlesHistory.History.Any())
{
_log.WriteErrorAsync(GetComponentName(), nameof(GetAvg),
new Exception("No candles history found for " + assetPairId)
{
Data = { { "AssetPairId", assetPairId } }
});
return(null);
}
return((decimal)candlesHistory.History.SelectMany(h => new[] { h.Open, h.Close }).Average());
}
public async Task <IEnumerable <ICandle> > GetLastCandlesAsync(CandlePriceType priceType, CandleTimeInterval interval, DateTime to, int number)
{
if (priceType == CandlePriceType.Unspecified)
{
throw new ArgumentException(nameof(priceType));
}
var query = GetTableQuery(priceType, interval, DateTime.MinValue, to);
var entities = await _tableStorage.WhereAsync(query);
var candles = entities
.SelectMany(e => e.Candles.Select(ci => ci.ToCandle(_assetPairId, e.PriceType, e.DateTime, interval)));
return(candles.Where(c => c.Timestamp < to)
.OrderByDescending(c => c.Timestamp)
.Take(number)
.OrderBy(c => c.Timestamp));
}
public ICandle ToCandle(string assetPairId, CandlePriceType priceType, DateTime baseTime)
{
var timeStamp = baseTime.AddSeconds(Tick);
return(Candle.Create(
open: Open,
close: Close,
high: High,
low: Low,
assetPair: assetPairId,
priceType: priceType,
timeInterval: CandleTimeInterval.Sec,
timestamp: timeStamp,
tradingVolume: 0,
tradingOppositeVolume: 0,
lastTradePrice: 0,
lastUpdateTimestamp: timeStamp));
}
public IEnumerable <ICandle> GenerateCandles(
AssetPair assetPair,
CandlePriceType priceType,
DateTime exclusiveStartDate,
DateTime exclusiveEndDate,
double exclusiveStartPrice,
double exclusiveEndPrice,
double spread)
{
return(GenerateCandles(
assetPair,
priceType,
exclusiveStartDate,
exclusiveEndDate,
ConvertToDecimal(exclusiveStartPrice),
ConvertToDecimal(exclusiveEndPrice),
ConvertToDecimal(spread)));
}
public float GetPrice(CandlePriceType priceType)
{
switch (priceType)
{
case CandlePriceType.Close: return(close);
case CandlePriceType.Open: return(open);
case CandlePriceType.High: return(high);
case CandlePriceType.Low: return(low);
case CandlePriceType.HighLowMid: return((high + low) * 0.5f);
case CandlePriceType.OpenCloseMid: return((open + close) * 0.5f);
}
return(close);
}
public ICandle ToCandle(string assetPairId, CandlePriceType priceType, DateTime baseTime, CandleTimeInterval timeInterval)
{
var normalizedTick = Tick - GetIntervalTickOrigin(timeInterval);
return(Candle.Create
(
open: Open,
close: Close,
high: High,
low: Low,
assetPair: assetPairId,
priceType: priceType,
timeInterval: timeInterval,
timestamp: baseTime.AddIntervalTicks(normalizedTick, timeInterval),
tradingVolume: TradingVolume,
tradingOppositeVolume: TradingOppositeVolume,
lastTradePrice: LastTradePrice,
lastUpdateTimestamp: LastUpdateTimestamp
));
}
public void UpdateCurrentHistoryDate(DateTime date, CandlePriceType priceType)
{
switch (priceType)
{
case CandlePriceType.Bid:
CurrentBidDate = date;
break;
case CandlePriceType.Ask:
CurrentAskDate = date;
break;
case CandlePriceType.Mid:
CurrentMidDate = date;
break;
default:
throw new ArgumentOutOfRangeException(nameof(priceType), priceType, "Invalid price type");
}
}
public float GetPrice(CandlePriceType priceType)
{
switch (priceType)
{
case CandlePriceType.Close : return close;
case CandlePriceType.Open : return open;
case CandlePriceType.High : return high;
case CandlePriceType.Low : return low;
case CandlePriceType.HighLowMid : return (high + low) * 0.5f;
case CandlePriceType.OpenCloseMid : return (open + close) * 0.5f;
}
return close;
}
