public static void MergeCandles(this CandleTableEntity entity, IEnumerable <CandleItem> candles, TimeInterval interval)
{
foreach (var candle in candles)
{
entity.MergeCandle(candle, interval);
}
}
public static void MergeCandle(this CandleTableEntity entity, IFeedCandle candle, TimeInterval interval)
{
if (entity == null)
{
throw new ArgumentNullException(nameof(entity));
}
// 1. Check if candle with specified time already exist
// 2. If found - merge, else - add to list
//
var cell = candle.DateTime.GetIntervalCell(interval);
var tick = candle.DateTime.GetIntervalTick(interval);
var existingCandle = entity.Candles.FirstOrDefault(ci => ci.Tick == tick && ci.Cell == cell);
if (existingCandle != null)
{
// Merge in list
var mergedCandle = existingCandle
.ToCandle(entity.IsBuy, entity.DateTime, interval)
.MergeWith(candle);
entity.Candles.Remove(existingCandle);
entity.Candles.Add(mergedCandle.ToItem(interval));
}
else
{
// Add to list
entity.Candles.Add(candle.ToItem(interval));
}
}
public async Task <IEnumerable <IFeedCandle> > GetCandlesAsync(PriceType priceType, TimeInterval interval, DateTime from, DateTime to)
{
if (priceType == PriceType.Unspecified)
{
throw new ArgumentException(nameof(priceType));
}
string partitionKey = CandleTableEntity.GeneratePartitionKey(priceType);
string rowKeyFrom = CandleTableEntity.GenerateRowKey(from, interval);
string rowKeyTo = CandleTableEntity.GenerateRowKey(to, interval);
TableQuery <CandleTableEntity> query = new TableQuery <CandleTableEntity>();
string pkeyFilter = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey);
var rowkeyCondFrom = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, rowKeyFrom);
var rowkeyCondTo = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, rowKeyTo);
var rowkeyFilter = TableQuery.CombineFilters(rowkeyCondFrom, TableOperators.And, rowkeyCondTo);
query.FilterString = TableQuery.CombineFilters(pkeyFilter, TableOperators.And, rowkeyFilter);
IEnumerable <CandleTableEntity> entities = await _tableStorage.WhereAsync(query);
var result = from e in entities
select e.Candles.Select(ci => ci.ToCandle(e.PriceType == PriceType.Bid, e.DateTime, interval));
return(result
.SelectMany(c => c)
.Where(c => c.DateTime >= from && c.DateTime < to));
}
public static string RowKey(this IFeedCandle candle, TimeInterval interval)
{
if (candle == null)
{
throw new ArgumentNullException(nameof(candle));
}
return(CandleTableEntity.GenerateRowKey(candle.DateTime, candle.IsBuy, interval));
}
public static string PartitionKey(this IFeedCandle candle, string asset)
{
if (candle == null)
{
throw new ArgumentNullException(nameof(candle));
}
return(CandleTableEntity.GeneratePartitionKey(asset));
}
public static void MergeCandle(this CandleTableEntity entity, CandleItem candle, TimeInterval interval)
{
if (entity == null)
{
throw new ArgumentNullException(nameof(entity));
}
IFeedCandle fc = candle.ToCandle(entity.IsBuy, entity.DateTime, interval);
entity.MergeCandle(fc, interval);
}
private async Task InsertOrMergeAsync(IEnumerable <IFeedCandle> candles, string partitionKey, string rowKey, TimeInterval interval)
{
// Read row
CandleTableEntity entity = await _tableStorage.GetDataAsync(partitionKey, rowKey);
if (entity == null)
{
entity = new CandleTableEntity(partitionKey, rowKey);
}
// Merge all candles
entity.MergeCandles(candles, interval);
// Update
await _tableStorage.InsertOrMergeAsync(entity);
}
public async Task InsertOrMergeAsync(IFeedCandle candle, PriceType priceType, TimeInterval interval)
{
// Get candle table entity
string partitionKey = CandleTableEntity.GeneratePartitionKey(priceType);
string rowKey = CandleTableEntity.GenerateRowKey(candle.DateTime, interval);
CandleTableEntity entity = await _tableStorage.GetDataAsync(partitionKey, rowKey);
if (entity == null)
{
entity = new CandleTableEntity(partitionKey, rowKey);
}
// Merge candle
entity.MergeCandle(candle, interval);
// Update
await _tableStorage.InsertOrMergeAsync(entity);
}
public async Task <IFeedCandle> GetCandleAsync(string asset, TimeInterval interval, bool isBuy, DateTime dateTime)
{
if (string.IsNullOrEmpty(asset))
{
throw new ArgumentNullException(nameof(asset));
}
// 1. Get candle table entity
string partitionKey = CandleTableEntity.GeneratePartitionKey(asset);
string rowKey = CandleTableEntity.GenerateRowKey(dateTime, isBuy, interval);
//CandleTableEntity entity = await _tableStorage.GetDataAsync(partitionKey, rowKey);
//---------------
TableQuery <CandleTableEntity> query = new TableQuery <CandleTableEntity>();
string pkeyFilter = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey);
string rowkeyFilter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, rowKey);
query.FilterString = TableQuery.CombineFilters(pkeyFilter, TableOperators.And, rowkeyFilter);
query.SelectColumns = CandleTableEntity.GetStoreFields(interval, dateTime);
List <CandleTableEntity> entities = new List <CandleTableEntity>(1);
await _tableStorage.ScanDataAsync(query, list =>
{
entities.AddRange(list);
return(Task.FromResult(0));
});
//-----------
// 2. Find required candle in candle list by tick
if (entities.Count > 0)
{
var cell = dateTime.GetIntervalCell(interval);
var tick = dateTime.GetIntervalTick(interval);
var candleItem = entities[0].Candles.FirstOrDefault(ci => ci.Tick == tick && ci.Cell == cell);
return(candleItem.ToCandle(isBuy, entities[0].DateTime, interval));
}
return(null);
}
public async Task <IFeedCandle> GetCandleAsync(PriceType priceType, TimeInterval interval, DateTime dateTime)
{
if (priceType == PriceType.Unspecified)
{
throw new ArgumentException(nameof(priceType));
}
// 1. Get candle table entity
string partitionKey = CandleTableEntity.GeneratePartitionKey(priceType);
string rowKey = CandleTableEntity.GenerateRowKey(dateTime, interval);
CandleTableEntity entity = await _tableStorage.GetDataAsync(partitionKey, rowKey);
// 2. Find required candle in candle list by tick
if (entity != null)
{
var tick = dateTime.GetIntervalTick(interval);
var candleItem = entity.Candles.FirstOrDefault(ci => ci.Tick == tick);
return(candleItem.ToCandle(priceType == PriceType.Bid, entity.DateTime, interval));
}
return(null);
}
public async Task <IEnumerable <IFeedCandle> > GetCandlesAsync(string asset, TimeInterval interval, bool isBuy, DateTime from, DateTime to)
{
if (string.IsNullOrEmpty(asset))
{
throw new ArgumentNullException(nameof(asset));
}
string partitionKey = CandleTableEntity.GeneratePartitionKey(asset);
string rowKeyFrom = CandleTableEntity.GenerateRowKey(from, isBuy, interval);
string rowKeyTo = CandleTableEntity.GenerateRowKey(to, isBuy, interval);
//IEnumerable<CandleTableEntity> candleEntities = await _tableStorage.WhereAsync(partitionKey, from, to, ToIntervalOption.ExcludeTo, null, includeTime: true);
//---------------
TableQuery <CandleTableEntity> query = new TableQuery <CandleTableEntity>();
string pkeyFilter = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey);
string fromFilter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, rowKeyFrom);
string toFilter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, rowKeyTo);
query.FilterString = TableQuery.CombineFilters(pkeyFilter, TableOperators.And,
TableQuery.CombineFilters(fromFilter, TableOperators.And, toFilter));
List <CandleTableEntity> entities = new List <CandleTableEntity>(1);
await _tableStorage.ScanDataAsync(query, list =>
{
entities.AddRange(list);
return(Task.FromResult(0));
});
//-----------
var result = from e in entities
select e.Candles.Select(ci => ci.ToCandle(e.IsBuy, e.DateTime, interval));
return(result
.SelectMany(c => c)
.Where(c => c.DateTime >= from && c.DateTime < to));
}
public async Task InsertOrMergeAsync(IReadOnlyDictionary <TimeInterval, IEnumerable <IFeedCandle> > dict, string asset)
{
if (dict == null)
{
throw new ArgumentNullException(nameof(dict));
}
if (string.IsNullOrEmpty(asset))
{
throw new ArgumentNullException(nameof(asset));
}
if (!dict.Any() && dict.Values.All(e => e != null) && dict.Values.Any(e => e.Count() > 0))
{
return;
}
var partitionKey = CandleTableEntity.GeneratePartitionKey(asset);
var rowKeys = new List <string>();
var fields = new HashSet <string>(); // which fields to read from table
// 1. Read all { pkey, rowkey } rows
//
var updateEntities = new List <CandleTableEntity>();
foreach (var interval in dict.Keys)
{
var candles = dict[interval];
if (candles != null && candles.Any())
{
// Inside one interval group all candles by distinct row
var groups = candles.GroupBy(candle => candle.RowKey(interval));
rowKeys.AddRange(groups.Select(g => g.Key));
foreach (var group in groups)
{
// Create entity with candles and add it to list
var e = new CandleTableEntity(partitionKey, group.Key); // group.Key = rowKey
e.MergeCandles(group, interval);
updateEntities.Add(e);
// update field
var dates = group.Select(c => c.DateTime);
fields.UnionWith(CandleTableEntity.GetStoreFields(interval, dates.Min(), dates.Max()));
}
}
}
// ... prepare get query
// ... partitionKey = ? AND (rokey = ? OR rowkey = ? OR rowkey = ? OR ...)
TableQuery <CandleTableEntity> query = new TableQuery <CandleTableEntity>();
string pkeyFilter = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey);
var rowkeyFilters = rowKeys.Select(rowKey => TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, rowKey));
var rowkeyFilter = rowkeyFilters.Aggregate((cond1, cond2) => TableQuery.CombineFilters(cond1, TableOperators.Or, cond2));
query.FilterString = TableQuery.CombineFilters(pkeyFilter, TableOperators.And, rowkeyFilter);
query.SelectColumns = fields.ToList();
// ... reading rows from azure table
List <CandleTableEntity> origEntities = new List <CandleTableEntity>(1);
await _tableStorage.ScanDataAsync(query, list =>
{
origEntities.AddRange(list);
return(Task.FromResult(0));
});
// 2. Update rows (merge entities)
//
var listToUpdate = new List <CandleTableEntity>();
foreach (var updateEntity in updateEntities)
{
var origEntity = origEntities.Where(e => e.PartitionKey == updateEntity.PartitionKey && e.RowKey == updateEntity.RowKey).FirstOrDefault();
if (origEntity != null)
{
origEntity.MergeCandles(updateEntity.Candles, updateEntity.Interval);
listToUpdate.Add(origEntity);
}
else
{
listToUpdate.Add(updateEntity);
}
}
// 3. Write rows in batch
// ... Only 100 records with the same pKey can be updated in one batch operation
foreach (var collection in listToUpdate.ToPieces(100))
{
await _tableStorage.InsertOrMergeBatchAsync(collection);
}
}