public Task ScanByDtAsync(Func <IEnumerable <IClientTrade>, Task> chunk, DateTime @from, DateTime to)
{
var rangeQuery = AzureStorageUtils.QueryGenerator <ClientTradeEntity>
.RowKeyOnly.BetweenQuery(ClientTradeEntity.ByDt.GetRowKeyPart(from),
ClientTradeEntity.ByDt.GetRowKeyPart(to), ToIntervalOption.IncludeTo);
return(_tableStorage.ScanDataAsync(rangeQuery, chunk));
}
public Task ScanDataAsync(string partitionKey, Func <IEnumerable <T>, Task> chunk)
{
return(_storage.ScanDataAsync(partitionKey, Map(chunk)));
}
public Task ScanDataAsync(string partitionKey, Func <IEnumerable <TEntity>, Task> chunk)
{
return(_impl.ScanDataAsync(partitionKey, chunk));
}
public Task ScanAllAsync(Func <IEnumerable <IWalletCredentials>, Task> chunk)
{
var partitionKey = WalletCredentialsEntity.ByClientId.GeneratePartitionKey();
return(_tableStorage.ScanDataAsync(partitionKey, chunk));
}
public async Task InsertOrMergeAsync(IReadOnlyDictionary <TimeInterval, IEnumerable <IFeedCandle> > dict, string asset)
{
if (dict == null)
{
throw new ArgumentNullException(nameof(dict));
}
if (string.IsNullOrEmpty(asset))
{
throw new ArgumentNullException(nameof(asset));
}
if (!dict.Any() && dict.Values.All(e => e != null) && dict.Values.Any(e => e.Count() > 0))
{
return;
}
var partitionKey = CandleTableEntity.GeneratePartitionKey(asset);
var rowKeys = new List <string>();
var fields = new HashSet <string>(); // which fields to read from table
// 1. Read all { pkey, rowkey } rows
//
var updateEntities = new List <CandleTableEntity>();
foreach (var interval in dict.Keys)
{
var candles = dict[interval];
if (candles != null && candles.Any())
{
// Inside one interval group all candles by distinct row
var groups = candles.GroupBy(candle => candle.RowKey(interval));
rowKeys.AddRange(groups.Select(g => g.Key));
foreach (var group in groups)
{
// Create entity with candles and add it to list
var e = new CandleTableEntity(partitionKey, group.Key); // group.Key = rowKey
e.MergeCandles(group, interval);
updateEntities.Add(e);
// update field
var dates = group.Select(c => c.DateTime);
fields.UnionWith(CandleTableEntity.GetStoreFields(interval, dates.Min(), dates.Max()));
}
}
}
// ... prepare get query
// ... partitionKey = ? AND (rokey = ? OR rowkey = ? OR rowkey = ? OR ...)
TableQuery <CandleTableEntity> query = new TableQuery <CandleTableEntity>();
string pkeyFilter = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey);
var rowkeyFilters = rowKeys.Select(rowKey => TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, rowKey));
var rowkeyFilter = rowkeyFilters.Aggregate((cond1, cond2) => TableQuery.CombineFilters(cond1, TableOperators.Or, cond2));
query.FilterString = TableQuery.CombineFilters(pkeyFilter, TableOperators.And, rowkeyFilter);
query.SelectColumns = fields.ToList();
// ... reading rows from azure table
List <CandleTableEntity> origEntities = new List <CandleTableEntity>(1);
await _tableStorage.ScanDataAsync(query, list =>
{
origEntities.AddRange(list);
return(Task.FromResult(0));
});
// 2. Update rows (merge entities)
//
var listToUpdate = new List <CandleTableEntity>();
foreach (var updateEntity in updateEntities)
{
var origEntity = origEntities.Where(e => e.PartitionKey == updateEntity.PartitionKey && e.RowKey == updateEntity.RowKey).FirstOrDefault();
if (origEntity != null)
{
origEntity.MergeCandles(updateEntity.Candles, updateEntity.Interval);
listToUpdate.Add(origEntity);
}
else
{
listToUpdate.Add(updateEntity);
}
}
// 3. Write rows in batch
// ... Only 100 records with the same pKey can be updated in one batch operation
foreach (var collection in listToUpdate.ToPieces(100))
{
await _tableStorage.InsertOrMergeBatchAsync(collection);
}
}
public Task ScanAllAsync(Func <IEnumerable <IMonitoringRecord>, Task> chunk)
{
var partitionKey = MonitoringRecordEntity.GeneratePartitionKey();
return(_tableStorage.ScanDataAsync(partitionKey, chunk));
}
public Task ScanDataAsync(string partitionKey, Func <IEnumerable <TEntity>, Task> chunk)
=> WrapAsync(() => _impl.ScanDataAsync(partitionKey, chunk), nameof(ScanDataAsync), new { partitionKey });
