public Task ScanByDtAsync(Func <IEnumerable <IClientTrade>, Task> chunk, DateTime @from, DateTime to) { var rangeQuery = AzureStorageUtils.QueryGenerator <ClientTradeEntity> .RowKeyOnly.BetweenQuery(ClientTradeEntity.ByDt.GetRowKeyPart(from), ClientTradeEntity.ByDt.GetRowKeyPart(to), ToIntervalOption.IncludeTo); return(_tableStorage.ScanDataAsync(rangeQuery, chunk)); }
public Task ScanDataAsync(string partitionKey, Func <IEnumerable <T>, Task> chunk) { return(_storage.ScanDataAsync(partitionKey, Map(chunk))); }
public Task ScanDataAsync(string partitionKey, Func <IEnumerable <TEntity>, Task> chunk) { return(_impl.ScanDataAsync(partitionKey, chunk)); }
public Task ScanAllAsync(Func <IEnumerable <IWalletCredentials>, Task> chunk) { var partitionKey = WalletCredentialsEntity.ByClientId.GeneratePartitionKey(); return(_tableStorage.ScanDataAsync(partitionKey, chunk)); }
public async Task InsertOrMergeAsync(IReadOnlyDictionary <TimeInterval, IEnumerable <IFeedCandle> > dict, string asset) { if (dict == null) { throw new ArgumentNullException(nameof(dict)); } if (string.IsNullOrEmpty(asset)) { throw new ArgumentNullException(nameof(asset)); } if (!dict.Any() && dict.Values.All(e => e != null) && dict.Values.Any(e => e.Count() > 0)) { return; } var partitionKey = CandleTableEntity.GeneratePartitionKey(asset); var rowKeys = new List <string>(); var fields = new HashSet <string>(); // which fields to read from table // 1. Read all { pkey, rowkey } rows // var updateEntities = new List <CandleTableEntity>(); foreach (var interval in dict.Keys) { var candles = dict[interval]; if (candles != null && candles.Any()) { // Inside one interval group all candles by distinct row var groups = candles.GroupBy(candle => candle.RowKey(interval)); rowKeys.AddRange(groups.Select(g => g.Key)); foreach (var group in groups) { // Create entity with candles and add it to list var e = new CandleTableEntity(partitionKey, group.Key); // group.Key = rowKey e.MergeCandles(group, interval); updateEntities.Add(e); // update field var dates = group.Select(c => c.DateTime); fields.UnionWith(CandleTableEntity.GetStoreFields(interval, dates.Min(), dates.Max())); } } } // ... prepare get query // ... partitionKey = ? AND (rokey = ? OR rowkey = ? OR rowkey = ? OR ...) TableQuery <CandleTableEntity> query = new TableQuery <CandleTableEntity>(); string pkeyFilter = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey); var rowkeyFilters = rowKeys.Select(rowKey => TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, rowKey)); var rowkeyFilter = rowkeyFilters.Aggregate((cond1, cond2) => TableQuery.CombineFilters(cond1, TableOperators.Or, cond2)); query.FilterString = TableQuery.CombineFilters(pkeyFilter, TableOperators.And, rowkeyFilter); query.SelectColumns = fields.ToList(); // ... reading rows from azure table List <CandleTableEntity> origEntities = new List <CandleTableEntity>(1); await _tableStorage.ScanDataAsync(query, list => { origEntities.AddRange(list); return(Task.FromResult(0)); }); // 2. Update rows (merge entities) // var listToUpdate = new List <CandleTableEntity>(); foreach (var updateEntity in updateEntities) { var origEntity = origEntities.Where(e => e.PartitionKey == updateEntity.PartitionKey && e.RowKey == updateEntity.RowKey).FirstOrDefault(); if (origEntity != null) { origEntity.MergeCandles(updateEntity.Candles, updateEntity.Interval); listToUpdate.Add(origEntity); } else { listToUpdate.Add(updateEntity); } } // 3. Write rows in batch // ... Only 100 records with the same pKey can be updated in one batch operation foreach (var collection in listToUpdate.ToPieces(100)) { await _tableStorage.InsertOrMergeBatchAsync(collection); } }
public Task ScanAllAsync(Func <IEnumerable <IMonitoringRecord>, Task> chunk) { var partitionKey = MonitoringRecordEntity.GeneratePartitionKey(); return(_tableStorage.ScanDataAsync(partitionKey, chunk)); }
public Task ScanDataAsync(string partitionKey, Func <IEnumerable <TEntity>, Task> chunk) => WrapAsync(() => _impl.ScanDataAsync(partitionKey, chunk), nameof(ScanDataAsync), new { partitionKey });