public async Task InsertManyAsync(IEnumerable <IBestBidAsk> olapEntities) { var total = olapEntities.Select(BestBidAskEntity.CreateEntity); var grouping = from e in total group e by new { e.PartitionKey } into cms select new { key = cms.Key, val = cms.ToList() }; foreach (var item in grouping) { var list = item.val; do { int bufferLen = maxbuffer; if (list.Count < maxbuffer) { bufferLen = list.Count; } var buffer = list.Take(bufferLen); //Console.WriteLine("Inserting {0} records", bufferLen); await _storage.InsertOrMergeBatchAsync(buffer); list.RemoveRange(0, bufferLen); } while (list.Count > 0); } }
public async Task <bool> UpdateKeyValueAsync(IEnumerable <IKeyValueEntity> keyValueList) { if (!keyValueList.Any()) { return(true); } var list = new List <KeyValueEntity>(); foreach (var item in keyValueList) { var kv = item as KeyValueEntity; list.Add(kv ?? new KeyValueEntity(item)); } try { await _tableStorage.InsertOrMergeBatchAsync(list); } catch (Exception ex) { Console.WriteLine(ex); return(false); } return(true); }
public async Task SaveAsync(IEnumerable <FixQuoteModel> quotes) { const int batchSize = 100; var entities = quotes.Select(q => new FixQuoteEntity(q.AssetPair.ToLowerInvariant(), q.Ask, q.Bid, q.FixingTime, q.TradeTime)); foreach (var batch in entities.Batch(batchSize)) { await _storage.InsertOrMergeBatchAsync(batch); } }
public async Task AddOrUpdateBatchAsync(IEnumerable <LimitOrder> orders) { var models = orders.Select(x => { var entity = new LimitOrderEntity(GetPartitionKey(x.AssetPairId), GetRowKey(x.Id)); Mapper.Map(x, entity); return(entity); }); await _storage.InsertOrMergeBatchAsync(models); }
public async Task <bool> UpdateKeyValueAsync(IEnumerable <IKeyValueEntity> keyValueList) { try { await _tableStorage.InsertOrMergeBatchAsync(keyValueList.Cast <KeyValueEntity>()); } catch (Exception ex) { Console.WriteLine(ex); return(false); } return(true); }
public async Task <IEnumerable <IConnectionUrlHistory> > GetByRepositoryIdAsync(string repositoryId) { var list = await _tableStorage.GetDataAsync(repositoryId); if (!list.Any()) { var pk = ConnectionUrlHistory.GeneratePartitionKey(); list = await _tableStorage.GetDataAsync(pk, x => x.RepositoryId == repositoryId); foreach (var item in list) { item.PartitionKey = item.RepositoryId; } await _tableStorage.InsertOrMergeBatchAsync(list); } return(list); }
public async Task InsertOrMergeBatchAsync(IEnumerable <TEntity> items) { await _retryService.RetryAsync(async() => await _impl.InsertOrMergeBatchAsync(items), _onModificationsRetryCount); }
public async Task AddAsync(IEnumerable <IPriceCandlesConsistencyResult> entities, DateTime checkDate) { await _tableStorage.InsertOrMergeBatchAsync(PriceCandlesConsistencyResultEntity.CreateBatch(entities, checkDate)); }
public async Task AddAsync(IEnumerable <IHedgingServiceCheckResult> entities, DateTime checkDate) { await _tableStorage.InsertOrMergeBatchAsync(HedgingServiceCheckResultEntity.CreateBatch(entities, checkDate)); }
public Task InsertOrMergeBatchAsync(IEnumerable <TEntity> items) => WrapAsync(() => _impl.InsertOrMergeBatchAsync(items), nameof(InsertOrMergeBatchAsync), items);
public async Task AddAsync(IEnumerable <IBalanceAndTransactionAmountCheckResult> entities, DateTime checkDate) { await _tableStorage.InsertOrMergeBatchAsync(BalanceAndTransactionAmountCheckResultEntity.CreateBatch(entities, checkDate)); }
public Task AddAsync(string clientId, Choice[] choices) { var entities = choices.Select(x => UserChoiceEntity.Create(clientId, x)); return(_tableStorage.InsertOrMergeBatchAsync(entities)); }
public async Task InsertOrMergeAsync(IReadOnlyDictionary <TimeInterval, IEnumerable <IFeedCandle> > dict, string asset) { if (dict == null) { throw new ArgumentNullException(nameof(dict)); } if (string.IsNullOrEmpty(asset)) { throw new ArgumentNullException(nameof(asset)); } if (!dict.Any() && dict.Values.All(e => e != null) && dict.Values.Any(e => e.Count() > 0)) { return; } var partitionKey = CandleTableEntity.GeneratePartitionKey(asset); var rowKeys = new List <string>(); var fields = new HashSet <string>(); // which fields to read from table // 1. Read all { pkey, rowkey } rows // var updateEntities = new List <CandleTableEntity>(); foreach (var interval in dict.Keys) { var candles = dict[interval]; if (candles != null && candles.Any()) { // Inside one interval group all candles by distinct row var groups = candles.GroupBy(candle => candle.RowKey(interval)); rowKeys.AddRange(groups.Select(g => g.Key)); foreach (var group in groups) { // Create entity with candles and add it to list var e = new CandleTableEntity(partitionKey, group.Key); // group.Key = rowKey e.MergeCandles(group, interval); updateEntities.Add(e); // update field var dates = group.Select(c => c.DateTime); fields.UnionWith(CandleTableEntity.GetStoreFields(interval, dates.Min(), dates.Max())); } } } // ... prepare get query // ... partitionKey = ? AND (rokey = ? OR rowkey = ? OR rowkey = ? OR ...) TableQuery <CandleTableEntity> query = new TableQuery <CandleTableEntity>(); string pkeyFilter = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, partitionKey); var rowkeyFilters = rowKeys.Select(rowKey => TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, rowKey)); var rowkeyFilter = rowkeyFilters.Aggregate((cond1, cond2) => TableQuery.CombineFilters(cond1, TableOperators.Or, cond2)); query.FilterString = TableQuery.CombineFilters(pkeyFilter, TableOperators.And, rowkeyFilter); query.SelectColumns = fields.ToList(); // ... reading rows from azure table List <CandleTableEntity> origEntities = new List <CandleTableEntity>(1); await _tableStorage.ScanDataAsync(query, list => { origEntities.AddRange(list); return(Task.FromResult(0)); }); // 2. Update rows (merge entities) // var listToUpdate = new List <CandleTableEntity>(); foreach (var updateEntity in updateEntities) { var origEntity = origEntities.Where(e => e.PartitionKey == updateEntity.PartitionKey && e.RowKey == updateEntity.RowKey).FirstOrDefault(); if (origEntity != null) { origEntity.MergeCandles(updateEntity.Candles, updateEntity.Interval); listToUpdate.Add(origEntity); } else { listToUpdate.Add(updateEntity); } } // 3. Write rows in batch // ... Only 100 records with the same pKey can be updated in one batch operation foreach (var collection in listToUpdate.ToPieces(100)) { await _tableStorage.InsertOrMergeBatchAsync(collection); } }
public Task AddAnswersAsync(IEnumerable <IAnswer> answers) { return(_tableStorage.InsertOrMergeBatchAsync(answers.Select(AnswerEntity.Create))); }
public virtual async Task InsertOrMergeBatchAsync(IEnumerable<TD> objects) { await _tableStorage.InsertOrMergeBatchAsync(objects.Select(InvokeCreate)); }
public async Task AddAsync(IEnumerable <IMarginEventsAccountStatusCheckResult> entities, DateTime checkDate) { await _tableStorage.InsertOrMergeBatchAsync(MarginEventsAccountStatusCheckResultEntity.CreateBatch(entities, checkDate)); }
public Task AddOrMergeMarketData(IEnumerable <IMarketData> data) { var entities = data.Select(MarketDataEntity.Create); return(_tableStorage.InsertOrMergeBatchAsync(entities)); }
public Task InsertOrMergeBatchAsync(IEnumerable <T> items) { var cryptoItems = items.Select(Encrypt); return(_storage.InsertOrMergeBatchAsync(cryptoItems)); }
public async Task InsertManyAsync(IEnumerable <IBoxSize> olapEntities) { await _storage.InsertOrMergeBatchAsync(olapEntities.Select(BoxSizeEntity.CreateEntity)); }