private async Task SaveCandlesBatchAsync(IEnumerable <IGrouping <string, ICandle> > candleByRowsChunk, string partitionKey) { var candleByRows = candleByRowsChunk.ToDictionary(g => g.Key, g => g.AsEnumerable()); // updates existing entities var existingEntities = (await _tableStorage.GetDataAsync(partitionKey, candleByRows.Keys)).ToArray(); foreach (var entity in existingEntities) { entity.MergeCandles(candleByRows[entity.RowKey], _assetPairId, _timeInterval); } // creates new entities var newEntityKeys = candleByRows.Keys.Except(existingEntities.Select(e => e.RowKey)); var newEntities = newEntityKeys.Select(k => new CandleHistoryEntity(partitionKey, k)).ToArray(); foreach (var entity in newEntities) { entity.MergeCandles(candleByRows[entity.RowKey], _assetPairId, _timeInterval); } // save changes _healthService.TraceCandleRowsPersisted(existingEntities.Length + newEntities.Length); await _tableStorage.InsertOrReplaceBatchAsync(existingEntities.Concat(newEntities)); }
public async Task <bool> ReplaceKeyValueAsync(IEnumerable <IKeyValueEntity> keyValueList) { if (!keyValueList.Any()) { return(true); } var list = new List <KeyValueEntity>(); foreach (var item in keyValueList) { var kv = item as KeyValueEntity; list.Add(kv ?? new KeyValueEntity(item)); } try { await _tableStorage.InsertOrReplaceBatchAsync(list); } catch (Exception ex) { Console.WriteLine(ex); return(false); } return(true); }
public async Task InsertGameBetAsync(IEnumerable <IGameBetItem> olapEntity) { var total = olapEntity.Select(GameBetEntity.CreateEntity); // Group by partition key var grouping = from e in total group e by new { e.PartitionKey } into cms select new { key = cms.Key, val = cms.ToList() }; // Insert grouped baches foreach (var item in grouping) { var list = item.val; do { int bufferLen = 128; if (list.Count < 128) { bufferLen = list.Count; } var buffer = list.Take(bufferLen); await _betstorage.InsertOrReplaceBatchAsync(buffer); list.RemoveRange(0, bufferLen); } while (list.Count > 0); } }
public async Task SaveAsync( SnapshotAggregate snapshot) { await _balanceTable.InsertOrReplaceBatchAsync ( snapshot.Balances.Select(x => new SnapshotBalanceEntity { BalanceValue = x.Value, SnapshotTimestamp = snapshot.Timestamp, WalletId = x.WalletId, PartitionKey = $"{GetSnapshotPartitionKey(snapshot.Timestamp)}-{GetSnapshotRowKey(snapshot.Timestamp)}", RowKey = $"{x.WalletId}" }) ); _chaosKitty.Meow(snapshot.Timestamp); await _snapshotTable.InsertOrReplaceAsync(new SnapshotEntity { SnapshotTimestamp = snapshot.Timestamp, PartitionKey = GetSnapshotPartitionKey(snapshot.Timestamp), RowKey = GetSnapshotRowKey(snapshot.Timestamp) }); }
public Task InsertOrReplaceBatchAsync(IEnumerable <IAccountsStatReport> stats) { var tasks = BatchEntityInsertHelper.MakeBatchesByPartitionKey(stats.Select(m => AccountsStatReportEntity.Create(m))) .Select(b => _tableStorage.InsertOrReplaceBatchAsync(b)); return(Task.WhenAll(tasks)); }
public async Task InsertOrReplaceBatchAsync(IEnumerable <T> entites) { var myArray = entites as T[] ?? entites.ToArray(); await _table.InsertOrReplaceBatchAsync(myArray); await _cache.InsertOrReplaceBatchAsync(myArray); }
public async Task SaveAsync(OrderBookEvent orderBookEvent) { if (_lastSavedMinute == default) { _lastSavedMinute = DateTime.UtcNow.TruncSeconds(); } foreach (var orderItem in orderBookEvent.OrderItems) { _orderBookEventEntities.Enqueue(new OrderBookEventEntity( orderBookEvent.SnapshotId, orderBookEvent.OrderEventTimestamp, orderItem.Id) { OrderId = orderItem.Id, IsBuy = orderItem.IsBuy, Symbol = orderItem.Symbol, EventType = (int)orderBookEvent.EventType, Price = orderItem.Price, Size = orderItem.Size }); } var currentTimeMinute = DateTime.UtcNow.TruncSeconds(); var isDifferentMinute = currentTimeMinute != _lastSavedMinute; var isQueueMaxLength = _orderBookEventEntities.Count > _desiredQueueLength; // Save on certain amount of time or items count if (!isDifferentMinute && !isQueueMaxLength) { return; } var tableEntityBatches = _orderBookEventEntities .DequeueChunk(_desiredQueueLength) .GroupBy(ee => ee.PartitionKey); foreach (var currentBatch in tableEntityBatches) { try { await _tableStorage.InsertOrReplaceBatchAsync(currentBatch); _lastSavedMinute = currentTimeMinute; } catch (Exception ex) { _orderBookEventEntities.AddRange(currentBatch); // Queue the list back await _log.WriteErrorAsync(_className, $"Can't write to Azure Table {_tableStorage.Name} for snapshot {currentBatch.Key}, " + $"will try later. Now in queue: {_orderBookEventEntities.Count}", ex); } } }
public async Task <bool> ReplaceKeyValueAsync(IEnumerable <IKeyValueEntity> keyValueList) { try { await _tableStorage.InsertOrReplaceBatchAsync(keyValueList.Cast <KeyValueEntity>()); } catch (Exception ex) { Console.WriteLine(ex); return(false); } return(true); }
public Task SaveBatchAsync( IEnumerable <BalanceUpdateAggregate> balanceUpdates) { var entities = balanceUpdates.Select(x => new BalanceUpdateEntity { EventTimestamp = x.EventTimestamp, NewBalance = x.NewBalance, WalletId = x.WalletId, PartitionKey = GetPartitionKey(x.EventTimestamp), RowKey = GetRowKey(x.EventTimestamp, x.WalletId) }); return(_balanceUpdateTable.InsertOrReplaceBatchAsync(entities)); }
public async Task SaveAsync( DistributionPlanAggregate distributionPlan) { var planEntity = new DistributionPlanEntity { PlanId = distributionPlan.Id, PlanTimestamp = distributionPlan.Timestamp, PartitionKey = GetPlanPartitionKey(distributionPlan.Timestamp), RowKey = GetPlanRowKey(distributionPlan.Timestamp) }; var indexEntity = new AzureIndex { PrimaryPartitionKey = planEntity.PartitionKey, PrimaryRowKey = planEntity.RowKey, PartitionKey = GetIndexPartitionKey(distributionPlan.Id), RowKey = GetIndexRowKey() }; var amountsPartition = GetAmountPartitionKey(distributionPlan.Timestamp); var amountEntities = distributionPlan.Amounts.Select(x => new DistributionAmountEntity { AmountValue = x.Value, AmountId = x.Id, WalletId = x.WalletId, PartitionKey = amountsPartition, RowKey = GetAmountRowKey(x.WalletId) }); await _distributionAmountTable.InsertOrReplaceBatchAsync(amountEntities); _chaosKitty.Meow(distributionPlan.Id); await _distributionPlanIndexTable.InsertOrReplaceAsync(indexEntity); _chaosKitty.Meow(distributionPlan.Id); await _distributionPlanTable.InsertOrReplaceAsync(planEntity); }
public Task InsertOrReplaceBatchAsync(IEnumerable <T> entities) { var cryptoItems = entities.Select(Encrypt); return(_storage.InsertOrReplaceBatchAsync(cryptoItems)); }
public async Task InsertOrReplaceBatchAsync(IEnumerable <TEntity> entites) { await _retryService.RetryAsync(async() => await _impl.InsertOrReplaceBatchAsync(entites), _onModificationsRetryCount); }
public async Task InsertOrReplaceActivities(IEnumerable <IActivity> entities) { await _storage.InsertOrReplaceBatchAsync(entities.Select(x => ActivityEntity.CreateEntity(x))); }
public Task InsertOrReplaceBatchAsync(IEnumerable <TEntity> entities) => WrapAsync(() => _impl.InsertOrReplaceBatchAsync(entities), nameof(InsertOrReplaceBatchAsync), entities);
public Task InsertOrReplaceBatchAsync(IEnumerable <IEmployeeNotificationId> employeeNotificationIds) { return(_storage.InsertOrReplaceBatchAsync( employeeNotificationIds.Select(i => new EmployeeNotificationIdEntity(i)))); }
public Task InsertOrReplaceBatchAsync(IEnumerable <IMerchantNotificationId> merchantNotificationIds) { return(_storage.InsertOrReplaceBatchAsync( merchantNotificationIds.Select(i => new MerchantNotificationIdEntity(i)))); }
public Task Insert(IReadOnlyCollection <MarginTradingAccountHistoryEntity> src) { return(_tableStorage.InsertOrReplaceBatchAsync(src)); }
public virtual async Task InsertOrReplaceBatchAsync(IEnumerable<TD> objects) { await _tableStorage.InsertOrReplaceBatchAsync(objects.Select(InvokeCreate)); }
public async Task InsertOrReplaceAsync(IAssetDefinition[] assetsDefinition) { await _assetTableStorage.InsertOrReplaceBatchAsync(assetsDefinition.Select(AssetDefinitionDefinitionEntity.Create)); }
public Task AddAsync(IEnumerable <IClientRegulation> clientRegulations) { IEnumerable <ClientRegulationEntity> entities = clientRegulations.Select(Create); return(_tableStorage.InsertOrReplaceBatchAsync(entities)); }
public Task InsertOrReplaceOutputsAsync(IEnumerable <IInternalOutput> outputs) { var entities = outputs.Select(InternalOutputEntity.Create); return(_storage.InsertOrReplaceBatchAsync(entities)); }