private async Task SaveCandlesBatchAsync(IEnumerable <IGrouping <string, ICandle> > candleByRowsChunk, string partitionKey)
{
var candleByRows = candleByRowsChunk.ToDictionary(g => g.Key, g => g.AsEnumerable());
// updates existing entities
var existingEntities = (await _tableStorage.GetDataAsync(partitionKey, candleByRows.Keys)).ToArray();
foreach (var entity in existingEntities)
{
entity.MergeCandles(candleByRows[entity.RowKey], _assetPairId, _timeInterval);
}
// creates new entities
var newEntityKeys = candleByRows.Keys.Except(existingEntities.Select(e => e.RowKey));
var newEntities = newEntityKeys.Select(k => new CandleHistoryEntity(partitionKey, k)).ToArray();
foreach (var entity in newEntities)
{
entity.MergeCandles(candleByRows[entity.RowKey], _assetPairId, _timeInterval);
}
// save changes
_healthService.TraceCandleRowsPersisted(existingEntities.Length + newEntities.Length);
await _tableStorage.InsertOrReplaceBatchAsync(existingEntities.Concat(newEntities));
}
public async Task <bool> ReplaceKeyValueAsync(IEnumerable <IKeyValueEntity> keyValueList)
{
if (!keyValueList.Any())
{
return(true);
}
var list = new List <KeyValueEntity>();
foreach (var item in keyValueList)
{
var kv = item as KeyValueEntity;
list.Add(kv ?? new KeyValueEntity(item));
}
try
{
await _tableStorage.InsertOrReplaceBatchAsync(list);
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(false);
}
return(true);
}
public async Task InsertGameBetAsync(IEnumerable <IGameBetItem> olapEntity)
{
var total = olapEntity.Select(GameBetEntity.CreateEntity);
// Group by partition key
var grouping = from e in total
group e by new { e.PartitionKey } into cms
select new { key = cms.Key, val = cms.ToList() };
// Insert grouped baches
foreach (var item in grouping)
{
var list = item.val;
do
{
int bufferLen = 128;
if (list.Count < 128)
{
bufferLen = list.Count;
}
var buffer = list.Take(bufferLen);
await _betstorage.InsertOrReplaceBatchAsync(buffer);
list.RemoveRange(0, bufferLen);
} while (list.Count > 0);
}
}
public async Task SaveAsync(
SnapshotAggregate snapshot)
{
await _balanceTable.InsertOrReplaceBatchAsync
(
snapshot.Balances.Select(x => new SnapshotBalanceEntity
{
BalanceValue = x.Value,
SnapshotTimestamp = snapshot.Timestamp,
WalletId = x.WalletId,
PartitionKey = $"{GetSnapshotPartitionKey(snapshot.Timestamp)}-{GetSnapshotRowKey(snapshot.Timestamp)}",
RowKey = $"{x.WalletId}"
})
);
_chaosKitty.Meow(snapshot.Timestamp);
await _snapshotTable.InsertOrReplaceAsync(new SnapshotEntity
{
SnapshotTimestamp = snapshot.Timestamp,
PartitionKey = GetSnapshotPartitionKey(snapshot.Timestamp),
RowKey = GetSnapshotRowKey(snapshot.Timestamp)
});
}
public Task InsertOrReplaceBatchAsync(IEnumerable <IAccountsStatReport> stats)
{
var tasks = BatchEntityInsertHelper.MakeBatchesByPartitionKey(stats.Select(m => AccountsStatReportEntity.Create(m)))
.Select(b => _tableStorage.InsertOrReplaceBatchAsync(b));
return(Task.WhenAll(tasks));
}
public async Task InsertOrReplaceBatchAsync(IEnumerable <T> entites)
{
var myArray = entites as T[] ?? entites.ToArray();
await _table.InsertOrReplaceBatchAsync(myArray);
await _cache.InsertOrReplaceBatchAsync(myArray);
}
public async Task SaveAsync(OrderBookEvent orderBookEvent)
{
if (_lastSavedMinute == default)
{
_lastSavedMinute = DateTime.UtcNow.TruncSeconds();
}
foreach (var orderItem in orderBookEvent.OrderItems)
{
_orderBookEventEntities.Enqueue(new OrderBookEventEntity(
orderBookEvent.SnapshotId,
orderBookEvent.OrderEventTimestamp, orderItem.Id)
{
OrderId = orderItem.Id,
IsBuy = orderItem.IsBuy,
Symbol = orderItem.Symbol,
EventType = (int)orderBookEvent.EventType,
Price = orderItem.Price,
Size = orderItem.Size
});
}
var currentTimeMinute = DateTime.UtcNow.TruncSeconds();
var isDifferentMinute = currentTimeMinute != _lastSavedMinute;
var isQueueMaxLength = _orderBookEventEntities.Count > _desiredQueueLength;
// Save on certain amount of time or items count
if (!isDifferentMinute && !isQueueMaxLength)
{
return;
}
var tableEntityBatches = _orderBookEventEntities
.DequeueChunk(_desiredQueueLength)
.GroupBy(ee => ee.PartitionKey);
foreach (var currentBatch in tableEntityBatches)
{
try
{
await _tableStorage.InsertOrReplaceBatchAsync(currentBatch);
_lastSavedMinute = currentTimeMinute;
}
catch (Exception ex)
{
_orderBookEventEntities.AddRange(currentBatch); // Queue the list back
await _log.WriteErrorAsync(_className,
$"Can't write to Azure Table {_tableStorage.Name} for snapshot {currentBatch.Key}, " +
$"will try later. Now in queue: {_orderBookEventEntities.Count}", ex);
}
}
}
public async Task <bool> ReplaceKeyValueAsync(IEnumerable <IKeyValueEntity> keyValueList)
{
try
{
await _tableStorage.InsertOrReplaceBatchAsync(keyValueList.Cast <KeyValueEntity>());
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(false);
}
return(true);
}
public Task SaveBatchAsync(
IEnumerable <BalanceUpdateAggregate> balanceUpdates)
{
var entities = balanceUpdates.Select(x => new BalanceUpdateEntity
{
EventTimestamp = x.EventTimestamp,
NewBalance = x.NewBalance,
WalletId = x.WalletId,
PartitionKey = GetPartitionKey(x.EventTimestamp),
RowKey = GetRowKey(x.EventTimestamp, x.WalletId)
});
return(_balanceUpdateTable.InsertOrReplaceBatchAsync(entities));
}
public async Task SaveAsync(
DistributionPlanAggregate distributionPlan)
{
var planEntity = new DistributionPlanEntity
{
PlanId = distributionPlan.Id,
PlanTimestamp = distributionPlan.Timestamp,
PartitionKey = GetPlanPartitionKey(distributionPlan.Timestamp),
RowKey = GetPlanRowKey(distributionPlan.Timestamp)
};
var indexEntity = new AzureIndex
{
PrimaryPartitionKey = planEntity.PartitionKey,
PrimaryRowKey = planEntity.RowKey,
PartitionKey = GetIndexPartitionKey(distributionPlan.Id),
RowKey = GetIndexRowKey()
};
var amountsPartition = GetAmountPartitionKey(distributionPlan.Timestamp);
var amountEntities = distributionPlan.Amounts.Select(x => new DistributionAmountEntity
{
AmountValue = x.Value,
AmountId = x.Id,
WalletId = x.WalletId,
PartitionKey = amountsPartition,
RowKey = GetAmountRowKey(x.WalletId)
});
await _distributionAmountTable.InsertOrReplaceBatchAsync(amountEntities);
_chaosKitty.Meow(distributionPlan.Id);
await _distributionPlanIndexTable.InsertOrReplaceAsync(indexEntity);
_chaosKitty.Meow(distributionPlan.Id);
await _distributionPlanTable.InsertOrReplaceAsync(planEntity);
}
public Task InsertOrReplaceBatchAsync(IEnumerable <T> entities)
{
var cryptoItems = entities.Select(Encrypt);
return(_storage.InsertOrReplaceBatchAsync(cryptoItems));
}
public async Task InsertOrReplaceBatchAsync(IEnumerable <TEntity> entites)
{
await _retryService.RetryAsync(async() => await _impl.InsertOrReplaceBatchAsync(entites), _onModificationsRetryCount);
}
public async Task InsertOrReplaceActivities(IEnumerable <IActivity> entities)
{
await _storage.InsertOrReplaceBatchAsync(entities.Select(x => ActivityEntity.CreateEntity(x)));
}
public Task InsertOrReplaceBatchAsync(IEnumerable <TEntity> entities) => WrapAsync(() => _impl.InsertOrReplaceBatchAsync(entities), nameof(InsertOrReplaceBatchAsync), entities);
public Task InsertOrReplaceBatchAsync(IEnumerable <IEmployeeNotificationId> employeeNotificationIds)
{
return(_storage.InsertOrReplaceBatchAsync(
employeeNotificationIds.Select(i => new EmployeeNotificationIdEntity(i))));
}
public Task InsertOrReplaceBatchAsync(IEnumerable <IMerchantNotificationId> merchantNotificationIds)
{
return(_storage.InsertOrReplaceBatchAsync(
merchantNotificationIds.Select(i => new MerchantNotificationIdEntity(i))));
}
public Task Insert(IReadOnlyCollection <MarginTradingAccountHistoryEntity> src)
{
return(_tableStorage.InsertOrReplaceBatchAsync(src));
}
public virtual async Task InsertOrReplaceBatchAsync(IEnumerable<TD> objects)
{
await _tableStorage.InsertOrReplaceBatchAsync(objects.Select(InvokeCreate));
}
public async Task InsertOrReplaceAsync(IAssetDefinition[] assetsDefinition)
{
await _assetTableStorage.InsertOrReplaceBatchAsync(assetsDefinition.Select(AssetDefinitionDefinitionEntity.Create));
}
public Task AddAsync(IEnumerable <IClientRegulation> clientRegulations)
{
IEnumerable <ClientRegulationEntity> entities = clientRegulations.Select(Create);
return(_tableStorage.InsertOrReplaceBatchAsync(entities));
}
public Task InsertOrReplaceOutputsAsync(IEnumerable <IInternalOutput> outputs)
{
var entities = outputs.Select(InternalOutputEntity.Create);
return(_storage.InsertOrReplaceBatchAsync(entities));
}
