public async Task <WalletDto> TryGetAsync(string blockchainType, Guid clientId)
{
var(partitionKey, rowKey) = GetClientBlockchainTypeIndexKeys(blockchainType, clientId);
var indexes = await _clientBlockchainTypeIndexTable.GetDataWithContinuationTokenAsync(partitionKey, 10, null);
var latestIndex = indexes.Entities.FirstOrDefault();
var entity = await _walletsTable.GetDataAsync(latestIndex);
return(entity != null
? ConvertEntityToDto(entity)
: null);
}
public async Task <bool> TryDeleteObservationAsync(string address)
{
// delete wallet if exists
var existed = await _walletStorage.DeleteIfExistAsync(DepositWalletEntity.Partition(address), DepositWalletEntity.Row());
// if not deleted earlier then delete balances
if (existed)
{
string continuation = null;
do
{
var query = new TableQuery <DepositWalletBalanceEntity>().Where($"PartitionKey eq '{DepositWalletBalanceEntity.Partition(address)}'");
var chunk = await _walletBalanceStorage.GetDataWithContinuationTokenAsync(query, 100, continuation);
var batch = new TableBatchOperation();
continuation = chunk.ContinuationToken;
foreach (var balance in chunk.Entities)
{
batch.Delete(balance);
}
if (batch.Any())
{
await _walletBalanceStorage.DoBatchAsync(batch);
}
} while (!string.IsNullOrEmpty(continuation));
}
return(existed);
}
public async Task <IReadOnlyCollection <ClaimedGasAmountAggregate> > GetAsync(
DateTime from,
DateTime to)
{
var claimedGasAmounts = new List <ClaimedGasAmountAggregate>();
foreach (var rangeQuery in GetRangeQueries(from, to))
{
var continuationToken = (string)null;
do
{
IEnumerable <ClaimedGasAmountEntity> entities;
(entities, continuationToken) = await _claimedGasAmountTable
.GetDataWithContinuationTokenAsync(rangeQuery, continuationToken);
claimedGasAmounts.AddRange(entities.Select(x => ClaimedGasAmountAggregate.CreateOrRestore
(
amount: x.Amount,
transactionId: x.TransactionId,
transactionBroadcastingMoment: x.TransactionBroadcastingMoment
)));
} while (continuationToken != null);
}
return(claimedGasAmounts);
}
public async Task <IReadOnlyCollection <BalanceUpdateAggregate> > GetAsync(
DateTime from,
DateTime to)
{
var balanceUpdates = new List <BalanceUpdateAggregate>();
foreach (var rangeQuery in GetRangeQueries(from, to))
{
var continuationToken = (string)null;
do
{
IEnumerable <BalanceUpdateEntity> entities;
(entities, continuationToken) = await _balanceUpdateTable
.GetDataWithContinuationTokenAsync(rangeQuery, 1000, continuationToken);
balanceUpdates.AddRange(entities.Select(x => BalanceUpdateAggregate.CreateOrRestore
(
x.EventTimestamp,
x.NewBalance,
x.WalletId
)));
} while (continuationToken != null);
}
return(balanceUpdates);
}
private async Task <IReadOnlyCollection <SnapshotBalanceEntity> > GetBalanceEntitiesAsync(
params SnapshotEntity[] snapshotEntities)
{
var balanceEntities = new List <SnapshotBalanceEntity>();
foreach (var snapshotEntity in snapshotEntities)
{
var continuationToken = (string)null;
var rangeQuery = new TableQuery <SnapshotBalanceEntity>().Where(TableQuery.GenerateFilterCondition
(
propertyName: nameof(AzureTableEntity.PartitionKey),
operation: QueryComparisons.Equal,
givenValue: $"{GetSnapshotPartitionKey(snapshotEntity.SnapshotTimestamp)}-{GetSnapshotRowKey(snapshotEntity.SnapshotTimestamp)}"
));
do
{
IEnumerable <SnapshotBalanceEntity> entities;
(entities, continuationToken) = await _balanceTable
.GetDataWithContinuationTokenAsync(rangeQuery, 1000, continuationToken);
balanceEntities.AddRange(entities);
} while (continuationToken != null);
}
return(balanceEntities);
}
public async Task <IPaginationResult <IWalletBalance> > GetBalances(int take, string continuation)
{
var t = await _storage.GetDataAsync(WalletBalanceEntity.GeneratePartitionKey());
var result = await _storage.GetDataWithContinuationTokenAsync(WalletBalanceEntity.GeneratePartitionKey(), take, continuation);
return(PaginationResult <IWalletBalance> .Create(result.Entities.Cast <IWalletBalance>(), result.ContinuationToken));
}
public async Task <(List <WalletBalance> Entities, string ContinuationToken)> GetAllAsync(int take, string continuationToken)
{
var data = await _balanceTable.GetDataWithContinuationTokenAsync(take, continuationToken);
var balances = data.Entities.Select(x => x.ToDomain()).ToList();
return(balances, data.ContinuationToken);
}
internal async Task <(IEnumerable <WalletDto> Wallets, string ContinuationToken)> GetAsync(string blockchainType, string assetId, int take, string continuationToken)
{
var filterCondition = TableQuery.CombineFilters
(
TableQuery.GenerateFilterCondition("IntegrationLayerId", QueryComparisons.Equal, blockchainType),
TableOperators.And,
TableQuery.GenerateFilterCondition("AssetId", QueryComparisons.Equal, assetId)
);
var query = new TableQuery <WalletEntity>().Where(filterCondition);
IEnumerable <WalletEntity> entities;
(entities, continuationToken) = await _walletsTable.GetDataWithContinuationTokenAsync(query, take, continuationToken);
return(entities.Select(ConvertEntityToDto), continuationToken);
}
public async Task<(IEnumerable<BlockchainExplorer>, string continuationToken)> GetAllAsync(int take,
string continuationToken = null)
{
var (entities, newToken) = await _table.GetDataWithContinuationTokenAsync(take, continuationToken);
var mapped = entities.Select(entity => entity.ToDomain());
return (mapped, newToken);
}
public async Task <(string continuation, IEnumerable <string> items)> GetHistoryAddressesChunkAsync(string continuation = null, int take = 100)
{
var chunk = await _historyAddressesStorage.GetDataWithContinuationTokenAsync(take, continuation);
return(
chunk.ContinuationToken,
chunk.Entities.Select(e => e.PartitionKey).ToArray()
);
}
private async Task <(IEnumerable <(string WalletPartitionKey, string WalletRowKey)> wallets, string continuationToken)> GetForClientIndicesAsync(Guid clientId, int take, string continuationToken)
{
var partitionKey = GetClientPartitionKey(clientId);
var indexes = await _clientIndexTable.GetDataWithContinuationTokenAsync(partitionKey, take, continuationToken);
var values = indexes.Entities.Select(x => (x.PrimaryPartitionKey, x.PrimaryRowKey));
return(values, indexes.ContinuationToken);
}
GetForClientIndicesAsync(Guid clientId, int take, string continuationToken)
{
var partitionKey = GetClientLatestIndexPartitionKey(clientId);
var indexes = await _clientLatestDepositsIndexTable.GetDataWithContinuationTokenAsync(partitionKey, take, continuationToken);
var values = indexes.Entities.Select(x => (x.PrimaryPartitionKey, x.PrimaryRowKey));
return(values, indexes.ContinuationToken);
}
public async Task <(IReadOnlyCollection <Balance> Balances, string ContinuationToken)> GetAllAsync(
int take,
string continuationToken)
{
var(entities, newContinuationToken) = await _balances
.GetDataWithContinuationTokenAsync(take, continuationToken);
var balances = entities
.Select(x => new Balance
(
address: x.RowKey,
amount: x.Amount,
blockNumber: x.BlockNumber
))
.ToImmutableList();
return(balances, newContinuationToken);
}
public async Task <(List <TU> Items, string ContinuationToken)> GetAllAsync(int take, string continuationToken)
{
var query = new TableQuery <T>().Take(take);
var data = await _table.GetDataWithContinuationTokenAsync(query, continuationToken);
var observations = data.Entities.Select(x => x.ToDomain()).ToList();
return(observations, data.ContinuationToken);
}
public async Task <(IReadOnlyCollection <WhitelistedAddress> Addresses, string ContinuationToken)> GetAllAsync(
int take,
string continuationToken)
{
IEnumerable <WhitelistedAddressEntity> addresses;
(addresses, continuationToken) = await _whitelistedAddresses.GetDataWithContinuationTokenAsync(take, continuationToken);
return(addresses.Select(x => new WhitelistedAddress(x.RowKey, x.MaxGasAmount)).ToImmutableList(), continuationToken);
}
public async Task <(IEnumerable <BlacklistedAddress> Addresses, string ContinuationToken)> GetAllAsync(
int take,
string continuationToken)
{
IEnumerable <BlacklistedAddressEntity> addresses;
(addresses, continuationToken) = await _blacklistedAddresses.GetDataWithContinuationTokenAsync(take, continuationToken);
return(addresses.Select(x => new BlacklistedAddress(x.RowKey, x.Reason)), continuationToken);
}
public async Task <(IEnumerable <WalletDto> Wallets, string ContinuationToken)> GetAllAsync(int take, string continuationToken)
{
IEnumerable <BlockchainWalletEntity> entities;
(entities, continuationToken) = await _walletsTable.GetDataWithContinuationTokenAsync(take, continuationToken);
var wallets = entities.Select(ConvertEntityToDto);
return(wallets, continuationToken);
}
public Task <(IEnumerable <BalanceEntity> balances, string continuationToken)> GetBalancesAsync(
string assetId,
int take,
string continuationToken)
{
var filter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, assetId);
var query = new TableQuery <BalanceEntity>().Where(filter);
return(_balances.GetDataWithContinuationTokenAsync(query, take, continuationToken));
}
GetAsync(string exchangeName, DateTime date, int take = 100, string continuationToken = null)
{
var result = await _storage.GetDataWithContinuationTokenAsync(
GetPartitionKey(exchangeName, date),
take,
continuationToken);
return(result.ContinuationToken, result.Entities.Select(x =>
(x.PartitionKey.Split(_splittingChars, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(),
x.MessagePayload)));
}
public async Task <(IEnumerable <T> Entities, string ContinuationToken)> GetDataWithContinuationTokenAsync(int take, string continuation)
{
try
{
return(await _azureRepo.GetDataWithContinuationTokenAsync(take, continuation));
}
catch (Exception ex)
{
throw new BusinessException(ErrorReason.BadRequest,
$"Invalid take or continuation token", ex);
}
}
public async Task RemoveOldOutputs(DateTime bound)
{
string continuation = null;
IEnumerable <SpentOutputEntity> outputs = null;
do
{
(outputs, continuation) = await _table.GetDataWithContinuationTokenAsync(100, continuation);
await Task.WhenAll(outputs.Where(o => o.Timestamp < bound).GroupBy(o => o.PartitionKey).Select(group => _table.DeleteAsync(group)));
} while (continuation != null);
}
private async Task AddNewAsync()
{
_log.Info("Add new indexes >>");
string continuationToken = null;
do
{
var result = await _oldStorage.GetDataWithContinuationTokenAsync(300, continuationToken);
continuationToken = result.ContinuationToken;
var tasks = new List <Task>();
foreach (var entity in result.Entities)
{
tasks.Add(AddNewAsync(entity));
}
await Task.WhenAll(tasks);
} while (continuationToken != null);
_log.Info("Add new indexes <<");
}
public async Task RemoveOldOutputsAsync(DateTime bound)
{
string continuation = null;
do
{
IEnumerable <SpentOutputEntity> outputs;
(outputs, continuation) = await _table.GetDataWithContinuationTokenAsync(100, continuation);
await outputs.Where(o => o.Timestamp < bound)
.GroupBy(o => o.PartitionKey)
.ForEachAsyncSemaphore(8, group => _table.DeleteAsync(group));
} while (continuation != null);
}
public async Task <(List <WalletBalance> Entities, string ContinuationToken)> GetAllAsync(int take, string continuationToken)
{
var query = new TableQuery <WalletBalanceEntity>().Take(take);
var data = await _table.GetDataWithContinuationTokenAsync(query, continuationToken);
var balances = new List <WalletBalance>();
foreach (var entity in data.Entities)
{
var balance = entity.ToDomain();
balances.Add(balance);
}
return(balances, data.ContinuationToken);
}
public async Task <DistributionPlanAggregate> TryGetAsync(
Guid planId)
{
var indexEntity = await _distributionPlanIndexTable.GetDataAsync
(
partition : GetIndexPartitionKey(planId),
row : GetIndexRowKey()
);
if (indexEntity == null)
{
return(null);
}
var planEntity = await _distributionPlanTable.GetDataAsync
(
partition : indexEntity.PrimaryPartitionKey,
row : indexEntity.PrimaryRowKey
);
if (planEntity == null)
{
return(null);
}
var amountEntities = new List <DistributionAmountEntity>();
var amountPartitionKey = GetAmountPartitionKey(planEntity.PlanTimestamp);
var continuationToken = (string)null;
do
{
IEnumerable <DistributionAmountEntity> entities;
(entities, continuationToken) = await _distributionAmountTable
.GetDataWithContinuationTokenAsync(amountPartitionKey, 1000, continuationToken);
amountEntities.AddRange(entities);
} while (continuationToken != null);
return(DistributionPlanAggregate.Restore
(
planEntity.PlanId,
planEntity.PlanTimestamp,
amountEntities
.Select(x => DistributionPlanAggregate.Amount.Restore(walletId: x.WalletId, amountId: x.AmountId, value: x.AmountValue))
.ToArray()
));
}
public async Task <(List <U> Items, string ContinuationToken)> GetAllAsync(int take, string continuationToken)
{
var query = new TableQuery <T>().Where(TableQuery.GenerateFilterCondition(nameof(ITableEntity.PartitionKey), QueryComparisons.Equal, typeof(U).Name))
.Take(take);
var data = await _table.GetDataWithContinuationTokenAsync(query, continuationToken);
var observations = new List <U>();
foreach (var entity in data.Entities)
{
var observation = entity.ToDomain();
observations.Add(observation);
}
return(observations, data.ContinuationToken);
}
public async Task RemoveOldOutputsAsync(DateTime bound)
{
string continuation = null;
do
{
IEnumerable <SpentOutputEntity> outputs;
(outputs, continuation) = await _table.GetDataWithContinuationTokenAsync(100, continuation);
await outputs.Where(o => o.Timestamp < bound)
.ForEachAsyncSemaphore(8, async output =>
{
await _table.DeleteIfExistAsync(output.PartitionKey, output.RowKey);
});
} while (continuation != null);
}
public async Task <(IReadOnlyList <ExternalTrade> entities, string continuationToken)> GetAsync(DateTime startDate,
DateTime endDate, int?limit, string continuationToken)
{
var filter = TableQuery.CombineFilters(
TableQuery.GenerateFilterCondition(nameof(ExternalTradeEntity.PartitionKey), QueryComparisons.GreaterThan,
GetPartitionKey(startDate)),
TableOperators.And,
TableQuery.GenerateFilterCondition(nameof(ExternalTradeEntity.PartitionKey), QueryComparisons.LessThan,
GetPartitionKey(endDate)));
var query = new TableQuery <ExternalTradeEntity>().Where(filter).Take(limit);
(IEnumerable <ExternalTradeEntity> entities, string token) =
await _storage.GetDataWithContinuationTokenAsync(query, continuationToken);
return(Mapper.Map <List <ExternalTrade> >(entities), token);
}
public async Task <(IEnumerable <BlackListModel>, string continuationToken)> TryGetAllAsync(string blockchainType, int take, string continuationToken = null)
{
if (!string.IsNullOrEmpty(continuationToken))
{
try
{
JsonConvert.DeserializeObject <TableContinuationToken>(Utils.HexToString(continuationToken));
}
catch
{
throw new ArgumentValidationException($"{continuationToken} continuationToken is not valid", $"{nameof(continuationToken)}");
}
}
var partitionKey = BlackListEntity.GetPartitionKey(blockchainType);
var(entities, newToken) = await _storage.GetDataWithContinuationTokenAsync(partitionKey, take, continuationToken);
return(entities?.Select(x => x.ToDomain()), newToken);
}
private async Task <SnapshotEntity[]> GetSnapshotEntitiesAsync(
DateTime from,
DateTime to)
{
var snapshotEntities = new List <SnapshotEntity>();
foreach (var rangeQuery in GetRangeQueries(from, to))
{
var continuationToken = (string)null;
do
{
IEnumerable <SnapshotEntity> entities;
(entities, continuationToken) = await _snapshotTable
.GetDataWithContinuationTokenAsync(rangeQuery, continuationToken);
snapshotEntities.AddRange(entities);
} while (continuationToken != null);
}
return(snapshotEntities.ToArray());
}
