public async Task ExportAsync(TransactionsReportBuilder reportBuilder)
{
var depositWallets = await LoadDepositWalletsAsync();
_totalDepositWalletsCount = depositWallets.Count;
var tasks = new List <Task>(512);
_log.Info("Exporting deposits...", new
{
DepositsHistoryProvider = _depositsHistoryProviders.Select(x => x.GetType().Name)
});
foreach (var wallet in depositWallets)
{
await _concurrencySemaphore.WaitAsync();
tasks.Add(ProcessDepositWalletAsync(reportBuilder, wallet));
if (tasks.Count >= 500)
{
await Task.WhenAll(tasks);
tasks.Clear();
}
}
await Task.WhenAll(tasks);
_log.Info($"Deposits exporting done. {_processedWalletsCount} deposit wallets processed. {_exportedDepositsCount} deposits exported");
}
public Exporter(
ILogFactory logFactory,
TransactionsReportBuilder transactionsReportBuilder,
IServiceProvider serviceProvider,
DepositsExporter depositsExporter)
{
_log = logFactory.CreateLog(this);
_transactionsReportBuilder = transactionsReportBuilder;
_serviceProvider = serviceProvider;
_depositsExporter = depositsExporter;
}
public async Task ExportAsync(TransactionsReportBuilder reportBuilder)
{
_log.Info("Exporting withdrawals...", new
{
WithdrawalHistoryProviders = _withdrawalsHistoryProviders.Select(x => x.GetType().Name)
});
var tasks = new List <Task>();
foreach (var historyProvider in _withdrawalsHistoryProviders)
{
tasks.Add(ExportProviderWithdrawals(reportBuilder, historyProvider));
}
await Task.WhenAll(tasks);
_log.Info($"Withdrawals exporting done. {_exportedWithdrawalsCount} withdrawals exported");
}
private async Task ExportProviderWithdrawals(TransactionsReportBuilder reportBuilder, IWithdrawalsHistoryProvider historyProvider)
{
PaginatedList <Transaction> transactions = null;
do
{
transactions = await Policy
.Handle <Exception>(ex =>
{
_log.Warning
(
"Failed to get withdrawals history. Operation will be retried.",
context: new
{
WithdrawalHistoryProvider = historyProvider.GetType().Name
},
exception: ex
);
return(true);
})
.WaitAndRetryForeverAsync(i => TimeSpan.FromSeconds(Math.Min(i, 5)))
.ExecuteAsync(async() => await historyProvider.GetHistoryAsync(transactions?.Continuation));
foreach (var tx in transactions.Items)
{
var normalizedTransaction = NormalizeTransactionOrDefault(tx);
if (normalizedTransaction == null)
{
continue;
}
reportBuilder.AddTransaction(normalizedTransaction);
var exportedWithdrawalsCount = Interlocked.Increment(ref _exportedWithdrawalsCount);
if (exportedWithdrawalsCount % 1000 == 0)
{
_log.Info($"{exportedWithdrawalsCount} withdrawals exported so far");
}
}
} while (transactions.Continuation != null);
}
private async Task ProcessDepositWalletAsync(TransactionsReportBuilder reportBuilder, DepositWallet wallet)
{
try
{
foreach (var historyProvider in _depositsHistoryProviders)
{
if (!historyProvider.CanProvideHistoryFor(wallet))
{
continue;
}
PaginatedList <Transaction> transactions = null;
var processedWalletTransactionsCount = 0;
do
{
transactions = await Policy
.Handle <Exception>(ex =>
{
_log.Warning
(
"Failed to get deposits history. Operation will be retried.",
context: new
{
Address = wallet.Address,
CryptoCurrency = wallet.CryptoCurrency,
HistoryProvider = historyProvider.GetType().Name
},
exception: ex
);
return(true);
})
.WaitAndRetryForeverAsync(i => TimeSpan.FromSeconds(Math.Min(i, 5)))
.ExecuteAsync(async() => await historyProvider.GetHistoryAsync(wallet, transactions?.Continuation));
foreach (var tx in transactions.Items)
{
var normalizedTransaction = NormalizeTransactionOrDefault(tx);
if (normalizedTransaction == null)
{
continue;
}
reportBuilder.AddTransaction(normalizedTransaction);
Interlocked.Increment(ref _exportedDepositsCount);
++processedWalletTransactionsCount;
if (processedWalletTransactionsCount % 100 == 0)
{
_log.Info($"{processedWalletTransactionsCount} deposits processed so far of {wallet.CryptoCurrency}:{wallet.Address} wallet using {historyProvider.GetType().Name}");
}
}
} while (transactions.Continuation != null);
}
}
catch (Exception ex)
{
throw new InvalidOperationException($"Failed to process deposit wallet: {wallet.CryptoCurrency}:{wallet.Address}:{wallet.UserId}", ex);
}
finally
{
var processedWalletsCount = Interlocked.Increment(ref _processedWalletsCount);
if (processedWalletsCount % 100 == 0)
{
var completedPercent = processedWalletsCount * 100 / _totalDepositWalletsCount;
_log.Info($"{processedWalletsCount} wallets processed so far ({completedPercent}%). {_exportedDepositsCount} deposits exported so far.");
}
_concurrencySemaphore.Release();
}
}
