public async Task ExportAsync(TransactionsReportBuilder reportBuilder) { var depositWallets = await LoadDepositWalletsAsync(); _totalDepositWalletsCount = depositWallets.Count; var tasks = new List <Task>(512); _log.Info("Exporting deposits...", new { DepositsHistoryProvider = _depositsHistoryProviders.Select(x => x.GetType().Name) }); foreach (var wallet in depositWallets) { await _concurrencySemaphore.WaitAsync(); tasks.Add(ProcessDepositWalletAsync(reportBuilder, wallet)); if (tasks.Count >= 500) { await Task.WhenAll(tasks); tasks.Clear(); } } await Task.WhenAll(tasks); _log.Info($"Deposits exporting done. {_processedWalletsCount} deposit wallets processed. {_exportedDepositsCount} deposits exported"); }
public Exporter( ILogFactory logFactory, TransactionsReportBuilder transactionsReportBuilder, IServiceProvider serviceProvider, DepositsExporter depositsExporter) { _log = logFactory.CreateLog(this); _transactionsReportBuilder = transactionsReportBuilder; _serviceProvider = serviceProvider; _depositsExporter = depositsExporter; }
public async Task ExportAsync(TransactionsReportBuilder reportBuilder) { _log.Info("Exporting withdrawals...", new { WithdrawalHistoryProviders = _withdrawalsHistoryProviders.Select(x => x.GetType().Name) }); var tasks = new List <Task>(); foreach (var historyProvider in _withdrawalsHistoryProviders) { tasks.Add(ExportProviderWithdrawals(reportBuilder, historyProvider)); } await Task.WhenAll(tasks); _log.Info($"Withdrawals exporting done. {_exportedWithdrawalsCount} withdrawals exported"); }
private async Task ExportProviderWithdrawals(TransactionsReportBuilder reportBuilder, IWithdrawalsHistoryProvider historyProvider) { PaginatedList <Transaction> transactions = null; do { transactions = await Policy .Handle <Exception>(ex => { _log.Warning ( "Failed to get withdrawals history. Operation will be retried.", context: new { WithdrawalHistoryProvider = historyProvider.GetType().Name }, exception: ex ); return(true); }) .WaitAndRetryForeverAsync(i => TimeSpan.FromSeconds(Math.Min(i, 5))) .ExecuteAsync(async() => await historyProvider.GetHistoryAsync(transactions?.Continuation)); foreach (var tx in transactions.Items) { var normalizedTransaction = NormalizeTransactionOrDefault(tx); if (normalizedTransaction == null) { continue; } reportBuilder.AddTransaction(normalizedTransaction); var exportedWithdrawalsCount = Interlocked.Increment(ref _exportedWithdrawalsCount); if (exportedWithdrawalsCount % 1000 == 0) { _log.Info($"{exportedWithdrawalsCount} withdrawals exported so far"); } } } while (transactions.Continuation != null); }
private async Task ProcessDepositWalletAsync(TransactionsReportBuilder reportBuilder, DepositWallet wallet) { try { foreach (var historyProvider in _depositsHistoryProviders) { if (!historyProvider.CanProvideHistoryFor(wallet)) { continue; } PaginatedList <Transaction> transactions = null; var processedWalletTransactionsCount = 0; do { transactions = await Policy .Handle <Exception>(ex => { _log.Warning ( "Failed to get deposits history. Operation will be retried.", context: new { Address = wallet.Address, CryptoCurrency = wallet.CryptoCurrency, HistoryProvider = historyProvider.GetType().Name }, exception: ex ); return(true); }) .WaitAndRetryForeverAsync(i => TimeSpan.FromSeconds(Math.Min(i, 5))) .ExecuteAsync(async() => await historyProvider.GetHistoryAsync(wallet, transactions?.Continuation)); foreach (var tx in transactions.Items) { var normalizedTransaction = NormalizeTransactionOrDefault(tx); if (normalizedTransaction == null) { continue; } reportBuilder.AddTransaction(normalizedTransaction); Interlocked.Increment(ref _exportedDepositsCount); ++processedWalletTransactionsCount; if (processedWalletTransactionsCount % 100 == 0) { _log.Info($"{processedWalletTransactionsCount} deposits processed so far of {wallet.CryptoCurrency}:{wallet.Address} wallet using {historyProvider.GetType().Name}"); } } } while (transactions.Continuation != null); } } catch (Exception ex) { throw new InvalidOperationException($"Failed to process deposit wallet: {wallet.CryptoCurrency}:{wallet.Address}:{wallet.UserId}", ex); } finally { var processedWalletsCount = Interlocked.Increment(ref _processedWalletsCount); if (processedWalletsCount % 100 == 0) { var completedPercent = processedWalletsCount * 100 / _totalDepositWalletsCount; _log.Info($"{processedWalletsCount} wallets processed so far ({completedPercent}%). {_exportedDepositsCount} deposits exported so far."); } _concurrencySemaphore.Release(); } }