public void ProcessEvent(WalletWithdrawalEventEntry eventEntry) { // This event was created by Wallet Service, so that it couldn't really access user's reserved balance. // You may ask, how can we make sure no withdrawal occurs concurrently during a trade order creation? // The trade command processor has strict sequential rules => such trade event persistence won't be allowed. // Another issue occurs when a user has balance reserved in an open position. // In that case we close his positions, so the trade command processor no longer uses them in calculation. // Lastly, there is an issue with Wallet Service not having access to the user's current balance. // This is solved using a Saga approach, so that we must validate the Withdrawal Event entry here, // while the Wallet Service actively waits using a parallel thread. If the thread ever gets killed, // the withdrawal process will get stuck, waiting for administrator to resolve the conflict. // In the future, an alternative approach to unstuck such ignored withdrawals can be added. if (eventEntry.Validated == false) { // Withdrawal was invalid anyway return; } var overdrawn = eventEntry.OverdrawnAndCanceledOrders; if (!overdrawn || eventEntry.Validated == null) { var(balance, reservedBalance) = _userService .GetBalanceAndReservedBalance(eventEntry.User, eventEntry.AccountId, eventEntry.CoinSymbol); if (eventEntry.Validated == null) { if (!IsValidWithdrawal(eventEntry, balance)) { _eventHistoryService.ReportWithdrawalValidation(eventEntry, false); // Withdrawal is invalid return; } _eventHistoryService.ReportWithdrawalValidation(eventEntry, true); } if (!overdrawn && balance - reservedBalance < eventEntry.WithdrawalQty) { // First time the event has detected an overdraw, it will be persisted for reference purposes // (and for a marginal calculation speedup next time the event gets processed after restart) _eventHistoryService.ReportOverdrawnWithdrawal(eventEntry); overdrawn = true; } } if (overdrawn) { _logger.LogInformation( $"User {eventEntry.User} accountId {eventEntry.AccountId} has overdrawn his {eventEntry.CoinSymbol} balance by a withdrawal so large, that all his positions need to be closed"); CloseUserOrders(eventEntry.User, eventEntry.AccountId, eventEntry.CoinSymbol, eventEntry.EntryTime); } _userService.ModifyBalance( eventEntry.User, eventEntry.AccountId, eventEntry.CoinSymbol, -eventEntry.WithdrawalQty - eventEntry.WithdrawalCombinedFee ).Wait(); }
public void ReportWithdrawalExecuted(WalletWithdrawalEventEntry withdrawal, Action afterMarkedExecuted) { // Note: this must occur after withdrawal event processing by this own service // TODO: try to use VersionControl.WaitForIntegration instead (same functionality) var retry = true; while (retry) { VersionControl.ExecuteUsingFixedVersion(currentVersion => { if (currentVersion < withdrawal.VersionNumber) { return; } EventHistoryRepository.Events().FindOneAndUpdate( eventEntry => eventEntry.Id.Equals(withdrawal.Id), Builders <EventEntry> .Update.Set( eventEntry => ((WalletWithdrawalEventEntry)eventEntry).Executed, true ) ); afterMarkedExecuted(); _logger.LogInformation( $"Reported executed withdrawal of {withdrawal.WithdrawalQty} {withdrawal.CoinSymbol}"); retry = false; }); if (retry) { _logger.LogInformation( $"{nameof(ReportWithdrawalExecuted)} waiting for integration of version number {withdrawal.VersionNumber}"); Task.Delay(1000).Wait(); } } }
private void ProcessEvent(WalletWithdrawalEventEntry eventEntry) { // We cannot reduce our cached balance, unlocking it for detection as a deposit, // until we are sure the withdrawal is valid and it has been executed in a previous service run while (eventEntry.Validated == null) { _logger.LogWarning( $"Withdrawal event {eventEntry.Id} is not validated by Trading Service yet, waiting..."); Task.Delay(1000).Wait(); eventEntry = (WalletWithdrawalEventEntry)_eventHistoryService.FindById(eventEntry.Id); } if (eventEntry.Validated == false) { return; } OnWithdrawal(eventEntry); if (eventEntry.Executed) { // We only process the balance unlock for deposit if it isn't being executed in this service run UnlockAfterWithdrawal(eventEntry); } }
public void ReportOverdrawnWithdrawal(WalletWithdrawalEventEntry withdrawal) { EventHistoryRepository.Events().FindOneAndUpdate( eventEntry => eventEntry.Id.Equals(withdrawal.Id), Builders <EventEntry> .Update.Set( eventEntry => ((WalletWithdrawalEventEntry)eventEntry).OverdrawnAndCanceledOrders, true ) ); }
public void ReportWithdrawalValidation(WalletWithdrawalEventEntry withdrawal, bool validation) { _logger.LogInformation( $"Validation of {withdrawal.WithdrawalQty} {withdrawal.CoinSymbol} withdrawal {(validation ? "successful" : "failed")}"); EventHistoryRepository.Events().FindOneAndUpdate( eventEntry => eventEntry.Id.Equals(withdrawal.Id), Builders <EventEntry> .Update.Set( eventEntry => ((WalletWithdrawalEventEntry)eventEntry).Validated, validation ) ); }
private void OnWithdrawal(WalletWithdrawalEventEntry eventEntry) { var oldBalance = GetCurrentlyCachedBalance(eventEntry.WithdrawalSourcePublicKey).Result; var newBalance = oldBalance - eventEntry.WithdrawalQty - eventEntry.WithdrawalSingleFee; if (newBalance != eventEntry.NewSourcePublicKeyBalance) { throw new Exception( $"{GetType().Name} withdrawal event processing detected fatal event inconsistency of wallet values, new balance event value of {eventEntry.NewSourcePublicKeyBalance} should be {newBalance}"); } // We unlock the balance for deposit, and immediately lock it back until withdrawal gets processed _knownPublicKeyBalances[eventEntry.WithdrawalSourcePublicKey] = eventEntry.NewSourcePublicKeyBalance; _lockedPublicKeyBalances[eventEntry.WithdrawalSourcePublicKey] += eventEntry.WithdrawalQty + eventEntry.WithdrawalSingleFee; }
private bool IsValidWithdrawal(WalletWithdrawalEventEntry eventEntry, decimal userCoinBalance) { return(userCoinBalance >= eventEntry.WithdrawalQty + eventEntry.WithdrawalCombinedFee); }
public override async Task PrepareWithdrawalAsync( WalletWithdrawalEventEntry withdrawalEventEntry, Action revocationAction) { try { var withdrawalDescription = $"Withdrawal of {withdrawalEventEntry.WithdrawalQty} {withdrawalEventEntry.CoinSymbol} of user {withdrawalEventEntry.User} to wallet {withdrawalEventEntry.WithdrawalTargetPublicKey}"; // Saga operation: we wait for TradingService to confirm the withdrawal event for (var i = 0; i < 60; i++) { await Task.Delay(1000); var validated = ((WalletWithdrawalEventEntry) _eventHistoryService.FindById(withdrawalEventEntry.Id)).Validated; switch (validated) { case true: { // We have to be sure all consolidations have also finished _versionControl.WaitForIntegration(withdrawalEventEntry.VersionNumber); // And now we are ready for the main goal var success = Withdraw( withdrawalEventEntry.WithdrawalSourcePublicKey, withdrawalEventEntry.WithdrawalTargetPublicKey, withdrawalEventEntry.WithdrawalQty ).Result; _logger.LogInformation( $"{withdrawalDescription} {(success ? "successful" : "has failed due to blockchain response, this is a critical error and the event will be revoked")}"); if (success) { EnsureWithdrawn( withdrawalEventEntry.WithdrawalSourcePublicKey, withdrawalEventEntry.NewSourcePublicKeyBalance, withdrawalEventEntry.WithdrawalQty + withdrawalEventEntry.WithdrawalSingleFee, withdrawalEventEntry.VersionNumber); // Note: this report will occur after event gets processed by this own service _eventHistoryService.ReportWithdrawalExecuted(withdrawalEventEntry, () => { // We will unlock the amount for deposit // (We want this to occur inside a version number lock, so that no other events // apply themselves right after the withdrawal event processing) UnlockAfterWithdrawal(withdrawalEventEntry); }); } else { // Validation successful, yet the blockchain refused the transaction, // so we can immediately unlock user's balance for trading or another retry revocationAction(); } return; } case false: // Validation failed _logger.LogInformation( $"{withdrawalDescription} has failed due to negative response of a Trading Backend validation, revocation initiated"); revocationAction(); return; case null: _logger.LogInformation($"{withdrawalDescription} still waiting for a validation..."); break; } } // Timed out _logger.LogInformation($"{withdrawalDescription} has failed due to a timeout, revocation initiated"); revocationAction(); } catch (AggregateException e) { _logger.LogError( $"{e.GetType()} happened during {nameof(PrepareWithdrawalAsync)}, this was unexpected."); foreach (var inner in e.InnerExceptions) { _logger.LogError($"\n{inner.Message}\n{inner.StackTrace}"); } } catch (Exception e) { _logger.LogError( $"{e.GetType()} happened during {nameof(PrepareWithdrawalAsync)}, this was unexpected.\n{e.Message}\n{e.StackTrace}"); } }
protected virtual void UnlockAfterWithdrawal(WalletWithdrawalEventEntry eventEntry) { _lockedPublicKeyBalances[eventEntry.WithdrawalSourcePublicKey] -= eventEntry.WithdrawalQty + eventEntry.WithdrawalSingleFee; }
public abstract Task PrepareWithdrawalAsync( WalletWithdrawalEventEntry withdrawalEventEntry, Action revocationAction);