public void ProcessEvent(WalletWithdrawalEventEntry eventEntry)
{
// This event was created by Wallet Service, so that it couldn't really access user's reserved balance.
// You may ask, how can we make sure no withdrawal occurs concurrently during a trade order creation?
// The trade command processor has strict sequential rules => such trade event persistence won't be allowed.
// Another issue occurs when a user has balance reserved in an open position.
// In that case we close his positions, so the trade command processor no longer uses them in calculation.
// Lastly, there is an issue with Wallet Service not having access to the user's current balance.
// This is solved using a Saga approach, so that we must validate the Withdrawal Event entry here,
// while the Wallet Service actively waits using a parallel thread. If the thread ever gets killed,
// the withdrawal process will get stuck, waiting for administrator to resolve the conflict.
// In the future, an alternative approach to unstuck such ignored withdrawals can be added.
if (eventEntry.Validated == false)
{
// Withdrawal was invalid anyway
return;
}
var overdrawn = eventEntry.OverdrawnAndCanceledOrders;
if (!overdrawn || eventEntry.Validated == null)
{
var(balance, reservedBalance) = _userService
.GetBalanceAndReservedBalance(eventEntry.User, eventEntry.AccountId, eventEntry.CoinSymbol);
if (eventEntry.Validated == null)
{
if (!IsValidWithdrawal(eventEntry, balance))
{
_eventHistoryService.ReportWithdrawalValidation(eventEntry, false);
// Withdrawal is invalid
return;
}
_eventHistoryService.ReportWithdrawalValidation(eventEntry, true);
}
if (!overdrawn && balance - reservedBalance < eventEntry.WithdrawalQty)
{
// First time the event has detected an overdraw, it will be persisted for reference purposes
// (and for a marginal calculation speedup next time the event gets processed after restart)
_eventHistoryService.ReportOverdrawnWithdrawal(eventEntry);
overdrawn = true;
}
}
if (overdrawn)
{
_logger.LogInformation(
$"User {eventEntry.User} accountId {eventEntry.AccountId} has overdrawn his {eventEntry.CoinSymbol} balance by a withdrawal so large, that all his positions need to be closed");
CloseUserOrders(eventEntry.User, eventEntry.AccountId, eventEntry.CoinSymbol, eventEntry.EntryTime);
}
_userService.ModifyBalance(
eventEntry.User,
eventEntry.AccountId,
eventEntry.CoinSymbol,
-eventEntry.WithdrawalQty - eventEntry.WithdrawalCombinedFee
).Wait();
}
public void ReportWithdrawalExecuted(WalletWithdrawalEventEntry withdrawal, Action afterMarkedExecuted)
{
// Note: this must occur after withdrawal event processing by this own service
// TODO: try to use VersionControl.WaitForIntegration instead (same functionality)
var retry = true;
while (retry)
{
VersionControl.ExecuteUsingFixedVersion(currentVersion =>
{
if (currentVersion < withdrawal.VersionNumber)
{
return;
}
EventHistoryRepository.Events().FindOneAndUpdate(
eventEntry => eventEntry.Id.Equals(withdrawal.Id),
Builders <EventEntry> .Update.Set(
eventEntry => ((WalletWithdrawalEventEntry)eventEntry).Executed,
true
)
);
afterMarkedExecuted();
_logger.LogInformation(
$"Reported executed withdrawal of {withdrawal.WithdrawalQty} {withdrawal.CoinSymbol}");
retry = false;
});
if (retry)
{
_logger.LogInformation(
$"{nameof(ReportWithdrawalExecuted)} waiting for integration of version number {withdrawal.VersionNumber}");
Task.Delay(1000).Wait();
}
}
}
private void ProcessEvent(WalletWithdrawalEventEntry eventEntry)
{
// We cannot reduce our cached balance, unlocking it for detection as a deposit,
// until we are sure the withdrawal is valid and it has been executed in a previous service run
while (eventEntry.Validated == null)
{
_logger.LogWarning(
$"Withdrawal event {eventEntry.Id} is not validated by Trading Service yet, waiting...");
Task.Delay(1000).Wait();
eventEntry = (WalletWithdrawalEventEntry)_eventHistoryService.FindById(eventEntry.Id);
}
if (eventEntry.Validated == false)
{
return;
}
OnWithdrawal(eventEntry);
if (eventEntry.Executed)
{
// We only process the balance unlock for deposit if it isn't being executed in this service run
UnlockAfterWithdrawal(eventEntry);
}
}
public void ReportOverdrawnWithdrawal(WalletWithdrawalEventEntry withdrawal)
{
EventHistoryRepository.Events().FindOneAndUpdate(
eventEntry => eventEntry.Id.Equals(withdrawal.Id),
Builders <EventEntry> .Update.Set(
eventEntry => ((WalletWithdrawalEventEntry)eventEntry).OverdrawnAndCanceledOrders,
true
)
);
}
public void ReportWithdrawalValidation(WalletWithdrawalEventEntry withdrawal, bool validation)
{
_logger.LogInformation(
$"Validation of {withdrawal.WithdrawalQty} {withdrawal.CoinSymbol} withdrawal {(validation ? "successful" : "failed")}");
EventHistoryRepository.Events().FindOneAndUpdate(
eventEntry => eventEntry.Id.Equals(withdrawal.Id),
Builders <EventEntry> .Update.Set(
eventEntry => ((WalletWithdrawalEventEntry)eventEntry).Validated,
validation
)
);
}
private void OnWithdrawal(WalletWithdrawalEventEntry eventEntry)
{
var oldBalance = GetCurrentlyCachedBalance(eventEntry.WithdrawalSourcePublicKey).Result;
var newBalance = oldBalance - eventEntry.WithdrawalQty - eventEntry.WithdrawalSingleFee;
if (newBalance != eventEntry.NewSourcePublicKeyBalance)
{
throw new Exception(
$"{GetType().Name} withdrawal event processing detected fatal event inconsistency of wallet values, new balance event value of {eventEntry.NewSourcePublicKeyBalance} should be {newBalance}");
}
// We unlock the balance for deposit, and immediately lock it back until withdrawal gets processed
_knownPublicKeyBalances[eventEntry.WithdrawalSourcePublicKey] = eventEntry.NewSourcePublicKeyBalance;
_lockedPublicKeyBalances[eventEntry.WithdrawalSourcePublicKey] +=
eventEntry.WithdrawalQty + eventEntry.WithdrawalSingleFee;
}
private bool IsValidWithdrawal(WalletWithdrawalEventEntry eventEntry, decimal userCoinBalance)
{
return(userCoinBalance >= eventEntry.WithdrawalQty + eventEntry.WithdrawalCombinedFee);
}
public override async Task PrepareWithdrawalAsync(
WalletWithdrawalEventEntry withdrawalEventEntry, Action revocationAction)
{
try
{
var withdrawalDescription =
$"Withdrawal of {withdrawalEventEntry.WithdrawalQty} {withdrawalEventEntry.CoinSymbol} of user {withdrawalEventEntry.User} to wallet {withdrawalEventEntry.WithdrawalTargetPublicKey}";
// Saga operation: we wait for TradingService to confirm the withdrawal event
for (var i = 0; i < 60; i++)
{
await Task.Delay(1000);
var validated = ((WalletWithdrawalEventEntry)
_eventHistoryService.FindById(withdrawalEventEntry.Id)).Validated;
switch (validated)
{
case true:
{
// We have to be sure all consolidations have also finished
_versionControl.WaitForIntegration(withdrawalEventEntry.VersionNumber);
// And now we are ready for the main goal
var success = Withdraw(
withdrawalEventEntry.WithdrawalSourcePublicKey,
withdrawalEventEntry.WithdrawalTargetPublicKey,
withdrawalEventEntry.WithdrawalQty
).Result;
_logger.LogInformation(
$"{withdrawalDescription} {(success ? "successful" : "has failed due to blockchain response, this is a critical error and the event will be revoked")}");
if (success)
{
EnsureWithdrawn(
withdrawalEventEntry.WithdrawalSourcePublicKey,
withdrawalEventEntry.NewSourcePublicKeyBalance,
withdrawalEventEntry.WithdrawalQty + withdrawalEventEntry.WithdrawalSingleFee,
withdrawalEventEntry.VersionNumber);
// Note: this report will occur after event gets processed by this own service
_eventHistoryService.ReportWithdrawalExecuted(withdrawalEventEntry, () =>
{
// We will unlock the amount for deposit
// (We want this to occur inside a version number lock, so that no other events
// apply themselves right after the withdrawal event processing)
UnlockAfterWithdrawal(withdrawalEventEntry);
});
}
else
{
// Validation successful, yet the blockchain refused the transaction,
// so we can immediately unlock user's balance for trading or another retry
revocationAction();
}
return;
}
case false:
// Validation failed
_logger.LogInformation(
$"{withdrawalDescription} has failed due to negative response of a Trading Backend validation, revocation initiated");
revocationAction();
return;
case null:
_logger.LogInformation($"{withdrawalDescription} still waiting for a validation...");
break;
}
}
// Timed out
_logger.LogInformation($"{withdrawalDescription} has failed due to a timeout, revocation initiated");
revocationAction();
}
catch (AggregateException e)
{
_logger.LogError(
$"{e.GetType()} happened during {nameof(PrepareWithdrawalAsync)}, this was unexpected.");
foreach (var inner in e.InnerExceptions)
{
_logger.LogError($"\n{inner.Message}\n{inner.StackTrace}");
}
}
catch (Exception e)
{
_logger.LogError(
$"{e.GetType()} happened during {nameof(PrepareWithdrawalAsync)}, this was unexpected.\n{e.Message}\n{e.StackTrace}");
}
}
protected virtual void UnlockAfterWithdrawal(WalletWithdrawalEventEntry eventEntry)
{
_lockedPublicKeyBalances[eventEntry.WithdrawalSourcePublicKey] -=
eventEntry.WithdrawalQty + eventEntry.WithdrawalSingleFee;
}
public abstract Task PrepareWithdrawalAsync(
WalletWithdrawalEventEntry withdrawalEventEntry, Action revocationAction);