private async Task Handle(DeleteAccountsStartedInternalEvent e, ICommandSender sender)
{
var executionInfo = await _executionInfoRepository.GetAsync <DeleteAccountsData>(
OperationName,
e.OperationId
);
if (executionInfo == null)
{
return;
}
if (executionInfo.Data.SwitchState(DeleteAccountsState.Initiated, DeleteAccountsState.Started))
{
executionInfo.Data.AddFailedIfNotExist(e.FailedAccountIds);
sender.SendCommand(new BlockAccountsForDeletionCommand
{
OperationId = e.OperationId,
Timestamp = _systemClock.UtcNow.UtcDateTime,
AccountIds = executionInfo.Data.GetAccountsToDelete(),
},
_contextNames.TradingEngine);
_chaosKitty.Meow(
$"{nameof(DeleteAccountsStartedInternalEvent)}: " +
"Save_OperationExecutionInfo: " +
$"{e.OperationId}");
await _executionInfoRepository.SaveAsync(executionInfo);
}
}
private async Task Handle(SpecialLiquidationStartedInternalEvent e, ICommandSender sender)
{
var executionInfo = await _operationExecutionInfoRepository.GetAsync <SpecialLiquidationOperationData>(
operationName : OperationName,
id : e.OperationId);
if (executionInfo?.Data == null)
{
return;
}
if (executionInfo.Data.SwitchState(SpecialLiquidationOperationState.Started,
SpecialLiquidationOperationState.PriceRequested))
{
var positionsVolume = GetNetPositionCloseVolume(executionInfo.Data.PositionIds, executionInfo.Data.AccountId);
executionInfo.Data.Instrument = e.Instrument;
executionInfo.Data.Volume = positionsVolume;
executionInfo.Data.RequestNumber = 1;
RequestPrice(sender, executionInfo);
_chaosKitty.Meow(e.OperationId);
await _operationExecutionInfoRepository.Save(executionInfo);
}
}
public async Task <RfqPauseErrorCode> AddAsync(string operationId, PauseSource source, Initiator initiator)
{
if (string.IsNullOrEmpty(operationId))
{
throw new ArgumentNullException(nameof(operationId));
}
var locker = _lock.GetOrAdd(operationId, new SemaphoreSlim(1, 1));
await locker.WaitAsync();
try
{
var existingPause = (await _pauseRepository.FindAsync(
operationId,
SpecialLiquidationSaga.OperationName,
NotCancelledPredicate))
.SingleOrDefault();
if (existingPause != null)
{
await _log.WriteWarningAsync(nameof(RfqPauseService), nameof(AddAsync), $"There is already pause with state [{existingPause.State}] for operation with id [{operationId}]");
return(RfqPauseErrorCode.AlreadyExists);
}
var executionInfo = await _executionInfoRepository
.GetAsync <SpecialLiquidationOperationData>(SpecialLiquidationSaga.OperationName, operationId);
if (executionInfo == null)
{
return(RfqPauseErrorCode.NotFound);
}
if (!AllowedOperationStatesToPauseIn.Contains(executionInfo.Data.State))
{
await _log.WriteWarningAsync(nameof(RfqPauseService), nameof(AddAsync),
$"There was an attempt to pause special liquidation with id {operationId} and state {executionInfo.Data.State}. Pause is possible in [{string.Join(',', AllowedOperationStatesToPauseIn)}] states only");
return(RfqPauseErrorCode.InvalidOperationState);
}
var pause = Pause.Create(
operationId,
SpecialLiquidationSaga.OperationName,
source,
initiator,
_dateService.Now());
await _pauseRepository.AddAsync(pause);
// todo: add audit log
}
finally
{
locker.Release();
}
return(RfqPauseErrorCode.None);
}
private async Task Handle(WithdrawalStartedInternalEvent e, ICommandSender sender)
{
var executionInfo = await _executionInfoRepository.GetAsync <WithdrawalDepositData>(OperationName, e.OperationId);
if (executionInfo == null)
{
return;
}
if (SwitchState(executionInfo.Data, WithdrawalState.Created, WithdrawalState.FreezingAmount))
{
sender.SendCommand(
new FreezeAmountForWithdrawalCommand(
executionInfo.Id,
_systemClock.UtcNow.UtcDateTime,
executionInfo.Data.AccountId,
executionInfo.Data.Amount,
executionInfo.Data.Comment),
_contextNames.TradingEngine);
_chaosKitty.Meow(e.OperationId);
await _executionInfoRepository.SaveAsync(executionInfo);
}
}
private async Task <CommandHandlingResult> Handle(GetPriceForSpecialLiquidationTimeoutInternalCommand command,
IEventPublisher publisher)
{
var executionInfo = await _operationExecutionInfoRepository.GetAsync <SpecialLiquidationOperationData>(
operationName : SpecialLiquidationSaga.OperationName,
id : command.OperationId);
if (executionInfo?.Data != null)
{
if (executionInfo.Data.State > SpecialLiquidationOperationState.PriceRequested ||
executionInfo.Data.RequestNumber > command.RequestNumber)
{
return(CommandHandlingResult.Ok());
}
if (_dateService.Now() >= command.CreationTime.AddSeconds(command.TimeoutSeconds))
{
publisher.PublishEvent(new SpecialLiquidationFailedEvent
{
OperationId = command.OperationId,
CreationTime = _dateService.Now(),
Reason = $"Timeout of {command.TimeoutSeconds} seconds from {command.CreationTime:s}",
CanRetryPriceRequest = true
});
return(CommandHandlingResult.Ok());
}
}
return(CommandHandlingResult.Fail(_marginTradingSettings.SpecialLiquidation.PriceRequestTimeoutCheckPeriod));
}
public async Task Handle(FailLiquidationInternalCommand command,
IEventPublisher publisher)
{
var executionInfo = await _operationExecutionInfoRepository.GetAsync <LiquidationOperationData>(
operationName : LiquidationSaga.OperationName,
id : command.OperationId);
if (executionInfo?.Data == null)
{
return;
}
_accountUpdateService.RemoveLiquidationStateIfNeeded(executionInfo.Data.AccountId,
$"Liquidation [{command.OperationId}] failed ({command.Reason})", command.OperationId,
executionInfo.Data.LiquidationType);
_chaosKitty.Meow(
$"{nameof(FailLiquidationInternalCommand)}:" +
$"Publish_LiquidationFailedInternalEvent:" +
$"{command.OperationId}");
publisher.PublishEvent(new LiquidationFailedEvent
{
OperationId = command.OperationId,
CreationTime = _dateService.Now(),
Reason = command.Reason,
LiquidationType = command.LiquidationType.ToType <LiquidationTypeContract>(),
AccountId = executionInfo.Data.AccountId,
AssetPairId = executionInfo.Data.AssetPairId,
Direction = executionInfo.Data.Direction?.ToType <PositionDirectionContract>(),
QuoteInfo = executionInfo.Data.QuoteInfo,
ProcessedPositionIds = executionInfo.Data.ProcessedPositionIds,
LiquidatedPositionIds = executionInfo.Data.LiquidatedPositionIds,
OpenPositionsRemainingOnAccount = _ordersCache.Positions.GetPositionsByAccountIds(executionInfo.Data.AccountId).Count,
CurrentTotalCapital = _accountsCache.Get(executionInfo.Data.AccountId).GetTotalCapital(),
});
_liquidationEndEventChannel.SendEvent(this, new LiquidationEndEventArgs
{
OperationId = command.OperationId,
CreationTime = _dateService.Now(),
AccountId = executionInfo.Data.AccountId,
LiquidatedPositionIds = executionInfo.Data.LiquidatedPositionIds,
FailReason = command.Reason,
});
}
private async Task Handle(FailWithdrawalInternalCommand command, IEventPublisher publisher)
{
var executionInfo = await _executionInfoRepository.GetAsync <WithdrawalDepositData>(
OperationName,
command.OperationId
);
if (executionInfo == null)
{
return;
}
var account = await _accountsRepository.GetAsync(executionInfo.Data.AccountId);
publisher.PublishEvent(new WithdrawalFailedEvent(
command.OperationId,
_systemClock.UtcNow.UtcDateTime,
command.Reason,
executionInfo.Data.AccountId,
account?.ClientId,
executionInfo.Data.Amount));
}
public async Task Handle(RevokeTemporaryCapitalStartedInternalEvent e, ICommandSender sender)
{
var executionInfo = await _executionInfoRepository.GetAsync <RevokeTemporaryCapitalData>(
OperationName,
e.OperationId
);
if (executionInfo == null)
{
return;
}
if (executionInfo.Data.SwitchState(TemporaryCapitalState.Initiated, TemporaryCapitalState.Started))
{
executionInfo.Data.RevokedTemporaryCapital = e.RevokedTemporaryCapital;
sender.SendCommand(
new UpdateBalanceInternalCommand(
e.OperationId,
executionInfo.Data.AccountId,
-executionInfo.Data.RevokedTemporaryCapital.Sum(x => x.Amount),
$"{executionInfo.Data.Comment} *** Revoked eventSourceIds: [{string.Join(",", executionInfo.Data.RevokedTemporaryCapital.Select(x => x.Id))}]",
executionInfo.Data.AdditionalInfo,
OperationName,
AccountBalanceChangeReasonType.TemporaryCashAdjustment,
executionInfo.Data.EventSourceId,
string.Empty,
_systemClock.UtcNow.UtcDateTime),
_contextNames.AccountsManagement);
_chaosKitty.Meow(
$"{nameof(RevokeTemporaryCapitalStartedInternalEvent)}: " +
"Save_OperationExecutionInfo:" +
$"{e.OperationId}");
await _executionInfoRepository.SaveAsync(executionInfo);
}
}
private async Task Handle(DepositStartedInternalEvent e, ICommandSender sender)
{
var executionInfo = await _executionInfoRepository.GetAsync <WithdrawalDepositData>(
OperationName,
e.OperationId
);
if (executionInfo == null)
{
return;
}
if (SwitchState(executionInfo.Data, WithdrawalState.Created, WithdrawalState.FreezingAmount))
{
sender.SendCommand(
new FreezeAmountForDepositInternalCommand(e.OperationId),
_contextNames.AccountsManagement);
_chaosKitty.Meow(e.OperationId);
await _executionInfoRepository.SaveAsync(executionInfo);
}
}
private async Task Handle(SpecialLiquidationStartedInternalEvent e, ICommandSender sender)
{
var executionInfo = await _operationExecutionInfoRepository.GetAsync <SpecialLiquidationOperationData>(
operationName : OperationName,
id : e.OperationId);
if (executionInfo?.Data == null)
{
return;
}
if (executionInfo.Data.SwitchState(SpecialLiquidationOperationState.Started,
SpecialLiquidationOperationState.PriceRequested))
{
var positionsVolume = GetNetPositionCloseVolume(executionInfo.Data.PositionIds, executionInfo.Data.AccountId);
//special command is sent instantly for timeout control.. it is retried until timeout occurs
sender.SendCommand(new GetPriceForSpecialLiquidationTimeoutInternalCommand
{
OperationId = e.OperationId,
CreationTime = _dateService.Now(),
TimeoutSeconds = _marginTradingSettings.SpecialLiquidation.PriceRequestTimeoutSec,
}, _cqrsContextNamesSettings.TradingEngine);
executionInfo.Data.Instrument = e.Instrument;
executionInfo.Data.Volume = positionsVolume;
if (_marginTradingSettings.ExchangeConnector == ExchangeConnectorType.RealExchangeConnector)
{
//send it to the Gavel
sender.SendCommand(new GetPriceForSpecialLiquidationCommand
{
OperationId = e.OperationId,
CreationTime = _dateService.Now(),
Instrument = e.Instrument,
Volume = positionsVolume != 0 ? positionsVolume : 1,//hack, requested by the bank
RequestNumber = 1,
}, _cqrsContextNamesSettings.Gavel);
}
else
{
_specialLiquidationService.FakeGetPriceForSpecialLiquidation(e.OperationId, e.Instrument,
positionsVolume);
}
_chaosKitty.Meow(e.OperationId);
await _operationExecutionInfoRepository.Save(executionInfo);
}
}
public async Task Handle(FinishGiveTemporaryCapitalInternalCommand c, IEventPublisher publisher)
{
var executionInfo = await _executionInfoRepository.GetAsync <GiveTemporaryCapitalData>(OperationName, c.OperationId);
if (executionInfo == null)
{
return;
}
if (!new[] { TemporaryCapitalState.ChargedOnAccount, TemporaryCapitalState.Failing }
.Contains(executionInfo.Data.State))
{
throw new Exception($"{nameof(FinishGiveTemporaryCapitalInternalCommand)} have state {executionInfo.Data.State.ToString()}, but one of [{TemporaryCapitalState.ChargedOnAccount}, {TemporaryCapitalState.Failing}] was expected. Throwing to retry in {(long) _settings.Cqrs.RetryDelay.TotalMilliseconds}ms.");
}
if (executionInfo.Data.State == TemporaryCapitalState.ChargedOnAccount)
{
publisher.PublishEvent(new GiveTemporaryCapitalSucceededEvent(
c.OperationId,
_systemClock.UtcNow.UtcDateTime,
executionInfo.Data.OperationId,
executionInfo.Data.AccountId,
executionInfo.Data.Amount,
executionInfo.Data.Reason,
executionInfo.Data.Comment,
executionInfo.Data.AdditionalInfo));
}
else if (executionInfo.Data.State == TemporaryCapitalState.Failing)
{
//rollback account. if exception is thrown here, it will be retried until success
await _accountsRepository.UpdateAccountTemporaryCapitalAsync(executionInfo.Data.AccountId,
AccountManagementService.UpdateTemporaryCapital,
new InternalModels.TemporaryCapital
{
Id = executionInfo.Data.OperationId,
Amount = executionInfo.Data.Amount,
},
false);
_chaosKitty.Meow($"{nameof(FinishGiveTemporaryCapitalInternalCommand)}: " +
"publisher.PublishEvent: " + c.OperationId);
publisher.PublishEvent(new GiveTemporaryCapitalFailedEvent(c.OperationId,
_systemClock.UtcNow.UtcDateTime, executionInfo.Data.FailReason));
}
}
private async Task Handle(MtCoreFinishAccountsDeletionCommand command, IEventPublisher publisher)
{
var executionInfo = await _operationExecutionInfoRepository.GetAsync <DeleteAccountsOperationData>(
operationName : OperationName,
id : command.OperationId
);
if (executionInfo == null)
{
return;
}
if (executionInfo.Data.SwitchState(OperationState.Started, OperationState.Finished))
{
foreach (var failedAccountId in command.FailedAccountIds)
{
try
{
var account = _accountsCacheService.Get(failedAccountId);
await UpdateAccount(account, false, r => { }, command.Timestamp);
}
catch (Exception exception)
{
await _log.WriteErrorAsync(nameof(DeleteAccountsCommandsHandler),
nameof(MtCoreFinishAccountsDeletionCommand), exception);
}
}
foreach (var accountId in command.AccountIds)
{
_accountsCacheService.Remove(accountId);
}
publisher.PublishEvent(new MtCoreDeleteAccountsFinishedEvent(command.OperationId, _dateService.Now()));
_chaosKitty.Meow($"{nameof(MtCoreFinishAccountsDeletionCommand)}: " +
"Save_OperationExecutionInfo: " +
$"{command.OperationId}");
await _operationExecutionInfoRepository.Save(executionInfo);
}
}
private async Task Handle(UnfreezeMarginOnFailWithdrawalCommand command, IEventPublisher publisher)
{
var executionInfo = await _operationExecutionInfoRepository.GetAsync <WithdrawalFreezeOperationData>(
operationName : OperationName,
id : command.OperationId
);
if (executionInfo == null)
{
return;
}
if (executionInfo.Data.SwitchState(OperationState.Started, OperationState.Finished))
{
await _accountUpdateService.UnfreezeWithdrawalMargin(executionInfo.Data.AccountId, command.OperationId);
publisher.PublishEvent(new UnfreezeMarginOnFailSucceededWithdrawalEvent(command.OperationId,
_dateService.Now(), executionInfo.Data.AccountId, executionInfo.Data.Amount));
_chaosKitty.Meow(command.OperationId);
await _operationExecutionInfoRepository.Save(executionInfo);
}
}
public async Task Handle(LiquidationStartedInternalEvent e, ICommandSender sender)
{
var executionInfo = await _operationExecutionInfoRepository.GetAsync <LiquidationOperationData>(
operationName : OperationName,
id : e.OperationId);
if (executionInfo?.Data == null)
{
return;
}
if (executionInfo.Data.SwitchState(LiquidationOperationState.Initiated,
LiquidationOperationState.Started))
{
LiquidatePositionsIfAnyAvailable(e.OperationId, executionInfo.Data, sender);
_chaosKitty.Meow(
$"{nameof(LiquidationStartedInternalEvent)}:" +
$"Save_OperationExecutionInfo:" +
$"{e.OperationId}");
await _operationExecutionInfoRepository.Save(executionInfo);
}
}
public async Task ExecuteAsync(IEventPublisher failurePublisher, string accountId, string operationId, string reason)
{
if (failurePublisher == null)
{
throw new ArgumentNullException(nameof(failurePublisher));
}
if (string.IsNullOrEmpty(accountId))
{
throw new ArgumentNullException(nameof(accountId));
}
if (string.IsNullOrEmpty(operationId))
{
throw new ArgumentNullException(nameof(operationId));
}
var executionInfo = await _operationExecutionInfoRepository.GetAsync <LiquidationOperationData>(
LiquidationSaga.OperationName,
operationId);
if (executionInfo?.Data == null)
{
await _log.WriteWarningAsync(nameof(LiquidationFailureExecutor),
nameof(ExecuteAsync),
new { operationId, accountId }.ToJson(),
$"Unable to execute failure. Liquidation execution info was not found.");
return;
}
_accountUpdateService.RemoveLiquidationStateIfNeeded(accountId, reason, operationId, executionInfo.Data.LiquidationType);
var account = _accountsCache.Get(accountId);
failurePublisher.PublishEvent(new LiquidationFailedEvent
{
OperationId = operationId,
CreationTime = _dateService.Now(),
Reason = reason,
LiquidationType = executionInfo.Data.LiquidationType.ToType <LiquidationTypeContract>(),
AccountId = executionInfo.Data.AccountId,
AssetPairId = executionInfo.Data.AssetPairId,
Direction = executionInfo.Data.Direction?.ToType <PositionDirectionContract>(),
QuoteInfo = executionInfo.Data.QuoteInfo,
ProcessedPositionIds = executionInfo.Data.ProcessedPositionIds,
LiquidatedPositionIds = executionInfo.Data.LiquidatedPositionIds,
OpenPositionsRemainingOnAccount = _ordersCache.Positions.GetPositionsByAccountIds(executionInfo.Data.AccountId).Count,
CurrentTotalCapital = account.GetTotalCapital(),
});
await _log.WriteInfoAsync(nameof(LiquidationFailureExecutor),
nameof(ExecuteAsync),
new { account, operationId, reason }.ToJson(),
$"Successfully published {nameof(LiquidationFailedEvent)} event");
_liquidationEndEventChannel.SendEvent(this, new LiquidationEndEventArgs
{
OperationId = operationId,
CreationTime = _dateService.Now(),
AccountId = executionInfo.Data.AccountId,
LiquidatedPositionIds = executionInfo.Data.LiquidatedPositionIds,
FailReason = reason,
});
await _log.WriteInfoAsync(nameof(LiquidationFailureExecutor),
nameof(ExecuteAsync),
new { account, operationId, reason }.ToJson(),
$"Successfully published {nameof(LiquidationEndEventArgs)} event");
}
private async Task Handle(MarkAccountsAsDeletedInternalCommand command, IEventPublisher publisher)
{
var executionInfo = await _executionInfoRepository.GetAsync <DeleteAccountsData>(OperationName,
command.OperationId);
if (executionInfo == null || executionInfo.Data.State > DeleteAccountsState.MtCoreAccountsBlocked)
{
return;
}
if (executionInfo.Data.State < DeleteAccountsState.MtCoreAccountsBlocked)
{
throw new Exception($"{nameof(MarkAccountsAsDeletedInternalCommand)} have state {executionInfo.Data.State.ToString()}, but [{DeleteAccountsState.MtCoreAccountsBlocked}] was expected. Throwing to retry in {(long) _settings.Cqrs.RetryDelay.TotalMilliseconds}ms.");
}
var validationFailedAccountIds = await ValidateAccountsAsync(executionInfo.Data.AccountIds);
foreach (var accountToDelete in executionInfo.Data.GetAccountsToDelete()
.Except(validationFailedAccountIds.Keys))
{
try
{
var account = await _accountsRepository.DeleteAsync(accountToDelete);
publisher.PublishEvent(
new AccountChangedEvent(
account.ModificationTimestamp,
OperationName,
_convertService.Convert <IAccount, AccountContract>(account),
AccountChangedEventTypeContract.Deleted,
null,
command.OperationId,
null));
}
catch (Exception exception)
{
await _log.WriteErrorAsync(nameof(DeleteAccountsCommandsHandler), nameof(MarkAccountsAsDeletedInternalCommand),
$"OperationId: [{command.OperationId}]", exception);
validationFailedAccountIds.Add(accountToDelete, exception.Message);
}
}
foreach (var failedAccountId in executionInfo.Data.FailedAccountIds.Keys
.Concat(validationFailedAccountIds.Keys))
{
try
{
var account = (await _accountsRepository.GetAsync(failedAccountId))
.RequiredNotNull(nameof(failedAccountId), $"Account {failedAccountId} does not exist.");
var result = await _accountsRepository.UpdateAccountAsync(
failedAccountId,
false,
false);
_eventSender.SendAccountChangedEvent(
nameof(MarkAccountsAsDeletedInternalCommand),
result,
AccountChangedEventTypeContract.Updated,
$"{command.OperationId}_{failedAccountId}",
previousSnapshot: account);
}
catch (Exception exception)
{
await _log.WriteErrorAsync(nameof(DeleteAccountsCommandsHandler),
nameof(MarkAccountsAsDeletedInternalCommand), exception);
}
}
_chaosKitty.Meow($"{nameof(MarkAccountsAsDeletedInternalCommand)}: " +
"AccountsMarkedAsDeletedEvent: " +
$"{command.OperationId}");
publisher.PublishEvent(new AccountsMarkedAsDeletedEvent(
command.OperationId,
_systemClock.UtcNow.UtcDateTime,
validationFailedAccountIds
));
}