private async Task <MeResponseModel> RefundAsync(IRefund refund)
{
var policy = Policy
.Handle <TaskCanceledException>(exception =>
{
_log.Warning($"Retry on TaskCanceledException", context: $"clientId = {refund.ClientId}, assetId = {refund.AssetId}, amount = {refund.Amount + refund.FeeAmount}");
return(true);
})
.OrResult <MeResponseModel>(r =>
{
_log.Warning($"Response from ME: {(r == null ? "null" : r.ToJson())}");
_withdrawalLogsRepository.AddAsync(refund.Id, "Response from ME",
new { refund.ClientId, refund.OperationId, refund.AssetId, refund.Amount, refund.FeeAmount, meResponse = r == null ? "null" : r.ToJson() }.ToJson()).GetAwaiter().GetResult();
return(r == null || r.Status == MeStatusCodes.Runtime);
})
.WaitAndRetryAsync(5, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)));
var result = await policy.ExecuteAsync(async() =>
{
await _withdrawalLogsRepository.AddAsync(refund.Id, "Send refund to ME",
new { refund.ClientId, refund.OperationId, refund.AssetId, refund.Amount }.ToJson()
);
var res = await _meClient.CashInOutAsync(refund.OperationId,
refund.WalletId ?? refund.ClientId,
refund.AssetId,
Convert.ToDouble(refund.Amount + refund.FeeAmount)
);
return(res);
});
return(result);
}
private async Task TransferAsync(
IReadOnlyList <BalanceEntity> balances)
{
for (var i = 0; i < balances.Count; i++)
{
var balance = balances[i];
try
{
var cashInId = await _migrationRepository.GetOrCreateCashInOutIdAsync(balance.ClientId, _options.TargetAssetId);
var cashOutId = await _migrationRepository.GetOrCreateCashInOutIdAsync(balance.ClientId, _options.SourceAssetId);
var cashInAmount = ((double)balance.Balance * _options.Multiplier).TruncateDecimalPlaces((int)_options.TargetAssetAccuracy);
var cashOutAmount = (double)balance.Balance * -1;
var cashInResult = await _meClient.CashInOutAsync
(
id : cashInId.ToString(),
clientId : balance.ClientId,
assetId : _options.TargetAssetId,
amount : cashInAmount
);
if (cashInResult.Status != MeStatusCodes.Ok && cashInResult.Status != MeStatusCodes.Duplicate)
{
_log.Warning($"CashIn [{cashInAmount}] for client [{balance.ClientId}] completed with [{cashInResult.Status.ToString()}] status.");
}
var cashOutResult = await _meClient.CashInOutAsync
(
id : cashOutId.ToString(),
clientId : balance.ClientId,
assetId : _options.SourceAssetId,
amount : cashOutAmount
);
if (cashOutResult.Status != MeStatusCodes.Ok || cashOutResult.Status != MeStatusCodes.Duplicate)
{
_log.Warning($"CashOut [{cashOutAmount}] completed for client {balance.ClientId} with [{cashOutResult.Status.ToString()}] status.");
}
_log.Info($"Completed {i + 1} of {balances.Count} transfers.");
}
catch (Exception e)
{
_log.Error(e, $"Failed to transfer balance [{balance.Balance}] for client [{balance.ClientId}].");
}
}
}
private async Task BurnAsync(
IReadOnlyList <BalanceEntity> balances)
{
for (var i = 0; i < balances.Count; i++)
{
var balance = balances[i];
try
{
var burnAmount = ((double)balance.Balance).TruncateDecimalPlaces((int)_options.AssetAccuracy) * -1;
var burnResult = await _meClient.CashInOutAsync
(
id : Guid.NewGuid().ToString(),
clientId : balance.ClientId,
assetId : _options.AssetId,
amount : burnAmount
);
if (burnResult.Status != MeStatusCodes.Ok)
{
_log.Warning($"CashOut [{burnAmount}] for client [{balance.ClientId}] completed with [{burnResult.Status.ToString()}] status.");
}
_log.Info($"Burned {i + 1} of {balances.Count} balances.");
}
catch (Exception e)
{
_log.Error(e, $"Failed to copy balance [{balance.Balance}] for client [{balance.ClientId}].");
}
}
}
public async Task <CommandHandlingResult> Handle(ProcessCashInCommand command, IEventPublisher eventPublisher)
{
var id = command.CommandId;
var asset = command.Asset;
var amount = command.Amount;
var transaction = command.Transaction;
ChaosKitty.Meow();
var responseModel = await _matchingEngineClient.CashInOutAsync(id, transaction.ClientId, asset.Id, amount.TruncateDecimalPlaces(asset.Accuracy));
if (responseModel.Status != MeStatusCodes.Ok && responseModel.Status != MeStatusCodes.AlreadyProcessed && responseModel.Status != MeStatusCodes.Duplicate)
{
_log.WriteInfo(nameof(ProcessCashInCommand), command, responseModel.ToJson());
throw new ProcessingException(responseModel.ToJson());
}
ChaosKitty.Meow();
eventPublisher.PublishEvent(new TransactionProcessedEvent {
ClientId = command.Transaction.ClientId, Asset = command.Asset, Amount = command.Amount
});
return(CommandHandlingResult.Ok());
}
public async Task HandleCashInOperation(IAsset asset, double amount, string clientId, string clientAddress, string hash)
{
var id = Guid.NewGuid().ToString("N");
var pt = await _paymentTransactionsRepository.TryCreateAsync(PaymentTransaction.Create(hash,
CashInPaymentSystem.Ethereum, clientId, amount,
asset.DisplayId ?? asset.Id, status: PaymentStatus.Processing));
if (pt == null)
{
await
_log.WriteWarningAsync(nameof(EthereumEventsQueue), nameof(HandleCashInOperation), hash,
"Transaction already handled");
//return if was handled previously
return;
}
var result = await _matchingEngineClient.CashInOutAsync(id, clientId, asset.Id, amount);
if (result == null || result.Status != MeStatusCodes.Ok)
{
await
_log.WriteWarningAsync(nameof(EthereumEventsQueue), nameof(HandleCashInOperation), "ME error",
result.ToJson());
}
else
{
var walletCreds = await _walletCredentialsRepository.GetAsync(clientId);
await _cashOperationsRepository.RegisterAsync(new CashInOutOperation
{
Id = id,
ClientId = clientId,
Multisig = walletCreds.MultiSig,
AssetId = asset.Id,
Amount = amount,
BlockChainHash = hash,
DateTime = DateTime.UtcNow,
AddressTo = clientAddress,
State = TransactionStates.SettledOnchain
});
var clientAcc = await _clientAccountsRepository.GetByIdAsync(clientId);
await _srvEmailsFacade.SendNoRefundDepositDoneMail(clientAcc.Email, amount, asset.Id);
await _paymentTransactionsRepository.SetStatus(hash, PaymentStatus.NotifyProcessed);
}
}
public async Task <CommandHandlingResult> Handle(EnrollEthCashinToMatchingEngineCommand command, IEventPublisher eventPublisher)
{
try
{
var cashinId = command.CashinOperationId.ToString("N");
var clientId = command.ClientId;
var hash = command.TransactionHash;
var amount = command.Amount;
var asset = await _assetsServiceWithCache.TryGetAssetAsync(command.AssetId);
var createPendingActions = command.CreatePendingActions;
var paymentTransaction = PaymentTransaction.Create(hash,
CashInPaymentSystem.Ethereum, clientId.ToString(), (double)amount,
asset.DisplayId ?? asset.Id, status: PaymentStatus.Processing);
var exists = await _paymentTransactionsRepository.CheckExistsAsync(paymentTransaction);
if (exists)
{
_log.Warning(command.TransactionHash ?? "Empty", $"Transaction already handled {hash}",
context: command);
return(CommandHandlingResult.Ok());
}
if (createPendingActions && asset.IsTrusted)
{
await _ethererumPendingActionsRepository.CreateAsync(clientId.ToString(), Guid.NewGuid().ToString());
}
ChaosKitty.Meow();
MeResponseModel result = null;
await ExecuteWithTimeoutHelper.ExecuteWithTimeoutAsync(async() =>
{
result = await _matchingEngineClient.CashInOutAsync(cashinId, clientId.ToString(), asset.Id, (double)amount);
}, 5 * 60 * 1000); // 5 min in ms
if (result == null ||
(result.Status != MeStatusCodes.Ok &&
result.Status != MeStatusCodes.Duplicate))
{
_log.Warning(command.TransactionHash ?? "Empty", "ME error", context: result);
return(CommandHandlingResult.Fail(TimeSpan.FromMinutes(1)));
}
eventPublisher.PublishEvent(new EthCashinEnrolledToMatchingEngineEvent()
{
TransactionHash = hash,
});
ChaosKitty.Meow();
await _paymentTransactionsRepository.TryCreateAsync(paymentTransaction);
return(CommandHandlingResult.Ok());
}
catch (Exception e)
{
_log.Error(nameof(EnrollEthCashinToMatchingEngineCommand), e, context: command);
throw;
}
}
public async Task <CommandHandlingResult> Handle(EnrollToMatchingEngineCommand command, IEventPublisher publisher)
{
// First level deduplication just to reduce traffic to the ME
if (await _deduplicationRepository.IsExistsAsync(command.CashinOperationId))
{
_log.Info("Deduplicated at first level", command);
// Workflow should be continued
publisher.PublishEvent(new CashinEnrolledToMatchingEngineEvent
{
CashoutOperationId = command.CashoutOperationId
});
return(CommandHandlingResult.Ok());
}
var cashInResult = await _meClient.CashInOutAsync(
command.CashinOperationId.ToString(),
command.RecipientClientId.ToString(),
command.AssetId,
(double)command.Amount);
_chaosKitty.Meow(command.CashoutOperationId);
if (cashInResult == null)
{
throw new InvalidOperationException("ME response is null, don't know what to do");
}
switch (cashInResult.Status)
{
case MeStatusCodes.Ok:
case MeStatusCodes.Duplicate:
if (cashInResult.Status == MeStatusCodes.Duplicate)
{
_log.Info("Deduplicated by the ME", command);
}
publisher.PublishEvent(new CashinEnrolledToMatchingEngineEvent
{
CashoutOperationId = command.CashoutOperationId
});
_chaosKitty.Meow(command.CashinOperationId);
await _deduplicationRepository.InsertOrReplaceAsync(command.CashinOperationId);
_chaosKitty.Meow(command.CashinOperationId);
return(CommandHandlingResult.Ok());
case MeStatusCodes.Runtime:
// Retry forever with the default delay + log the error outside.
throw new Exception($"Cashin into the ME is failed. ME status: {cashInResult.Status}, ME message: {cashInResult.Message}");
default:
// Just abort cashout for further manual processing. ME call could not be retried anyway if responce was received.
_log.Warning(
$"Unexpected response from ME. Status: {cashInResult.Status}, ME message: {cashInResult.Message}",
context: command);
return(CommandHandlingResult.Ok());
}
}
public async Task <CommandHandlingResult> Handle(EnrollToMatchingEngineCommand command, IEventPublisher publisher)
{
var clientId = command.ClientId;
var amountDecimal = (decimal)command.MatchingEngineOperationAmount;
var scale = amountDecimal.GetScale();
var amount = command.MatchingEngineOperationAmount.TruncateDecimalPlaces(scale);
if (clientId == null)
{
clientId = await _walletsClient.TryGetClientIdAsync(
command.BlockchainType,
command.DepositWalletAddress);
}
if (clientId == null)
{
throw new InvalidOperationException("Client ID for the blockchain deposit wallet address is not found");
}
// First level deduplication just to reduce traffic to the ME
if (await _deduplicationRepository.IsExistsAsync(command.OperationId))
{
_log.Info(nameof(EnrollToMatchingEngineCommand), "Deduplicated at first level", command.OperationId);
// Workflow should be continued
publisher.PublishEvent(new CashinEnrolledToMatchingEngineEvent
{
ClientId = clientId.Value,
OperationId = command.OperationId
});
return(CommandHandlingResult.Ok());
}
var cashInResult = await _meClient.CashInOutAsync
(
id : command.OperationId.ToString(),
clientId : clientId.Value.ToString(),
assetId : command.AssetId,
amount : amount
);
_chaosKitty.Meow(command.OperationId);
if (cashInResult == null)
{
throw new InvalidOperationException("ME response is null, don't know what to do");
}
switch (cashInResult.Status)
{
case MeStatusCodes.Ok:
case MeStatusCodes.Duplicate:
if (cashInResult.Status == MeStatusCodes.Duplicate)
{
_log.Info(nameof(EnrollToMatchingEngineCommand), "Deduplicated by the ME", command.OperationId);
}
publisher.PublishEvent(new CashinEnrolledToMatchingEngineEvent
{
ClientId = clientId.Value,
OperationId = command.OperationId
});
_chaosKitty.Meow(command.OperationId);
await _deduplicationRepository.InsertOrReplaceAsync(command.OperationId);
_chaosKitty.Meow(command.OperationId);
return(CommandHandlingResult.Ok());
case MeStatusCodes.Runtime:
// Retry forever with the default delay + log the error outside.
throw new Exception($"Cashin into the ME is failed. ME status: {cashInResult.Status}, ME message: {cashInResult.Message}");
default:
// Just abort cashin for futher manual processing. ME call could not be retried anyway if responce was received.
_log.Error(nameof(EnrollToMatchingEngineCommand), null, $"Unexpected response from ME. Status: {cashInResult.Status}, ME message: {cashInResult.Message}", context: command.OperationId);
return(CommandHandlingResult.Ok());
}
}
public async Task ExecutePlanAsync(
Guid planId)
{
_log.Info("Distribution plan execution started...", new
{
planId
});
var distributionPlan = await _distributionPlanRepository.TryGetAsync(planId);
if (distributionPlan != null)
{
_log.Info($"Plan contains {distributionPlan.Amounts} amounts to distribute");
foreach (var distributionAmount in distributionPlan.Amounts)
{
var result = await _matchingEngineClient.CashInOutAsync
(
id : distributionAmount.Id.ToString(),
clientId : distributionAmount.WalletId.ToString(),
assetId : _gasAssetId,
amount : ((double)distributionAmount.Value)
);
// ReSharper disable once SwitchStatementMissingSomeCases
switch (result.Status)
{
case MeStatusCodes.Ok:
_log.Info
(
$"{distributionAmount.Value} of gas has been distributed to {distributionAmount.WalletId}.",
distributionAmount
);
break;
case MeStatusCodes.Duplicate:
_log.Info
(
$"Distribution of {distributionAmount.Value} gas to {distributionAmount.WalletId} has been deduplicated by ME.",
distributionAmount
);
break;
case MeStatusCodes.Runtime:
throw new Exception($"Distribution failed. ME status: {result.Status}. ME message: {result.Message}.");
default:
_log.Warning
(
$"Got unexpected response from ME. ME status: {result.Status}, ME message: {result.Message}.",
context: distributionAmount
);
break;
}
}
}
else
{
throw new InvalidOperationException($"Distribution plan [{planId}] has not been found.");
}
}
private async Task ProcessDepositsAsync()
{
_cancellationTokenSource = new CancellationTokenSource();
while (!_cancellationTokenSource.IsCancellationRequested)
{
_lastCursor = await _lastCursorRepository.GetAsync(_brokerAccountId);
var assets = await _assetsService.GetAllAssetsAsync(false, _cancellationTokenSource.Token);
try
{
var request = new DepositUpdateSearchRequest
{
BrokerAccountId = _brokerAccountId,
Cursor = _lastCursor
};
request.State.Add(DepositState.Confirmed);
_log.Info("Getting updates...", context: $"request: {request.ToJson()}");
var updates = _apiClient.Deposits.GetUpdates(request);
while (await updates.ResponseStream.MoveNext(_cancellationTokenSource.Token).ConfigureAwait(false))
{
DepositUpdateArrayResponse update = updates.ResponseStream.Current;
foreach (var item in update.Items)
{
if (item.DepositUpdateId <= _lastCursor)
{
continue;
}
if (string.IsNullOrWhiteSpace(item.UserNativeId))
{
_log.Warning("UserNativeId is empty");
continue;
}
_log.Info("Deposit detected", context: $"deposit: {item.ToJson()}");
Guid operationId = await _operationIdsRepository.GetOperationIdAsync(item.DepositId);
string assetId = assets.FirstOrDefault(x => x.SiriusAssetId == item.AssetId)?.Id;
if (string.IsNullOrEmpty(assetId))
{
_log.Warning("Lykke asset not found", context: new { siriusAssetId = item.AssetId, depositId = item.DepositId });
continue;
}
var cashInResult = await _meClient.CashInOutAsync
(
id : operationId.ToString(),
clientId : item.ReferenceId ?? item.UserNativeId,
assetId : assetId,
amount : double.Parse(item.Amount.Value)
);
if (cashInResult == null)
{
throw new InvalidOperationException("ME response is null, don't know what to do");
}
switch (cashInResult.Status)
{
case MeStatusCodes.Ok:
case MeStatusCodes.Duplicate:
if (cashInResult.Status == MeStatusCodes.Duplicate)
{
_log.Info(message: "Deduplicated by the ME", context: new { operationId, depositId = item.DepositId });
}
_log.Info("Deposit processed", context: new { cashInResult.TransactionId });
_cqrsEngine.PublishEvent(new CashinCompletedEvent
{
ClientId = item.UserNativeId,
AssetId = assetId,
Amount = decimal.Parse(item.Amount.Value),
OperationId = operationId,
TransactionHash = item.TransactionInfo.TransactionId,
WalletId = item.ReferenceId == item.UserNativeId ? default(string) : item.ReferenceId,
}, SiriusDepositsDetectorBoundedContext.Name);
await _lastCursorRepository.AddAsync(_brokerAccountId, item.DepositUpdateId);
_lastCursor = item.DepositUpdateId;
break;
case MeStatusCodes.Runtime:
throw new Exception($"Cashin into the ME is failed. ME status: {cashInResult.Status}, ME message: {cashInResult.Message}");
default:
_log.Warning($"Unexpected response from ME. Status: {cashInResult.Status}, ME message: {cashInResult.Message}", context: operationId.ToString());
break;
}
}
}
_log.Info("End of stream");
}
catch (RpcException ex)
{
if (ex.StatusCode == StatusCode.ResourceExhausted)
{
_log.Warning($"Rate limit has been reached. Waiting 1 minute...", ex);
await Task.Delay(60000);
}
else
{
_log.Warning($"RpcException. {ex.Status}; {ex.StatusCode}", ex);
}
}
catch (Exception ex)
{
_log.Error(ex);
}
await Task.Delay(5000);
}
}
