public override async Task <Empty> AcceptOperation(OperationIdRequest request, ServerCallContext context)
{
var result = new Empty();
if (string.IsNullOrEmpty(request.Id))
{
return(result);
}
if (!Guid.TryParse(request.Id, out var operationId))
{
return(result);
}
var operation = await _validationRepository.TryGetAsync(operationId);
if (operation == null)
{
_log.Warning("Operation not found", context: request.Id);
return(result);
}
operation.Resolve(OperationValidationResolution.Accepted);
await _validationRepository.SaveAsync(operation);
_cqrsEngine.PublishEvent(new OperationAcceptedEvent {
OperationId = operationId
},
BlockchainRiskControlBoundedContext.Name);
return(result);
}
public async Task BroadcastTransaction(BroadcastingTransaction transaction, QueueTriggeringContext context)
{
try
{
var signedByClientTr = await _transactionBlobStorage.GetTransaction(transaction.TransactionId, TransactionBlobType.Initial);
var signedByExchangeTr = await _exchangeSignatureApi.SignTransaction(signedByClientTr);
if (!await _settingsRepository.Get(Constants.CanBeBroadcastedSetting, true))
{
await _transactionBlobStorage.AddOrReplaceTransaction(transaction.TransactionId, TransactionBlobType.Signed, signedByExchangeTr);
context.MoveMessageToPoison(transaction.ToJson());
return;
}
var tr = new Transaction(signedByExchangeTr);
await _broadcastService.BroadcastTransaction(transaction.TransactionId, tr);
if (transaction.TransactionCommandType == TransactionCommandType.SegwitTransferToHotwallet)
{
_cqrsEngine.PublishEvent(new CashinCompletedEvent {
OperationId = transaction.TransactionId, TxHash = tr.GetHash().ToString()
}, BitcoinBoundedContext.Name);
}
if (transaction.TransactionCommandType == TransactionCommandType.Transfer)
{
_cqrsEngine.PublishEvent(new CashoutCompletedEvent {
OperationId = transaction.TransactionId, TxHash = tr.GetHash().ToString()
}, BitcoinBoundedContext.Name);
}
}
catch (RPCException e)
{
if (e.Message != transaction.LastError)
{
await _logger.WriteWarningAsync("BroadcastingTransactionFunction", "BroadcastTransaction", $"Id: [{transaction.TransactionId}]", $"Message: {e.Message} Code:{e.RPCCode} CodeMessage:{e.RPCCodeMessage}");
}
transaction.LastError = e.Message;
var unacceptableTx = _unacceptableTxErrors.Any(o => e.Message.Contains(o));
if (transaction.DequeueCount >= _settings.MaxDequeueCount || unacceptableTx)
{
context.MoveMessageToPoison();
}
else
{
transaction.DequeueCount++;
context.MoveMessageToEnd(transaction.ToJson());
context.SetCountQueueBasedDelay(_settings.MaxQueueDelay, 200);
}
}
}
public async Task <IActionResult> EmailRequestAsync(EmailRequestCommand cmd)
{
var swiftCredentials = await _swiftCredentialsService.GetAsync(cmd.RegulationId, cmd.AssetId);
var pd = await _personalDataService.GetAsync(cmd.ClientId);
var asset = await _assetsService.TryGetAssetAsync(cmd.AssetId);
_cqrsEngine.PublishEvent(new SwiftCredentialsRequestedEvent
{
Email = pd.Email,
ClientName = pd.FullName,
Amount = cmd.Amount,
Year = DateTime.UtcNow.Year.ToString(),
AccountName = swiftCredentials.AccountName,
AccountNumber = swiftCredentials.AccountNumber,
AssetId = swiftCredentials.AssetId,
AssetSymbol = asset.Symbol ?? asset.DisplayId,
BankAddress = swiftCredentials.BankAddress,
Bic = swiftCredentials.BIC,
CompanyAddress = swiftCredentials.CompanyAddress,
CorrespondentAccount = swiftCredentials.CorrespondentAccount,
PurposeOfPayment = await _purposeOfPaymentBuilder.Build(
swiftCredentials.PurposeOfPayment,
cmd.ClientId,
cmd.AssetId),
RegulatorId = swiftCredentials.RegulatorId
}, SwiftCredentialsBoundedContext.Name);
return(Ok());
}
private async Task ProcessMessageAsync(CashInEvent item)
{
if (!_depositCurrencies.Contains(item.CashIn.AssetId, StringComparer.InvariantCultureIgnoreCase))
{
return;
}
_log.Info("CashIn event", context: item.ToJson());
double volume = Convert.ToDouble(item.CashIn.Volume);
(double convertedVolume, string assetId) = await _currencyConverter.ConvertAsync(item.CashIn.AssetId, volume);
if (convertedVolume == 0)
{
return;
}
_cqrsEngine.PublishEvent(new ClientDepositedEvent
{
ClientId = item.CashIn.WalletId,
OperationId = item.Header.MessageId ?? item.Header.RequestId,
Asset = item.CashIn.AssetId,
Amount = volume,
BaseAsset = assetId,
BaseVolume = convertedVolume,
OperationType = "CashIn",
Timestamp = item.Header.Timestamp
}, TierBoundedContext.Name);
}
private static async Task Migrate(IBlockchainWalletsRepository walletRepository, ISigningServiceApi signingServiceApi,
IBlockchainSignFacadeClient blockchainSignFacade, ICqrsEngine cqrs,
IBcnCredentialsRecord bcnCredentialsRecord)
{
var clientId = Guid.Parse(bcnCredentialsRecord.ClientId);
var existingWallet = await walletRepository.TryGetAsync(BlockchainType, clientId);
if (existingWallet != null)
{
return;
}
var address = bcnCredentialsRecord.AssetAddress;
var privateKey = await GetPrivateKey(signingServiceApi, address);
if (privateKey == null)
{
return;
}
await ImportWalletToSignFacade(blockchainSignFacade, privateKey, address);
await walletRepository.AddAsync(BlockchainType, clientId, address, CreatorType.LykkeWallet);
var @event = new WalletCreatedEvent
{
Address = address,
AssetId = AssetId,
BlockchainType = BlockchainType,
IntegrationLayerId = BlockchainType,
ClientId = clientId,
CreatedBy = CreatorType.LykkeWallet
};
cqrs.PublishEvent(@event, BlockchainWalletsBoundedContext.Name);
}
public async Task RemoveOperationAsync(string clientId, string operationId)
{
if (string.IsNullOrWhiteSpace(clientId))
{
throw new ArgumentException(nameof(clientId));
}
if (string.IsNullOrWhiteSpace(operationId))
{
throw new ArgumentException(nameof(operationId));
}
bool removedCashOperation = await _cashOperationsCollector.RemoveClientOperationAsync(clientId, operationId);
bool removedTransferOperation = false;
if (!removedCashOperation)
{
removedTransferOperation = await _cashTransfersCollector.RemoveClientOperationAsync(clientId, operationId);
}
if (removedCashOperation || removedTransferOperation)
{
_cqrsEngine.PublishEvent(
new ClientOperationRemovedEvent
{
ClientId = clientId,
OperationId = operationId
},
LimitationsBoundedContext.Name);
}
}
public async Task SendAssetPairChangedEvent(AssetPairChangedEvent @event)
{
try
{
_cqrsEngine.PublishEvent(@event, _contextNames.SettingsService);
}
catch (Exception ex)
{
await _log.WriteErrorAsync(nameof(CqrsMessageSender), nameof(SendAssetPairChangedEvent), ex);
}
}
public async Task SendEvent <TEvent>(TEvent @event)
{
try
{
_cqrsEngine.PublishEvent(@event, _contextNames.AssetService);
}
catch (Exception ex)
{
await _log.WriteErrorAsync(nameof(CqrsMessageSender), nameof(TEvent), ex);
}
}
private void SendCompleteCashoutEvents(IEnumerable <ICashoutRequest> completedRequests, string txHash)
{
foreach (var cashoutRequest in completedRequests)
{
_cqrsEngine.PublishEvent(new CashoutCompletedEvent()
{
OperationId = cashoutRequest.CashoutRequestId,
TxHash = txHash
}, BitcoinBoundedContext.Name);
}
}
private void UpdateBalances(Lykke.MatchingEngine.Connector.Models.Events.Common.Header header,
List <Lykke.MatchingEngine.Connector.Models.Events.Common.BalanceUpdate> updates)
{
foreach (var wallet in updates)
{
_cqrsEngine.PublishEvent(new BalanceUpdatedEvent
{
WalletId = wallet.WalletId,
AssetId = wallet.AssetId,
Balance = wallet.NewBalance,
Reserved = wallet.NewReserved,
SequenceNumber = header.SequenceNumber
}, "balances");
}
}
private void UpdateBalances(Lykke.MatchingEngine.Connector.Models.Events.Common.Header header,
List <Lykke.MatchingEngine.Connector.Models.Events.Common.BalanceUpdate> updates)
{
foreach (var wallet in updates)
{
_cqrsEngine.PublishEvent(new BalanceUpdatedEvent
{
WalletId = wallet.WalletId,
AssetId = wallet.AssetId,
Balance = wallet.NewBalance,
Reserved = wallet.NewReserved,
OldBalance = wallet.OldBalance,
OldReserved = wallet.OldReserved,
SequenceNumber = header.SequenceNumber,
Timestamp = header.Timestamp
}, BoundedContext.Name);
}
}
public override async Task Execute()
{
var soonest = await _expiryWatcher.GetSoonestAsync(100);
foreach (var entry in soonest)
{
if (entry.IsDue())
{
_cqrsEngine.PublishEvent(
new ClientHistoryExpiredEvent
{
ClientId = entry.ClientId,
Id = entry.RequestId
},
HistoryExportBuilderBoundedContext.Name);
await _expiryWatcher.RemoveAsync(entry);
}
}
}
private async Task ProcessMessageAsync(CashTransferEvent item)
{
if (!_depositCurrencies.Contains(item.CashTransfer.AssetId, StringComparer.InvariantCultureIgnoreCase))
{
return;
}
var transfer = item.CashTransfer;
if (await IsTransferBetweenClientWalletsAsync(transfer.FromWalletId, transfer.ToWalletId))
{
_log.Info("Skip transfer between client wallets", context: item.ToJson());
return;
}
double volume = Convert.ToDouble(transfer.Volume);
(double convertedVolume, string assetId) = await _currencyConverter.ConvertAsync(transfer.AssetId, volume);
if (convertedVolume == 0)
{
return;
}
string operationType = GetOperationType(item.CashTransfer);
_cqrsEngine.PublishEvent(new ClientDepositedEvent
{
ClientId = transfer.ToWalletId,
FromClientId = transfer.FromWalletId,
OperationId = item.Header.MessageId ?? item.Header.RequestId,
Asset = transfer.AssetId,
Amount = volume,
BaseAsset = assetId,
BaseVolume = convertedVolume,
OperationType = operationType,
Timestamp = item.Header.Timestamp
}, TierBoundedContext.Name);
}
public async Task <WalletWithAddressExtensionDto> CreateWalletAsync(string blockchainType, string assetId,
Guid clientId)
{
string address;
if (blockchainType != SpecialBlockchainTypes.FirstGenerationBlockchain)
{
var wallet = await _blockchainSignFacadeClient.CreateWalletAsync(blockchainType);
address = wallet.PublicAddress;
var isAddressMappingRequired = _blockchainExtensionsService.IsAddressMappingRequired(blockchainType);
var underlyingAddress = await _addressService.GetUnderlyingAddressAsync(blockchainType, address);
if (isAddressMappingRequired.HasValue && isAddressMappingRequired.Value && underlyingAddress == null)
{
throw new ArgumentException(
$"Failed to get UnderlyingAddress for blockchainType={blockchainType} and address={address}");
}
await _walletRepository.AddAsync(blockchainType, clientId, address, CreatorType.LykkeWallet);
var @event = new WalletCreatedEvent
{
Address = address,
AssetId = assetId,
BlockchainType = blockchainType,
IntegrationLayerId = blockchainType,
ClientId = clientId
};
await _firstGenerationBlockchainWalletRepository.InsertOrReplaceAsync(new BcnCredentialsRecord
{
Address = string.Empty,
AssetAddress = address,
ClientId = clientId.ToString(),
EncodedKey = string.Empty,
PublicKey = string.Empty,
AssetId = $"{blockchainType} ({assetId})"
});
_cqrsEngine.PublishEvent
(
@event,
BlockchainWalletsBoundedContext.Name
);
return(ConvertWalletToWalletWithAddressExtension
(
new WalletDto
{
Address = isAddressMappingRequired.HasValue && isAddressMappingRequired.Value ? underlyingAddress : address,
AssetId = assetId,
BlockchainType = blockchainType,
ClientId = clientId,
CreatorType = CreatorType.LykkeWallet
}
));
}
address = await _legacyWalletService.CreateWalletAsync(clientId, assetId);
return(new WalletWithAddressExtensionDto()
{
Address = address,
AssetId = assetId,
BlockchainType = blockchainType,
ClientId = clientId,
BaseAddress = address
});
}
private Task ProcessMessageAsync(CashInEvent message)
{
var fees = message.CashIn.Fees;
var fee = fees?.FirstOrDefault()?.Transfer;
var @event = new CashInProcessedEvent
{
OperationId = Guid.Parse(message.Header.RequestId),
RequestId = Guid.Parse(message.Header.MessageId),
WalletId = Guid.Parse(message.CashIn.WalletId),
Volume = decimal.Parse(message.CashIn.Volume),
AssetId = message.CashIn.AssetId,
Timestamp = message.Header.Timestamp,
FeeSize = ParseNullabe(fee?.Volume)
};
_cqrsEngine.PublishEvent(@event, BoundedContext.Name);
if (fees != null)
{
var feeEvent = new FeeChargedEvent
{
OperationId = message.Header.MessageId,
OperationType = FeeOperationType.CashInOut,
Fee = fees.Where(x => x.Transfer != null).Select(x => x.Transfer).ToJson()
};
_cqrsEngine.PublishEvent(feeEvent, BoundedContext.Name);
}
return(Task.CompletedTask);
}
private async Task ProcessDepositsAsync()
{
_cancellationTokenSource = new CancellationTokenSource();
while (!_cancellationTokenSource.IsCancellationRequested)
{
_lastCursor = await _lastCursorRepository.GetAsync(_brokerAccountId);
var assets = await _assetsService.GetAllAssetsAsync(false, _cancellationTokenSource.Token);
try
{
var request = new DepositUpdateSearchRequest
{
BrokerAccountId = _brokerAccountId,
Cursor = _lastCursor
};
request.State.Add(DepositState.Confirmed);
_log.Info("Getting updates...", context: $"request: {request.ToJson()}");
var updates = _apiClient.Deposits.GetUpdates(request);
while (await updates.ResponseStream.MoveNext(_cancellationTokenSource.Token).ConfigureAwait(false))
{
DepositUpdateArrayResponse update = updates.ResponseStream.Current;
foreach (var item in update.Items)
{
if (item.DepositUpdateId <= _lastCursor)
{
continue;
}
if (string.IsNullOrWhiteSpace(item.UserNativeId))
{
_log.Warning("UserNativeId is empty");
continue;
}
_log.Info("Deposit detected", context: $"deposit: {item.ToJson()}");
Guid operationId = await _operationIdsRepository.GetOperationIdAsync(item.DepositId);
string assetId = assets.FirstOrDefault(x => x.SiriusAssetId == item.AssetId)?.Id;
if (string.IsNullOrEmpty(assetId))
{
_log.Warning("Lykke asset not found", context: new { siriusAssetId = item.AssetId, depositId = item.DepositId });
continue;
}
var cashInResult = await _meClient.CashInOutAsync
(
id : operationId.ToString(),
clientId : item.ReferenceId ?? item.UserNativeId,
assetId : assetId,
amount : double.Parse(item.Amount.Value)
);
if (cashInResult == null)
{
throw new InvalidOperationException("ME response is null, don't know what to do");
}
switch (cashInResult.Status)
{
case MeStatusCodes.Ok:
case MeStatusCodes.Duplicate:
if (cashInResult.Status == MeStatusCodes.Duplicate)
{
_log.Info(message: "Deduplicated by the ME", context: new { operationId, depositId = item.DepositId });
}
_log.Info("Deposit processed", context: new { cashInResult.TransactionId });
_cqrsEngine.PublishEvent(new CashinCompletedEvent
{
ClientId = item.UserNativeId,
AssetId = assetId,
Amount = decimal.Parse(item.Amount.Value),
OperationId = operationId,
TransactionHash = item.TransactionInfo.TransactionId,
WalletId = item.ReferenceId == item.UserNativeId ? default(string) : item.ReferenceId,
}, SiriusDepositsDetectorBoundedContext.Name);
await _lastCursorRepository.AddAsync(_brokerAccountId, item.DepositUpdateId);
_lastCursor = item.DepositUpdateId;
break;
case MeStatusCodes.Runtime:
throw new Exception($"Cashin into the ME is failed. ME status: {cashInResult.Status}, ME message: {cashInResult.Message}");
default:
_log.Warning($"Unexpected response from ME. Status: {cashInResult.Status}, ME message: {cashInResult.Message}", context: operationId.ToString());
break;
}
}
}
_log.Info("End of stream");
}
catch (RpcException ex)
{
if (ex.StatusCode == StatusCode.ResourceExhausted)
{
_log.Warning($"Rate limit has been reached. Waiting 1 minute...", ex);
await Task.Delay(60000);
}
else
{
_log.Warning($"RpcException. {ex.Status}; {ex.StatusCode}", ex);
}
}
catch (Exception ex)
{
_log.Error(ex);
}
await Task.Delay(5000);
}
}
internal async Task <bool> AddDataItemAsync(T item)
{
var now = DateTime.UtcNow;
var ttl = item.DateTime.AddMonths(1).Subtract(now);
if (ttl.Ticks <= 0)
{
return(true);
}
if (!item.OperationType.HasValue)
{
item.OperationType = _opTypeResolver(item);
}
await _lock.WaitAsync();
try
{
var(clientData, _) = await FetchClientDataAsync(item.ClientId, item.OperationType);
if (clientData.Any(i => i.Id == item.Id))
{
return(false);
}
clientData.Add(item);
string clientKey = string.Format(ClientSetKeyPattern, _instanceName, _cacheType, item.ClientId);
string operationSuffix = string.Format(OpKeySuffixPattern, item.OperationType.Value, item.Id);
var operationKey = $"{clientKey}:{operationSuffix}";
var tx = _db.CreateTransaction();
var tasks = new List <Task>
{
tx.SortedSetAddAsync(clientKey, operationSuffix, DateTime.UtcNow.Ticks),
_stateRepository.SaveClientStateAsync($"{item.ClientId}-{item.OperationType.Value}", clientData),
};
var setKeyTask = tx.StringSetAsync(operationKey, item.ToJson(), ttl);
tasks.Add(setKeyTask);
if (!await tx.ExecuteAsync())
{
throw new InvalidOperationException($"Error during operations update for client {item.ClientId} with operation type {item.OperationType}");
}
await Task.WhenAll(tasks);
if (!setKeyTask.Result)
{
throw new InvalidOperationException($"Error during operations update for client {item.ClientId} with operation type {item.OperationType}");
}
switch (item.OperationType)
{
case CurrencyOperationType.CardCashIn:
case CurrencyOperationType.CryptoCashIn:
case CurrencyOperationType.SwiftTransfer:
_cqrsEngine.PublishEvent(new ClientDepositEvent
{
ClientId = item.ClientId,
OperationId = item.Id,
Asset = item.Asset,
Amount = item.Volume,
BaseAsset = item.BaseAsset,
BaseVolume = item.BaseVolume,
OperationType = item.OperationType.ToString(),
Timestamp = item.DateTime
}, LimitationsBoundedContext.Name);
break;
case CurrencyOperationType.CardCashOut:
case CurrencyOperationType.CryptoCashOut:
case CurrencyOperationType.SwiftTransferOut:
_cqrsEngine.PublishEvent(new ClientWithdrawEvent
{
ClientId = item.ClientId,
OperationId = item.Id,
Asset = item.Asset,
Amount = item.Volume,
BaseAsset = item.BaseAsset,
BaseVolume = item.BaseVolume,
OperationType = item.OperationType.ToString(),
Timestamp = item.DateTime
}, LimitationsBoundedContext.Name);
break;
}
}
finally
{
_lock.Release();
}
return(true);
}
private async Task ProcessMessageAsync(CashInEvent message)
{
var fees = message.CashIn.Fees;
var fee = fees?.FirstOrDefault()?.Transfer;
var @event = new CashInProcessedEvent
{
OperationId = Guid.Parse(message.Header.RequestId),
RequestId = Guid.Parse(message.Header.MessageId),
WalletId = Guid.Parse(message.CashIn.WalletId),
Volume = decimal.Parse(message.CashIn.Volume),
AssetId = message.CashIn.AssetId,
Timestamp = message.Header.Timestamp,
FeeSize = ParseNullabe(fee?.Volume)
};
_cqrsEngine.PublishEvent(@event, BoundedContext.Name);
if (fees != null)
{
var feeEvent = new FeeChargedEvent
{
OperationId = message.Header.MessageId,
OperationType = FeeOperationType.CashInOut,
Fee = fees.Where(x => x.Transfer != null).Select(x => x.Transfer).ToJson()
};
_cqrsEngine.PublishEvent(feeEvent, BoundedContext.Name);
}
await _lykkeMailerliteClient.Customers.UpdateDepositAsync(new UpdateCustomerDepositRequest
{
CustomerId = message.CashIn.WalletId, RequestId = Guid.NewGuid().ToString(), Timestamp = message.Header.Timestamp.ToTimestamp()
});
}
public async Task ProcessMessage(TransactionQueueMessage message, QueueTriggeringContext context)
{
CreateTransactionResponse transactionResponse;
try
{
var request = await _signRequestRepository.GetSignRequest(message.TransactionId);
if (request?.Invalidated == true)
{
context.MoveMessageToPoison(message.ToJson());
return;
}
switch (message.Type)
{
case TransactionCommandType.Issue:
var issue = message.Command.DeserializeJson <IssueCommand>();
transactionResponse = await _lykkeTransactionBuilderService.GetIssueTransaction(
OpenAssetsHelper.ParseAddress(issue.Address),
issue.Amount, await _assetRepository.GetAssetById(issue.Asset), message.TransactionId);
break;
case TransactionCommandType.Transfer:
var transfer = message.Command.DeserializeJson <TransferCommand>();
transactionResponse = await _lykkeTransactionBuilderService.GetTransferTransaction(
OpenAssetsHelper.ParseAddress(transfer.SourceAddress),
OpenAssetsHelper.ParseAddress(transfer.DestinationAddress), transfer.Amount,
await _assetRepository.GetAssetById(transfer.Asset), message.TransactionId);
break;
case TransactionCommandType.TransferAll:
var transferAll = message.Command.DeserializeJson <TransferAllCommand>();
transactionResponse = await _lykkeTransactionBuilderService.GetTransferAllTransaction(
OpenAssetsHelper.ParseAddress(transferAll.SourceAddress),
OpenAssetsHelper.ParseAddress(transferAll.DestinationAddress),
message.TransactionId);
break;
case TransactionCommandType.Swap:
var swap = message.Command.DeserializeJson <SwapCommand>();
transactionResponse = await _lykkeTransactionBuilderService.GetSwapTransaction(
OpenAssetsHelper.ParseAddress(swap.MultisigCustomer1),
swap.Amount1,
await _assetRepository.GetAssetById(swap.Asset1),
OpenAssetsHelper.ParseAddress(swap.MultisigCustomer2),
swap.Amount2,
await _assetRepository.GetAssetById(swap.Asset2),
message.TransactionId);
break;
case TransactionCommandType.Destroy:
var destroy = message.Command.DeserializeJson <DestroyCommand>();
transactionResponse = await _lykkeTransactionBuilderService.GetDestroyTransaction(
OpenAssetsHelper.ParseAddress(destroy.Address),
destroy.Amount,
await _assetRepository.GetAssetById(destroy.Asset),
message.TransactionId);
break;
case TransactionCommandType.SegwitTransferToHotwallet:
var segwitTransfer = message.Command.DeserializeJson <SegwitTransferCommand>();
transactionResponse = await _lykkeTransactionBuilderService.GetTransferFromSegwitWallet(
OpenAssetsHelper.ParseAddress(segwitTransfer.SourceAddress), message.TransactionId);
break;
default:
throw new ArgumentOutOfRangeException();
}
}
catch (BackendException e) when(e.Code == ErrorCode.NoCoinsFound)
{
if (message.Type == TransactionCommandType.SegwitTransferToHotwallet)
{
_cqrsEngine.PublishEvent(new CashinCompletedEvent {
OperationId = message.TransactionId
}, BitcoinBoundedContext.Name);
}
return;
}
catch (BackendException e)
{
if (e.Text != message.LastError)
{
await _logger.WriteWarningAsync("TransactionBuildFunction", "ProcessMessage", $"Id: [{message.TransactionId}], cmd: [{message.Command}]", e.Text);
}
message.LastError = e.Text;
if (message.DequeueCount >= _settings.MaxDequeueCount)
{
context.MoveMessageToPoison(message.ToJson());
}
else
{
message.DequeueCount++;
context.MoveMessageToEnd(message.ToJson());
context.SetCountQueueBasedDelay(_settings.MaxQueueDelay, 200);
}
return;
}
await _transactionBlobStorage.AddOrReplaceTransaction(message.TransactionId, TransactionBlobType.Initial, transactionResponse.Transaction);
await _queueFactory(Constants.BroadcastingQueue).PutRawMessageAsync(new BroadcastingTransaction
{
TransactionCommandType = message.Type,
TransactionId = message.TransactionId
}.ToJson());
}
private async Task ProcessLimitOrdersAsync(LimitOrdersMessage message)
{
if (message.Orders == null || !message.Orders.Any())
{
return;
}
HashSet <string> limitOrderIds = message.Orders
.Select(o => o.Order.Id)
.ToHashSet();
foreach (var orderMessage in message.Orders)
{
if (orderMessage.Trades == null || !orderMessage.Trades.Any())
{
continue;
}
string assetPairId = orderMessage.Order.AssetPairId;
AssetPair assetPair = _assetPairsRepository.TryGet(assetPairId);
if (assetPair == null)
{
_log.Error($"Asset pair {assetPairId} not found");
continue;
}
List <LimitOrdersMessage.Trade> allTrades = message.Orders.SelectMany(x => x.Trades).ToList();
string marketDataKey = RedisService.GetMarketDataKey(assetPairId);
string baseVolumeKey = RedisService.GetMarketDataBaseVolumeKey(assetPairId);
string quoteVolumeKey = RedisService.GetMarketDataQuoteVolumeKey(assetPairId);
string priceKey = RedisService.GetMarketDataPriceKey(assetPairId);
foreach (var tradeMessage in orderMessage.Trades.OrderBy(t => t.Timestamp).ThenBy(t => t.Index))
{
long maxIndex = allTrades
.Where(x => x.OppositeOrderId == tradeMessage.OppositeOrderId)
.Max(t => t.Index);
var price = (decimal)tradeMessage.Price;
string priceString = price.ToString(CultureInfo.InvariantCulture);
var nowDate = tradeMessage.Timestamp;
var nowTradeDate = nowDate.AddMilliseconds(tradeMessage.Index);
await Task.WhenAll(
_database.HashSetAsync(marketDataKey, nameof(MarketSlice.LastPrice), priceString),
_database.SortedSetAddAsync(priceKey, RedisExtensions.SerializeWithTimestamp(priceString, nowTradeDate), nowTradeDate.ToUnixTime())
);
var isOppositeOrderIsLimit = limitOrderIds.Contains(tradeMessage.OppositeOrderId);
// If opposite order is market order, then unconditionally takes the given limit order.
// But if both of orders are limit orders, we should take only one of them.
if (isOppositeOrderIsLimit)
{
var isBuyOrder = orderMessage.Order.Volume > 0;
// Takes trade only for the sell limit orders
if (isBuyOrder)
{
continue;
}
}
decimal baseVolume;
decimal quotingVolume;
if (tradeMessage.Asset == assetPair.BaseAssetId)
{
baseVolume = (decimal)tradeMessage.Volume;
quotingVolume = (decimal)tradeMessage.OppositeVolume;
}
else
{
baseVolume = (decimal)tradeMessage.OppositeVolume;
quotingVolume = (decimal)tradeMessage.Volume;
}
if (tradeMessage.Price > 0 && baseVolume > 0 && quotingVolume > 0)
{
double now = nowDate.ToUnixTime();
double from = (nowDate - _marketDataInterval).ToUnixTime();
decimal baseVolumeSum = baseVolume;
decimal quoteVolumeSum = quotingVolume;
decimal priceChange = 0;
decimal highValue = (decimal)tradeMessage.Price;
decimal lowValue = (decimal)tradeMessage.Price;
var tasks = new List <Task>();
var baseVolumesDataTask = _database.SortedSetRangeByScoreAsync(baseVolumeKey, from, now);
var quoteVolumesDataTask = _database.SortedSetRangeByScoreAsync(quoteVolumeKey, from, now);
var priceDataTask = _database.SortedSetRangeByScoreAsync(priceKey, from, now);
await Task.WhenAll(baseVolumesDataTask, quoteVolumesDataTask, priceDataTask);
baseVolumeSum += baseVolumesDataTask.Result
.Where(x => x.HasValue)
.Sum(x => RedisExtensions.DeserializeTimestamped <decimal>(x));
quoteVolumeSum += quoteVolumesDataTask.Result
.Where(x => x.HasValue)
.Sum(x => RedisExtensions.DeserializeTimestamped <decimal>(x));
var currentHigh = priceDataTask.Result.Any(x => x.HasValue)
? priceDataTask.Result
.Where(x => x.HasValue)
.Max(x => RedisExtensions.DeserializeTimestamped <decimal>(x))
: (decimal?)null;
if (currentHigh.HasValue && currentHigh.Value > highValue)
{
highValue = currentHigh.Value;
}
var currentLow = priceDataTask.Result.Any(x => x.HasValue)
? priceDataTask.Result
.Where(x => x.HasValue)
.Min(x => RedisExtensions.DeserializeTimestamped <decimal>(x))
: (decimal?)null;
if (currentLow.HasValue && currentLow.Value < lowValue)
{
lowValue = currentLow.Value;
}
var pricesData = priceDataTask.Result;
if (pricesData.Any() && pricesData[0].HasValue)
{
decimal openPrice = RedisExtensions.DeserializeTimestamped <decimal>(pricesData[0]);
if (openPrice > 0)
{
priceChange = ((decimal)tradeMessage.Price - openPrice) / openPrice;
}
}
tasks.Add(_database.SortedSetAddAsync(baseVolumeKey, RedisExtensions.SerializeWithTimestamp(baseVolume, nowTradeDate), now));
tasks.Add(_database.SortedSetAddAsync(quoteVolumeKey, RedisExtensions.SerializeWithTimestamp(quotingVolume, nowTradeDate), now));
await Task.WhenAll(tasks);
//send event only for the last trade in the order
if (tradeMessage.Index == maxIndex)
{
var evt = new MarketDataChangedEvent
{
AssetPairId = assetPairId,
VolumeBase = baseVolumeSum,
VolumeQuote = quoteVolumeSum,
PriceChange = priceChange,
LastPrice = (decimal)tradeMessage.Price,
High = highValue,
Low = lowValue
};
_cqrsEngine.PublishEvent(evt, MarketDataBoundedContext.Name);
try
{
await _tickerWriter.InsertOrReplaceAsync(new Ticker(assetPairId)
{
VolumeBase = baseVolumeSum,
VolumeQuote = quoteVolumeSum,
PriceChange = priceChange,
LastPrice = (decimal)tradeMessage.Price,
High = highValue,
Low = lowValue,
UpdatedDt = nowTradeDate
});
}
catch (Exception ex)
{
_log.Error(ex, "Error sending ticker to MyNySqlServer", context: evt.ToJson());
}
}
}
}
}
}
private async Task ProcessCashoutsAsync()
{
_cancellationTokenSource = new CancellationTokenSource();
while (!_cancellationTokenSource.IsCancellationRequested)
{
_lastCursor = await _lastCursorRepository.GetAsync(_brokerAccountId);
var assets = await _assetsService.GetAllAssetsAsync(false, _cancellationTokenSource.Token);
var streamId = Guid.NewGuid().ToString();
try
{
var request = new WithdrawalUpdateSearchRequest
{
StreamId = streamId,
BrokerAccountId = _brokerAccountId,
Cursor = _lastCursor
};
_log.Info("Getting updates...", context: new
{
StreamId = request.StreamId,
BrokerAccountId = request.BrokerAccountId,
Cursor = request.Cursor,
});
var updates = _apiClient.Withdrawals.GetUpdates(request);
while (await updates.ResponseStream.MoveNext(_cancellationTokenSource.Token).ConfigureAwait(false))
{
WithdrawalUpdateArrayResponse update = updates.ResponseStream.Current;
if (!update.Items.Any())
{
_log.Warning("Empty collection of update items:", context: new
{
StreamId = request.StreamId,
BrokerAccountId = request.BrokerAccountId,
Cursor = request.Cursor,
});
}
foreach (var item in update.Items)
{
if (item.WithdrawalUpdateId <= _lastCursor)
{
continue;
}
if (string.IsNullOrWhiteSpace(item.Withdrawal.UserNativeId))
{
_log.Warning("UserNativeId is empty", context: new
{
StreamId = request.StreamId,
BrokerAccountId = request.BrokerAccountId,
Cursor = request.Cursor,
});
_log.Warning(message: "Withdrawal update body", context: new
{
Item = item.Withdrawal.ToJson(),
StreamId = request.StreamId,
BrokerAccountId = request.BrokerAccountId,
Cursor = request.Cursor,
});
continue;
}
_log.Info("Withdrawal update", context: new
{
Withdrawal = item.ToJson(),
StreamId = request.StreamId,
BrokerAccountId = request.BrokerAccountId,
Cursor = request.Cursor,
});
Asset asset = assets.FirstOrDefault(x => x.SiriusAssetId == item.Withdrawal.AssetId);
if (asset == null)
{
_log.Warning(
"Lykke asset not found",
context: new
{
siriusAssetId = item.Withdrawal.AssetId,
withdrawalId = item.Withdrawal.Id,
StreamId = request.StreamId,
BrokerAccountId = request.BrokerAccountId,
Cursor = request.Cursor,
});
continue;
}
await _withdrawalLogsRepository.AddAsync(item.Withdrawal.TransferContext.WithdrawalReferenceId, $"Withdrawal update (state: {item.Withdrawal.State.ToString()})",
new
{
siriusWithdrawalId = item.Withdrawal.Id,
clientId = item.Withdrawal.UserNativeId,
walletId = item.Withdrawal.AccountReferenceId == item.Withdrawal.UserNativeId ? item.Withdrawal.UserNativeId : item.Withdrawal.AccountReferenceId,
fees = item.Withdrawal.ActualFees.ToJson(),
item.Withdrawal.State,
TransactionHash = item.Withdrawal.TransactionInfo?.TransactionId
}.ToJson()
);
if (!Guid.TryParse(item.Withdrawal.TransferContext.WithdrawalReferenceId, out var operationId))
{
operationId = Guid.Empty;
}
Guid?walletId = item.Withdrawal.AccountReferenceId == item.Withdrawal.UserNativeId ? null : Guid.Parse(item.Withdrawal.AccountReferenceId);
switch (item.Withdrawal.State)
{
case WithdrawalState.Completed:
_cqrsEngine.PublishEvent(new CashoutCompletedEvent
{
OperationId = operationId,
ClientId = Guid.Parse(item.Withdrawal.UserNativeId),
WalletId = walletId,
AssetId = asset.Id,
Amount = Convert.ToDecimal(item.Withdrawal.Amount.Value),
Address = item.Withdrawal.DestinationDetails.Address,
Tag = item.Withdrawal.DestinationDetails.Tag,
TransactionHash = item.Withdrawal.TransactionInfo?.TransactionId,
Timestamp = item.Withdrawal.UpdatedAt.ToDateTime().ToUniversalTime(),
}, SiriusCashoutProcessorBoundedContext.Name);
await _lastCursorRepository.AddAsync(_brokerAccountId, item.WithdrawalUpdateId);
_lastCursor = item.WithdrawalUpdateId;
break;
case WithdrawalState.Failed:
await _withdrawalLogsRepository.AddAsync(
item.Withdrawal.TransferContext.WithdrawalReferenceId,
"Withdrawal failed, finishing without Refund",
null);
await _lastCursorRepository.AddAsync(_brokerAccountId, item.WithdrawalUpdateId);
_lastCursor = item.WithdrawalUpdateId;
break;
case WithdrawalState.Rejected:
case WithdrawalState.Refunded:
{
await _withdrawalLogsRepository.AddAsync(
item.Withdrawal.TransferContext.WithdrawalReferenceId,
"Withdrawal failed, processing refund in ME",
new { WithdrawalError = item.Withdrawal.Error?.ToJson() }.ToJson());
decimal amount = Convert.ToDecimal(item.Withdrawal.Amount.Value);
var operation = await _operationsClient.Get(operationId);
var operationContext = JsonConvert.DeserializeObject <OperationContext>(operation.ContextJson);
decimal fee = operationContext.Fee.Type == "Absolute"
? operationContext.Fee.Size.TruncateDecimalPlaces(asset.Accuracy, true)
: (amount * operationContext.Fee.Size).TruncateDecimalPlaces(asset.Accuracy, true);
var refund = await _refundsRepository.GetAsync(item.Withdrawal.UserNativeId,
item.Withdrawal.TransferContext.WithdrawalReferenceId) ??
await _refundsRepository.AddAsync(
item.Withdrawal.TransferContext.WithdrawalReferenceId,
item.Withdrawal.UserNativeId,
walletId?.ToString() ?? item.Withdrawal.UserNativeId,
operationContext.GlobalSettings.FeeSettings.TargetClients.Cashout,
asset.Id,
asset.SiriusAssetId,
amount, fee);
if (refund.FeeAmount > 0)
{
var feeReturnResult = await ReturnFeeAsync(refund);
if (feeReturnResult != null && (feeReturnResult.Status == MeStatusCodes.Ok || feeReturnResult.Status == MeStatusCodes.Duplicate))
{
_log.Info("Fee cashed out from fee wallet", new
{
OperationId = refund.FeeOperationId,
StreamId = request.StreamId,
BrokerAccountId = request.BrokerAccountId,
Cursor = request.Cursor,
});
await _withdrawalLogsRepository.AddAsync(refund.Id, "Fee cashed out from fee wallet",
new { refund.FeeOperationId, refund.FeeClientId, refund.FeeAmount, refund.AssetId }.ToJson());
}
else
{
_log.Info("Can't cashout fee from fee wallet", context: new
{
OperationId = refund.FeeOperationId,
StreamId = request.StreamId,
BrokerAccountId = request.BrokerAccountId,
Cursor = request.Cursor,
});
await _withdrawalLogsRepository.AddAsync(refund.Id, "Can't cashout fee from fee wallet",
new { refund.FeeOperationId, refund.FeeClientId, refund.FeeAmount, refund.AssetId,
error = feeReturnResult == null
? "response from ME is null"
: $"{feeReturnResult.Status}: {feeReturnResult.Message}" }.ToJson());
}
}
var result = await RefundAsync(refund);
if (result != null && (result.Status == MeStatusCodes.Ok || result.Status == MeStatusCodes.Duplicate))
{
_log.Info("Refund processed", context: new
{
OperationId = refund.OperationId,
StreamId = request.StreamId,
BrokerAccountId = request.BrokerAccountId,
Cursor = request.Cursor,
});
await _refundsRepository.UpdateAsync(refund.ClientId, refund.Id, WithdrawalState.Completed.ToString());
await _withdrawalLogsRepository.AddAsync(refund.Id, "Refund processed in ME",
new { result.Status, refund.OperationId }.ToJson());
_cqrsEngine.PublishEvent(new CashoutFailedEvent
{
OperationId = refund.Id,
RefundId = refund.OperationId,
Status = MeStatusCodes.Ok.ToString()
}, SiriusCashoutProcessorBoundedContext.Name);
await _lastCursorRepository.AddAsync(_brokerAccountId, item.WithdrawalUpdateId);
_lastCursor = item.WithdrawalUpdateId;
}
else
{
_log.Info("Refund failed", context: new
{
OperationId = refund.OperationId,
StreamId = request.StreamId,
BrokerAccountId = request.BrokerAccountId,
Cursor = request.Cursor,
});
await _refundsRepository.UpdateAsync(refund.ClientId, refund.Id, WithdrawalState.Failed.ToString());
await _withdrawalLogsRepository.AddAsync(refund.Id, "Refund in ME failed",
new
{
Error = result == null
? "response from ME is null"
: $"{result.Status}: {result.Message}",
OperationId = refund.Id
}.ToJson());
_cqrsEngine.PublishEvent(new CashoutFailedEvent
{
OperationId = refund.Id,
RefundId = refund.OperationId,
Status = result?.Status.ToString(),
Error = result == null
? "response from ME is null"
: result.Message
}, SiriusCashoutProcessorBoundedContext.Name);
await _lastCursorRepository.AddAsync(_brokerAccountId, item.WithdrawalUpdateId);
_lastCursor = item.WithdrawalUpdateId;
}
break;
}
}
}
}
_log.Info("End of stream", context: new { request.StreamId });
}
catch (RpcException ex)
{
if (ex.StatusCode == StatusCode.ResourceExhausted)
{
_log.Warning($"Rate limit has been reached. Waiting 1 minute...", ex, context: new
{
StreamId = streamId
});
await Task.Delay(60000);
}
else
{
_log.Warning($"RpcException. {ex.Status}; {ex.StatusCode}", ex, context: new
{
StreamId = streamId
});
}
}
catch (Exception ex)
{
_log.Error(ex);
}
await Task.Delay(5000);
}
}