private static void MoveFromPoisonToProcessing()
{
try
{
Console.WriteLine("Are you sure?: Y/N");
var input = Console.ReadLine();
if (input.ToLower() != "y")
{
Console.WriteLine("Cancel");
return;
}
Console.WriteLine("Started");
var queueFactory = ServiceProvider.GetService <IQueueFactory>();
var queuePoison = queueFactory.Build(Constants.PendingOperationsQueue + "-poison");
var queue = queueFactory.Build(Constants.PendingOperationsQueue);
var count = queuePoison.Count().Result;
for (int i = 0; i < count; i++)
{
var message = queuePoison.GetRawMessageAsync().Result;
OperationHashMatchMessage newMessage = JsonConvert.DeserializeObject <OperationHashMatchMessage>(message.AsString);
queue.PutRawMessageAsync(message.AsString).Wait();
queuePoison.FinishRawMessageAsync(message);
}
Console.WriteLine("All Processed");
}
catch (Exception e)
{
}
}
public async Task MonitoringOperationJobUnitTest_SkipIfNoBalance()
{
string operationId = Guid.NewGuid().ToString();
OperationHashMatchMessage message = new OperationHashMatchMessage()
{
OperationId = operationId,
PutDateTime = DateTime.UtcNow,
TransactionHash = null
};
#region Arrange Mocks
IPendingOperation pendingOperation = new PendingOperation()
{
Amount = "1000000000000000000",
FromAddress = "fromAddress",
OperationId = operationId,
OperationType = OperationTypes.Transfer,
ToAddress = "toAddress",
CoinAdapterAddress = "coinAdapter",
};
_pendingOperationService.Setup(x => x.GetOperationAsync(pendingOperation.OperationId)).Returns(Task.FromResult(pendingOperation));
_transferContractService.Setup(x => x.GetBalanceOnAdapter(pendingOperation.CoinAdapterAddress, pendingOperation.FromAddress, true))
.Returns(Task.FromResult(new BigInteger(0)));
_coinEventService.Setup(x => x.PublishEvent(It.IsAny <ICoinEvent>(), It.IsAny <bool>())).Returns(Task.FromResult(0)).Verifiable();
#endregion
MonitoringOperationJob job = GetJob();
await job.Execute(message, new Lykke.JobTriggers.Triggers.Bindings.QueueTriggeringContext(DateTimeOffset.UtcNow));
_coinEventService.Verify(x => x.PublishEvent(It.IsAny <ICoinEvent>(), It.IsAny <bool>()), Times.Never);
}
private static void MoveToPendingOperationQueue(string fromQueue)
{
var queueFactory = ServiceProvider.GetService <IQueueFactory>();
var queuePoison = queueFactory.Build(fromQueue);
var queue = queueFactory.Build(Constants.PendingOperationsQueue);
var count = queuePoison.Count().Result;
for (int i = 0; i < count; i++)
{
var message = queuePoison.GetRawMessageAsync().Result;
OperationHashMatchMessage newMessage = JsonConvert.DeserializeObject <OperationHashMatchMessage>(message.AsString);
newMessage.DequeueCount = 0;
queue.PutRawMessageAsync(message.AsString).Wait();
queuePoison.FinishRawMessageAsync(message).Wait();
}
}
public async Task RemoveFromPendingOperationQueue(string opId)
{
var count = await _queue.Count();
for (int i = 0; i < count; i++)
{
var message = await _queue.GetRawMessageAsync();
OperationHashMatchMessage newMessage = JsonConvert.DeserializeObject <OperationHashMatchMessage>(message.AsString);
await _queue.FinishRawMessageAsync(message);
if (newMessage.OperationId?.ToLower() != opId?.ToLower())
{
await _queue.PutRawMessageAsync(message.AsString);
}
}
}
public async Task ProcessOperation(OperationHashMatchMessage opMessage, QueueTriggeringContext context,
Func <Guid, string, string, string, BigInteger, string, Task <string> > transferDelegate)
{
try
{
var operation = await _pendingOperationService.GetOperationAsync(opMessage.OperationId);
if (operation == null)
{
await _coinEventResubmittQueue.PutRawMessageAsync(JsonConvert.SerializeObject(opMessage));
return;
}
if (_hotWalletAddress == operation.FromAddress.ToLower() &&
opMessage.DequeueCount == 0)
{
opMessage.DequeueCount++;
context.MoveMessageToEnd(opMessage.ToJson());
context.SetCountQueueBasedDelay(_settings.MaxQueueDelay, 200);
return;
}
var guid = Guid.Parse(operation.OperationId);
var amount = BigInteger.Parse(operation.Amount);
BigInteger resultAmount;
string transactionHash = null;
CoinEventType?eventType = null;
BigInteger currentBalance = await _transferContractService.GetBalanceOnAdapter(
operation.CoinAdapterAddress,
operation.FromAddress,
checkInPendingBlock : true);
switch (operation.OperationType)
{
case OperationTypes.Cashout:
eventType = CoinEventType.CashoutStarted;
resultAmount = amount;
if (!CheckBalance(currentBalance, resultAmount))
{
break;
}
transactionHash = await _exchangeContractService.CashOut(guid,
operation.CoinAdapterAddress,
operation.FromAddress,
operation.ToAddress, amount, operation.SignFrom);
break;
case OperationTypes.Transfer:
eventType = CoinEventType.TransferStarted;
resultAmount = amount;
if (!CheckBalance(currentBalance, resultAmount))
{
break;
}
transactionHash = await transferDelegate(guid, operation.CoinAdapterAddress,
operation.FromAddress,
operation.ToAddress, amount, operation.SignFrom);
break;
case OperationTypes.TransferWithChange:
eventType = CoinEventType.TransferStarted;
BigInteger change = BigInteger.Parse(operation.Change);
resultAmount = amount - change;
if (!CheckBalance(currentBalance, resultAmount))
{
break;
}
transactionHash = await _exchangeContractService.TransferWithChange(guid, operation.CoinAdapterAddress,
operation.FromAddress,
operation.ToAddress, amount, operation.SignFrom, change, operation.SignTo);
break;
default:
await _log.WriteWarningAsync("MonitoringOperationJob", "Execute", $"Can't find right operation type for {opMessage.OperationId}", "");
break;
}
if (transactionHash != null && eventType != null)
{
await _pendingOperationService.MatchHashToOpId(transactionHash, operation.OperationId);
await _coinEventService.PublishEvent(new CoinEvent(operation.OperationId.ToString(), transactionHash, operation.FromAddress, operation.ToAddress, resultAmount.ToString(), eventType.Value, operation.CoinAdapterAddress));
await _eventTraceRepository.InsertAsync(new EventTrace()
{
Note = $"Operation Processed. Put it in the {Constants.TransactionMonitoringQueue}. With hash {transactionHash}",
OperationId = operation.OperationId,
TraceDate = DateTime.UtcNow
});
return;
}
}
catch (ClientSideException clientSideExc) when(clientSideExc.ExceptionType == ExceptionType.OperationWithIdAlreadyExists)
{
await _coinEventResubmittQueue.PutRawMessageAsync(JsonConvert.SerializeObject(opMessage));
return;
}
catch (RpcClientException exc)
{
await _log.WriteErrorAsync("MonitoringOperationJob", "Execute", "RpcException", exc);
opMessage.LastError = exc.Message;
opMessage.DequeueCount++;
if (opMessage.DequeueCount < 6)
{
context.MoveMessageToEnd(opMessage.ToJson());
context.SetCountQueueBasedDelay(_settings.MaxQueueDelay, 200);
}
else
{
context.MoveMessageToPoison(opMessage.ToJson());
}
return;
}
catch (Exception ex)
{
if (ex.Message != opMessage.LastError)
{
await _log.WriteWarningAsync("MonitoringOperationJob", "Execute", $"OperationId: [{opMessage.OperationId}]", "");
}
opMessage.LastError = ex.Message;
opMessage.DequeueCount++;
context.MoveMessageToPoison(opMessage.ToJson());
await _log.WriteErrorAsync("MonitoringOperationJob", "Execute", "", ex);
return;
}
opMessage.DequeueCount++;
context.MoveMessageToEnd(opMessage.ToJson());
context.SetCountQueueBasedDelay(_settings.MaxQueueDelay, 200);
}
public async Task Execute(OperationHashMatchMessage opMessage, QueueTriggeringContext context)
{
await ProcessOperation(opMessage, context, _exchangeContractService.Transfer);
}
public async Task Execute(OperationHashMatchMessage opMessage, QueueTriggeringContext context)
{
try
{
string operatioId = opMessage.OperationId;
IOperationResubmitt opResubmitCounter = await _operationResubmittRepository.GetAsync(operatioId);
if (opResubmitCounter == null)
{
opResubmitCounter = new OperationResubmitt()
{
OperationId = operatioId,
ResubmittCount = 0
};
}
if (opResubmitCounter.ResubmittCount > 2)
{
await _log.WriteWarningAsync("CoinEventResubmittJob", "Execute", "", $"Message put to poison {opMessage.OperationId}");
context.MoveMessageToPoison(opMessage.ToJson());
return;
}
var historicalMessages = await _pendingOperationService.GetHistoricalAsync(opMessage.OperationId);
if (historicalMessages == null || historicalMessages.Count() == 0)
{
//Process cashin operations
var coinEvent = await _coinEventService.GetCoinEventById(opMessage.OperationId);
if (coinEvent != null &&
await _ethereumTransactionService.IsTransactionExecuted(coinEvent.TransactionHash, Constants.GasForCoinTransaction))
{
await ResubmittTransactionAsync(coinEvent.TransactionHash, operatioId, opResubmitCounter);
return;
}
else
{
context.MoveMessageToPoison(opMessage.ToJson());
await _slackNotifier.ErrorAsync($"Moved message {opMessage.OperationId} to poison: no corresponding coinEvent");
}
}
else
{
//Process transfer/cashout operations
foreach (var match in historicalMessages)
{
if (!string.IsNullOrEmpty(match.TransactionHash) &&
await _ethereumTransactionService.IsTransactionExecuted(match.TransactionHash, Constants.GasForCoinTransaction))
{
var coinEvent = await _coinEventService.GetCoinEventById(match.OperationId);
if (coinEvent != null && coinEvent.TransactionHash.ToLower() == match.TransactionHash.ToLower())
{
await ResubmittTransactionAsync(match.TransactionHash, operatioId, opResubmitCounter);
break;
}
}
}
}
}
catch (Exception ex)
{
if (opMessage.DequeueCount > 100000)
{
context.MoveMessageToPoison(opMessage.ToJson());
await _slackNotifier.ErrorAsync($"Moved message {opMessage.OperationId} to poison: dequeue count is {opMessage.DequeueCount }" +
$" error is {ex.Message}");
return;
}
opMessage.LastError = ex.Message;
opMessage.DequeueCount++;
context.MoveMessageToEnd(opMessage.ToJson());
await _log.WriteErrorAsync("MonitoringOperationJob", "Execute", "", ex);
return;
}
if (opMessage.DequeueCount > 100000)
{
context.MoveMessageToPoison(opMessage.ToJson());
await _slackNotifier.ErrorAsync($"Moved message {opMessage.OperationId} to poison: dequeue count is {opMessage.DequeueCount }");
}
else
{
opMessage.DequeueCount++;
context.MoveMessageToEnd(opMessage.ToJson());
}
}
private void MoveMessageToQueueEnd(OperationHashMatchMessage opMessage, QueueTriggeringContext context)
{
opMessage.DequeueCount++;
context.MoveMessageToEnd(opMessage.ToJson());
context.SetCountQueueBasedDelay(_settings.MaxQueueDelay, 200);
}
