public override Task RenewLockAsync(
IQueueMessage message,
CancellationToken cancellationToken = default(CancellationToken))
{
// TODO Implement automatic `nack`-ing.
throw new NotImplementedException();
}
private void QueueReader()
{
isActive = true;
while (running)
{
IQueueMessage msg = inputQueue.Dequeue() as IQueueMessage;
try
{
if (msg == null)
{
continue;
}
msg.Run();
}
catch (ThreadInterruptedException)
{
}
catch (Exception e)
{
instrumentationProvider.FireCacheFailed(Resources.BackgroundSchedulerProducerConsumerQueueFailure, e);
}
}
isActive = false;
}
public JogadorService(IJogadorRepository repository, IBlobService blobService, IQueueMessage queueService, IJogadorHistoricoRepository historicoRepository)
{
_repository = repository;
_queueService = queueService;
_blobService = blobService;
_historicoRepository = historicoRepository;
}
public PostService(
IPostRepository postRepository, IBlobService blobService, IQueueMessage queueService)
{
_postRepository = postRepository;
_blobService = blobService;
_queueService = queueService;
}
private void SendQueue(IQueueMessage mess, string[] queues, string entityType = "entity empty", int?delay = 1, int tryAttempt = 1)
{
try
{
_queueProvider.PushMessage(mess, queues);
_logger.Info($"Success send entity - {entityType}");
}
catch (Exception exp)
{
if (tryAttempt < 5)
{
_logger.Warning($"Failed send entity {mess.Body} after {tryAttempt} attempt. We will try again. Exception - {exp.Message}");
Thread.Sleep(2000);
// повторить попытку отправления
SendQueue(mess, queues, entityType, delay, ++tryAttempt);
}
else
{
_logger.Error($"Failed send entity {mess.Body} after {tryAttempt} attempt. Lost entity. Exception - {exp.Message}");
}
}
}
private async Task MessageRecieved(IQueueMessage message)
{
using (message)
{
var details = JsonConvert.DeserializeObject <StoreDataDetails>(message.Message);
var location = details.GetLocation();
var data = await _originalStore.LoadData(location).ConfigureAwait(false);
if (data == null)
{
// Need to make sure to soft delete in our backup...
// We don't have metadata about who did it though...
await _backupStore.SoftDelete(location, new UpdateAuditInfo()).ConfigureAwait(false);
}
else
{
// Just save it right back into the backup!
var ct = CancellationToken.None;
await _backupStore.SaveData(location, data.Metadata, data.Metadata.Audit.ToUpdateAuditInfo(), async (s) =>
{
await data.Stream.CopyToAsync(s, ct).ConfigureAwait(false);
return(data.Metadata.ContentLength);
}, ct).ConfigureAwait(false);
data.Stream.Dispose();
}
await message.Complete().ConfigureAwait(false);
}
}
private string FindKey(IQueueMessage message)
{
var details = JsonConvert.DeserializeObject <StoreDataDetails>(message.Message);
var firstPath = details.BasePath.Split(new char[] { '\\', '/' }, StringSplitOptions.RemoveEmptyEntries)[0];
return(details.Container + "_" + firstPath);
}
private async Task MoveToEnd(IQueueMessage message, string newMessageVersion)
{
newMessageVersion = newMessageVersion ?? message.AsString;
await _queueReader.AddMessageAsync(newMessageVersion);
await _queueReader.FinishMessageAsync(message);
}
internal TransStream ExecQuery(IQueueMessage request)
{
if (request == null)
{
throw new ArgumentNullException("QueueManager.ExecQuery request");
}
if (request.Host == null)
{
throw new ArgumentNullException("QueueManager.ExecQuery request.Host is invalid");
}
MQueue Q = Get(request.Host);
if (Q == null)
{
throw new MessageException(MessageState.InvalidMessageHost, "request.HostName not found " + request.Host);
}
switch (request.QCommand)
{
case QueueCmd.QueueProperty:
var res = Q.Property(); //.QueueProperty();
return(new TransStream(res));
case QueueCmd.ReportQueueItems:
var items = Q.QueryItems();
return(new TransStream(items));
default:
throw new NotSupportedException(request.QCommand.ToString());
}
}
internal IQueueItem ExecGet(IQueueMessage request)
{
if (request.Host == null)
{
throw new MessageException(MessageState.InvalidMessageHost, "Invalid message.Host ");
}
MQueue Q = Get(request.Host);
if (Q == null)
{
throw new MessageException(MessageState.InvalidMessageHost, "message.HostName not found " + request.Host);
}
switch (request.QCommand)
{
case QueueCmd.Dequeue:
return(Q.Dequeue());
case QueueCmd.DequeuePriority:
return(Q.Dequeue(request.Priority));
case QueueCmd.Peek:
return(Q.Peek());
case QueueCmd.PeekPriority:
return(Q.Peek(request.Priority));
}
return(null);
}
public QueueWatcherException(string message, IQueueMessage queueMessage, Exception innerExeception)
: base(message, innerExeception)
{
QueueMessage = queueMessage;
this.Source = queueMessage.Body;
}
private async Task <bool> TryInvokeMessageHandlerAsync(IQueueMessage message)
{
try
{
if (_messageHandler != null)
{
await _messageHandler(message, CancellationToken.None);
return(true);
}
}
catch (Exception ex)
{
var exceptionReceivedHandler = _messageHandlerOptions?.ExceptionReceivedHandler;
if (exceptionReceivedHandler != null)
{
var context = new MessageHandlerExceptionContext(ex);
await exceptionReceivedHandler(context);
}
}
return(false);
}
private async Task MessageRecieved(IQueueMessage message)
{
using(message)
{
var details = JsonConvert.DeserializeObject<StoreDataDetails>(message.Message);
var location = details.GetLocation();
var data = await _originalStore.LoadData(location).ConfigureAwait(false);
if (data == null)
{
// Need to make sure to soft delete in our backup...
// We don't have metadata about who did it though...
await _backupStore.SoftDelete(location, new UpdateAuditInfo()).ConfigureAwait(false);
}
else
{
// Just save it right back into the backup!
var ct = CancellationToken.None;
await _backupStore.SaveData(location, data.Metadata, data.Metadata.Audit.ToUpdateAuditInfo(), async (s) =>
{
await data.Stream.CopyToAsync(s, ct).ConfigureAwait(false);
return data.Metadata.ContentLength;
}, ct).ConfigureAwait(false);
data.Stream.Dispose();
}
await message.Complete().ConfigureAwait(false);
}
}
private void QueueReader()
{
isActive = true;
while (running)
{
IQueueMessage msg = inputQueue.Dequeue() as IQueueMessage;
try
{
if (msg == null)
{
continue;
}
msg.Run();
}
catch (ThreadInterruptedException)
{
}
catch (Exception e)
{
CachingServiceInternalFailureEvent.Fire(SR.BackgroundSchedulerProducerConsumerQueueFailure, e);
}
catch
{
CachingServiceInternalFailureEvent.Fire(SR.BackgroundSchedulerProducerConsumerQueueFailure, new Exception(SR.UnknownFailureReason));
}
}
isActive = false;
}
private void ParallelFinallyHandler(
[CanBeNull] HandleMessagesParallelOptions messageOptions,
[CanBeNull] long[] activeMessageSlots,
[CanBeNull] Task keepAliveTask,
[CanBeNull] IQueueMessage currentMessage,
[CanBeNull] CancellationTokenSource messageSpecificCancellationTokenSource)
{
if (Guard.IsAnyNull(messageOptions, activeMessageSlots, messageSpecificCancellationTokenSource))
{
return;
}
Interlocked.Decrement(ref activeMessageSlots[0]);
// Cancel any outstanding jobs due to the faulted operation (the keepalive task should have been cancelled)
if (keepAliveTask != null && !keepAliveTask.IsCompleted)
{
if (currentMessage != null)
{
this.Top.LogAction(LogSeverity.Warning,
"Message processing was faulted; cancelling",
"Queue's '{0}' message '{1}', processing faulted; cancelling related jobs for this message",
this.Name,
currentMessage.Id);
}
messageSpecificCancellationTokenSource.Cancel();
}
}
private Message CreateMessage(IQueueMessage message)
{
return(new Message(message.Body)
{
ContentType = "application/json"
});
}
public TransStream GetQueueReport(IQueueMessage message)
{
MQueue queue = Get(message.Host);
if (queue == null)
{
var ack = new QueueItem()//MessageState.QueueNotFound, "QueueNotFound: " + message.Host, null, message.Host);
{
MessageState = MessageState.QueueNotFound,
Label = "QueueNotFound: " + message.Host,
Host = message.Host
};
Logger.Info("QueueController GetQueueReport QueueNotFound : {0}", message.Host);
return(ack.ToTransStream());
}
var report = queue.GetReport();
string result = null;
if (report != null)
{
result = Nistec.Serialization.JsonSerializer.Serialize(report);
}
var item = new QueueItem()//MessageState.Ok, result, null, message.Host);
{
MessageState = MessageState.Ok,
Label = result,
Host = message.Host
};
item.SetBody(report);
Logger.Info("QueueController GetQueueReport : {0}", result);
return(item.ToTransStream());
}
protected internal override void SerialFinallyHandler(
HandleMessagesSerialOptions messageOptions,
Task keepAliveTask,
IQueueMessage message,
CancellationTokenSource messageSpecificCancellationTokenSource)
{
this.DecoratedQueue.SerialFinallyHandler(messageOptions, keepAliveTask, message, messageSpecificCancellationTokenSource);
}
/// <summary>
/// Initializes a new instance of the
/// <see cref="QueueMessageWrapper" />
/// class.
/// </summary>
/// <param name="queue">The queue the message belongs to.</param>
/// <param name="message">The underlying message.</param>
public QueueMessageWrapper([NotNull] IExtendedQueue queue, [NotNull] IQueueMessage message)
{
Guard.NotNull(queue, "queue");
Guard.NotNull(message, "message");
this.ParentQueue = queue;
this.ActualMessage = message;
}
public QueueAck(MessageState state, IQueueMessage item, string label)
{
this.Identifier = item.Identifier;
this.Label = label;
this.MessageState = state;
this.Creation = DateTime.Now;
this.Host = item.Host;
}
public MessageParser(IQueueMessage queueMessage, IMessageTemplateProvider messageTemplateProvider,
IMessageTemplateParser messageTemplateParser, Site site)
{
_queueMessage = queueMessage;
_messageTemplateProvider = messageTemplateProvider;
_messageTemplateParser = messageTemplateParser;
_site = site;
}
public Task Pop(IQueueMessage queueMessage)
{
return Task.Factory.StartNew(() =>
{
var messageQueue = FindOrCreateQueue(QueuePath);
messageQueue.ReceiveById(queueMessage.Id, MessageQueueTransactionType.Automatic);
});
}
public void Ack(IQueueMessage msg)
{
if (msg == null) throw new ArgumentNullException("msg");
var fsMessafe = (FileSystemQueueMessage)msg;
//hurray, this message is finally acked! delete it!
fsMessafe.MessageFile.Delete();
}
public override Task EnqueueAsync(
IQueueMessage message,
CancellationToken cancellationToken = default(CancellationToken))
{
ThrowIfDisposed();
return(_queue.EnqueueAsync(message));
}
public override Task CompleteAsync(
IQueueMessage message,
CancellationToken cancellationToken = default(CancellationToken))
{
ThrowIfDisposed();
return(Task.CompletedTask);
}
public TaskRequest(IQueueMessage message)
{
var content = message.Content;
var serializer = new JavaScriptSerializer();
ContentData = serializer.Deserialize<Dictionary<string, string>>(content);
TaskName = ContentData["task"];
ActionName = ContentData["action"];
Id = ContentData["id"];
}
public virtual async Task <QueueServiceBatchResult> FetchAndProcessDataAsync(
Func <TData, Task <TResult> > processFunc,
long totalCount)
{
var batchResult = new QueueServiceBatchResult();
foreach (var fetchTask in DequeueMany(totalCount, Configuration.BatchSize))
{
IQueueMessage <TData> message = null;
try
{
message = await GetQueueExceptionPolicy()
.ExecuteAsync(async() => await fetchTask.ConfigureAwait(false))
.ConfigureAwait(false);
if (!Validate(message))
{
_requestValidator.HandleInvalidMessage(message, batchResult, nameof(FetchAndProcessDataAsync));
continue;
}
var result = await Policy.Handle <MessageFetchAndProcessException>()
.WaitAndRetryAsync(GetRetrySleepDurations(Configuration.RetryInMessageProcess))
.ExecuteAsync(async() => await QueueServiceMessageHelpers
.ProcessMessageAsync(message, processFunc)
.ConfigureAwait(false))
.ConfigureAwait(false);
_logger.LogDebug(result.ItemDescription);
batchResult.SuccessCount++;
}
catch (MessageQueueException e)
{
_logger.LogError("Error while fetching queue item. Item will remain in the queue.", e);
batchResult.FailureCount++;
}
catch (MessageFetchAndProcessException e)
{
_logger.LogError(
"Error while processing message data. Adding the data back to the message queue for the next run.", e);
batchResult.FailureCount++;
if (message == null)
{
_logger.LogWarning("Queue message is null, skipping.");
continue;
}
await MessageQueue.EnqueueAsync(message.Data, message.Id).ConfigureAwait(false);
}
}
return(batchResult);
}
public void DeleteMessage(IQueue queue, IQueueMessage message)
{
var azureSbMessage = message as AzureServiceBusMessage;
if (azureSbMessage == null)
throw new InvalidOperationException("Can't delete non Azure Service Bus message");
azureSbMessage.BrokeredMessage.Complete();
}
public override Task AddMessageAsync(IQueueMessage <object, int> message, QueueSettings queueSettings, CancellationToken cancellationToken = default(CancellationToken))
{
message.Id = IdInternal;
Queue.Add(IdInternal, SerializerBinary.CloneStatic(message));
++IdInternal;
return(Task.Delay(0, cancellationToken));
}
public override Task EnqueueAsync(
IQueueMessage message,
CancellationToken cancellationToken = default(CancellationToken))
{
return(_messageSender.SendAsync(new Message(message.Body)
{
ContentType = "application/json"
}));
}
public void DeleteMessage(IQueue queue, IQueueMessage message)
{
// Retrieve a reference to a queue
CloudQueue queueC = _client.GetQueueReference(queue.Id);
var azureMessage = (AzureQueueMessage) message;
queueC.DeleteMessage(message.Id, azureMessage.ReceiptHandle);
}
/// <summary>
/// Enqueue a message to the queue
/// </summary>
/// <param name="message">Message to be enqueued</param>
/// <param name="availabilityDate">Date and time when the message should be available for processing.</param>
public void Enqueue(IQueueMessage message, DateTime availabilityDate)
{
//TODO:Should availabilityDate be converted to UTC?
TimeSpan initialVisibilityDelay = availabilityDate - DateTime.UtcNow;
if (initialVisibilityDelay.Ticks < 0)
initialVisibilityDelay = TimeSpan.Zero;
this.InternalEnqueue(message, initialVisibilityDelay);
}
public void Nack(IQueueMessage msg, Exception exception = null)
{
if (msg == null) throw new ArgumentNullException("msg");
var fsMessafe = (FileSystemQueueMessage)msg;
var poisonedFolder = EnsurePoisonedQueueFolder();
var fileNameInPoisonedFolder = Path.Combine(poisonedFolder, fsMessafe.MessageFile.Name);
fsMessafe.MessageFile.MoveTo(fileNameInPoisonedFolder);
}
public void DeleteMessage(IQueue queue, IQueueMessage message)
{
var awsMessage = (AwsQueueMessage)message;
DeleteMessageRequest request = new DeleteMessageRequest()
.WithQueueUrl(queue.Id)
.WithReceiptHandle(awsMessage.ReceiptHandle);
_sqsClient.DeleteMessage(request);
}
private void AssumeQueueIsInitialised()
{
this.storageQueue = Substitute.For <IStorageQueue>();
this.queueMessage = Substitute.For <IQueueMessage>();
this.storageQueue.GetMessage(Arg.Any <TimeSpan>())
.Returns(this.queueMessage);
this.queueMessage.AsString()
.Returns(MessageContent);
}
public void DeleteMessage(IQueueMessage message)
{
var queueMessage = message as QueueMessage;
if (queueMessage != null)
{
this.cloudQueue.DeleteMessageAsync(queueMessage.CloudQueueMessage);
}
}
public void Write(IQueueMessage queueMessage)
{
using (this.metricFactory.CreateLoggingTimerMetric(this.logWriter)
.Start("QueueWriter.Write"))
{
this.logWriter.Trace($"Adding message to queue with content {queueMessage.AsString()}");
this.storageQueue.AddMessage(queueMessage);
}
}
public void HandleMessage(IQueueMessage message)
{
_handler.HandleMessage(
new Message <T>(
(T)message.Message,
message.Id,
message.MessageType,
message.ConsumeDate));
}
public AlbumService(
IAlbumRepository albumRepository,
IQueueMessage queueMessage,
IAlbumHistoricoRepository albumHistoricoRepository)
{
_repository = albumRepository;
_queue = queueMessage;
_historicoRepository = albumHistoricoRepository;
}
public static bool TryDeserializeEvent(IQueueMessage queueMessage, out Event @event)
{
@event = default(Event);
try
{
@event = JsonConvert.DeserializeObject<Event>(queueMessage.Data);
return true;
}
catch (Exception)
{
return false;
}
}
private void HandleMessage(IQueueMessage receivedMessage)
{
//get the real message
var messageBody = receivedMessage.GetBody();
object realMessage;
//if cannot deserialize the message => Nack it immediately, retries will not help
try
{
realMessage = _serializer.Deserialize(messageBody);
}
catch (Exception ex)
{
receivedMessage.Queue.Nack(receivedMessage, ex);
return;
}
var tryCount = 0;
Exception dispatchError = null;
do
{
tryCount++;
dispatchError = TryDispatchMessage(realMessage);
if (dispatchError == null)
{
//Important bit: acking the message!
receivedMessage.Queue.Ack(receivedMessage);
return;
}
//something happened, message was not dispatched.
//wait a bit and try again
Thread.Sleep(50);
} while (tryCount < 5);
// If we are here it means the message has not been dispatched for several times,
// Nack it and move on
receivedMessage.Queue.Nack(receivedMessage, dispatchError);
}
public bool TryReceive(CancellationToken cancellation, out IQueueMessage message)
{
while (!cancellation.IsCancellationRequested)
{
//enumerate all the readers, check if there is a message somewhere
for (var i = 0; i < _readers.Length; i++)
{
var queueMessage = _readers[i].Peek();
if (queueMessage == null) continue;
//a message found, hurray! return it and report success!
message = queueMessage;
return true;
}
//there were no messages in any readers. Wait for a bit and try again
var waiter = cancellation.WaitHandle.WaitOne(250);
}
message = null;
return false;
}
public Task Pop(IQueueMessage queueMessage)
{
return Task.Factory.StartNew(() =>
{
//acknowledgment message (remove from queue)
ulong id;
if (ulong.TryParse(queueMessage.Id, out id))
{
Channel.BasicAck(id, false);
}
});
}
/// <summary>
/// Extends the lease of the message.
/// </summary>
/// <param name="message">Message whose lease needs to be extended</param>
public void ExtendLease(IQueueMessage message)
{
ArgumentValidator.ValidateNonNullReference(message, "message", "AzureQueue.ExtendLease");
ArgumentValidator.ValidateIsAssignableFromType(message.GetType(), "message", "AzureQueue.ExtendLease", typeof(AzureQueueMessage));
try
{
this._internalQueue.UpdateMessage(((AzureQueueMessage)message).CloudQueueMessage, this._visibilityTimeout, MessageUpdateFields.Visibility);
}
catch (StorageException ex)
{
throw new Exception(QueueResource.QueueExtendLeaseError, ex);
}
}
/// <summary>
/// Enqueue a message to the queue
/// </summary>
/// <param name="message">Message to be enqueued</param>
/// <param name="initialVisibilityDelay">Initial time delay before the message can be made visible for processing</param>
/// <remarks>
/// Message a set to live for a maximum of 7 days.
/// </remarks>
private void InternalEnqueue(IQueueMessage message, TimeSpan initialVisibilityDelay)
{
try
{
ArgumentValidator.ValidateNonNullReference(message, "message", "AzureQueue.InternalEnqueue");
ArgumentValidator.ValidateIsAssignableFromType(message.GetType(), "message", "AzureQueue.InternalEnqueue", typeof(AzureQueueMessage));
this._internalQueue.AddMessage(((AzureQueueMessage)message).CloudQueueMessage, this._maxMessageLifeSpan, initialVisibilityDelay);
}
catch (StorageException ex)
{
throw new Exception(QueueResource.QueueEnqueueError, ex);
}
}
public void Acknowledge(IQueueMessage message)
{
var azureQueueMessage = (AzureQueueMessage)message;
azureQueueMessage.Acknowledge(m_Queue);
}
/// <summary>
/// Enqueue a message to the queue
/// </summary>
/// <param name="message">Message to be enqueued</param>
public void Enqueue(IQueueMessage message)
{
this.InternalEnqueue(message, TimeSpan.Zero);
}
/// <summary>
/// Deletes a specific message from the queue
/// </summary>
/// <param name="message">Message to be deleted</param>
public void Delete(IQueueMessage message)
{
ArgumentValidator.ValidateNonNullReference(message, "message", "AzureQueue.Delete");
ArgumentValidator.ValidateIsAssignableFromType(message.GetType(), "message", "AzureQueue.Delete", typeof(AzureQueueMessage));
try
{
this._internalQueue.DeleteMessage(((AzureQueueMessage)message).CloudQueueMessage);
}
catch (StorageException ex)
{
throw new Exception(QueueResource.QueueDeleteError, ex);
}
}
public void Acknowledge(IQueueMessage message)
{
}
public void AddMessage(IQueueMessage message)
{
m_Queue.Enqueue(message.Content);
}
public async Task Pop(IQueueMessage queueMessage)
{
var request = new HttpRequestMessage(HttpMethod.Delete, string.Format("messages/{0}", queueMessage.Id));
var response = await HttpClient.SendAsync(request);
var responseContent = await response.Content.ReadAsStringAsync();
var responseObj = JsonConvert.DeserializeObject<IronIoPopResponse>(responseContent);
}
public void Pop(IQueueMessage queueMessage)
{
var uri = string.Format("{0}/messages/{1}", _baseUri, queueMessage.Id);
var httpResponse = _httpClient.DeleteAsync(uri).Result;
}
public void AddMessage(IQueueMessage message)
{
m_Queue.AddMessage(new CloudQueueMessage(message.Content));
}
