public void Enqueue(CrawlingQueueItem queueItem)
{
var writeQueueProxy = QueueProxiesWrite.FirstOrDefault();
// if too many in queue already and proxy present, add to proxy (global azure queue), else add to local queue
if (writeQueueProxy != null && LocalQueue.Count >= MaxLocalQueueSize)
{
writeQueueProxy.EnqueueAsync(queueItem, QueueCancellationTokenSource) // No need to wait for the operation to complete successfully
.ContinueWith(pred =>
{
if (pred.Status != TaskStatus.RanToCompletion)
{
queueItem.ChangeStatus(CrawlingQueueItem.CrawlingStatuses.InLocalQueue);
lock (LocalQueue)
LocalQueue.Enqueue(queueItem);
}
})
.ContinueWith(task =>
{
if (task.IsFaulted)
{
Trace.TraceError("CrawlingQueue.[CrawlingQueueProxy].EnqueueAsync: Exception while trying to enqueue {0} [{1}]", queueItem.ResourceLink, task.Exception);
}
});
}
else
{
queueItem.ChangeStatus(CrawlingQueueItem.CrawlingStatuses.InLocalQueue);
lock (LocalQueue)
LocalQueue.Enqueue(queueItem);
}
}
public async Task EnqueueAsync(CrawlingQueueItem crawlingQueueItem, CancellationTokenSource cts)
{
// 1. Asign an AsyncState object when creating a task, so that if remote action fails, we still have the object to insert into local queue
// 2. Use a timeout value for remote operation
crawlingQueueItem.ChangeStatus(CrawlingQueueItem.CrawlingStatuses.InProxyQueue);
await _azureCrawlingQueue.AddMessageAsync(
CloudQueueMessage.CreateCloudQueueMessageFromByteArray(crawlingQueueItem.ToByteArray()), //new CloudQueueMessage(crawlingQueueItem.ToByteArray()),
null,
null,
new QueueRequestOptions
{
},
new OperationContext
{
},
cts.Token
);
}