private void KeepAliveJobCallback(object obj)
{
lock (syncRoot)
{
if (reQueued || removedFromQueue)
{
return;
}
try
{
Documents.Queue queue = (Documents.Queue)obj;
queue.FetchedAt = DateTime.UtcNow;
Uri replaceUri = new Uri(queue.SelfLink, UriKind.Relative);
Task <ResourceResponse <Document> > task = storage.Client.ReplaceDocumentWithRetriesAsync(replaceUri, queue, new RequestOptions {
PartitionKey = partitionKey
});
task.Wait();
logger.Trace($"Keep-alive query for job: {queue.Id} sent");
}
catch (Exception ex)
{
logger.DebugException($"Unable to execute keep-alive query for job: {data.Id}", ex);
}
}
}
public FetchedJob(DocumentDbStorage storage, Documents.Queue data)
{
this.storage = storage;
Id = data.Id;
JobId = data.JobId;
Queue = data.Name;
}
public IFetchedJob Dequeue(string[] queues, CancellationToken cancellationToken)
{
lock (syncLock)
{
IEnumerable <string> queueParams = Enumerable.Range(0, queues.Length).Select((q, i) => $"@queue_{i}");
string query = $"SELECT TOP 1 * FROM doc WHERE doc.type = @type AND doc.name IN ({string.Join(", ", queueParams)}) " +
"AND (NOT IS_DEFINED(doc.fetched_at) OR doc.fetched_at < @timeout) ORDER BY doc.created_on";
List <SqlParameter> parameters = new List <SqlParameter> {
new SqlParameter("@type", (int)Documents.DocumentTypes.Queue)
};
for (int index = 0; index < queues.Length; index++)
{
string queue = queues[index];
parameters.Add(new SqlParameter($"@queue_{index}", queue));
}
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
logger.Trace("Looking for any jobs from the queue");
using (new DocumentDbDistributedLock(DISTRIBUTED_LOCK_KEY, defaultLockTimeout, storage))
{
int invisibilityTimeoutEpoch = DateTime.UtcNow.Add(invisibilityTimeout.Negate()).ToEpoch();
SqlQuerySpec sql = new SqlQuerySpec
{
QueryText = query,
Parameters = new SqlParameterCollection(parameters)
};
sql.Parameters.Add(new SqlParameter("@timeout", invisibilityTimeoutEpoch));
Documents.Queue data = storage.Client.CreateDocumentQueryAsync <Documents.Queue>(storage.CollectionUri, sql, new FeedOptions {
PartitionKey = partitionKey
})
.ToQueryResult()
.FirstOrDefault();
if (data != null)
{
// mark the document
data.FetchedAt = DateTime.UtcNow;
Uri replaceUri = new Uri(data.SelfLink, UriKind.Relative);
Task <ResourceResponse <Document> > task = storage.Client.ReplaceDocumentWithRetriesAsync(replaceUri, data, new RequestOptions {
PartitionKey = partitionKey
}, cancellationToken: cancellationToken);
task.Wait(cancellationToken);
logger.Trace($"Found job {data.JobId} from the queue : {data.Name}");
return(new FetchedJob(storage, data));
}
}
logger.Trace($"Unable to find any jobs in the queue. Will check the queue for jobs in {storage.Options.QueuePollInterval.TotalSeconds} seconds");
cancellationToken.WaitHandle.WaitOne(storage.Options.QueuePollInterval);
}
}
}
private void KeepAliveJobCallback(object obj)
{
lock (syncRoot)
{
if (reQueued || removedFromQueue)
{
return;
}
try
{
Documents.Queue queue = (Documents.Queue)obj;
queue.FetchedAt = DateTime.UtcNow;
Task <ItemResponse <Documents.Queue> > task = storage.Container.ReplaceItemWithRetriesAsync(queue, queue.Id, partitionKey);
task.Wait();
logger.Trace($"Keep-alive query for job: {queue.Id} sent");
}
catch (Exception ex)
{
logger.DebugException($"Unable to execute keep-alive query for job: {data.Id}", ex);
}
}
}
public FetchedJob(DocumentDbStorage storage, Documents.Queue data)
{
this.storage = storage;
this.data = data;
TimeSpan keepAliveInterval = TimeSpan.FromMinutes(5);
timer = new Timer(KeepAliveJobCallback, data, keepAliveInterval, keepAliveInterval);
}
public FetchedJob(CouchbaseStorage storage, Documents.Queue data)
{
bucket = storage.Client.OpenBucket(storage.Options.DefaultBucket);
Id = data.Id;
JobId = data.JobId;
TimeSpan keepAliveInterval = TimeSpan.FromMinutes(5);
timer = new Timer(KeepAliveJobCallback, null, keepAliveInterval, keepAliveInterval);
}
public void Enqueue(string queue, string jobId)
{
Documents.Queue data = new Documents.Queue
{
Name = queue,
JobId = jobId,
CreatedOn = DateTime.UtcNow
};
Task <ResourceResponse <Document> > task = storage.Client.CreateDocumentWithRetriesAsync(storage.CollectionUri, data);
task.Wait();
}
public void Enqueue(string queue, string jobId)
{
Documents.Queue data = new Documents.Queue
{
Name = queue,
JobId = jobId,
CreatedOn = DateTime.UtcNow
};
Task <ItemResponse <Documents.Queue> > task = storage.Container.CreateItemWithRetriesAsync(data, partitionKey);
task.Wait();
}
public void Enqueue(string queue, string jobId)
{
Documents.Queue data = new Documents.Queue
{
Name = queue,
JobId = jobId,
CreatedOn = DateTime.UtcNow
};
using (IBucket bucket = storage.Client.OpenBucket(storage.Options.DefaultBucket))
{
bucket.Insert(data.Id, data);
}
}
public IFetchedJob Dequeue(string[] queues, CancellationToken cancellationToken)
{
lock (syncLock)
{
IEnumerable <string> queueParams = Enumerable.Range(0, queues.Length).Select((q, i) => $"@queue_{i}");
string query = $"SELECT TOP 1 * FROM doc WHERE doc.type = @type AND doc.name IN ({string.Join(", ", queueParams)}) " +
"AND (NOT IS_DEFINED(doc.fetched_at) OR doc.fetched_at < @timeout) ORDER BY doc.created_on";
QueryDefinition sql = new QueryDefinition(query)
.WithParameter("@type", (int)DocumentTypes.Queue);
for (int index = 0; index < queues.Length; index++)
{
string queue = queues[index];
sql.WithParameter($"@queue_{index}", queue);
}
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
logger.Trace("Looking for any jobs from the queue");
using (new CosmosDbDistributedLock(DISTRIBUTED_LOCK_KEY, defaultLockTimeout, storage))
{
int invisibilityTimeoutEpoch = DateTime.UtcNow.Add(invisibilityTimeout.Negate()).ToEpoch();
sql.WithParameter("@timeout", invisibilityTimeoutEpoch);
Documents.Queue data = storage.Container.GetItemQueryIterator <Documents.Queue>(sql, requestOptions: new QueryRequestOptions {
PartitionKey = partitionKey
})
.ToQueryResult()
.FirstOrDefault();
if (data != null)
{
// mark the document
data.FetchedAt = DateTime.UtcNow;
Task <ItemResponse <Documents.Queue> > task = storage.Container.ReplaceItemWithRetriesAsync(data, data.Id, partitionKey, cancellationToken: cancellationToken);
task.Wait(cancellationToken);
logger.Trace($"Found job {data.JobId} from the queue : {data.Name}");
return(new FetchedJob(storage, data));
}
}
logger.Trace($"Unable to find any jobs in the queue. Will check the queue for jobs in {storage.StorageOptions.QueuePollInterval.TotalSeconds} seconds");
cancellationToken.WaitHandle.WaitOne(storage.StorageOptions.QueuePollInterval);
}
}
}
public void Requeue()
{
Documents.Queue data = new Documents.Queue
{
Id = Id,
Name = Queue,
JobId = JobId,
CreatedOn = DateTime.UtcNow,
FetchedAt = null
};
Task <ResourceResponse <Document> > task = storage.Client.UpsertDocumentAsync(storage.CollectionUri, data);
task.Wait();
}
public IFetchedJob Dequeue(string[] queues, CancellationToken cancellationToken)
{
lock (syncLock)
{
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
int invisibilityTimeoutEpoch = DateTime.UtcNow.Add(invisibilityTimeout.Negate()).ToEpoch();
logger.Trace("Looking for any jobs from the queue");
using (new CouchbaseDistributedLock(DISTRIBUTED_LOCK_KEY, defaultLockTimeout, storage))
{
using (IBucket bucket = storage.Client.OpenBucket(storage.Options.DefaultBucket))
{
BucketContext context = new BucketContext(bucket);
Documents.Queue data = context.Query <Documents.Queue>()
.Where(q => q.DocumentType == DocumentTypes.Queue && queues.Contains(q.Name) && (N1QlFunctions.IsMissing(q.FetchedAt) || q.FetchedAt < invisibilityTimeoutEpoch))
.OrderBy(q => q.CreatedOn)
.FirstOrDefault();
if (data != null)
{
IDocumentFragment <Documents.Queue> result = bucket.MutateIn <Documents.Queue>(data.Id)
.Upsert(q => q.FetchedAt, DateTime.UtcNow.ToEpoch(), true)
.Execute();
if (result.Success)
{
logger.Trace($"Found job {data.JobId} from the queue {data.Name}");
return(new FetchedJob(storage, data));
}
}
}
}
logger.Trace($"Unable to find any jobs in the queue. Will check the queue for jobs in {storage.Options.QueuePollInterval.TotalSeconds} seconds");
cancellationToken.WaitHandle.WaitOne(storage.Options.QueuePollInterval);
}
}
}
