private async Task RetryJob(Job job, ExecutedJob run, ITransaction transaction)
{
run.Retries++;
run.Status = JobStatus.Retrying;
run.NextRetry = CalculateNextRetry(run);
await SaveJobRun(run, transaction);
transaction.SetAddAsync($"{_options.KeyPrefix}_job_retries", job.RunId.ToString());
}
private async Task SaveJobRun(Job job, JobStatus status, ITransaction transaction)
{
var run = await GetRun(job.RunId.ToString());
var execution = new ExecutedJob(job, status, _id);
if (run != null)
{
execution.Retries = run.Retries;
execution.NextRetry = run.NextRetry;
}
await SaveJobRun(execution, transaction);
}
private async Task SaveJobRun(ExecutedJob job, ITransaction transaction)
{
var last = await GetRun(job.RunId.ToString());
if (last != null)
{
transaction.SetRemoveAsync($"{_options.KeyPrefix}_job_status_{last.Status.ToString()}",
last.RunId.ToString());
}
job.Timestamp = DateTime.UtcNow.Ticks;
var lastRun = $"{_options.KeyPrefix}_{job.Name}_last_run";
var key = $"{_options.KeyPrefix}_{job.Name}_runs";
transaction.StringSetAsync($"{_options.KeyPrefix}_{job.RunId}_run", BsonSerializer.ToBson(job));
transaction.SetAddAsync(key, job.RunId.ToString());
transaction.StringSetAsync(lastRun, job.Timestamp);
transaction.SetAddAsync($"{_options.KeyPrefix}_job_status_{job.Status.ToString()}", job.RunId.ToString());
}
private async Task ExecuteJob(ExecutedJob job, ITransaction transaction)
{
if (!_listeners.ContainsKey(job.Name) && !_listenersNoArgs.ContainsKey(job.Name))
{
await OnJobError(job, new NoHandlerFoundException(job), transaction);
return;
}
await OnJobStart(job, transaction);
await transaction.ExecuteAsync();
transaction = _connection.GetDatabase().CreateTransaction();
try
{
if (_listeners.ContainsKey(job.Name))
{
await _listeners[job.Name](job.Parameters);
}
else
{
await _listenersNoArgs[job.Name]();
}
}
catch (Exception e)
{
await OnJobError(job, e, transaction);
return;
}
try
{
transaction.StringIncrementAsync($"{_options.KeyPrefix}_job_{job.Name}_run_count");
transaction.ListRemoveAsync($"{_options.KeyPrefix}_progress", job.RunId.ToString());
transaction.StringIncrementAsync($"{_options.KeyPrefix}_stats_total_runs");
await SaveJobRun(job, JobStatus.Success, transaction);
switch (job.Type)
{
case JobType.Immediate:
await OnImmediateJobFinish(job, transaction);
break;
case JobType.Scheduled:
await OnScheduledJobFinish(job, transaction);
break;
}
}
catch (Exception e)
{
Console.WriteLine(e);
}
finally
{
await transaction.ExecuteAsync(CommandFlags.FireAndForget);
}
}
private async Task ExecuteQueue()
{
await _semaphore.WaitAsync();
ExecutedJob job = null;
string runId = string.Empty;
try
{
var db = _connection.GetDatabase();
var trans = _connection.GetDatabase().CreateTransaction();
var value = await db.ListRightPopLeftPushAsync($"{_options.KeyPrefix}_enqueued",
$"{_options.KeyPrefix}_progress");
if (!value.HasValue)
{
return;
}
runId = value.ToString();
job = await GetRun(runId);
if (job == null)
{
throw new Exception($"Failed to find job run details relating to job run {runId}");
}
var key = $"{_options.KeyPrefix}_{job.Name}_lock";
if (job.Options != null && !job.Options.AllowConcurrentExecution)
{
using (var jobLock = await _lockFactory.CreateLockAsync(key, TimeSpan.FromSeconds(30)))
{
if (!jobLock.IsAcquired)
{
trans.ListRightPushAsync($"{_options.KeyPrefix}_enqueued", value);
trans.ListRemoveAsync($"{_options.KeyPrefix}_progress", value);
await trans.ExecuteAsync(CommandFlags.FireAndForget);
return;
}
await ExecuteJob(job, trans);
}
}
else
{
await ExecuteJob(job, trans);
}
}
catch (Exception e)
{
var trans = _connection.GetDatabase().CreateTransaction();
if (_options.OnQueueError != null)
{
await _options.OnQueueError(e);
}
if (job != null)
{
await OnJobError(job, e, trans);
await trans.ExecuteAsync(CommandFlags.FireAndForget);
}
if (_options.OnQueueError == null)
{
throw;
}
}
finally
{
_semaphore.Release();
}
}
private long CalculateNextRetry(ExecutedJob run)
{
var ticks = (run.Retries * _options.RetryBackOff).Ticks;
return(DateTime.UtcNow.AddTicks(ticks).Ticks);
}
private async Task Enqueue(ExecutedJob run)
{
await EnqueueMany(new[] { run });
}
