private async Task <IState> PerformJob(BackgroundProcessContext context, IStorageConnection connection, string jobId)
{
try
{
var jobData = connection.GetJobData(jobId);
if (jobData == null)
{
// Job expired just after moving to a processing state. This is an
// unreal scenario, but shit happens. Returning null instead of throwing
// an exception and rescuing from en-queueing a poisoned jobId back
// to a queue.
return(null);
}
jobData.EnsureLoaded();
var backgroundJob = new BackgroundJob(jobId, jobData.Job, jobData.CreatedAt);
var jobToken = new ServerJobCancellationToken(connection, jobId, context.ServerId, _workerId, context.CancellationToken);
var performContext = new PerformContext(connection, backgroundJob, jobToken);
var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds;
var duration = Stopwatch.StartNew();
var result = await _performer.PerformAsync(performContext).ConfigureAwait(false);
duration.Stop();
// SHOULD BE: return new SucceededState(result, (long)latency, duration.ElapsedMilliseconds);
return(CreateSucceededState(result, (long)latency, duration.ElapsedMilliseconds));
}
catch (JobAbortedException)
{
// Background job performance was aborted due to a
// state change, so it's idenfifier should be removed
// from a queue.
return(null);
}
catch (JobPerformanceException ex)
{
return(new FailedState(ex.InnerException)
{
Reason = ex.Message
});
}
catch (Exception ex)
{
if (ex is OperationCanceledException && context.IsShutdownRequested)
{
throw;
}
return(new FailedState(ex)
{
Reason = "An exception occurred during processing of a background job."
});
}
}
public void CheckAllCancellationTokens_DoesNotAbortCancellationToken_IfNothingChanged()
{
var token = CreateToken();
Assert.False(token.ShutdownToken.IsCancellationRequested);
ServerJobCancellationToken.CheckAllCancellationTokens(ServerId, _connection.Object, _cts.Token);
Assert.False(token.IsAborted);
token.ShutdownToken.ThrowIfCancellationRequested(); // does not throw
}
public void CheckAllCancellationTokens_DoesNotPerformChecks_WhenShutdownTokenWasNotInitialized()
{
_stateData.Name = "NotProcessing";
var token = CreateToken();
ServerJobCancellationToken.CheckAllCancellationTokens(ServerId, _connection.Object, _cts.Token);
Assert.False(token.IsAborted);
_connection.Verify(x => x.GetStateData(It.IsAny <string>()), Times.Never);
}
public void CheckAllCancellationTokens_DoesNotAbortJobsFromOtherServers()
{
_stateData.Name = "NotProcessing";
var token = CreateToken();
Assert.False(token.ShutdownToken.IsCancellationRequested);
ServerJobCancellationToken.CheckAllCancellationTokens("another-id", _connection.Object, _cts.Token);
token.ShutdownToken.ThrowIfCancellationRequested();
Assert.False(token.IsAborted);
}
public ServerJobCancellationWatcherFacts()
{
_checkInterval = Timeout.InfiniteTimeSpan;
_context = new BackgroundProcessContextMock();
_context.StoppingTokenSource.Cancel();
_connection = new Mock <IStorageConnection>();
_context.Storage.Setup(x => x.GetConnection()).Returns(_connection.Object);
ServerJobCancellationToken.AddServer(_context.ServerId);
}
public void CheckAllCancellationTokens_AbortsCancellationToken_IfWorkerIdWasChanged()
{
_stateData.Data["WorkerId"] = "999";
var token = CreateToken();
Assert.False(token.ShutdownToken.IsCancellationRequested);
ServerJobCancellationToken.CheckAllCancellationTokens(ServerId, _connection.Object, _cts.Token);
Assert.Throws <OperationCanceledException>(
() => token.ShutdownToken.ThrowIfCancellationRequested());
Assert.True(token.IsAborted);
}
public void CheckAllCancellationTokens_AbortsCancellationToken_IfJobIsNotInProcessingState()
{
_stateData.Name = "NotProcessing";
var token = CreateToken();
Assert.False(token.ShutdownToken.IsCancellationRequested);
ServerJobCancellationToken.CheckAllCancellationTokens(ServerId, _connection.Object, _cts.Token);
Assert.Throws <OperationCanceledException>(
() => token.ShutdownToken.ThrowIfCancellationRequested());
Assert.True(token.IsAborted);
}
public void CheckAllCancellationTokens_AbortsCancellationToken_IfStateDataDoesNotExist()
{
_connection.Setup(x => x.GetStateData(It.IsAny <string>())).Returns((StateData)null);
var token = CreateToken();
Assert.False(token.ShutdownToken.IsCancellationRequested);
ServerJobCancellationToken.CheckAllCancellationTokens(ServerId, _connection.Object, _cts.Token);
Assert.Throws <OperationCanceledException>(
() => token.ShutdownToken.ThrowIfCancellationRequested());
Assert.True(token.IsAborted);
}
public void Execute_DelegatesCancellationToServerJobCancellationToken()
{
var token = new ServerJobCancellationToken(_connection.Object, "job-id", _context.ServerId, "1", _context.StoppedTokenSource.Token);
Assert.False(token.ShutdownToken.IsCancellationRequested);
_connection.Setup(x => x.GetStateData(It.IsAny <string>())).Returns((StateData)null);
var watchdog = new ServerJobCancellationWatcher(_checkInterval);
watchdog.Execute(_context.Object);
_connection.Verify(x => x.GetStateData("job-id"));
_connection.Verify(x => x.Dispose(), Times.Once);
Assert.True(token.IsAborted);
}
public void CheckAllCancellationTokens_PerformsAdditionalChecks_WhenPriorOnesDidNotLeadToAbort()
{
var token = CreateToken();
Assert.False(token.ShutdownToken.IsCancellationRequested);
ServerJobCancellationToken.CheckAllCancellationTokens(ServerId, _connection.Object, _cts.Token);
Assert.False(token.IsAborted);
_stateData.Name = "NotProcessing";
ServerJobCancellationToken.CheckAllCancellationTokens(ServerId, _connection.Object, _cts.Token);
Assert.True(token.IsAborted);
_connection.Verify(x => x.GetStateData(It.IsAny <string>()), Times.Exactly(2));
}
public ServerJobCancellationTokenFacts()
{
_stateData = new StateData
{
Name = ProcessingState.StateName,
Data = new Dictionary <string, string>
{
{ "ServerId", ServerId },
{ "WorkerId", WorkerId },
}
};
_connection = new Mock <IStorageConnection>();
_connection.Setup(x => x.GetStateData(JobId)).Returns(_stateData);
_cts = new CancellationTokenSource();
_shutdownCts = new CancellationTokenSource();
ServerJobCancellationToken.AddServer(ServerId);
}
public void Dispose()
{
ServerJobCancellationToken.RemoveServer(_context.ServerId);
}