protected internal virtual Task Dispatch(InvocationState invocation, CancellationToken cancelToken)
{
var logCapture = _logContainer == null ?
new InvocationLogCapture(invocation) :
new BlobInvocationLogCapture(invocation, _logContainer);
return(Dispatch(invocation, logCapture, cancelToken));
}
private bool ProcessResult(InvocationState invocation, InvocationState.InvocationRow result)
{
if (result == null)
{
return(false);
}
else
{
invocation.Update(result);
return(true);
}
}
public virtual async Task <bool> UpdateStatus(InvocationState invocation, InvocationStatus status, ExecutionResult result)
{
var newVersion = await ConnectAndExec(
"work.SetInvocationStatus",
new
{
Id = invocation.Id,
Version = invocation.CurrentVersion,
Status = status,
Result = result,
InstanceName = InstanceName
});
return(ProcessResult(invocation, newVersion));
}
/// <summary>
/// Extends the visibility timeout of the request. That is, the time during which the
/// queue message is hidden from other clients
/// </summary>
/// <param name="request">The request to extend</param>
/// <param name="duration">The duration from the time of invocation to hide the message</param>
public virtual async Task <bool> Extend(InvocationState invocation, TimeSpan duration)
{
var invisibleUntil = _clock.UtcNow + duration;
var newVersion = await ConnectAndExec(
"work.ExtendInvocation",
new
{
Id = invocation.Id,
Version = invocation.CurrentVersion,
ExtendTo = invisibleUntil.UtcDateTime,
InstanceName = InstanceName
});
return(ProcessResult(invocation, newVersion));
}
/// <summary>
/// Acknowledges that the request has completed successfully, removing the message from the queue.
/// </summary>
/// <param name="request">The request to acknowledge</param>
/// <returns>A boolean indicating if the request was successful. Failure indicates that another node has updated the invocation since we last checked</returns>
public virtual async Task <bool> Complete(InvocationState invocation, ExecutionResult result, string resultMessage, string logUrl)
{
// Try to complete the row
var newVersion = await ConnectAndExec(
"work.CompleteInvocation",
new
{
Id = invocation.Id,
Version = invocation.CurrentVersion,
Result = (int)result,
ResultMessage = resultMessage,
LogUrl = logUrl,
InstanceName = InstanceName
});
return(ProcessResult(invocation, newVersion));
}
public IObservable <EventEntry> RunJob(string job, string payload)
{
var runner = new JobRunner(
new JobDispatcher(
GetAllAvailableJobs(),
Container),
InvocationQueue.Null,
Container.Resolve <ConfigurationHub>(),
Clock.RealClock);
var invocation =
new InvocationState(
new InvocationState.InvocationRow()
{
Payload = payload,
Status = (int)InvocationStatus.Executing,
Result = (int)ExecutionResult.Incomplete,
Source = Constants.Source_LocalJob,
Id = Guid.NewGuid(),
Job = job,
UpdatedBy = Environment.MachineName,
UpdatedAt = DateTime.UtcNow,
QueuedAt = DateTime.UtcNow,
NextVisibleAt = DateTime.UtcNow + TimeSpan.FromMinutes(5)
});
return(Observable.Create <EventEntry>(observer =>
{
var capture = new InvocationLogCapture(invocation);
capture.Subscribe(e => observer.OnNext(e), ex => observer.OnError(ex));
runner.Dispatch(invocation, capture, CancellationToken.None, includeContinuations: true).ContinueWith(t =>
{
if (t.IsFaulted)
{
observer.OnError(t.Exception);
}
else
{
observer.OnCompleted();
}
return t;
});
return () => { }; // No action on unsubscribe
}));
}
public virtual async Task <bool> Suspend(InvocationState invocation, Dictionary <string, string> newPayload, TimeSpan suspendFor, string logUrl)
{
var suspendUntil = _clock.UtcNow + suspendFor;
var serializedPayload = InvocationPayloadSerializer.Serialize(newPayload);
var newVersion = await ConnectAndExec(
"work.SuspendInvocation",
new
{
Id = invocation.Id,
Version = invocation.CurrentVersion,
Payload = serializedPayload,
SuspendUntil = suspendUntil.UtcDateTime,
LogUrl = logUrl,
InstanceName = InstanceName
});
return(ProcessResult(invocation, newVersion));
}
public override Task <bool> Complete(InvocationState invocation, ExecutionResult result, string resultMessage, string logUrl)
{
return(Task.FromResult(true));
}
private bool ProcessResult(InvocationState invocation, InvocationState.InvocationRow result)
{
if (result == null)
{
return false;
}
else
{
invocation.Update(result);
return true;
}
}
public InvocationContext(InvocationState invocation, InvocationQueue queue, CancellationToken cancelToken, InvocationLogCapture capture)
: this(invocation, queue, cancelToken)
{
_capture = capture;
}
public InvocationContext(InvocationState invocation, InvocationQueue queue)
: this(invocation, queue, CancellationToken.None)
{
}
private Task <InvocationState> EnqueueRepeat(InvocationState repeat, InvocationResult result)
{
return(Queue.Enqueue(repeat.Job, Constants.Source_RepeatingJob, repeat.Payload, result.RescheduleIn.Value));
}
protected internal virtual Task Dispatch(InvocationState invocation, InvocationLogCapture capture, CancellationToken cancelToken)
{
return(Dispatch(invocation, capture, cancelToken, includeContinuations: false));
}
public override Task <bool> UpdateStatus(InvocationState invocation, InvocationStatus status, ExecutionResult result)
{
return(Task.FromResult(true));
}
protected internal virtual async Task Dispatch(InvocationState invocation, InvocationLogCapture capture, CancellationToken cancelToken, bool includeContinuations)
{
InvocationContext.SetCurrentInvocationId(invocation.Id);
if (invocation.IsContinuation)
{
InvocationEventSource.Log.Resumed();
}
else
{
InvocationEventSource.Log.Started();
}
// Record that we are executing the job
if (!await Queue.UpdateStatus(invocation, InvocationStatus.Executing, ExecutionResult.Incomplete))
{
InvocationEventSource.Log.Aborted(invocation);
return;
}
// Create the request.Invocation context and start capturing the logs
await capture.Start();
var context = new InvocationContext(invocation, Queue, cancelToken, capture);
InvocationResult result = null;
try
{
result = await Dispatcher.Dispatch(context);
// TODO: If response.Continuation != null, enqueue continuation
switch (result.Result)
{
case ExecutionResult.Completed:
InvocationEventSource.Log.Succeeded(result);
break;
case ExecutionResult.Faulted:
InvocationEventSource.Log.Faulted(result);
break;
case ExecutionResult.Incomplete:
InvocationEventSource.Log.Suspended(result);
break;
default:
InvocationEventSource.Log.UnknownStatus(result);
break;
}
if (invocation.NextVisibleAt < Clock.UtcNow)
{
InvocationEventSource.Log.InvocationTookTooLong(invocation);
}
}
catch (Exception ex)
{
InvocationEventSource.Log.DispatchError(ex);
result = new InvocationResult(ExecutionResult.Crashed, ex);
}
// Stop capturing and set the log url
string logUrl = null;
if (capture != null)
{
var logUri = await capture.End();
if (logUri != null)
{
logUrl = logUri.AbsoluteUri;
}
}
if (result.Result != ExecutionResult.Incomplete)
{
// If we're not suspended, the invocation has completed
InvocationEventSource.Log.Ended();
await Queue.Complete(invocation, result.Result, result.Exception == null ? null : result.Exception.ToString(), logUrl);
// If we've completed and there's a repeat, queue the repeat
if (result.RescheduleIn != null)
{
// Rescheule it to run again
var repeat = await EnqueueRepeat(invocation, result);
InvocationEventSource.Log.ScheduledRepeat(invocation, repeat, result.RescheduleIn.Value);
}
}
else
{
if (includeContinuations)
{
invocation.Update(new InvocationState.InvocationRow()
{
Id = Guid.NewGuid(),
Version = invocation.CurrentVersion + 1,
Job = invocation.Job,
Source = invocation.Id.ToString("N"),
Status = (int)InvocationStatus.Suspended,
Result = (int)ExecutionResult.Incomplete,
LastDequeuedAt = invocation.LastDequeuedAt == null ? (DateTime?)null : invocation.LastDequeuedAt.Value.UtcDateTime,
LastSuspendedAt = DateTime.UtcNow,
CompletedAt = null,
QueuedAt = invocation.QueuedAt.UtcDateTime,
NextVisibleAt = invocation.NextVisibleAt.UtcDateTime + DefaultInvisibilityPeriod,
UpdatedAt = invocation.UpdatedAt.UtcDateTime,
Payload = InvocationPayloadSerializer.Serialize(result.Continuation.Parameters),
IsContinuation = true
});
// Run the continuation inline after waiting
await Task.Delay(result.Continuation.WaitPeriod);
await Dispatch(invocation, capture, cancelToken, includeContinuations);
}
else
{
// Suspend the job until the continuation is ready to run
await Queue.Suspend(invocation, result.Continuation.Parameters, result.Continuation.WaitPeriod, logUrl);
}
}
}
protected internal virtual Task Dispatch(InvocationState invocation, InvocationLogCapture capture, CancellationToken cancelToken)
{
return Dispatch(invocation, capture, cancelToken, includeContinuations: false);
}
protected internal virtual Task Dispatch(InvocationState invocation, CancellationToken cancelToken)
{
var logCapture = _logContainer == null ?
new InvocationLogCapture(invocation) :
new BlobInvocationLogCapture(invocation, _logContainer);
return Dispatch(invocation, logCapture, cancelToken);
}
public override Task<bool> UpdateStatus(InvocationState invocation, InvocationStatus status, ExecutionResult result)
{
return Task.FromResult(true);
}
public override Task<bool> Suspend(InvocationState invocation, Dictionary<string, string> newPayload, TimeSpan suspendFor, string logUrl)
{
return Task.FromResult(true);
}
public override Task<bool> Extend(InvocationState invocation, TimeSpan duration)
{
return Task.FromResult(true);
}
public override Task<bool> Complete(InvocationState invocation, ExecutionResult result, string resultMessage, string logUrl)
{
return Task.FromResult(true);
}
public override Task <bool> Extend(InvocationState invocation, TimeSpan duration)
{
return(Task.FromResult(true));
}
public override Task <bool> Suspend(InvocationState invocation, Dictionary <string, string> newPayload, TimeSpan suspendFor, string logUrl)
{
return(Task.FromResult(true));
}
private Task<InvocationState> EnqueueRepeat(InvocationState repeat, InvocationResult result)
{
return Queue.Enqueue(repeat.Job, Constants.Source_RepeatingJob, repeat.Payload, result.RescheduleIn.Value);
}
public InvocationContext(InvocationState invocation, InvocationQueue queue, CancellationToken cancelToken, InvocationLogCapture capture)
: this(invocation, queue, cancelToken)
{
_capture = capture;
}
public virtual async Task<bool> Suspend(InvocationState invocation, Dictionary<string, string> newPayload, TimeSpan suspendFor, string logUrl)
{
var suspendUntil = _clock.UtcNow + suspendFor;
var serializedPayload = InvocationPayloadSerializer.Serialize(newPayload);
var newVersion = await ConnectAndExec(
"work.SuspendInvocation",
new
{
Id = invocation.Id,
Version = invocation.CurrentVersion,
Payload = serializedPayload,
SuspendUntil = suspendUntil.UtcDateTime,
LogUrl = logUrl,
InstanceName = InstanceName
});
return ProcessResult(invocation, newVersion);
}
protected internal virtual async Task Dispatch(InvocationState invocation, InvocationLogCapture capture, CancellationToken cancelToken, bool includeContinuations)
{
InvocationContext.SetCurrentInvocationId(invocation.Id);
if (invocation.IsContinuation)
{
InvocationEventSource.Log.Resumed();
}
else
{
InvocationEventSource.Log.Started();
}
// Record that we are executing the job
if (!await Queue.UpdateStatus(invocation, InvocationStatus.Executing, ExecutionResult.Incomplete))
{
InvocationEventSource.Log.Aborted(invocation);
return;
}
// Create the request.Invocation context and start capturing the logs
await capture.Start();
var context = new InvocationContext(invocation, Queue, cancelToken, capture);
InvocationResult result = null;
try
{
result = await Dispatcher.Dispatch(context);
// TODO: If response.Continuation != null, enqueue continuation
switch (result.Result)
{
case ExecutionResult.Completed:
InvocationEventSource.Log.Succeeded(result);
break;
case ExecutionResult.Faulted:
InvocationEventSource.Log.Faulted(result);
break;
case ExecutionResult.Incomplete:
InvocationEventSource.Log.Suspended(result);
break;
default:
InvocationEventSource.Log.UnknownStatus(result);
break;
}
if (invocation.NextVisibleAt < Clock.UtcNow)
{
InvocationEventSource.Log.InvocationTookTooLong(invocation);
}
}
catch (Exception ex)
{
InvocationEventSource.Log.DispatchError(ex);
result = new InvocationResult(ExecutionResult.Crashed, ex);
}
// Stop capturing and set the log url
string logUrl = null;
if (capture != null)
{
var logUri = await capture.End();
if (logUri != null)
{
logUrl = logUri.AbsoluteUri;
}
}
if (result.Result != ExecutionResult.Incomplete)
{
// If we're not suspended, the invocation has completed
InvocationEventSource.Log.Ended();
await Queue.Complete(invocation, result.Result, result.Exception == null?null : result.Exception.ToString(), logUrl);
// If we've completed and there's a repeat, queue the repeat
if (result.RescheduleIn != null)
{
// Rescheule it to run again
var repeat = await EnqueueRepeat(invocation, result);
InvocationEventSource.Log.ScheduledRepeat(invocation, repeat, result.RescheduleIn.Value);
}
}
else
{
if (includeContinuations)
{
invocation.Update(new InvocationState.InvocationRow()
{
Id = Guid.NewGuid(),
Version = invocation.CurrentVersion + 1,
Job = invocation.Job,
Source = invocation.Id.ToString("N"),
Status = (int)InvocationStatus.Suspended,
Result = (int)ExecutionResult.Incomplete,
LastDequeuedAt = invocation.LastDequeuedAt == null ? (DateTime?)null : invocation.LastDequeuedAt.Value.UtcDateTime,
LastSuspendedAt = DateTime.UtcNow,
CompletedAt = null,
QueuedAt = invocation.QueuedAt.UtcDateTime,
NextVisibleAt = invocation.NextVisibleAt.UtcDateTime + DefaultInvisibilityPeriod,
UpdatedAt = invocation.UpdatedAt.UtcDateTime,
Payload = InvocationPayloadSerializer.Serialize(result.Continuation.Parameters),
IsContinuation = true
});
// Run the continuation inline after waiting
await Task.Delay(result.Continuation.WaitPeriod);
await Dispatch(invocation, capture, cancelToken, includeContinuations);
}
else
{
// Suspend the job until the continuation is ready to run
await Queue.Suspend(invocation, result.Continuation.Parameters, result.Continuation.WaitPeriod, logUrl);
}
}
}
public InvocationContext(InvocationState invocation, InvocationQueue queue, CancellationToken cancelToken)
{
Invocation = invocation;
Queue = queue;
CancelToken = cancelToken;
}
public virtual async Task Run(CancellationToken cancelToken)
{
WorkServiceEventSource.Log.ReinitializingInvocationState(Queue.InstanceName);
await Queue.ReinitializeInvocationState();
WorkServiceEventSource.Log.ReinitializedInvocationState();
WorkServiceEventSource.Log.DispatchLoopStarted();
try
{
while (!cancelToken.IsCancellationRequested)
{
InvocationState invocation = null;
try
{
Status = RunnerStatus.Dequeuing;
invocation = await Queue.Dequeue(DefaultInvisibilityPeriod, cancelToken);
Status = RunnerStatus.Working;
}
catch (Exception ex)
{
WorkServiceEventSource.Log.ErrorRetrievingInvocation(ex);
}
// Check Cancellation
if (cancelToken.IsCancellationRequested)
{
break;
}
if (invocation == null)
{
Status = RunnerStatus.Sleeping;
WorkServiceEventSource.Log.DispatchLoopWaiting(_pollInterval);
await Clock.Delay(_pollInterval, cancelToken);
WorkServiceEventSource.Log.DispatchLoopResumed();
Status = RunnerStatus.Working;
}
else if (invocation.Status == InvocationStatus.Cancelled)
{
// Job was cancelled by the user, so just continue.
WorkServiceEventSource.Log.Cancelled(invocation);
}
else
{
Status = RunnerStatus.Dispatching;
_currentInvocationId = invocation.Id.ToByteArray();
Exception dispatchError = null;
try
{
await Dispatch(invocation, cancelToken);
}
catch (Exception ex)
{
dispatchError = ex;
}
if (dispatchError != null)
{
await Queue.Complete(
invocation,
ExecutionResult.Crashed,
dispatchError.ToString(),
null);
}
_currentInvocationId = null;
_lastInvocationId = invocation.Id.ToByteArray();
Status = RunnerStatus.Working;
}
}
Status = RunnerStatus.Stopping;
}
catch (Exception ex)
{
WorkServiceEventSource.Log.DispatchLoopError(ex);
Status = RunnerStatus.Error;
_error = ex;
throw;
}
WorkServiceEventSource.Log.DispatchLoopEnded();
}
/// <summary>
/// Extends the visibility timeout of the request. That is, the time during which the
/// queue message is hidden from other clients
/// </summary>
/// <param name="request">The request to extend</param>
/// <param name="duration">The duration from the time of invocation to hide the message</param>
public virtual async Task<bool> Extend(InvocationState invocation, TimeSpan duration)
{
var invisibleUntil = _clock.UtcNow + duration;
var newVersion = await ConnectAndExec(
"work.ExtendInvocation",
new
{
Id = invocation.Id,
Version = invocation.CurrentVersion,
ExtendTo = invisibleUntil.UtcDateTime,
InstanceName = InstanceName
});
return ProcessResult(invocation, newVersion);
}
public InvocationContext(InvocationState invocation, InvocationQueue queue, CancellationToken cancelToken)
{
Invocation = invocation;
Queue = queue;
CancelToken = cancelToken;
}
/// <summary>
/// Acknowledges that the request has completed successfully, removing the message from the queue.
/// </summary>
/// <param name="request">The request to acknowledge</param>
/// <returns>A boolean indicating if the request was successful. Failure indicates that another node has updated the invocation since we last checked</returns>
public virtual async Task<bool> Complete(InvocationState invocation, ExecutionResult result, string resultMessage, string logUrl)
{
// Try to complete the row
var newVersion = await ConnectAndExec(
"work.CompleteInvocation",
new
{
Id = invocation.Id,
Version = invocation.CurrentVersion,
Result = (int)result,
ResultMessage = resultMessage,
LogUrl = logUrl,
InstanceName = InstanceName
});
return ProcessResult(invocation, newVersion);
}
public InvocationContext(InvocationState invocation, InvocationQueue queue)
: this(invocation, queue, CancellationToken.None)
{
}
public virtual async Task<bool> UpdateStatus(InvocationState invocation, InvocationStatus status, ExecutionResult result)
{
var newVersion = await ConnectAndExec(
"work.SetInvocationStatus",
new
{
Id = invocation.Id,
Version = invocation.CurrentVersion,
Status = status,
Result = result,
InstanceName = InstanceName
});
return ProcessResult(invocation, newVersion);
}