protected internal override Task Dispatch(InvocationState invocation, InvocationLogCapture capture, CancellationToken cancelToken)
{
LastDispatched = invocation;
if (_skipDispatch)
{
return(DispatchTCS.Task);
}
else
{
return(base.Dispatch(invocation, capture, cancelToken));
}
}
public IObservable <EventEntry> RunJob(string job, string payload)
{
var runner = new JobRunner(
new JobDispatcher(
GetAllAvailableJobs(),
Container),
InvocationQueue.Null,
Container.Resolve <ConfigurationHub>(),
Clock.RealClock);
var invocation =
new InvocationState(
new InvocationState.InvocationRow()
{
Payload = payload,
Status = (int)InvocationStatus.Executing,
Result = (int)ExecutionResult.Incomplete,
Source = Constants.Source_LocalJob,
Id = Guid.NewGuid(),
Job = job,
UpdatedBy = Environment.MachineName,
UpdatedAt = DateTime.UtcNow,
QueuedAt = DateTime.UtcNow,
NextVisibleAt = DateTime.UtcNow + TimeSpan.FromMinutes(5)
});
return(Observable.Create <EventEntry>(observer =>
{
var capture = new InvocationLogCapture(invocation);
capture.Subscribe(e => observer.OnNext(e), ex => observer.OnError(ex));
runner.Dispatch(invocation, capture, CancellationToken.None, includeContinuations: true).ContinueWith(t =>
{
if (t.IsFaulted)
{
observer.OnError(t.Exception);
}
else
{
observer.OnCompleted();
}
return t;
});
return () => { }; // No action on unsubscribe
}));
}
protected internal override Task Dispatch(InvocationState invocation, InvocationLogCapture capture, CancellationToken cancelToken)
{
LastDispatched = invocation;
if (_skipDispatch)
{
return DispatchTCS.Task;
}
else
{
return base.Dispatch(invocation, capture, cancelToken);
}
}
public InvocationContext(InvocationState invocation, InvocationQueue queue, CancellationToken cancelToken, InvocationLogCapture capture)
: this(invocation, queue, cancelToken)
{
_capture = capture;
}
protected internal virtual async Task Dispatch(InvocationState invocation, InvocationLogCapture capture, CancellationToken cancelToken, bool includeContinuations)
{
InvocationContext.SetCurrentInvocationId(invocation.Id);
if (invocation.IsContinuation)
{
InvocationEventSource.Log.Resumed();
}
else
{
InvocationEventSource.Log.Started();
}
// Record that we are executing the job
if (!await Queue.UpdateStatus(invocation, InvocationStatus.Executing, ExecutionResult.Incomplete))
{
InvocationEventSource.Log.Aborted(invocation);
return;
}
// Create the request.Invocation context and start capturing the logs
await capture.Start();
var context = new InvocationContext(invocation, Queue, cancelToken, capture);
InvocationResult result = null;
try
{
result = await Dispatcher.Dispatch(context);
// TODO: If response.Continuation != null, enqueue continuation
switch (result.Result)
{
case ExecutionResult.Completed:
InvocationEventSource.Log.Succeeded(result);
break;
case ExecutionResult.Faulted:
InvocationEventSource.Log.Faulted(result);
break;
case ExecutionResult.Incomplete:
InvocationEventSource.Log.Suspended(result);
break;
default:
InvocationEventSource.Log.UnknownStatus(result);
break;
}
if (invocation.NextVisibleAt < Clock.UtcNow)
{
InvocationEventSource.Log.InvocationTookTooLong(invocation);
}
}
catch (Exception ex)
{
InvocationEventSource.Log.DispatchError(ex);
result = new InvocationResult(ExecutionResult.Crashed, ex);
}
// Stop capturing and set the log url
string logUrl = null;
if (capture != null)
{
var logUri = await capture.End();
if (logUri != null)
{
logUrl = logUri.AbsoluteUri;
}
}
if (result.Result != ExecutionResult.Incomplete)
{
// If we're not suspended, the invocation has completed
InvocationEventSource.Log.Ended();
await Queue.Complete(invocation, result.Result, result.Exception == null?null : result.Exception.ToString(), logUrl);
// If we've completed and there's a repeat, queue the repeat
if (result.RescheduleIn != null)
{
// Rescheule it to run again
var repeat = await EnqueueRepeat(invocation, result);
InvocationEventSource.Log.ScheduledRepeat(invocation, repeat, result.RescheduleIn.Value);
}
}
else
{
if (includeContinuations)
{
invocation.Update(new InvocationState.InvocationRow()
{
Id = Guid.NewGuid(),
Version = invocation.CurrentVersion + 1,
Job = invocation.Job,
Source = invocation.Id.ToString("N"),
Status = (int)InvocationStatus.Suspended,
Result = (int)ExecutionResult.Incomplete,
LastDequeuedAt = invocation.LastDequeuedAt == null ? (DateTime?)null : invocation.LastDequeuedAt.Value.UtcDateTime,
LastSuspendedAt = DateTime.UtcNow,
CompletedAt = null,
QueuedAt = invocation.QueuedAt.UtcDateTime,
NextVisibleAt = invocation.NextVisibleAt.UtcDateTime + DefaultInvisibilityPeriod,
UpdatedAt = invocation.UpdatedAt.UtcDateTime,
Payload = InvocationPayloadSerializer.Serialize(result.Continuation.Parameters),
IsContinuation = true
});
// Run the continuation inline after waiting
await Task.Delay(result.Continuation.WaitPeriod);
await Dispatch(invocation, capture, cancelToken, includeContinuations);
}
else
{
// Suspend the job until the continuation is ready to run
await Queue.Suspend(invocation, result.Continuation.Parameters, result.Continuation.WaitPeriod, logUrl);
}
}
}
protected internal virtual Task Dispatch(InvocationState invocation, InvocationLogCapture capture, CancellationToken cancelToken)
{
return(Dispatch(invocation, capture, cancelToken, includeContinuations: false));
}
protected internal virtual async Task Dispatch(InvocationState invocation, InvocationLogCapture capture, CancellationToken cancelToken, bool includeContinuations)
{
InvocationContext.SetCurrentInvocationId(invocation.Id);
if (invocation.IsContinuation)
{
InvocationEventSource.Log.Resumed();
}
else
{
InvocationEventSource.Log.Started();
}
// Record that we are executing the job
if (!await Queue.UpdateStatus(invocation, InvocationStatus.Executing, ExecutionResult.Incomplete))
{
InvocationEventSource.Log.Aborted(invocation);
return;
}
// Create the request.Invocation context and start capturing the logs
await capture.Start();
var context = new InvocationContext(invocation, Queue, cancelToken, capture);
InvocationResult result = null;
try
{
result = await Dispatcher.Dispatch(context);
// TODO: If response.Continuation != null, enqueue continuation
switch (result.Result)
{
case ExecutionResult.Completed:
InvocationEventSource.Log.Succeeded(result);
break;
case ExecutionResult.Faulted:
InvocationEventSource.Log.Faulted(result);
break;
case ExecutionResult.Incomplete:
InvocationEventSource.Log.Suspended(result);
break;
default:
InvocationEventSource.Log.UnknownStatus(result);
break;
}
if (invocation.NextVisibleAt < Clock.UtcNow)
{
InvocationEventSource.Log.InvocationTookTooLong(invocation);
}
}
catch (Exception ex)
{
InvocationEventSource.Log.DispatchError(ex);
result = new InvocationResult(ExecutionResult.Crashed, ex);
}
// Stop capturing and set the log url
string logUrl = null;
if (capture != null)
{
var logUri = await capture.End();
if (logUri != null)
{
logUrl = logUri.AbsoluteUri;
}
}
if (result.Result != ExecutionResult.Incomplete)
{
// If we're not suspended, the invocation has completed
InvocationEventSource.Log.Ended();
await Queue.Complete(invocation, result.Result, result.Exception == null ? null : result.Exception.ToString(), logUrl);
// If we've completed and there's a repeat, queue the repeat
if (result.RescheduleIn != null)
{
// Rescheule it to run again
var repeat = await EnqueueRepeat(invocation, result);
InvocationEventSource.Log.ScheduledRepeat(invocation, repeat, result.RescheduleIn.Value);
}
}
else
{
if (includeContinuations)
{
invocation.Update(new InvocationState.InvocationRow()
{
Id = Guid.NewGuid(),
Version = invocation.CurrentVersion + 1,
Job = invocation.Job,
Source = invocation.Id.ToString("N"),
Status = (int)InvocationStatus.Suspended,
Result = (int)ExecutionResult.Incomplete,
LastDequeuedAt = invocation.LastDequeuedAt == null ? (DateTime?)null : invocation.LastDequeuedAt.Value.UtcDateTime,
LastSuspendedAt = DateTime.UtcNow,
CompletedAt = null,
QueuedAt = invocation.QueuedAt.UtcDateTime,
NextVisibleAt = invocation.NextVisibleAt.UtcDateTime + DefaultInvisibilityPeriod,
UpdatedAt = invocation.UpdatedAt.UtcDateTime,
Payload = InvocationPayloadSerializer.Serialize(result.Continuation.Parameters),
IsContinuation = true
});
// Run the continuation inline after waiting
await Task.Delay(result.Continuation.WaitPeriod);
await Dispatch(invocation, capture, cancelToken, includeContinuations);
}
else
{
// Suspend the job until the continuation is ready to run
await Queue.Suspend(invocation, result.Continuation.Parameters, result.Continuation.WaitPeriod, logUrl);
}
}
}
protected internal virtual Task Dispatch(InvocationState invocation, InvocationLogCapture capture, CancellationToken cancelToken)
{
return Dispatch(invocation, capture, cancelToken, includeContinuations: false);
}
public InvocationContext(InvocationState invocation, InvocationQueue queue, CancellationToken cancelToken, InvocationLogCapture capture)
: this(invocation, queue, cancelToken)
{
_capture = capture;
}
