Contains all information about the current status of an invocation, serves as the central status record of an invocation
Example #1
0
        protected internal virtual Task Dispatch(InvocationState invocation, CancellationToken cancelToken)
        {
            var logCapture = _logContainer == null ?
                             new InvocationLogCapture(invocation) :
                             new BlobInvocationLogCapture(invocation, _logContainer);

            return(Dispatch(invocation, logCapture, cancelToken));
        }
Example #2
0
 private bool ProcessResult(InvocationState invocation, InvocationState.InvocationRow result)
 {
     if (result == null)
     {
         return(false);
     }
     else
     {
         invocation.Update(result);
         return(true);
     }
 }
Example #3
0
        public virtual async Task <bool> UpdateStatus(InvocationState invocation, InvocationStatus status, ExecutionResult result)
        {
            var newVersion = await ConnectAndExec(
                "work.SetInvocationStatus",
                new
            {
                Id           = invocation.Id,
                Version      = invocation.CurrentVersion,
                Status       = status,
                Result       = result,
                InstanceName = InstanceName
            });

            return(ProcessResult(invocation, newVersion));
        }
Example #4
0
        /// <summary>
        /// Extends the visibility timeout of the request. That is, the time during which the
        /// queue message is hidden from other clients
        /// </summary>
        /// <param name="request">The request to extend</param>
        /// <param name="duration">The duration from the time of invocation to hide the message</param>
        public virtual async Task <bool> Extend(InvocationState invocation, TimeSpan duration)
        {
            var invisibleUntil = _clock.UtcNow + duration;
            var newVersion     = await ConnectAndExec(
                "work.ExtendInvocation",
                new
            {
                Id           = invocation.Id,
                Version      = invocation.CurrentVersion,
                ExtendTo     = invisibleUntil.UtcDateTime,
                InstanceName = InstanceName
            });

            return(ProcessResult(invocation, newVersion));
        }
Example #5
0
        /// <summary>
        /// Acknowledges that the request has completed successfully, removing the message from the queue.
        /// </summary>
        /// <param name="request">The request to acknowledge</param>
        /// <returns>A boolean indicating if the request was successful. Failure indicates that another node has updated the invocation since we last checked</returns>
        public virtual async Task <bool> Complete(InvocationState invocation, ExecutionResult result, string resultMessage, string logUrl)
        {
            // Try to complete the row
            var newVersion = await ConnectAndExec(
                "work.CompleteInvocation",
                new
            {
                Id            = invocation.Id,
                Version       = invocation.CurrentVersion,
                Result        = (int)result,
                ResultMessage = resultMessage,
                LogUrl        = logUrl,
                InstanceName  = InstanceName
            });

            return(ProcessResult(invocation, newVersion));
        }
        public IObservable <EventEntry> RunJob(string job, string payload)
        {
            var runner = new JobRunner(
                new JobDispatcher(
                    GetAllAvailableJobs(),
                    Container),
                InvocationQueue.Null,
                Container.Resolve <ConfigurationHub>(),
                Clock.RealClock);

            var invocation =
                new InvocationState(
                    new InvocationState.InvocationRow()
            {
                Payload       = payload,
                Status        = (int)InvocationStatus.Executing,
                Result        = (int)ExecutionResult.Incomplete,
                Source        = Constants.Source_LocalJob,
                Id            = Guid.NewGuid(),
                Job           = job,
                UpdatedBy     = Environment.MachineName,
                UpdatedAt     = DateTime.UtcNow,
                QueuedAt      = DateTime.UtcNow,
                NextVisibleAt = DateTime.UtcNow + TimeSpan.FromMinutes(5)
            });

            return(Observable.Create <EventEntry>(observer =>
            {
                var capture = new InvocationLogCapture(invocation);
                capture.Subscribe(e => observer.OnNext(e), ex => observer.OnError(ex));
                runner.Dispatch(invocation, capture, CancellationToken.None, includeContinuations: true).ContinueWith(t =>
                {
                    if (t.IsFaulted)
                    {
                        observer.OnError(t.Exception);
                    }
                    else
                    {
                        observer.OnCompleted();
                    }
                    return t;
                });
                return () => { }; // No action on unsubscribe
            }));
        }
Example #7
0
        public virtual async Task <bool> Suspend(InvocationState invocation, Dictionary <string, string> newPayload, TimeSpan suspendFor, string logUrl)
        {
            var suspendUntil      = _clock.UtcNow + suspendFor;
            var serializedPayload = InvocationPayloadSerializer.Serialize(newPayload);
            var newVersion        = await ConnectAndExec(
                "work.SuspendInvocation",
                new
            {
                Id           = invocation.Id,
                Version      = invocation.CurrentVersion,
                Payload      = serializedPayload,
                SuspendUntil = suspendUntil.UtcDateTime,
                LogUrl       = logUrl,
                InstanceName = InstanceName
            });

            return(ProcessResult(invocation, newVersion));
        }
Example #8
0
 public override Task <bool> Complete(InvocationState invocation, ExecutionResult result, string resultMessage, string logUrl)
 {
     return(Task.FromResult(true));
 }
 private bool ProcessResult(InvocationState invocation, InvocationState.InvocationRow result)
 {
     if (result == null)
     {
         return false;
     }
     else
     {
         invocation.Update(result);
         return true;
     }
 }
 public InvocationContext(InvocationState invocation, InvocationQueue queue, CancellationToken cancelToken, InvocationLogCapture capture)
     : this(invocation, queue, cancelToken)
 {
     _capture = capture;
 }
 public InvocationContext(InvocationState invocation, InvocationQueue queue)
     : this(invocation, queue, CancellationToken.None)
 {
 }
Example #12
0
 private Task <InvocationState> EnqueueRepeat(InvocationState repeat, InvocationResult result)
 {
     return(Queue.Enqueue(repeat.Job, Constants.Source_RepeatingJob, repeat.Payload, result.RescheduleIn.Value));
 }
Example #13
0
 protected internal virtual Task Dispatch(InvocationState invocation, InvocationLogCapture capture, CancellationToken cancelToken)
 {
     return(Dispatch(invocation, capture, cancelToken, includeContinuations: false));
 }
Example #14
0
 public override Task <bool> UpdateStatus(InvocationState invocation, InvocationStatus status, ExecutionResult result)
 {
     return(Task.FromResult(true));
 }
Example #15
0
        protected internal virtual async Task Dispatch(InvocationState invocation, InvocationLogCapture capture, CancellationToken cancelToken, bool includeContinuations)
        {
            InvocationContext.SetCurrentInvocationId(invocation.Id);
            
            if (invocation.IsContinuation)
            {
                InvocationEventSource.Log.Resumed();
            }
            else
            {
                InvocationEventSource.Log.Started();
            }
            
            // Record that we are executing the job
            if (!await Queue.UpdateStatus(invocation, InvocationStatus.Executing, ExecutionResult.Incomplete))
            {
                InvocationEventSource.Log.Aborted(invocation);
                return;
            }

            // Create the request.Invocation context and start capturing the logs
            await capture.Start();
            var context = new InvocationContext(invocation, Queue, cancelToken, capture);

            InvocationResult result = null;
            try
            {
                result = await Dispatcher.Dispatch(context);

                // TODO: If response.Continuation != null, enqueue continuation
                switch (result.Result)
                {
                    case ExecutionResult.Completed:
                        InvocationEventSource.Log.Succeeded(result);
                        break;
                    case ExecutionResult.Faulted:
                        InvocationEventSource.Log.Faulted(result);
                        break;
                    case ExecutionResult.Incomplete:
                        InvocationEventSource.Log.Suspended(result);
                        break;
                    default:
                        InvocationEventSource.Log.UnknownStatus(result);
                        break;
                }

                if (invocation.NextVisibleAt < Clock.UtcNow)
                {
                    InvocationEventSource.Log.InvocationTookTooLong(invocation);
                }
            }
            catch (Exception ex)
            {
                InvocationEventSource.Log.DispatchError(ex);
                result = new InvocationResult(ExecutionResult.Crashed, ex);
            }

            // Stop capturing and set the log url
            string logUrl = null;
            if (capture != null)
            {
                var logUri = await capture.End();
                if (logUri != null)
                {
                    logUrl = logUri.AbsoluteUri;
                }
            }

            if (result.Result != ExecutionResult.Incomplete)
            {
                // If we're not suspended, the invocation has completed
                InvocationEventSource.Log.Ended();
                await Queue.Complete(invocation, result.Result, result.Exception == null ? null : result.Exception.ToString(), logUrl);

                // If we've completed and there's a repeat, queue the repeat
                if (result.RescheduleIn != null)
                {
                    // Rescheule it to run again
                    var repeat = await EnqueueRepeat(invocation, result);
                    InvocationEventSource.Log.ScheduledRepeat(invocation, repeat, result.RescheduleIn.Value);
                }
            }
            else
            {
                if (includeContinuations)
                {
                    invocation.Update(new InvocationState.InvocationRow()
                    {
                        Id = Guid.NewGuid(),
                        Version = invocation.CurrentVersion + 1,
                        Job = invocation.Job,
                        Source = invocation.Id.ToString("N"),
                        Status = (int)InvocationStatus.Suspended,
                        Result = (int)ExecutionResult.Incomplete,
                        LastDequeuedAt = invocation.LastDequeuedAt == null ? (DateTime?)null : invocation.LastDequeuedAt.Value.UtcDateTime,
                        LastSuspendedAt = DateTime.UtcNow,
                        CompletedAt = null,
                        QueuedAt = invocation.QueuedAt.UtcDateTime,
                        NextVisibleAt = invocation.NextVisibleAt.UtcDateTime + DefaultInvisibilityPeriod,
                        UpdatedAt = invocation.UpdatedAt.UtcDateTime,
                        Payload = InvocationPayloadSerializer.Serialize(result.Continuation.Parameters),
                        IsContinuation = true
                    });

                    // Run the continuation inline after waiting
                    await Task.Delay(result.Continuation.WaitPeriod);

                    await Dispatch(invocation, capture, cancelToken, includeContinuations);
                }
                else
                {
                    // Suspend the job until the continuation is ready to run
                    await Queue.Suspend(invocation, result.Continuation.Parameters, result.Continuation.WaitPeriod, logUrl);
                }
            }
        }
Example #16
0
 protected internal virtual Task Dispatch(InvocationState invocation, InvocationLogCapture capture, CancellationToken cancelToken)
 {
     return Dispatch(invocation, capture, cancelToken, includeContinuations: false);
 }
Example #17
0
 protected internal virtual Task Dispatch(InvocationState invocation, CancellationToken cancelToken)
 {
     var logCapture = _logContainer == null ?
         new InvocationLogCapture(invocation) :
         new BlobInvocationLogCapture(invocation, _logContainer);
     return Dispatch(invocation, logCapture, cancelToken);
 }
 public override Task<bool> UpdateStatus(InvocationState invocation, InvocationStatus status, ExecutionResult result)
 {
     return Task.FromResult(true);
 }
 public override Task<bool> Suspend(InvocationState invocation, Dictionary<string, string> newPayload, TimeSpan suspendFor, string logUrl)
 {
     return Task.FromResult(true);
 }
 public override Task<bool> Extend(InvocationState invocation, TimeSpan duration)
 {
     return Task.FromResult(true);
 }
 public override Task<bool> Complete(InvocationState invocation, ExecutionResult result, string resultMessage, string logUrl)
 {
     return Task.FromResult(true);
 }
Example #22
0
 public override Task <bool> Extend(InvocationState invocation, TimeSpan duration)
 {
     return(Task.FromResult(true));
 }
Example #23
0
 public override Task <bool> Suspend(InvocationState invocation, Dictionary <string, string> newPayload, TimeSpan suspendFor, string logUrl)
 {
     return(Task.FromResult(true));
 }
Example #24
0
 private Task<InvocationState> EnqueueRepeat(InvocationState repeat, InvocationResult result)
 {
     return Queue.Enqueue(repeat.Job, Constants.Source_RepeatingJob, repeat.Payload, result.RescheduleIn.Value);
 }
 public InvocationContext(InvocationState invocation, InvocationQueue queue, CancellationToken cancelToken, InvocationLogCapture capture)
     : this(invocation, queue, cancelToken)
 {
     _capture = capture;
 }
 public virtual async Task<bool> Suspend(InvocationState invocation, Dictionary<string, string> newPayload, TimeSpan suspendFor, string logUrl)
 {
     var suspendUntil = _clock.UtcNow + suspendFor;
     var serializedPayload = InvocationPayloadSerializer.Serialize(newPayload);
     var newVersion = await ConnectAndExec(
         "work.SuspendInvocation",
         new
         {
             Id = invocation.Id,
             Version = invocation.CurrentVersion,
             Payload = serializedPayload,
             SuspendUntil = suspendUntil.UtcDateTime,
             LogUrl = logUrl,
             InstanceName = InstanceName
         });
     return ProcessResult(invocation, newVersion);
 }
Example #27
0
        protected internal virtual async Task Dispatch(InvocationState invocation, InvocationLogCapture capture, CancellationToken cancelToken, bool includeContinuations)
        {
            InvocationContext.SetCurrentInvocationId(invocation.Id);

            if (invocation.IsContinuation)
            {
                InvocationEventSource.Log.Resumed();
            }
            else
            {
                InvocationEventSource.Log.Started();
            }

            // Record that we are executing the job
            if (!await Queue.UpdateStatus(invocation, InvocationStatus.Executing, ExecutionResult.Incomplete))
            {
                InvocationEventSource.Log.Aborted(invocation);
                return;
            }

            // Create the request.Invocation context and start capturing the logs
            await capture.Start();

            var context = new InvocationContext(invocation, Queue, cancelToken, capture);

            InvocationResult result = null;

            try
            {
                result = await Dispatcher.Dispatch(context);

                // TODO: If response.Continuation != null, enqueue continuation
                switch (result.Result)
                {
                case ExecutionResult.Completed:
                    InvocationEventSource.Log.Succeeded(result);
                    break;

                case ExecutionResult.Faulted:
                    InvocationEventSource.Log.Faulted(result);
                    break;

                case ExecutionResult.Incomplete:
                    InvocationEventSource.Log.Suspended(result);
                    break;

                default:
                    InvocationEventSource.Log.UnknownStatus(result);
                    break;
                }

                if (invocation.NextVisibleAt < Clock.UtcNow)
                {
                    InvocationEventSource.Log.InvocationTookTooLong(invocation);
                }
            }
            catch (Exception ex)
            {
                InvocationEventSource.Log.DispatchError(ex);
                result = new InvocationResult(ExecutionResult.Crashed, ex);
            }

            // Stop capturing and set the log url
            string logUrl = null;

            if (capture != null)
            {
                var logUri = await capture.End();

                if (logUri != null)
                {
                    logUrl = logUri.AbsoluteUri;
                }
            }

            if (result.Result != ExecutionResult.Incomplete)
            {
                // If we're not suspended, the invocation has completed
                InvocationEventSource.Log.Ended();
                await Queue.Complete(invocation, result.Result, result.Exception == null?null : result.Exception.ToString(), logUrl);

                // If we've completed and there's a repeat, queue the repeat
                if (result.RescheduleIn != null)
                {
                    // Rescheule it to run again
                    var repeat = await EnqueueRepeat(invocation, result);

                    InvocationEventSource.Log.ScheduledRepeat(invocation, repeat, result.RescheduleIn.Value);
                }
            }
            else
            {
                if (includeContinuations)
                {
                    invocation.Update(new InvocationState.InvocationRow()
                    {
                        Id              = Guid.NewGuid(),
                        Version         = invocation.CurrentVersion + 1,
                        Job             = invocation.Job,
                        Source          = invocation.Id.ToString("N"),
                        Status          = (int)InvocationStatus.Suspended,
                        Result          = (int)ExecutionResult.Incomplete,
                        LastDequeuedAt  = invocation.LastDequeuedAt == null ? (DateTime?)null : invocation.LastDequeuedAt.Value.UtcDateTime,
                        LastSuspendedAt = DateTime.UtcNow,
                        CompletedAt     = null,
                        QueuedAt        = invocation.QueuedAt.UtcDateTime,
                        NextVisibleAt   = invocation.NextVisibleAt.UtcDateTime + DefaultInvisibilityPeriod,
                        UpdatedAt       = invocation.UpdatedAt.UtcDateTime,
                        Payload         = InvocationPayloadSerializer.Serialize(result.Continuation.Parameters),
                        IsContinuation  = true
                    });

                    // Run the continuation inline after waiting
                    await Task.Delay(result.Continuation.WaitPeriod);

                    await Dispatch(invocation, capture, cancelToken, includeContinuations);
                }
                else
                {
                    // Suspend the job until the continuation is ready to run
                    await Queue.Suspend(invocation, result.Continuation.Parameters, result.Continuation.WaitPeriod, logUrl);
                }
            }
        }
 public InvocationContext(InvocationState invocation, InvocationQueue queue, CancellationToken cancelToken)
 {
     Invocation  = invocation;
     Queue       = queue;
     CancelToken = cancelToken;
 }
Example #29
0
        public virtual async Task Run(CancellationToken cancelToken)
        {
            WorkServiceEventSource.Log.ReinitializingInvocationState(Queue.InstanceName);
            await Queue.ReinitializeInvocationState();

            WorkServiceEventSource.Log.ReinitializedInvocationState();


            WorkServiceEventSource.Log.DispatchLoopStarted();
            try
            {
                while (!cancelToken.IsCancellationRequested)
                {
                    InvocationState invocation = null;
                    try
                    {
                        Status     = RunnerStatus.Dequeuing;
                        invocation = await Queue.Dequeue(DefaultInvisibilityPeriod, cancelToken);

                        Status = RunnerStatus.Working;
                    }
                    catch (Exception ex)
                    {
                        WorkServiceEventSource.Log.ErrorRetrievingInvocation(ex);
                    }

                    // Check Cancellation
                    if (cancelToken.IsCancellationRequested)
                    {
                        break;
                    }

                    if (invocation == null)
                    {
                        Status = RunnerStatus.Sleeping;
                        WorkServiceEventSource.Log.DispatchLoopWaiting(_pollInterval);
                        await Clock.Delay(_pollInterval, cancelToken);

                        WorkServiceEventSource.Log.DispatchLoopResumed();
                        Status = RunnerStatus.Working;
                    }
                    else if (invocation.Status == InvocationStatus.Cancelled)
                    {
                        // Job was cancelled by the user, so just continue.
                        WorkServiceEventSource.Log.Cancelled(invocation);
                    }
                    else
                    {
                        Status = RunnerStatus.Dispatching;
                        _currentInvocationId = invocation.Id.ToByteArray();

                        Exception dispatchError = null;
                        try
                        {
                            await Dispatch(invocation, cancelToken);
                        }
                        catch (Exception ex)
                        {
                            dispatchError = ex;
                        }
                        if (dispatchError != null)
                        {
                            await Queue.Complete(
                                invocation,
                                ExecutionResult.Crashed,
                                dispatchError.ToString(),
                                null);
                        }
                        _currentInvocationId = null;
                        _lastInvocationId    = invocation.Id.ToByteArray();
                        Status = RunnerStatus.Working;
                    }
                }
                Status = RunnerStatus.Stopping;
            }
            catch (Exception ex)
            {
                WorkServiceEventSource.Log.DispatchLoopError(ex);
                Status = RunnerStatus.Error;
                _error = ex;
                throw;
            }
            WorkServiceEventSource.Log.DispatchLoopEnded();
        }
        /// <summary>
        /// Extends the visibility timeout of the request. That is, the time during which the 
        /// queue message is hidden from other clients
        /// </summary>
        /// <param name="request">The request to extend</param>
        /// <param name="duration">The duration from the time of invocation to hide the message</param>
        public virtual async Task<bool> Extend(InvocationState invocation, TimeSpan duration)
        {
            var invisibleUntil = _clock.UtcNow + duration;
            var newVersion = await ConnectAndExec(
                "work.ExtendInvocation",
                new
                {
                    Id = invocation.Id,
                    Version = invocation.CurrentVersion,
                    ExtendTo = invisibleUntil.UtcDateTime,
                    InstanceName = InstanceName
                });

            return ProcessResult(invocation, newVersion);
        }
 public InvocationContext(InvocationState invocation, InvocationQueue queue, CancellationToken cancelToken)
 {
     Invocation = invocation;
     Queue = queue;
     CancelToken = cancelToken;
 }
 /// <summary>
 /// Acknowledges that the request has completed successfully, removing the message from the queue.
 /// </summary>
 /// <param name="request">The request to acknowledge</param>
 /// <returns>A boolean indicating if the request was successful. Failure indicates that another node has updated the invocation since we last checked</returns>
 public virtual async Task<bool> Complete(InvocationState invocation, ExecutionResult result, string resultMessage, string logUrl)
 {
     // Try to complete the row
     var newVersion = await ConnectAndExec(
         "work.CompleteInvocation",
         new
         {
             Id = invocation.Id,
             Version = invocation.CurrentVersion,
             Result = (int)result,
             ResultMessage = resultMessage,
             LogUrl = logUrl,
             InstanceName = InstanceName
         });
     return ProcessResult(invocation, newVersion);
 }
 public InvocationContext(InvocationState invocation, InvocationQueue queue)
     : this(invocation, queue, CancellationToken.None)
 {
 }
 public virtual async Task<bool> UpdateStatus(InvocationState invocation, InvocationStatus status, ExecutionResult result)
 {
     var newVersion = await ConnectAndExec(
         "work.SetInvocationStatus",
         new
         {
             Id = invocation.Id,
             Version = invocation.CurrentVersion,
             Status = status,
             Result = result,
             InstanceName = InstanceName
         });
     return ProcessResult(invocation, newVersion);
 }