Beispiel #1
0
        public void Execute(BackgroundProcessContext context)
        {
            if (context == null)
            {
                throw new ArgumentNullException(nameof(context));
            }

            var jobsEnqueued = 0;

            while (EnqueueNextScheduledJob(context))
            {
                jobsEnqueued++;

                if (context.IsShutdownRequested)
                {
                    break;
                }
            }

            if (jobsEnqueued != 0)
            {
                _logger.Log($"{jobsEnqueued} scheduled job(s) enqueued.");
            }

            context.Wait(_pollingDelay);
        }
Beispiel #2
0
        private void Process(BackgroundProcessContext context)
        {
            try
            {
                Logger.Info("Processing execution in-progress.");

                var stopwatch = new Stopwatch();
                stopwatch.Start();
                Process(context.CancellationToken).GetAwaiter().GetResult();
                stopwatch.Stop();

                Logger.Info($"Processing executed successfully in {stopwatch.ElapsedMilliseconds}ms.");

                var localInterval = Interval;
                Logger.Info($"Processing {Name} server will now sleep for {localInterval}.");

                var endTime = DateTimeOffset.Now;
                while (DateTimeOffset.Now - endTime < localInterval)
                {
                    if (localInterval != Interval)
                    {
                        localInterval = Interval;
                        Logger.Info($"Processing server {Name} interval was changed to {localInterval}.");
                    }

                    context.Wait(TimeSpan.FromSeconds(5));
                }
            }
            catch (Exception e)
            {
                Logger.Warn(e, "An error occured while processing, but was handled by the processing server. The processing server will restart.");
                throw;
            }
        }
        public void Execute(BackgroundProcessContext context)
        {
            try
            {
                var jobsEnqueued = 0;
                while (context.Storage.EnqueueNextScheduledItem((CalculateNextInvocation)))
                {
                    jobsEnqueued++;

                    if (context.IsShutdownRequested)
                    {
                        break;
                    }
                }
                ;
                if (jobsEnqueued != 0)
                {
                    _logger.Log($"{jobsEnqueued} scheduled job(s)/workflow(s) enqueued by Recurring Tasks Process.");
                }
            }
            //TODO: refactor to catch exception in storage layer
            catch (SqlException sqlEx) when(sqlEx.Number == 1205)
            {
                //deadlock, just skip, other process on other server is taking care of it...
            }
            context.Wait(TimeSpan.FromMinutes(1));
        }
Beispiel #4
0
        public void Execute(BackgroundProcessContext context)
        {
            context.CancellationToken.ThrowIfCancellationRequested();
            var queueJob = _storage.FetchNextJob(this._queues, this._workerId);

            if (queueJob == null)
            {
                context.Wait(_workerFetchIdleSleep);
                return;
            }
            _storage.SetJobState(queueJob.JobId, new ProcessingState(context.ServerId, this._workerId));
            _eventManager.RaiseOnProcessing(queueJob, context.ServerId, this._workerId);
            var perfomContext = new PerformContext(context.CancellationToken, queueJob);
            var resultState   = PerformJob(perfomContext);

            if (resultState is FailedState)
            {
                var failedState = resultState as FailedState;
                if (queueJob.RetryCount < queueJob.MaxRetries)
                {
                    //schedule new run
                    var nextRun = DateTime.UtcNow.AddSeconds(SecondsToDelay(queueJob.RetryCount));

                    const int maxMessageLength = 50;
                    var       exceptionMessage = failedState.Exception.Message.Length > maxMessageLength
                        ? failedState.Exception.Message.Substring(0, maxMessageLength - 1) + "…"
                        : failedState.Exception.Message;

                    var scheduledState = new ScheduledState(nextRun)
                    {
                        Reason = $"Retry attempt { queueJob.RetryCount } of { queueJob.MaxRetries }: { exceptionMessage}"
                    };
                    _storage.UpgradeFailedToScheduled(queueJob.JobId, resultState, scheduledState, nextRun, queueJob.RetryCount + 1);
                    _eventManager.RaiseOnReschedule(queueJob, context.ServerId, this._workerId, failedState.Exception, nextRun);
                }
                else
                {
                    //final failed state
                    _storage.SetJobState(queueJob.JobId, resultState);
                    _eventManager.RaiseOnFail(queueJob, context.ServerId, this._workerId, failedState.Exception);

                    if (queueJob.WorkflowId.HasValue)
                    {
                        //mark dependent jobs consequently failed
                        _storage.MarkConsequentlyFailedJobs(queueJob.JobId);
                    }
                }
            }
            else
            {
                //Succeeded
                _storage.SetJobState(queueJob.JobId, resultState);
                _storage.ExpireJob(queueJob.JobId);
                _storage.EnqueueAwaitingWorkflowJobs(queueJob.JobId);
                _eventManager.RaiseOnSuccess(queueJob, context.ServerId, this._workerId);
            }
            _storage.RemoveFromQueue(queueJob.QueueJobId);
        }
Beispiel #5
0
        public void Execute(BackgroundProcessContext context)
        {
            var serversRemoved = context.Storage.RemoveTimedOutServers(_serverTimeout);

            if (serversRemoved != 0)
            {
                _logger.Log(LogLevel.Information, $"{serversRemoved} servers were removed due to timeout");
            }

            context.Wait(_checkInterval);
        }
 public void Execute([NotNull] BackgroundProcessContext context)
 {
     using (var connetion = context.Storage.GetConnection())
     {
         //申请分布式锁
         using (connetion.AcquireDistributedLock($"{_backWorker.JobName}:secondsJob", TimeSpan.FromSeconds(_backWorker.Internal)))
         {
             BackgroundJob.Enqueue <DoTest>(job => job.SendRequest(_backWorker.JobName, _backWorker, null));
         }
     }
     context.Wait(TimeSpan.FromSeconds(_backWorker.Internal));
 }
        public void Execute(BackgroundProcessContext context)
        {
            using (var connection = context.Storage.GetConnection())
            {
                var cpuPercentUsage = ComputeCpuUsage(context.CancellationToken);

                if (context.IsShutdownRequested)
                {
                    CleanupState(context, connection);
                    return;
                }

                var values = new Dictionary <string, string>
                {
                    [ProcessId]   = _currentProcess.Id.ToString(CultureInfo.InvariantCulture),
                    [ProcessName] = _currentProcess.ProcessName,
                    [CpuUsage]    = cpuPercentUsage.ToString(CultureInfo.InvariantCulture),
                    [WorkingSet]  = _currentProcess.WorkingSet64.ToString(CultureInfo.InvariantCulture),
                    [Timestamp]   = DateTimeOffset.UtcNow.ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture)
                };

                using (var transaction = connection.CreateWriteTransaction())
                {
                    var hashKey = Utils.FormatKey(context.ServerId);

                    transaction.SetRangeInHash(hashKey, values);

                    if (transaction is JobStorageTransaction jobStorageTransaction)
                    {
                        // set expiration (if supported by storage)
                        jobStorageTransaction.ExpireHash(hashKey, _expireIn);
                    }

                    transaction.Commit();
                }
            }

            if (_checkInterval != TimeSpan.Zero)
            {
                context.Wait(_checkInterval);
            }
        }
Beispiel #8
0
        public void Execute(BackgroundProcessContext context)
        {
            var monitoringApi = context.Storage.GetMonitoringApi();

            if (monitoringApi.EnqueuedCount("default") > _count)
            {
                context.Wait(_delay);
                return;
            }

            var client = new BackgroundJobClient(context.Storage)
            {
                RetryAttempts = 15
            };

            for (var i = 0; i < _count; i++)
            {
                context.StoppingToken.ThrowIfCancellationRequested();
                client.Enqueue <IHarnessV1>(x => x.Perform(0));
            }
        }
Beispiel #9
0
        public void Execute(BackgroundProcessContext context)
        {
            for (var i = 0; i <= MaxRetryAttempts; i++)
            {
                try
                {
                    InnerProcess.Execute(context);
                    return;
                }
                catch (Exception ex)
                {
                    if (ex is OperationCanceledException && context.IsShutdownRequested)
                    {
                        throw;
                    }

                    // Break the loop after the retry attempts number exceeded.
                    if (i >= MaxRetryAttempts - 1)
                    {
                        throw;
                    }

                    var nextTry  = DelayCallback(i);
                    var logLevel = GetLogLevel(i);

                    _logger.Log(
                        logLevel,
                        // ReSharper disable once AccessToModifiedClosure
                        () => $"Error occurred during execution of '{InnerProcess}' process. Execution will be retried (attempt #{i + 1}) in {nextTry} seconds.",
                        ex);

                    context.Wait(nextTry);

                    if (context.IsShutdownRequested)
                    {
                        break;
                    }
                }
            }
        }
Beispiel #10
0
        public void Execute(BackgroundProcessContext context)
        {
            var api    = context.Storage.GetMonitoringApi();
            var queues = api.Queues();

            _logger.Trace("Checking queue length to decide whether to change gate levels...");

            if (queues.Any(queue => _queues.Contains(queue.Name) && queue.Length > 0))
            {
                if (_gate.TryIncreaseLevel(out var level))
                {
                    _logger.Debug($"Gate level for queues ({_queueNames}) increased to {level} / {_gate.MaxLevel}");
                }
            }
            else
            {
                if (_gate.TryDecreaseLevel(out var level))
                {
                    _logger.Debug($"Gate level for queues ({_queueNames}) decreased to {level} / {_gate.MaxLevel}");
                }
            }

            context.Wait(_delay);
        }
Beispiel #11
0
        public void Execute(BackgroundProcessContext context)
        {
            for (var i = 0; i <= MaxRetryAttempts; i++)
            {
                try
                {
                    _innerProcess.Execute(context);
                    return;
                }
                catch (Exception ex)
                {
                    if (ex is OperationCanceledException && context.IsShutdownRequested)
                    {
                        throw;
                    }

                    // Break the loop after the retry attempts number exceeded.
                    if (i >= MaxRetryAttempts - 1)
                    {
                        _logger.Log(LogLevel.Error, $"Error occurred during execution of '{_innerProcess}' process. No more retries. Exiting background process.", ex);

                        throw;
                    }

                    var nextTry = DelayCallback(i);
                    _logger.Log(LogLevel.Warning, $"Error occurred during execution of '{_innerProcess}' process. Execution will be retried (attempt #{i + 1}) in {nextTry} seconds.", ex);

                    context.Wait(nextTry);

                    if (context.IsShutdownRequested)
                    {
                        break;
                    }
                }
            }
        }
Beispiel #12
0
 //https://www.hangfire.io/overview.html
 public void Execute([NotNull] BackgroundProcessContext context)
 {
     context.Wait(TimeSpan.FromHours(1));
 }
 public void Execute(BackgroundProcessContext context)
 {
     this.GetValue();
     context.Wait(new TimeSpan(0, 0, 5));
 }
Beispiel #14
0
        public void Execute(BackgroundProcessContext context)
        {
            context.Storage.HeartbeatServer(context.ServerId, JobHelper.ToJson(context.ServerContext));

            context.Wait(_heartbeatInterval);
        }