public void Execute(BackgroundProcessContext context) { if (context == null) { throw new ArgumentNullException(nameof(context)); } var jobsEnqueued = 0; while (EnqueueNextScheduledJob(context)) { jobsEnqueued++; if (context.IsShutdownRequested) { break; } } if (jobsEnqueued != 0) { _logger.Log($"{jobsEnqueued} scheduled job(s) enqueued."); } context.Wait(_pollingDelay); }
private void Process(BackgroundProcessContext context) { try { Logger.Info("Processing execution in-progress."); var stopwatch = new Stopwatch(); stopwatch.Start(); Process(context.CancellationToken).GetAwaiter().GetResult(); stopwatch.Stop(); Logger.Info($"Processing executed successfully in {stopwatch.ElapsedMilliseconds}ms."); var localInterval = Interval; Logger.Info($"Processing {Name} server will now sleep for {localInterval}."); var endTime = DateTimeOffset.Now; while (DateTimeOffset.Now - endTime < localInterval) { if (localInterval != Interval) { localInterval = Interval; Logger.Info($"Processing server {Name} interval was changed to {localInterval}."); } context.Wait(TimeSpan.FromSeconds(5)); } } catch (Exception e) { Logger.Warn(e, "An error occured while processing, but was handled by the processing server. The processing server will restart."); throw; } }
public void Execute(BackgroundProcessContext context) { try { var jobsEnqueued = 0; while (context.Storage.EnqueueNextScheduledItem((CalculateNextInvocation))) { jobsEnqueued++; if (context.IsShutdownRequested) { break; } } ; if (jobsEnqueued != 0) { _logger.Log($"{jobsEnqueued} scheduled job(s)/workflow(s) enqueued by Recurring Tasks Process."); } } //TODO: refactor to catch exception in storage layer catch (SqlException sqlEx) when(sqlEx.Number == 1205) { //deadlock, just skip, other process on other server is taking care of it... } context.Wait(TimeSpan.FromMinutes(1)); }
public void Execute(BackgroundProcessContext context) { context.CancellationToken.ThrowIfCancellationRequested(); var queueJob = _storage.FetchNextJob(this._queues, this._workerId); if (queueJob == null) { context.Wait(_workerFetchIdleSleep); return; } _storage.SetJobState(queueJob.JobId, new ProcessingState(context.ServerId, this._workerId)); _eventManager.RaiseOnProcessing(queueJob, context.ServerId, this._workerId); var perfomContext = new PerformContext(context.CancellationToken, queueJob); var resultState = PerformJob(perfomContext); if (resultState is FailedState) { var failedState = resultState as FailedState; if (queueJob.RetryCount < queueJob.MaxRetries) { //schedule new run var nextRun = DateTime.UtcNow.AddSeconds(SecondsToDelay(queueJob.RetryCount)); const int maxMessageLength = 50; var exceptionMessage = failedState.Exception.Message.Length > maxMessageLength ? failedState.Exception.Message.Substring(0, maxMessageLength - 1) + "…" : failedState.Exception.Message; var scheduledState = new ScheduledState(nextRun) { Reason = $"Retry attempt { queueJob.RetryCount } of { queueJob.MaxRetries }: { exceptionMessage}" }; _storage.UpgradeFailedToScheduled(queueJob.JobId, resultState, scheduledState, nextRun, queueJob.RetryCount + 1); _eventManager.RaiseOnReschedule(queueJob, context.ServerId, this._workerId, failedState.Exception, nextRun); } else { //final failed state _storage.SetJobState(queueJob.JobId, resultState); _eventManager.RaiseOnFail(queueJob, context.ServerId, this._workerId, failedState.Exception); if (queueJob.WorkflowId.HasValue) { //mark dependent jobs consequently failed _storage.MarkConsequentlyFailedJobs(queueJob.JobId); } } } else { //Succeeded _storage.SetJobState(queueJob.JobId, resultState); _storage.ExpireJob(queueJob.JobId); _storage.EnqueueAwaitingWorkflowJobs(queueJob.JobId); _eventManager.RaiseOnSuccess(queueJob, context.ServerId, this._workerId); } _storage.RemoveFromQueue(queueJob.QueueJobId); }
public void Execute(BackgroundProcessContext context) { var serversRemoved = context.Storage.RemoveTimedOutServers(_serverTimeout); if (serversRemoved != 0) { _logger.Log(LogLevel.Information, $"{serversRemoved} servers were removed due to timeout"); } context.Wait(_checkInterval); }
public void Execute([NotNull] BackgroundProcessContext context) { using (var connetion = context.Storage.GetConnection()) { //申请分布式锁 using (connetion.AcquireDistributedLock($"{_backWorker.JobName}:secondsJob", TimeSpan.FromSeconds(_backWorker.Internal))) { BackgroundJob.Enqueue <DoTest>(job => job.SendRequest(_backWorker.JobName, _backWorker, null)); } } context.Wait(TimeSpan.FromSeconds(_backWorker.Internal)); }
public void Execute(BackgroundProcessContext context) { using (var connection = context.Storage.GetConnection()) { var cpuPercentUsage = ComputeCpuUsage(context.CancellationToken); if (context.IsShutdownRequested) { CleanupState(context, connection); return; } var values = new Dictionary <string, string> { [ProcessId] = _currentProcess.Id.ToString(CultureInfo.InvariantCulture), [ProcessName] = _currentProcess.ProcessName, [CpuUsage] = cpuPercentUsage.ToString(CultureInfo.InvariantCulture), [WorkingSet] = _currentProcess.WorkingSet64.ToString(CultureInfo.InvariantCulture), [Timestamp] = DateTimeOffset.UtcNow.ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture) }; using (var transaction = connection.CreateWriteTransaction()) { var hashKey = Utils.FormatKey(context.ServerId); transaction.SetRangeInHash(hashKey, values); if (transaction is JobStorageTransaction jobStorageTransaction) { // set expiration (if supported by storage) jobStorageTransaction.ExpireHash(hashKey, _expireIn); } transaction.Commit(); } } if (_checkInterval != TimeSpan.Zero) { context.Wait(_checkInterval); } }
public void Execute(BackgroundProcessContext context) { var monitoringApi = context.Storage.GetMonitoringApi(); if (monitoringApi.EnqueuedCount("default") > _count) { context.Wait(_delay); return; } var client = new BackgroundJobClient(context.Storage) { RetryAttempts = 15 }; for (var i = 0; i < _count; i++) { context.StoppingToken.ThrowIfCancellationRequested(); client.Enqueue <IHarnessV1>(x => x.Perform(0)); } }
public void Execute(BackgroundProcessContext context) { for (var i = 0; i <= MaxRetryAttempts; i++) { try { InnerProcess.Execute(context); return; } catch (Exception ex) { if (ex is OperationCanceledException && context.IsShutdownRequested) { throw; } // Break the loop after the retry attempts number exceeded. if (i >= MaxRetryAttempts - 1) { throw; } var nextTry = DelayCallback(i); var logLevel = GetLogLevel(i); _logger.Log( logLevel, // ReSharper disable once AccessToModifiedClosure () => $"Error occurred during execution of '{InnerProcess}' process. Execution will be retried (attempt #{i + 1}) in {nextTry} seconds.", ex); context.Wait(nextTry); if (context.IsShutdownRequested) { break; } } } }
public void Execute(BackgroundProcessContext context) { var api = context.Storage.GetMonitoringApi(); var queues = api.Queues(); _logger.Trace("Checking queue length to decide whether to change gate levels..."); if (queues.Any(queue => _queues.Contains(queue.Name) && queue.Length > 0)) { if (_gate.TryIncreaseLevel(out var level)) { _logger.Debug($"Gate level for queues ({_queueNames}) increased to {level} / {_gate.MaxLevel}"); } } else { if (_gate.TryDecreaseLevel(out var level)) { _logger.Debug($"Gate level for queues ({_queueNames}) decreased to {level} / {_gate.MaxLevel}"); } } context.Wait(_delay); }
public void Execute(BackgroundProcessContext context) { for (var i = 0; i <= MaxRetryAttempts; i++) { try { _innerProcess.Execute(context); return; } catch (Exception ex) { if (ex is OperationCanceledException && context.IsShutdownRequested) { throw; } // Break the loop after the retry attempts number exceeded. if (i >= MaxRetryAttempts - 1) { _logger.Log(LogLevel.Error, $"Error occurred during execution of '{_innerProcess}' process. No more retries. Exiting background process.", ex); throw; } var nextTry = DelayCallback(i); _logger.Log(LogLevel.Warning, $"Error occurred during execution of '{_innerProcess}' process. Execution will be retried (attempt #{i + 1}) in {nextTry} seconds.", ex); context.Wait(nextTry); if (context.IsShutdownRequested) { break; } } } }
//https://www.hangfire.io/overview.html public void Execute([NotNull] BackgroundProcessContext context) { context.Wait(TimeSpan.FromHours(1)); }
public void Execute(BackgroundProcessContext context) { this.GetValue(); context.Wait(new TimeSpan(0, 0, 5)); }
public void Execute(BackgroundProcessContext context) { context.Storage.HeartbeatServer(context.ServerId, JobHelper.ToJson(context.ServerContext)); context.Wait(_heartbeatInterval); }