public void Execute(BackgroundProcessContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
var jobsEnqueued = 0;
while (EnqueueNextScheduledJob(context))
{
jobsEnqueued++;
if (context.IsShutdownRequested)
{
break;
}
}
if (jobsEnqueued != 0)
{
_logger.Log($"{jobsEnqueued} scheduled job(s) enqueued.");
}
context.Wait(_pollingDelay);
}
private void Process(BackgroundProcessContext context)
{
try
{
Logger.Info("Processing execution in-progress.");
var stopwatch = new Stopwatch();
stopwatch.Start();
Process(context.CancellationToken).GetAwaiter().GetResult();
stopwatch.Stop();
Logger.Info($"Processing executed successfully in {stopwatch.ElapsedMilliseconds}ms.");
var localInterval = Interval;
Logger.Info($"Processing {Name} server will now sleep for {localInterval}.");
var endTime = DateTimeOffset.Now;
while (DateTimeOffset.Now - endTime < localInterval)
{
if (localInterval != Interval)
{
localInterval = Interval;
Logger.Info($"Processing server {Name} interval was changed to {localInterval}.");
}
context.Wait(TimeSpan.FromSeconds(5));
}
}
catch (Exception e)
{
Logger.Warn(e, "An error occured while processing, but was handled by the processing server. The processing server will restart.");
throw;
}
}
public void Execute(BackgroundProcessContext context)
{
try
{
var jobsEnqueued = 0;
while (context.Storage.EnqueueNextScheduledItem((CalculateNextInvocation)))
{
jobsEnqueued++;
if (context.IsShutdownRequested)
{
break;
}
}
;
if (jobsEnqueued != 0)
{
_logger.Log($"{jobsEnqueued} scheduled job(s)/workflow(s) enqueued by Recurring Tasks Process.");
}
}
//TODO: refactor to catch exception in storage layer
catch (SqlException sqlEx) when(sqlEx.Number == 1205)
{
//deadlock, just skip, other process on other server is taking care of it...
}
context.Wait(TimeSpan.FromMinutes(1));
}
public void Execute(BackgroundProcessContext context)
{
context.CancellationToken.ThrowIfCancellationRequested();
var queueJob = _storage.FetchNextJob(this._queues, this._workerId);
if (queueJob == null)
{
context.Wait(_workerFetchIdleSleep);
return;
}
_storage.SetJobState(queueJob.JobId, new ProcessingState(context.ServerId, this._workerId));
_eventManager.RaiseOnProcessing(queueJob, context.ServerId, this._workerId);
var perfomContext = new PerformContext(context.CancellationToken, queueJob);
var resultState = PerformJob(perfomContext);
if (resultState is FailedState)
{
var failedState = resultState as FailedState;
if (queueJob.RetryCount < queueJob.MaxRetries)
{
//schedule new run
var nextRun = DateTime.UtcNow.AddSeconds(SecondsToDelay(queueJob.RetryCount));
const int maxMessageLength = 50;
var exceptionMessage = failedState.Exception.Message.Length > maxMessageLength
? failedState.Exception.Message.Substring(0, maxMessageLength - 1) + "…"
: failedState.Exception.Message;
var scheduledState = new ScheduledState(nextRun)
{
Reason = $"Retry attempt { queueJob.RetryCount } of { queueJob.MaxRetries }: { exceptionMessage}"
};
_storage.UpgradeFailedToScheduled(queueJob.JobId, resultState, scheduledState, nextRun, queueJob.RetryCount + 1);
_eventManager.RaiseOnReschedule(queueJob, context.ServerId, this._workerId, failedState.Exception, nextRun);
}
else
{
//final failed state
_storage.SetJobState(queueJob.JobId, resultState);
_eventManager.RaiseOnFail(queueJob, context.ServerId, this._workerId, failedState.Exception);
if (queueJob.WorkflowId.HasValue)
{
//mark dependent jobs consequently failed
_storage.MarkConsequentlyFailedJobs(queueJob.JobId);
}
}
}
else
{
//Succeeded
_storage.SetJobState(queueJob.JobId, resultState);
_storage.ExpireJob(queueJob.JobId);
_storage.EnqueueAwaitingWorkflowJobs(queueJob.JobId);
_eventManager.RaiseOnSuccess(queueJob, context.ServerId, this._workerId);
}
_storage.RemoveFromQueue(queueJob.QueueJobId);
}
public void Execute(BackgroundProcessContext context)
{
var serversRemoved = context.Storage.RemoveTimedOutServers(_serverTimeout);
if (serversRemoved != 0)
{
_logger.Log(LogLevel.Information, $"{serversRemoved} servers were removed due to timeout");
}
context.Wait(_checkInterval);
}
public void Execute([NotNull] BackgroundProcessContext context)
{
using (var connetion = context.Storage.GetConnection())
{
//申请分布式锁
using (connetion.AcquireDistributedLock($"{_backWorker.JobName}:secondsJob", TimeSpan.FromSeconds(_backWorker.Internal)))
{
BackgroundJob.Enqueue <DoTest>(job => job.SendRequest(_backWorker.JobName, _backWorker, null));
}
}
context.Wait(TimeSpan.FromSeconds(_backWorker.Internal));
}
public void Execute(BackgroundProcessContext context)
{
using (var connection = context.Storage.GetConnection())
{
var cpuPercentUsage = ComputeCpuUsage(context.CancellationToken);
if (context.IsShutdownRequested)
{
CleanupState(context, connection);
return;
}
var values = new Dictionary <string, string>
{
[ProcessId] = _currentProcess.Id.ToString(CultureInfo.InvariantCulture),
[ProcessName] = _currentProcess.ProcessName,
[CpuUsage] = cpuPercentUsage.ToString(CultureInfo.InvariantCulture),
[WorkingSet] = _currentProcess.WorkingSet64.ToString(CultureInfo.InvariantCulture),
[Timestamp] = DateTimeOffset.UtcNow.ToUnixTimeSeconds().ToString(CultureInfo.InvariantCulture)
};
using (var transaction = connection.CreateWriteTransaction())
{
var hashKey = Utils.FormatKey(context.ServerId);
transaction.SetRangeInHash(hashKey, values);
if (transaction is JobStorageTransaction jobStorageTransaction)
{
// set expiration (if supported by storage)
jobStorageTransaction.ExpireHash(hashKey, _expireIn);
}
transaction.Commit();
}
}
if (_checkInterval != TimeSpan.Zero)
{
context.Wait(_checkInterval);
}
}
public void Execute(BackgroundProcessContext context)
{
var monitoringApi = context.Storage.GetMonitoringApi();
if (monitoringApi.EnqueuedCount("default") > _count)
{
context.Wait(_delay);
return;
}
var client = new BackgroundJobClient(context.Storage)
{
RetryAttempts = 15
};
for (var i = 0; i < _count; i++)
{
context.StoppingToken.ThrowIfCancellationRequested();
client.Enqueue <IHarnessV1>(x => x.Perform(0));
}
}
public void Execute(BackgroundProcessContext context)
{
for (var i = 0; i <= MaxRetryAttempts; i++)
{
try
{
InnerProcess.Execute(context);
return;
}
catch (Exception ex)
{
if (ex is OperationCanceledException && context.IsShutdownRequested)
{
throw;
}
// Break the loop after the retry attempts number exceeded.
if (i >= MaxRetryAttempts - 1)
{
throw;
}
var nextTry = DelayCallback(i);
var logLevel = GetLogLevel(i);
_logger.Log(
logLevel,
// ReSharper disable once AccessToModifiedClosure
() => $"Error occurred during execution of '{InnerProcess}' process. Execution will be retried (attempt #{i + 1}) in {nextTry} seconds.",
ex);
context.Wait(nextTry);
if (context.IsShutdownRequested)
{
break;
}
}
}
}
public void Execute(BackgroundProcessContext context)
{
var api = context.Storage.GetMonitoringApi();
var queues = api.Queues();
_logger.Trace("Checking queue length to decide whether to change gate levels...");
if (queues.Any(queue => _queues.Contains(queue.Name) && queue.Length > 0))
{
if (_gate.TryIncreaseLevel(out var level))
{
_logger.Debug($"Gate level for queues ({_queueNames}) increased to {level} / {_gate.MaxLevel}");
}
}
else
{
if (_gate.TryDecreaseLevel(out var level))
{
_logger.Debug($"Gate level for queues ({_queueNames}) decreased to {level} / {_gate.MaxLevel}");
}
}
context.Wait(_delay);
}
public void Execute(BackgroundProcessContext context)
{
for (var i = 0; i <= MaxRetryAttempts; i++)
{
try
{
_innerProcess.Execute(context);
return;
}
catch (Exception ex)
{
if (ex is OperationCanceledException && context.IsShutdownRequested)
{
throw;
}
// Break the loop after the retry attempts number exceeded.
if (i >= MaxRetryAttempts - 1)
{
_logger.Log(LogLevel.Error, $"Error occurred during execution of '{_innerProcess}' process. No more retries. Exiting background process.", ex);
throw;
}
var nextTry = DelayCallback(i);
_logger.Log(LogLevel.Warning, $"Error occurred during execution of '{_innerProcess}' process. Execution will be retried (attempt #{i + 1}) in {nextTry} seconds.", ex);
context.Wait(nextTry);
if (context.IsShutdownRequested)
{
break;
}
}
}
}
//https://www.hangfire.io/overview.html
public void Execute([NotNull] BackgroundProcessContext context)
{
context.Wait(TimeSpan.FromHours(1));
}
public void Execute(BackgroundProcessContext context)
{
this.GetValue();
context.Wait(new TimeSpan(0, 0, 5));
}
public void Execute(BackgroundProcessContext context)
{
context.Storage.HeartbeatServer(context.ServerId, JobHelper.ToJson(context.ServerContext));
context.Wait(_heartbeatInterval);
}