public void Execute(BackgroundProcessContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
var jobsEnqueued = 0;
while (EnqueueNextScheduledJob(context))
{
jobsEnqueued++;
if (context.IsShutdownRequested)
{
break;
}
}
if (jobsEnqueued != 0)
{
_logger.Log($"{jobsEnqueued} scheduled job(s) enqueued.");
}
context.Wait(_pollingDelay);
}
public void Execute([NotNull] BackgroundProcessContext context)
{
Console.WriteLine(DateTime.Now.ToString() + ": Start refresh refresh-token background job. ");
_accessTokenManager.UpdateRefreshToken();
// context.Wait(TimeSpan.FromSeconds(5));
Console.WriteLine(DateTime.Now.ToString() + ": Finish refresh refresh-token background job. ");
}
public void Execute([NotNull] BackgroundProcessContext context)
{
Logger.Debug("Aggregating records in 'Counter' table...");
int removedCount = 0;
do
{
_storage.UseConnection(null, connection =>
{
removedCount = connection.Execute(
GetAggregationQuery(_storage),
new { now = DateTime.UtcNow, count = NumberOfRecordsInSinglePass },
commandTimeout: 0);
});
if (removedCount >= NumberOfRecordsInSinglePass)
{
context.CancellationToken.WaitHandle.WaitOne(DelayBetweenPasses);
context.CancellationToken.ThrowIfCancellationRequested();
}
// ReSharper disable once LoopVariableIsNeverChangedInsideLoop
} while (removedCount >= NumberOfRecordsInSinglePass);
Logger.Trace("Records from the 'Counter' table aggregated.");
context.CancellationToken.WaitHandle.WaitOne(_interval);
}
public void Execute([NotNull] BackgroundProcessContext context)
{
Console.WriteLine("Begin to refresh access token background job. " + DateTime.Now.ToString());
_accessTokenManager.UpdateToken();
// context.Wait(TimeSpan.FromSeconds(5));
Console.WriteLine("Finish refresh access token background job. " + DateTime.Now.ToString());
}
private async Task ProcessQueue(string topic, BackgroundProcessContext context)
{
var startProcessTime = DateTime.Now;
int deleteCount = 0;
var length = 0;
var start = BatchCount * -1;
var end = -1;
do
{
var list = await _redisStorage.GetErrorJobId(topic, start, end);// -100,-1 //从队列的尾部抓取 -1=最后一个，-2=倒数第二个，...
length = list.Length;
if (context.IsShutdownRequested)
{
return;
}
deleteCount = await ProcessFailedJob(topic, list);//倒序处理
end = 0 - (length + 1 - deleteCount);
start = end - BatchCount + 1;
if (DateTime.Now - startProcessTime >= lockTimeSpan)
{
break;
}
}while (length > 0);
}
public async Task ExecuteAsync(BackgroundProcessContext context)
{
while (!context.IsShutdownRequested)
{
await InnerTask.ExecuteAsync(context).ConfigureAwait(false);
}
}
public ExpirationManagerTests()
{
_storage = ConnectionUtils.GetStorage();
var cts = new CancellationTokenSource();
_context = new BackgroundProcessContext("dummy", _storage, new Dictionary <string, object>(), cts.Token);
}
public void Execute(BackgroundProcessContext context)
{
foreach (var table in ProcessedTables)
{
if (context.CancellationToken.IsCancellationRequested)
{
return;
}
_logger.Log($"Removing outdated records from the '{table}' table...");
_storage.UseConnection((conn) => {
try
{
int affected;
do
{
affected = ExecuteNonQuery(
conn,
GetQuery(_schema, table),
context.CancellationToken,
new SqlParameter("@count", NumberOfRecordsInSinglePass),
new SqlParameter("@now", DateTime.UtcNow));
} while (affected == NumberOfRecordsInSinglePass);
}
finally
{
}
});
_logger.Log($"Outdated records removed from the '{table}' table.");
}
context.CancellationToken.WaitHandle.WaitOne(_checkInterval);
}
private void Process(BackgroundProcessContext context)
{
try
{
Logger.Info("Processing execution in-progress.");
var stopwatch = new Stopwatch();
stopwatch.Start();
Process(context.CancellationToken).GetAwaiter().GetResult();
stopwatch.Stop();
Logger.Info($"Processing executed successfully in {stopwatch.ElapsedMilliseconds}ms.");
var localInterval = Interval;
Logger.Info($"Processing {Name} server will now sleep for {localInterval}.");
var endTime = DateTimeOffset.Now;
while (DateTimeOffset.Now - endTime < localInterval)
{
if (localInterval != Interval)
{
localInterval = Interval;
Logger.Info($"Processing server {Name} interval was changed to {localInterval}.");
}
context.Wait(TimeSpan.FromSeconds(5));
}
}
catch (Exception e)
{
Logger.Warn(e, "An error occured while processing, but was handled by the processing server. The processing server will restart.");
throw;
}
}
public void Execute(BackgroundProcessContext context)
{
while (!context.IsShutdownRequested)
{
InnerProcess.Execute(context);
}
}
public void Execute([NotNull] BackgroundProcessContext context)
{
Console.WriteLine(DateTime.Now.ToString() + ": Start to alibaba callback message processing.");
tradeManager.CallbackMessageTrigger();
// context.Wait(TimeSpan.FromSeconds(5));
Console.WriteLine(DateTime.Now.ToString() + ": Finish alibaba callback message processing.");
}
public void Execute(BackgroundProcessContext context)
{
try
{
var jobsEnqueued = 0;
while (context.Storage.EnqueueNextScheduledItem((CalculateNextInvocation)))
{
jobsEnqueued++;
if (context.IsShutdownRequested)
{
break;
}
}
;
if (jobsEnqueued != 0)
{
_logger.Log($"{jobsEnqueued} scheduled job(s)/workflow(s) enqueued by Recurring Tasks Process.");
}
}
//TODO: refactor to catch exception in storage layer
catch (SqlException sqlEx) when(sqlEx.Number == 1205)
{
//deadlock, just skip, other process on other server is taking care of it...
}
context.Wait(TimeSpan.FromMinutes(1));
}
public void Execute([NotNull] BackgroundProcessContext context)
{
foreach (var table in ProcessedTables)
{
Logger.Debug($"Removing outdated records from the '{table}' table...");
UseConnectionDistributedLock(_storage, connection =>
{
int affected;
do
{
affected = ExecuteNonQuery(
connection,
GetQuery(_storage.SchemaName, table),
context.CancellationToken,
new SqlParameter("@count", NumberOfRecordsInSinglePass),
new SqlParameter("@now", DateTime.UtcNow));
} while (affected == NumberOfRecordsInSinglePass);
});
Logger.Trace($"Outdated records removed from the '{table}' table.");
}
context.CancellationToken.WaitHandle.WaitOne(_checkInterval);
}
public void Execute([NotNull] BackgroundProcessContext context)
{
Console.WriteLine("Begin to start data sync service background job. " + DateTime.Now.ToString());
dataSyncServiceManager.Execution();
// context.Wait(TimeSpan.FromSeconds(5));
Console.WriteLine("Finish data sync service background job. " + DateTime.Now.ToString());
}
public ExpirationManagerTests()
{
_storage = ConnectionUtils.GetStorage();
_storage.Options.JobExpirationCheckInterval = TimeSpan.Zero;
var cts = new CancellationTokenSource();
_context = new BackgroundProcessContext("dummy", _storage, new Dictionary <string, object>(), cts.Token);
}
private static void CleanupState(BackgroundProcessContext context, IStorageConnection connection)
{
using (var transaction = connection.CreateWriteTransaction())
{
transaction.RemoveHash(Utils.FormatKey(context.ServerId));
transaction.Commit();
}
}
public void Execute(BackgroundProcessContext context)
{
context.CancellationToken.ThrowIfCancellationRequested();
var queueJob = _storage.FetchNextJob(this._queues, this._workerId);
if (queueJob == null)
{
context.Wait(_workerFetchIdleSleep);
return;
}
_storage.SetJobState(queueJob.JobId, new ProcessingState(context.ServerId, this._workerId));
_eventManager.RaiseOnProcessing(queueJob, context.ServerId, this._workerId);
var perfomContext = new PerformContext(context.CancellationToken, queueJob);
var resultState = PerformJob(perfomContext);
if (resultState is FailedState)
{
var failedState = resultState as FailedState;
if (queueJob.RetryCount < queueJob.MaxRetries)
{
//schedule new run
var nextRun = DateTime.UtcNow.AddSeconds(SecondsToDelay(queueJob.RetryCount));
const int maxMessageLength = 50;
var exceptionMessage = failedState.Exception.Message.Length > maxMessageLength
? failedState.Exception.Message.Substring(0, maxMessageLength - 1) + "…"
: failedState.Exception.Message;
var scheduledState = new ScheduledState(nextRun)
{
Reason = $"Retry attempt { queueJob.RetryCount } of { queueJob.MaxRetries }: { exceptionMessage}"
};
_storage.UpgradeFailedToScheduled(queueJob.JobId, resultState, scheduledState, nextRun, queueJob.RetryCount + 1);
_eventManager.RaiseOnReschedule(queueJob, context.ServerId, this._workerId, failedState.Exception, nextRun);
}
else
{
//final failed state
_storage.SetJobState(queueJob.JobId, resultState);
_eventManager.RaiseOnFail(queueJob, context.ServerId, this._workerId, failedState.Exception);
if (queueJob.WorkflowId.HasValue)
{
//mark dependent jobs consequently failed
_storage.MarkConsequentlyFailedJobs(queueJob.JobId);
}
}
}
else
{
//Succeeded
_storage.SetJobState(queueJob.JobId, resultState);
_storage.ExpireJob(queueJob.JobId);
_storage.EnqueueAwaitingWorkflowJobs(queueJob.JobId);
_eventManager.RaiseOnSuccess(queueJob, context.ServerId, this._workerId);
}
_storage.RemoveFromQueue(queueJob.QueueJobId);
}
private async Task <IState> PerformJob(BackgroundProcessContext context, IStorageConnection connection, string jobId)
{
try
{
var jobData = connection.GetJobData(jobId);
if (jobData == null)
{
// Job expired just after moving to a processing state. This is an
// unreal scenario, but shit happens. Returning null instead of throwing
// an exception and rescuing from en-queueing a poisoned jobId back
// to a queue.
return(null);
}
jobData.EnsureLoaded();
var backgroundJob = new BackgroundJob(jobId, jobData.Job, jobData.CreatedAt);
var jobToken = new ServerJobCancellationToken(connection, jobId, context.ServerId, _workerId, context.CancellationToken);
var performContext = new PerformContext(connection, backgroundJob, jobToken);
var latency = (DateTime.UtcNow - jobData.CreatedAt).TotalMilliseconds;
var duration = Stopwatch.StartNew();
var result = await _performer.PerformAsync(performContext).ConfigureAwait(false);
duration.Stop();
// SHOULD BE: return new SucceededState(result, (long)latency, duration.ElapsedMilliseconds);
return(CreateSucceededState(result, (long)latency, duration.ElapsedMilliseconds));
}
catch (JobAbortedException)
{
// Background job performance was aborted due to a
// state change, so it's idenfifier should be removed
// from a queue.
return(null);
}
catch (JobPerformanceException ex)
{
return(new FailedState(ex.InnerException)
{
Reason = ex.Message
});
}
catch (Exception ex)
{
if (ex is OperationCanceledException && context.IsShutdownRequested)
{
throw;
}
return(new FailedState(ex)
{
Reason = "An exception occurred during processing of a background job."
});
}
}
public void Ctor_CorrectlyInitializes_AllTheProperties()
{
var context = new BackgroundProcessContext(_serverId, _storage.Object, _properties, _cts.Token);
Assert.Equal(_serverId, context.ServerId);
Assert.True(_properties.SequenceEqual(context.Properties));
Assert.Same(_storage.Object, context.Storage);
Assert.Equal(_cts.Token, context.CancellationToken);
}
public Task ExecuteAsync(BackgroundProcessContext context)
{
using (var connection = context.Storage.GetConnection())
{
connection.Heartbeat(context.ServerId);
}
return(context.WaitAsync(_heartbeatInterval));
}
public void Execute(BackgroundProcessContext context)
{
var serversRemoved = context.Storage.RemoveTimedOutServers(_serverTimeout);
if (serversRemoved != 0)
{
_logger.Log(LogLevel.Information, $"{serversRemoved} servers were removed due to timeout");
}
context.Wait(_checkInterval);
}
public void Execute([NotNull] BackgroundProcessContext context)
{
using (var connetion = context.Storage.GetConnection())
{
//申请分布式锁
using (connetion.AcquireDistributedLock($"{_backWorker.JobName}:secondsJob", TimeSpan.FromSeconds(_backWorker.Internal)))
{
BackgroundJob.Enqueue <DoTest>(job => job.SendRequest(_backWorker.JobName, _backWorker, null));
}
}
context.Wait(TimeSpan.FromSeconds(_backWorker.Internal));
}
public Task ExecuteAsync(BackgroundProcessContext context)
{
using (var connection = context.Storage.GetConnection())
{
var serversRemoved = connection.RemoveTimedOutServers(_serverTimeout);
if (serversRemoved != 0)
{
Logger.Info($"{serversRemoved} servers were removed due to timeout");
}
}
return(context.WaitAsync(_checkInterval));
}
public void Execute(BackgroundProcessContext context)
{
using var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(context.ShutdownToken, context.StoppingToken);
using (new HangfireNinjectResolutionScope(_kernel))
{
var backgroundJobLauncher = _kernel.GetRequiredService <IBackgroundJobLauncher>();
PurgeDuplicateUpdates(backgroundJobLauncher, combinedTokenSource);
ScheduleUpdatesCausedByDependencyChanges(backgroundJobLauncher, combinedTokenSource);
ProcessPendingUpdates(backgroundJobLauncher, combinedTokenSource);
}
CoolDown();
}
public void Execute(BackgroundProcessContext context)
{
try
{
SetCultureFromWebConfig();
ECommerceLog.WriteLog("Worker is starting, orderprocessor is " + ECommerce.OrderProcessor.GetType().FullName);
var ticker = 60;
using (ThreadDataManager.EnsureInitialize())
{
while (!context.IsShutdownRequested)
{
try
{
if (!_processOrdersNow && ticker != 60)
{
continue;
}
_processOrdersNow = false;
PostProcessPendingOrders(context);
}
catch (Exception ex)
{
ECommerceLog.WriteLog("Unhandled error when postprocessing orders", ex);
}
finally
{
if (ticker == 60)
{
ticker = 0;
}
ticker = ticker + 1;
context.CancellationToken.WaitHandle.WaitOne(OneSecond);
context.CancellationToken.ThrowIfCancellationRequested();
}
}
}
}
catch (OperationCanceledException) { }
catch (Exception ex)
{
ECommerceLog.WriteLog("Unhandled error in ThreadDataManager, worker is stopping", ex);
}
}
private IDisposable AcquireDistributedLock(BackgroundProcessContext context)
{
if (!RequiresDistributedLock || _configuration.DisableDistributedLocks)
{
Logger.Debug("Distibuted lock is not reqired, none will be aquired.");
return(null);
}
Logger.Info("Attemping to aquire distibuted lock.");
var connection = context.Storage.GetConnection();
var rock = connection.AcquireDistributedLock(Name, TimeSpan.FromHours(24));
Logger.Info("Distibuted lock achived.");
return(rock);
}
/// <summary>
/// 服务启动时先处理pel中的数据（上次服务结束时拉去到但是未执行的数据/未ack）
/// </summary>
/// <param name="context"></param>
/// <returns></returns>
public async Task ProcessPel(BackgroundProcessContext context)
{
while (!context.IsShutdownRequested)
{
//读取pending 中的消息
var list = await _database.StreamReadGroupAsync(_topic, _groupName, _consumerName, "0-0", 20);
if (list == null || list.Length == 0)
{
return;
}
await ProcessList(list, false);//每次启动时处理pel中的数据，要同步处理，等处理结束，再拉去新消息处理
}
}
public void Execute(BackgroundProcessContext context)
{
_logger.Trace("Acquiring a gate...");
_gate.Wait(context.StoppingToken);
_logger.Trace("Gate acquired.");
try
{
_innerWorker.Execute(context);
}
finally
{
_logger.Trace("Releasing gate");
_gate.Release();
}
}
private static void PostProcessPendingOrders(BackgroundProcessContext context)
{
using (var data = new DataConnection())
{
var orders = data.Get <IShopOrder>()
.Where(s => s.PaymentStatus == (int)PaymentStatus.Authorized && !s.PostProcessed)
.Take(NumberOfRecordsInSinglePass).ToList();
foreach (var order in orders)
{
PostProcessOrder(order, data);
context.CancellationToken.ThrowIfCancellationRequested();
}
}
}
public void Execute(BackgroundProcessContext context)
{
Logger.Info($"Beginning execution of processing server {Name} with a execution interval of {Interval}.");
var rock = AcquireDistributedLock(context);
try
{
while (!context.IsShutdownRequested)
{
Process(context);
}
}
finally
{
rock?.Dispose();
}
}
