/// <summary>
/// 定时处理 执行错误的任务，进行重试
/// </summary>
/// <returns></returns>
public Task Start()
{
Task.Run(async() =>
{
await Task.Delay(TimeSpan.FromSeconds(3));
while (!_scheduleTaskLifetime.ScheduleTaskStopping.IsCancellationRequested)
{
_scheduleTaskLifetime.ScheduleTaskStopping.ThrowIfCancellationRequested();
try
{
await ScheduleProcessErrorTask();
}
catch (OperationCanceledException ex)
{
_logger.LogError(ex, "Aix.ScheduleTask任务取消");
}
catch (Exception ex)
{
_logger.LogError(ex, "处理失败任务异常");
}
await TaskEx.DelayNoException(TimeSpan.FromSeconds(10), _scheduleTaskLifetime.ScheduleTaskStopping);
}
});
return(Task.CompletedTask);
}
private async Task RunProcess(IBackgroundProcess process)
{
try
{
while (_isStart && !_backgroundProcessContext.IsShutdownRequested)
{
try
{
await process.Execute(_backgroundProcessContext); //内部控制异常
}
catch (OperationCanceledException ex)
{
_logger.LogError(ex, "redis任务取消");
}
catch (RedisException ex)
{
_logger.LogError(ex, "redis错误");
await TaskEx.DelayNoException(TimeSpan.FromSeconds(10), _backgroundProcessContext.CancellationToken);
}
catch (Exception ex)
{
string errorMsg = $"执行任务{process.GetType().FullName}异常";
_logger.LogError(ex, errorMsg);
}
}
}
catch (Exception)
{
}
}
/// <summary>
/// 清除过期日志
/// </summary>
/// <returns></returns>
public Task Start()
{
Task.Run(async() =>
{
await Task.Delay(TimeSpan.FromSeconds(30));
var logExpireHour = _options.LogExpireHour > 0 ? _options.LogExpireHour : 168;
while (!_scheduleTaskLifetime.ScheduleTaskStopping.IsCancellationRequested)
{
_scheduleTaskLifetime.ScheduleTaskStopping.ThrowIfCancellationRequested();
try
{
var expiration = DateTime.Now.AddHours(0 - logExpireHour);
await _aixScheduleTaskLogRepository.Delete(expiration);
}
catch (OperationCanceledException ex)
{
_logger.LogError(ex, "Aix.ScheduleTask任务取消");
}
catch (Exception ex)
{
_logger.LogError(ex, "清除过期日志异常");
}
await TaskEx.DelayNoException(TimeSpan.FromHours(2), _scheduleTaskLifetime.ScheduleTaskStopping);
}
});
return(Task.CompletedTask);
}
public async Task Execute(BackgroundProcessContext context)
{
var lockKey = $"{_options.TopicPrefix}delay:lock";
long delay = 0; //毫秒
await _redisStorage.Lock(lockKey, lockTimeSpan, async() =>
{
var now = DateTime.Now;
var maxScore = DateUtils.GetTimeStamp(now);
var list = await _redisStorage.GetTopDueDealyJobId(maxScore + PreReadSecond * 1000, BatchCount); //多查询1秒的数据，便于精确控制延迟
foreach (var item in list)
{
if (context.IsShutdownRequested)
{
return;
}
if (_isStart == false)
{
return; //已经关闭了 就直接返回吧
}
if (item.Value > maxScore)
{
delay = item.Value - maxScore;
break;
}
var jobId = item.Key;
// 延时任务到期加入即时任务队列
await _redisStorage.DueDealyJobEnqueue(jobId);
}
if (list.Count == 0)//没有数据时
{
delay = PreReadSecond * 1000;
}
}, async() => await TaskEx.DelayNoException(PreReadSecond * 1000, context.CancellationToken)); //出现并发也休息一会
if (delay > 0)
{
var minDelay = Math.Min((int)delay, PreReadSecond * 1000);
_redisStorage.WaitForDelayJob(TimeSpan.FromMilliseconds(minDelay), context.CancellationToken);
//await TaskEx.DelayNoException(Math.Min((int)delay, PreReadSecond * 1000), context.CancellationToken);
}
}
/// <summary>
/// 根据是否集群环境是否增加分布式锁
/// </summary>
/// <returns></returns>
private async Task DistributionLockWrap()
{
List <TimeSpan> nextExecuteDelays = null;
await _scheduleTaskDistributedLock.Lock(ScheduleTaskLock, TimeSpan.FromSeconds(300), async() =>
{
nextExecuteDelays = await Execute();
}, async() =>
{
//出现并发 休息一会，说明其他服务器已经在执行了
await TaskEx.DelayNoException(TimeSpan.FromSeconds(PreReadSecond), _scheduleTaskLifetime.ScheduleTaskStopping);
});
var depay = nextExecuteDelays.Any() ? nextExecuteDelays.Min() : TimeSpan.FromSeconds(PreReadSecond);
if (depay > TimeSpan.FromSeconds(PreReadSecond))
{
depay = TimeSpan.FromSeconds(PreReadSecond);
}
await TaskEx.DelayNoException(depay, _scheduleTaskLifetime.ScheduleTaskStopping);
}
public async Task Execute(BackgroundProcessContext context)
{
if (_taskExecutor.GetTaskCount() > _options.TaskExecutorMaxTaskCount) //队列任务积压超过1000就不要再拉取了，不然也处理不过来
{
await TaskEx.DelayNoException(TimeSpan.FromMilliseconds(_options.ConsumePullIntervalMillisecond), context.CancellationToken);
return;
}
//// >：读取未分配给其他消费者的消息(未被拉去的) 0-0或id： 读取pending 中的消息
var list = await _database.StreamReadGroupAsync(_topic, _groupName, _consumerName, ">", BatchCount);
if (list.Length == 0)
{
await TaskEx.DelayNoException(TimeSpan.FromMilliseconds(_options.ConsumePullIntervalMillisecond), context.CancellationToken);
return;
}
await ProcessList(list, true);
}
public async Task Execute(BackgroundProcessContext context)
{
// 循环所有的stream和group进行处理，订阅时保存 stream和对应的group（一个stream可能有多个groupo）
List <long> waitTimes = new List <long>();
await _redisStorage.Lock("pendingreadlock", TimeSpan.FromSeconds(60), async() =>
{
foreach (var item in context.SubscriberTopics)
{
if (_isStart == false)
{
return; //及时关闭
}
var currentWaitTime = await ProcessPel(item);
waitTimes.Add(currentWaitTime);
}
}, () => Task.CompletedTask);
var waitTime = waitTimes.Any() ? waitTimes.Min() : ExecuteTimeoutMilliseconds / 2;
await TaskEx.DelayNoException(TimeSpan.FromMilliseconds(waitTime), context.CancellationToken);
}
private async Task StartProcessTask()
{
while (!_scheduleTaskLifetime.ScheduleTaskStopping.IsCancellationRequested)
{
try
{
await DistributionLockWrap();
}
catch (OperationCanceledException ex)
{
_logger.LogError(ex, "Aix.ScheduleTask定时任务取消");
}
catch (Exception ex)
{
if (ex.Message.ToLower().IndexOf("timeout") < 0)
{
_logger.LogError(ex, "Aix.ScheduleTask定时任务执行出错");
await TaskEx.DelayNoException(TimeSpan.FromSeconds(5), _scheduleTaskLifetime.ScheduleTaskStopping);
}
}
}
}
public async Task Execute(BackgroundProcessContext context)
{
//var lockKey = $"{_options.TopicPrefix}delay:lock";
var lockKey = $"{_delayTopicName}:lock";
long delay = 0; //毫秒
await _redisStorage.Lock(lockKey, lockTimeSpan, async() =>
{
var now = DateTime.Now;
var maxScore = DateUtils.GetTimeStamp(now);
var list = await _redisStorage.GetTopDueDealyJobId(_delayTopicName, maxScore + PreReadSecond * 1000, BatchCount); //多查询1秒的数据，便于精确控制延迟
foreach (var item in list)
{
if (context.IsShutdownRequested)
{
return;
}
if (_isStart == false)
{
return; //已经关闭了 就直接返回吧
}
if (item.Value > maxScore) //预拉去了PreReadSecond秒的数据，可能有还没到时间的
{
delay = item.Value - maxScore;
break;
}
var jobId = item.Key;
// 延时任务到期加入即时任务队列
var hashEntities = await _redisStorage.HashGetAll(Helper.GetJobHashId(_options, jobId));//这里要出错了呢
JobData jobData = null;
try
{
jobData = JobData.ToJobData(hashEntities);
}
catch (Exception ex)
{
_logger.LogError(ex, $"RedisMessageBus解析延迟任务数据报错");
}
if (jobData != null)
{
await _redisStorage.DueDealyJobEnqueue(_delayTopicName, jobData);
}
else
{
_logger.LogError("RedisMessageBus延迟任务解析出错为空，这里就从hash中删除了");
await _redisStorage.RemoveNullDealyJob(_delayTopicName, jobId);
}
}
if (list.Count == 0)//没有数据时
{
delay = PreReadSecond * 1000;
}
}, async() => await TaskEx.DelayNoException(PreReadSecond * 1000, context.CancellationToken)); //出现并发也休息一会
if (delay > 0)
{
var minDelay = Math.Min((int)delay, PreReadSecond * 1000);
_redisStorage.WaitForDelayJob(TimeSpan.FromMilliseconds(minDelay), context.CancellationToken);
//await TaskEx.DelayNoException(Math.Min((int)delay, PreReadSecond * 1000), context.CancellationToken);
}
}