private void Handle(IReadOnlyCollection <FactOperation> operations)
{
_tracer.Debug("Handing fact operations started");
var result = _factsReplicator.Replicate(operations);
_telemetryPublisher.Publish <ErmProcessedOperationCountIdentity>(operations.Count);
var statistics = result.OfType <RecalculateStatisticsOperation>().ToArray();
var aggregates = result.OfType <AggregateOperation>().ToArray();
// We always need to use different transaction scope to operate with operation sender because it has its own store
using (var pushTransaction = new TransactionScope(TransactionScopeOption.RequiresNew,
new TransactionOptions {
IsolationLevel = IsolationLevel.ReadCommitted, Timeout = TimeSpan.Zero
}))
{
_tracer.Debug("Pushing events for statistics recalculation");
_statisticsSender.Push(statistics);
_telemetryPublisher.Publish <StatisticsEnqueuedOperationCountIdentity>(statistics.Length);
_tracer.Debug("Pushing events for aggregates recalculation");
_aggregateSender.Push(aggregates);
_telemetryPublisher.Publish <AggregateEnqueuedOperationCountIdentity>(aggregates.Length);
pushTransaction.Complete();
}
_tracer.Debug("Handing fact operations finished");
}
/// <summary>
/// 配置实例
/// </summary>
/// <returns></returns>
public static TaskConfig GetInstance(bool isReload = false)
{
if (_instance == null)
{
lock (syncObject)
{
if (isReload)
{
_instance = null;
}
if (_instance != null)
{
return(_instance);
}
logRun.Debug("TaskConfig实例化");
_instance = new TaskConfig();
_instance.Initialize();
}
}
else
{
logRun.Debug("返回缓存的的TaskConfig实例");
}
return(_instance);
}
public void Execute()
{
var actualQueuedBytes = _messageQueue.ActualQueuedBytes;
if (_maxQueuedBytesThreshold <= actualQueuedBytes)
{
if (_isBackpressureMode)
{
return;
}
_tracer.Debug($"KafkaAudit. Backpressure mode. Limit for messages in queue is {_maxQueuedBytesThreshold} bytes. Actual queued {actualQueuedBytes} bytes");
var pausedPartitions = _consumer.Pause(_targetPartitions);
foreach (var partition in pausedPartitions)
{
_tracer.Debug($"{partition.TopicPartition} pause attempt finished with result {partition.Error}");
}
_isBackpressureMode = true;
}
else if (_isBackpressureMode)
{
var resumedPartitions = _consumer.Resume(_targetPartitions);
foreach (var partition in resumedPartitions)
{
_tracer.Debug($"{partition.TopicPartition} resume attempt finished with result {partition.Error}");
}
_isBackpressureMode = false;
}
}
private Task DiscoverAssembly(ITestCaseDiscoverySink discoverySink, Assembly asm,
string source)
{
_tracer.Debug($"Processing assembly {asm.FullName}");
return
(Task.WhenAll(
asm.GetTypes()
.Where(IsUnitTestType)
.Select(type => DiscoverType(discoverySink, type, source))));
}
public void DiscoverTests(IEnumerable <string> sources, IDiscoveryContext discoveryContext, IMessageLogger logger, ITestCaseDiscoverySink discoverySink, CancellationToken token)
{
_tracer = new MessageLoggerTracer(logger);
_tracer.Debug(
$"Test discovery starting - using {RuntimeInformation.FrameworkDescription}, PID {Process.GetCurrentProcess().Id}");
DiscoverTestsAsync(sources, discoverySink).Wait(token);
}
public int Apply(long changeset)
{
CheckInitialized();
RamDriverChangeset changesetRec;
if (!m_changesets.TryRemove(changeset, out changesetRec))
{
throw new ArgumentException("Invalid changeset handle: " + changeset, "changeset");
}
try
{
InvalidateIndexes(changesetRec);
}
finally
{
changesetRec.DocumentContainer.StructureLock.ExitReadLock();
}
if (changesetRec.ChangeCount > 0)
{
if (m_tracer.IsDebugEnabled)
{
m_tracer.Debug(
string.Format(
"Applied {0} changed rows on changeset {1}", changesetRec.ChangeCount, changeset));
}
}
else
{
if (m_tracer.IsDebugEnabled)
{
m_tracer.Debug("Did not have any changes to apply on changeset " + changeset);
}
}
return(changesetRec.ChangeCount);
}
/// <summary>
/// Initializes driver with settings object.
/// </summary>
/// <param name="tracer">Tracer sink</param>
/// <param name="settings">Arbitrary settings object, type specific to implementation</param>
/// <seealso cref="IStorageDriverFactory.Create"/>
/// <seealso cref="IStorageDriverFactory.GetDriverConfig"/>
public void Initialize(ITracer tracer, object settings)
{
if (m_initialized)
{
throw new InvalidOperationException("Already initialized");
}
if (tracer == null)
{
throw new ArgumentNullException("tracer");
}
m_tracer = tracer;
if (m_tracer.IsDebugEnabled)
{
m_tracer.Debug(string.Concat("Initializing RAM storage driver for settings string: ", settings));
}
m_changesets = new ConcurrentDictionary <long, RamDriverChangeset>();
var initString = settings as string;
if (!string.IsNullOrEmpty(initString))
{
m_settings = new RamDriverSettings();
if (0 == StringComparer.OrdinalIgnoreCase.Compare("demo", initString))
{
m_settings.InitializationCommand = initString;
}
else
{
m_settings.StorageRoot = initString;
}
}
else
{
m_settings = new RamDriverSettings(settings as RamDriverSettings);
}
CheckInitialized();
if (m_tracer.IsInfoEnabled)
{
m_tracer.Info("RAM storage driver ready");
}
}
/// <summary>
/// 监视轮询工作
/// 发现有可执行的任务时，启动该任务所在的驱动以使其执行。
/// Note:Working时就要每次重读配置，释放后的任务再也回不来了。
/// </summary>
public void Working(object sender, ElapsedEventArgs args)
{
//Note:线程锁定
lock (this)
{
_watcher.Stop();
_log.Debug($"↓---- 任务监视轮询开始 ---- [下次轮询延时: {_watcher.Interval/1000}秒 ; 当前时间:{DateTime.Now.ToString("HH:mm:ss ffffff")}]");
TryRemoveTask();
StartSettingTasks();
_watcher.Interval = ((TimeSpan)TaskSetting.WatchTimer.WorkingInterval).TotalMilliseconds;
_workTimes++;
_log.DebugFormat($"↑---- 任务监视轮询结束[Times:{_workTimes}] ---- [当前时间:{DateTime.Now.ToString("HH:mm:ss ffffff")}]");
_watcher.Start();
}
}
public IReadOnlyCollection <IOperation> Replicate(IEnumerable <FactOperation> operations)
{
using (Probe.Create("ETL1 Transforming"))
{
var result = Enumerable.Empty <IOperation>();
var slices = operations.GroupBy(operation => new { operation.FactType })
.OrderByDescending(slice => slice.Key.FactType, _factTypePriorityComparer);
foreach (var slice in slices)
{
var factType = slice.Key.FactType;
IMetadataElement factMetadata;
var metadataId = ReplicationMetadataIdentity.Instance.Id.WithRelative(new Uri(string.Format("Facts/{0}", factType.Name), UriKind.Relative));
if (!_metadataProvider.TryGetMetadata(metadataId, out factMetadata))
{
throw new NotSupportedException(string.Format("The fact of type '{0}' is not supported.", factType));
}
var factIds = slice.Select(x => x.FactId).Distinct();
using (Probe.Create("ETL1 Transforming", factType.Name))
{
var processor = _factProcessorFactory.Create(factMetadata);
foreach (var batch in factIds.CreateBatches(_replicationSettings.ReplicationBatchSize))
{
_tracer.Debug("Apply changes to target facts storage");
var aggregateOperations = processor.ApplyChanges(batch);
result = result.Concat(aggregateOperations);
}
}
}
return(result.Distinct().ToArray());
}
}
/// <summary>
/// Initializes driver with settings object.
/// </summary>
/// <param name="tracer">Tracer sink</param>
/// <param name="settings">Arbitrary settings object, type specific to implementation</param>
/// <seealso cref="IStorageDriverFactory.Create"/>
/// <seealso cref="IStorageDriverFactory.GetDriverConfig"/>
public void Initialize(ITracer tracer, object settings)
{
if (m_initialized)
{
throw new InvalidOperationException("Already initialized");
}
if (tracer == null)
{
throw new ArgumentNullException("tracer");
}
m_tracer = tracer;
if (m_tracer.IsDebugEnabled)
{
m_tracer.Debug(string.Concat("Initializing RAM storage driver for settings string: ", settings));
}
m_changesets = new ConcurrentDictionary<long, RamDriverChangeset>();
var initString = settings as string;
if (!string.IsNullOrEmpty(initString))
{
m_settings = new RamDriverSettings();
if (0 == StringComparer.OrdinalIgnoreCase.Compare("demo", initString))
{
m_settings.InitializationCommand = initString;
}
else
{
m_settings.StorageRoot = initString;
}
}
else
{
m_settings = new RamDriverSettings(settings as RamDriverSettings);
}
CheckInitialized();
if (m_tracer.IsInfoEnabled)
{
m_tracer.Info("RAM storage driver ready");
}
}
public static void Debug(string message, params object[] args)
{
_tracer.Debug(message, args);
}
/// <summary>
/// 工作运行
/// Note:两次间隔有100豪秒左右的误差
/// </summary>
public virtual void Working()
{
try
{
TaskWorker.Change(Timeout.Infinite, Timeout.Infinite); //暂停计时。
if (IsRemoving)
{
Log.Warn("任务被标起为移除，回调中断，且任务不会被再次调用。");
return;
}
//执行今日任务
_runTimes++;
var now = SystemTime.Now();
//Log.Info($"[{this}] 第{_runTimes}次执行开始。[{now:HH:mm:ss ffff}] ◇");
ChangeStatus(TaskRunStatusType.Working);
var val = new TaskResult();
try
{
val = WorkHandler(); //同步委托，任务执行[可能较耗时]
}
catch (Exception ex)
{
Log.Error($"执行任务<{Task}>时发生异常:", ex);
//throw;
val.Result = TaskResultType.Error;
val.ExtendMessage = ex.Message;
}
finally
{
ChangeStatus(TaskRunStatusType.Worked);
Task.Execution.LastRun = now;
}
var runSpan = SystemTime.Now() - now;
Log.Info($"[{this}] 第{_runTimes}次执行结果[{val.Result} : {val.Message}] [Execution:{runSpan}]");
//Note:工作完成后的状态处理
//Note:注意，这里的错误次数实际上是执行失败的次数
if (val.Result.HasFlag(TaskResultType.Error))
{
var sleepInterval = ((TimeSpan)Task.WorkSetting.SleepInterval);
Task.Execution.SleepTimes++;
Log.Warn($"[{this}] 状态更新[{val.Result}],休眠次数++ ，准备[{sleepInterval}]后再次执行");
TaskWorker.Change(sleepInterval, TimeSpan.FromMilliseconds(-1));
return;
}
else
{
Task.Execution.RunTimes++;
var runInterval = Task.GetNextInterval();
if (runInterval == null)
{
ChangeStatus(TaskRunStatusType.Removing);
Log.Debug($"[{this}] 下次运行时间为null，当前任务停止。");
return;
}
if (runInterval.Value.TotalMilliseconds > WorkerInterval * 5)
{
ChangeStatus(TaskRunStatusType.Removing);
Log.Debug($"[{this}] 下次运行时间{runInterval}，超过5倍工作线程间隔，暂时移除执行队列。当前任务停止。");
return;
}
//var runInterval = runTime.Value.Subtract(now);
SaveExecution();
Log.Debug($"[{this}]第{_runTimes}次执行结束。 运行成功[Times:{Task.Execution.RunTimes}] ，准备[{runInterval}]后再次执行 ◆");
TaskWorker.Change(runInterval.Value, TimeSpan.FromMilliseconds(-1)); //Note:更改计时器约50多毫秒
}
#region 根据任务配置做出相应动作
//本次任务已完成,Note:只有本次任务达到所设条件才算是正常完成，正常完成后才更新最后成功完成的时间。
if ((Task.Execution.RunTimes >= Task.WorkSetting.Times && Task.WorkSetting.Times > 0) ||
(val.Result.HasFlag(TaskResultType.Finished)))
{
//Task.Meta.Execution.LastSucceedRun = PathDate ?? now; //Note:可自动补全点
//Task.Meta.Execution.RunStatus = TaskRunStatusType.TodayComplete;
ChangeStatus(TaskRunStatusType.Removing);
Log.Debug($"■ [{this}] ({Task.Execution.LastSucceedRun})完成。■");
return;
}
//根据设定，一旦有错误发生。立即停止
if (val.Result.HasFlag(TaskResultType.Error) && Task.WorkSetting.ErrorWay == ErrorWayType.Stop)
{
ChangeStatus(TaskRunStatusType.Removing);
Log.Info($"▲ [{this}] 根据设定Stop，发生了错误一次，等待移除。▲");
return;
}
//根据设定，有错误时。休眠超期后停止
if (Task.Execution.SleepTimes >= Task.WorkSetting.SleepInterval.Times &&
Task.WorkSetting.SleepInterval.Times > 0 && Task.WorkSetting.ErrorWay == ErrorWayType.TryAndStop)
{
ChangeStatus(TaskRunStatusType.Removing);
Log.Info($"▲ [{this}] 根据设定Sleep，发生了错误{Task.Execution.SleepTimes}次，等待移除。▲");
return;
}
#endregion
}
catch (Exception ex)
{
//Note:异常发生后停止该任务，不管任何原因
Log.Error($"[{this}] 执行异常，停止执行。", ex);
Stop();
//throw;
}
finally
{
}
}