//object _syncLock=new object();
public ProcessWatchDog(ILogger logger, IStateManager stateManager,
IBatchEngineSubscribers batchEngineSubscribers, ICacheAside cacheAside,
ISerializersFactory serializersFactory, IEntityFactory entityFactory, IEventAggregator eventAggregator,
IProcessDataStorage storage, IPubSubFactory pubSubFactory, IResolver resolver,
IBatchLoggerFactory loggerFactory, IProcessRepository registeredProcesses,
ProcessVolumePipeline volumePipeline) : base("WatchDog", logger)
{
_stateManager = stateManager;
_batchEngineSubscribers = batchEngineSubscribers;
_cacheAside = cacheAside;
_eventAggregator = eventAggregator;
_storage = storage;
_pubSubFactory = pubSubFactory;
_resolver = resolver;
_loggerFactory = loggerFactory;
_registeredProcesses = registeredProcesses;
_volumePipeline = volumePipeline;
_groupsHandler = new GroupsHandler(logger, batchEngineSubscribers, stateManager, serializersFactory, entityFactory, resolver, loggerFactory);
Interval = TimeSpan.FromMinutes(3);
_subGroupRemoved = eventAggregator.Subscribe4Broadcast <ProcessGroupRemovedMessage>(RemoveGroup);
_subRem = eventAggregator.Subscribe <TextMessage>(ProcessRemoved, Constants.EventProcessFinished);
_serializer = SerializersFactory.Instance.GetSerializer(typeof(GroupMessage));
this._volumeGenSub = eventAggregator.Subscribe <TextMessage>(OnVolumeGenerated, Constants.EventProcessVolumeGenerated);
_checkGroupSub =
eventAggregator.Subscribe <TextMessage>(CheckProcessGroup, Constants.EventCheckGroupCommand);
_retrySub = eventAggregator.Subscribe <TextMessage>(OnVolumeGenerated, Constants.EventProcessRetry);
_healthSub = eventAggregator.Subscribe4Broadcast <HealthMessage>(PublishHealth);
_systemLogger = _loggerFactory.GetSystemLogger();
this._processAddedSub = eventAggregator.Subscribe <TextMessage>(OnProcessGroupAdded, Constants.EventProcessGroupAdded);
_volErrorSub = eventAggregator.Subscribe <VolumeErrorMessage>(OnProcessVolumeError);
}
protected DistributedMutex(string key, Func <CancellationToken, Task> taskToRunWhenLockAcquired, Action secondaryAction, IFrameworkLogger logger)
{
Key = key;
_taskToRunWhenLockAcquired = taskToRunWhenLockAcquired;
_secondaryAction = secondaryAction;
Logger = logger;
}
public UdpPubSubFactory(string ip, int port, IBatchLoggerFactory logger)
{
_ip = ip;
_port = port;
_systemLogger = logger.GetSystemLogger();
_portSubscriber = new UdpPortSubscriber(_ip, port, _systemLogger, OnUdpMessage);
}
public Bus(IEntityFactory entityFactory, IVolumeHandler volumeHandler, IPubSubFactory pubSubFactory, IStateManager stateManager, IProcessDataStorage storage,
IDistributedMutexFactory distributedMutexFactory, IResolver resolver, IBatchLoggerFactory batchLoggerFactory)
{
EntityFactory = entityFactory;
VolumeHandler = volumeHandler;
PubSubFactory = pubSubFactory;
_stateManager = stateManager;
_storage = storage;
DistributedMutexFactory = distributedMutexFactory;
_resolver = resolver;
_batchLoggerFactory = batchLoggerFactory;
_cts = new CancellationTokenSource();
_cancellationToken = _cts.Token;
HookExceptionEvents();
_logger = batchLoggerFactory.GetSystemLogger();
EventAggregator = new TinyEventAggregator();
var wrapper = new BusStateManager(_stateManager, _logger, resolver);
_stateManager = wrapper;
var originalStorage = _storage;
_storage = new CacheBusWrapper(_logger, originalStorage, resolver);
_cacheCommandPipeline = new CacheStoragePipeline(_logger, _cancellationToken, originalStorage);
_cacheAside = new CacheAside(_stateManager, _storage, EventAggregator, _logger, batchLoggerFactory);
_processRepository = _resolver.Resolve <IProcessRepository>();
//var taskListener = resolver.Resolve<ITaskListener>();
//if (!ReferenceEquals(taskListener, _processRepository))
//{
// Console.WriteLine("ALERT");
//}
_taskExecutorsRepo = new TaskExecutorsPool(_logger, _cacheAside, _cancellationToken, _stateManager, _processRepository, EventAggregator, resolver, _logger);
//BuildCommandHandlerPipeline();
_statePersistencePipeline = new StatePersistencePipeline(_logger, _cancellationToken);
this._databasePipeline = new DatabasePipeline(_logger, _cancellationToken, 0);//todo 500
_taskProcessorPipeline = GetTaskProcessorPipeLine();
//_grouPipeline=new GroupHandlerPipeline(_stateManager, _logger, _branchEngineSubscriber);
//_volumePipeline = new ProcessVolumePipeline(_cancellationToken, _logger, _stateManager, _cacheAside, _processRepository, VolumeHandler, resolver, EventAggregator, _branchEngineSubscriber);
_branchEngineSubscriber = new BatchEngineSubscribers();
//_watchDog = new ProcessWatchDog(_logger, StateManager, _branchEngineSubscriber, _cacheAside, SerializersFactory.Instance, EntityFactory, EventAggregator, Storage);
//watchDog.Start(_cancellationToken);//todo
// _grouPipeline = new Pipeline<GroupMessage>(_watchDog);
//_watchDogPipeline = new Pipeline<IWatchDogMessage>(_watchDog);
_taskProducer = new TaskProducerWorker(_logger, _cacheAside, VolumeHandler, resolver, batchLoggerFactory);
_leaderManager = DistributedMutexFactory.CreateDistributedMutex(NodeSettings.Instance.LockKey, RunLocalWatchDog, () => SwitchToPubSubWatchDog(null), batchLoggerFactory.GetSystemLogger());
}
public JobScheduler(ISerializersFactory factory, IPubSubFactory pubSubFactory, IBatchLoggerFactory loggerFactory)
{
SystemLogger = loggerFactory.GetSystemLogger();
_publisher = pubSubFactory.GetPublisher(CancellationToken.None, SystemLogger, nameof(IWatchDogMessage));
//_ser = SerializersFactory.Instance.GetSerializer<List<JobCriteria>>();
//JsonSerializer serializer=new JsonSerializer();
_ser = factory.GetSerializer(typeof(JobCriteria));
}
public ProcessVolumePipeline(CancellationToken token, IFrameworkLogger logger, IStateManager stateManager,
ICacheAside cacheAside, IProcessRepository processRepository, IVolumeHandler volumeHandler,
IResolver resolver, IEventAggregator eventAggregator, IBatchEngineSubscribers batchEngineSubscribers)
: base(new ProcessVolumeRequestHandler(logger, stateManager, cacheAside, processRepository, volumeHandler,
token, resolver, eventAggregator, batchEngineSubscribers))
{
RegisterFeatureDecorator(new ConsumerFilter <ProcessExecutionContext>(token, logger, "VolumeGeneratorConsumer"));
}
public static void MarkAsError(this ProcessExecutionContext context, IStateManager stateManager,
string errorMessage, IProcessRepository registeredProcesses, IBatchEngineSubscribers batchEngineSubscribers,
IFrameworkLogger fLogger)
{
context.Logger.Error(errorMessage);
context.WritableProcessState.Status = CompletionStatus.Finished;
context.WritableProcessState.Result = ResultStatus.Error;
context.WritableProcessState.CompleteTime = DateTime.UtcNow;
stateManager.SaveProcess(context.WritableProcessState);
context.WritableProcessState.TriggerProcessEvents(registeredProcesses, context, true, context.ProcessState.IsStopped, batchEngineSubscribers, fLogger, errorMessage);
}
public ProcessConsumer(CancellationToken parentToken, int processKey, long processId,
long groupId,
IStateManager stateManager,
ILogger logger, ITask taskHandler, ISerializer serializer, Bus bus,
IFrameworkLogger frameworkLogger, ITaskListenerHandler taskListener,
IProcessExecutionContext executionContext) //, ICacheAside cacheAside //, ProcessConfiguration processConfiguration
{
_systemLogger = frameworkLogger;
_taskListener = taskListener;
//_cacheAside = cacheAside;
_processContext = executionContext; //cacheAside.GetProcessExecutionContext(processId);
_processConfiguration = _processContext.Configuration;
_lastInputTime = DateTime.UtcNow;
//_processId = processId;
_maxConsumers = ProcessConfiguration.BatchSize;
_parentToken = parentToken;
_logger = logger;
//_volumeHandler = volumeHandler;
_processId = processId; // processConfiguration.ProcessKey;
_groupId = groupId;
_processKey = processKey;
_stateManager = stateManager;
int queueSize = _processConfiguration.QueueSize ?? DefaultQueueSize; //Environment.ProcessorCount
queueSize = queueSize <= 0 ? 1 : queueSize;
_taskContextsQueue = new BlockingCollection <TaskContext>(queueSize);
_processTokenSource = CancellationTokenSource.CreateLinkedTokenSource(parentToken);
_processToken = _processTokenSource.Token;
_handler = taskHandler;// ApplicationTasksHandlers.Instance.GetProcessTaskHandler(_processConfiguration.ProcessKey);
if (_handler == null)
{
var error = $"Task processor for processKey {_processConfiguration.ProcessKey} not found.";
logger.Fatal(error);
throw new FrameworkException(error);
}
Type[] interfaces = _handler.GetType().GetInterfaces().Where(t => t.IsGenericType).ToArray();
bool isStateful = interfaces.Any(x => x.GetGenericTypeDefinition() == typeof(ITaskSaga <>)); // (typeof(ITaskSaga<>).IsAssignableFrom(handler.GetType()))
_isStateful = isStateful;
_serializer = serializer; // ApplicationTasksHandlers.Instance.GetSerializer(_handler);
_bus = bus;
}
//private readonly Action<ProcessExecutionContext, bool> _invokeProcessCompletion;
public VolumeGenerator(IBaseProcess process, IStateManager stateManager, IVolumeHandler volumeHandler,
IResolver resolver, CancellationToken cancellationToken, IProcessRepository registeredProcesses,
IBatchEngineSubscribers batchEngineSubscribers, IFrameworkLogger logger)
{
_process = process;
_stateManager = stateManager;
this._volumeHandler = volumeHandler;
_resolver = resolver;
_cancellationToken = cancellationToken;
_registeredProcesses = registeredProcesses;
_batchEngineSubscribers = batchEngineSubscribers;
_logger = logger;
}
public ProcessVolumeRequestHandler(IFrameworkLogger logger, IStateManager stateManager, ICacheAside cacheAside,
IProcessRepository registeredProcesses, IVolumeHandler volumeHandler, CancellationToken token,
IResolver resolver, IEventAggregator eventAggregator, IBatchEngineSubscribers batchEngineSubscribers)
{
_logger = logger;
_stateManager = stateManager;
_cacheAside = cacheAside;
_registeredProcesses = registeredProcesses;
_volumeHandler = volumeHandler;
_token = token;
_resolver = resolver;
_eventAggregator = eventAggregator;
_batchEngineSubscribers = batchEngineSubscribers;
}
//private Task GetTimeoutObserverTask()
//{
// var task = Task.Factory.StartNew(async () =>
// {
// while (!_token.IsCancellationRequested)
// {
// try
// {
// await Task.Delay(Math.Min(TaskTimoutCheckInterval, _processNotificationThresholdMilliSec), _token);
// if (_token.IsCancellationRequested)
// return;
// await SweepTimedoutTasks();
// CheckLastInputInterval();
// }
// catch (TaskCanceledException e)
// {
// var msg =
// $"Timeout observer task canceled Process: {_processKey} by token {(_processToken.IsCancellationRequested ? "ProcessToken" : (_parentToken.IsCancellationRequested ? "ParentToken" : "NoToken"))} with msg {e.Message}";
// Logger.Info(msg);
// }
// catch (OperationCanceledException e)
// {
// var msg =
// $"Timeout observer task canceled Process: {_processKey} by token {(_processToken.IsCancellationRequested ? "ProcessToken" : (_parentToken.IsCancellationRequested ? "ParentToken" : "NoToken"))} with msg {e.Message}";
// _logger.Info(msg);
// }
// catch (Exception e)
// {
// _logger.Error($"Timeout observer got unexpected error with message {e.Message}", e);
// }
// }
// _logger.Trace($"Timeout observer stopped for Process: {_processKey} by token {(_processToken.IsCancellationRequested ? "ProcessToken" : (_parentToken.IsCancellationRequested ? "ParentToken" : "NoToken"))}");
// }, _processToken);
// return task;
//}
#endregion
public TaskExecutorsPool(ILogger logger, ICacheAside cacheAside, CancellationToken token, IStateManager stateManager, IProcessRepository processRepository, IEventAggregator eventAggregator
, IResolver resolver, IFrameworkLogger frameworkLogger) : base(nameof(TaskExecutorsPool), logger)
{
_cacheAside = cacheAside;
_token = token;
_stateManager = stateManager;
_processRepository = processRepository;
_eventAggregator = eventAggregator;
_resolver = resolver;
_frameworkLogger = frameworkLogger;
//Start(_token);
Interval = TimeSpan.FromMilliseconds(1500);
//TaskTimoutCheckInterval = 1500;
}
public static void MarkProcessStatus(this IReadWritableProcessState state, CompletionStatus completionStatus,
ResultStatus result, string reason, IStateManager stateManager, IProcessRepository registeredProcesses,
IProcessExecutionContext executionContext, IBatchEngineSubscribers batchEngineSubscribers,
IFrameworkLogger fLogger)
{
state.Status = completionStatus;
state.Result = result;
if (completionStatus.IsDone())
{
state.CompleteTime = DateTime.UtcNow;
}
state.IsFinished = completionStatus.Id == CompletionStatus.Finished.Id;
state.IsStopped = completionStatus.Id == CompletionStatus.Stopped.Id;
stateManager.SaveProcess(state);
state.TriggerProcessEvents(registeredProcesses, executionContext, result.Id == ResultStatus.Error.Id, state.IsStopped,
batchEngineSubscribers, fLogger, reason);
}
public SqlDistributedLock(string key, Func <CancellationToken, Task> taskToRunWhenLockAcquired, Action secondaryAction, IFrameworkLogger logger) : base(key, taskToRunWhenLockAcquired, secondaryAction, logger)
{
}
public DistributedMutex CreateDistributedMutex(string key,
Func <CancellationToken, Task> taskToRunWhenLockAcquired, Action secondaryAction, IFrameworkLogger logger)
{
return(new RedisDistributedLock(_connection, key, taskToRunWhenLockAcquired, secondaryAction, logger));
}
static void TriggerProcessEvents(this IReadWritableProcessState state, IProcessRepository registeredProcesses, IProcessExecutionContext context,
bool isFailed, bool isStopped, IBatchEngineSubscribers batchEngineSubscribers, IFrameworkLogger fLogger, string reason)
{
var process = registeredProcesses.GetRegisteredProcesses().FirstOrDefault(p => p.ProcessKey == context.Configuration.ProcessKey);
//var context = _cacheAside.GetProcessExecutionContext(state.Id);
//process?.InvokeProcessCompeteEvent(context, isFailed);
var subscribers = batchEngineSubscribers.GetProcessSubscribers().Where(p => p.ProcessKey == context.Configuration.ProcessKey);
//ProcessRetryContext ct =new ProcessRetryContext(context.ProcessState.Id, context.ProcessState.ProcessKey, context.Logger);
ProcessCompleteContext ct = new ProcessCompleteContext(context.ProcessState.Id, context.ProcessState.ProcessId, false, false, context.Logger);
//IProcessCompleteContext ctx=new proccom
ProcessStoppedContext processStoppedContext = null;
foreach (var subscriber in subscribers)
{
if (isFailed)
{
Robustness.Instance.SafeCall(() =>
subscriber.ProcessFailed(ct)
, fLogger);
}
else if (isStopped)
{
if (processStoppedContext == null)
{
processStoppedContext = new ProcessStoppedContext(state.Id, state.ProcessId, context.Configuration.ProcessKey, reason, context.Logger);
}
var stoppedContext = processStoppedContext;//access to modified closure
Robustness.Instance.SafeCall(() =>
subscriber.OnProcessStop(stoppedContext)
, fLogger);
}
else
{
Robustness.Instance.SafeCall(() =>
subscriber.OnProcessComplete(ct)
, fLogger);
}
}
if (isStopped)
{
Robustness.Instance.SafeCall(() =>
{
if (processStoppedContext == null)
{
processStoppedContext = new ProcessStoppedContext(state.Id, state.ProcessId, context.Configuration.ProcessKey, reason, context.Logger);
}
process?.ProcessStopped(processStoppedContext);
});
}
else if (isFailed)
{
Robustness.Instance.SafeCall(() =>
{
process?.ProcessFailed(context);
});
}
else
{
Robustness.Instance.SafeCall(() =>
{
process?.ProcessCompleted(context);
});
}
Robustness.Instance.SafeCall(() =>
{
process?.ProcessFinalizer(context);
});
}
internal static bool CanRetryProcess(this IReadWritableProcessState state,
IProcessExecutionContext executionContext,
ILogger processLogger, IProcessRepository registeredProcesses,
IBatchEngineSubscribers batchEngineSubscribers, IStateManager stateManager, IFrameworkLogger fLogger,
out bool stop, out string stopMessage)
{
stop = false;
stopMessage = string.Empty;
var process = registeredProcesses.GetRegisteredProcesses().FirstOrDefault(p => p.ProcessKey == executionContext.Configuration.ProcessKey);
ProcessRetryContext context = new ProcessRetryContext(state.Id, state.ProcessId, processLogger, executionContext);
process?.InvokeProcessRetry(context);
if (context.StopFlag)
{
processLogger.Warn("Retry stopped by process class {processType}", process?.GetType());
//state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error,
// "Retry stopped by process class", stateManager, registeredProcesses, executionContext, batchEngineSubscribers, fLogger);
stopMessage = "Retry stopped by process class";
stop = true;
return(false);
}
foreach (var processSubscriber in batchEngineSubscribers.GetProcessSubscribers().Where(s => s.ProcessKey == context.Configuration.ProcessKey))
{
Robustness.Instance.SafeCall(() => processSubscriber.OnProcessRetry(context),
executionContext.Logger);
if (context.StopFlag)
{
processLogger.Warn($"Retry stopped by extension {processSubscriber.GetType()}");
stop = true;
stopMessage = $"Retry stopped by extension {processSubscriber.GetType()}";
//state.MarkProcessStatus(CompletionStatus.Finished, ResultStatus.Error,
// $"Retry stopped by extension {processSubscriber.GetType()}", stateManager, registeredProcesses, executionContext, batchEngineSubscribers, fLogger);
//_eventAggregator.Publish(this, Constants.EventProcessStop,
// processId.ToString());
return(false);
}
}
return(true);
}
public RedisDistributedLock(string connection, string key,
Func <CancellationToken, Task> taskToRunWhenLockAcquired, Action secondaryAction, IFrameworkLogger logger) : base(key, taskToRunWhenLockAcquired, secondaryAction, logger)
{
_connection = connection;
_serializer = new RedisSerializer();
}
public BluetoothManager(IFrameworkLogger _logger)
{
logger = _logger;
}