void currentDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e)
{
if (e.ExceptionObject is Exception)
{
_logger.Exception("Unhandled exception", e.ExceptionObject as Exception);
}
else
{
_logger.Error("Unknown exception");
}
}
void processRetryMail(CancellationToken token)
{
if (_activeRetryCount > _maxParallelMessages)
{
return;
}
Interlocked.Increment(ref _activeRetryCount);
QueuedMail msg = null;
try {
if (!token.IsCancellationRequested && _retryQueue.TryDequeue(out msg))
{
if (++msg.RetryCount > _maxRetry)
{
_logger.Error("Sending mail '{0}' to {1}: maxretry has been exceeded {2}\r\nLast error is: {3}", msg.Msg.Subject, msg.Msg.Recipients, _maxRetry, msg.LastError.ToString());
msg.Msg.Dispose();
}
else
{
DateTime dt = DateTime.UtcNow;
using (ISmpClient cli = _clifac.Create())
{
cli.Send(msg.Msg);
}
if ((DateTime.UtcNow - dt).TotalSeconds > WarnTimeInSec)
{
_logger.Warn("Sending mail '{0}', to {1} took extratime: {2}s", msg.Msg.Subject, msg.Msg.Recipients, (DateTime.UtcNow - dt).TotalSeconds);
}
msg.Msg.Dispose();
}
}
}
catch (Exception ex)
{
_logger.Exception(string.Format("On sending news, queue size = {0}", _retryQueue.Count), ex);
if (!token.IsCancellationRequested)
{
msg.LastError = ex;
_retryQueue.Enqueue(msg);
}
}
finally {
Interlocked.Decrement(ref _activeRetryCount);
}
if (_retryQueue.Count > 0 && !token.IsCancellationRequested)
{
startRetrying();
}
}
void wkfn()
{
try {
WaitHandle[] handles = new WaitHandle[] { _evHasData, _evStop.WaitHandle };
while (true)
{
_operationStarted = 0;
int idx = WaitHandle.WaitAny(handles);
if (idx == 0)
{
T workUnit;
while (_queue.TryDequeue(out workUnit))
{
Interlocked.Exchange(ref _operationStarted, DateTime.UtcNow.ToBinary());
try {
if (null != workUnit)
{
try {
ExecuteTask(workUnit);
}
catch (Exception ex)
{
_logger.Exception(string.Format("AT processing {0}", GetTaskDescription(workUnit)), ex);
}
}
else
{
_logger.Error("Thread {0}: null work in queue.", Name);
}
}
finally {
_operationStarted = 0;
}
}
}
else
{
break;
}
}
}
catch (ThreadAbortException)
{
Thread.ResetAbort();
}
catch (Exception ex)
{
_logger.Exception(string.Format("Main wkfn of '{0}' was broken", Name), ex);
}
}
/// <summary>
/// Выполняет задачу.
/// </summary>
void executer(object task)
{
#if DEBUG
int ec = Interlocked.Increment(ref _entranceCount);
if (ec > 1)
{
_logger.Error("ThreadPoolSequentialExecuter: task parallel execution is detected '{0}'", ec);
}
#endif
if (Thread.VolatileRead(ref _disposed) == 0)
{
T tt = (T)task;
Exception excKeep = null;
try {
ExecuteTask(tt);
}
catch (Exception ex)
{
excKeep = ex;
_logger.Exception("Error of a task execution", ex);
}
try {
onTaskFinished(tt, excKeep);
}
catch (Exception ex)
{
_logger.Exception("Error of a task execution/finishing", ex);
}
}
#if DEBUG
Interlocked.Decrement(ref _entranceCount);
#endif
trySpawnNextTask();
}
protected override void OnContinue()
{
base.OnContinue();
base.OnPause();
CancellationTokenSource src = new CancellationTokenSource();
_logger.Info("Service '{0}' recieved OnContinue command", WinServiceName);
Task <RuntimeServiceState> tsk = _stateMachine.Transit(RuntimeServiceState.Started, src.Token);
tsk.ContinueWith((t) => src.Dispose());
bool res = false;
System.Diagnostics.Stopwatch w = System.Diagnostics.Stopwatch.StartNew();
bool hasError = false;
try {
res = tsk.Wait(pauseResumeTimeoutMS);
}
catch (AggregateException ex)
{
hasError = true;
_logger.Error("Service '{0}' failed to process OnContinue command", WinServiceName);
ex.Handle((err) => {
if (err is OperationCanceledException)
{
_logger.Warn("Service '{0}' OnContinue was cancelled", WinServiceName);
if (_testMode)
{
Console.WriteLine(string.Format("Service '{0}' OnContinue was cancelled", WinServiceName));
}
return(true);
}
else
{
_logger.Exception(string.Format("Service '{0}' failed to process OnContinue command", WinServiceName), err);
if (_testMode)
{
Console.WriteLine(err.ToString());
}
}
return(true);
});
}
catch (OperationCanceledException)
{
hasError = true;
_logger.Warn("Service '{0}' OnContinue was cancelled", WinServiceName);
if (_testMode)
{
Console.WriteLine("Resuming has been cancelled");
}
}
catch (Exception ex)
{
hasError = true;
_logger.Exception(string.Format("Service '{0}' failed to process OnContinue command", WinServiceName), ex);
if (_testMode)
{
Console.WriteLine(ex.ToString());
}
}
finally {
w.Stop();
}
if (!res)
{
try { src.Cancel(); } catch {}
if (!hasError)
{
string msg = string.Format("Timeout of resuming service '{0}', elapsed: {1:hh\\:mm\\:ss\\.ff}", WinServiceName, w.Elapsed);
_logger.Error(msg);
if (_testMode)
{
Console.WriteLine(msg);
}
}
#pragma warning disable 4014
_logger.Info("Requesting additional time and stop...");
requestPauseResumeTime();
//try {_stateMachine.Transit(RuntimeServiceState.Stopped, CancellationToken.None).Wait(startStopTimeoutMS);} catch {}
//часть сервисов могла запуститься, поэтому состояние хоста не целостное (сервисы в разных состояниях), и мы всё останавливаем
try { stop(_stateMachine.State, RuntimeServiceState.Stopped, CancellationToken.None); } catch {}
#pragma warning restore 4014
throw new Exception(hasError ? "OnContinue rolled back by error":"OnContinue rolled back by timeout");
}
_logger.Info("Service '{0}' successfully processed OnContinue command, elapsed: {1:hh\\:mm\\:ss\\.ff}", WinServiceName, w.Elapsed);
}
public async Task <TState> Transit(TState newState, CancellationToken cts, object args = null)
{
if (_logTransitions)
{
_logger.Info("Machine '{0}' is about to transit {1} --> {2}", Name, State, newState);
}
if (!StateDef <TState> .IsValid(newState))
{
return(await compliteWithError(new Exception(string.Format("State '{0}' is invalid for machine '{1}'", newState, Name))));
}
if (IsInFinalState)
{
return(await compliteWithError(new Exception(string.Format("Machine '{0}' can't transit to state '{1}' because it is in final state '{2}'", Name, newState, _state.Final))));
}
int res = Interlocked.Exchange(ref _isInTransition, 1); //блокировка транзишенов до тех пор, пока не заевршится текущий транзишен
if (res != 0)
{
if (EqualityComparer <TState> .Default.Equals(newState, FinalState)) //разрешаем только рекурсивный Close
{
_logger.Warn("Machine '{0}' started reqursive transition {1} --> {2}", Name, State, newState);
}
else
{
return(await compliteWithError(new Exception(string.Format("In machine '{0}' reqursive transition {1} --> {2} has been detected", Name, State, newState))));
}
}
TState transitionResult = State;
CancellationTokenRegistration reg = default(CancellationTokenRegistration);
bool lockCleared = false;
try {
TState stateFrom = State;
TransitionDef <TState> transition = getTransition(stateFrom, newState);
if (transition == TransitionDef <TState> .Empty)
{
throw new Exception(string.Format("Machine '{0}' didn't find transition {1} --> {2}", Name, stateFrom, newState));
}
if (_logTransitions)
{
_logger.Info("Machine '{0}' is transiting {1} --> {2}", Name, stateFrom, newState);
}
if (transition.StateActivityAsync != null)
{
try {
TransitionState = _state.GetTransitionalState(State, newState);
if (TransitionState != null)
{
onStateChanging(TransitionState.Value);
}
reg = cts.Register(() => { lockCleared = true; Interlocked.Exchange(ref _isInTransition, 0); });
transitionResult = await transition.StateActivityAsync(stateFrom, newState, cts, args).WithAllExceptions().ConfigureAwait(false);
cts.ThrowIfCancellationRequested();
State = transitionResult;
if (_logTransitions)
{
_logger.Info("Machine '{0}' transited {1} --> {2}", Name, stateFrom, newState);
}
onStateChanged(stateFrom, transitionResult);
}
catch (AggregateException ex)
{
_logger.Error("Machine '{0}' failed to transit {1} --> {2}", Name, stateFrom, newState);
onStateChanged(stateFrom, transitionResult, ex);
ex.Handle((err) => {
if (err is OperationCanceledException)
{
_logger.Warn("Machine '{0}' transition {1} --> {2} was cancelled", Name, stateFrom, newState);
return(true);
}
return(false);
});
}
catch (OperationCanceledException ce)
{
_logger.Warn("Machine '{0}' transition {1} --> {2} was cancelled", Name, stateFrom, newState);
onStateChanged(stateFrom, transitionResult, ce);
}
catch (Exception ex)
{
_logger.Exception(string.Format("Machine '{0}' failed to transit {1} --> {2}", Name, stateFrom, newState), ex);
onStateChanged(stateFrom, transitionResult, ex);
throw;
}
}
else
{
State = newState;
transitionResult = newState;
if (_logTransitions)
{
_logger.Info("Machine '{0}' transited {1} --> {2} with no action", Name, stateFrom, newState);
}
onStateChanged(stateFrom, newState);
}
}
finally {
if (!lockCleared)
{
Interlocked.Exchange(ref _isInTransition, 0);
TransitionState = null;
}
}
return(transitionResult);
}