private void StartJob(ContinuousJob continuousJob)
{
// Do not go further if already started or job is disabled
if (IsDisabled)
{
UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
return;
}
if (Interlocked.Exchange(ref _started, 1) == 1)
{
return;
}
_continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);
_continuousJobThread = new Thread(() =>
{
try
{
while (_started == 1 && !IsDisabled)
{
// Try getting the singleton lock if single is enabled
if (!TryGetLockIfSingleton())
{
// Wait 5 seconds and retry to take the lock
WaitForTimeOrStop(TimeSpan.FromSeconds(5));
continue;
}
_continuousJobLogger.StartingNewRun();
InitializeJobInstance(continuousJob, _continuousJobLogger);
RunJobInstance(continuousJob, _continuousJobLogger, String.Empty);
if (_started == 1 && !IsDisabled)
{
TimeSpan webJobsRestartTime = Settings.GetWebJobsRestartTime();
_continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
_continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
WaitForTimeOrStop(webJobsRestartTime);
}
}
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
finally
{
ReleaseSingletonLock();
}
});
_continuousJobThread.Start();
}
private void StartJob(ContinuousJob continuousJob)
{
// Do not go further if already started or job is disabled
if (IsDisabled)
{
UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
return;
}
if (Interlocked.Exchange(ref _started, 1) == 1)
{
return;
}
_continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);
CheckAlwaysOn();
_continuousJobThread = new Thread(() =>
{
try
{
while (_started == 1 && !IsDisabled)
{
// Try getting the singleton lock if single is enabled
if (!TryGetLockIfSingleton())
{
// Wait 5 seconds and retry to take the lock
WaitForTimeOrStop(TimeSpan.FromSeconds(5));
continue;
}
Stopwatch liveStopwatch = Stopwatch.StartNew();
_continuousJobLogger.StartingNewRun();
using (new Timer(LogStillRunning, null, TimeSpan.FromHours(1), TimeSpan.FromHours(12)))
{
InitializeJobInstance(continuousJob, _continuousJobLogger);
RunJobInstance(continuousJob, _continuousJobLogger, String.Empty, String.Empty);
}
if (_started == 1 && !IsDisabled)
{
// The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob
// Was running for at least 2 minutes there is no wait time.
TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero;
_continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
_continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
WaitForTimeOrStop(webJobsRestartTime);
}
// Make sure lock is released before re-iterating and trying to get the lock again
ReleaseSingletonLock();
}
}
catch (ThreadAbortException)
{
TraceFactory.GetTracer().TraceWarning("Thread was aborted, make sure WebJob was about to stop.");
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
finally
{
ReleaseSingletonLock();
}
});
_continuousJobThread.Start();
}
private void StartJob(ContinuousJob continuousJob)
{
// Do not go further if already started or job is disabled
if (IsDisabled)
{
UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
return;
}
if (Interlocked.Exchange(ref _started, 1) == 1)
{
return;
}
_continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);
CheckAlwaysOn();
_continuousJobThread = new Thread(() =>
{
var threadAborted = false;
while (!threadAborted && _started == 1 && !IsDisabled)
{
try
{
// Try getting the singleton lock if single is enabled
bool acquired;
if (!TryGetLockIfSingleton(out acquired))
{
// Wait 5 seconds and retry to take the lock
WaitForTimeOrStop(TimeSpan.FromSeconds(5));
continue;
}
try
{
Stopwatch liveStopwatch = Stopwatch.StartNew();
_continuousJobLogger.StartingNewRun();
var tracer = TraceFactory.GetTracer();
using (tracer.Step("Run {0} {1}", continuousJob.JobType, continuousJob.Name))
using (new Timer(LogStillRunning, null, TimeSpan.FromHours(1), TimeSpan.FromHours(12)))
{
InitializeJobInstance(continuousJob, _continuousJobLogger);
WebJobPort = GetAvailableJobPort();
RunJobInstance(continuousJob, _continuousJobLogger, String.Empty, String.Empty, tracer, WebJobPort);
}
if (_started == 1 && !IsDisabled)
{
// The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob
// Was running for at least 2 minutes there is no wait time.
TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero;
_continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
_continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
WaitForTimeOrStop(webJobsRestartTime);
}
}
finally
{
if (acquired)
{
// Make sure lock is released before re-iterating and trying to get the lock again
ReleaseSingletonLock();
}
}
}
catch (ThreadAbortException ex)
{
// by nature, ThreadAbortException will be rethrown at the end of this catch block and
// this bool may not be neccessary since while loop will be exited anyway. we added
// it to be explicit.
threadAborted = true;
if (!ex.AbortedByKudu())
{
TraceFactory.GetTracer().TraceWarning("Thread was aborted, make sure WebJob was about to stop.");
}
}
catch (Exception ex)
{
_analytics.UnexpectedException(ex, trace: true);
// sleep to avoid tight exception loop
WaitForTimeOrStop(TimeSpan.FromSeconds(60));
}
}
});
_continuousJobThread.Start();
}
