public ContinuousJobRunnerFacts()
{
FileSystemHelpers.Instance = new FileSystem();
_job = new ContinuousJob
{
Name = "testjob",
JobBinariesRootPath = @"c:\test\data\continuous\testjob"
};
_environment = new TestEnvironment
{
TempPath = @"c:\temp",
JobsBinariesPath = @"c:\test\data\continuous\testjob",
JobsDataPath = Path.GetTempPath(),
DataPath = @"c:\test\data"
};
string jobDirectory = Path.Combine(_environment.JobsDataPath, "continuous", _job.Name);
_logFilePath = Path.Combine(jobDirectory, "job_log.txt");
MockDeploymentSettingsManager mockSettingsManager = new MockDeploymentSettingsManager();
Mock<ITraceFactory> mockTraceFactory = new Mock<ITraceFactory>(MockBehavior.Strict);
Mock<IAnalytics> mockAnalytics = new Mock<IAnalytics>(MockBehavior.Strict);
Mock<ITracer> mockTracer = new Mock<ITracer>(MockBehavior.Strict);
mockTracer.Setup(p => p.Trace(It.IsAny<string>(), It.IsAny<IDictionary<string, string>>()));
mockTraceFactory.Setup(p => p.GetTracer()).Returns(mockTracer.Object);
_runner = new ContinuousJobRunner(_job, _environment, mockSettingsManager, mockTraceFactory.Object, mockAnalytics.Object);
FileSystemHelpers.DeleteFileSafe(_logFilePath);
}
public ContinuousJobRunner(ContinuousJob continuousJob, IEnvironment environment, IDeploymentSettingsManager settings, ITraceFactory traceFactory, IAnalytics analytics)
: base(continuousJob.Name, Constants.ContinuousPath, environment, settings, traceFactory, analytics)
{
_continuousJobLogger = new ContinuousJobLogger(continuousJob.Name, Environment, TraceFactory);
_disableFilePath = Path.Combine(continuousJob.JobBinariesRootPath, "disable.job");
_singletonLock = new LockFile(Path.Combine(JobDataPath, "singleton.job.lock"), TraceFactory);
}
private void StartJob(ContinuousJob continuousJob)
{
// Do not go further if already started or job is disabled
if (IsDisabled)
{
UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
return;
}
if (Interlocked.Exchange(ref _started, 1) == 1)
{
return;
}
_continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);
_continuousJobThread = new Thread(() =>
{
try
{
while (_started == 1 && !IsDisabled)
{
// Try getting the singleton lock if single is enabled
if (!TryGetLockIfSingleton())
{
// Wait 5 seconds and retry to take the lock
WaitForTimeOrStop(TimeSpan.FromSeconds(5));
continue;
}
_continuousJobLogger.StartingNewRun();
InitializeJobInstance(continuousJob, _continuousJobLogger);
RunJobInstance(continuousJob, _continuousJobLogger);
if (_started == 1 && !IsDisabled)
{
TimeSpan jobsInterval = Settings.GetJobsInterval();
_continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(jobsInterval.TotalSeconds));
_continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
WaitForTimeOrStop(jobsInterval);
}
}
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
finally
{
DisposeSingletonLock();
}
});
_continuousJobThread.Start();
}
private void PushAndVerifyConsoleWorker(ApplicationManager appManager, TestRepository testRepository, string[] expectedVerificationFileLines, int expectedDeployments = 1)
{
appManager.GitDeploy(testRepository.PhysicalPath);
var results = appManager.DeploymentManager.GetResultsAsync().Result.ToList();
Assert.Equal(expectedDeployments, results.Count);
for (int i = 0; i < expectedDeployments; i++)
{
Assert.Equal(DeployStatus.Success, results[i].Status);
}
var expectedContinuousJob = new ContinuousJob()
{
Name = "deployedJob",
JobType = "continuous",
Status = "Running",
RunCommand = "ConsoleWorker.exe"
};
WaitUntilAssertVerified(
"verify continuous job",
TimeSpan.FromSeconds(60),
() =>
{
ContinuousJob deployedJob = appManager.JobsManager.GetContinuousJobAsync("deployedJob").Result;
AssertContinuousJob(expectedContinuousJob, deployedJob);
});
WaitUntilAssertVerified(
"verification file",
TimeSpan.FromSeconds(30),
() =>
{
VerifyVerificationFile(appManager, expectedVerificationFileLines);
});
}
public void ContinuousJobStartsAfterGoingDown()
{
RunScenario("ContinuousJobStartsAfterGoingDown", appManager =>
{
TestTracer.Trace("Copying the script to the continuous job directory");
appManager.JobsManager.CreateContinuousJobAsync("basicJob1", "run.cmd", JobScript).Wait();
var expectedContinuousJob = new ContinuousJob()
{
Name = "basicJob1",
JobType = "continuous",
Status = "PendingRestart",
RunCommand = "run.cmd"
};
WaitUntilAssertVerified(
"verify continuous job",
TimeSpan.FromSeconds(60),
() =>
{
ContinuousJob deployedJob = appManager.JobsManager.GetContinuousJobAsync("basicJob1").Result;
AssertContinuousJob(expectedContinuousJob, deployedJob);
});
TestTracer.Trace("Waiting for verification file to have 2 lines (which means it ran twice)");
var expectedVerificationFileContents = new List<string>();
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
WaitUntilAssertVerified(
"verification file",
TimeSpan.FromSeconds(30),
() => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));
TestTracer.Trace("Verify continuous job settings and set it to isSingleton: true");
JobSettings continuousJobSettings =
appManager.JobsManager.GetContinuousJobSettingsAsync(expectedContinuousJob.Name).Result;
Assert.False(continuousJobSettings.IsSingleton);
continuousJobSettings.SetSetting("is_singleton", true);
appManager.JobsManager.SetContinuousJobSettingsAsync(expectedContinuousJob.Name, continuousJobSettings).Wait();
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
WaitUntilAssertVerified(
"verification file",
TimeSpan.FromSeconds(30),
() => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));
TestTracer.Trace("Verify continuous job settings and set it to isSingleton: false");
continuousJobSettings =
appManager.JobsManager.GetContinuousJobSettingsAsync(expectedContinuousJob.Name).Result;
Assert.True(continuousJobSettings.GetSetting<bool>("is_singleton"));
continuousJobSettings.SetSetting("is_singleton", false);
appManager.JobsManager.SetContinuousJobSettingsAsync(expectedContinuousJob.Name, continuousJobSettings).Wait();
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
WaitUntilAssertVerified(
"verification file",
TimeSpan.FromSeconds(30),
() => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));
});
}
private void AssertContinuousJob(ContinuousJob expectedContinuousJob, ContinuousJob actualContinuousJob)
{
AssertJob(expectedContinuousJob, actualContinuousJob);
Assert.NotNull(actualContinuousJob);
Assert.Equal(expectedContinuousJob.Status, actualContinuousJob.Status);
Assert.NotNull(actualContinuousJob.LogUrl);
}
private void StartJob(ContinuousJob continuousJob)
{
// Do not go further if already started or job is disabled
if (IsDisabled)
{
UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
return;
}
if (Interlocked.Exchange(ref _started, 1) == 1)
{
return;
}
_continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);
CheckAlwaysOn();
_continuousJobThread = new Thread(() =>
{
try
{
while (_started == 1 && !IsDisabled)
{
// Try getting the singleton lock if single is enabled
if (!TryGetLockIfSingleton())
{
// Wait 5 seconds and retry to take the lock
WaitForTimeOrStop(TimeSpan.FromSeconds(5));
continue;
}
Stopwatch liveStopwatch = Stopwatch.StartNew();
_continuousJobLogger.StartingNewRun();
var tracer = TraceFactory.GetTracer();
using (tracer.Step("Run {0} {1}", continuousJob.JobType, continuousJob.Name))
using (new Timer(LogStillRunning, null, TimeSpan.FromHours(1), TimeSpan.FromHours(12)))
{
InitializeJobInstance(continuousJob, _continuousJobLogger);
WebJobPort = GetAvailableJobPort();
RunJobInstance(continuousJob, _continuousJobLogger, String.Empty, String.Empty, tracer, WebJobPort);
}
if (_started == 1 && !IsDisabled)
{
// The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob
// Was running for at least 2 minutes there is no wait time.
TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero;
_continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
_continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
WaitForTimeOrStop(webJobsRestartTime);
}
// Make sure lock is released before re-iterating and trying to get the lock again
ReleaseSingletonLock();
}
}
catch (ThreadAbortException)
{
TraceFactory.GetTracer().TraceWarning("Thread was aborted, make sure WebJob was about to stop.");
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
finally
{
ReleaseSingletonLock();
}
});
_continuousJobThread.Start();
}
public void RefreshJob(ContinuousJob continuousJob, JobSettings jobSettings, bool logRefresh)
{
if (logRefresh)
{
_continuousJobLogger.LogInformation("Detected WebJob file/s were updated, refreshing WebJob");
}
StopJob();
JobSettings = jobSettings;
StartJob(continuousJob);
}
private void StartJob(ContinuousJob continuousJob)
{
// Do not go further if already started or job is disabled
if (IsDisabled)
{
UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
return;
}
if (Interlocked.Exchange(ref _started, 1) == 1)
{
return;
}
_continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);
_continuousJobThread = new Thread(() =>
{
try
{
while (_started == 1 && !IsDisabled)
{
// Try getting the singleton lock if single is enabled
if (!TryGetLockIfSingleton())
{
// Wait 5 seconds and retry to take the lock
WaitForTimeOrStop(TimeSpan.FromSeconds(5));
continue;
}
Stopwatch liveStopwatch = Stopwatch.StartNew();
_continuousJobLogger.StartingNewRun();
InitializeJobInstance(continuousJob, _continuousJobLogger);
RunJobInstance(continuousJob, _continuousJobLogger, String.Empty);
if (_started == 1 && !IsDisabled)
{
// The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob
// Was running for at least 2 minutes there is no wait time.
TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero;
_continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
_continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
WaitForTimeOrStop(webJobsRestartTime);
}
}
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
finally
{
ReleaseSingletonLock();
}
});
_continuousJobThread.Start();
}
public void EnableJob(ContinuousJob continuousJob)
{
OperationManager.Attempt(() => FileSystemHelpers.DeleteFile(_disableFilePath));
StartJob(continuousJob);
}
public void RefreshJob(ContinuousJob continuousJob, JobSettings jobSettings)
{
StopJob();
_jobSettings = jobSettings;
StartJob(continuousJob);
}
private void StartJob(ContinuousJob continuousJob)
{
// Do not go further if already started or job is disabled
if (IsDisabled)
{
UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
return;
}
if (Interlocked.Exchange(ref _started, 1) == 1)
{
return;
}
_continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);
CheckAlwaysOn();
_continuousJobThread = new Thread(() =>
{
var threadAborted = false;
while (!threadAborted && _started == 1 && !IsDisabled)
{
try
{
// Try getting the singleton lock if single is enabled
bool acquired;
if (!TryGetLockIfSingleton(out acquired))
{
// Wait 5 seconds and retry to take the lock
WaitForTimeOrStop(TimeSpan.FromSeconds(5));
continue;
}
try
{
Stopwatch liveStopwatch = Stopwatch.StartNew();
_continuousJobLogger.StartingNewRun();
var tracer = TraceFactory.GetTracer();
using (tracer.Step("Run {0} {1}", continuousJob.JobType, continuousJob.Name))
using (new Timer(LogStillRunning, null, TimeSpan.FromHours(1), TimeSpan.FromHours(12)))
{
InitializeJobInstance(continuousJob, _continuousJobLogger);
WebJobPort = GetAvailableJobPort();
RunJobInstance(continuousJob, _continuousJobLogger, String.Empty, String.Empty, tracer, WebJobPort);
}
if (_started == 1 && !IsDisabled)
{
// The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob
// Was running for at least 2 minutes there is no wait time.
TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero;
_continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
_continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
WaitForTimeOrStop(webJobsRestartTime);
}
}
finally
{
if (acquired)
{
// Make sure lock is released before re-iterating and trying to get the lock again
ReleaseSingletonLock();
}
}
}
catch (ThreadAbortException ex)
{
// by nature, ThreadAbortException will be rethrown at the end of this catch block and
// this bool may not be neccessary since while loop will be exited anyway. we added
// it to be explicit.
threadAborted = true;
if (!ex.AbortedByKudu())
{
TraceFactory.GetTracer().TraceWarning("Thread was aborted, make sure WebJob was about to stop.");
}
}
catch (Exception ex)
{
_analytics.UnexpectedException(ex, trace: true);
// sleep to avoid tight exception loop
WaitForTimeOrStop(TimeSpan.FromSeconds(60));
}
}
});
_continuousJobThread.Start();
}
public void RefreshJob(ContinuousJob continuousJob)
{
StopJob();
StartJob(continuousJob);
}
public void ContinuousJobStartsAfterGoingDown()
{
RunScenario("ContinuousJobStartsAfterGoingDown", appManager =>
{
TestTracer.Trace("Copying the script to the continuous job directory");
WaitUntilAssertVerified(
"writing file",
TimeSpan.FromSeconds(10),
() => appManager.VfsManager.WriteAllText(BasicContinuousJobExecutablePath, JobScript));
var expectedContinuousJob = new ContinuousJob()
{
Name = "basicJob1",
JobType = "continuous",
Status = "PendingRestart",
RunCommand = "run.cmd"
};
WaitUntilAssertVerified(
"verify continuous job",
TimeSpan.FromSeconds(60),
() =>
{
ContinuousJob deployedJob = appManager.JobsManager.GetContinuousJobAsync("basicJob1").Result;
AssertContinuousJob(expectedContinuousJob, deployedJob);
});
TestTracer.Trace("Waiting for verification file to have 2 lines (which means it ran twice)");
var expectedVerificationFileContents = new List<string>();
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
WaitUntilAssertVerified(
"verification file",
TimeSpan.FromSeconds(30),
() => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));
appManager.JobsManager.SetSingletonContinuousJobAsync(expectedContinuousJob.Name, true).Wait();
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
WaitUntilAssertVerified(
"verification file",
TimeSpan.FromSeconds(30),
() => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));
appManager.JobsManager.SetSingletonContinuousJobAsync(expectedContinuousJob.Name, false).Wait();
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
WaitUntilAssertVerified(
"verification file",
TimeSpan.FromSeconds(30),
() => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));
});
}