public ContinuousJobRunnerFacts()
{
_job = new ContinuousJob
{
Name = "testjob",
JobBinariesRootPath = @"c:\test\data\continuous\testjob"
};
_environment = new TestEnvironment
{
TempPath = @"c:\temp",
JobsBinariesPath = @"c:\test\data\continuous\testjob",
JobsDataPath = Path.GetTempPath(),
DataPath = @"c:\test\data"
};
string jobDirectory = Path.Combine(_environment.JobsDataPath, "continuous", _job.Name);
_logFilePath = Path.Combine(jobDirectory, "job_log.txt");
MockDeploymentSettingsManager mockSettingsManager = new MockDeploymentSettingsManager();
Mock <ITraceFactory> mockTraceFactory = new Mock <ITraceFactory>(MockBehavior.Strict);
Mock <IAnalytics> mockAnalytics = new Mock <IAnalytics>(MockBehavior.Strict);
Mock <ITracer> mockTracer = new Mock <ITracer>(MockBehavior.Strict);
mockTracer.Setup(p => p.Trace(It.IsAny <string>(), It.IsAny <IDictionary <string, string> >()));
mockTraceFactory.Setup(p => p.GetTracer()).Returns(mockTracer.Object);
_runner = new ContinuousJobRunner(_job, _environment, mockSettingsManager, mockTraceFactory.Object, mockAnalytics.Object);
FileSystemHelpers.DeleteFileSafe(_logFilePath);
}
private void AssertContinuousJob(ContinuousJob expectedContinuousJob, ContinuousJob actualContinuousJob)
{
AssertJob(expectedContinuousJob, actualContinuousJob);
Assert.NotNull(actualContinuousJob);
Assert.Equal(expectedContinuousJob.Status, actualContinuousJob.Status);
Assert.NotNull(actualContinuousJob.LogUrl);
}
/// <summary>
/// Initializes a new instance of the <see cref="HighResolutionTimer{TState}" /> class.
/// </summary>
internal HighResolutionTimer(TState state)
{
this.state = state;
this.timerJob = new CancellableJob(this.Timer);
this.timerActionJob = new ContinuousJob <BlockingCollection <Action> >(this.TimerControl, new BlockingCollection <Action>(new ConcurrentQueue <Action>()));
this.disposeCancellationToken = this.timerJob.Start().Value;
this.timerActionJob.Start();
}
public ContinuousJobRunner(ContinuousJob continuousJob, IEnvironment environment, IDeploymentSettingsManager settings, ITraceFactory traceFactory, IAnalytics analytics)
: base(continuousJob.Name, Constants.ContinuousPath, environment, settings, traceFactory, analytics)
{
_continuousJobLogger = new ContinuousJobLogger(continuousJob.Name, Environment, TraceFactory);
_disableFilePath = Path.Combine(continuousJob.JobBinariesRootPath, "disable.job");
_singletonLock = new LockFile(Path.Combine(JobDataPath, "singleton.job.lock"), TraceFactory);
}
private void StartJob(ContinuousJob continuousJob)
{
// Do not go further if already started or job is disabled
if (IsDisabled)
{
UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
return;
}
if (Interlocked.Exchange(ref _started, 1) == 1)
{
return;
}
_continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);
_continuousJobThread = new Thread(() =>
{
try
{
while (_started == 1 && !IsDisabled)
{
// Try getting the singleton lock if single is enabled
if (!TryGetLockIfSingleton())
{
// Wait 5 seconds and retry to take the lock
WaitForTimeOrStop(TimeSpan.FromSeconds(5));
continue;
}
_continuousJobLogger.StartingNewRun();
InitializeJobInstance(continuousJob, _continuousJobLogger);
RunJobInstance(continuousJob, _continuousJobLogger, String.Empty);
if (_started == 1 && !IsDisabled)
{
TimeSpan webJobsRestartTime = Settings.GetWebJobsRestartTime();
_continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
_continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
WaitForTimeOrStop(webJobsRestartTime);
}
}
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
finally
{
ReleaseSingletonLock();
}
});
_continuousJobThread.Start();
}
public void RefreshJob(ContinuousJob continuousJob, JobSettings jobSettings, bool logRefresh)
{
if (logRefresh)
{
_continuousJobLogger.LogInformation("Detected WebJob file/s were updated, refreshing WebJob");
}
StopJob();
JobSettings = jobSettings;
StartJob(continuousJob);
}
public HttpResponseMessage GetContinuousJob(string jobName)
{
ContinuousJob continuousJob = _continuousJobsManager.GetJob(jobName);
if (continuousJob != null)
{
return(Request.CreateResponse(HttpStatusCode.OK, continuousJob));
}
return(Request.CreateResponse(HttpStatusCode.NotFound));
}
public HttpResponseMessage GetContinuousJob(string jobName)
{
ContinuousJob continuousJob = _continuousJobsManager.GetJob(jobName);
if (continuousJob != null)
{
return(Request.CreateResponse(HttpStatusCode.OK, ArmUtils.AddEnvelopeOnArmRequest(continuousJob, Request)));
}
return(Request.CreateResponse(HttpStatusCode.NotFound));
}
public ContinuousJobRunner(ContinuousJob continuousJob, string basePath, IEnvironment environment, IDeploymentSettingsManager settings, ITraceFactory traceFactory, IAnalytics analytics)
: base(continuousJob.Name, Constants.ContinuousPath, basePath, environment, settings, traceFactory, analytics)
{
_analytics = analytics;
_continuousJobLogger = new ContinuousJobLogger(continuousJob.Name, Environment, TraceFactory);
_continuousJobLogger.RolledLogFile += OnLogFileRolled;
_isUsingSdk = continuousJob.UsingSdk;
_disableFilePath = Path.Combine(continuousJob.JobBinariesRootPath, "disable.job");
_singletonLock = new LockFile(Path.Combine(JobDataPath, "singleton.job.lock"), TraceFactory, ensureLock: true);
}
public void Stop_Then_ResultShouldBeSuccess()
{
using var resetEvent = new ManualResetEventSlim();
using var testee = new ContinuousJob(_ => resetEvent.Set());
testee.Start();
resetEvent.Wait();
Thread.Sleep(1);
var result = testee.Stop();
((bool)result).Should().BeTrue();
}
public void Stop_When_ExceptionIsThrown_Then_ResultShouldContainThrownException()
{
using var resetEvent = new ManualResetEventSlim();
using var testee = new ContinuousJob(_ =>
{
resetEvent.Set();
throw new InvalidOperationException();
});
testee.Start();
resetEvent.Wait();
Thread.Sleep(1);
var result = testee.Stop();
result.Error !.InnerException.Should().BeOfType <InvalidOperationException>();
}
/// <summary>
/// Initializes a new instance of the <see cref="Mfrc522Connection" /> class.
/// </summary>
/// <param name="spiDevicePath">The spi device path.</param>
/// <param name="resetConnectorPin">The reset connector pin.</param>
/// <param name="gpioConnectionDriverFactory">The gpio connection driver factory.</param>
/// <param name="threadFactory">The thread factory.</param>
/// <param name="rfidConnectionReporter">The rfid connection reporter.</param>
public Mfrc522Connection(
string spiDevicePath,
ConnectorPin?resetConnectorPin,
IGpioConnectionDriverFactory?gpioConnectionDriverFactory = null,
IThreadFactory?threadFactory = null,
IRfidConnectionReporter?rfidConnectionReporter = null)
{
this.spiDevicePath = spiDevicePath;
this.resetConnectorPin = resetConnectorPin;
this.rfidConnectionReporter = rfidConnectionReporter;
this.rfidConnectionReporter?.SetSource(typeof(IRfidConnectionReporter), this);
this.thread = ThreadFactory.EnsureThreadFactory(threadFactory).Create();
this.gpioConnectionDriverFactory =
GpioConnectionDriverFactory.EnsureGpioConnectionDriverFactory(gpioConnectionDriverFactory);
this.gpioConnectionDriver = this.gpioConnectionDriverFactory.Get();
this.mfrc522Device = new Mfrc522Device(this.thread, this.gpioConnectionDriver);
this.scanningJob = new ContinuousJob(this.CheckForTags, (Exception exception, ref bool _) => this.rfidConnectionReporter?.OnException(exception));
}
/// <summary>
/// Initializes a new instance of the <see cref="Mfrc522Device" /> class.
/// </summary>
/// <param name="gpioController">The gpio controller.</param>
/// <param name="busId">The bus identifier.</param>
/// <param name="chipLineSelect">The chip line select.</param>
/// <param name="resetPin">The reset pin.</param>
/// <param name="activate">if set to <c>true</c> [activate].</param>
/// <param name="rfidDeviceReporter">The rfid connection reporter.</param>
public Mfrc522Device(
GpioController gpioController,
int busId,
int chipLineSelect,
int? resetPin,
bool activate,
IRfidDeviceReporter? rfidDeviceReporter = null)
{
this.gpioController = gpioController;
this.rfidDeviceReporter = rfidDeviceReporter;
this.rfidDeviceReporter?.SetSource(typeof(IRfidDeviceReporter), this);
this.thread = new CurrentThread();
this.mfrc522 = new Mfrc522(this.thread, this.gpioController);
this.scanningJob = new ContinuousJob(this.CheckForTags, (Exception e, ref bool _) => this.rfidDeviceReporter?.OnException(e));
this.activation = new Activation(
() => this.scanningJob.IsRunning,
() =>
{
this.mfrc522.Initialize(busId, chipLineSelect, resetPin, AntennaGain.Gain48);
this.scanningJob.Start();
},
() => this.scanningJob.Stop(),
activate);
}
private void PushAndVerifyConsoleWorker(ApplicationManager appManager, TestRepository testRepository, string[] expectedVerificationFileLines, int expectedDeployments = 1)
{
appManager.GitDeploy(testRepository.PhysicalPath);
var results = appManager.DeploymentManager.GetResultsAsync().Result.ToList();
Assert.Equal(expectedDeployments, results.Count);
for (int i = 0; i < expectedDeployments; i++)
{
Assert.Equal(DeployStatus.Success, results[i].Status);
}
var expectedContinuousJob = new ContinuousJob()
{
Name = "deployedJob",
JobType = "continuous",
Status = "Running",
RunCommand = "ConsoleWorker.exe"
};
WaitUntilAssertVerified(
"verify continuous job",
TimeSpan.FromSeconds(60),
() =>
{
ContinuousJob deployedJob = appManager.JobsManager.GetContinuousJobAsync("deployedJob").Result;
AssertContinuousJob(expectedContinuousJob, deployedJob);
});
WaitUntilAssertVerified(
"verification file",
TimeSpan.FromSeconds(30),
() =>
{
VerifyVerificationFile(appManager, expectedVerificationFileLines);
});
}
public void EnableJob(ContinuousJob continuousJob)
{
OperationManager.Attempt(() => FileSystemHelpers.DeleteFile(_disableFilePath));
StartJob(continuousJob);
}
public void RefreshJob(ContinuousJob continuousJob, JobSettings jobSettings)
{
StopJob();
_jobSettings = jobSettings;
StartJob(continuousJob);
}
public void ContinuousJobStartsAfterGoingDown()
{
RunScenario("ContinuousJobStartsAfterGoingDown", appManager =>
{
TestTracer.Trace("Copying the script to the continuous job directory");
appManager.JobsManager.CreateContinuousJobAsync("basicJob1", "run.cmd", JobScript).Wait();
var expectedContinuousJob = new ContinuousJob()
{
Name = "basicJob1",
JobType = "continuous",
Status = "PendingRestart",
RunCommand = "run.cmd"
};
WaitUntilAssertVerified(
"verify continuous job",
TimeSpan.FromSeconds(60),
() =>
{
ContinuousJob deployedJob = appManager.JobsManager.GetContinuousJobAsync("basicJob1").Result;
AssertContinuousJob(expectedContinuousJob, deployedJob);
});
TestTracer.Trace("Waiting for verification file to have 2 lines (which means it ran twice)");
var expectedVerificationFileContents = new List <string>();
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
WaitUntilAssertVerified(
"verification file",
TimeSpan.FromSeconds(30),
() => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));
TestTracer.Trace("Verify continuous job settings and set it to isSingleton: true");
JobSettings continuousJobSettings =
appManager.JobsManager.GetContinuousJobSettingsAsync(expectedContinuousJob.Name).Result;
Assert.False(continuousJobSettings.IsSingleton);
continuousJobSettings.SetSetting("is_singleton", true);
appManager.JobsManager.SetContinuousJobSettingsAsync(expectedContinuousJob.Name, continuousJobSettings).Wait();
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
WaitUntilAssertVerified(
"verification file",
TimeSpan.FromSeconds(30),
() => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));
TestTracer.Trace("Verify continuous job settings and set it to isSingleton: false");
continuousJobSettings =
appManager.JobsManager.GetContinuousJobSettingsAsync(expectedContinuousJob.Name).Result;
Assert.True(continuousJobSettings.GetSetting <bool>("is_singleton"));
continuousJobSettings.SetSetting("is_singleton", false);
appManager.JobsManager.SetContinuousJobSettingsAsync(expectedContinuousJob.Name, continuousJobSettings).Wait();
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
WaitUntilAssertVerified(
"verification file",
TimeSpan.FromSeconds(30),
() => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));
});
}
private void StartJob(ContinuousJob continuousJob)
{
// Do not go further if already started or job is disabled
if (IsDisabled)
{
UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
return;
}
if (Interlocked.Exchange(ref _started, 1) == 1)
{
return;
}
_continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);
CheckAlwaysOn();
_continuousJobThread = new Thread(() =>
{
try
{
while (_started == 1 && !IsDisabled)
{
// Try getting the singleton lock if single is enabled
if (!TryGetLockIfSingleton())
{
// Wait 5 seconds and retry to take the lock
WaitForTimeOrStop(TimeSpan.FromSeconds(5));
continue;
}
Stopwatch liveStopwatch = Stopwatch.StartNew();
_continuousJobLogger.StartingNewRun();
using (new Timer(LogStillRunning, null, TimeSpan.FromHours(1), TimeSpan.FromHours(12)))
{
InitializeJobInstance(continuousJob, _continuousJobLogger);
RunJobInstance(continuousJob, _continuousJobLogger, String.Empty, String.Empty);
}
if (_started == 1 && !IsDisabled)
{
// The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob
// Was running for at least 2 minutes there is no wait time.
TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero;
_continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
_continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
WaitForTimeOrStop(webJobsRestartTime);
}
// Make sure lock is released before re-iterating and trying to get the lock again
ReleaseSingletonLock();
}
}
catch (ThreadAbortException)
{
TraceFactory.GetTracer().TraceWarning("Thread was aborted, make sure WebJob was about to stop.");
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
finally
{
ReleaseSingletonLock();
}
});
_continuousJobThread.Start();
}
private void StartJob(ContinuousJob continuousJob)
{
// Do not go further if already started or job is disabled
if (IsDisabled)
{
UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
return;
}
if (Interlocked.Exchange(ref _started, 1) == 1)
{
return;
}
_continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);
CheckAlwaysOn();
_continuousJobThread = new Thread(() =>
{
var threadAborted = false;
while (!threadAborted && _started == 1 && !IsDisabled)
{
try
{
// Try getting the singleton lock if single is enabled
bool acquired;
if (!TryGetLockIfSingleton(out acquired))
{
// Wait 5 seconds and retry to take the lock
WaitForTimeOrStop(TimeSpan.FromSeconds(5));
continue;
}
try
{
Stopwatch liveStopwatch = Stopwatch.StartNew();
_continuousJobLogger.StartingNewRun();
var tracer = TraceFactory.GetTracer();
using (tracer.Step("Run {0} {1}", continuousJob.JobType, continuousJob.Name))
using (new Timer(LogStillRunning, null, TimeSpan.FromHours(1), TimeSpan.FromHours(12)))
{
InitializeJobInstance(continuousJob, _continuousJobLogger);
WebJobPort = GetAvailableJobPort();
RunJobInstance(continuousJob, _continuousJobLogger, String.Empty, String.Empty, tracer, WebJobPort);
}
if (_started == 1 && !IsDisabled)
{
// The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob
// Was running for at least 2 minutes there is no wait time.
TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero;
_continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
_continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
WaitForTimeOrStop(webJobsRestartTime);
}
}
finally
{
if (acquired)
{
// Make sure lock is released before re-iterating and trying to get the lock again
ReleaseSingletonLock();
}
}
}
catch (ThreadAbortException ex)
{
// by nature, ThreadAbortException will be rethrown at the end of this catch block and
// this bool may not be neccessary since while loop will be exited anyway. we added
// it to be explicit.
threadAborted = true;
if (!ex.AbortedByKudu())
{
TraceFactory.GetTracer().TraceWarning("Thread was aborted, make sure WebJob was about to stop.");
}
}
catch (Exception ex)
{
_analytics.UnexpectedException(ex, trace: true);
// sleep to avoid tight exception loop
WaitForTimeOrStop(TimeSpan.FromSeconds(60));
}
}
});
_continuousJobThread.Start();
}
