public ContinuousJobRunnerFacts() { FileSystemHelpers.Instance = new FileSystem(); _job = new ContinuousJob { Name = "testjob", JobBinariesRootPath = @"c:\test\data\continuous\testjob" }; _environment = new TestEnvironment { TempPath = @"c:\temp", JobsBinariesPath = @"c:\test\data\continuous\testjob", JobsDataPath = Path.GetTempPath(), DataPath = @"c:\test\data" }; string jobDirectory = Path.Combine(_environment.JobsDataPath, "continuous", _job.Name); _logFilePath = Path.Combine(jobDirectory, "job_log.txt"); MockDeploymentSettingsManager mockSettingsManager = new MockDeploymentSettingsManager(); Mock<ITraceFactory> mockTraceFactory = new Mock<ITraceFactory>(MockBehavior.Strict); Mock<IAnalytics> mockAnalytics = new Mock<IAnalytics>(MockBehavior.Strict); Mock<ITracer> mockTracer = new Mock<ITracer>(MockBehavior.Strict); mockTracer.Setup(p => p.Trace(It.IsAny<string>(), It.IsAny<IDictionary<string, string>>())); mockTraceFactory.Setup(p => p.GetTracer()).Returns(mockTracer.Object); _runner = new ContinuousJobRunner(_job, _environment, mockSettingsManager, mockTraceFactory.Object, mockAnalytics.Object); FileSystemHelpers.DeleteFileSafe(_logFilePath); }
public ContinuousJobRunner(ContinuousJob continuousJob, IEnvironment environment, IDeploymentSettingsManager settings, ITraceFactory traceFactory, IAnalytics analytics) : base(continuousJob.Name, Constants.ContinuousPath, environment, settings, traceFactory, analytics) { _continuousJobLogger = new ContinuousJobLogger(continuousJob.Name, Environment, TraceFactory); _disableFilePath = Path.Combine(continuousJob.JobBinariesRootPath, "disable.job"); _singletonLock = new LockFile(Path.Combine(JobDataPath, "singleton.job.lock"), TraceFactory); }
private void StartJob(ContinuousJob continuousJob) { // Do not go further if already started or job is disabled if (IsDisabled) { UpdateStatusIfChanged(ContinuousJobStatus.Stopped); return; } if (Interlocked.Exchange(ref _started, 1) == 1) { return; } _continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting); _continuousJobThread = new Thread(() => { try { while (_started == 1 && !IsDisabled) { // Try getting the singleton lock if single is enabled if (!TryGetLockIfSingleton()) { // Wait 5 seconds and retry to take the lock WaitForTimeOrStop(TimeSpan.FromSeconds(5)); continue; } _continuousJobLogger.StartingNewRun(); InitializeJobInstance(continuousJob, _continuousJobLogger); RunJobInstance(continuousJob, _continuousJobLogger); if (_started == 1 && !IsDisabled) { TimeSpan jobsInterval = Settings.GetJobsInterval(); _continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(jobsInterval.TotalSeconds)); _continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart); WaitForTimeOrStop(jobsInterval); } } } catch (Exception ex) { TraceFactory.GetTracer().TraceError(ex); } finally { DisposeSingletonLock(); } }); _continuousJobThread.Start(); }
private void PushAndVerifyConsoleWorker(ApplicationManager appManager, TestRepository testRepository, string[] expectedVerificationFileLines, int expectedDeployments = 1) { appManager.GitDeploy(testRepository.PhysicalPath); var results = appManager.DeploymentManager.GetResultsAsync().Result.ToList(); Assert.Equal(expectedDeployments, results.Count); for (int i = 0; i < expectedDeployments; i++) { Assert.Equal(DeployStatus.Success, results[i].Status); } var expectedContinuousJob = new ContinuousJob() { Name = "deployedJob", JobType = "continuous", Status = "Running", RunCommand = "ConsoleWorker.exe" }; WaitUntilAssertVerified( "verify continuous job", TimeSpan.FromSeconds(60), () => { ContinuousJob deployedJob = appManager.JobsManager.GetContinuousJobAsync("deployedJob").Result; AssertContinuousJob(expectedContinuousJob, deployedJob); }); WaitUntilAssertVerified( "verification file", TimeSpan.FromSeconds(30), () => { VerifyVerificationFile(appManager, expectedVerificationFileLines); }); }
public void ContinuousJobStartsAfterGoingDown() { RunScenario("ContinuousJobStartsAfterGoingDown", appManager => { TestTracer.Trace("Copying the script to the continuous job directory"); appManager.JobsManager.CreateContinuousJobAsync("basicJob1", "run.cmd", JobScript).Wait(); var expectedContinuousJob = new ContinuousJob() { Name = "basicJob1", JobType = "continuous", Status = "PendingRestart", RunCommand = "run.cmd" }; WaitUntilAssertVerified( "verify continuous job", TimeSpan.FromSeconds(60), () => { ContinuousJob deployedJob = appManager.JobsManager.GetContinuousJobAsync("basicJob1").Result; AssertContinuousJob(expectedContinuousJob, deployedJob); }); TestTracer.Trace("Waiting for verification file to have 2 lines (which means it ran twice)"); var expectedVerificationFileContents = new List<string>(); expectedVerificationFileContents.Add(ExpectedVerificationFileContent); expectedVerificationFileContents.Add(ExpectedVerificationFileContent); WaitUntilAssertVerified( "verification file", TimeSpan.FromSeconds(30), () => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray())); TestTracer.Trace("Verify continuous job settings and set it to isSingleton: true"); JobSettings continuousJobSettings = appManager.JobsManager.GetContinuousJobSettingsAsync(expectedContinuousJob.Name).Result; Assert.False(continuousJobSettings.IsSingleton); continuousJobSettings.SetSetting("is_singleton", true); appManager.JobsManager.SetContinuousJobSettingsAsync(expectedContinuousJob.Name, continuousJobSettings).Wait(); expectedVerificationFileContents.Add(ExpectedVerificationFileContent); expectedVerificationFileContents.Add(ExpectedVerificationFileContent); WaitUntilAssertVerified( "verification file", TimeSpan.FromSeconds(30), () => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray())); TestTracer.Trace("Verify continuous job settings and set it to isSingleton: false"); continuousJobSettings = appManager.JobsManager.GetContinuousJobSettingsAsync(expectedContinuousJob.Name).Result; Assert.True(continuousJobSettings.GetSetting<bool>("is_singleton")); continuousJobSettings.SetSetting("is_singleton", false); appManager.JobsManager.SetContinuousJobSettingsAsync(expectedContinuousJob.Name, continuousJobSettings).Wait(); expectedVerificationFileContents.Add(ExpectedVerificationFileContent); expectedVerificationFileContents.Add(ExpectedVerificationFileContent); WaitUntilAssertVerified( "verification file", TimeSpan.FromSeconds(30), () => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray())); }); }
private void AssertContinuousJob(ContinuousJob expectedContinuousJob, ContinuousJob actualContinuousJob) { AssertJob(expectedContinuousJob, actualContinuousJob); Assert.NotNull(actualContinuousJob); Assert.Equal(expectedContinuousJob.Status, actualContinuousJob.Status); Assert.NotNull(actualContinuousJob.LogUrl); }
private void StartJob(ContinuousJob continuousJob) { // Do not go further if already started or job is disabled if (IsDisabled) { UpdateStatusIfChanged(ContinuousJobStatus.Stopped); return; } if (Interlocked.Exchange(ref _started, 1) == 1) { return; } _continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting); CheckAlwaysOn(); _continuousJobThread = new Thread(() => { try { while (_started == 1 && !IsDisabled) { // Try getting the singleton lock if single is enabled if (!TryGetLockIfSingleton()) { // Wait 5 seconds and retry to take the lock WaitForTimeOrStop(TimeSpan.FromSeconds(5)); continue; } Stopwatch liveStopwatch = Stopwatch.StartNew(); _continuousJobLogger.StartingNewRun(); var tracer = TraceFactory.GetTracer(); using (tracer.Step("Run {0} {1}", continuousJob.JobType, continuousJob.Name)) using (new Timer(LogStillRunning, null, TimeSpan.FromHours(1), TimeSpan.FromHours(12))) { InitializeJobInstance(continuousJob, _continuousJobLogger); WebJobPort = GetAvailableJobPort(); RunJobInstance(continuousJob, _continuousJobLogger, String.Empty, String.Empty, tracer, WebJobPort); } if (_started == 1 && !IsDisabled) { // The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob // Was running for at least 2 minutes there is no wait time. TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero; _continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds)); _continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart); WaitForTimeOrStop(webJobsRestartTime); } // Make sure lock is released before re-iterating and trying to get the lock again ReleaseSingletonLock(); } } catch (ThreadAbortException) { TraceFactory.GetTracer().TraceWarning("Thread was aborted, make sure WebJob was about to stop."); } catch (Exception ex) { TraceFactory.GetTracer().TraceError(ex); } finally { ReleaseSingletonLock(); } }); _continuousJobThread.Start(); }
public void RefreshJob(ContinuousJob continuousJob, JobSettings jobSettings, bool logRefresh) { if (logRefresh) { _continuousJobLogger.LogInformation("Detected WebJob file/s were updated, refreshing WebJob"); } StopJob(); JobSettings = jobSettings; StartJob(continuousJob); }
private void StartJob(ContinuousJob continuousJob) { // Do not go further if already started or job is disabled if (IsDisabled) { UpdateStatusIfChanged(ContinuousJobStatus.Stopped); return; } if (Interlocked.Exchange(ref _started, 1) == 1) { return; } _continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting); _continuousJobThread = new Thread(() => { try { while (_started == 1 && !IsDisabled) { // Try getting the singleton lock if single is enabled if (!TryGetLockIfSingleton()) { // Wait 5 seconds and retry to take the lock WaitForTimeOrStop(TimeSpan.FromSeconds(5)); continue; } Stopwatch liveStopwatch = Stopwatch.StartNew(); _continuousJobLogger.StartingNewRun(); InitializeJobInstance(continuousJob, _continuousJobLogger); RunJobInstance(continuousJob, _continuousJobLogger, String.Empty); if (_started == 1 && !IsDisabled) { // The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob // Was running for at least 2 minutes there is no wait time. TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero; _continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds)); _continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart); WaitForTimeOrStop(webJobsRestartTime); } } } catch (Exception ex) { TraceFactory.GetTracer().TraceError(ex); } finally { ReleaseSingletonLock(); } }); _continuousJobThread.Start(); }
public void EnableJob(ContinuousJob continuousJob) { OperationManager.Attempt(() => FileSystemHelpers.DeleteFile(_disableFilePath)); StartJob(continuousJob); }
public void RefreshJob(ContinuousJob continuousJob, JobSettings jobSettings) { StopJob(); _jobSettings = jobSettings; StartJob(continuousJob); }
private void StartJob(ContinuousJob continuousJob) { // Do not go further if already started or job is disabled if (IsDisabled) { UpdateStatusIfChanged(ContinuousJobStatus.Stopped); return; } if (Interlocked.Exchange(ref _started, 1) == 1) { return; } _continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting); CheckAlwaysOn(); _continuousJobThread = new Thread(() => { var threadAborted = false; while (!threadAborted && _started == 1 && !IsDisabled) { try { // Try getting the singleton lock if single is enabled bool acquired; if (!TryGetLockIfSingleton(out acquired)) { // Wait 5 seconds and retry to take the lock WaitForTimeOrStop(TimeSpan.FromSeconds(5)); continue; } try { Stopwatch liveStopwatch = Stopwatch.StartNew(); _continuousJobLogger.StartingNewRun(); var tracer = TraceFactory.GetTracer(); using (tracer.Step("Run {0} {1}", continuousJob.JobType, continuousJob.Name)) using (new Timer(LogStillRunning, null, TimeSpan.FromHours(1), TimeSpan.FromHours(12))) { InitializeJobInstance(continuousJob, _continuousJobLogger); WebJobPort = GetAvailableJobPort(); RunJobInstance(continuousJob, _continuousJobLogger, String.Empty, String.Empty, tracer, WebJobPort); } if (_started == 1 && !IsDisabled) { // The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob // Was running for at least 2 minutes there is no wait time. TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero; _continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds)); _continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart); WaitForTimeOrStop(webJobsRestartTime); } } finally { if (acquired) { // Make sure lock is released before re-iterating and trying to get the lock again ReleaseSingletonLock(); } } } catch (ThreadAbortException ex) { // by nature, ThreadAbortException will be rethrown at the end of this catch block and // this bool may not be neccessary since while loop will be exited anyway. we added // it to be explicit. threadAborted = true; if (!ex.AbortedByKudu()) { TraceFactory.GetTracer().TraceWarning("Thread was aborted, make sure WebJob was about to stop."); } } catch (Exception ex) { _analytics.UnexpectedException(ex, trace: true); // sleep to avoid tight exception loop WaitForTimeOrStop(TimeSpan.FromSeconds(60)); } } }); _continuousJobThread.Start(); }
public void RefreshJob(ContinuousJob continuousJob) { StopJob(); StartJob(continuousJob); }
public void ContinuousJobStartsAfterGoingDown() { RunScenario("ContinuousJobStartsAfterGoingDown", appManager => { TestTracer.Trace("Copying the script to the continuous job directory"); WaitUntilAssertVerified( "writing file", TimeSpan.FromSeconds(10), () => appManager.VfsManager.WriteAllText(BasicContinuousJobExecutablePath, JobScript)); var expectedContinuousJob = new ContinuousJob() { Name = "basicJob1", JobType = "continuous", Status = "PendingRestart", RunCommand = "run.cmd" }; WaitUntilAssertVerified( "verify continuous job", TimeSpan.FromSeconds(60), () => { ContinuousJob deployedJob = appManager.JobsManager.GetContinuousJobAsync("basicJob1").Result; AssertContinuousJob(expectedContinuousJob, deployedJob); }); TestTracer.Trace("Waiting for verification file to have 2 lines (which means it ran twice)"); var expectedVerificationFileContents = new List<string>(); expectedVerificationFileContents.Add(ExpectedVerificationFileContent); expectedVerificationFileContents.Add(ExpectedVerificationFileContent); WaitUntilAssertVerified( "verification file", TimeSpan.FromSeconds(30), () => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray())); appManager.JobsManager.SetSingletonContinuousJobAsync(expectedContinuousJob.Name, true).Wait(); expectedVerificationFileContents.Add(ExpectedVerificationFileContent); expectedVerificationFileContents.Add(ExpectedVerificationFileContent); WaitUntilAssertVerified( "verification file", TimeSpan.FromSeconds(30), () => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray())); appManager.JobsManager.SetSingletonContinuousJobAsync(expectedContinuousJob.Name, false).Wait(); expectedVerificationFileContents.Add(ExpectedVerificationFileContent); expectedVerificationFileContents.Add(ExpectedVerificationFileContent); WaitUntilAssertVerified( "verification file", TimeSpan.FromSeconds(30), () => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray())); }); }