Пример #1
0
        public ContinuousJobRunnerFacts()
        {
            FileSystemHelpers.Instance = new FileSystem();

            _job = new ContinuousJob
            {
                Name = "testjob",
                JobBinariesRootPath = @"c:\test\data\continuous\testjob"
            };
            _environment = new TestEnvironment
            {
                TempPath = @"c:\temp",
                JobsBinariesPath = @"c:\test\data\continuous\testjob",
                JobsDataPath = Path.GetTempPath(),
                DataPath = @"c:\test\data"
            };

            string jobDirectory = Path.Combine(_environment.JobsDataPath, "continuous", _job.Name);
            _logFilePath = Path.Combine(jobDirectory, "job_log.txt");

            MockDeploymentSettingsManager mockSettingsManager = new MockDeploymentSettingsManager();
            Mock<ITraceFactory> mockTraceFactory = new Mock<ITraceFactory>(MockBehavior.Strict);
            Mock<IAnalytics> mockAnalytics = new Mock<IAnalytics>(MockBehavior.Strict);

            Mock<ITracer> mockTracer = new Mock<ITracer>(MockBehavior.Strict);
            mockTracer.Setup(p => p.Trace(It.IsAny<string>(), It.IsAny<IDictionary<string, string>>()));
            mockTraceFactory.Setup(p => p.GetTracer()).Returns(mockTracer.Object);

            _runner = new ContinuousJobRunner(_job, _environment, mockSettingsManager, mockTraceFactory.Object, mockAnalytics.Object);

            FileSystemHelpers.DeleteFileSafe(_logFilePath);
        }
Пример #2
0
        public ContinuousJobRunner(ContinuousJob continuousJob, IEnvironment environment, IDeploymentSettingsManager settings, ITraceFactory traceFactory, IAnalytics analytics)
            : base(continuousJob.Name, Constants.ContinuousPath, environment, settings, traceFactory, analytics)
        {
            _continuousJobLogger = new ContinuousJobLogger(continuousJob.Name, Environment, TraceFactory);

            _disableFilePath = Path.Combine(continuousJob.JobBinariesRootPath, "disable.job");

            _singletonLock = new LockFile(Path.Combine(JobDataPath, "singleton.job.lock"), TraceFactory);
        }
Пример #3
0
        private void StartJob(ContinuousJob continuousJob)
        {
            // Do not go further if already started or job is disabled

            if (IsDisabled)
            {
                UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
                return;
            }

            if (Interlocked.Exchange(ref _started, 1) == 1)
            {
                return;
            }

            _continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);

            _continuousJobThread = new Thread(() =>
            {
                try
                {
                    while (_started == 1 && !IsDisabled)
                    {
                        // Try getting the singleton lock if single is enabled
                        if (!TryGetLockIfSingleton())
                        {
                            // Wait 5 seconds and retry to take the lock
                            WaitForTimeOrStop(TimeSpan.FromSeconds(5));
                            continue;
                        }

                        _continuousJobLogger.StartingNewRun();

                        InitializeJobInstance(continuousJob, _continuousJobLogger);
                        RunJobInstance(continuousJob, _continuousJobLogger);

                        if (_started == 1 && !IsDisabled)
                        {
                            TimeSpan jobsInterval = Settings.GetJobsInterval();
                            _continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(jobsInterval.TotalSeconds));
                            _continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
                            WaitForTimeOrStop(jobsInterval);
                        }
                    }
                }
                catch (Exception ex)
                {
                    TraceFactory.GetTracer().TraceError(ex);
                }
                finally
                {
                    DisposeSingletonLock();
                }
            });

            _continuousJobThread.Start();
        }
Пример #4
0
        private void PushAndVerifyConsoleWorker(ApplicationManager appManager, TestRepository testRepository, string[] expectedVerificationFileLines, int expectedDeployments = 1)
        {
            appManager.GitDeploy(testRepository.PhysicalPath);
            var results = appManager.DeploymentManager.GetResultsAsync().Result.ToList();

            Assert.Equal(expectedDeployments, results.Count);
            for (int i = 0; i < expectedDeployments; i++)
            {
                Assert.Equal(DeployStatus.Success, results[i].Status);
            }

            var expectedContinuousJob = new ContinuousJob()
            {
                Name = "deployedJob",
                JobType = "continuous",
                Status = "Running",
                RunCommand = "ConsoleWorker.exe"
            };

            WaitUntilAssertVerified(
                "verify continuous job",
                TimeSpan.FromSeconds(60),
                () =>
                {
                    ContinuousJob deployedJob = appManager.JobsManager.GetContinuousJobAsync("deployedJob").Result;
                    AssertContinuousJob(expectedContinuousJob, deployedJob);
                });

            WaitUntilAssertVerified(
                "verification file",
                TimeSpan.FromSeconds(30),
                () =>
                {
                    VerifyVerificationFile(appManager, expectedVerificationFileLines);
                });
        }
Пример #5
0
        public void ContinuousJobStartsAfterGoingDown()
        {
            RunScenario("ContinuousJobStartsAfterGoingDown", appManager =>
            {
                TestTracer.Trace("Copying the script to the continuous job directory");

                appManager.JobsManager.CreateContinuousJobAsync("basicJob1", "run.cmd", JobScript).Wait();

                var expectedContinuousJob = new ContinuousJob()
                {
                    Name = "basicJob1",
                    JobType = "continuous",
                    Status = "PendingRestart",
                    RunCommand = "run.cmd"
                };

                WaitUntilAssertVerified(
                    "verify continuous job",
                    TimeSpan.FromSeconds(60),
                    () =>
                    {
                        ContinuousJob deployedJob = appManager.JobsManager.GetContinuousJobAsync("basicJob1").Result;
                        AssertContinuousJob(expectedContinuousJob, deployedJob);
                    });

                TestTracer.Trace("Waiting for verification file to have 2 lines (which means it ran twice)");

                var expectedVerificationFileContents = new List<string>();
                expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
                expectedVerificationFileContents.Add(ExpectedVerificationFileContent);

                WaitUntilAssertVerified(
                    "verification file",
                    TimeSpan.FromSeconds(30),
                    () => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));

                TestTracer.Trace("Verify continuous job settings and set it to isSingleton: true");
                JobSettings continuousJobSettings =
                    appManager.JobsManager.GetContinuousJobSettingsAsync(expectedContinuousJob.Name).Result;

                Assert.False(continuousJobSettings.IsSingleton);

                continuousJobSettings.SetSetting("is_singleton", true);
                appManager.JobsManager.SetContinuousJobSettingsAsync(expectedContinuousJob.Name, continuousJobSettings).Wait();

                expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
                expectedVerificationFileContents.Add(ExpectedVerificationFileContent);

                WaitUntilAssertVerified(
                    "verification file",
                    TimeSpan.FromSeconds(30),
                    () => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));

                TestTracer.Trace("Verify continuous job settings and set it to isSingleton: false");
                continuousJobSettings =
                    appManager.JobsManager.GetContinuousJobSettingsAsync(expectedContinuousJob.Name).Result;

                Assert.True(continuousJobSettings.GetSetting<bool>("is_singleton"));

                continuousJobSettings.SetSetting("is_singleton", false);
                appManager.JobsManager.SetContinuousJobSettingsAsync(expectedContinuousJob.Name, continuousJobSettings).Wait();

                expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
                expectedVerificationFileContents.Add(ExpectedVerificationFileContent);

                WaitUntilAssertVerified(
                    "verification file",
                    TimeSpan.FromSeconds(30),
                    () => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));
            });
        }
Пример #6
0
 private void AssertContinuousJob(ContinuousJob expectedContinuousJob, ContinuousJob actualContinuousJob)
 {
     AssertJob(expectedContinuousJob, actualContinuousJob);
     Assert.NotNull(actualContinuousJob);
     Assert.Equal(expectedContinuousJob.Status, actualContinuousJob.Status);
     Assert.NotNull(actualContinuousJob.LogUrl);
 }
Пример #7
0
        private void StartJob(ContinuousJob continuousJob)
        {
            // Do not go further if already started or job is disabled
            if (IsDisabled)
            {
                UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
                return;
            }

            if (Interlocked.Exchange(ref _started, 1) == 1)
            {
                return;
            }

            _continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);

            CheckAlwaysOn();

            _continuousJobThread = new Thread(() =>
            {
                try
                {
                    while (_started == 1 && !IsDisabled)
                    {
                        // Try getting the singleton lock if single is enabled
                        if (!TryGetLockIfSingleton())
                        {
                            // Wait 5 seconds and retry to take the lock
                            WaitForTimeOrStop(TimeSpan.FromSeconds(5));
                            continue;
                        }

                        Stopwatch liveStopwatch = Stopwatch.StartNew();

                        _continuousJobLogger.StartingNewRun();

                        var tracer = TraceFactory.GetTracer();
                        using (tracer.Step("Run {0} {1}", continuousJob.JobType, continuousJob.Name))
                        using (new Timer(LogStillRunning, null, TimeSpan.FromHours(1), TimeSpan.FromHours(12)))
                        {
                            InitializeJobInstance(continuousJob, _continuousJobLogger);
                            WebJobPort = GetAvailableJobPort();
                            RunJobInstance(continuousJob, _continuousJobLogger, String.Empty, String.Empty, tracer, WebJobPort);
                        }

                        if (_started == 1 && !IsDisabled)
                        {
                            // The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob
                            // Was running for at least 2 minutes there is no wait time.
                            TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero;
                            _continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
                            _continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
                            WaitForTimeOrStop(webJobsRestartTime);
                        }

                        // Make sure lock is released before re-iterating and trying to get the lock again
                        ReleaseSingletonLock();
                    }
                }
                catch (ThreadAbortException)
                {
                    TraceFactory.GetTracer().TraceWarning("Thread was aborted, make sure WebJob was about to stop.");
                }
                catch (Exception ex)
                {
                    TraceFactory.GetTracer().TraceError(ex);
                }
                finally
                {
                    ReleaseSingletonLock();
                }
            });

            _continuousJobThread.Start();
        }
Пример #8
0
        public void RefreshJob(ContinuousJob continuousJob, JobSettings jobSettings, bool logRefresh)
        {
            if (logRefresh)
            {
                _continuousJobLogger.LogInformation("Detected WebJob file/s were updated, refreshing WebJob");
            }

            StopJob();
            JobSettings = jobSettings;
            StartJob(continuousJob);
        }
Пример #9
0
        private void StartJob(ContinuousJob continuousJob)
        {
            // Do not go further if already started or job is disabled

            if (IsDisabled)
            {
                UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
                return;
            }

            if (Interlocked.Exchange(ref _started, 1) == 1)
            {
                return;
            }

            _continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);

            _continuousJobThread = new Thread(() =>
            {
                try
                {
                    while (_started == 1 && !IsDisabled)
                    {
                        // Try getting the singleton lock if single is enabled
                        if (!TryGetLockIfSingleton())
                        {
                            // Wait 5 seconds and retry to take the lock
                            WaitForTimeOrStop(TimeSpan.FromSeconds(5));
                            continue;
                        }

                        Stopwatch liveStopwatch = Stopwatch.StartNew();

                        _continuousJobLogger.StartingNewRun();

                        InitializeJobInstance(continuousJob, _continuousJobLogger);
                        RunJobInstance(continuousJob, _continuousJobLogger, String.Empty);

                        if (_started == 1 && !IsDisabled)
                        {
                            // The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob
                            // Was running for at least 2 minutes there is no wait time.
                            TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero;
                            _continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
                            _continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
                            WaitForTimeOrStop(webJobsRestartTime);
                        }
                    }
                }
                catch (Exception ex)
                {
                    TraceFactory.GetTracer().TraceError(ex);
                }
                finally
                {
                    ReleaseSingletonLock();
                }
            });

            _continuousJobThread.Start();
        }
Пример #10
0
 public void EnableJob(ContinuousJob continuousJob)
 {
     OperationManager.Attempt(() => FileSystemHelpers.DeleteFile(_disableFilePath));
     StartJob(continuousJob);
 }
Пример #11
0
 public void RefreshJob(ContinuousJob continuousJob, JobSettings jobSettings)
 {
     StopJob();
     _jobSettings = jobSettings;
     StartJob(continuousJob);
 }
Пример #12
0
        private void StartJob(ContinuousJob continuousJob)
        {
            // Do not go further if already started or job is disabled
            if (IsDisabled)
            {
                UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
                return;
            }

            if (Interlocked.Exchange(ref _started, 1) == 1)
            {
                return;
            }

            _continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);

            CheckAlwaysOn();

            _continuousJobThread = new Thread(() =>
            {
                var threadAborted = false;
                while (!threadAborted && _started == 1 && !IsDisabled)
                {
                    try
                    {
                        // Try getting the singleton lock if single is enabled
                        bool acquired;
                        if (!TryGetLockIfSingleton(out acquired))
                        {
                            // Wait 5 seconds and retry to take the lock
                            WaitForTimeOrStop(TimeSpan.FromSeconds(5));
                            continue;
                        }

                        try
                        {
                            Stopwatch liveStopwatch = Stopwatch.StartNew();

                            _continuousJobLogger.StartingNewRun();

                            var tracer = TraceFactory.GetTracer();
                            using (tracer.Step("Run {0} {1}", continuousJob.JobType, continuousJob.Name))
                            using (new Timer(LogStillRunning, null, TimeSpan.FromHours(1), TimeSpan.FromHours(12)))
                            {
                                InitializeJobInstance(continuousJob, _continuousJobLogger);
                                WebJobPort = GetAvailableJobPort();
                                RunJobInstance(continuousJob, _continuousJobLogger, String.Empty, String.Empty, tracer, WebJobPort);
                            }

                            if (_started == 1 && !IsDisabled)
                            {
                                // The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob
                                // Was running for at least 2 minutes there is no wait time.
                                TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero;
                                _continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
                                _continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
                                WaitForTimeOrStop(webJobsRestartTime);
                            }
                        }
                        finally
                        {
                            if (acquired)
                            {
                                // Make sure lock is released before re-iterating and trying to get the lock again
                                ReleaseSingletonLock();
                            }
                        }
                    }
                    catch (ThreadAbortException ex)
                    {
                        // by nature, ThreadAbortException will be rethrown at the end of this catch block and
                        // this bool may not be neccessary since while loop will be exited anyway.  we added
                        // it to be explicit.
                        threadAborted = true;

                        if (!ex.AbortedByKudu())
                        {
                            TraceFactory.GetTracer().TraceWarning("Thread was aborted, make sure WebJob was about to stop.");
                        }
                    }
                    catch (Exception ex)
                    {
                        _analytics.UnexpectedException(ex, trace: true);

                        // sleep to avoid tight exception loop
                        WaitForTimeOrStop(TimeSpan.FromSeconds(60));
                    }
                }
            });

            _continuousJobThread.Start();
        }
Пример #13
0
 public void RefreshJob(ContinuousJob continuousJob)
 {
     StopJob();
     StartJob(continuousJob);
 }
Пример #14
0
        public void ContinuousJobStartsAfterGoingDown()
        {
            RunScenario("ContinuousJobStartsAfterGoingDown", appManager =>
            {
                TestTracer.Trace("Copying the script to the continuous job directory");

                WaitUntilAssertVerified(
                    "writing file",
                    TimeSpan.FromSeconds(10),
                    () => appManager.VfsManager.WriteAllText(BasicContinuousJobExecutablePath, JobScript));

                var expectedContinuousJob = new ContinuousJob()
                {
                    Name = "basicJob1",
                    JobType = "continuous",
                    Status = "PendingRestart",
                    RunCommand = "run.cmd"
                };

                WaitUntilAssertVerified(
                    "verify continuous job",
                    TimeSpan.FromSeconds(60),
                    () =>
                    {
                        ContinuousJob deployedJob = appManager.JobsManager.GetContinuousJobAsync("basicJob1").Result;
                        AssertContinuousJob(expectedContinuousJob, deployedJob);
                    });

                TestTracer.Trace("Waiting for verification file to have 2 lines (which means it ran twice)");

                var expectedVerificationFileContents = new List<string>();
                expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
                expectedVerificationFileContents.Add(ExpectedVerificationFileContent);

                WaitUntilAssertVerified(
                    "verification file",
                    TimeSpan.FromSeconds(30),
                    () => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));

                appManager.JobsManager.SetSingletonContinuousJobAsync(expectedContinuousJob.Name, true).Wait();
                expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
                expectedVerificationFileContents.Add(ExpectedVerificationFileContent);

                WaitUntilAssertVerified(
                    "verification file",
                    TimeSpan.FromSeconds(30),
                    () => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));

                appManager.JobsManager.SetSingletonContinuousJobAsync(expectedContinuousJob.Name, false).Wait();
                expectedVerificationFileContents.Add(ExpectedVerificationFileContent);
                expectedVerificationFileContents.Add(ExpectedVerificationFileContent);

                WaitUntilAssertVerified(
                    "verification file",
                    TimeSpan.FromSeconds(30),
                    () => VerifyVerificationFile(appManager, expectedVerificationFileContents.ToArray()));
            });
        }