public void SafeKillAllRunningJobInstances(IJobLogger logger)
{
try
{
Process[] processes = Process.GetProcesses();
foreach (Process process in processes)
{
StringDictionary processEnvironment;
bool success = ProcessEnvironment.TryGetEnvironmentVariables(process, out processEnvironment);
if (success && processEnvironment.ContainsKey(GetJobEnvironmentKey()))
{
try
{
process.Kill(true, TraceFactory.GetTracer());
}
catch (Exception ex)
{
if (!process.HasExited)
{
logger.LogWarning("Failed to kill process - {0} for job - {1}\n{2}".FormatInvariant(process.ProcessName, JobName, ex));
}
}
}
}
}
catch (Exception ex)
{
logger.LogWarning(ex.ToString());
}
}
private void OnError(object sender, ErrorEventArgs e)
{
Exception ex = e.GetException();
TraceFactory.GetTracer().TraceError(ex.ToString());
ResetWatcher();
}
public void StartJobRun(TriggeredJob triggeredJob, JobSettings jobSettings, string trigger, Action <string, string> reportAction)
{
JobSettings = jobSettings;
if (Settings.IsWebJobsStopped())
{
throw new WebJobsStoppedException();
}
if (!_lockFile.Lock())
{
throw new ConflictException();
}
TriggeredJobRunLogger logger = TriggeredJobRunLogger.LogNewRun(triggeredJob, trigger, Environment, TraceFactory, Settings);
Debug.Assert(logger != null);
try
{
if (_currentRunningJobWaitHandle != null)
{
_currentRunningJobWaitHandle.Dispose();
_currentRunningJobWaitHandle = null;
}
_currentRunningJobWaitHandle = new ManualResetEvent(false);
var tracer = TraceFactory.GetTracer();
var step = tracer.Step("Run {0} {1}", triggeredJob.JobType, triggeredJob.Name);
ThreadPool.QueueUserWorkItem(_ =>
{
try
{
InitializeJobInstance(triggeredJob, logger);
RunJobInstance(triggeredJob, logger, logger.Id, trigger, tracer);
}
catch (Exception ex)
{
logger.LogError("WebJob run failed due to: " + ex);
}
finally
{
step.Dispose();
logger.ReportEndRun();
_lockFile.Release();
reportAction(triggeredJob.Name, logger.Id);
_currentRunningJobWaitHandle.Set();
}
});
}
catch (Exception ex)
{
logger.LogError("Failed to start job due to: " + ex);
_lockFile.Release();
throw;
}
}
private void StartJob(ContinuousJob continuousJob)
{
// Do not go further if already started or job is disabled
if (IsDisabled)
{
UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
return;
}
if (Interlocked.Exchange(ref _started, 1) == 1)
{
return;
}
_continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);
_continuousJobThread = new Thread(() =>
{
try
{
while (_started == 1 && !IsDisabled)
{
// Try getting the singleton lock if single is enabled
if (!TryGetLockIfSingleton())
{
// Wait 5 seconds and retry to take the lock
WaitForTimeOrStop(TimeSpan.FromSeconds(5));
continue;
}
_continuousJobLogger.StartingNewRun();
InitializeJobInstance(continuousJob, _continuousJobLogger);
RunJobInstance(continuousJob, _continuousJobLogger, String.Empty);
if (_started == 1 && !IsDisabled)
{
TimeSpan webJobsRestartTime = Settings.GetWebJobsRestartTime();
_continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
_continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
WaitForTimeOrStop(webJobsRestartTime);
}
}
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
finally
{
ReleaseSingletonLock();
}
});
_continuousJobThread.Start();
}
protected void RunJobInstance(JobBase job, IJobLogger logger, string runId)
{
string scriptFileName = Path.GetFileName(job.ScriptFilePath);
string scriptFileExtension = Path.GetExtension(job.ScriptFilePath);
logger.LogInformation("Run script '{0}' with script host - '{1}'".FormatCurrentCulture(scriptFileName, job.ScriptHost.GetType().Name));
string siteMode = Settings.GetWebSitePolicy();
_analytics.JobStarted(job.Name.Fuzz(), scriptFileExtension, job.JobType, siteMode);
try
{
var exe = _externalCommandFactory.BuildCommandExecutable(job.ScriptHost.HostPath, WorkingDirectory, IdleTimeout, NullLogger.Instance);
// Set environment variable to be able to identify all processes spawned for this job
exe.EnvironmentVariables[GetJobEnvironmentKey()] = "true";
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRootPath] = WorkingDirectory;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsName] = job.Name;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsType] = job.JobType;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsDataPath] = JobDataPath;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRunId] = runId;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsExtraUrlPath] = JobsManagerBase.GetJobExtraInfoUrlFilePath(JobDataPath);
UpdateStatus(logger, "Running");
int exitCode =
exe.ExecuteReturnExitCode(
TraceFactory.GetTracer(),
logger.LogStandardOutput,
logger.LogStandardError,
job.ScriptHost.ArgumentsFormat,
job.RunCommand);
if (exitCode != 0)
{
logger.LogError("Job failed due to exit code " + exitCode);
}
else
{
UpdateStatus(logger, "Success");
}
}
catch (Exception ex)
{
if (ex is ThreadAbortException)
{
// We kill the process when refreshing the job
logger.LogInformation("Job aborted");
UpdateStatus(logger, "Aborted");
return;
}
logger.LogError(ex.ToString());
}
}
protected void NotifyShutdownJob()
{
try
{
FileSystemHelpers.EnsureDirectory(Path.GetDirectoryName(_shutdownNotificationFilePath));
OperationManager.Attempt(() => FileSystemHelpers.WriteAllText(_shutdownNotificationFilePath, DateTime.UtcNow.ToString()));
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
_analytics.UnexpectedException(ex);
}
}
public void ResetLockedStatusFile()
{
try
{
FileStream newLockedStatusFile = File.Open(GetStatusFilePath(), FileMode.OpenOrCreate, FileAccess.Read, FileShare.ReadWrite);
OperationManager.SafeExecute(() => _lockedStatusFile?.Dispose());
_lockedStatusFile = newLockedStatusFile;
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
throw;
}
}
protected override void ReportStatus <TJobStatus>(TJobStatus status, bool logStatus)
{
try
{
if (!FileSystemHelpers.FileExists(GetStatusFilePath()))
{
ResetLockedStatusFile();
}
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
base.ReportStatus(status, logStatus);
}
protected void ReportStatus <TJobStatus>(TJobStatus status, bool logStatus) where TJobStatus : class, IJobStatus
{
try
{
string content = JsonConvert.SerializeObject(status, JsonSerializerSettings);
SafeLogToFile(GetStatusFilePath(), content, isAppend: false);
if (logStatus)
{
LogInformation("Status changed to " + status.Status);
}
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
}
private void ResetLockedStatusFile()
{
try
{
if (_lockedStatusFile != null)
{
_lockedStatusFile.Dispose();
}
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
_lockedStatusFile = File.Open(GetStatusFilePath(), FileMode.OpenOrCreate, FileAccess.Read, FileShare.ReadWrite);
}
private void UpdateAppConfig(string configFilePath)
{
try
{
var settings = SettingsProcessor.Instance;
bool updateXml = false;
// Read app.config
string exeFilePath = configFilePath.Substring(0, configFilePath.Length - ".config".Length);
Configuration config = ConfigurationManager.OpenExeConfiguration(exeFilePath);
foreach (var appSetting in settings.AppSettings)
{
config.AppSettings.Settings.Remove(appSetting.Key);
config.AppSettings.Settings.Add(appSetting.Key, appSetting.Value);
updateXml = true;
}
foreach (ConnectionStringSettings connectionString in settings.ConnectionStrings)
{
ConnectionStringSettings currentConnectionString = config.ConnectionStrings.ConnectionStrings[connectionString.Name];
if (currentConnectionString != null)
{
// Update provider name if connection string already exists and provider name is null (custom type)
connectionString.ProviderName = connectionString.ProviderName ?? currentConnectionString.ProviderName;
}
config.ConnectionStrings.ConnectionStrings.Remove(connectionString.Name);
config.ConnectionStrings.ConnectionStrings.Add(connectionString);
updateXml = true;
}
if (updateXml)
{
// Write updated app.config
config.Save();
}
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
_analytics.UnexpectedException(ex);
}
}
/// <summary>
/// Updates the app.config using XML directly for injecting trace providers.
/// </summary>
private void UpdateAppConfigAddTraceListeners(string configFilePath)
{
try
{
var xmlConfig = XDocument.Load(configFilePath);
// Make sure the trace listeners section available otherwise create it
var configurationElement = GetOrCreateElement(xmlConfig, "configuration");
var systemDiagnosticsElement = GetOrCreateElement(configurationElement, "system.diagnostics");
var traceElement = GetOrCreateElement(systemDiagnosticsElement, "trace");
var listenersElement = GetOrCreateElement(traceElement, "listeners");
// Inject existing trace providers to the target app.config
foreach (TraceListener listener in Trace.Listeners)
{
// Ignore the default trace provider
if (String.Equals(listener.Name, "default", StringComparison.OrdinalIgnoreCase))
{
continue;
}
// Do not add a trace provider if it already exists (by name)
XElement listenerElement = listenersElement.Elements().FirstOrDefault(xElement =>
{
XAttribute nameAttribute = xElement.Attribute("name");
return(nameAttribute != null && String.Equals(nameAttribute.Value, listener.Name, StringComparison.OrdinalIgnoreCase));
});
if (listenerElement == null)
{
var addElement = new XElement("add");
addElement.Add(new XAttribute("name", listener.Name));
addElement.Add(new XAttribute("type", listener.GetType().AssemblyQualifiedName));
listenersElement.AddFirst(addElement);
}
}
FileSystemHelpers.WriteAllText(configFilePath, xmlConfig.ToString());
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
_analytics.UnexpectedException(ex);
}
}
protected void SafeLogToFile(string path, string content, bool isAppend = true)
{
try
{
if (isAppend)
{
OperationManager.Attempt(() => FileSystemHelpers.AppendAllTextToFile(path, content));
}
else
{
OperationManager.Attempt(() => FileSystemHelpers.WriteAllTextToFile(path, content));
}
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
}
protected virtual void ReportStatus <TJobStatus>(TJobStatus status, bool logStatus) where TJobStatus : class, IJobStatus
{
try
{
string content = JsonConvert.SerializeObject(status, JsonSerializerSettings);
SafeLogToFile(GetStatusFilePath(), content, isAppend: false);
if (logStatus)
{
LogInformation("Status changed to " + status.Status);
}
// joblistcache has info about job status, so when changing the status
// the cache should be invalidated.
ClearJobsListCache();
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
}
private void OnMakeChanges(object state)
{
HashSet <string> updatedJobs;
lock (_lockObject)
{
if (_makingChanges)
{
_makeChangesTimer.Change(TimeoutUntilMakingChanges, Timeout.Infinite);
return;
}
_makingChanges = true;
updatedJobs = _updatedJobs;
_updatedJobs = new HashSet <string>();
}
foreach (string updatedJobName in updatedJobs)
{
try
{
ContinuousJob continuousJob = GetJob(updatedJobName);
if (continuousJob == null || !String.IsNullOrEmpty(continuousJob.Error))
{
RemoveJob(updatedJobName);
}
else
{
RefreshJob(continuousJob, logRefresh: !_firstTimeMakingChanges);
}
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
}
_makingChanges = false;
_firstTimeMakingChanges = false;
}
private void StartJob(ContinuousJob continuousJob)
{
// Do not go further if already started or job is disabled
if (IsDisabled)
{
UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
return;
}
if (Interlocked.Exchange(ref _started, 1) == 1)
{
return;
}
_continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);
CheckAlwaysOn();
_continuousJobThread = new Thread(() =>
{
try
{
while (_started == 1 && !IsDisabled)
{
// Try getting the singleton lock if single is enabled
if (!TryGetLockIfSingleton())
{
// Wait 5 seconds and retry to take the lock
WaitForTimeOrStop(TimeSpan.FromSeconds(5));
continue;
}
Stopwatch liveStopwatch = Stopwatch.StartNew();
_continuousJobLogger.StartingNewRun();
using (new Timer(LogStillRunning, null, TimeSpan.FromHours(1), TimeSpan.FromHours(12)))
{
InitializeJobInstance(continuousJob, _continuousJobLogger);
RunJobInstance(continuousJob, _continuousJobLogger, String.Empty, String.Empty);
}
if (_started == 1 && !IsDisabled)
{
// The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob
// Was running for at least 2 minutes there is no wait time.
TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero;
_continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
_continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
WaitForTimeOrStop(webJobsRestartTime);
}
// Make sure lock is released before re-iterating and trying to get the lock again
ReleaseSingletonLock();
}
}
catch (ThreadAbortException)
{
TraceFactory.GetTracer().TraceWarning("Thread was aborted, make sure WebJob was about to stop.");
}
catch (Exception ex)
{
TraceFactory.GetTracer().TraceError(ex);
}
finally
{
ReleaseSingletonLock();
}
});
_continuousJobThread.Start();
}
private void StartJob(ContinuousJob continuousJob)
{
// Do not go further if already started or job is disabled
if (IsDisabled)
{
UpdateStatusIfChanged(ContinuousJobStatus.Stopped);
return;
}
if (Interlocked.Exchange(ref _started, 1) == 1)
{
return;
}
_continuousJobLogger.ReportStatus(ContinuousJobStatus.Starting);
CheckAlwaysOn();
_continuousJobThread = new Thread(() =>
{
var threadAborted = false;
while (!threadAborted && _started == 1 && !IsDisabled)
{
try
{
// Try getting the singleton lock if single is enabled
bool acquired;
if (!TryGetLockIfSingleton(out acquired))
{
// Wait 5 seconds and retry to take the lock
WaitForTimeOrStop(TimeSpan.FromSeconds(5));
continue;
}
try
{
Stopwatch liveStopwatch = Stopwatch.StartNew();
_continuousJobLogger.StartingNewRun();
var tracer = TraceFactory.GetTracer();
using (tracer.Step("Run {0} {1}", continuousJob.JobType, continuousJob.Name))
using (new Timer(LogStillRunning, null, TimeSpan.FromHours(1), TimeSpan.FromHours(12)))
{
InitializeJobInstance(continuousJob, _continuousJobLogger);
WebJobPort = GetAvailableJobPort();
RunJobInstance(continuousJob, _continuousJobLogger, String.Empty, String.Empty, tracer, WebJobPort);
}
if (_started == 1 && !IsDisabled)
{
// The wait time between WebJob invocations is either WebJobsRestartTime (60 seconds by default) or if the WebJob
// Was running for at least 2 minutes there is no wait time.
TimeSpan webJobsRestartTime = liveStopwatch.Elapsed < WarmupTimeSpan ? Settings.GetWebJobsRestartTime() : TimeSpan.Zero;
_continuousJobLogger.LogInformation("Process went down, waiting for {0} seconds".FormatInvariant(webJobsRestartTime.TotalSeconds));
_continuousJobLogger.ReportStatus(ContinuousJobStatus.PendingRestart);
WaitForTimeOrStop(webJobsRestartTime);
}
}
finally
{
if (acquired)
{
// Make sure lock is released before re-iterating and trying to get the lock again
ReleaseSingletonLock();
}
}
}
catch (ThreadAbortException ex)
{
// by nature, ThreadAbortException will be rethrown at the end of this catch block and
// this bool may not be neccessary since while loop will be exited anyway. we added
// it to be explicit.
threadAborted = true;
if (!ex.AbortedByKudu())
{
TraceFactory.GetTracer().TraceWarning("Thread was aborted, make sure WebJob was about to stop.");
}
}
catch (Exception ex)
{
_analytics.UnexpectedException(ex, trace: true);
// sleep to avoid tight exception loop
WaitForTimeOrStop(TimeSpan.FromSeconds(60));
}
}
});
_continuousJobThread.Start();
}
protected void RunJobInstance(JobBase job, IJobLogger logger, string runId)
{
string scriptFileName = Path.GetFileName(job.ScriptFilePath);
string scriptFileFullPath = Path.Combine(WorkingDirectory, job.RunCommand);
string workingDirectoryForScript = Path.GetDirectoryName(scriptFileFullPath);
logger.LogInformation("Run script '{0}' with script host - '{1}'".FormatCurrentCulture(scriptFileName, job.ScriptHost.GetType().Name));
using (var jobStartedReporter = new JobStartedReporter(_analytics, job, Settings.GetWebSitePolicy(), JobDataPath))
{
try
{
var exe = _externalCommandFactory.BuildCommandExecutable(job.ScriptHost.HostPath, workingDirectoryForScript, IdleTimeout, NullLogger.Instance);
_shutdownNotificationFilePath = RefreshShutdownNotificationFilePath(job.Name, job.JobType);
// Set environment variable to be able to identify all processes spawned for this job
exe.EnvironmentVariables[GetJobEnvironmentKey()] = "true";
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRootPath] = WorkingDirectory;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsName] = job.Name;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsType] = job.JobType;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsDataPath] = JobDataPath;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRunId] = runId;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsShutdownNotificationFile] = _shutdownNotificationFilePath;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsCommandArguments] = job.CommandArguments;
UpdateStatus(logger, "Running");
int exitCode =
exe.ExecuteReturnExitCode(
TraceFactory.GetTracer(),
logger.LogStandardOutput,
logger.LogStandardError,
job.ScriptHost.ArgumentsFormat,
scriptFileName,
job.CommandArguments != null ? " " + job.CommandArguments : String.Empty);
if (exitCode != 0)
{
string errorMessage = "Job failed due to exit code " + exitCode;
logger.LogError(errorMessage);
jobStartedReporter.Error = errorMessage;
}
else
{
UpdateStatus(logger, "Success");
}
}
catch (Exception ex)
{
if (ex is ThreadAbortException)
{
// We kill the process when refreshing the job
logger.LogInformation("Job aborted");
UpdateStatus(logger, "Aborted");
return;
}
logger.LogError(ex.ToString());
jobStartedReporter.Error = ex.Message;
}
}
}
