public void RunJob(IJobPublisher publisher, IMessageReceiver receiver, JobInfo jobInfo, ulong deliveryTag, IJobServiceScope serviceScope, CancellationToken cancellationToken)
{
try
{
using (var timeoutTokenSource = new CancellationTokenSource(jobInfo.TimeoutMs ?? DefaultJobTimeoutMs))
{
var timeoutToken = timeoutTokenSource.Token;
using (var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutToken))
{
try
{
InvokePerform(publisher, receiver, jobInfo, deliveryTag, serviceScope, combinedTokenSource.Token);
}
catch (OperationCanceledException)
{
if (timeoutToken.IsCancellationRequested)
{
throw new OperationCanceledException("A job timed out.");
}
throw;
}
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, $"Failed running job: {jobInfo.JobType} / {jobInfo.ArgsType} / {jobInfo.GroupKey}");
publisher.RescheduleJob(jobInfo, ex);
receiver.AckBatchMessageProcessed(serviceScope, deliveryTag);
}
}
public void StartJobWorkers()
{
lock (WorkersLock)
{
if (Workers.Count > 0)
{
return;
}
try
{
CreateWorkers();
foreach (var worker in Workers)
{
worker.Start();
}
// reset reconnect time index upon first successful connect
_reconnectTimeIndex = 0;
}
catch (Exception ex)
{
Logger.LogError(ex, "Failed starting workers");
StopJobWorkers(true, true);
}
}
}
private void CacheJobBinaries(IJobLogger logger)
{
if (WorkingDirectory != null)
{
try
{
int currentHash = CalculateHashForJob(JobBinariesPath);
int lastHash = CalculateHashForJob(WorkingDirectory);
if (lastHash == currentHash)
{
return;
}
}
catch (Exception ex)
{
// Log error and ignore it as it's not critical to cache job binaries
logger.LogError("Failed to calculate hash for WebJob: " + ex);
_analytics.UnexpectedException(ex);
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true);
}
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
OperationManager.Attempt(() =>
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath);
WorkingDirectory = tempJobInstancePath;
});
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
_analytics.UnexpectedException(ex);
// job disabled
WorkingDirectory = null;
}
}
protected void RunJobInstance(JobBase job, IJobLogger logger, string runId)
{
string scriptFileName = Path.GetFileName(job.ScriptFilePath);
string scriptFileExtension = Path.GetExtension(job.ScriptFilePath);
logger.LogInformation("Run script '{0}' with script host - '{1}'".FormatCurrentCulture(scriptFileName, job.ScriptHost.GetType().Name));
string siteMode = Settings.GetWebSitePolicy();
_analytics.JobStarted(job.Name.Fuzz(), scriptFileExtension, job.JobType, siteMode);
try
{
var exe = _externalCommandFactory.BuildCommandExecutable(job.ScriptHost.HostPath, WorkingDirectory, IdleTimeout, NullLogger.Instance);
// Set environment variable to be able to identify all processes spawned for this job
exe.EnvironmentVariables[GetJobEnvironmentKey()] = "true";
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRootPath] = WorkingDirectory;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsName] = job.Name;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsType] = job.JobType;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsDataPath] = JobDataPath;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRunId] = runId;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsExtraUrlPath] = JobsManagerBase.GetJobExtraInfoUrlFilePath(JobDataPath);
UpdateStatus(logger, "Running");
int exitCode =
exe.ExecuteReturnExitCode(
TraceFactory.GetTracer(),
logger.LogStandardOutput,
logger.LogStandardError,
job.ScriptHost.ArgumentsFormat,
job.RunCommand);
if (exitCode != 0)
{
logger.LogError("Job failed due to exit code " + exitCode);
}
else
{
UpdateStatus(logger, "Success");
}
}
catch (Exception ex)
{
if (ex is ThreadAbortException)
{
// We kill the process when refreshing the job
logger.LogInformation("Job aborted");
UpdateStatus(logger, "Aborted");
return;
}
logger.LogError(ex.ToString());
}
}
private void TryOnStop()
{
try
{
OnStop();
}
catch (Exception ex)
{
Logger.LogError(ex, "Exception raised while executing OnStopped");
}
}
public static void SafeDispose(this IDisposable disposable, IJobLogger logger = null)
{
try
{
if (disposable == null)
{
return;
}
if (logger?.IsEnabled(JobLogLevel.Debug) ?? false)
{
logger.LogDebug($"Disposing {disposable.GetType()}");
}
disposable.Dispose();
if (logger?.IsEnabled(JobLogLevel.Debug) ?? false)
{
logger.LogDebug($"Disposed {disposable.GetType()}");
}
}
catch (Exception ex)
{
logger?.LogError(ex, "Disposal error");
}
}
protected void EnsureConnectionExists()
{
lock (_connectionLock)
{
if (Connection != null)
{
return;
}
Logger.LogDebug("Connecting...");
try
{
Connection = _connectionFactory.CreateConnection(_rabbitMqSettings.HostNames, $"MassiveJobs.NET/{GetEntryFileName()}");
Connection.CallbackException += ConnectionOnCallbackException;
Connection.ConnectionBlocked += ConnectionOnConnectionBlocked;
Connection.ConnectionUnblocked += ConnectionOnConnectionUnblocked;
Connection.ConnectionShutdown += ConnectionOnConnectionShutdown;
ModelPool = new ModelPool(Connection, 2, Logger);
var model = ModelPool.Get();
try
{
DeclareTopology(model.Model);
}
finally
{
ModelPool.Return(model);
}
}
catch (Exception ex)
{
Logger.LogError(ex, "Failed connection create");
throw;
}
}
Logger.LogWarning("Connected");
}
public void LogConsoleError_TestOk(string message)
{
try
{
loggerConsole.LogError(message);
Assert.IsTrue(true);
}
catch (Exception ex)
{
Assert.IsTrue(false, ex.ToString());
}
}
public static void SafeClose(this IModel model, IJobLogger logger = null)
{
try
{
if (model == null || model.IsClosed)
{
return;
}
model.Close();
}
catch (Exception ex)
{
logger?.LogError(ex, "Failed closing RabbitMQ model");
}
}
public static void SafeClose(this IConnection connection, IJobLogger logger = null)
{
try
{
if (connection == null || !connection.IsOpen)
{
return;
}
connection.Close();
}
catch (Exception ex)
{
logger?.LogError(ex, "Failed closing RabbitMQ connection");
}
}
public int MessageQueueInsert(DbContext dbContext, string routingKey, string argsType, string messageData, DateTime publishedAtUtc)
{
try
{
return(dbContext.Database.ExecuteSqlInterpolated($@"
INSERT INTO massive_jobs.message_queue (routing_key, args_type, message_data, published_at_utc)
VALUES ({routingKey}, {argsType}, {messageData}, {publishedAtUtc})
"));
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed db access");
throw;
}
}
private void ConsumerOnReceived(object sender, BasicDeliverEventArgs e)
{
try
{
if (MessageReceived == null)
{
return;
}
var message = new RawMessage
{
TypeTag = e.BasicProperties?.Type,
IsPersistent = e.BasicProperties?.Persistent ?? false,
Body = e.Body.ToArray(),
DeliveryTag = e.DeliveryTag
};
MessageReceived(this, message);
}
catch (Exception ex)
{
_logger.LogError(ex, $"Error in {nameof(ConsumerOnReceived)}");
}
}
public void ConsumerFunction()
{
try
{
// consumer function will run at least once (useful for tests)
do
{
if (_messages.GetMessages(_queueName, out var batch))
{
foreach (var msg in batch)
{
MessageReceived?.Invoke(this, msg);
}
}
}while (_disposed == 0);
}
catch (Exception ex)
{
_logger.LogError(ex, "Error in consumer function");
}
_messages.MoveUnackToReady(_queueName);
_stopEvent.Set();
}
private void CacheJobBinaries(JobBase job, IJobLogger logger)
{
bool isInPlaceDefault = job.ScriptHost.GetType() == typeof(NodeScriptHost);
if (JobSettings.GetIsInPlace(isInPlaceDefault))
{
_inPlaceWorkingDirectory = JobBinariesPath;
SafeKillAllRunningJobInstances(logger);
UpdateAppConfigs(WorkingDirectory);
return;
}
_inPlaceWorkingDirectory = null;
if (WorkingDirectory != null)
{
try
{
int currentHash = CalculateHashForJob(JobBinariesPath);
int lastHash = CalculateHashForJob(WorkingDirectory);
if (lastHash == currentHash)
{
return;
}
}
catch (Exception ex)
{
// Log error and ignore it as it's not critical to cache job binaries
logger.LogWarning("Failed to calculate hash for WebJob, continue to copy WebJob binaries (this will not affect WebJob run)\n" + ex);
_analytics.UnexpectedException(ex);
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true);
}
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
OperationManager.Attempt(() =>
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath);
_workingDirectory = tempJobInstancePath;
});
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
_analytics.UnexpectedException(ex);
// job disabled
_workingDirectory = null;
}
}
protected void RunJobInstance(JobBase job, IJobLogger logger, string runId, string trigger, int port = -1)
{
string scriptFileName = Path.GetFileName(job.ScriptFilePath);
string scriptFileFullPath = Path.Combine(WorkingDirectory, job.RunCommand);
string workingDirectoryForScript = Path.GetDirectoryName(scriptFileFullPath);
logger.LogInformation("Run script '{0}' with script host - '{1}'".FormatCurrentCulture(scriptFileName, job.ScriptHost.GetType().Name));
using (var jobStartedReporter = new JobStartedReporter(_analytics, job, trigger, Settings.GetWebSiteSku(), JobDataPath))
{
try
{
var exe = _externalCommandFactory.BuildCommandExecutable(job.ScriptHost.HostPath, workingDirectoryForScript, IdleTimeout, NullLogger.Instance);
_shutdownNotificationFilePath = RefreshShutdownNotificationFilePath(job.Name, job.JobType);
// Set environment variable to be able to identify all processes spawned for this job
exe.EnvironmentVariables[GetJobEnvironmentKey()] = "true";
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRootPath] = WorkingDirectory;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsName] = job.Name;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsType] = job.JobType;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsDataPath] = JobDataPath;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRunId] = runId;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsCommandArguments] = job.CommandArguments;
if (port != -1)
{
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsPort] = port.ToString();
}
if (_shutdownNotificationFilePath != null)
{
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsShutdownNotificationFile] = _shutdownNotificationFilePath;
}
UpdateStatus(logger, "Running");
int exitCode =
exe.ExecuteReturnExitCode(
TraceFactory.GetTracer(),
logger.LogStandardOutput,
logger.LogStandardError,
job.ScriptHost.ArgumentsFormat,
scriptFileName,
job.CommandArguments != null ? " " + job.CommandArguments : String.Empty);
if (exitCode != 0)
{
string errorMessage = "Job failed due to exit code " + exitCode;
logger.LogError(errorMessage);
jobStartedReporter.Error = errorMessage;
}
else
{
UpdateStatus(logger, "Success");
}
}
catch (ThreadAbortException)
{
// We kill the process when refreshing the job
logger.LogInformation("WebJob process was aborted");
UpdateStatus(logger, "Stopped");
}
catch (Exception ex)
{
logger.LogError(ex.ToString());
jobStartedReporter.Error = ex.Message;
}
}
}
private void CacheJobBinaries(JobBase job, IJobLogger logger)
{
bool isInPlaceDefault = job.ScriptHost.GetType() == typeof(NodeScriptHost);
if (JobSettings.GetIsInPlace(isInPlaceDefault))
{
_inPlaceWorkingDirectory = JobBinariesPath;
SafeKillAllRunningJobInstances(logger);
UpdateAppConfigs(WorkingDirectory, _analytics);
return;
}
_inPlaceWorkingDirectory = null;
Dictionary<string, FileInfoBase> sourceDirectoryFileMap = GetJobDirectoryFileMap(JobBinariesPath);
if (WorkingDirectory != null)
{
try
{
var workingDirectoryFileMap = GetJobDirectoryFileMap(WorkingDirectory);
if (!JobDirectoryHasChanged(sourceDirectoryFileMap, workingDirectoryFileMap, _cachedSourceDirectoryFileMap, logger))
{
// no changes detected, so skip the cache/copy step below
return;
}
}
catch (Exception ex)
{
// Log error and ignore it, since this diff optimization isn't critical.
// We'll just do a full copy in this case.
logger.LogWarning("Failed to diff WebJob directories for changes. Continuing to copy WebJob binaries (this will not affect the WebJob run)\n" + ex);
_analytics.UnexpectedException(ex);
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true);
}
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
OperationManager.Attempt(() =>
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath, _analytics);
_workingDirectory = tempJobInstancePath;
// cache the file map snapshot for next time (to aid in detecting
// file deletions)
_cachedSourceDirectoryFileMap = sourceDirectoryFileMap;
});
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
_analytics.UnexpectedException(ex);
// job disabled
_workingDirectory = null;
}
}
private void CacheJobBinaries(IJobLogger logger)
{
if (WorkingDirectory != null)
{
try
{
int currentHash = CalculateHashForJob(JobBinariesPath);
int lastHash = CalculateHashForJob(WorkingDirectory);
if (lastHash == currentHash)
{
return;
}
}
catch (Exception ex)
{
// Log error and ignore it as it's not critical to cache job binaries
logger.LogError("Failed to calculate hash for WebJob: " + ex);
_analytics.UnexpectedException(ex);
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true);
}
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
OperationManager.Attempt(() =>
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath);
WorkingDirectory = tempJobInstancePath;
});
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
_analytics.UnexpectedException(ex);
// job disabled
WorkingDirectory = null;
}
}
protected void RunJobInstance(JobBase job, IJobLogger logger, string runId)
{
string scriptFileName = Path.GetFileName(job.ScriptFilePath);
string scriptFileFullPath = Path.Combine(WorkingDirectory, job.RunCommand);
string workingDirectoryForScript = Path.GetDirectoryName(scriptFileFullPath);
logger.LogInformation("Run script '{0}' with script host - '{1}'".FormatCurrentCulture(scriptFileName, job.ScriptHost.GetType().Name));
using (var jobStartedReporter = new JobStartedReporter(_analytics, job, Settings.GetWebSitePolicy(), JobDataPath))
{
try
{
var exe = _externalCommandFactory.BuildCommandExecutable(job.ScriptHost.HostPath, workingDirectoryForScript, IdleTimeout, NullLogger.Instance);
_shutdownNotificationFilePath = RefreshShutdownNotificationFilePath(job.Name, job.JobType);
// Set environment variable to be able to identify all processes spawned for this job
exe.EnvironmentVariables[GetJobEnvironmentKey()] = "true";
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRootPath] = WorkingDirectory;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsName] = job.Name;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsType] = job.JobType;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsDataPath] = JobDataPath;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRunId] = runId;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsShutdownNotificationFile] = _shutdownNotificationFilePath;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsCommandArguments] = job.CommandArguments;
UpdateStatus(logger, "Running");
int exitCode =
exe.ExecuteReturnExitCode(
TraceFactory.GetTracer(),
logger.LogStandardOutput,
logger.LogStandardError,
job.ScriptHost.ArgumentsFormat,
scriptFileName,
job.CommandArguments != null ? " " + job.CommandArguments : String.Empty);
if (exitCode != 0)
{
string errorMessage = "Job failed due to exit code " + exitCode;
logger.LogError(errorMessage);
jobStartedReporter.Error = errorMessage;
}
else
{
UpdateStatus(logger, "Success");
}
}
catch (Exception ex)
{
if (ex is ThreadAbortException)
{
// We kill the process when refreshing the job
logger.LogInformation("Job aborted");
UpdateStatus(logger, "Aborted");
return;
}
logger.LogError(ex.ToString());
jobStartedReporter.Error = ex.Message;
}
}
}
private void CacheJobBinaries(IJobLogger logger)
{
if (WorkingDirectory != null)
{
int currentHash = CalculateHashForJob(JobBinariesPath);
int lastHash = CalculateHashForJob(WorkingDirectory);
if (lastHash == currentHash)
{
return;
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystem.Directory.Exists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, true);
}
if (FileSystem.Directory.Exists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(FileSystem, JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath);
WorkingDirectory = tempJobInstancePath;
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
// job disabled
WorkingDirectory = null;
}
}
private void CacheJobBinaries(JobBase job, IJobLogger logger)
{
bool isInPlaceDefault = job.ScriptHost.GetType() == typeof(NodeScriptHost);
if (JobSettings.GetIsInPlace(isInPlaceDefault))
{
_inPlaceWorkingDirectory = JobBinariesPath;
SafeKillAllRunningJobInstances(logger);
UpdateAppConfigs(WorkingDirectory);
return;
}
_inPlaceWorkingDirectory = null;
if (WorkingDirectory != null)
{
try
{
int currentHash = CalculateHashForJob(JobBinariesPath);
int lastHash = CalculateHashForJob(WorkingDirectory);
if (lastHash == currentHash)
{
return;
}
}
catch (Exception ex)
{
// Log error and ignore it as it's not critical to cache job binaries
logger.LogWarning("Failed to calculate hash for WebJob, continue to copy WebJob binaries (this will not affect WebJob run)\n" + ex);
_analytics.UnexpectedException(ex);
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true);
}
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
OperationManager.Attempt(() =>
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath);
_workingDirectory = tempJobInstancePath;
});
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
_analytics.UnexpectedException(ex);
// job disabled
_workingDirectory = null;
}
}
protected void RunJobInstance(JobBase job, IJobLogger logger, string runId)
{
string scriptFileName = Path.GetFileName(job.ScriptFilePath);
string scriptFileExtension = Path.GetExtension(job.ScriptFilePath);
logger.LogInformation("Run script '{0}' with script host - '{1}'".FormatCurrentCulture(scriptFileName, job.ScriptHost.GetType().Name));
string siteMode = Settings.GetWebSitePolicy();
_analytics.JobStarted(job.Name.Fuzz(), scriptFileExtension, job.JobType, siteMode);
try
{
var exe = _externalCommandFactory.BuildCommandExecutable(job.ScriptHost.HostPath, WorkingDirectory, IdleTimeout, NullLogger.Instance);
// Set environment variable to be able to identify all processes spawned for this job
exe.EnvironmentVariables[GetJobEnvironmentKey()] = "true";
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRootPath] = WorkingDirectory;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsName] = job.Name;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsType] = job.JobType;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsDataPath] = JobDataPath;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRunId] = runId;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsExtraUrlPath] = JobsManagerBase.GetJobExtraInfoUrlFilePath(JobDataPath);
UpdateStatus(logger, "Running");
int exitCode =
exe.ExecuteReturnExitCode(
TraceFactory.GetTracer(),
logger.LogStandardOutput,
logger.LogStandardError,
job.ScriptHost.ArgumentsFormat,
job.RunCommand);
if (exitCode != 0)
{
logger.LogError("Job failed due to exit code " + exitCode);
}
else
{
UpdateStatus(logger, "Success");
}
}
catch (Exception ex)
{
if (ex is ThreadAbortException)
{
// We kill the process when refreshing the job
logger.LogInformation("Job aborted");
UpdateStatus(logger, "Aborted");
return;
}
logger.LogError(ex.ToString());
}
}
private void CacheJobBinaries(JobBase job, IJobLogger logger)
{
bool isInPlaceDefault = job.ScriptHost.GetType() == typeof(NodeScriptHost);
if (JobSettings.GetIsInPlace(isInPlaceDefault))
{
_inPlaceWorkingDirectory = JobBinariesPath;
SafeKillAllRunningJobInstances(logger);
UpdateAppConfigs(WorkingDirectory, _analytics);
return;
}
_inPlaceWorkingDirectory = null;
Dictionary <string, FileInfoBase> sourceDirectoryFileMap = GetJobDirectoryFileMap(JobBinariesPath);
if (WorkingDirectory != null)
{
try
{
var workingDirectoryFileMap = GetJobDirectoryFileMap(WorkingDirectory);
if (!JobDirectoryHasChanged(sourceDirectoryFileMap, workingDirectoryFileMap, _cachedSourceDirectoryFileMap, logger))
{
// no changes detected, so skip the cache/copy step below
return;
}
}
catch (Exception ex)
{
// Log error and ignore it, since this diff optimization isn't critical.
// We'll just do a full copy in this case.
logger.LogWarning("Failed to diff WebJob directories for changes. Continuing to copy WebJob binaries (this will not affect the WebJob run)\n" + ex);
_analytics.UnexpectedException(ex);
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true);
}
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
OperationManager.Attempt(() =>
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath, _analytics);
_workingDirectory = tempJobInstancePath;
// cache the file map snapshot for next time (to aid in detecting
// file deletions)
_cachedSourceDirectoryFileMap = sourceDirectoryFileMap;
});
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
_analytics.UnexpectedException(ex);
// job disabled
_workingDirectory = null;
}
}