public void RunJob(IJobPublisher publisher, IMessageReceiver receiver, JobInfo jobInfo, ulong deliveryTag, IJobServiceScope serviceScope, CancellationToken cancellationToken) { try { using (var timeoutTokenSource = new CancellationTokenSource(jobInfo.TimeoutMs ?? DefaultJobTimeoutMs)) { var timeoutToken = timeoutTokenSource.Token; using (var combinedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutToken)) { try { InvokePerform(publisher, receiver, jobInfo, deliveryTag, serviceScope, combinedTokenSource.Token); } catch (OperationCanceledException) { if (timeoutToken.IsCancellationRequested) { throw new OperationCanceledException("A job timed out."); } throw; } } } } catch (Exception ex) { _logger.LogError(ex, $"Failed running job: {jobInfo.JobType} / {jobInfo.ArgsType} / {jobInfo.GroupKey}"); publisher.RescheduleJob(jobInfo, ex); receiver.AckBatchMessageProcessed(serviceScope, deliveryTag); } }
public void StartJobWorkers() { lock (WorkersLock) { if (Workers.Count > 0) { return; } try { CreateWorkers(); foreach (var worker in Workers) { worker.Start(); } // reset reconnect time index upon first successful connect _reconnectTimeIndex = 0; } catch (Exception ex) { Logger.LogError(ex, "Failed starting workers"); StopJobWorkers(true, true); } } }
private void CacheJobBinaries(IJobLogger logger) { if (WorkingDirectory != null) { try { int currentHash = CalculateHashForJob(JobBinariesPath); int lastHash = CalculateHashForJob(WorkingDirectory); if (lastHash == currentHash) { return; } } catch (Exception ex) { // Log error and ignore it as it's not critical to cache job binaries logger.LogError("Failed to calculate hash for WebJob: " + ex); _analytics.UnexpectedException(ex); } } SafeKillAllRunningJobInstances(logger); if (FileSystemHelpers.DirectoryExists(JobTempPath)) { FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true); } if (FileSystemHelpers.DirectoryExists(JobTempPath)) { logger.LogWarning("Failed to delete temporary directory"); } try { OperationManager.Attempt(() => { var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName()); FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath); UpdateAppConfigs(tempJobInstancePath); WorkingDirectory = tempJobInstancePath; }); } catch (Exception ex) { //Status = "Worker is not running due to an error"; //TraceError("Failed to copy bin directory: " + ex); logger.LogError("Failed to copy job files: " + ex); _analytics.UnexpectedException(ex); // job disabled WorkingDirectory = null; } }
protected void RunJobInstance(JobBase job, IJobLogger logger, string runId) { string scriptFileName = Path.GetFileName(job.ScriptFilePath); string scriptFileExtension = Path.GetExtension(job.ScriptFilePath); logger.LogInformation("Run script '{0}' with script host - '{1}'".FormatCurrentCulture(scriptFileName, job.ScriptHost.GetType().Name)); string siteMode = Settings.GetWebSitePolicy(); _analytics.JobStarted(job.Name.Fuzz(), scriptFileExtension, job.JobType, siteMode); try { var exe = _externalCommandFactory.BuildCommandExecutable(job.ScriptHost.HostPath, WorkingDirectory, IdleTimeout, NullLogger.Instance); // Set environment variable to be able to identify all processes spawned for this job exe.EnvironmentVariables[GetJobEnvironmentKey()] = "true"; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRootPath] = WorkingDirectory; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsName] = job.Name; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsType] = job.JobType; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsDataPath] = JobDataPath; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRunId] = runId; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsExtraUrlPath] = JobsManagerBase.GetJobExtraInfoUrlFilePath(JobDataPath); UpdateStatus(logger, "Running"); int exitCode = exe.ExecuteReturnExitCode( TraceFactory.GetTracer(), logger.LogStandardOutput, logger.LogStandardError, job.ScriptHost.ArgumentsFormat, job.RunCommand); if (exitCode != 0) { logger.LogError("Job failed due to exit code " + exitCode); } else { UpdateStatus(logger, "Success"); } } catch (Exception ex) { if (ex is ThreadAbortException) { // We kill the process when refreshing the job logger.LogInformation("Job aborted"); UpdateStatus(logger, "Aborted"); return; } logger.LogError(ex.ToString()); } }
private void TryOnStop() { try { OnStop(); } catch (Exception ex) { Logger.LogError(ex, "Exception raised while executing OnStopped"); } }
public static void SafeDispose(this IDisposable disposable, IJobLogger logger = null) { try { if (disposable == null) { return; } if (logger?.IsEnabled(JobLogLevel.Debug) ?? false) { logger.LogDebug($"Disposing {disposable.GetType()}"); } disposable.Dispose(); if (logger?.IsEnabled(JobLogLevel.Debug) ?? false) { logger.LogDebug($"Disposed {disposable.GetType()}"); } } catch (Exception ex) { logger?.LogError(ex, "Disposal error"); } }
protected void EnsureConnectionExists() { lock (_connectionLock) { if (Connection != null) { return; } Logger.LogDebug("Connecting..."); try { Connection = _connectionFactory.CreateConnection(_rabbitMqSettings.HostNames, $"MassiveJobs.NET/{GetEntryFileName()}"); Connection.CallbackException += ConnectionOnCallbackException; Connection.ConnectionBlocked += ConnectionOnConnectionBlocked; Connection.ConnectionUnblocked += ConnectionOnConnectionUnblocked; Connection.ConnectionShutdown += ConnectionOnConnectionShutdown; ModelPool = new ModelPool(Connection, 2, Logger); var model = ModelPool.Get(); try { DeclareTopology(model.Model); } finally { ModelPool.Return(model); } } catch (Exception ex) { Logger.LogError(ex, "Failed connection create"); throw; } } Logger.LogWarning("Connected"); }
public void LogConsoleError_TestOk(string message) { try { loggerConsole.LogError(message); Assert.IsTrue(true); } catch (Exception ex) { Assert.IsTrue(false, ex.ToString()); } }
public static void SafeClose(this IModel model, IJobLogger logger = null) { try { if (model == null || model.IsClosed) { return; } model.Close(); } catch (Exception ex) { logger?.LogError(ex, "Failed closing RabbitMQ model"); } }
public static void SafeClose(this IConnection connection, IJobLogger logger = null) { try { if (connection == null || !connection.IsOpen) { return; } connection.Close(); } catch (Exception ex) { logger?.LogError(ex, "Failed closing RabbitMQ connection"); } }
public int MessageQueueInsert(DbContext dbContext, string routingKey, string argsType, string messageData, DateTime publishedAtUtc) { try { return(dbContext.Database.ExecuteSqlInterpolated($@" INSERT INTO massive_jobs.message_queue (routing_key, args_type, message_data, published_at_utc) VALUES ({routingKey}, {argsType}, {messageData}, {publishedAtUtc}) ")); } catch (Exception ex) { _logger.LogError(ex, "Failed db access"); throw; } }
private void ConsumerOnReceived(object sender, BasicDeliverEventArgs e) { try { if (MessageReceived == null) { return; } var message = new RawMessage { TypeTag = e.BasicProperties?.Type, IsPersistent = e.BasicProperties?.Persistent ?? false, Body = e.Body.ToArray(), DeliveryTag = e.DeliveryTag }; MessageReceived(this, message); } catch (Exception ex) { _logger.LogError(ex, $"Error in {nameof(ConsumerOnReceived)}"); } }
public void ConsumerFunction() { try { // consumer function will run at least once (useful for tests) do { if (_messages.GetMessages(_queueName, out var batch)) { foreach (var msg in batch) { MessageReceived?.Invoke(this, msg); } } }while (_disposed == 0); } catch (Exception ex) { _logger.LogError(ex, "Error in consumer function"); } _messages.MoveUnackToReady(_queueName); _stopEvent.Set(); }
private void CacheJobBinaries(JobBase job, IJobLogger logger) { bool isInPlaceDefault = job.ScriptHost.GetType() == typeof(NodeScriptHost); if (JobSettings.GetIsInPlace(isInPlaceDefault)) { _inPlaceWorkingDirectory = JobBinariesPath; SafeKillAllRunningJobInstances(logger); UpdateAppConfigs(WorkingDirectory); return; } _inPlaceWorkingDirectory = null; if (WorkingDirectory != null) { try { int currentHash = CalculateHashForJob(JobBinariesPath); int lastHash = CalculateHashForJob(WorkingDirectory); if (lastHash == currentHash) { return; } } catch (Exception ex) { // Log error and ignore it as it's not critical to cache job binaries logger.LogWarning("Failed to calculate hash for WebJob, continue to copy WebJob binaries (this will not affect WebJob run)\n" + ex); _analytics.UnexpectedException(ex); } } SafeKillAllRunningJobInstances(logger); if (FileSystemHelpers.DirectoryExists(JobTempPath)) { FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true); } if (FileSystemHelpers.DirectoryExists(JobTempPath)) { logger.LogWarning("Failed to delete temporary directory"); } try { OperationManager.Attempt(() => { var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName()); FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath); UpdateAppConfigs(tempJobInstancePath); _workingDirectory = tempJobInstancePath; }); } catch (Exception ex) { //Status = "Worker is not running due to an error"; //TraceError("Failed to copy bin directory: " + ex); logger.LogError("Failed to copy job files: " + ex); _analytics.UnexpectedException(ex); // job disabled _workingDirectory = null; } }
protected void RunJobInstance(JobBase job, IJobLogger logger, string runId, string trigger, int port = -1) { string scriptFileName = Path.GetFileName(job.ScriptFilePath); string scriptFileFullPath = Path.Combine(WorkingDirectory, job.RunCommand); string workingDirectoryForScript = Path.GetDirectoryName(scriptFileFullPath); logger.LogInformation("Run script '{0}' with script host - '{1}'".FormatCurrentCulture(scriptFileName, job.ScriptHost.GetType().Name)); using (var jobStartedReporter = new JobStartedReporter(_analytics, job, trigger, Settings.GetWebSiteSku(), JobDataPath)) { try { var exe = _externalCommandFactory.BuildCommandExecutable(job.ScriptHost.HostPath, workingDirectoryForScript, IdleTimeout, NullLogger.Instance); _shutdownNotificationFilePath = RefreshShutdownNotificationFilePath(job.Name, job.JobType); // Set environment variable to be able to identify all processes spawned for this job exe.EnvironmentVariables[GetJobEnvironmentKey()] = "true"; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRootPath] = WorkingDirectory; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsName] = job.Name; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsType] = job.JobType; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsDataPath] = JobDataPath; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRunId] = runId; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsCommandArguments] = job.CommandArguments; if (port != -1) { exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsPort] = port.ToString(); } if (_shutdownNotificationFilePath != null) { exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsShutdownNotificationFile] = _shutdownNotificationFilePath; } UpdateStatus(logger, "Running"); int exitCode = exe.ExecuteReturnExitCode( TraceFactory.GetTracer(), logger.LogStandardOutput, logger.LogStandardError, job.ScriptHost.ArgumentsFormat, scriptFileName, job.CommandArguments != null ? " " + job.CommandArguments : String.Empty); if (exitCode != 0) { string errorMessage = "Job failed due to exit code " + exitCode; logger.LogError(errorMessage); jobStartedReporter.Error = errorMessage; } else { UpdateStatus(logger, "Success"); } } catch (ThreadAbortException) { // We kill the process when refreshing the job logger.LogInformation("WebJob process was aborted"); UpdateStatus(logger, "Stopped"); } catch (Exception ex) { logger.LogError(ex.ToString()); jobStartedReporter.Error = ex.Message; } } }
private void CacheJobBinaries(JobBase job, IJobLogger logger) { bool isInPlaceDefault = job.ScriptHost.GetType() == typeof(NodeScriptHost); if (JobSettings.GetIsInPlace(isInPlaceDefault)) { _inPlaceWorkingDirectory = JobBinariesPath; SafeKillAllRunningJobInstances(logger); UpdateAppConfigs(WorkingDirectory, _analytics); return; } _inPlaceWorkingDirectory = null; Dictionary<string, FileInfoBase> sourceDirectoryFileMap = GetJobDirectoryFileMap(JobBinariesPath); if (WorkingDirectory != null) { try { var workingDirectoryFileMap = GetJobDirectoryFileMap(WorkingDirectory); if (!JobDirectoryHasChanged(sourceDirectoryFileMap, workingDirectoryFileMap, _cachedSourceDirectoryFileMap, logger)) { // no changes detected, so skip the cache/copy step below return; } } catch (Exception ex) { // Log error and ignore it, since this diff optimization isn't critical. // We'll just do a full copy in this case. logger.LogWarning("Failed to diff WebJob directories for changes. Continuing to copy WebJob binaries (this will not affect the WebJob run)\n" + ex); _analytics.UnexpectedException(ex); } } SafeKillAllRunningJobInstances(logger); if (FileSystemHelpers.DirectoryExists(JobTempPath)) { FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true); } if (FileSystemHelpers.DirectoryExists(JobTempPath)) { logger.LogWarning("Failed to delete temporary directory"); } try { OperationManager.Attempt(() => { var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName()); FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath); UpdateAppConfigs(tempJobInstancePath, _analytics); _workingDirectory = tempJobInstancePath; // cache the file map snapshot for next time (to aid in detecting // file deletions) _cachedSourceDirectoryFileMap = sourceDirectoryFileMap; }); } catch (Exception ex) { //Status = "Worker is not running due to an error"; //TraceError("Failed to copy bin directory: " + ex); logger.LogError("Failed to copy job files: " + ex); _analytics.UnexpectedException(ex); // job disabled _workingDirectory = null; } }
protected void RunJobInstance(JobBase job, IJobLogger logger, string runId) { string scriptFileName = Path.GetFileName(job.ScriptFilePath); string scriptFileFullPath = Path.Combine(WorkingDirectory, job.RunCommand); string workingDirectoryForScript = Path.GetDirectoryName(scriptFileFullPath); logger.LogInformation("Run script '{0}' with script host - '{1}'".FormatCurrentCulture(scriptFileName, job.ScriptHost.GetType().Name)); using (var jobStartedReporter = new JobStartedReporter(_analytics, job, Settings.GetWebSitePolicy(), JobDataPath)) { try { var exe = _externalCommandFactory.BuildCommandExecutable(job.ScriptHost.HostPath, workingDirectoryForScript, IdleTimeout, NullLogger.Instance); _shutdownNotificationFilePath = RefreshShutdownNotificationFilePath(job.Name, job.JobType); // Set environment variable to be able to identify all processes spawned for this job exe.EnvironmentVariables[GetJobEnvironmentKey()] = "true"; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRootPath] = WorkingDirectory; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsName] = job.Name; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsType] = job.JobType; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsDataPath] = JobDataPath; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRunId] = runId; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsShutdownNotificationFile] = _shutdownNotificationFilePath; exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsCommandArguments] = job.CommandArguments; UpdateStatus(logger, "Running"); int exitCode = exe.ExecuteReturnExitCode( TraceFactory.GetTracer(), logger.LogStandardOutput, logger.LogStandardError, job.ScriptHost.ArgumentsFormat, scriptFileName, job.CommandArguments != null ? " " + job.CommandArguments : String.Empty); if (exitCode != 0) { string errorMessage = "Job failed due to exit code " + exitCode; logger.LogError(errorMessage); jobStartedReporter.Error = errorMessage; } else { UpdateStatus(logger, "Success"); } } catch (Exception ex) { if (ex is ThreadAbortException) { // We kill the process when refreshing the job logger.LogInformation("Job aborted"); UpdateStatus(logger, "Aborted"); return; } logger.LogError(ex.ToString()); jobStartedReporter.Error = ex.Message; } } }
private void CacheJobBinaries(IJobLogger logger) { if (WorkingDirectory != null) { int currentHash = CalculateHashForJob(JobBinariesPath); int lastHash = CalculateHashForJob(WorkingDirectory); if (lastHash == currentHash) { return; } } SafeKillAllRunningJobInstances(logger); if (FileSystem.Directory.Exists(JobTempPath)) { FileSystemHelpers.DeleteDirectorySafe(JobTempPath, true); } if (FileSystem.Directory.Exists(JobTempPath)) { logger.LogWarning("Failed to delete temporary directory"); } try { var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName()); FileSystemHelpers.CopyDirectoryRecursive(FileSystem, JobBinariesPath, tempJobInstancePath); UpdateAppConfigs(tempJobInstancePath); WorkingDirectory = tempJobInstancePath; } catch (Exception ex) { //Status = "Worker is not running due to an error"; //TraceError("Failed to copy bin directory: " + ex); logger.LogError("Failed to copy job files: " + ex); // job disabled WorkingDirectory = null; } }
private void CacheJobBinaries(JobBase job, IJobLogger logger) { bool isInPlaceDefault = job.ScriptHost.GetType() == typeof(NodeScriptHost); if (JobSettings.GetIsInPlace(isInPlaceDefault)) { _inPlaceWorkingDirectory = JobBinariesPath; SafeKillAllRunningJobInstances(logger); UpdateAppConfigs(WorkingDirectory, _analytics); return; } _inPlaceWorkingDirectory = null; Dictionary <string, FileInfoBase> sourceDirectoryFileMap = GetJobDirectoryFileMap(JobBinariesPath); if (WorkingDirectory != null) { try { var workingDirectoryFileMap = GetJobDirectoryFileMap(WorkingDirectory); if (!JobDirectoryHasChanged(sourceDirectoryFileMap, workingDirectoryFileMap, _cachedSourceDirectoryFileMap, logger)) { // no changes detected, so skip the cache/copy step below return; } } catch (Exception ex) { // Log error and ignore it, since this diff optimization isn't critical. // We'll just do a full copy in this case. logger.LogWarning("Failed to diff WebJob directories for changes. Continuing to copy WebJob binaries (this will not affect the WebJob run)\n" + ex); _analytics.UnexpectedException(ex); } } SafeKillAllRunningJobInstances(logger); if (FileSystemHelpers.DirectoryExists(JobTempPath)) { FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true); } if (FileSystemHelpers.DirectoryExists(JobTempPath)) { logger.LogWarning("Failed to delete temporary directory"); } try { OperationManager.Attempt(() => { var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName()); FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath); UpdateAppConfigs(tempJobInstancePath, _analytics); _workingDirectory = tempJobInstancePath; // cache the file map snapshot for next time (to aid in detecting // file deletions) _cachedSourceDirectoryFileMap = sourceDirectoryFileMap; }); } catch (Exception ex) { //Status = "Worker is not running due to an error"; //TraceError("Failed to copy bin directory: " + ex); logger.LogError("Failed to copy job files: " + ex); _analytics.UnexpectedException(ex); // job disabled _workingDirectory = null; } }