public static void Decompress(BinaryReader reader, BinaryWriter writer, int jobCount, int chunkSize = Constants.DEFAULT_CHUNK_SIZE,
CancellationToken cancellationToken = null, Loggers loggers = null)
{
Guard.NotNull(reader, nameof(reader));
Guard.NotNull(writer, nameof(writer));
Guard.NotZeroOrNegative(jobCount, nameof(jobCount));
Guard.NotZeroOrNegative(chunkSize, nameof(chunkSize));
Console.WriteLine("Decompress as Enumerable");
IDefaultLogger defaultLogger = loggers?.DefaultLogger;
IChunkLogger chunkLogger = loggers?.ChunkLogger;
IJobLogger jobLogger = loggers?.JobLogger;
ChunkSource.ReadHeader(reader, out int chunkCount);
var chunks = ChunkSource.ReadChunkCompressed(reader, chunkCount)
.AsParallel(jobCount)
.Do(x => chunkLogger?.LogChunk("Read", x))
.Map(ChunkConverter.Unzip)
.Do(x => chunkLogger?.LogChunk("Proc", x))
.AsEnumerable(cancellationToken, jobLogger);
int index = 0;
foreach (var chunk in chunks)
{
ChunkTarget.WriteChunk(chunk, writer, chunkSize);
chunkLogger?.LogChunk("Write", chunk);
defaultLogger?.LogChunksProcessed(++index, chunkCount);
}
}
protected override void UpdateStatus(IJobLogger logger, string status)
{
logger.ReportStatus(new ContinuousJobStatus()
{
Status = status
});
}
public DefaultJobPublisher(MassiveJobsSettings settings,
IMessagePublisher messagePublisher,
IJobTypeProvider jobTypeProvider,
IJobSerializer jobSerializer,
IJobLogger <DefaultJobPublisher> logger)
{
_batchSize = settings.PublishBatchSize;
Settings = settings;
MessagePublisher = messagePublisher;
JobTypeProvider = jobTypeProvider;
JobSerializer = jobSerializer;
Logger = logger;
// This is just to avoid always publishing to a first worker
// since the new instance of publisher is created on each job batch
// for scheduled/periodic workers. Actually, immediate workers will
// also create one instance of publisher per batch to inject into
// jobs if jobs require it in their constructors.
var tickCount = Math.Abs(Environment.TickCount);
if (settings.ImmediateWorkersCount > 0)
{
_nextImmediateWorkerIndex = tickCount % settings.ImmediateWorkersCount;
}
if (settings.ScheduledWorkersCount > 0)
{
_nextScheduledWorkerIndex = tickCount % settings.ScheduledWorkersCount;
}
if (settings.LongRunningWorkersCount > 0)
{
_nextLongRunningWorkerIndex = tickCount % settings.LongRunningWorkersCount;
}
}
//-----------------------------------------------------------------------------------------------------------------------------------------------------
public SchedulerLifecycleManager(IFramework framework, AutofacJobFactory jobFactory, IEnumerable<IJobDetail> jobDetails, Auto<IJobLogger> logger)
{
_framework = framework;
_logger = logger.Instance;
_jobDetails = jobDetails.ToArray();
_jobFactory = jobFactory;
}
public static void SafeDispose(this IDisposable disposable, IJobLogger logger = null)
{
try
{
if (disposable == null)
{
return;
}
if (logger?.IsEnabled(JobLogLevel.Debug) ?? false)
{
logger.LogDebug($"Disposing {disposable.GetType()}");
}
disposable.Dispose();
if (logger?.IsEnabled(JobLogLevel.Debug) ?? false)
{
logger.LogDebug($"Disposed {disposable.GetType()}");
}
}
catch (Exception ex)
{
logger?.LogError(ex, "Disposal error");
}
}
public ActionResult DeleteLesson(string RowKey)
{
//NetTcpBinding binding = new NetTcpBinding();
//ChannelFactory<IJobReader> factory = new ChannelFactory<IJobReader>(binding, new EndpointAddress("net.tcp://localhost:10101/InputRequest"));
//IJobReader proxy = factory.CreateChannel();
//List<Lesson> listaSub = new List<Lesson>();
//listaSub = proxy.ReadFromDatabase2();
//Lesson p = listaSub.Find(st => st.RowKey.Equals(RowKey));
NetTcpBinding binding1 = new NetTcpBinding();
ChannelFactory <IJobWriter> factory1 = new ChannelFactory <IJobWriter>(binding1, new EndpointAddress("net.tcp://localhost:10100/InputRequest"));
IJobWriter proxy1 = factory1.CreateChannel();
NetTcpBinding binding2 = new NetTcpBinding();
ChannelFactory <IJobLogger> factory2 = new ChannelFactory <IJobLogger>(binding1, new EndpointAddress("net.tcp://localhost:10102/InputRequest"));
IJobLogger proxy2 = factory2.CreateChannel();
proxy1.DeleteLesson(RowKey);
proxy2.Logger("Izbrisan iz tabele", RowKey, "deleted");
return(RedirectToAction("GetLesson", "Subject"));
}
public ActionResult AddLesson(string RowKey, string Title, DateTime PlannedDate, string Description)
{
NetTcpBinding binding = new NetTcpBinding();
ChannelFactory <IJobReader> factory = new ChannelFactory <IJobReader>(binding, new EndpointAddress("net.tcp://localhost:10101/InputRequest"));
IJobReader proxy = factory.CreateChannel();
NetTcpBinding binding1 = new NetTcpBinding();
ChannelFactory <IJobWriter> factory1 = new ChannelFactory <IJobWriter>(binding, new EndpointAddress("net.tcp://localhost:10100/InputRequest"));
IJobWriter proxy1 = factory1.CreateChannel();
NetTcpBinding binding2 = new NetTcpBinding();
ChannelFactory <IJobLogger> factory2 = new ChannelFactory <IJobLogger>(binding, new EndpointAddress("net.tcp://localhost:10102/InputRequest"));
IJobLogger proxy2 = factory2.CreateChannel();
Lesson l = new Lesson(RowKey)
{
Title = Title, PlannedDate = PlannedDate, Description = Description
};
if (!proxy.ExistsLesson(l))
{
proxy1.AddLesson(l);
proxy2.LoggerLesson("Upisano u Table", RowKey, Title, PlannedDate, Description);
return(RedirectToAction("Index", "Subject"));
}
return(RedirectToAction("AddLesson", "Subject"));
}
protected BatchProcessor(int batchSize, IJobLogger <BatchProcessor <TMessage> > logger)
{
_batchSize = batchSize;
_messages = new ConcurrentQueue <TMessage>();
Logger = logger;
}
public ActionResult AddProfessor(string RowKey, string Name, string Lastname)
{
NetTcpBinding binding = new NetTcpBinding();
ChannelFactory <IJobReader> factory = new ChannelFactory <IJobReader>(binding, new EndpointAddress("net.tcp://localhost:10101/InputRequest"));
IJobReader proxy = factory.CreateChannel();
NetTcpBinding binding1 = new NetTcpBinding();
ChannelFactory <IJobWriter> factory1 = new ChannelFactory <IJobWriter>(binding, new EndpointAddress("net.tcp://localhost:10100/InputRequest"));
IJobWriter proxy1 = factory1.CreateChannel();
NetTcpBinding binding2 = new NetTcpBinding();
ChannelFactory <IJobLogger> factory2 = new ChannelFactory <IJobLogger>(binding, new EndpointAddress("net.tcp://localhost:10102/InputRequest"));
IJobLogger proxy2 = factory2.CreateChannel();
Professor professor = new Professor(RowKey)
{
Name = Name, Surname = Lastname
};
if (!proxy.ExistsProfessor(professor))
{
proxy1.AddProfessor(professor);
proxy2.LoggerProfessor("Upisano u Table", RowKey, Name, Lastname);
// proxy2.Download(RowKey + Name);
return(RedirectToAction("Index", "Subject"));
}
return(RedirectToAction("AddProfessor", "Subject"));
}
public ModelPool(IConnection connection, int maxRetained, IJobLogger logger)
{
_connection = connection;
_maxRetained = maxRetained;
_logger = logger;
_models = new Queue <ModelPoolEntry>();
}
public void SafeKillAllRunningJobInstances(IJobLogger logger)
{
try
{
Process[] processes = Process.GetProcesses();
foreach (Process process in processes)
{
StringDictionary processEnvironment;
bool success = ProcessEnvironment.TryGetEnvironmentVariables(process, out processEnvironment);
if (success && processEnvironment.ContainsKey(GetJobEnvironmentKey()))
{
try
{
process.Kill(true, TraceFactory.GetTracer());
}
catch (Exception ex)
{
if (!process.HasExited)
{
logger.LogWarning("Failed to kill process - {0} for job - {1}\n{2}".FormatInvariant(process.ProcessName, JobName, ex));
}
}
}
}
}
catch (Exception ex)
{
logger.LogWarning(ex.ToString());
}
}
public RabbitMqMessageBroker(RabbitMqSettings rabbitMqSettings, MassiveJobsSettings massiveJobsSettings, bool automaticRecoveryEnabled, IJobLogger logger)
{
Logger = logger;
_rabbitMqSettings = rabbitMqSettings;
_massiveJobsSettings = massiveJobsSettings;
_connectionFactory = new ConnectionFactory
{
Port = rabbitMqSettings.Port,
UserName = rabbitMqSettings.Username,
Password = rabbitMqSettings.Password,
VirtualHost = rabbitMqSettings.VirtualHost,
AutomaticRecoveryEnabled = automaticRecoveryEnabled,
TopologyRecoveryEnabled = automaticRecoveryEnabled,
RequestedHeartbeat = TimeSpan.FromSeconds(10),
Ssl =
{
Enabled = rabbitMqSettings.SslEnabled,
ServerName = rabbitMqSettings.SslServerName,
CertPath = rabbitMqSettings.SslClientCertPath,
CertPassphrase = rabbitMqSettings.SslClientCertPassphrase
}
};
}
public static void Init(IJobRepository repository, IJobExecutor executor, IJobLogger logger)
{
Repository = repository ?? throw new ArgumentNullException(nameof(repository));
Executor = executor ?? throw new ArgumentNullException(nameof(executor));
Logger = logger;
IsInit = true;
}
private void CacheJobBinaries(IJobLogger logger)
{
if (WorkingDirectory != null)
{
try
{
int currentHash = CalculateHashForJob(JobBinariesPath);
int lastHash = CalculateHashForJob(WorkingDirectory);
if (lastHash == currentHash)
{
return;
}
}
catch (Exception ex)
{
// Log error and ignore it as it's not critical to cache job binaries
logger.LogError("Failed to calculate hash for WebJob: " + ex);
_analytics.UnexpectedException(ex);
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true);
}
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
OperationManager.Attempt(() =>
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath);
WorkingDirectory = tempJobInstancePath;
});
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
_analytics.UnexpectedException(ex);
// job disabled
WorkingDirectory = null;
}
}
private void CacheJobBinaries(IJobLogger logger)
{
if (WorkingDirectory != null)
{
try
{
int currentHash = CalculateHashForJob(JobBinariesPath);
int lastHash = CalculateHashForJob(WorkingDirectory);
if (lastHash == currentHash)
{
return;
}
}
catch (Exception ex)
{
// Log error and ignore it as it's not critical to cache job binaries
logger.LogError("Failed to calculate hash for WebJob: " + ex);
_analytics.UnexpectedException(ex);
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true);
}
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
OperationManager.Attempt(() =>
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath);
WorkingDirectory = tempJobInstancePath;
});
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
_analytics.UnexpectedException(ex);
// job disabled
WorkingDirectory = null;
}
}
public RabbitMqMessageConsumer(RabbitMqSettings rmqSettings,
MassiveJobsSettings jobsSettings,
IJobLoggerFactory loggerFactory,
IJobLogger <RabbitMqMessageConsumer> consumerLogger)
: base(rmqSettings, jobsSettings, false, consumerLogger)
{
_loggerFactory = loggerFactory;
_prefetchCount = rmqSettings.PrefetchCount;
}
public JobWorker(
JobQueue queue,
IJobLogger logger,
TimeSpan executePeriod
)
{
_queue = queue;
_logger = logger;
_executePeriod = executePeriod;
}
public InMemoryMessageReceiver(InMemoryMessages messages, string queueName, IJobLogger <InMemoryMessageReceiver> logger)
{
_messages = messages;
_queueName = queueName;
_logger = logger;
_consumerThread = new Thread(ConsumerFunction)
{
IsBackground = true
};
}
protected void RunJobInstance(JobBase job, IJobLogger logger, string runId)
{
string scriptFileName = Path.GetFileName(job.ScriptFilePath);
string scriptFileExtension = Path.GetExtension(job.ScriptFilePath);
logger.LogInformation("Run script '{0}' with script host - '{1}'".FormatCurrentCulture(scriptFileName, job.ScriptHost.GetType().Name));
string siteMode = Settings.GetWebSitePolicy();
_analytics.JobStarted(job.Name.Fuzz(), scriptFileExtension, job.JobType, siteMode);
try
{
var exe = _externalCommandFactory.BuildCommandExecutable(job.ScriptHost.HostPath, WorkingDirectory, IdleTimeout, NullLogger.Instance);
// Set environment variable to be able to identify all processes spawned for this job
exe.EnvironmentVariables[GetJobEnvironmentKey()] = "true";
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRootPath] = WorkingDirectory;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsName] = job.Name;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsType] = job.JobType;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsDataPath] = JobDataPath;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRunId] = runId;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsExtraUrlPath] = JobsManagerBase.GetJobExtraInfoUrlFilePath(JobDataPath);
UpdateStatus(logger, "Running");
int exitCode =
exe.ExecuteReturnExitCode(
TraceFactory.GetTracer(),
logger.LogStandardOutput,
logger.LogStandardError,
job.ScriptHost.ArgumentsFormat,
job.RunCommand);
if (exitCode != 0)
{
logger.LogError("Job failed due to exit code " + exitCode);
}
else
{
UpdateStatus(logger, "Success");
}
}
catch (Exception ex)
{
if (ex is ThreadAbortException)
{
// We kill the process when refreshing the job
logger.LogInformation("Job aborted");
UpdateStatus(logger, "Aborted");
return;
}
logger.LogError(ex.ToString());
}
}
public MessageReceiver(IConnection connection, string queueName, ushort prefetchCount,
IJobLogger <MessageReceiver> logger)
{
_queueName = queueName;
_logger = logger;
_model = connection.CreateModel();
_model.BasicQos(0, prefetchCount, false);
_consumer = new EventingBasicConsumer(_model);
_consumer.Received += ConsumerOnReceived;
}
public CallbackTaskQueue(AdminClient adminClient, IJobLogger jobLogger, IOptions <XxlJobExecutorOptions> optionsAccessor
, ILoggerFactory loggerFactory)
{
this._adminClient = adminClient;
this._jobLogger = jobLogger;
this._retryQueue = new RetryCallbackTaskQueue(optionsAccessor.Value.LogPath,
Push,
loggerFactory.CreateLogger <RetryCallbackTaskQueue>());
this._logger = loggerFactory.CreateLogger <CallbackTaskQueue>();
}
protected Worker(string queueName, int batchSize, int masMaxDegreeOfParallelism, bool singleActiveConsumer,
IJobServiceScopeFactory serviceScopeFactory, IJobLogger <Worker> logger)
: base(batchSize, logger)
{
ServiceScopeFactory = serviceScopeFactory;
ServiceScope = ServiceScopeFactory.CreateScope();
_maxDegreeOfParallelism = masMaxDegreeOfParallelism;
_singleActiveConsumer = singleActiveConsumer;
_messageConsumer = ServiceScope.GetRequiredService <IMessageConsumer>();
QueueName = queueName;
}
public JobDispatcher(
TaskExecutorFactory executorFactory,
CallbackTaskQueue callbackTaskQueue,
IJobLogger jobLogger,
ILoggerFactory loggerFactory
)
{
this._executorFactory = executorFactory;
this._callbackTaskQueue = callbackTaskQueue;
this._jobLogger = jobLogger;
this._jobQueueLogger = loggerFactory.CreateLogger <JobTaskQueue>();
this._logger = loggerFactory.CreateLogger <JobDispatcher>();
}
public static void SafeClose(this IModel model, IJobLogger logger = null)
{
try
{
if (model == null || model.IsClosed)
{
return;
}
model.Close();
}
catch (Exception ex)
{
logger?.LogError(ex, "Failed closing RabbitMQ model");
}
}
public XxlRestfulServiceHandler(IOptions <XxlJobExecutorOptions> optionsAccessor,
JobDispatcher jobDispatcher,
IJobLogger jobLogger,
ILogger <XxlRestfulServiceHandler> logger)
{
this._jobDispatcher = jobDispatcher;
this._jobLogger = jobLogger;
this._logger = logger;
this._options = optionsAccessor.Value;
if (this._options == null)
{
throw new ArgumentNullException(nameof(XxlJobExecutorOptions));
}
}
public SqlServerMessageReceiver(string queueName, bool singleActiveConsumer, IJobServiceScopeFactory scopeFactory, IJobLogger <SqlServerMessageReceiver <TDbContext> > logger)
{
_scopeFactory = scopeFactory;
_scope = scopeFactory.CreateScope();
_queueName = queueName;
_singleActiveConsumer = singleActiveConsumer;
_sqlDialect = _scope.GetRequiredService <ISqlDialect>();
_timeProvider = _scope.GetRequiredService <ITimeProvider>();
_logger = logger;
_instanceName = $"{Environment.MachineName}/{Guid.NewGuid()}";
_timer = _scope.GetRequiredService <ITimer>();
_timer.TimeElapsed += TimerCallback;
}
public static void SafeClose(this IConnection connection, IJobLogger logger = null)
{
try
{
if (connection == null || !connection.IsOpen)
{
return;
}
connection.Close();
}
catch (Exception ex)
{
logger?.LogError(ex, "Failed closing RabbitMQ connection");
}
}
public WorkerCoordinator(IJobServiceFactory serviceFactory, IJobLogger <WorkerCoordinator> logger = null)
{
ServiceScopeFactory = serviceFactory.GetRequiredService <IJobServiceScopeFactory>();
ServiceScope = ServiceScopeFactory.CreateScope();
_settings = ServiceScope.GetRequiredService <MassiveJobsSettings>();
_reconnectTimer = ServiceScope.GetRequiredService <ITimer>();
_reconnectTimer.TimeElapsed += Reconnect;
Workers = new List <IWorker>();
Logger = logger ?? ServiceScope.GetRequiredService <IJobLogger <WorkerCoordinator> >();
MessageConsumer = ServiceScope.GetRequiredService <IMessageConsumer>();
MessageConsumer.Disconnected += MessageBrokerDisconnected;
}
public JobQueue(
QueueOptions options,
IJobRepository repository,
IJobExecutor executor,
IJobLogger logger = null
)
{
_options = options;
_repository = repository;
_executor = executor;
_logger = logger;
_workers = Enumerable.Range(0, options.WorkersNumber)
.Select(x => new JobWorker(this, _logger, _options.ExecutePeriod))
.ToArray();
}
internal static bool JobDirectoryHasChanged(
Dictionary <string, FileInfoBase> sourceDirectoryFileMap,
Dictionary <string, FileInfoBase> workingDirectoryFileMap,
Dictionary <string, FileInfoBase> cachedSourceDirectoryFileMap,
IJobLogger logger)
{
// enumerate all source directory files, and compare against the files
// in the working directory (i.e. the cached directory)
FileInfoBase foundEntry = null;
foreach (var entry in sourceDirectoryFileMap)
{
if (workingDirectoryFileMap.TryGetValue(entry.Key, out foundEntry))
{
if (entry.Value.LastWriteTimeUtc != foundEntry.LastWriteTimeUtc)
{
// source file has changed since we last cached it, or a source
// file has been modified in the working directory.
logger.LogInformation(string.Format("Job directory change detected: Job file '{0}' timestamp differs between source and working directories.", entry.Key));
return(true);
}
}
else
{
// A file exists in source that doesn't exist in our working
// directory. This is either because a file was actually added
// to source, or a file that previously existed in source has been
// deleted from the working directory.
logger.LogInformation(string.Format("Job directory change detected: Job file '{0}' exists in source directory but not in working directory.", entry.Key));
return(true);
}
}
// if we've previously run this check in the current process,
// look for any file deletions by ensuring all our previously
// cached entries are still present
foreach (var entry in cachedSourceDirectoryFileMap)
{
if (!sourceDirectoryFileMap.TryGetValue(entry.Key, out foundEntry))
{
logger.LogInformation(string.Format("Job directory change detected: Job file '{0}' has been deleted.", entry.Key));
return(true);
}
}
return(false);
}
public JobFactoryService(ConfigService configService, RuriLibSettingsService settingsService, PluginRepository pluginRepo,
IHitRepository hitRepo, ProxySourceFactoryService proxySourceFactory, DataPoolFactoryService dataPoolFactory,
ProxyReloadService proxyReloadService, IRandomUAProvider randomUAProvider, IRNGProvider rngProvider, IJobLogger logger,
IProxyRepository proxyRepo)
{
this.configService = configService;
this.settingsService = settingsService;
this.pluginRepo = pluginRepo;
this.hitRepo = hitRepo;
this.proxySourceFactory = proxySourceFactory;
this.dataPoolFactory = dataPoolFactory;
this.proxyReloadService = proxyReloadService;
this.randomUAProvider = randomUAProvider;
this.rngProvider = rngProvider;
this.logger = logger;
this.proxyRepo = proxyRepo;
}
internal static bool JobDirectoryHasChanged(
Dictionary<string, FileInfoBase> sourceDirectoryFileMap,
Dictionary<string, FileInfoBase> workingDirectoryFileMap,
Dictionary<string, FileInfoBase> cachedSourceDirectoryFileMap,
IJobLogger logger)
{
// enumerate all source directory files, and compare against the files
// in the working directory (i.e. the cached directory)
FileInfoBase foundEntry = null;
foreach (var entry in sourceDirectoryFileMap)
{
if (workingDirectoryFileMap.TryGetValue(entry.Key, out foundEntry))
{
if (entry.Value.LastWriteTimeUtc != foundEntry.LastWriteTimeUtc)
{
// source file has changed since we last cached it, or a source
// file has been modified in the working directory.
logger.LogInformation(string.Format("Job directory change detected: Job file '{0}' timestamp differs between source and working directories.", entry.Key));
return true;
}
}
else
{
// A file exists in source that doesn't exist in our working
// directory. This is either because a file was actually added
// to source, or a file that previously existed in source has been
// deleted from the working directory.
logger.LogInformation(string.Format("Job directory change detected: Job file '{0}' exists in source directory but not in working directory.", entry.Key));
return true;
}
}
// if we've previously run this check in the current process,
// look for any file deletions by ensuring all our previously
// cached entries are still present
foreach (var entry in cachedSourceDirectoryFileMap)
{
if (!sourceDirectoryFileMap.TryGetValue(entry.Key, out foundEntry))
{
logger.LogInformation(string.Format("Job directory change detected: Job file '{0}' has been deleted.", entry.Key));
return true;
}
}
return false;
}
static JobScheduler()
{
_isInit = Global.IsInit;
if (!_isInit)
return;
_options = Global.Configuration.Options;
_repository = Global.Repository;
_executor = Global.Executor;
_logger = Global.Logger;
_repository?.SetLogger(_logger);
_queues = _options.Queues
.Select(x => new JobQueue(x, _repository, _executor, _logger))
.ToDictionary(x => x.Name);
}
private void CacheJobBinaries(IJobLogger logger)
{
if (WorkingDirectory != null)
{
int currentHash = CalculateHashForJob(JobBinariesPath);
int lastHash = CalculateHashForJob(WorkingDirectory);
if (lastHash == currentHash)
{
return;
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystem.Directory.Exists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, true);
}
if (FileSystem.Directory.Exists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(FileSystem, JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath);
WorkingDirectory = tempJobInstancePath;
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
// job disabled
WorkingDirectory = null;
}
}
public static void SetUpJobLoggers(IJobLogger textFileJobLogger
, IJobLogger consoleJobLogger
, IJobLogger databaseJobLogger)
{
if (textFileJobLogger != null)
{
_textFileJobLogger = textFileJobLogger;
}
if (consoleJobLogger != null)
{
_consoleJobLogger = consoleJobLogger;
}
if (databaseJobLogger != null)
{
_databaseJobLogger = databaseJobLogger;
}
}
protected override void UpdateStatus(IJobLogger logger, string status)
{
logger.ReportStatus(new ContinuousJobStatus() { Status = status });
}
private void CacheJobBinaries(JobBase job, IJobLogger logger)
{
bool isInPlaceDefault = job.ScriptHost.GetType() == typeof(NodeScriptHost);
if (JobSettings.GetIsInPlace(isInPlaceDefault))
{
_inPlaceWorkingDirectory = JobBinariesPath;
SafeKillAllRunningJobInstances(logger);
UpdateAppConfigs(WorkingDirectory);
return;
}
_inPlaceWorkingDirectory = null;
if (WorkingDirectory != null)
{
try
{
int currentHash = CalculateHashForJob(JobBinariesPath);
int lastHash = CalculateHashForJob(WorkingDirectory);
if (lastHash == currentHash)
{
return;
}
}
catch (Exception ex)
{
// Log error and ignore it as it's not critical to cache job binaries
logger.LogWarning("Failed to calculate hash for WebJob, continue to copy WebJob binaries (this will not affect WebJob run)\n" + ex);
_analytics.UnexpectedException(ex);
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true);
}
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
OperationManager.Attempt(() =>
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath);
_workingDirectory = tempJobInstancePath;
});
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
_analytics.UnexpectedException(ex);
// job disabled
_workingDirectory = null;
}
}
protected abstract void UpdateStatus(IJobLogger logger, string status);
public void SafeKillAllRunningJobInstances(IJobLogger logger)
{
try
{
Process[] processes = Process.GetProcesses();
foreach (Process process in processes)
{
Dictionary<string, string> processEnvironment;
bool success = process.TryGetEnvironmentVariables(out processEnvironment);
if (success && processEnvironment.ContainsKey(GetJobEnvironmentKey()))
{
try
{
process.Kill(true, TraceFactory.GetTracer());
}
catch (Exception ex)
{
if (!process.HasExited)
{
logger.LogWarning("Failed to kill process - {0} for job - {1}\n{2}".FormatInvariant(process.ProcessName, JobName, ex));
}
}
}
}
}
catch (Exception ex)
{
logger.LogWarning(ex.ToString());
}
}
protected void RunJobInstance(JobBase job, IJobLogger logger, string runId, string trigger, int port = -1)
{
string scriptFileName = Path.GetFileName(job.ScriptFilePath);
string scriptFileFullPath = Path.Combine(WorkingDirectory, job.RunCommand);
string workingDirectoryForScript = Path.GetDirectoryName(scriptFileFullPath);
logger.LogInformation("Run script '{0}' with script host - '{1}'".FormatCurrentCulture(scriptFileName, job.ScriptHost.GetType().Name));
using (var jobStartedReporter = new JobStartedReporter(_analytics, job, trigger, Settings.GetWebSiteSku(), JobDataPath))
{
try
{
var exe = _externalCommandFactory.BuildCommandExecutable(job.ScriptHost.HostPath, workingDirectoryForScript, IdleTimeout, NullLogger.Instance);
_shutdownNotificationFilePath = RefreshShutdownNotificationFilePath(job.Name, job.JobType);
// Set environment variable to be able to identify all processes spawned for this job
exe.EnvironmentVariables[GetJobEnvironmentKey()] = "true";
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRootPath] = WorkingDirectory;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsName] = job.Name;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsType] = job.JobType;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsDataPath] = JobDataPath;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRunId] = runId;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsCommandArguments] = job.CommandArguments;
if (port != -1)
{
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsPort] = port.ToString();
}
if (_shutdownNotificationFilePath != null)
{
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsShutdownNotificationFile] = _shutdownNotificationFilePath;
}
UpdateStatus(logger, "Running");
int exitCode =
exe.ExecuteReturnExitCode(
TraceFactory.GetTracer(),
logger.LogStandardOutput,
logger.LogStandardError,
job.ScriptHost.ArgumentsFormat,
scriptFileName,
job.CommandArguments != null ? " " + job.CommandArguments : String.Empty);
if (exitCode != 0)
{
string errorMessage = "Job failed due to exit code " + exitCode;
logger.LogError(errorMessage);
jobStartedReporter.Error = errorMessage;
}
else
{
UpdateStatus(logger, "Success");
}
}
catch (ThreadAbortException)
{
// We kill the process when refreshing the job
logger.LogInformation("WebJob process was aborted");
UpdateStatus(logger, "Stopped");
}
catch (Exception ex)
{
logger.LogError(ex.ToString());
jobStartedReporter.Error = ex.Message;
}
}
}
protected void InitializeJobInstance(JobBase job, IJobLogger logger)
{
if (!String.Equals(JobName, job.Name, StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException(
"The job runner can only run jobs with the same name it was configured, configured - {0}, trying to run - {1}".FormatInvariant(
JobName, job.Name));
}
if (!FileSystemHelpers.FileExists(job.ScriptFilePath))
{
throw new InvalidOperationException("Missing job script to run - {0}".FormatInvariant(job.ScriptFilePath));
}
CacheJobBinaries(job, logger);
if (WorkingDirectory == null)
{
throw new InvalidOperationException("Missing working directory");
}
}
internal void SetJobLogger(IJobLogger jobLogger)
{
this.jobLoggers.Add(jobLogger);
}
protected override void UpdateStatus(IJobLogger logger, string status)
{
((TriggeredJobRunLogger)logger).ReportStatus(status);
}
public JobLoggerFactory(IJobLogger consoleLogger, IJobLogger databaseLogger, IJobLogger fileLogger)
{
this.loggersDic.Add(EnumJobLoggerType.CONSOLE, consoleLogger);
this.loggersDic.Add(EnumJobLoggerType.DATABASE, databaseLogger);
this.loggersDic.Add(EnumJobLoggerType.FILE, fileLogger);
}
private void CacheJobBinaries(JobBase job, IJobLogger logger)
{
bool isInPlaceDefault = job.ScriptHost.GetType() == typeof(NodeScriptHost);
if (JobSettings.GetIsInPlace(isInPlaceDefault))
{
_inPlaceWorkingDirectory = JobBinariesPath;
SafeKillAllRunningJobInstances(logger);
UpdateAppConfigs(WorkingDirectory, _analytics);
return;
}
_inPlaceWorkingDirectory = null;
Dictionary<string, FileInfoBase> sourceDirectoryFileMap = GetJobDirectoryFileMap(JobBinariesPath);
if (WorkingDirectory != null)
{
try
{
var workingDirectoryFileMap = GetJobDirectoryFileMap(WorkingDirectory);
if (!JobDirectoryHasChanged(sourceDirectoryFileMap, workingDirectoryFileMap, _cachedSourceDirectoryFileMap, logger))
{
// no changes detected, so skip the cache/copy step below
return;
}
}
catch (Exception ex)
{
// Log error and ignore it, since this diff optimization isn't critical.
// We'll just do a full copy in this case.
logger.LogWarning("Failed to diff WebJob directories for changes. Continuing to copy WebJob binaries (this will not affect the WebJob run)\n" + ex);
_analytics.UnexpectedException(ex);
}
}
SafeKillAllRunningJobInstances(logger);
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
FileSystemHelpers.DeleteDirectorySafe(JobTempPath, ignoreErrors: true);
}
if (FileSystemHelpers.DirectoryExists(JobTempPath))
{
logger.LogWarning("Failed to delete temporary directory");
}
try
{
OperationManager.Attempt(() =>
{
var tempJobInstancePath = Path.Combine(JobTempPath, Path.GetRandomFileName());
FileSystemHelpers.CopyDirectoryRecursive(JobBinariesPath, tempJobInstancePath);
UpdateAppConfigs(tempJobInstancePath, _analytics);
_workingDirectory = tempJobInstancePath;
// cache the file map snapshot for next time (to aid in detecting
// file deletions)
_cachedSourceDirectoryFileMap = sourceDirectoryFileMap;
});
}
catch (Exception ex)
{
//Status = "Worker is not running due to an error";
//TraceError("Failed to copy bin directory: " + ex);
logger.LogError("Failed to copy job files: " + ex);
_analytics.UnexpectedException(ex);
// job disabled
_workingDirectory = null;
}
}
protected void RunJobInstance(JobBase job, IJobLogger logger, string runId)
{
string scriptFileName = Path.GetFileName(job.ScriptFilePath);
string scriptFileExtension = Path.GetExtension(job.ScriptFilePath);
logger.LogInformation("Run script '{0}' with script host - '{1}'".FormatCurrentCulture(scriptFileName, job.ScriptHost.GetType().Name));
string siteMode = Settings.GetWebSitePolicy();
_analytics.JobStarted(job.Name.Fuzz(), scriptFileExtension, job.JobType, siteMode);
try
{
var exe = _externalCommandFactory.BuildCommandExecutable(job.ScriptHost.HostPath, WorkingDirectory, IdleTimeout, NullLogger.Instance);
// Set environment variable to be able to identify all processes spawned for this job
exe.EnvironmentVariables[GetJobEnvironmentKey()] = "true";
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRootPath] = WorkingDirectory;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsName] = job.Name;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsType] = job.JobType;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsDataPath] = JobDataPath;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsRunId] = runId;
exe.EnvironmentVariables[WellKnownEnvironmentVariables.WebJobsExtraUrlPath] = JobsManagerBase.GetJobExtraInfoUrlFilePath(JobDataPath);
UpdateStatus(logger, "Running");
int exitCode =
exe.ExecuteReturnExitCode(
TraceFactory.GetTracer(),
logger.LogStandardOutput,
logger.LogStandardError,
job.ScriptHost.ArgumentsFormat,
job.RunCommand);
if (exitCode != 0)
{
logger.LogError("Job failed due to exit code " + exitCode);
}
else
{
UpdateStatus(logger, "Success");
}
}
catch (Exception ex)
{
if (ex is ThreadAbortException)
{
// We kill the process when refreshing the job
logger.LogInformation("Job aborted");
UpdateStatus(logger, "Aborted");
return;
}
logger.LogError(ex.ToString());
}
}
