/// <summary> /// Adds a task to the execution list. /// </summary> /// <param name="task">The task.</param> public static void AddTask(TaskWrapper task) { if (_taskQueue == null) { throw new Exception("TaskManager failed to initialize."); } task.NextAttempt = DateTime.Now; lock (_taskLock) { _tasks.Add(task); _taskQueue.Enqueue(task); } _stats.ScheduledTasks.Increment(); _stats.Tasks.Increment(); EnsureExecution(); }
/// <summary> /// Terminates the task. /// </summary> /// <param name="task">The task.</param> /// <returns>True whether the task was being executed at the time, false otherwise.</returns> private static bool AbortTaskThread(TaskWrapper task) { bool found = false; lock (_threadsLock) { for (int i = _threads.Count - 1; i >= 0; i--) { TaskThread tt = _threads[i]; if (tt.CurrentTask == task) { RemoveTaskThread(tt); _stats.Tasks.Decrement(); found = true; } } SpawnTaskThread(); } return(found); }
/// <summary> /// Removes a task from the execution list. /// </summary> /// <param name="task">The task.</param> /// <returns>True whether the task was queued for execution, false otherwise.</returns> private static bool RemoveTaskFromQueue(TaskWrapper task) { bool found = false; lock (_taskLock) { int before, spawnCount; before = _taskQueue.Count; lock (_threadsLock) { spawnCount = _taskQueue.Count(t => t == task) + _threads.Count(t => t.CurrentTask == task); } for (int index = 0, total = _taskQueue.Count; index < total; index++) { TaskWrapper front = _taskQueue.Dequeue(); if (front != task) { _taskQueue.Enqueue(front); } else { _stats.ScheduledTasks.Decrement(); _stats.Tasks.Decrement(); found = true; } } Debug.Assert(_taskQueue.Count == before - spawnCount, "Lost track of tasks."); } return(found); }
/// <summary> /// Reschedules a task for its next execution time. /// </summary> /// <param name="task">The task.</param> /// <param name="hasMoreWork">Boolean indicating whether the previous execution identified more work available immediately. If true, and the task is configured to allow execution bursts, it might be scheduled for execution immediately.</param> /// <remarks>Burst is enabled by allowing more than 1 MaxRuns in a TimeUnit.</remarks> private static void RescheduleTask(TaskWrapper task, bool hasMoreWork) { ScheduleData currentSchedule = task.ConfigurationData.GetScheduleFor(DateTime.Now); DateTime nextAttempt = DateTime.MinValue; task.BurstCounter++; int currentBurstCount = task.BurstCounter; if (hasMoreWork) { if (currentBurstCount < currentSchedule.MaxRuns) { nextAttempt = DateTime.Now; } else { task.BurstCounter = 0; nextAttempt = task.BurstStart.Add(currentSchedule.TimeUnit.Value); if (nextAttempt < DateTime.Now) { nextAttempt = DateTime.Now; } } } else { task.BurstCounter = 0; nextAttempt = DateTime.Now.Add(currentSchedule.Wait.Value); } task.NextAttempt = nextAttempt; ScheduleData nextSchedule = task.ConfigurationData.GetScheduleFor(nextAttempt); lock (_taskLock) { int currentSpawnCount; lock (_threadsLock) { currentSpawnCount = _taskQueue.Count(t => t == task) + _threads.Count(t => t.CurrentTask == task); } bool extraSpawn = false; for (int index = currentSpawnCount, total = nextSchedule.Spawn.Value; index < total; index++) { _taskQueue.Enqueue(task); _stats.ScheduledTasks.Increment(); if (extraSpawn) { _stats.Tasks.Increment(); // TODO: missing .Decrement() when a task is not rescheduled. } extraSpawn = true; } Thread.MemoryBarrier(); } }
/// <summary> /// Loads a module from a zip file. /// </summary> /// <param name="zipFile">The path to the zip file.</param> /// <param name="startImmediately">True if execution should be started immediately after configuring the module, false if module should be configured but not started.</param> private static void LoadZipAndConfigure(string zipFile, bool startImmediately) { string tempPath = null; int i = 0; while (Directory.Exists(tempPath = Path.Combine(Path.Combine(Path.GetTempPath(), "TaskManager"), Path.GetRandomFileName()))) { i++; if (i == 10) { throw new Exception("Failed to create a new temporary folder."); } } if (!tempPath.EndsWith(Path.DirectorySeparatorChar.ToString())) { tempPath += Path.DirectorySeparatorChar; } string overridePath = Path.Combine(Path.GetDirectoryName(zipFile), Path.GetFileNameWithoutExtension(zipFile)) + Path.DirectorySeparatorChar; string loaderDll = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TaskManager.Loader.dll"); string tempLoaderDll = Path.Combine(tempPath, "TaskManager.Loader.dll"); string commonDll = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TaskManager.Common.dll"); string tempCommonDll = Path.Combine(tempPath, "TaskManager.Common.dll"); List <string> dependencies = new List <string>(); DirectoryInfo directoryInfo = Directory.CreateDirectory(tempPath); try { using (Stream fileStream = File.Open(zipFile, FileMode.Open, FileAccess.Read)) { using (ZipArchive zip = new ZipArchive(fileStream, ZipArchiveMode.Read)) { var directory = zip.Entries; foreach (var compressedFile in directory) { string destinationFile = Path.Combine(tempPath, compressedFile.FullName); string overrideFile = Path.Combine(overridePath, compressedFile.FullName); dependencies.Add(overrideFile); compressedFile.ExtractToFile(destinationFile, true); } } } if (Directory.Exists(overridePath)) { foreach (string overrideFile in Directory.GetFiles(overridePath, "*.*", SearchOption.AllDirectories)) { if (!dependencies.Contains(overrideFile)) { dependencies.Add(overrideFile); } dependencies.Add(Path.Combine(overridePath, overrideFile.Replace(tempPath, string.Empty))); string relativeName = overrideFile.Replace(overridePath, string.Empty); string destination = Path.Combine(tempPath, relativeName); string destinationPath = Path.GetDirectoryName(destination); if (!Directory.Exists(destinationPath)) { Directory.CreateDirectory(destinationPath); } File.Copy(overrideFile, destination, true); } } List <string> possibleFiles = new List <string>(); foreach (string moduleFile in Directory.GetFiles(tempPath, "*" + ModuleFileDefaultExtension, SearchOption.AllDirectories)) { string xmlFile = Path.ChangeExtension(moduleFile, ConfigFileDefaultExtension); if (File.Exists(xmlFile)) { possibleFiles.Add(moduleFile); } } File.Copy(loaderDll, tempLoaderDll, true); File.Copy(commonDll, tempCommonDll, true); foreach (string dllFile in possibleFiles) { string xmlFile = Path.ChangeExtension(dllFile, ConfigFileDefaultExtension); if (File.Exists(xmlFile)) { if (!xmlFile.IsValidConfigurationFile()) { continue; } string modulePath = Path.GetDirectoryName(dllFile); string[] files = Directory.GetFiles(modulePath, "*.*", SearchOption.AllDirectories); AppDomainSetup domainSetup = new AppDomainSetup(); domainSetup.ShadowCopyFiles = "true"; domainSetup.ApplicationBase = tempPath; domainSetup.ConfigurationFile = dllFile + ".config"; AppDomain domain = AppDomain.CreateDomain(dllFile, null, domainSetup); TaskWrapper[] tasks = new TaskWrapper[0]; try { TaskManagerService.Logger.Log(string.Format("Module found: '{0}', configuration file: '{1}', scanning assembly for tasks...", dllFile, xmlFile)); AssemblyName loaderAssemblyName = AssemblyName.GetAssemblyName(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TaskManager.Loader.dll")); Loader loader = (Loader)domain.CreateInstanceAndUnwrap(loaderAssemblyName.ToString(), "TaskManager.Loader"); tasks = loader.LoadAndConfigure(dllFile, xmlFile, TaskManagerService.Logger, AppDomain.CurrentDomain.BaseDirectory); ModuleData newModule = new ModuleData(); newModule.BasePath = overridePath; newModule.Tasks = tasks; newModule.DllFile = dllFile; newModule.XmlFile = xmlFile; newModule.ZipFile = zipFile; newModule.ZipDirectory = tempPath; newModule.Files = new List <string>(files); newModule.Files.AddRange(dependencies); newModule.Files.Add(zipFile); newModule.Domain = domain; if (startImmediately) { foreach (TaskWrapper task in newModule.Tasks) { TaskSupervisor.ScheduleTask(task); } } _moduleList.Add(newModule); } catch (Exception ex) { foreach (TaskWrapper task in tasks) { try { TaskSupervisor.RemoveTask(task); } catch { } } AppDomain.Unload(domain); TaskManagerService.Logger.Log(string.Format("Unable to load module '{0}' from zipped file '{1}'.", dllFile, zipFile), ex); } } } foreach (ModuleData module in _moduleList) { if (module.ZipFile == zipFile) { return; } } throw new Exception(string.Format("Unable to find tasks in zipped file '{0}'.", zipFile)); } catch { try { Directory.Delete(tempPath, true); } catch (Exception ex) { TaskManagerService.Logger.Log(string.Format("Unable to remove temporary directory '{0}'.", tempPath), ex); } throw; } }
/// <summary> /// The thread loop. /// </summary> private void Loop() { DateTime lastRun = DateTime.Now; while (true) { lock (TaskSupervisor._taskLock) { this._task = TaskSupervisor.GetScheduledTask(this); } System.Threading.Thread.MemoryBarrier(); if (null != this._task) { bool result = false; long runTime = -1; TaskSupervisor.NotifyStart(this._task); this.StartingTime = lastRun = DateTime.Now; this.IsRunning = true; System.Threading.Thread.MemoryBarrier(); Stopwatch timer = Stopwatch.StartNew(); try { result = this._task.Execute(); runTime = timer.ElapsedTicks; } catch (ThreadAbortException) { this._task = null; throw; } catch (AppDomainUnloadedException) { this._task = null; throw; } catch (Exception ex) { this.IsRunning = false; if (!TaskSupervisor.NotifyException("Exception caught while executing task.", ex)) { this._task = null; System.Threading.Thread.MemoryBarrier(); throw; } } finally { lastRun = DateTime.Now; this.IsRunning = false; this.StartingTime = DateTime.MinValue; System.Threading.Thread.MemoryBarrier(); TaskSupervisor.NotifyEnd(this._task, runTime); } lock (TaskSupervisor._taskLock) { try { TaskWrapper toReturn = this._task; this._task = null; TaskSupervisor.RescheduleTask(toReturn, result); } catch (AppDomainUnloadedException) { } this._task = null; } System.Threading.Thread.MemoryBarrier(); Thread.Sleep(100); } else { if (lastRun.AddSeconds(IdleTimeout) <= DateTime.Now) { TaskSupervisor.NotifyIdleThread(this); } Thread.Sleep(250); } } }
/// <summary> /// Terminates the task. /// </summary> /// <param name="task">The task.</param> /// <returns>True whether the task was being executed at the time, false otherwise.</returns> private static bool AbortTaskThread(TaskWrapper task) { bool found = false; lock (_threadsLock) { for (int i = _threads.Count - 1; i >= 0; i--) { TaskThread tt = _threads[i]; if (tt.CurrentTask == task) { RemoveTaskThread(tt); _stats.Tasks.Decrement(); found = true; } } SpawnTaskThread(); } return found; }
/// <summary> /// Removes a task from the execution list. /// </summary> /// <param name="task">The task.</param> /// <returns>True whether the task was queued for execution, false otherwise.</returns> private static bool RemoveTaskFromQueue(TaskWrapper task) { bool found = false; lock (_taskLock) { int before, spawnCount; before = _taskQueue.Count; lock (_threadsLock) { spawnCount = _taskQueue.Count(t => t == task) + _threads.Count(t => t.CurrentTask == task); } for (int index = 0, total = _taskQueue.Count; index < total; index++) { TaskWrapper front = _taskQueue.Dequeue(); if (front != task) { _taskQueue.Enqueue(front); } else { _stats.ScheduledTasks.Decrement(); _stats.Tasks.Decrement(); found = true; } } Debug.Assert(_taskQueue.Count == before - spawnCount, "Lost track of tasks."); } return found; }
/// <summary> /// Terminates a task and removes it from the execution list. /// </summary> /// <param name="task">The task.</param> public static void RemoveTask(TaskWrapper task) { if (_taskQueue == null) { throw new Exception("TaskManager failed to initialize."); } try { lock (_taskLock) { lock (_threadsLock) { AbortTaskThread(task); if (_taskQueue.Count > 0 && _taskQueue.Contains(task)) { RemoveTaskFromQueue(task); } } } } catch (Exception ex) { TaskManagerService.Logger.Log("Exception caught while removing task.", ex); } finally { _tasks.Remove(task); } }
/// <summary> /// Prepares and schedules a task for execution. /// </summary> /// <param name="task">The task.</param> public static void ScheduleTask(TaskWrapper task) { if (_taskQueue == null) { throw new Exception("TaskManager failed to initialize."); } var nextAttempt = task.NextAttempt = DateTime.Now + (task.ConfigurationData.DelayStart ?? ConfigurationHelpers.DefaultDelayStart); ScheduleData current = task.ConfigurationData.GetScheduleFor(task.NextAttempt); lock (_taskLock) { _tasks.Add(task); for (int index = 0, total = current.Spawn.Value; index < total; index++) { _taskQueue.Enqueue(task); _stats.ScheduledTasks.Increment(); _stats.Tasks.Increment(); } } EnsureExecution(); }
/// <summary> /// The SLA Thread entry point. /// </summary> /// <remarks> /// The SLA Thread ensures that all tasks ready to be executed will start ASAP by spawning more execution threads, if possible, /// and that tasks whose execution time exceeds the configured execution timeout will be aborted. /// </remarks> private static void SLALoopWorker() { while (true) { bool spawnNewThread = false; bool hasThreadAbort = false; lock (_taskLock) { lock (_threadsLock) { if (_taskQueue.Count > 0) { for (int index = 0, total = _taskQueue.Count; index < total; index++) { TaskWrapper task = _taskQueue.Dequeue(); #region Figure out if we need to spawn a new thread, if this task is late. try { DateTime nextAttemptDate = task.NextAttempt; ScheduleData nextSchedule = task.ConfigurationData.GetScheduleFor(nextAttemptDate); if (nextAttemptDate.AddSeconds(nextSchedule.SLA.Value.TotalSeconds) < DateTime.Now) { spawnNewThread = true; } _taskQueue.Enqueue(task); } catch (AppDomainUnloadedException) { _stats.ScheduledTasks.Decrement(); Thread.MemoryBarrier(); } #endregion } } if (_tasks.Count > 0) { for (int index = 0, total = _tasks.Count; index < total; index++) { TaskWrapper task = _tasks[index]; #region Figure out if we need to spawn task clones, if the current schedule requires more spawned clones than there are clones currently running try { ScheduleData currentSchedule = task.ConfigurationData.GetScheduleFor(DateTime.Now); int currentSpawnCount = _taskQueue.Count(t => t == task) + _threads.Count(t => t.CurrentTask == task); for (int spawnIndex = currentSpawnCount, spawnTotal = currentSchedule.Spawn.Value; spawnIndex < spawnTotal; spawnIndex++) { _taskQueue.Enqueue(task); _stats.ScheduledTasks.Increment(); _stats.Tasks.Increment(); } } catch (AppDomainUnloadedException) { // The AppDomain is gone. // May happens during module shutdown (files were updated) or service shutdown; that's ok. // The spawn a new clone check will deal with it. } #endregion } } for (int i = _threads.Count - 1; i >= 0; i--) { TaskThread thread = _threads[i]; try { if (thread.IsRunning && thread.StartingTime != DateTime.MinValue && null != thread.CurrentTask && thread.CurrentTask.HasTimedOut() && _threads.Count > 1) { TaskWrapper currentTask = thread.CurrentTask; if (null != currentTask) { RemoveTaskThread(thread); SpawnTaskThread(); _stats.TotalTimeouts.Increment(); _stats.TimeoutsPerSecond.Increment(); Thread.MemoryBarrier(); RescheduleTask(currentTask, false); hasThreadAbort = true; } } } catch (AppDomainUnloadedException) { // The AppDomain is gone. // May happens during module shutdown (files were updated) or service shutdown; that's ok. // The spawn a new clone check will deal with it. } } } } if (spawnNewThread && !hasThreadAbort) { SpawnTaskThread(); } Thread.Sleep(1000); } }
/// <summary> /// Notifies the supervisor that a task has started executing. /// </summary> /// <param name="task">The task.</param> private static void NotifyStart(TaskWrapper task) { _stats.TasksRunning.Increment(); }
/// <summary> /// Notifies the supervisor that a task has finished executing. /// </summary> /// <param name="task">The task.</param> /// <param name="ticks">How long the task took to run, in system ticks.</param> private static void NotifyEnd(TaskWrapper task, long ticks) { if (ticks != -1) { _stats.AverageExecutionTime.IncrementBy(ticks); _stats.BaseAverageExecutionTime.Increment(); } _stats.TasksRunning.Decrement(); }
/// <summary> /// Loads a module from a zip file. /// </summary> /// <param name="zipFile">The path to the zip file.</param> /// <param name="startImmediately">True if execution should be started immediately after configuring the module, false if module should be configured but not started.</param> private static void LoadZipAndConfigure(string zipFile, bool startImmediately) { string tempPath = null; int i = 0; while (Directory.Exists(tempPath = Path.Combine(Path.Combine(Path.GetTempPath(), "TaskManager"), Path.GetRandomFileName()))) { i++; if (i == 10) throw new Exception("Failed to create a new temporary folder."); } if (!tempPath.EndsWith(Path.DirectorySeparatorChar.ToString())) { tempPath += Path.DirectorySeparatorChar; } string overridePath = Path.Combine(Path.GetDirectoryName(zipFile), Path.GetFileNameWithoutExtension(zipFile)) + Path.DirectorySeparatorChar; string loaderDll = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TaskManager.Loader.dll"); string tempLoaderDll = Path.Combine(tempPath, "TaskManager.Loader.dll"); string commonDll = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TaskManager.Common.dll"); string tempCommonDll = Path.Combine(tempPath, "TaskManager.Common.dll"); List<string> dependencies = new List<string>(); DirectoryInfo directoryInfo = Directory.CreateDirectory(tempPath); try { using (Stream fileStream = File.Open(zipFile, FileMode.Open, FileAccess.Read)) { using (ZipArchive zip = new ZipArchive(fileStream, ZipArchiveMode.Read)) { var directory = zip.Entries; foreach (var compressedFile in directory) { string destinationFile = Path.Combine(tempPath, compressedFile.FullName); string overrideFile = Path.Combine(overridePath, compressedFile.FullName); dependencies.Add(overrideFile); compressedFile.ExtractToFile(destinationFile, true); } } } if (Directory.Exists(overridePath)) { foreach (string overrideFile in Directory.GetFiles(overridePath, "*.*", SearchOption.AllDirectories)) { if (!dependencies.Contains(overrideFile)) { dependencies.Add(overrideFile); } dependencies.Add(Path.Combine(overridePath, overrideFile.Replace(tempPath, string.Empty))); string relativeName = overrideFile.Replace(overridePath, string.Empty); string destination = Path.Combine(tempPath, relativeName); string destinationPath = Path.GetDirectoryName(destination); if (!Directory.Exists(destinationPath)) { Directory.CreateDirectory(destinationPath); } File.Copy(overrideFile, destination, true); } } List<string> possibleFiles = new List<string>(); foreach (string moduleFile in Directory.GetFiles(tempPath, "*" + ModuleFileDefaultExtension, SearchOption.AllDirectories)) { string xmlFile = Path.ChangeExtension(moduleFile, ConfigFileDefaultExtension); if (File.Exists(xmlFile)) { possibleFiles.Add(moduleFile); } } File.Copy(loaderDll, tempLoaderDll, true); File.Copy(commonDll, tempCommonDll, true); foreach (string dllFile in possibleFiles) { string xmlFile = Path.ChangeExtension(dllFile, ConfigFileDefaultExtension); if (File.Exists(xmlFile)) { if (!xmlFile.IsValidConfigurationFile()) { continue; } string modulePath = Path.GetDirectoryName(dllFile); string[] files = Directory.GetFiles(modulePath, "*.*", SearchOption.AllDirectories); AppDomainSetup domainSetup = new AppDomainSetup(); domainSetup.ShadowCopyFiles = "true"; domainSetup.ApplicationBase = tempPath; domainSetup.ConfigurationFile = dllFile + ".config"; AppDomain domain = AppDomain.CreateDomain(dllFile, null, domainSetup); TaskWrapper[] tasks = new TaskWrapper[0]; try { TaskManagerService.Logger.Log(string.Format("Module found: '{0}', configuration file: '{1}', scanning assembly for tasks...", dllFile, xmlFile)); AssemblyName loaderAssemblyName = AssemblyName.GetAssemblyName(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "TaskManager.Loader.dll")); Loader loader = (Loader)domain.CreateInstanceAndUnwrap(loaderAssemblyName.ToString(), "TaskManager.Loader"); tasks = loader.LoadAndConfigure(dllFile, xmlFile, TaskManagerService.Logger, AppDomain.CurrentDomain.BaseDirectory); ModuleData newModule = new ModuleData(); newModule.BasePath = overridePath; newModule.Tasks = tasks; newModule.DllFile = dllFile; newModule.XmlFile = xmlFile; newModule.ZipFile = zipFile; newModule.ZipDirectory = tempPath; newModule.Files = new List<string>(files); newModule.Files.AddRange(dependencies); newModule.Files.Add(zipFile); newModule.Domain = domain; if (startImmediately) { foreach (TaskWrapper task in newModule.Tasks) { TaskSupervisor.ScheduleTask(task); } } _moduleList.Add(newModule); } catch (Exception ex) { foreach (TaskWrapper task in tasks) { try { TaskSupervisor.RemoveTask(task); } catch { } } AppDomain.Unload(domain); TaskManagerService.Logger.Log(string.Format("Unable to load module '{0}' from zipped file '{1}'.", dllFile, zipFile), ex); } } } foreach (ModuleData module in _moduleList) { if (module.ZipFile == zipFile) { return; } } throw new Exception(string.Format("Unable to find tasks in zipped file '{0}'.", zipFile)); } catch { try { Directory.Delete(tempPath, true); } catch (Exception ex) { TaskManagerService.Logger.Log(string.Format("Unable to remove temporary directory '{0}'.", tempPath), ex); } throw; } }
/// <summary> /// Notifies the supervisor that a task has started executing. /// </summary> /// <param name="task">The task.</param> private static void NotifyStart(TaskWrapper task) { _perfCounterTasksRunning.Increment(); }