protected virtual void CompleteFileInfoForType(FileSpecification fileInfo, TaskSpecification task, SynchronizableFiles fileType) { switch (fileType) { case SynchronizableFiles.StandardOutputFile: fileInfo.RelativePath = task.StandardOutputFile; fileInfo.NameSpecification = FileNameSpecification.FullName; fileInfo.SynchronizationType = FileSynchronizationType.IncrementalAppend; break; case SynchronizableFiles.StandardErrorFile: fileInfo.RelativePath = task.StandardErrorFile; fileInfo.NameSpecification = FileNameSpecification.FullName; fileInfo.SynchronizationType = FileSynchronizationType.IncrementalAppend; break; case SynchronizableFiles.LogFile: fileInfo.RelativePath = task.LogFile.RelativePath; fileInfo.NameSpecification = task.LogFile.NameSpecification; fileInfo.SynchronizationType = task.LogFile.SynchronizationType; break; case SynchronizableFiles.ProgressFile: fileInfo.RelativePath = task.ProgressFile.RelativePath; fileInfo.NameSpecification = task.ProgressFile.NameSpecification; fileInfo.SynchronizationType = task.ProgressFile.SynchronizationType; break; } }
private static TaskSpecification ConvertExtToInt(this TaskSpecificationExt taskSpecificationExt, JobSpecification jobSpecification) { var result = new TaskSpecification { Name = taskSpecificationExt.Name, MinCores = taskSpecificationExt.MinCores, MaxCores = taskSpecificationExt.MaxCores, WalltimeLimit = taskSpecificationExt.WalltimeLimit, PlacementPolicy = taskSpecificationExt.PlacementPolicy, RequiredNodes = taskSpecificationExt.RequiredNodes? .Select(s => new TaskSpecificationRequiredNode { NodeName = s }) .ToList(), Priority = taskSpecificationExt.Priority.ConvertExtToInt(), Project = jobSpecification.Project, JobArrays = taskSpecificationExt.JobArrays, IsExclusive = taskSpecificationExt.IsExclusive ?? false, IsRerunnable = !string.IsNullOrEmpty(taskSpecificationExt.JobArrays) || (taskSpecificationExt.IsRerunnable ?? false), StandardInputFile = taskSpecificationExt.StandardInputFile, StandardOutputFile = taskSpecificationExt.StandardOutputFile ?? "stdout.txt", StandardErrorFile = taskSpecificationExt.StandardErrorFile ?? "stderr.txt", ClusterTaskSubdirectory = taskSpecificationExt.ClusterTaskSubdirectory, ProgressFile = new FileSpecification { RelativePath = taskSpecificationExt.ProgressFile, NameSpecification = FileNameSpecification.FullName, SynchronizationType = FileSynchronizationType.IncrementalAppend }, LogFile = new FileSpecification { RelativePath = taskSpecificationExt.LogFile, NameSpecification = FileNameSpecification.FullName, SynchronizationType = FileSynchronizationType.IncrementalAppend }, ClusterNodeTypeId = taskSpecificationExt.ClusterNodeTypeId.Value, CommandTemplateId = taskSpecificationExt.CommandTemplateId ?? 0, EnvironmentVariables = taskSpecificationExt.EnvironmentVariables? .Select(s => s.ConvertExtToInt()) .ToList(), CpuHyperThreading = taskSpecificationExt.CpuHyperThreading, JobSpecification = jobSpecification, TaskParalizationSpecifications = taskSpecificationExt.TaskParalizationParameters? .Select(s => s.ConvertExtToInt()) .ToList(), CommandParameterValues = taskSpecificationExt.TemplateParameterValues? .Select(s => s.ConvertExtToInt()) .ToList(), }; result.DependsOn = taskSpecificationExt.DependsOn? .Select(s => new TaskDependency { TaskSpecification = result, ParentTaskSpecification = s.ConvertExtToInt(jobSpecification) }) .ToList(); return(result); }
protected virtual FullFileSpecification CreateSynchronizableFileInfoForType(TaskSpecification task, string taskClusterDirectoryPath, SynchronizableFiles fileType) { FullFileSpecification fileInfo = new FullFileSpecification { DestinationDirectory = task.LocalDirectory, SourceDirectory = taskClusterDirectoryPath, }; CompleteFileInfoForType(fileInfo, task, fileType); return(fileInfo); }
public void FilterUserOwned_Positive() { var collection = new List <TaskEntity> { new() { UserId = "userid1" }, new() { UserId = "userid2" } }; var service = new TaskSpecification().FilterUserOwned("userid1"); var result = service.Apply(collection.AsQueryable()).ToList(); Assert.Collection(result, _ => { }); }
public static TaskInfo Create(TaskSpecification taskSpecification, User user) { var taskInfo = new TaskInfo { TaskSpecificationId = taskSpecification.Id, TaskSpecification = taskSpecification }; if (user != null) { taskInfo.SetAuditInfo(user.Login); } return(taskInfo); }
/// <summary> /// Create task directory sym link /// </summary> /// <param name="taskSpecification">Task specification</param> /// <returns></returns> protected static string CreateTaskDirectorySymlinkCommand(TaskSpecification taskSpecification) { string symlinkCommand = string.Empty; if (taskSpecification.DependsOn.Any()) { long dependsOnIdLast = taskSpecification.DependsOn.Max(x => x.ParentTaskSpecificationId); return(taskSpecification.DependsOn.FirstOrDefault(x => x.ParentTaskSpecificationId == dependsOnIdLast) == null ? symlinkCommand : symlinkCommand = $"ln -s ../{dependsOnIdLast}/* ."); } return(symlinkCommand); }
public void FilterForeignUserOwned_Positive() { var collection = new List <TaskEntity> { new() { UserId = "userid1", Uid = "uid1" }, new() { UserId = "userid2", Uid = "uid2" } }; var service = new TaskSpecification().FilterForeignUserOwned("userid1", new[] { "uid1", "uid2" }); var result = service.Apply(collection.AsQueryable()).ToList(); Assert.Collection(result, x => { Assert.Equal("uid2", x.Uid); }); } }
private void ValidateWallTimeLimit(TaskSpecification task) { var clusterNodeType = LogicFactory.GetLogicFactory().CreateClusterInformationLogic(_unitOfWork) .GetClusterNodeTypeById(task.ClusterNodeTypeId); if (clusterNodeType == null) { _messageBuilder.AppendLine($"Requested ClusterNodeType with Id {task.ClusterNodeTypeId} does not exist in the system"); return; } if (task.WalltimeLimit.HasValue && task.WalltimeLimit.Value > clusterNodeType.MaxWalltime) { _messageBuilder.AppendLine( $"Defined task {task.Name} has set higher WalltimeLimit ({task.WalltimeLimit.Value}) than the maximum on this cluster node, " + $"maximal WallTimeLimit is {clusterNodeType.MaxWalltime}"); } }
private void ValidateTaskSpecification(TaskSpecification task) { if (task.Id != 0 && _unitOfWork.TaskSpecificationRepository.GetById(task.Id) == null) { _messageBuilder.AppendLine($"Task with Id {task.Id} does not exist in the system"); } ValidateWallTimeLimit(task); if (task.CommandTemplate == null) { _messageBuilder.AppendLine($"Command Template does not exist."); return; } foreach (CommandTemplateParameter parameter in task.CommandTemplate.TemplateParameters) { if (string.IsNullOrEmpty(parameter.Query) && (task.CommandParameterValues == null || !task.CommandParameterValues.Any(w => w.TemplateParameter == parameter))) { _messageBuilder.AppendLine($"Command Template parameter \"{parameter.Identifier}\" does not have a value."); } } if (task.ClusterNodeTypeId != task.CommandTemplate.ClusterNodeTypeId) { _messageBuilder.AppendLine($"Task {task.Name} has wrong CommandTemplate"); } if (!task.CommandTemplate.IsEnabled) { _messageBuilder.AppendLine($"Task {task.Name} has specified deleted CommandTemplateId \"{task.CommandTemplate.Id}\""); } if (task.CommandTemplate.IsGeneric) { ValidateGenericCommandTemplateSetup(task); } }
private void ValidateGenericCommandTemplateSetup(TaskSpecification task) { Dictionary <string, string> genericCommandParametres = new(); //Regex.Matches(task.CommandTemplate.CommandParameters, @"%%\{([\w\.]+)\}", RegexOptions.Compiled) foreach (var commandParameterValue in task.CommandParameterValues) { var key = commandParameterValue.CommandParameterIdentifier; string value = commandParameterValue.Value; if (!string.IsNullOrEmpty(value)) { genericCommandParametres.Add(key, value); } } Match scriptPathParameterName = Regex.Match(task.CommandTemplate.CommandParameters, @"%%\{([\w\.]+)\}", RegexOptions.Compiled); if (!scriptPathParameterName.Success) { _messageBuilder.AppendLine($"CommandTemplate is wrong"); } string clusterPathToUserScript = genericCommandParametres.FirstOrDefault(x => x.Key == scriptPathParameterName.Groups[1].Value).Value; if (string.IsNullOrWhiteSpace(clusterPathToUserScript)) { _messageBuilder.AppendLine($"User script path parameter, for generic command template, does not have a value."); } var scriptDefinedParametres = GetUserDefinedScriptParametres(task.ClusterNodeType.Cluster, clusterPathToUserScript); foreach (string parameter in scriptDefinedParametres) { if (!genericCommandParametres.Select(x => x.Value).Any(x => Regex.IsMatch(x, $"{parameter}=\\\\\".+\\\\\""))) { _messageBuilder.AppendLine($"Task specification does not contain '{parameter}' parameter."); } } }
private void ValidateTaskSpecificationInput(TaskSpecification task) { if (task.CommandTemplateId <= 0) { _messageBuilder.AppendLine("CommandTemplateId cannot be empty or <= 0."); } if (string.IsNullOrEmpty(task.Name)) { _messageBuilder.AppendLine("Task name cannot be empty."); } if (task.Name.Length > 50) { _messageBuilder.AppendLine($"Task name \"{task.Name}\" cannot be longer than 50 characters."); } if (ContainsIllegalCharacters(task.Name)) { _messageBuilder.AppendLine("Task name contains illegal characters."); } if (task.MinCores <= 0) { _messageBuilder.AppendLine($"Minimal number of cores for task \"{task.Name}\" has to be greater than 0."); } if (task.MaxCores <= 0) { _messageBuilder.AppendLine($"Maximal number of cores for task \"{task.Name}\" has to be greater than 0."); } if (task.MinCores > task.MaxCores) { _messageBuilder.AppendLine($"Minimal number of cores for task \"{task.Name}\" cannot be greater than maximal number of cores."); } if (task.WalltimeLimit <= 0) { _messageBuilder.AppendLine($"Walltime limit for task \"{task.Name}\" has to be greater than 0."); } if (task.JobArrays != null) { if (task.JobArrays == string.Empty) { task.JobArrays = null; } else { if (task.JobArrays.Length > 40) { _messageBuilder.AppendLine($"JobArrays specification for task \"{task.Name}\" cannot be longer than 40 characters."); } task.JobArrays = task.JobArrays.Replace(" ", string.Empty); var splittedArray = task.JobArrays.Split(':'); if (splittedArray.Length >= 1) { int step = 1; var interval = splittedArray[0].Split('-'); if ((splittedArray.Length == 1 || (splittedArray.Length == 2 && int.TryParse(splittedArray[1], out step))) && interval.Length == 2 && int.TryParse(interval[0], out int min) && int.TryParse(interval[1], out int max)) { if (!(min < max && min + step <= max)) { _messageBuilder.AppendLine($"JobArrays parameter for task \"{task.Name}\" has wrong filled minimum or maximum or step value."); } } else { _messageBuilder.AppendLine($"JobArrays parameter for task \"{task.Name}\" has wrong definition."); } } } } if (task.TaskParalizationSpecifications?.Sum(s => s.MaxCores) > task.MaxCores) { _messageBuilder.AppendLine($"TaskParalizationSpecifications count of maximal cores for task \"{task.Name}\" must be lower or equals to Maximal number of cores in task."); } if (!string.IsNullOrEmpty(task.PlacementPolicy)) { if (task.PlacementPolicy.Length > 40) { _messageBuilder.AppendLine($"Placement policy specification for task \"{task.Name}\" cannot be longer than 40 characters."); } if (ContainsIllegalCharacters(task.PlacementPolicy)) { _messageBuilder.AppendLine($"Placement policy specification for task \"{task.Name}\" contains illegal characters."); } } if (!string.IsNullOrEmpty(task.StandardInputFile)) { if (task.StandardInputFile.Length > 30) { _messageBuilder.AppendLine($"Standard input file for task \"{task.Name}\" cannot be longer than 30 characters."); } if (ContainsIllegalCharacters(task.StandardInputFile)) { _messageBuilder.AppendLine($"Standard input file for task \"{task.Name}\" contains illegal characters."); } } if (!string.IsNullOrEmpty(task.StandardOutputFile)) { if (task.StandardOutputFile.Length > 30) { _messageBuilder.AppendLine($"Standard output file for task \"{task.Name}\" cannot be longer than 30 characters."); } if (ContainsIllegalCharacters(task.StandardOutputFile)) { _messageBuilder.AppendLine($"Standard output file for task \"{task.Name}\" contains illegal characters."); } } if (!string.IsNullOrEmpty(task.StandardErrorFile)) { if (task.StandardErrorFile.Length > 30) { _messageBuilder.AppendLine($"Standard error file for task \"{task.Name}\" cannot be longer than 30 characters."); } if (ContainsIllegalCharacters(task.StandardErrorFile)) { _messageBuilder.AppendLine($"Standard error file for task \"{task.Name}\" contains illegal characters."); } } if (!string.IsNullOrEmpty(task.ProgressFile.RelativePath)) { if (task.ProgressFile.RelativePath.Length > 50) { _messageBuilder.AppendLine($"Progress file for task \"{task.Name}\" cannot be longer than 50 characters."); } if (ContainsIllegalCharactersForPath(task.ProgressFile.RelativePath)) { _messageBuilder.AppendLine($"Progress file for task \"{task.Name}\" contains illegal characters."); } } if (!string.IsNullOrEmpty(task.LogFile.RelativePath)) { if (task.LogFile.RelativePath.Length > 50) { _messageBuilder.AppendLine($"Log file for task \"{task.Name}\" cannot be longer than 50 characters."); } if (ContainsIllegalCharactersForPath(task.LogFile.RelativePath)) { _messageBuilder.AppendLine($"Log file for task \"{task.Name}\" contains illegal characters."); } } if (task.RequiredNodes != null) { foreach (TaskSpecificationRequiredNode requiredNode in task.RequiredNodes) { if (requiredNode.NodeName.Length > 40) { _messageBuilder.AppendLine($"Required node \"{requiredNode.NodeName}\" specification for task \"{task.Name}\" cannot be longer than 40 characters."); } if (ContainsIllegalCharacters(requiredNode.NodeName)) { _messageBuilder.AppendLine($"Required node \"{requiredNode.NodeName}\" specification for task \"{task.Name}\" contains illegal characters."); } } } if (task.EnvironmentVariables != null) { foreach (EnvironmentVariable variable in task.EnvironmentVariables) { if (string.IsNullOrEmpty(variable.Name)) { _messageBuilder.AppendLine($"Environment variable's name for task \"{task.Name}\" cannot be empty. ({variable.Name} = {variable.Value})"); } } } }
public static string GetTaskClusterDirectoryPath(string jobClusterDirectoryPath, TaskSpecification taskSpecification) { string taskSubdirectory = !string.IsNullOrEmpty(taskSpecification.ClusterTaskSubdirectory) ? $"{taskSpecification.Id}/{taskSpecification.ClusterTaskSubdirectory}" : $"{taskSpecification.Id}"; return(ConcatenatePaths(jobClusterDirectoryPath, taskSubdirectory)); }
/// <summary> /// Convert task specification to task /// </summary> /// <param name="jobSpecification">Job specification</param> /// <param name="taskSpecification">Task specification</param> /// <param name="schedulerAllocationCmd">Scheduler allocation cmd</param> /// <returns></returns> /// <exception cref="ApplicationException"></exception> public virtual object ConvertTaskSpecificationToTask(JobSpecification jobSpecification, TaskSpecification taskSpecification, object schedulerAllocationCmd) { ISchedulerTaskAdapter taskAdapter = _conversionAdapterFactory.CreateTaskAdapter(schedulerAllocationCmd); taskAdapter.DependsOn = taskSpecification.DependsOn; taskAdapter.SetEnvironmentVariablesToTask(taskSpecification.EnvironmentVariables); taskAdapter.IsExclusive = taskSpecification.IsExclusive; taskAdapter.SetRequestedResourceNumber(taskSpecification.ClusterNodeType.RequestedNodeGroups.Select(s => s.Name).ToList(), taskSpecification.RequiredNodes.Select(s => s.NodeName).ToList(), taskSpecification.PlacementPolicy, taskSpecification.TaskParalizationSpecifications, Convert.ToInt32(taskSpecification.MinCores), Convert.ToInt32(taskSpecification.MaxCores), taskSpecification.ClusterNodeType.CoresPerNode); // Do not change!!! Task name on the cluster is set as ID of the used task specification to enable pairing of cluster task info with DB task info. taskAdapter.Name = taskSpecification.Id.ToString(CultureInfo.InvariantCulture); if (Convert.ToInt32(taskSpecification.WalltimeLimit) > 0) { taskAdapter.Runtime = Convert.ToInt32(taskSpecification.WalltimeLimit); } string jobClusterDirectory = FileSystemUtils.GetJobClusterDirectoryPath(jobSpecification.FileTransferMethod.Cluster.LocalBasepath, jobSpecification); string workDirectory = FileSystemUtils.GetTaskClusterDirectoryPath(jobClusterDirectory, taskSpecification); string stdErrFilePath = FileSystemUtils.ConcatenatePaths(workDirectory, taskSpecification.StandardErrorFile); taskAdapter.StdErrFilePath = workDirectory.Equals(stdErrFilePath) ? string.Empty : stdErrFilePath; string stdInFilePath = FileSystemUtils.ConcatenatePaths(workDirectory, taskSpecification.StandardInputFile); taskAdapter.StdInFilePath = workDirectory.Equals(stdInFilePath) ? string.Empty : stdInFilePath; string stdOutFilePath = FileSystemUtils.ConcatenatePaths(workDirectory, taskSpecification.StandardOutputFile); taskAdapter.StdOutFilePath = workDirectory.Equals(stdOutFilePath) ? string.Empty : stdOutFilePath; taskAdapter.WorkDirectory = workDirectory; taskAdapter.JobArrays = taskSpecification.JobArrays; taskAdapter.IsRerunnable = !string.IsNullOrEmpty(taskSpecification.JobArrays) || taskSpecification.IsRerunnable; taskAdapter.Queue = taskSpecification.ClusterNodeType.Queue; taskAdapter.ClusterAllocationName = taskSpecification.ClusterNodeType.ClusterAllocationName; taskAdapter.CpuHyperThreading = taskSpecification.CpuHyperThreading ?? false; CommandTemplate template = taskSpecification.CommandTemplate; if (template is null) { throw new ApplicationException(@$ "Command Template " "{taskSpecification.CommandTemplate.Name}" " for task " "{taskSpecification.Name}" " does not exist in the adaptor configuration."); } Dictionary <string, string> templateParameters = CreateTemplateParameterValuesDictionary(jobSpecification, taskSpecification, template.TemplateParameters, taskSpecification.CommandParameterValues); taskAdapter.SetPreparationAndCommand(workDirectory, ReplaceTemplateDirectivesInCommand(template.PreparationScript, templateParameters), ReplaceTemplateDirectivesInCommand($"{template.ExecutableFile} {template.CommandParameters}", templateParameters), stdOutFilePath, stdErrFilePath, CreateTaskDirectorySymlinkCommand(taskSpecification)); return(taskAdapter.AllocationCmd); }
/// <summary> /// Create template parameter values dictionary /// </summary> /// <param name="jobSpecification">Job specification</param> /// <param name="taskSpecification">Task specification</param> /// <param name="templateParameters">Template parameters</param> /// <param name="taskParametersValues">Task parameters values</param> /// <returns></returns> protected static Dictionary <string, string> CreateTemplateParameterValuesDictionary(JobSpecification jobSpecification, TaskSpecification taskSpecification, ICollection <CommandTemplateParameter> templateParameters, ICollection <CommandTemplateParameterValue> taskParametersValues) { var finalParameters = new Dictionary <string, string>(); foreach (CommandTemplateParameter templateParameter in templateParameters) { var taskParametersValue = taskParametersValues.Where(w => w.TemplateParameter.Identifier == templateParameter.Identifier) .FirstOrDefault(); if (taskParametersValue is not null) { // If taskParametersValue represent already escaped string of generic key-value pairs, don't escape it again. var isStringOfGenericParameters = templateParameter.CommandTemplate.IsGeneric && Regex.IsMatch(taskParametersValue.Value, @""".+""", RegexOptions.IgnoreCase | RegexOptions.Compiled); finalParameters.Add(templateParameter.Identifier, isStringOfGenericParameters ? taskParametersValue.Value : Regex.Escape(taskParametersValue.Value)); } else { string templateParameterValueFromQuery = templateParameter.Query; if (templateParameter.Query.StartsWith("Job.")) { templateParameterValueFromQuery = GetPropertyValueForQuery(jobSpecification, templateParameter.Query); } if (templateParameter.Query == "Task.Workdir") { string taskClusterDirectory = FileSystemUtils.GetJobClusterDirectoryPath(jobSpecification.FileTransferMethod.Cluster.LocalBasepath, jobSpecification); templateParameterValueFromQuery = FileSystemUtils.GetTaskClusterDirectoryPath(taskClusterDirectory, taskSpecification); } if (templateParameter.Query.StartsWith("Task.")) { templateParameterValueFromQuery = GetPropertyValueForQuery(taskSpecification, templateParameter.Query); } finalParameters.Add(templateParameter.Identifier, templateParameterValueFromQuery); } } return(finalParameters); }