protected virtual void CompleteFileInfoForType(FileSpecification fileInfo, TaskSpecification task, SynchronizableFiles fileType)
{
switch (fileType)
{
case SynchronizableFiles.StandardOutputFile:
fileInfo.RelativePath = task.StandardOutputFile;
fileInfo.NameSpecification = FileNameSpecification.FullName;
fileInfo.SynchronizationType = FileSynchronizationType.IncrementalAppend;
break;
case SynchronizableFiles.StandardErrorFile:
fileInfo.RelativePath = task.StandardErrorFile;
fileInfo.NameSpecification = FileNameSpecification.FullName;
fileInfo.SynchronizationType = FileSynchronizationType.IncrementalAppend;
break;
case SynchronizableFiles.LogFile:
fileInfo.RelativePath = task.LogFile.RelativePath;
fileInfo.NameSpecification = task.LogFile.NameSpecification;
fileInfo.SynchronizationType = task.LogFile.SynchronizationType;
break;
case SynchronizableFiles.ProgressFile:
fileInfo.RelativePath = task.ProgressFile.RelativePath;
fileInfo.NameSpecification = task.ProgressFile.NameSpecification;
fileInfo.SynchronizationType = task.ProgressFile.SynchronizationType;
break;
}
}
private static TaskSpecification ConvertExtToInt(this TaskSpecificationExt taskSpecificationExt, JobSpecification jobSpecification)
{
var result = new TaskSpecification
{
Name = taskSpecificationExt.Name,
MinCores = taskSpecificationExt.MinCores,
MaxCores = taskSpecificationExt.MaxCores,
WalltimeLimit = taskSpecificationExt.WalltimeLimit,
PlacementPolicy = taskSpecificationExt.PlacementPolicy,
RequiredNodes = taskSpecificationExt.RequiredNodes?
.Select(s => new TaskSpecificationRequiredNode
{
NodeName = s
})
.ToList(),
Priority = taskSpecificationExt.Priority.ConvertExtToInt(),
Project = jobSpecification.Project,
JobArrays = taskSpecificationExt.JobArrays,
IsExclusive = taskSpecificationExt.IsExclusive ?? false,
IsRerunnable = !string.IsNullOrEmpty(taskSpecificationExt.JobArrays) || (taskSpecificationExt.IsRerunnable ?? false),
StandardInputFile = taskSpecificationExt.StandardInputFile,
StandardOutputFile = taskSpecificationExt.StandardOutputFile ?? "stdout.txt",
StandardErrorFile = taskSpecificationExt.StandardErrorFile ?? "stderr.txt",
ClusterTaskSubdirectory = taskSpecificationExt.ClusterTaskSubdirectory,
ProgressFile = new FileSpecification
{
RelativePath = taskSpecificationExt.ProgressFile,
NameSpecification = FileNameSpecification.FullName,
SynchronizationType = FileSynchronizationType.IncrementalAppend
},
LogFile = new FileSpecification
{
RelativePath = taskSpecificationExt.LogFile,
NameSpecification = FileNameSpecification.FullName,
SynchronizationType = FileSynchronizationType.IncrementalAppend
},
ClusterNodeTypeId = taskSpecificationExt.ClusterNodeTypeId.Value,
CommandTemplateId = taskSpecificationExt.CommandTemplateId ?? 0,
EnvironmentVariables = taskSpecificationExt.EnvironmentVariables?
.Select(s => s.ConvertExtToInt())
.ToList(),
CpuHyperThreading = taskSpecificationExt.CpuHyperThreading,
JobSpecification = jobSpecification,
TaskParalizationSpecifications = taskSpecificationExt.TaskParalizationParameters?
.Select(s => s.ConvertExtToInt())
.ToList(),
CommandParameterValues = taskSpecificationExt.TemplateParameterValues?
.Select(s => s.ConvertExtToInt())
.ToList(),
};
result.DependsOn = taskSpecificationExt.DependsOn?
.Select(s => new TaskDependency
{
TaskSpecification = result,
ParentTaskSpecification = s.ConvertExtToInt(jobSpecification)
})
.ToList();
return(result);
}
protected virtual FullFileSpecification CreateSynchronizableFileInfoForType(TaskSpecification task, string taskClusterDirectoryPath,
SynchronizableFiles fileType)
{
FullFileSpecification fileInfo = new FullFileSpecification
{
DestinationDirectory = task.LocalDirectory,
SourceDirectory = taskClusterDirectoryPath,
};
CompleteFileInfoForType(fileInfo, task, fileType);
return(fileInfo);
}
public void FilterUserOwned_Positive()
{
var collection = new List <TaskEntity>
{
new() { UserId = "userid1" },
new() { UserId = "userid2" }
};
var service = new TaskSpecification().FilterUserOwned("userid1");
var result = service.Apply(collection.AsQueryable()).ToList();
Assert.Collection(result, _ => { });
}
public static TaskInfo Create(TaskSpecification taskSpecification, User user)
{
var taskInfo = new TaskInfo
{
TaskSpecificationId = taskSpecification.Id,
TaskSpecification = taskSpecification
};
if (user != null)
{
taskInfo.SetAuditInfo(user.Login);
}
return(taskInfo);
}
/// <summary>
/// Create task directory sym link
/// </summary>
/// <param name="taskSpecification">Task specification</param>
/// <returns></returns>
protected static string CreateTaskDirectorySymlinkCommand(TaskSpecification taskSpecification)
{
string symlinkCommand = string.Empty;
if (taskSpecification.DependsOn.Any())
{
long dependsOnIdLast = taskSpecification.DependsOn.Max(x => x.ParentTaskSpecificationId);
return(taskSpecification.DependsOn.FirstOrDefault(x => x.ParentTaskSpecificationId == dependsOnIdLast) == null
? symlinkCommand
: symlinkCommand = $"ln -s ../{dependsOnIdLast}/* .");
}
return(symlinkCommand);
}
public void FilterForeignUserOwned_Positive()
{
var collection = new List <TaskEntity>
{
new() { UserId = "userid1", Uid = "uid1" },
new() { UserId = "userid2", Uid = "uid2" }
};
var service = new TaskSpecification().FilterForeignUserOwned("userid1", new[] { "uid1", "uid2" });
var result = service.Apply(collection.AsQueryable()).ToList();
Assert.Collection(result, x =>
{
Assert.Equal("uid2", x.Uid);
});
}
}
private void ValidateWallTimeLimit(TaskSpecification task)
{
var clusterNodeType = LogicFactory.GetLogicFactory().CreateClusterInformationLogic(_unitOfWork)
.GetClusterNodeTypeById(task.ClusterNodeTypeId);
if (clusterNodeType == null)
{
_messageBuilder.AppendLine($"Requested ClusterNodeType with Id {task.ClusterNodeTypeId} does not exist in the system");
return;
}
if (task.WalltimeLimit.HasValue && task.WalltimeLimit.Value > clusterNodeType.MaxWalltime)
{
_messageBuilder.AppendLine(
$"Defined task {task.Name} has set higher WalltimeLimit ({task.WalltimeLimit.Value}) than the maximum on this cluster node, " +
$"maximal WallTimeLimit is {clusterNodeType.MaxWalltime}");
}
}
private void ValidateTaskSpecification(TaskSpecification task)
{
if (task.Id != 0 && _unitOfWork.TaskSpecificationRepository.GetById(task.Id) == null)
{
_messageBuilder.AppendLine($"Task with Id {task.Id} does not exist in the system");
}
ValidateWallTimeLimit(task);
if (task.CommandTemplate == null)
{
_messageBuilder.AppendLine($"Command Template does not exist.");
return;
}
foreach (CommandTemplateParameter parameter in task.CommandTemplate.TemplateParameters)
{
if (string.IsNullOrEmpty(parameter.Query) &&
(task.CommandParameterValues == null ||
!task.CommandParameterValues.Any(w => w.TemplateParameter == parameter)))
{
_messageBuilder.AppendLine($"Command Template parameter \"{parameter.Identifier}\" does not have a value.");
}
}
if (task.ClusterNodeTypeId != task.CommandTemplate.ClusterNodeTypeId)
{
_messageBuilder.AppendLine($"Task {task.Name} has wrong CommandTemplate");
}
if (!task.CommandTemplate.IsEnabled)
{
_messageBuilder.AppendLine($"Task {task.Name} has specified deleted CommandTemplateId \"{task.CommandTemplate.Id}\"");
}
if (task.CommandTemplate.IsGeneric)
{
ValidateGenericCommandTemplateSetup(task);
}
}
private void ValidateGenericCommandTemplateSetup(TaskSpecification task)
{
Dictionary <string, string> genericCommandParametres = new();
//Regex.Matches(task.CommandTemplate.CommandParameters, @"%%\{([\w\.]+)\}", RegexOptions.Compiled)
foreach (var commandParameterValue in task.CommandParameterValues)
{
var key = commandParameterValue.CommandParameterIdentifier;
string value = commandParameterValue.Value;
if (!string.IsNullOrEmpty(value))
{
genericCommandParametres.Add(key, value);
}
}
Match scriptPathParameterName = Regex.Match(task.CommandTemplate.CommandParameters, @"%%\{([\w\.]+)\}", RegexOptions.Compiled);
if (!scriptPathParameterName.Success)
{
_messageBuilder.AppendLine($"CommandTemplate is wrong");
}
string clusterPathToUserScript = genericCommandParametres.FirstOrDefault(x => x.Key == scriptPathParameterName.Groups[1].Value).Value;
if (string.IsNullOrWhiteSpace(clusterPathToUserScript))
{
_messageBuilder.AppendLine($"User script path parameter, for generic command template, does not have a value.");
}
var scriptDefinedParametres = GetUserDefinedScriptParametres(task.ClusterNodeType.Cluster, clusterPathToUserScript);
foreach (string parameter in scriptDefinedParametres)
{
if (!genericCommandParametres.Select(x => x.Value).Any(x => Regex.IsMatch(x, $"{parameter}=\\\\\".+\\\\\"")))
{
_messageBuilder.AppendLine($"Task specification does not contain '{parameter}' parameter.");
}
}
}
private void ValidateTaskSpecificationInput(TaskSpecification task)
{
if (task.CommandTemplateId <= 0)
{
_messageBuilder.AppendLine("CommandTemplateId cannot be empty or <= 0.");
}
if (string.IsNullOrEmpty(task.Name))
{
_messageBuilder.AppendLine("Task name cannot be empty.");
}
if (task.Name.Length > 50)
{
_messageBuilder.AppendLine($"Task name \"{task.Name}\" cannot be longer than 50 characters.");
}
if (ContainsIllegalCharacters(task.Name))
{
_messageBuilder.AppendLine("Task name contains illegal characters.");
}
if (task.MinCores <= 0)
{
_messageBuilder.AppendLine($"Minimal number of cores for task \"{task.Name}\" has to be greater than 0.");
}
if (task.MaxCores <= 0)
{
_messageBuilder.AppendLine($"Maximal number of cores for task \"{task.Name}\" has to be greater than 0.");
}
if (task.MinCores > task.MaxCores)
{
_messageBuilder.AppendLine($"Minimal number of cores for task \"{task.Name}\" cannot be greater than maximal number of cores.");
}
if (task.WalltimeLimit <= 0)
{
_messageBuilder.AppendLine($"Walltime limit for task \"{task.Name}\" has to be greater than 0.");
}
if (task.JobArrays != null)
{
if (task.JobArrays == string.Empty)
{
task.JobArrays = null;
}
else
{
if (task.JobArrays.Length > 40)
{
_messageBuilder.AppendLine($"JobArrays specification for task \"{task.Name}\" cannot be longer than 40 characters.");
}
task.JobArrays = task.JobArrays.Replace(" ", string.Empty);
var splittedArray = task.JobArrays.Split(':');
if (splittedArray.Length >= 1)
{
int step = 1;
var interval = splittedArray[0].Split('-');
if ((splittedArray.Length == 1 || (splittedArray.Length == 2 && int.TryParse(splittedArray[1], out step))) &&
interval.Length == 2 && int.TryParse(interval[0], out int min) && int.TryParse(interval[1], out int max))
{
if (!(min < max && min + step <= max))
{
_messageBuilder.AppendLine($"JobArrays parameter for task \"{task.Name}\" has wrong filled minimum or maximum or step value.");
}
}
else
{
_messageBuilder.AppendLine($"JobArrays parameter for task \"{task.Name}\" has wrong definition.");
}
}
}
}
if (task.TaskParalizationSpecifications?.Sum(s => s.MaxCores) > task.MaxCores)
{
_messageBuilder.AppendLine($"TaskParalizationSpecifications count of maximal cores for task \"{task.Name}\" must be lower or equals to Maximal number of cores in task.");
}
if (!string.IsNullOrEmpty(task.PlacementPolicy))
{
if (task.PlacementPolicy.Length > 40)
{
_messageBuilder.AppendLine($"Placement policy specification for task \"{task.Name}\" cannot be longer than 40 characters.");
}
if (ContainsIllegalCharacters(task.PlacementPolicy))
{
_messageBuilder.AppendLine($"Placement policy specification for task \"{task.Name}\" contains illegal characters.");
}
}
if (!string.IsNullOrEmpty(task.StandardInputFile))
{
if (task.StandardInputFile.Length > 30)
{
_messageBuilder.AppendLine($"Standard input file for task \"{task.Name}\" cannot be longer than 30 characters.");
}
if (ContainsIllegalCharacters(task.StandardInputFile))
{
_messageBuilder.AppendLine($"Standard input file for task \"{task.Name}\" contains illegal characters.");
}
}
if (!string.IsNullOrEmpty(task.StandardOutputFile))
{
if (task.StandardOutputFile.Length > 30)
{
_messageBuilder.AppendLine($"Standard output file for task \"{task.Name}\" cannot be longer than 30 characters.");
}
if (ContainsIllegalCharacters(task.StandardOutputFile))
{
_messageBuilder.AppendLine($"Standard output file for task \"{task.Name}\" contains illegal characters.");
}
}
if (!string.IsNullOrEmpty(task.StandardErrorFile))
{
if (task.StandardErrorFile.Length > 30)
{
_messageBuilder.AppendLine($"Standard error file for task \"{task.Name}\" cannot be longer than 30 characters.");
}
if (ContainsIllegalCharacters(task.StandardErrorFile))
{
_messageBuilder.AppendLine($"Standard error file for task \"{task.Name}\" contains illegal characters.");
}
}
if (!string.IsNullOrEmpty(task.ProgressFile.RelativePath))
{
if (task.ProgressFile.RelativePath.Length > 50)
{
_messageBuilder.AppendLine($"Progress file for task \"{task.Name}\" cannot be longer than 50 characters.");
}
if (ContainsIllegalCharactersForPath(task.ProgressFile.RelativePath))
{
_messageBuilder.AppendLine($"Progress file for task \"{task.Name}\" contains illegal characters.");
}
}
if (!string.IsNullOrEmpty(task.LogFile.RelativePath))
{
if (task.LogFile.RelativePath.Length > 50)
{
_messageBuilder.AppendLine($"Log file for task \"{task.Name}\" cannot be longer than 50 characters.");
}
if (ContainsIllegalCharactersForPath(task.LogFile.RelativePath))
{
_messageBuilder.AppendLine($"Log file for task \"{task.Name}\" contains illegal characters.");
}
}
if (task.RequiredNodes != null)
{
foreach (TaskSpecificationRequiredNode requiredNode in task.RequiredNodes)
{
if (requiredNode.NodeName.Length > 40)
{
_messageBuilder.AppendLine($"Required node \"{requiredNode.NodeName}\" specification for task \"{task.Name}\" cannot be longer than 40 characters.");
}
if (ContainsIllegalCharacters(requiredNode.NodeName))
{
_messageBuilder.AppendLine($"Required node \"{requiredNode.NodeName}\" specification for task \"{task.Name}\" contains illegal characters.");
}
}
}
if (task.EnvironmentVariables != null)
{
foreach (EnvironmentVariable variable in task.EnvironmentVariables)
{
if (string.IsNullOrEmpty(variable.Name))
{
_messageBuilder.AppendLine($"Environment variable's name for task \"{task.Name}\" cannot be empty. ({variable.Name} = {variable.Value})");
}
}
}
}
public static string GetTaskClusterDirectoryPath(string jobClusterDirectoryPath, TaskSpecification taskSpecification)
{
string taskSubdirectory = !string.IsNullOrEmpty(taskSpecification.ClusterTaskSubdirectory)
? $"{taskSpecification.Id}/{taskSpecification.ClusterTaskSubdirectory}"
: $"{taskSpecification.Id}";
return(ConcatenatePaths(jobClusterDirectoryPath, taskSubdirectory));
}
/// <summary>
/// Convert task specification to task
/// </summary>
/// <param name="jobSpecification">Job specification</param>
/// <param name="taskSpecification">Task specification</param>
/// <param name="schedulerAllocationCmd">Scheduler allocation cmd</param>
/// <returns></returns>
/// <exception cref="ApplicationException"></exception>
public virtual object ConvertTaskSpecificationToTask(JobSpecification jobSpecification, TaskSpecification taskSpecification, object schedulerAllocationCmd)
{
ISchedulerTaskAdapter taskAdapter = _conversionAdapterFactory.CreateTaskAdapter(schedulerAllocationCmd);
taskAdapter.DependsOn = taskSpecification.DependsOn;
taskAdapter.SetEnvironmentVariablesToTask(taskSpecification.EnvironmentVariables);
taskAdapter.IsExclusive = taskSpecification.IsExclusive;
taskAdapter.SetRequestedResourceNumber(taskSpecification.ClusterNodeType.RequestedNodeGroups.Select(s => s.Name).ToList(),
taskSpecification.RequiredNodes.Select(s => s.NodeName).ToList(),
taskSpecification.PlacementPolicy,
taskSpecification.TaskParalizationSpecifications,
Convert.ToInt32(taskSpecification.MinCores),
Convert.ToInt32(taskSpecification.MaxCores),
taskSpecification.ClusterNodeType.CoresPerNode);
// Do not change!!! Task name on the cluster is set as ID of the used task specification to enable pairing of cluster task info with DB task info.
taskAdapter.Name = taskSpecification.Id.ToString(CultureInfo.InvariantCulture);
if (Convert.ToInt32(taskSpecification.WalltimeLimit) > 0)
{
taskAdapter.Runtime = Convert.ToInt32(taskSpecification.WalltimeLimit);
}
string jobClusterDirectory = FileSystemUtils.GetJobClusterDirectoryPath(jobSpecification.FileTransferMethod.Cluster.LocalBasepath, jobSpecification);
string workDirectory = FileSystemUtils.GetTaskClusterDirectoryPath(jobClusterDirectory, taskSpecification);
string stdErrFilePath = FileSystemUtils.ConcatenatePaths(workDirectory, taskSpecification.StandardErrorFile);
taskAdapter.StdErrFilePath = workDirectory.Equals(stdErrFilePath) ? string.Empty : stdErrFilePath;
string stdInFilePath = FileSystemUtils.ConcatenatePaths(workDirectory, taskSpecification.StandardInputFile);
taskAdapter.StdInFilePath = workDirectory.Equals(stdInFilePath) ? string.Empty : stdInFilePath;
string stdOutFilePath = FileSystemUtils.ConcatenatePaths(workDirectory, taskSpecification.StandardOutputFile);
taskAdapter.StdOutFilePath = workDirectory.Equals(stdOutFilePath) ? string.Empty : stdOutFilePath;
taskAdapter.WorkDirectory = workDirectory;
taskAdapter.JobArrays = taskSpecification.JobArrays;
taskAdapter.IsRerunnable = !string.IsNullOrEmpty(taskSpecification.JobArrays) || taskSpecification.IsRerunnable;
taskAdapter.Queue = taskSpecification.ClusterNodeType.Queue;
taskAdapter.ClusterAllocationName = taskSpecification.ClusterNodeType.ClusterAllocationName;
taskAdapter.CpuHyperThreading = taskSpecification.CpuHyperThreading ?? false;
CommandTemplate template = taskSpecification.CommandTemplate;
if (template is null)
{
throw new ApplicationException(@$ "Command Template " "{taskSpecification.CommandTemplate.Name}" " for task
" "{taskSpecification.Name}" " does not exist in the adaptor configuration.");
}
Dictionary <string, string> templateParameters = CreateTemplateParameterValuesDictionary(jobSpecification, taskSpecification,
template.TemplateParameters, taskSpecification.CommandParameterValues);
taskAdapter.SetPreparationAndCommand(workDirectory, ReplaceTemplateDirectivesInCommand(template.PreparationScript, templateParameters),
ReplaceTemplateDirectivesInCommand($"{template.ExecutableFile} {template.CommandParameters}", templateParameters),
stdOutFilePath, stdErrFilePath, CreateTaskDirectorySymlinkCommand(taskSpecification));
return(taskAdapter.AllocationCmd);
}
/// <summary>
/// Create template parameter values dictionary
/// </summary>
/// <param name="jobSpecification">Job specification</param>
/// <param name="taskSpecification">Task specification</param>
/// <param name="templateParameters">Template parameters</param>
/// <param name="taskParametersValues">Task parameters values</param>
/// <returns></returns>
protected static Dictionary <string, string> CreateTemplateParameterValuesDictionary(JobSpecification jobSpecification, TaskSpecification taskSpecification,
ICollection <CommandTemplateParameter> templateParameters, ICollection <CommandTemplateParameterValue> taskParametersValues)
{
var finalParameters = new Dictionary <string, string>();
foreach (CommandTemplateParameter templateParameter in templateParameters)
{
var taskParametersValue = taskParametersValues.Where(w => w.TemplateParameter.Identifier == templateParameter.Identifier)
.FirstOrDefault();
if (taskParametersValue is not null)
{
// If taskParametersValue represent already escaped string of generic key-value pairs, don't escape it again.
var isStringOfGenericParameters = templateParameter.CommandTemplate.IsGeneric && Regex.IsMatch(taskParametersValue.Value, @""".+""", RegexOptions.IgnoreCase | RegexOptions.Compiled);
finalParameters.Add(templateParameter.Identifier, isStringOfGenericParameters ? taskParametersValue.Value : Regex.Escape(taskParametersValue.Value));
}
else
{
string templateParameterValueFromQuery = templateParameter.Query;
if (templateParameter.Query.StartsWith("Job."))
{
templateParameterValueFromQuery = GetPropertyValueForQuery(jobSpecification, templateParameter.Query);
}
if (templateParameter.Query == "Task.Workdir")
{
string taskClusterDirectory = FileSystemUtils.GetJobClusterDirectoryPath(jobSpecification.FileTransferMethod.Cluster.LocalBasepath, jobSpecification);
templateParameterValueFromQuery = FileSystemUtils.GetTaskClusterDirectoryPath(taskClusterDirectory, taskSpecification);
}
if (templateParameter.Query.StartsWith("Task."))
{
templateParameterValueFromQuery = GetPropertyValueForQuery(taskSpecification, templateParameter.Query);
}
finalParameters.Add(templateParameter.Identifier, templateParameterValueFromQuery);
}
}
return(finalParameters);
}
