protected virtual FullFileSpecification CreateSynchronizableFileInfoForType(TaskSpecification task, string taskClusterDirectoryPath,
SynchronizableFiles fileType)
{
FullFileSpecification fileInfo = new FullFileSpecification
{
DestinationDirectory = task.LocalDirectory,
SourceDirectory = taskClusterDirectoryPath,
};
CompleteFileInfoForType(fileInfo, task, fileType);
return(fileInfo);
}
protected virtual ICollection <JobFileContent> SynchronizeAllFilesOfType(SynchronizableFiles fileType)
{
List <JobFileContent> results = new List <JobFileContent>();
foreach (IFileSynchronizer synchronizer in _fileSynchronizers[fileType].Values)
{
ICollection <JobFileContent> result = synchronizer.SynchronizeFiles();
if (result != null)
{
results.AddRange(result);
}
}
return(results);
}
public virtual ICollection <JobFileContent> DownloadPartOfJobFileFromCluster(SubmittedTaskInfo taskInfo, SynchronizableFiles fileType, long offset)
{
string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, taskInfo.Specification.JobSpecification);
string taskClusterDirectoryPath = FileSystemUtils.GetTaskClusterDirectoryPath(jobClusterDirectoryPath, taskInfo.Specification);
FullFileSpecification fileInfo = CreateSynchronizableFileInfoForType(taskInfo.Specification, taskClusterDirectoryPath, fileType);
IFileSynchronizer synchronizer = CreateFileSynchronizer(fileInfo, taskInfo.Specification.JobSpecification.ClusterUser);
synchronizer.Offset = offset;
synchronizer.SyncFileInfo.DestinationDirectory = null;
ICollection <JobFileContent> result = synchronizer.SynchronizeFiles();
if (result != null)
{
foreach (JobFileContent content in result)
{
content.FileType = fileType;
content.SubmittedTaskInfoId = taskInfo.Id;
}
}
return(result);
}
protected virtual void CompleteFileInfoForType(FileSpecification fileInfo, TaskSpecification task, SynchronizableFiles fileType)
{
switch (fileType)
{
case SynchronizableFiles.StandardOutputFile:
fileInfo.RelativePath = task.StandardOutputFile;
fileInfo.NameSpecification = FileNameSpecification.FullName;
fileInfo.SynchronizationType = FileSynchronizationType.IncrementalAppend;
break;
case SynchronizableFiles.StandardErrorFile:
fileInfo.RelativePath = task.StandardErrorFile;
fileInfo.NameSpecification = FileNameSpecification.FullName;
fileInfo.SynchronizationType = FileSynchronizationType.IncrementalAppend;
break;
case SynchronizableFiles.LogFile:
fileInfo.RelativePath = task.LogFile.RelativePath;
fileInfo.NameSpecification = task.LogFile.NameSpecification;
fileInfo.SynchronizationType = task.LogFile.SynchronizationType;
break;
case SynchronizableFiles.ProgressFile:
fileInfo.RelativePath = task.ProgressFile.RelativePath;
fileInfo.NameSpecification = task.ProgressFile.NameSpecification;
fileInfo.SynchronizationType = task.ProgressFile.SynchronizationType;
break;
}
}
protected virtual void CreateSynchronizersForType(JobSpecification jobSpecification, SynchronizableFiles fileType)
{
_fileSynchronizers[fileType] = new Dictionary <string, IFileSynchronizer>(jobSpecification.Tasks.Count);
foreach (TaskSpecification task in jobSpecification.Tasks)
{
string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobSpecification);
string taskClusterDirectoryPath = FileSystemUtils.GetTaskClusterDirectoryPath(jobClusterDirectoryPath, task);
FullFileSpecification fileInfo = CreateSynchronizableFileInfoForType(task, taskClusterDirectoryPath, fileType);
string sourceFilePath = FileSystemUtils.ConcatenatePaths(fileInfo.SourceDirectory, fileInfo.RelativePath);
if (!_fileSynchronizers[fileType].ContainsKey(sourceFilePath))
{
_fileSynchronizers[fileType][sourceFilePath] = CreateFileSynchronizer(fileInfo, jobSpecification.ClusterUser);
}
}
}
protected virtual ICollection <JobFileContent> PerformSynchronizationForType(SubmittedJobInfo jobInfo, SynchronizableFiles fileType)
{
List <JobFileContent> result = new List <JobFileContent>();
if (!_fileSynchronizers.ContainsKey(fileType))
{
_fileSynchronizers[fileType] = new Dictionary <string, IFileSynchronizer>(jobInfo.Tasks.Count);
}
foreach (SubmittedTaskInfo taskInfo in jobInfo.Tasks)
{
string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification);
string taskClusterDirectoryPath = FileSystemUtils.GetTaskClusterDirectoryPath(jobClusterDirectoryPath, taskInfo.Specification);
FullFileSpecification fileInfo = CreateSynchronizableFileInfoForType(taskInfo.Specification, taskClusterDirectoryPath, fileType);
string sourceFilePath = FileSystemUtils.ConcatenatePaths(fileInfo.SourceDirectory, fileInfo.RelativePath);
if (!_fileSynchronizers[fileType].ContainsKey(sourceFilePath))
{
_fileSynchronizers[fileType][sourceFilePath] = CreateFileSynchronizer(fileInfo, jobInfo.Specification.ClusterUser);
}
ICollection <JobFileContent> subresult = _fileSynchronizers[fileType][sourceFilePath].SynchronizeFiles();
if (subresult != null)
{
foreach (JobFileContent content in subresult)
{
content.FileType = fileType;
content.SubmittedTaskInfoId = taskInfo.Id;
result.Add(content);
}
}
}
return(result);
}