protected override void CopyAll(string source, string target, bool overwrite, DateTime?lastModificationLimit, string[] excludedFiles, ClusterAuthenticationCredentials credentials)
{
ConnectionInfo connection = _connectionPool.GetConnectionForUser(credentials);
try
{
var client = new SftpClientAdapter((ExtendedSftpClient)connection.Connection);
if (Uri.IsWellFormedUriString(target, UriKind.Absolute))
{
CopyAllToSftp(source, target, overwrite, lastModificationLimit, client, excludedFiles);
}
else
{
if (Uri.IsWellFormedUriString(source, UriKind.Absolute))
{
CopyAllFromSftp(source, target, overwrite, lastModificationLimit, client, excludedFiles);
}
else
{
FileSystemUtils.CopyAll(source, target, overwrite, lastModificationLimit, excludedFiles);
}
}
}
finally
{
_connectionPool.ReturnConnection(connection);
}
}
protected override ICollection <FileInformation> ListChangedFilesForTask(string taskClusterDirectoryPath, DateTime?lastModificationLimit, ClusterAuthenticationCredentials credentials)
{
ConnectionInfo connection = _connectionPool.GetConnectionForUser(credentials);
try
{
var client = new SftpClientAdapter((ExtendedSftpClient)connection.Connection);
return(ListChangedFilesInDirectory(taskClusterDirectoryPath, taskClusterDirectoryPath, lastModificationLimit, credentials, client));
}
finally
{
_connectionPool.ReturnConnection(connection);
}
}
public override void DeleteSessionFromCluster(SubmittedJobInfo jobInfo)
{
string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification);
ConnectionInfo connection = _connectionPool.GetConnectionForUser(jobInfo.Specification.ClusterUser);
try
{
string remotePath = jobClusterDirectoryPath;
var client = new SftpClientAdapter((ExtendedSftpClient)connection.Connection);
DeleteRemoteDirectory(remotePath, client);
}
finally
{
_connectionPool.ReturnConnection(connection);
}
}
public override byte[] DownloadFileFromClusterByAbsolutePath(JobSpecification jobSpecification, string absoluteFilePath)
{
ConnectionInfo connection = _connectionPool.GetConnectionForUser(jobSpecification.ClusterUser);
try
{
var client = new SftpClientAdapter((ExtendedSftpClient)connection.Connection);
using var stream = new MemoryStream();
var path = absoluteFilePath.Replace("~/", string.Empty).Replace("/~/", string.Empty);
client.DownloadFile(path, stream);
return(stream.ToArray());
}
finally
{
_connectionPool.ReturnConnection(connection);
}
}
public override byte[] DownloadFileFromCluster(SubmittedJobInfo jobInfo, string relativeFilePath)
{
string jobClusterDirectoryPath = FileSystemUtils.GetJobClusterDirectoryPath(_fileSystem.Cluster.LocalBasepath, jobInfo.Specification);
ConnectionInfo connection = _connectionPool.GetConnectionForUser(jobInfo.Specification.ClusterUser);
try
{
var client = new SftpClientAdapter((ExtendedSftpClient)connection.Connection);
using MemoryStream stream = new MemoryStream();
string file = jobClusterDirectoryPath + relativeFilePath;
client.DownloadFile(file, stream);
return(stream.ToArray());
}
finally
{
_connectionPool.ReturnConnection(connection);
}
}
public ICollection <JobFileContent> SynchronizeFiles()
{
ConnectionInfo connection = ConnectionPool.GetConnectionForUser(_credentials);
try
{
var client = new SftpClientAdapter((ExtendedSftpClient)connection.Connection);
string sourcePath = FileSystemUtils.ConcatenatePaths(SyncFileInfo.SourceDirectory, SyncFileInfo.RelativePath);
if (client.Exists(sourcePath))
{
using Stream sourceStream = client.OpenRead(sourcePath);
string synchronizedContent = FileSystemUtils.ReadStreamContentFromSpecifiedOffset(sourceStream, Offset);
if (!string.IsNullOrEmpty(SyncFileInfo.DestinationDirectory))
{
string destinationPath = Path.Combine(SyncFileInfo.DestinationDirectory, SyncFileInfo.RelativePath);
if (Offset == 0)
{
File.Delete(destinationPath);
}
FileSystemUtils.WriteStringToLocalFile(synchronizedContent, destinationPath);
}
JobFileContent[] result = new JobFileContent[1]
{
new JobFileContent {
RelativePath = SyncFileInfo.RelativePath,
Content = synchronizedContent,
Offset = Offset
}
};
if (SyncFileInfo.SynchronizationType == FileSynchronizationType.IncrementalAppend)
{
Offset = sourceStream.Position;
}
return(result);
}
}
finally
{
ConnectionPool.ReturnConnection(connection);
}
return(default);
private void DeleteRemoteDirectory(string remotePath, SftpClientAdapter client)
{
_logger.LogDebug($"Starting delete remote directory {remotePath}");
if (client.Exists(remotePath))
{
foreach (SftpFile file in client.ListDirectory(remotePath))
{
if (file.Name == "." || file.Name == "..")
{
continue;
}
if (file.IsSymbolicLink)
{
_logger.LogDebug($"Deleting symlink {file.Name}");
client.Delete(file.FullName);
}
else
{
if (file.IsDirectory)
{
_logger.LogDebug($"Deleting subdirectory {file.Name}");
DeleteRemoteDirectory(file.FullName, client);
}
else
{
_logger.LogDebug($"Deleting file {file.Name}");
client.DeleteFile(file.FullName);
}
}
}
_logger.LogDebug($"Deleting root directory {remotePath}");
client.DeleteDirectory(remotePath);
}
}
private ICollection <FileInformation> ListChangedFilesInDirectory(string rootDirectory, string currentDirectory, DateTime?lastModificationLimit, ClusterAuthenticationCredentials credentials, SftpClientAdapter client)
{
List <FileInformation> results = new List <FileInformation>();
foreach (SftpFile file in client.ListDirectory(currentDirectory))
{
if (file.Name == "." || file.Name == "..")
{
continue;
}
if (file.IsDirectory)
{
results.AddRange(ListChangedFilesInDirectory(rootDirectory, FileSystemUtils.ConcatenatePaths(currentDirectory, file.Name), lastModificationLimit, credentials, client));
}
else if (((!lastModificationLimit.HasValue) || (lastModificationLimit.Value <= file.LastWriteTime)))
{
results.Add(new FileInformation
{
FileName = file.FullName.Replace(rootDirectory, string.Empty),
LastModifiedDate = file.LastWriteTime
});
}
}
return(results);
}
private void CopyAllToSftp(string source, string target, bool overwrite, DateTime?lastModificationLimit, SftpClientAdapter client, string[] excludedFiles)
{
string targetPath = target;
if (!client.Exists(targetPath))
{
client.CreateDirectory(targetPath);
}
DirectoryInfo sourceDir = new DirectoryInfo(source);
foreach (FileInfo file in sourceDir.GetFiles())
{
if (excludedFiles != null && excludedFiles.Contains(file.Name))
{
continue;
}
string targetFilePath = FileSystemUtils.ConcatenatePaths(targetPath, file.Name);
if (((!client.Exists(targetFilePath)) || overwrite) && ((!lastModificationLimit.HasValue) || (lastModificationLimit.Value < file.LastWriteTime)))
{
using FileStream sourceStream = file.OpenRead();
client.UploadFile(sourceStream, targetFilePath, true);
}
}
foreach (DirectoryInfo directory in sourceDir.GetDirectories())
{
CopyAllToSftp(directory.FullName, FileSystemUtils.ConcatenatePaths(target, directory.Name), overwrite, lastModificationLimit, client, FileSystemUtils.GetExcludedFilesForSubdirectory(excludedFiles, directory.Name));
}
}
private void CopyAllFromSftp(string source, string target, bool overwrite, DateTime?lastModificationLimit, SftpClientAdapter client, string[] excludedFiles)
{
string sourcePath = source;
if (!Directory.Exists(target))
{
Directory.CreateDirectory(target);
}
foreach (SftpFile file in client.ListDirectory(sourcePath))
{
if (file.Name == "." || file.Name == "..")
{
continue;
}
if (file.IsDirectory)
{
CopyAllFromSftp(FileSystemUtils.ConcatenatePaths(source, file.Name), Path.Combine(target, file.Name), overwrite,
lastModificationLimit, client, FileSystemUtils.GetExcludedFilesForSubdirectory(excludedFiles, file.Name));
}
else
{
if (excludedFiles != null && excludedFiles.Contains(file.Name))
{
continue;
}
string targetFilePath = Path.Combine(target, file.Name);
if (((!File.Exists(targetFilePath)) || overwrite) &&
((!lastModificationLimit.HasValue) || (lastModificationLimit.Value < file.LastWriteTime)))
{
using FileStream targetStream = new FileStream(targetFilePath, FileMode.Create, FileAccess.Write, FileShare.None);
client.DownloadFile(file.FullName, targetStream);
}
}
}
}
