private async Task <Message> ReadAsync(string path, string fileName)
{
var stream = new MemoryStream();
var fullPath = Path.Combine(path, fileName).Replace('\\', '/');
using (var sftpClient = new Sftp())
{
sftpClient.TransferType = _transferType;
await sftpClient.ConnectAsync(_serverAddress, _port);
await sftpClient.LoginAsync(_userName, _password);
await sftpClient.GetFileAsync(fullPath, stream);
await sftpClient.DisconnectAsync();
}
// Rewind the stream:
stream.Position = 0;
return(new Message {
ContentStream = stream
});
}
private async Task UploadFile(JobInitResult jobInitResult, FileInfo toBeConverted)
{
Licensing.Key = sftpLicenseKey;
using (var client = new Sftp())
{
await client.ConnectAsync(jobInitResult.UploadUrl, jobInitResult.Port);
await client.LoginAsync(jobInitResult.User, jobInitResult.Password);
var uploadName = $"{client.GetCurrentDirectory()}{toBeConverted.Name}";
Log.Verbose("Uploading file {FullName} to {UploadName}", toBeConverted.FullName, uploadName);
if (toBeConverted.Exists)
{
await client.PutFileAsync(toBeConverted.FullName, uploadName);
Log.Verbose("Upload successfull");
}
else
{
Log.Verbose("File {FullName} does not exist, or cannot be accessed.", toBeConverted.FullName);
throw new FileNotFoundException("File could not be uploaded, because source file is not accessible or cannot be found",
toBeConverted.FullName);
}
}
}
public async Task <List <ReceiveFileResult> > ReceiveFilesAsync(string path, string fileMask, bool receiveMultiple = true)
{
var handledFiles = new List <ReceiveFileResult>();
using (var client = new Sftp())
{
client.TransferType = _transferType;
await client.ConnectAsync(_serverAddress, _port, SslMode.None);
await client.LoginAsync(_userName, _password);
// Get files from the specified path using the given file name mask:
var items = await client.GetListAsync(path);
var files = items.GetFiles(fileMask);
// Ignore files that are already transfering / failed and sort the remaining by filename
var sortedFiles = files.Where(f => !(f.StartsWith(".") && f.EndsWith(".tmp"))).OrderBy(f => f).ToList();
if (!receiveMultiple && (sortedFiles.Count > 1))
{
// only process the first file
var singleFile = sortedFiles.First();
sortedFiles = new List <string> {
singleFile
};
}
// Try to rename the files with temporary names:
foreach (var file in sortedFiles)
{
var result = new ReceiveFileResult
{
FileName = file,
TempFileName = $".{Guid.NewGuid()}.tmp",
Path = path,
IsError = false
};
try
{
await client.RenameAsync(Path.Combine(path, result.FileName).Replace('\\', '/'), Path.Combine(path, result.TempFileName).Replace('\\', '/'));
}
catch (Exception ex)
{
result.IsError = true;
result.Message = $"Renaming the file '{result.FileName}' to '{result.TempFileName}' failed: {ex.Message}.";
}
handledFiles.Add(result);
}
await client.DisconnectAsync();
}
return(handledFiles);
}
private void Client_ConnectCompleted(object sender, System.ComponentModel.AsyncCompletedEventArgs e)
{
if (e.Error == null)
{
client.AuthenticateAsync(Settings.MasterLogin.Username, Settings.MasterLogin.Password);
}
else
{
connectErrorCount += 1;
if (connectErrorCount < Settings.MaxTransferErrorCount)
{
client.ConnectAsync(Settings.MasterLogin.Url, 22);
}
else
{
Sleep();
}
}
}
private async Task UploadFile(JobInitResult jobInitResult, FileInfo toBeConverted)
{
Licensing.Key = sftpLicenseKey;
using (var client = new Sftp())
{
await client.ConnectAsync(jobInitResult.UploadUrl, jobInitResult.Port);
await client.LoginAsync(jobInitResult.User, jobInitResult.Password);
await client.PutFileAsync(toBeConverted.FullName, $"{client.GetCurrentDirectory()}{toBeConverted.Name}");
}
}
private async Task MoveAsync(string path, string fileName, string newPathAndFileName)
{
var oldPathAndFileName = Path.Combine(path, fileName).Replace('\\', '/');
var newFullPath = newPathAndFileName.Replace('\\', '/');
using (var sftpClient = new Sftp())
{
sftpClient.TransferType = _transferType;
await sftpClient.ConnectAsync(_serverAddress, _port);
await sftpClient.LoginAsync(_userName, _password);
var dir = Path.GetDirectoryName(newFullPath)?.Replace('\\', '/');
// If the directory hasn't been specified, use user home directory
if (!string.IsNullOrEmpty(dir) && !await sftpClient.DirectoryExistsAsync(dir))
{
// Examine the path step by step and create directories:
StringBuilder directory = null;
foreach (var directoryPart in dir.Split('/'))
{
// First directory should not be preceded by '/':
if (directory == null)
{
directory = new StringBuilder();
directory.Append(directoryPart);
}
else
{
directory.AppendFormat("/{0}", directoryPart);
}
// If this directory does not exist, create it and move to the next part:
var dirString = directory.ToString();
if (!string.IsNullOrWhiteSpace(dirString) && !await sftpClient.DirectoryExistsAsync(dirString))
{
await sftpClient.CreateDirectoryAsync(dirString);
}
}
}
if (await sftpClient.FileExistsAsync(newFullPath))
{
await sftpClient.DeleteFileAsync(newFullPath);
}
await sftpClient.RenameAsync(oldPathAndFileName, newFullPath);
await sftpClient.DisconnectAsync();
}
}
private async static Task <Sftp> Connect(IConfigurationRoot config)
{
var client = new Sftp();
Console.WriteLine($"Connecting to server {config["Server"]}");
await client.ConnectAsync(config["Server"]);
Console.WriteLine("Logging in...");
await client.LoginAsync(config["UserName"], config["Password"]);
Console.WriteLine("Logged In");
return(client);
}
public async Task <XchangeFile> Handle(XchangeFile xchangeFile)
{
Rebex.Licensing.Key = Runner.StartupValueOf(CommonProperties.LicenseKey);
switch (Runner.StartupValueOf(CommonProperties.Protocol).ToLower())
{
case "sftp":
var sftp = new Sftp();
int sftpPort = string.IsNullOrWhiteSpace(Runner.StartupValueOf(CommonProperties.Port)) ? 22 : Convert.ToInt32(Runner.StartupValueOf(CommonProperties.Port));
await sftp.ConnectAsync(Runner.StartupValueOf(CommonProperties.Host), sftpPort);
ftpOrSftp = sftp;
break;
case "ftp":
var ftp = new Rebex.Net.Ftp();
int ftpPort = string.IsNullOrWhiteSpace(Runner.StartupValueOf(CommonProperties.Port)) ? 21 : Convert.ToInt32(Runner.StartupValueOf(CommonProperties.Port));
await ftp.ConnectAsync(Runner.StartupValueOf(CommonProperties.Host), ftpPort);
ftpOrSftp = ftp;
break;
default:
throw new ArgumentException($"Unknown protocol '{Runner.StartupValueOf(CommonProperties.Protocol)}'");
}
await ftpOrSftp.LoginAsync(Runner.StartupValueOf(CommonProperties.Username), Runner.StartupValueOf(CommonProperties.Password));
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(xchangeFile.Data));
//Stream str = stream;
var filename = xchangeFile.Filename;
if (string.IsNullOrWhiteSpace(filename))
{
var currentDate = DateTime.UtcNow;
filename = $"{currentDate.Year:0000}{currentDate.Month:00}{currentDate.Day:00}{currentDate.Hour:00}{currentDate.Minute:00}{currentDate.Second:00}{currentDate.Millisecond:000}";
}
//if (!string.IsNullOrWhiteSpace(Runner.StartupValueOf(CommonProperties.FileNamePrefix)))
// filename += Runner.StartupValueOf(CommonProperties.FileNamePrefix) + "_";
//filename += Runner.StartupValueOf(CommonProperties.FileNamePrefix) + "_" + DateTime.UtcNow.Day + DateTime.UtcNow.Month + DateTime.UtcNow.Year + DateTime.UtcNow.Hour + DateTime.UtcNow.Minute + DateTime.UtcNow.Second + DateTime.UtcNow.Millisecond;
await ftpOrSftp.PutFileAsync(stream, Runner.StartupValueOf($"{CommonProperties.TargetPath}/{filename}"));
await ftpOrSftp.DisconnectAsync();
return(new XchangeFile(string.Empty));
}
public async Task DeleteFileAsync(string path, string fileName)
{
using (var sftpClient = new Sftp())
{
sftpClient.TransferType = _transferType;
await sftpClient.ConnectAsync(_serverAddress, _port);
await sftpClient.LoginAsync(_userName, _password);
await sftpClient.DeleteAsync(Path.Combine(path, fileName).Replace('\\', '/'), Rebex.IO.TraversalMode.MatchFilesShallow);
await sftpClient.DisconnectAsync();
}
}
private async Task <DownloadAndStoreFileResult> DownloadAndStoreFile(ConversionStartResult documentConversionResult,
FileSystemInfo targetFolder)
{
Licensing.Key = sftpLicenseKey;
try
{
using (var client = new Sftp())
{
await client.ConnectAsync(documentConversionResult.UploadUrl, documentConversionResult.Port);
await client.LoginAsync(documentConversionResult.User, documentConversionResult.Password);
var remotePath = $"{client.GetCurrentDirectory()}{documentConversionResult.ConvertedFileName}";
var targetPath = Path.Combine(targetFolder.FullName, $"{documentConversionResult.ConvertedFileName}");
if (!await client.FileExistsAsync(remotePath))
{
throw new FileNotFoundException($"File '{documentConversionResult.ConvertedFileName}' not found on SFTP server");
}
var lengthOfContent = await client.GetFileAsync(remotePath, targetPath);
if (!File.Exists(targetPath))
{
throw new InvalidOperationException($"Was unable to download file {targetPath} from sftp server");
}
return(new DownloadAndStoreFileResult {
LengthOfContent = lengthOfContent, TargetPath = targetPath
});
}
}
catch (Exception e)
{
Log.Error(e, e.Message);
throw;
}
}
public async Task <string> WriteFileAsync(string path, string fileName, Stream messageStream, bool overwriteIfExists = false, bool createDirectory = false)
{
var tempFileName = $".{Guid.NewGuid()}.tmp";
var destinationFilePath = Path.Combine(path, FileNameHelper.PopulateFileNameMacros(fileName)).Replace('\\', '/');
var tempFilePath = Path.Combine(path, tempFileName).Replace('\\', '/');
path = path.Replace('\\', '/');
using (var sftpClient = new Sftp())
{
sftpClient.TransferType = _transferType;
// Connect & authenticate:
await sftpClient.ConnectAsync(_serverAddress, _port);
await sftpClient.LoginAsync(_userName, _password);
// If the directory does not exist, create it if allowed:
if (!await sftpClient.DirectoryExistsAsync(path))
{
if (createDirectory)
{
// Examine the path step by step and create directories:
StringBuilder directory = null;
foreach (var directoryPart in path.Split('/'))
{
// First directory should not be preceded by '/':
if (directory == null)
{
directory = new StringBuilder();
directory.Append(directoryPart);
}
else
{
directory.AppendFormat("/{0}", directoryPart);
}
// If this directory does not exist, create it and move to the next part:
var dirString = directory.ToString();
if (!string.IsNullOrWhiteSpace(dirString) && !await sftpClient.DirectoryExistsAsync(dirString))
{
await sftpClient.CreateDirectoryAsync(dirString);
}
}
}
else
{
throw new Exception($"Directory '{path}' was not found.");
}
}
// Overwrite existing files if allowed:
if (await sftpClient.FileExistsAsync(destinationFilePath))
{
if (overwriteIfExists)
{
await sftpClient.DeleteFileAsync(destinationFilePath);
}
else
{
throw new Exception($"File '{destinationFilePath}' already exists.");
}
}
// Upload the file with a temporary file name:
await sftpClient.PutFileAsync(messageStream, tempFilePath);
await sftpClient.RenameAsync(tempFilePath, destinationFilePath);
await sftpClient.DisconnectAsync();
}
return(tempFileName);
}
private async Task <string[]> DoDownloadAsync(int siteId, SFtpOptions options)
{
var opts = _options.Value ?? options;
if (!opts.IsValid())
{
_logger.LogError($"{nameof(SFtpOptions)} is invalid, config: {opts}.");
return(null);
}
if (siteId <= 0)
{
_logger.LogError($"The {nameof(siteId)}[{siteId}] must be provided.");
return(null);
}
if (Interlocked.CompareExchange(ref _isDownloading, 1, 0) != 0)
{
_logger.LogWarning($"{nameof(SFTPFilesDownloader)} is downloading, please try again later.");
return(null);
}
string[] files = null;
try
{
var sftp = new Sftp();
_logger.LogInformation($"Connecting sftp server[{opts.Host}]...");
if (string.IsNullOrEmpty(opts.Host))
{
_logger.LogError($"The {nameof(opts.Host)}[{opts.Host}] must be provided.");
return(null);
}
await sftp.ConnectAsync(opts.Host, 22);
_logger.LogInformation($"Authenticating sftp user[UserName: {opts.UserName}]...");
switch (opts.AuthScheme)
{
case SFtpOptions.AuthenticateScheme.Password:
if (string.IsNullOrEmpty(opts.Password))
{
_logger.LogError($"In the password authenticate scheme, the password must be provided.");
return(null);
}
await sftp.AuthenticateAsync(opts.UserName, opts.Password);
break;
case SFtpOptions.AuthenticateScheme.SecurityKey:
if (opts.PrivateKey == null ||
!opts.PrivateKey.CanRead)
{
_logger.LogError($"In the security sey authenticate scheme, the Private Key must be provided.");
return(null);
}
var privateKey = new SecureShellPrivateKey(opts.PrivateKey);
await sftp.AuthenticateAsync(opts.UserName, privateKey);
break;
default:
throw new InvalidOperationException($"Only these authenticate schemes[{nameof(SFtpOptions.AuthenticateScheme.Password)},{nameof(SFtpOptions.AuthenticateScheme.SecurityKey)}] supported.");
}
var filter = _namingStrategy.GetFileRegexName();
files = await sftp.ListNameAsync(opts.RemoteDirectory, new NameRegexSearchCondition(filter));
if (files.Length <= 0)
{
_logger.LogWarning($"No reports files found from sftp server.");
return(null);
}
_logger.LogInformation($"The reports[{files.Aggregate((x, y) => $"{x},{y}")}] will be downloaded.");
if (string.IsNullOrEmpty(opts.LocalDirectory))
{
opts.LocalDirectory = Path.Combine(Directory.GetCurrentDirectory(), "reports", siteId.ToString());
}
if (!Directory.Exists(opts.LocalDirectory))
{
Directory.CreateDirectory(opts.LocalDirectory);
}
var hasNewFile = false;
foreach (var file in files)
{
var isDownloaded = await _manager.IsDownloadedAsync(siteId, opts.LocalDirectory, file);
if (isDownloaded)
{
continue;
}
_logger.LogInformation($"The file[{file}] downloading...");
var localPath = Path.Combine(opts.LocalDirectory, file);
if (File.Exists(localPath))
{
File.Delete(localPath);
}
using (var stream = new FileStream(localPath, FileMode.Create))
{
var remoteDir = opts.RemoteDirectory.TrimStart('/').TrimEnd('/').Replace('\\', '/');
var remotePath = $"/{remoteDir}/{file}";
await sftp.DownloadFileAsync(remotePath, stream);
}
hasNewFile = true;
_logger.LogInformation($"The file[{localPath}] was downloaded successfully.");
}
await sftp.DisconnectAsync();
if (hasNewFile)
{
_manager.TryRefreshSavePoint(siteId);
}
return(files.Select(it => Path.Combine(opts.LocalDirectory, it)).ToArray());
}
catch (Exception e)
{
var filesToRemove = files?.Select(it => Path.Combine(opts.LocalDirectory, it)).ToArray();
_manager.TryRemoveSavePoint(siteId, filesToRemove);
_logger.LogError($"{nameof(DownloadAsync)} has a unknown exception, ex: {e}");
return(null);
}
finally
{
_isDownloading = 0;
}
}
