public async Task <FtpClientResponse> WriteAsync(FtpConfig config, Func <string, Task <string> > securePasswordFunc, byte[] data, string fileName)
{
var response = new FtpClientResponse {
ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
};
try
{
await CreateFtpClientAsync(config, securePasswordFunc);
_ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy)
.Execute((context) => _ftpClient.Upload(data, $"{config.Ftp.Path}/{fileName}"),
new Polly.Context("WRITE_FILE"));
}
catch (FtpNotConnectException e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
}
catch (FtpSecurityNotAvailableException e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
}
catch (Exception)
{
throw;
}
return(response);
}
private async Task CreateFtpClientAsync(FtpConfig config, Func <string, Task <string> > securePasswordCallBack)
{
var passwd = await securePasswordCallBack.Invoke(config.Ftp.Credentials.AzureKeyVault.SecretName);
_ftpClient = (config.Ftp.Protocol.Equals(FtpConstants.Ftps, StringComparison.InvariantCultureIgnoreCase) || config.Ftp.Port.Equals(990)) ?
new FluentFTP.FtpClient(config.Ftp.Host)
{ //FTPS explicitly or Port = 990 => automatically create FTPS
Credentials = new NetworkCredential(config.Ftp.Credentials.Username, passwd),
Port = config.Ftp.Port,
EnableThreadSafeDataConnections = true,
DataConnectionType = config.Ftp.DataConnectionType == FtpConstants.FtpActiveMode ? FtpDataConnectionType.AutoActive : FtpDataConnectionType.AutoPassive,
EncryptionMode = config.Ftp.Port.Equals(990) ? FtpEncryptionMode.Implicit : FtpEncryptionMode.Explicit,
DataConnectionReadTimeout = 30000
}
:
new FluentFTP.FtpClient(config.Ftp.Host)
{
Credentials = new NetworkCredential(config.Ftp.Credentials.Username, await securePasswordCallBack.Invoke(config.Ftp.Credentials.AzureKeyVault.SecretName)),
Port = config.Ftp.Port,
EnableThreadSafeDataConnections = true,
Encoding = Encoding.UTF8,
DataConnectionType = config.Ftp.DataConnectionType == FtpConstants.FtpActiveMode ? FtpDataConnectionType.AutoActive : FtpDataConnectionType.AutoPassive,
DataConnectionReadTimeout = 30000
};
_ftpClient.ValidateCertificate += (control, e) =>
{
e.Accept = (string.IsNullOrEmpty(config.Ftp.ValidateServerCertificate) ||
config.Ftp.ValidateServerCertificate.Equals("false", StringComparison.InvariantCultureIgnoreCase)) ? true
: (e.Certificate.GetRawCertDataString() == config.Ftp.ValidateServerCertificate || e.PolicyErrors == SslPolicyErrors.None);
};
_ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpConnectPolicy).Execute(() => _ftpClient.Connect());
}
private async Task <ConnectionInfo> SetUpSftpAsync(FtpConfig config, Func <string, Task <string> > securePassword)
{
try
{
var passAuthenMethod = new PasswordAuthenticationMethod(config.Ftp.Credentials.Username, await securePassword.Invoke(config.Ftp.Credentials.AzureKeyVault.SecretName));
var privateKeyAuthenMethod = new PrivateKeyAuthenticationMethod(config.Ftp.Credentials.Username);
var connectionInfo = new ConnectionInfo(config.Ftp.Host, config.Ftp.Port, config.Ftp.Credentials.Username, passAuthenMethod, privateKeyAuthenMethod);
return(connectionInfo);
}
catch (Exception ex)
{
throw ex;
}
}
public async Task <FtpClientResponse> WriteAsync(FtpConfig config, Func <string, Task <string> > securePasswordFunc, byte[] data, string fileName)
{
var response = new FtpClientResponse {
ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
};
var connectionInfo = await SetUpSftpAsync(config, securePasswordFunc);
if (config.Encoding == null)
{
connectionInfo.Encoding = Encoding.UTF8;
}
else
{
if (config.Encoding == CommonConstants.ENCODING_DEFAULT_VALUE_TEXT)
{
connectionInfo.Encoding = Encoding.Default;
}
else
{
connectionInfo.Encoding = Encoding.GetEncoding(config.Encoding);
}
}
_sftpClient = new SftpClient(connectionInfo);
using (_sftpClient)
{
_ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpConnectPolicy).Execute(() => _sftpClient.Connect());
var stream = new MemoryStream();
stream.Write(data, 0, data.Length);
stream.Position = 0;
_ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute(() =>
{
_sftpClient.UploadFile(stream, $"{config.Ftp.Path}/{fileName}");
});
}
return(response);
}
public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > myFunc, Func <string, string, string, MemoryStream, string, string, bool> copyFileFunc)
{
var response = new FtpClientResponse {
ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
};
try
{
var connectionInfo = await SetUpSftpAsync(config, myFunc);
_sftpClient = new SftpClient(connectionInfo);
using (_sftpClient)
{
_ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpConnectPolicy).Execute(() => _sftpClient.Connect());
var listOfFiles = _sftpClient.ListDirectory(config.Ftp.Path).Where(f => !f.IsDirectory)
.Select(f => f)
.ToList();
if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath))
{
var folderExisted = _sftpClient.Exists(config.Ftp.ArchivedPath);
if (!folderExisted)
{
_sftpClient.CreateDirectory(config.Ftp.ArchivedPath);
}
}
var resolvedStorageAccountKey =
await myFunc.Invoke(config.AzureBlobStorage.StorageKeyVault);
foreach (var item in listOfFiles)
{
bool copied = false;
bool isArchived = false;
try
{
var fileName = item.Name;
using (var istream = _sftpClient.OpenRead(item.FullName))
{
MemoryStream memoryStream = new MemoryStream();
istream.CopyTo(memoryStream);
memoryStream.Position = 0;
copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName,
fileName,
config.Retry.StorageRetryPolicy, memoryStream, config.AzureBlobStorage.StorageAccountName, resolvedStorageAccountKey);
if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath))
{
var fileNameSplit = item.Name.Split('.');
var newFileName =
fileNameSplit[0] + "_" + DateTime.UtcNow.ToString("MMddyyyy_HHmmss") + "." +
fileNameSplit[1];
var path = config.Ftp.ArchivedPath + "/" + newFileName;
item.MoveTo(path);
isArchived = true;
}
var fres = new FtpClientResponseFile
{
Status = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus,
FileName = fileName,
ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError)
};
response.FileData.Add(fres);
}
}
catch (Exception ex)
{
_logger.LogError(ex, $"{nameof(ActorSftpClient)} failed to process the file {item.Name}, copy: {copied}, archived {isArchived}. Error: {ex.Message}");
}
}
}
await Stop();
}
catch (Exception e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
_logger.LogError(e, $"{nameof(ActorSftpClient)} failed to start creating the client of {config.Ftp.Host} : {e.Message}");
}
return(response);
}
public Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > securePassword, Func <string, string, string, MemoryStream, bool> copyFileFunc)
{
throw new NotImplementedException();
}
public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > securePasswordCallBack,
Func <string, string, string, MemoryStream, string, string, bool> copyFileFunc)
{
var response = new FtpClientResponse {
ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
};
try
{
if (_ftpClient == null)
{
await CreateFtpClientAsync(config, securePasswordCallBack);
}
else
{
if (!_ftpClient.IsConnected)
{
await CreateFtpClientAsync(config, securePasswordCallBack);
}
}
var listOfFiles = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.GetListing(config.Ftp.Path).Where(f => f.Type == FtpFileSystemObjectType.File), new Polly.Context($"LIST_FILE_{config.Ftp.Path}"));
//config.Ftp.ArchivedPath != config.Ftp.Path is delete mode
if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
{
var folderExisted = _ftpClient.DirectoryExists(config.Ftp.ArchivedPath);
if (!folderExisted)
{
_ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.CreateDirectory(config.Ftp.ArchivedPath), new Polly.Context("CREATE_ARCHIVE_FOLDER"));
}
}
var resolvedStorageAccountKey =
await securePasswordCallBack.Invoke(config.AzureBlobStorage.StorageKeyVault);
Parallel.ForEach(
listOfFiles,
new ParallelOptions {
MaxDegreeOfParallelism = config.Parellelism
},
item =>
{
try
{
var fileName = item.Name;
using (var istream = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.OpenRead(item.FullName), new Polly.Context($"READ_FILE_{item.FullName}")))
{
bool isArchived = false;
MemoryStream memoryStream = new MemoryStream();
istream.CopyTo(memoryStream);
memoryStream.Position = 0;
//_telemetryLogger.TrackTrace<FtpActor>("", "[ActorFtpClient] Reading File " + item.FullName, SeverityLevel.Information);
var copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName,
fileName,
config.Retry.StorageRetryPolicy, memoryStream, config.AzureBlobStorage.StorageAccountName, resolvedStorageAccountKey);
if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
{
isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.MoveFile(item.FullName, config.Ftp.ArchivedPath + "\\" + item.Name, FtpExists.Overwrite), new Polly.Context("COPY_FILE_TO_ARCHIVE"));
}
else if (config.Ftp.ArchivedPath == config.Ftp.Path)
{
//delete mode
isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) =>
{
_ftpClient.DeleteFile(item.FullName);
return(true);
}, new Polly.Context("DELETE_FILE_TO_ARCHIVE"));
}
var fres = new FtpClientResponseFile
{
Status = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus,
FileName = item.Name,
ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError)
};
var properties = new Dictionary <string, string> {
{ "payload", JsonConvert.SerializeObject(fres) }
};
//_telemetryLogger.TrackEvent("", EventConstant.FtpCustomEventName, properties);
response.FileData.Add(fres);
}
}
catch (Exception)
{
throw;
}
}
);
//_telemetryLogger.TrackMetric("", EventConstant.FtpCustomEventName, 1);
await Stop();
}
catch (FtpNotConnectException e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
}
catch (FtpSecurityNotAvailableException e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
}
catch (Exception e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
}
return(response);
}
public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > securePasswordCallBack,
Func <string, string, string, MemoryStream, string, string, bool> copyFileFunc)
{
var response = new FtpClientResponse {
ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
};
try
{
await CreateFtpClientAsync(config, securePasswordCallBack);
Func <FtpListItem, bool> predicate = f => f.Type == FtpFileSystemObjectType.File;
if (!string.IsNullOrEmpty(config.Ftp.FilenameRegex))
{
var rx = new Regex(config.Ftp.FilenameRegex, RegexOptions.IgnoreCase);
predicate = f => f.Type == FtpFileSystemObjectType.File && rx.IsMatch(f.Name);
}
var listOfFiles = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy)
.Execute((context) => _ftpClient.GetListing(config.Ftp.Path).Where(predicate), new Polly.Context($"LIST_FILE_{config.Ftp.Path}"));
if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
{
var folderExisted = _ftpClient.DirectoryExists(config.Ftp.ArchivedPath);
if (!folderExisted)
{
_ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.CreateDirectory(config.Ftp.ArchivedPath), new Polly.Context("CREATE_ARCHIVE_FOLDER"));
}
}
var resolvedStorageAccountKey =
await securePasswordCallBack.Invoke(config.AzureBlobStorage.StorageKeyVault);
foreach (var item in listOfFiles)
{
bool copied = false;
bool isArchived = false;
try
{
var fileName = item.Name;
using (var istream = await _ftpClient.OpenReadAsync(item.FullName))
{
MemoryStream memoryStream = new MemoryStream();
istream.CopyTo(memoryStream);
memoryStream.Position = 0;
copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName,
fileName,
config.Retry.StorageRetryPolicy, memoryStream, config.AzureBlobStorage.StorageAccountName, resolvedStorageAccountKey);
}
if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
{
isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.MoveFile(item.FullName, config.Ftp.ArchivedPath + "\\" + item.Name, FtpExists.Overwrite), new Polly.Context("COPY_FILE_TO_ARCHIVE"));
}
else if (config.Ftp.ArchivedPath == config.Ftp.Path)
{
//delete mode
isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) =>
{
_ftpClient.DeleteFile(item.FullName);
return(true);
}, new Polly.Context("DELETE_FILE_TO_ARCHIVE"));
}
var fres = new FtpClientResponseFile
{
Status = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus,
FileName = item.Name,
ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError)
};
response.FileData.Add(fres);
}
catch (Exception ex)
{
_logger.LogError(ex, $"ActorFtpClient failed to process the file {item.Name}, copy: {copied}, archived {isArchived}. Error: {ex.Message}");
}
}
await Stop();
}
catch (Exception e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
_logger.LogError(e, $"ActorFtpClient failed to start creating the client of {config.Ftp.Host} : {e.Message}");
}
return(response);
}
public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > securePasswordCallBack, Func <string, string, string, MemoryStream, bool> copyFileFunc)
{
var response = new FtpClientResponse {
ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
};
try
{
await CreateFtpClientAsync(config, securePasswordCallBack);
Func <FtpListItem, bool> predicate = f => f.Type == FtpFileSystemObjectType.File;
if (!string.IsNullOrEmpty(config.Ftp.FilenameRegex))
{
var rx = new Regex(config.Ftp.FilenameRegex, RegexOptions.IgnoreCase);
predicate = f => f.Type == FtpFileSystemObjectType.File && rx.IsMatch(f.Name);
}
var listOfFiles = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy)
.Execute((context) => _ftpClient.GetListing(config.Ftp.Path).Where(predicate), new Polly.Context($"LIST_FILE_{config.Ftp.Path}"));
if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath))
{
var folderExisted = _ftpClient.DirectoryExists(config.Ftp.ArchivedPath);
if (!folderExisted)
{
_ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.CreateDirectory(config.Ftp.ArchivedPath), new Polly.Context("CREATE_ARCHIVE_FOLDER"));
}
}
Parallel.ForEach(
listOfFiles,
new ParallelOptions {
MaxDegreeOfParallelism = config.Parellelism
},
item =>
{
try
{
var fileName = item.FullName.Remove(0, 1);
using (var istream = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.OpenRead(fileName), new Polly.Context($"READ_FILE_{item.FullName}")))
{
bool isArchived = false;
MemoryStream memoryStream = new MemoryStream();
istream.CopyTo(memoryStream);
memoryStream.Position = 0;
//_telemetryLogger.TrackTrace<FtpActor>("", "[ActorFtpClient] Reading File " + item.FullName, SeverityLevel.Information);
var copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName,
fileName,
config.Retry.StorageRetryPolicy, memoryStream);
if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath))
{
isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.MoveFile(item.FullName, config.Ftp.ArchivedPath + "\\" + item.Name, FtpExists.Overwrite), new Polly.Context("COPY_FILE_TO_ARCHIVE"));
}
var fres = new FtpClientResponseFile
{
Status = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus,
FileName = item.Name,
ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError)
};
response.FileData.Add(fres);
}
}
catch (Exception)
{
throw;
}
}
);
await Stop();
}
catch (FtpNotConnectException e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
}
catch (FtpSecurityNotAvailableException e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
}
catch (Exception e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
}
return(response);
}
public FtpWriterOption(FtpConfig ftpConfig, string writeData, string fileName) : base(ftpConfig)
{
WriteData = writeData;
FileName = fileName;
}
public FtpOption(FtpConfig ftpConfig)
{
FtpConfig = ftpConfig;
}