/// <summary>
/// Remove File on FTP server
/// </summary>
/// <param name="pathFileOnFTP">absolute path where is the file to remove</param>
/// <returns></returns>
public bool RemoveFileOnFTPServer(string pathFileOnFTP)
{
_FTPClient.GetListing("/");
if (_FTPClient.FileExists(pathFileOnFTP))
{
_FTPClient.DeleteFile(pathFileOnFTP);
}
return(!_FTPClient.FileExists(pathFileOnFTP));
}
/// <summary>
/// Transfer the specified file from the source FTP Server to the destination FTP Server using the FXP protocol.
/// High-level API that takes care of various edge cases internally.
/// </summary>
/// <param name="sourcePath">The full or relative path to the file on the source FTP Server</param>
/// <param name="remoteClient">Valid FTP connection to the destination FTP Server</param>
/// <param name="remotePath">The full or relative path to destination file on the remote FTP Server</param>
/// <param name="createRemoteDir">Indicates if the folder should be created on the remote FTP Server</param>
/// <param name="existsMode">If the file exists on disk, should we skip it, resume the download or restart the download?</param>
/// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param>
/// <param name="progress">Provide a callback to track download progress.</param>
/// Returns a FtpStatus indicating if the file was transfered.
/// <remarks>
/// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support
/// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful
/// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically be set to true for subsequent attempts.
/// </remarks>
public FtpStatus TransferFile(string sourcePath, FtpClient remoteClient, string remotePath,
bool createRemoteDir = false, FtpRemoteExists existsMode = FtpRemoteExists.Resume, FtpVerify verifyOptions = FtpVerify.None, Action <FtpProgress> progress = null, FtpProgress metaProgress = null)
{
sourcePath = sourcePath.GetFtpPath();
remotePath = remotePath.GetFtpPath();
LogFunc(nameof(TransferFile), new object[] { sourcePath, remoteClient, remotePath, FXPDataType, createRemoteDir, existsMode, verifyOptions });
// verify input params
VerifyTransferFileParams(sourcePath, remoteClient, remotePath, existsMode);
// ensure source file exists
if (!FileExists(sourcePath))
{
throw new FtpException("Source File " + sourcePath + " cannot be found or does not exists!");
}
bool fxpSuccess;
var verified = true;
var attemptsLeft = verifyOptions.HasFlag(FtpVerify.Retry) ? m_retryAttempts : 1;
do
{
fxpSuccess = TransferFileFXPInternal(sourcePath, remoteClient, remotePath, createRemoteDir, existsMode, progress, metaProgress is null ? new FtpProgress(1, 0) : metaProgress);
attemptsLeft--;
// if verification is needed
if (fxpSuccess && verifyOptions != FtpVerify.None)
{
verified = VerifyFXPTransfer(sourcePath, remoteClient, remotePath);
LogStatus(FtpTraceLevel.Info, "File Verification: " + (verified ? "PASS" : "FAIL"));
if (!verified && attemptsLeft > 0)
{
LogStatus(FtpTraceLevel.Verbose, "Retrying due to failed verification." + (existsMode == FtpRemoteExists.Resume ? " Overwrite will occur." : "") + " " + attemptsLeft + " attempts remaining");
// Force overwrite if a retry is required
existsMode = FtpRemoteExists.Overwrite;
}
}
} while (!verified && attemptsLeft > 0);
if (fxpSuccess && !verified && verifyOptions.HasFlag(FtpVerify.Delete))
{
remoteClient.DeleteFile(remotePath);
}
if (fxpSuccess && !verified && verifyOptions.HasFlag(FtpVerify.Throw))
{
throw new FtpException("Destination file checksum value does not match source file");
}
return(fxpSuccess && verified ? FtpStatus.Success : FtpStatus.Failed);
}
/// <summary>
/// 删除服务器文件
/// </summary>
/// <param name="file">文件路径</param>
/// <returns>是否成功</returns>
public bool DeleteFile(string file)
{
try
{
ftp.DeleteFile(file);
return(true);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
return(false);
}
}
/// <summary>
/// Transfers a file from the source FTP Server to the destination FTP Server via the FXP protocol
/// </summary>
private bool TransferFileFXPInternal(string sourcePath, FtpClient remoteClient, string remotePath, bool createRemoteDir, FtpRemoteExists existsMode,
Action <FtpProgress> progress, FtpProgress metaProgress)
{
FtpReply reply;
long offset = 0;
bool fileExists = false;
long fileSize = 0;
var ftpFxpSession = OpenPassiveFXPConnection(remoteClient, progress != null);
if (ftpFxpSession != null)
{
try {
ftpFxpSession.SourceServer.ReadTimeout = (int)TimeSpan.FromMinutes(30.0).TotalMilliseconds;
ftpFxpSession.TargetServer.ReadTimeout = (int)TimeSpan.FromMinutes(30.0).TotalMilliseconds;
// check if the file exists, and skip, overwrite or append
if (existsMode == FtpRemoteExists.AppendNoCheck)
{
offset = remoteClient.GetFileSize(remotePath);
if (offset == -1)
{
offset = 0; // start from the beginning
}
}
else
{
fileExists = remoteClient.FileExists(remotePath);
switch (existsMode)
{
case FtpRemoteExists.Skip:
if (fileExists)
{
LogStatus(FtpTraceLevel.Info, "Skip is selected => Destination file exists => skipping");
//Fix #413 - progress callback isn't called if the file has already been uploaded to the server
//send progress reports
if (progress != null)
{
progress(new FtpProgress(100.0, 0, 0, TimeSpan.FromSeconds(0), sourcePath, remotePath, metaProgress));
}
return(true);
}
break;
case FtpRemoteExists.Overwrite:
if (fileExists)
{
remoteClient.DeleteFile(remotePath);
}
break;
case FtpRemoteExists.Append:
if (fileExists)
{
offset = remoteClient.GetFileSize(remotePath);
if (offset == -1)
{
offset = 0; // start from the beginning
}
}
break;
}
}
fileSize = GetFileSize(sourcePath);
// ensure the remote dir exists .. only if the file does not already exist!
if (createRemoteDir && !fileExists)
{
var dirname = remotePath.GetFtpDirectoryName();
if (!remoteClient.DirectoryExists(dirname))
{
remoteClient.CreateDirectory(dirname);
}
}
if (offset == 0 && existsMode != FtpRemoteExists.AppendNoCheck)
{
// send command to tell the source server to 'send' the file to the destination server
if (!(reply = ftpFxpSession.SourceServer.Execute($"RETR {sourcePath}")).Success)
{
throw new FtpCommandException(reply);
}
//Instruct destination server to store the file
if (!(reply = ftpFxpSession.TargetServer.Execute($"STOR {remotePath}")).Success)
{
throw new FtpCommandException(reply);
}
}
else
{
//tell source server to restart / resume
if (!(reply = ftpFxpSession.SourceServer.Execute($"REST {offset}")).Success)
{
throw new FtpCommandException(reply);
}
// send command to tell the source server to 'send' the file to the destination server
if (!(reply = ftpFxpSession.SourceServer.Execute($"RETR {sourcePath}")).Success)
{
throw new FtpCommandException(reply);
}
//Instruct destination server to append the file
if (!(reply = ftpFxpSession.TargetServer.Execute($"APPE {remotePath}")).Success)
{
throw new FtpCommandException(reply);
}
}
var transferStarted = DateTime.Now;
long lastSize = 0;
var sourceFXPTransferReply = ftpFxpSession.SourceServer.GetReply();
var destinationFXPTransferReply = ftpFxpSession.TargetServer.GetReply();
// while the transfer is not complete
while (!sourceFXPTransferReply.Success || !destinationFXPTransferReply.Success)
{
// send progress reports every 1 second
if (ftpFxpSession.ProgressServer != null)
{
// send progress reports
if (progress != null && fileSize != -1)
{
offset = ftpFxpSession.ProgressServer.GetFileSize(remotePath);
if (offset != -1 && lastSize <= offset)
{
long bytesProcessed = offset - lastSize;
lastSize = offset;
ReportProgress(progress, fileSize, offset, bytesProcessed, DateTime.Now - transferStarted, sourcePath, remotePath, metaProgress);
}
}
}
#if CORE14
Task.Delay(FXPProgressInterval);
#else
Thread.Sleep(FXPProgressInterval);
#endif
}
FtpTrace.WriteLine(FtpTraceLevel.Info, $"FXP transfer of file {sourcePath} has completed");
Noop();
remoteClient.Noop();
ftpFxpSession.Dispose();
return(true);
}
// Fix: catch all exceptions and dispose off the FTP clients if one occurs
catch (Exception ex) {
ftpFxpSession.Dispose();
throw ex;
}
}
else
{
FtpTrace.WriteLine(FtpTraceLevel.Error, "Failed to open FXP passive Connection");
return(false);
}
}
public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > securePasswordCallBack,
Func <string, string, string, MemoryStream, string, string, bool> copyFileFunc)
{
var response = new FtpClientResponse {
ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
};
try
{
if (_ftpClient == null)
{
await CreateFtpClientAsync(config, securePasswordCallBack);
}
else
{
if (!_ftpClient.IsConnected)
{
await CreateFtpClientAsync(config, securePasswordCallBack);
}
}
var listOfFiles = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.GetListing(config.Ftp.Path).Where(f => f.Type == FtpFileSystemObjectType.File), new Polly.Context($"LIST_FILE_{config.Ftp.Path}"));
//config.Ftp.ArchivedPath != config.Ftp.Path is delete mode
if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
{
var folderExisted = _ftpClient.DirectoryExists(config.Ftp.ArchivedPath);
if (!folderExisted)
{
_ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.CreateDirectory(config.Ftp.ArchivedPath), new Polly.Context("CREATE_ARCHIVE_FOLDER"));
}
}
var resolvedStorageAccountKey =
await securePasswordCallBack.Invoke(config.AzureBlobStorage.StorageKeyVault);
Parallel.ForEach(
listOfFiles,
new ParallelOptions {
MaxDegreeOfParallelism = config.Parellelism
},
item =>
{
try
{
var fileName = item.Name;
using (var istream = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.OpenRead(item.FullName), new Polly.Context($"READ_FILE_{item.FullName}")))
{
bool isArchived = false;
MemoryStream memoryStream = new MemoryStream();
istream.CopyTo(memoryStream);
memoryStream.Position = 0;
//_telemetryLogger.TrackTrace<FtpActor>("", "[ActorFtpClient] Reading File " + item.FullName, SeverityLevel.Information);
var copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName,
fileName,
config.Retry.StorageRetryPolicy, memoryStream, config.AzureBlobStorage.StorageAccountName, resolvedStorageAccountKey);
if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
{
isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.MoveFile(item.FullName, config.Ftp.ArchivedPath + "\\" + item.Name, FtpExists.Overwrite), new Polly.Context("COPY_FILE_TO_ARCHIVE"));
}
else if (config.Ftp.ArchivedPath == config.Ftp.Path)
{
//delete mode
isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) =>
{
_ftpClient.DeleteFile(item.FullName);
return(true);
}, new Polly.Context("DELETE_FILE_TO_ARCHIVE"));
}
var fres = new FtpClientResponseFile
{
Status = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus,
FileName = item.Name,
ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError)
};
var properties = new Dictionary <string, string> {
{ "payload", JsonConvert.SerializeObject(fres) }
};
//_telemetryLogger.TrackEvent("", EventConstant.FtpCustomEventName, properties);
response.FileData.Add(fres);
}
}
catch (Exception)
{
throw;
}
}
);
//_telemetryLogger.TrackMetric("", EventConstant.FtpCustomEventName, 1);
await Stop();
}
catch (FtpNotConnectException e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
}
catch (FtpSecurityNotAvailableException e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
}
catch (Exception e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
}
return(response);
}
public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > securePasswordCallBack,
Func <string, string, string, MemoryStream, string, string, bool> copyFileFunc)
{
var response = new FtpClientResponse {
ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
};
try
{
await CreateFtpClientAsync(config, securePasswordCallBack);
Func <FtpListItem, bool> predicate = f => f.Type == FtpFileSystemObjectType.File;
if (!string.IsNullOrEmpty(config.Ftp.FilenameRegex))
{
var rx = new Regex(config.Ftp.FilenameRegex, RegexOptions.IgnoreCase);
predicate = f => f.Type == FtpFileSystemObjectType.File && rx.IsMatch(f.Name);
}
var listOfFiles = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy)
.Execute((context) => _ftpClient.GetListing(config.Ftp.Path).Where(predicate), new Polly.Context($"LIST_FILE_{config.Ftp.Path}"));
if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
{
var folderExisted = _ftpClient.DirectoryExists(config.Ftp.ArchivedPath);
if (!folderExisted)
{
_ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.CreateDirectory(config.Ftp.ArchivedPath), new Polly.Context("CREATE_ARCHIVE_FOLDER"));
}
}
var resolvedStorageAccountKey =
await securePasswordCallBack.Invoke(config.AzureBlobStorage.StorageKeyVault);
foreach (var item in listOfFiles)
{
bool copied = false;
bool isArchived = false;
try
{
var fileName = item.Name;
using (var istream = await _ftpClient.OpenReadAsync(item.FullName))
{
MemoryStream memoryStream = new MemoryStream();
istream.CopyTo(memoryStream);
memoryStream.Position = 0;
copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName,
fileName,
config.Retry.StorageRetryPolicy, memoryStream, config.AzureBlobStorage.StorageAccountName, resolvedStorageAccountKey);
}
if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
{
isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.MoveFile(item.FullName, config.Ftp.ArchivedPath + "\\" + item.Name, FtpExists.Overwrite), new Polly.Context("COPY_FILE_TO_ARCHIVE"));
}
else if (config.Ftp.ArchivedPath == config.Ftp.Path)
{
//delete mode
isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) =>
{
_ftpClient.DeleteFile(item.FullName);
return(true);
}, new Polly.Context("DELETE_FILE_TO_ARCHIVE"));
}
var fres = new FtpClientResponseFile
{
Status = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus,
FileName = item.Name,
ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError)
};
response.FileData.Add(fres);
}
catch (Exception ex)
{
_logger.LogError(ex, $"ActorFtpClient failed to process the file {item.Name}, copy: {copied}, archived {isArchived}. Error: {ex.Message}");
}
}
await Stop();
}
catch (Exception e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
_logger.LogError(e, $"ActorFtpClient failed to start creating the client of {config.Ftp.Host} : {e.Message}");
}
return(response);
}
