/// <summary>
/// Create all the sub directories within the main directory
/// </summary>
private void CreateSubDirectories(FtpClient client, List <FtpResult> dirsToUpload)
{
foreach (var result in dirsToUpload)
{
// absorb errors
try {
// create directory on the server
// to ensure we upload the blank remote dirs as well
if (client.CreateDirectory(result.RemotePath))
{
result.IsSuccess = true;
result.IsSkipped = false;
}
else
{
result.IsSkipped = true;
}
}
catch (Exception ex) {
// mark that the folder failed to upload
result.IsFailed = true;
result.Exception = ex;
}
}
}
public void ArchiveFile(string oldPath, string newPath, string fileName)
{
try
{
if (Connect())
{
if (!ftp.DirectoryExists(newPath))
{
ftp.CreateDirectory(newPath);
}
ftp.Rename(oldPath + "/" + fileName, newPath + "/" +
Path.GetFileNameWithoutExtension(fileName) + "_" +
DateTime.Now.ToString("yyyyMMddHHmmssffff") + Path.GetExtension(fileName));
}
}
catch (Exception ex)
{
throw ex;
}
}
/// <summary>
/// 新建目录
/// </summary>
/// <param name="remoteDic">远程目录("/test")</param>
/// <returns></returns>
public bool MakeDir(string remoteDic)
{
bool boolResult = false;
try
{
if (Connect())
{
ftpClient.CreateDirectory(remoteDic);
boolResult = true;
}
}
catch (Exception ex)
{
Log4netUtil.Log4NetHelper.Error(String.Format(@"MakeDir->新建目录 异常:{0} |*|remoteDic: {1}", ex.ToString(), remoteDic), @"Ftp");
}
finally
{
DisConnect();
}
return(boolResult);
}
/// <summary>
/// Transfer the specified directory from the source FTP Server onto the remote FTP Server using the FXP protocol.
/// You will need to create a valid connection to your remote FTP Server before calling this method.
/// In Update mode, we will only transfer missing files and preserve any extra files on the remote FTP Server. This is useful when you want to simply transfer missing files from an FTP directory.
/// Currently Mirror mode is not implemented.
/// Only transfers the files and folders matching all the rules provided, if any.
/// All exceptions during transfer are caught, and the exception is stored in the related FtpResult object.
/// </summary>
/// <param name="sourceFolder">The full or relative path to the folder on the source FTP Server. If it does not exist, an empty result list is returned.</param>
/// <param name="remoteClient">Valid FTP connection to the destination FTP Server</param>
/// <param name="remoteFolder">The full or relative path to destination folder on the remote FTP Server</param>
/// <param name="mode">Only Update mode is currently implemented</param>
/// <param name="existsMode">If the file exists on disk, should we skip it, resume the download or restart the download?</param>
/// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param>
/// <param name="rules">Only files and folders that pass all these rules are downloaded, and the files that don't pass are skipped. In the Mirror mode, the files that fail the rules are also deleted from the local folder.</param>
/// <param name="progress">Provide a callback to track download progress.</param>
/// <remarks>
/// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support
/// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful
/// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically switch to true for subsequent attempts.
/// If <see cref="FtpVerify.Throw"/> is set and <see cref="FtpError.Throw"/> is <i>not set</i>, then individual verification errors will not cause an exception
/// to propagate from this method.
/// </remarks>
/// <returns>
/// Returns a listing of all the remote files, indicating if they were downloaded, skipped or overwritten.
/// Returns a blank list if nothing was transfered. Never returns null.
/// </returns>
public List <FtpResult> TransferDirectory(string sourceFolder, FtpClient remoteClient, string remoteFolder, FtpFolderSyncMode mode = FtpFolderSyncMode.Update,
FtpRemoteExists existsMode = FtpRemoteExists.Skip, FtpVerify verifyOptions = FtpVerify.None, List <FtpRule> rules = null, Action <FtpProgress> progress = null)
{
if (sourceFolder.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "sourceFolder");
}
if (remoteFolder.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "remoteFolder");
}
LogFunc(nameof(TransferDirectory), new object[] { sourceFolder, remoteClient, remoteFolder, mode, existsMode, verifyOptions, (rules.IsBlank() ? null : rules.Count + " rules") });
var results = new List <FtpResult>();
// cleanup the FTP paths
sourceFolder = sourceFolder.GetFtpPath().EnsurePostfix("/");
remoteFolder = remoteFolder.GetFtpPath().EnsurePostfix("/");
// if the source dir does not exist, fail fast
if (!DirectoryExists(sourceFolder))
{
return(results);
}
// flag to determine if existence checks are required
var checkFileExistence = true;
// ensure the remote dir exists
if (!remoteClient.DirectoryExists(remoteFolder))
{
remoteClient.CreateDirectory(remoteFolder);
checkFileExistence = false;
}
// collect paths of the files that should exist (lowercase for CI checks)
var shouldExist = new Dictionary <string, bool>();
// get all the folders in the local directory
var dirListing = GetListing(sourceFolder, FtpListOption.Recursive).Where(x => x.Type == FtpFileSystemObjectType.Directory).Select(x => x.FullName).ToArray();
// get all the already existing files
var remoteListing = checkFileExistence ? remoteClient.GetListing(remoteFolder, FtpListOption.Recursive) : null;
// loop thru each folder and ensure it exists
var dirsToUpload = GetSubDirectoriesToTransfer(sourceFolder, remoteFolder, rules, results, dirListing);
CreateSubDirectories(remoteClient, dirsToUpload);
// get all the files in the local directory
var fileListing = GetListing(sourceFolder, FtpListOption.Recursive).Where(x => x.Type == FtpFileSystemObjectType.File).Select(x => x.FullName).ToArray();
// loop thru each file and transfer it
var filesToUpload = GetFilesToTransfer(sourceFolder, remoteFolder, rules, results, shouldExist, fileListing);
TransferServerFiles(filesToUpload, remoteClient, existsMode, verifyOptions, progress, remoteListing);
// delete the extra remote files if in mirror mode and the directory was pre-existing
// DeleteExtraServerFiles(mode, shouldExist, remoteListing);
return(results);
}
/// <summary>
/// Transfers a file from the source FTP Server to the destination FTP Server via the FXP protocol
/// </summary>
private bool TransferFileFXPInternal(string sourcePath, FtpClient remoteClient, string remotePath, bool createRemoteDir, FtpRemoteExists existsMode,
Action <FtpProgress> progress, FtpProgress metaProgress)
{
FtpReply reply;
long offset = 0;
bool fileExists = false;
long fileSize = 0;
var ftpFxpSession = OpenPassiveFXPConnection(remoteClient, progress != null);
if (ftpFxpSession != null)
{
try {
ftpFxpSession.SourceServer.ReadTimeout = (int)TimeSpan.FromMinutes(30.0).TotalMilliseconds;
ftpFxpSession.TargetServer.ReadTimeout = (int)TimeSpan.FromMinutes(30.0).TotalMilliseconds;
// check if the file exists, and skip, overwrite or append
if (existsMode == FtpRemoteExists.AppendNoCheck)
{
offset = remoteClient.GetFileSize(remotePath);
if (offset == -1)
{
offset = 0; // start from the beginning
}
}
else
{
fileExists = remoteClient.FileExists(remotePath);
switch (existsMode)
{
case FtpRemoteExists.Skip:
if (fileExists)
{
LogStatus(FtpTraceLevel.Info, "Skip is selected => Destination file exists => skipping");
//Fix #413 - progress callback isn't called if the file has already been uploaded to the server
//send progress reports
if (progress != null)
{
progress(new FtpProgress(100.0, 0, 0, TimeSpan.FromSeconds(0), sourcePath, remotePath, metaProgress));
}
return(true);
}
break;
case FtpRemoteExists.Overwrite:
if (fileExists)
{
remoteClient.DeleteFile(remotePath);
}
break;
case FtpRemoteExists.Append:
if (fileExists)
{
offset = remoteClient.GetFileSize(remotePath);
if (offset == -1)
{
offset = 0; // start from the beginning
}
}
break;
}
}
fileSize = GetFileSize(sourcePath);
// ensure the remote dir exists .. only if the file does not already exist!
if (createRemoteDir && !fileExists)
{
var dirname = remotePath.GetFtpDirectoryName();
if (!remoteClient.DirectoryExists(dirname))
{
remoteClient.CreateDirectory(dirname);
}
}
if (offset == 0 && existsMode != FtpRemoteExists.AppendNoCheck)
{
// send command to tell the source server to 'send' the file to the destination server
if (!(reply = ftpFxpSession.SourceServer.Execute($"RETR {sourcePath}")).Success)
{
throw new FtpCommandException(reply);
}
//Instruct destination server to store the file
if (!(reply = ftpFxpSession.TargetServer.Execute($"STOR {remotePath}")).Success)
{
throw new FtpCommandException(reply);
}
}
else
{
//tell source server to restart / resume
if (!(reply = ftpFxpSession.SourceServer.Execute($"REST {offset}")).Success)
{
throw new FtpCommandException(reply);
}
// send command to tell the source server to 'send' the file to the destination server
if (!(reply = ftpFxpSession.SourceServer.Execute($"RETR {sourcePath}")).Success)
{
throw new FtpCommandException(reply);
}
//Instruct destination server to append the file
if (!(reply = ftpFxpSession.TargetServer.Execute($"APPE {remotePath}")).Success)
{
throw new FtpCommandException(reply);
}
}
var transferStarted = DateTime.Now;
long lastSize = 0;
var sourceFXPTransferReply = ftpFxpSession.SourceServer.GetReply();
var destinationFXPTransferReply = ftpFxpSession.TargetServer.GetReply();
// while the transfer is not complete
while (!sourceFXPTransferReply.Success || !destinationFXPTransferReply.Success)
{
// send progress reports every 1 second
if (ftpFxpSession.ProgressServer != null)
{
// send progress reports
if (progress != null && fileSize != -1)
{
offset = ftpFxpSession.ProgressServer.GetFileSize(remotePath);
if (offset != -1 && lastSize <= offset)
{
long bytesProcessed = offset - lastSize;
lastSize = offset;
ReportProgress(progress, fileSize, offset, bytesProcessed, DateTime.Now - transferStarted, sourcePath, remotePath, metaProgress);
}
}
}
#if CORE14
Task.Delay(FXPProgressInterval);
#else
Thread.Sleep(FXPProgressInterval);
#endif
}
FtpTrace.WriteLine(FtpTraceLevel.Info, $"FXP transfer of file {sourcePath} has completed");
Noop();
remoteClient.Noop();
ftpFxpSession.Dispose();
return(true);
}
// Fix: catch all exceptions and dispose off the FTP clients if one occurs
catch (Exception ex) {
ftpFxpSession.Dispose();
throw ex;
}
}
else
{
FtpTrace.WriteLine(FtpTraceLevel.Error, "Failed to open FXP passive Connection");
return(false);
}
}
/// <summary>
/// 创建文件夹("/path/that")
/// </summary>
/// <param name="dir">服务器文件夹路径</param>
/// <returns></returns>
public bool CreateDirectory(string dir)
{
return(ftp.CreateDirectory(dir, true));
}
public bool Upload(string rootRemoteDirectory = null)
{
if (ftp == null || !ftp.IsConnected)
{
return(false);
}
if (!string.IsNullOrEmpty(rootRemoteDirectory))
{
ftp.SetWorkingDirectory(rootRemoteDirectory);
}
foreach (var dir in this.directorySet)
{
ftp.CreateDirectory(dir);
}
using (MemoryStream commandBuilder = new MemoryStream())
{
string command = $"#!/bin/sh\ncd \"{rootRemoteDirectory}\"\n";
commandBuilder.Write(Encoding.UTF8.GetBytes(command), 0, command.Length);
long currentPosition = 0;
foreach (var afi in localSet)
{
if (skipFiles != null && Array.IndexOf(skipFiles, Path.GetFileName(afi.FilePath)) != -1)
{
continue;
}
Stream inStream = null;
bool owned = true;
if (!string.IsNullOrEmpty(afi.LocalFilePath))
{
inStream = new FileStream(afi.LocalFilePath, FileMode.Open);
}
else if (afi.FileStream != null)
{
inStream = afi.FileStream;
inStream.Position = 0;
owned = false;
}
else if (this.rootDirectory != null)
{
string inPath = new Uri(this.rootDirectory + "/" + afi.FilePath).LocalPath;
inStream = new FileStream(inPath, FileMode.Open);
}
if (inStream != null)
{
ftp.Upload(inStream, afi.FilePath, FtpExists.NoCheck, false, new Progress <double>(p =>
{
if (p != -1)
{
OnReadProgress(currentPosition + (long)(p * afi.FileSize / 100), totalSize, afi.FilePath);
}
}));
if (owned)
{
inStream.Dispose();
}
command = string.Format("touch -m -t {1} '{0}'\n", afi.FilePath.Replace("'", @"'\''"), afi.ModifiedTime.ToString("yyyyMMddHHmm.ss"));
commandBuilder.Write(Encoding.UTF8.GetBytes(command), 0, command.Length);
}
else
{
Trace.WriteLine($"Error with file \"{afi.FilePath}\", no input data.");
OnFileInputError(afi.FilePath);
}
currentPosition += afi.FileSize;
OnReadProgress(currentPosition, totalSize, afi.FilePath);
}
// adjust time pointers!
hakchi.RunTemporaryScript(commandBuilder, "modtime.sh");
}
return(true);
}
/// <summary>
/// Create directory in FTP Server
/// </summary>
/// <param name="remotePath">Remote absolute path directory</param>
public void CreateDirectory(string remotePath)
{
_FTPClient.CreateDirectory(remotePath);
}
public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > securePasswordCallBack,
Func <string, string, string, MemoryStream, string, string, bool> copyFileFunc)
{
var response = new FtpClientResponse {
ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
};
try
{
if (_ftpClient == null)
{
await CreateFtpClientAsync(config, securePasswordCallBack);
}
else
{
if (!_ftpClient.IsConnected)
{
await CreateFtpClientAsync(config, securePasswordCallBack);
}
}
var listOfFiles = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.GetListing(config.Ftp.Path).Where(f => f.Type == FtpFileSystemObjectType.File), new Polly.Context($"LIST_FILE_{config.Ftp.Path}"));
//config.Ftp.ArchivedPath != config.Ftp.Path is delete mode
if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
{
var folderExisted = _ftpClient.DirectoryExists(config.Ftp.ArchivedPath);
if (!folderExisted)
{
_ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.CreateDirectory(config.Ftp.ArchivedPath), new Polly.Context("CREATE_ARCHIVE_FOLDER"));
}
}
var resolvedStorageAccountKey =
await securePasswordCallBack.Invoke(config.AzureBlobStorage.StorageKeyVault);
Parallel.ForEach(
listOfFiles,
new ParallelOptions {
MaxDegreeOfParallelism = config.Parellelism
},
item =>
{
try
{
var fileName = item.Name;
using (var istream = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.OpenRead(item.FullName), new Polly.Context($"READ_FILE_{item.FullName}")))
{
bool isArchived = false;
MemoryStream memoryStream = new MemoryStream();
istream.CopyTo(memoryStream);
memoryStream.Position = 0;
//_telemetryLogger.TrackTrace<FtpActor>("", "[ActorFtpClient] Reading File " + item.FullName, SeverityLevel.Information);
var copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName,
fileName,
config.Retry.StorageRetryPolicy, memoryStream, config.AzureBlobStorage.StorageAccountName, resolvedStorageAccountKey);
if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
{
isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.MoveFile(item.FullName, config.Ftp.ArchivedPath + "\\" + item.Name, FtpExists.Overwrite), new Polly.Context("COPY_FILE_TO_ARCHIVE"));
}
else if (config.Ftp.ArchivedPath == config.Ftp.Path)
{
//delete mode
isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) =>
{
_ftpClient.DeleteFile(item.FullName);
return(true);
}, new Polly.Context("DELETE_FILE_TO_ARCHIVE"));
}
var fres = new FtpClientResponseFile
{
Status = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus,
FileName = item.Name,
ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError)
};
var properties = new Dictionary <string, string> {
{ "payload", JsonConvert.SerializeObject(fres) }
};
//_telemetryLogger.TrackEvent("", EventConstant.FtpCustomEventName, properties);
response.FileData.Add(fres);
}
}
catch (Exception)
{
throw;
}
}
);
//_telemetryLogger.TrackMetric("", EventConstant.FtpCustomEventName, 1);
await Stop();
}
catch (FtpNotConnectException e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
}
catch (FtpSecurityNotAvailableException e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
}
catch (Exception e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
}
return(response);
}
public async Task <FtpClientResponse> ReadAsync(FtpConfig config, Func <string, Task <string> > securePasswordCallBack,
Func <string, string, string, MemoryStream, string, string, bool> copyFileFunc)
{
var response = new FtpClientResponse {
ErrorMessage = FtpConstants.NoError, Status = FtpConstants.SuccessStatus, FileData = new List <FtpClientResponseFile>()
};
try
{
await CreateFtpClientAsync(config, securePasswordCallBack);
Func <FtpListItem, bool> predicate = f => f.Type == FtpFileSystemObjectType.File;
if (!string.IsNullOrEmpty(config.Ftp.FilenameRegex))
{
var rx = new Regex(config.Ftp.FilenameRegex, RegexOptions.IgnoreCase);
predicate = f => f.Type == FtpFileSystemObjectType.File && rx.IsMatch(f.Name);
}
var listOfFiles = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy)
.Execute((context) => _ftpClient.GetListing(config.Ftp.Path).Where(predicate), new Polly.Context($"LIST_FILE_{config.Ftp.Path}"));
if (!string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
{
var folderExisted = _ftpClient.DirectoryExists(config.Ftp.ArchivedPath);
if (!folderExisted)
{
_ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.CreateDirectory(config.Ftp.ArchivedPath), new Polly.Context("CREATE_ARCHIVE_FOLDER"));
}
}
var resolvedStorageAccountKey =
await securePasswordCallBack.Invoke(config.AzureBlobStorage.StorageKeyVault);
foreach (var item in listOfFiles)
{
bool copied = false;
bool isArchived = false;
try
{
var fileName = item.Name;
using (var istream = await _ftpClient.OpenReadAsync(item.FullName))
{
MemoryStream memoryStream = new MemoryStream();
istream.CopyTo(memoryStream);
memoryStream.Position = 0;
copied = copyFileFunc.Invoke(config.AzureBlobStorage.ContainerName,
fileName,
config.Retry.StorageRetryPolicy, memoryStream, config.AzureBlobStorage.StorageAccountName, resolvedStorageAccountKey);
}
if (copied && !string.IsNullOrEmpty(config.Ftp.ArchivedPath) && config.Ftp.ArchivedPath != config.Ftp.Path)
{
isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) => _ftpClient.MoveFile(item.FullName, config.Ftp.ArchivedPath + "\\" + item.Name, FtpExists.Overwrite), new Polly.Context("COPY_FILE_TO_ARCHIVE"));
}
else if (config.Ftp.ArchivedPath == config.Ftp.Path)
{
//delete mode
isArchived = _ftpPolicyRegistry.GetPolicy(FtpPolicyName.FtpCommandPolicy).Execute((context) =>
{
_ftpClient.DeleteFile(item.FullName);
return(true);
}, new Polly.Context("DELETE_FILE_TO_ARCHIVE"));
}
var fres = new FtpClientResponseFile
{
Status = (copied && isArchived) ? FtpConstants.SuccessStatus : FtpConstants.FailureStatus,
FileName = item.Name,
ErrorMessage = (copied ? string.Empty : FtpConstants.BlobUploadError) + " " + (isArchived ? string.Empty : FtpConstants.ArchivedError)
};
response.FileData.Add(fres);
}
catch (Exception ex)
{
_logger.LogError(ex, $"ActorFtpClient failed to process the file {item.Name}, copy: {copied}, archived {isArchived}. Error: {ex.Message}");
}
}
await Stop();
}
catch (Exception e)
{
response.ErrorMessage = response.ErrorMessage + e.Message;
response.Status = FtpConstants.FailureStatus;
_logger.LogError(e, $"ActorFtpClient failed to start creating the client of {config.Ftp.Host} : {e.Message}");
}
return(response);
}