/// <summary>
/// Downloads the specified directory onto the local file system.
/// In Mirror mode, we will download missing files, and delete any extra files from disk that are not present on the server. This is very useful when creating an exact local backup of an FTP directory.
/// In Update mode, we will only download missing files and preserve any extra files on disk. This is useful when you want to simply download missing files from an FTP directory.
/// Only downloads the files and folders matching all the rules provided, if any.
/// All exceptions during downloading are caught, and the exception is stored in the related FtpResult object.
/// </summary>
/// <param name="localFolder">The full path of the local folder on disk to download into. It is created if it does not exist.</param>
/// <param name="remoteFolder">The full path of the remote FTP folder that you want to download. If it does not exist, an empty result list is returned.</param>
/// <param name="mode">Mirror or Update mode, as explained above</param>
/// <param name="existsMode">If the file exists on disk, should we skip it, resume the download or restart the download?</param>
/// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param>
/// <param name="rules">Only files and folders that pass all these rules are downloaded, and the files that don't pass are skipped. In the Mirror mode, the files that fail the rules are also deleted from the local folder.</param>
/// <param name="progress">Provide an implementation of IProgress to track upload progress.</param>
/// <param name="token">The token that can be used to cancel the entire process</param>
/// <remarks>
/// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support
/// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful
/// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically switch to true for subsequent attempts.
/// If <see cref="FtpVerify.Throw"/> is set and <see cref="FtpError.Throw"/> is <i>not set</i>, then individual verification errors will not cause an exception
/// to propagate from this method.
/// </remarks>
/// <returns>
/// Returns a listing of all the remote files, indicating if they were downloaded, skipped or overwritten.
/// Returns a blank list if nothing was transfered. Never returns null.
/// </returns>
public async Task <List <FtpResult> > DownloadDirectoryAsync(string localFolder, string remoteFolder, FtpFolderSyncMode mode = FtpFolderSyncMode.Update,
FtpLocalExists existsMode = FtpLocalExists.Skip, FtpVerify verifyOptions = FtpVerify.None, List <FtpRule> rules = null, IProgress <FtpProgress> progress = null, CancellationToken token = default(CancellationToken))
{
if (localFolder.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "localFolder");
}
if (remoteFolder.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "remoteFolder");
}
// ensure the local path ends with slash
localFolder = localFolder.EnsurePostfix(Path.DirectorySeparatorChar.ToString());
// cleanup the remote path
remoteFolder = remoteFolder.GetFtpPath().EnsurePostfix("/");
LogFunc(nameof(DownloadDirectoryAsync), new object[] { localFolder, remoteFolder, mode, existsMode, verifyOptions, (rules.IsBlank() ? null : rules.Count + " rules") });
var results = new List <FtpResult>();
// if the dir does not exist, fail fast
if (!await DirectoryExistsAsync(remoteFolder, token))
{
return(results);
}
// ensure the local dir exists
localFolder.EnsureDirectory();
// get all the files in the remote directory
var listing = await GetListingAsync(remoteFolder, FtpListOption.Recursive | FtpListOption.Size, token);
// break if task is cancelled
token.ThrowIfCancellationRequested();
// collect paths of the files that should exist (lowercase for CI checks)
var shouldExist = new Dictionary <string, bool>();
// loop thru each file and transfer it #1
var toDownload = GetFilesToDownload(localFolder, remoteFolder, rules, results, listing, shouldExist);
// break if task is cancelled
token.ThrowIfCancellationRequested();
/*-------------------------------------------------------------------------------------/
* Cancelling after this point would leave the FTP server in an inconsistant state *
*-------------------------------------------------------------------------------------*/
// loop thru each file and transfer it #2
await DownloadServerFilesAsync(toDownload, existsMode, verifyOptions, progress, token);
// delete the extra local files if in mirror mode
DeleteExtraLocalFiles(localFolder, mode, shouldExist, rules);
return(results);
}
public static async Task FtpDownloadFolder(BotData data, string remoteDir, string localDir,
FtpLocalExists existsPolicy = FtpLocalExists.Skip)
{
data.Logger.LogHeader();
var client = GetClient(data);
await client.DownloadDirectoryAsync(localDir, remoteDir, FtpFolderSyncMode.Update, existsPolicy,
FtpVerify.None, null, null, data.CancellationToken).ConfigureAwait(false);
data.Logger.Log($"{remoteDir} downloaded to {localDir}", LogColors.Maize);
}
void IFtpSession.Download(string remotePath, string localPath, FtpLocalExists overwrite, bool recursive)
{
if (string.IsNullOrWhiteSpace(remotePath))
{
throw new ArgumentNullException(nameof(remotePath));
}
if (string.IsNullOrWhiteSpace(localPath))
{
throw new ArgumentNullException(nameof(localPath));
}
FtpObjectType objectType = ((IFtpSession)this).GetObjectType(remotePath);
if (objectType == FtpObjectType.Directory)
{
IEnumerable <Tuple <string, string> > listing = GetRemoteListing(remotePath, localPath, recursive);
foreach (Tuple <string, string> file in listing)
{
string directoryPath = Path.GetDirectoryName(file.Item1);
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
using (Stream fileStream = File.OpenWrite(file.Item1))
{
_sftpClient.DownloadFile(file.Item2, fileStream);
}
}
}
else
{
if (objectType == FtpObjectType.File)
{
if (File.Exists(localPath) && overwrite != FtpLocalExists.Overwrite)
{
throw new IOException("FileExistsException");
}
using (Stream fileStream = File.OpenWrite(localPath))
{
_sftpClient.DownloadFile(remotePath, fileStream);
}
}
else
{
throw new NotImplementedException("UnsupportedObjectTypeException");
}
}
}
/// <summary>
/// Download all the listed files and folders from the main directory
/// </summary>
private async Task DownloadServerFilesAsync(List <FtpResult> toDownload, FtpLocalExists existsMode, FtpVerify verifyOptions, IProgress <FtpProgress> progress, CancellationToken token)
{
LogFunc("DownloadServerFilesAsync", new object[] { toDownload.Count + " files" });
// per object to download
var r = -1;
foreach (var result in toDownload)
{
r++;
if (result.Type == FtpFileSystemObjectType.File)
{
// absorb errors
try {
// create meta progress to store the file progress
var metaProgress = new FtpProgress(toDownload.Count, r);
// download the file
var transferred = await DownloadFileToFileAsync(result.LocalPath, result.RemotePath, existsMode, verifyOptions, progress, token, metaProgress);
result.IsSuccess = true;
result.IsSkipped = !transferred;
}
catch (Exception ex) {
LogStatus(FtpTraceLevel.Warn, "File failed to download: " + result.RemotePath);
// mark that the file failed to download
result.IsFailed = true;
result.Exception = ex;
}
}
else if (result.Type == FtpFileSystemObjectType.Directory)
{
// absorb errors
try {
// create directory on local filesystem
// to ensure we download the blank remote dirs as well
var created = result.LocalPath.EnsureDirectory();
result.IsSuccess = true;
result.IsSkipped = !created;
}
catch (Exception ex) {
// mark that the file failed to download
result.IsFailed = true;
result.Exception = ex;
}
}
}
}
/// <summary>
/// Downloads the specified directory onto the local file system.
/// In Mirror mode, we will download missing files, and delete any extra files from disk that are not present on the server. This is very useful when creating an exact local backup of an FTP directory.
/// In Update mode, we will only download missing files and preserve any extra files on disk. This is useful when you want to simply download missing files from an FTP directory.
/// Only downloads the files and folders matching all the rules provided, if any.
/// All exceptions during downloading are caught, and the exception is stored in the related FtpResult object.
/// </summary>
/// <param name="localFolder">The full path of the local folder on disk to download into. It is created if it does not exist.</param>
/// <param name="remoteFolder">The full path of the remote FTP folder that you want to download. If it does not exist, an empty result list is returned.</param>
/// <param name="mode">Mirror or Update mode, as explained above</param>
/// <param name="existsMode">If the file exists on disk, should we skip it, resume the download or restart the download?</param>
/// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param>
/// <param name="rules">Only files and folders that pass all these rules are downloaded, and the files that don't pass are skipped. In the Mirror mode, the files that fail the rules are also deleted from the local folder.</param>
/// <param name="progress">Provide a callback to track download progress.</param>
/// <remarks>
/// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support
/// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful
/// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically switch to true for subsequent attempts.
/// If <see cref="FtpVerify.Throw"/> is set and <see cref="FtpError.Throw"/> is <i>not set</i>, then individual verification errors will not cause an exception
/// to propagate from this method.
/// </remarks>
/// <returns>
/// Returns a listing of all the remote files, indicating if they were downloaded, skipped or overwritten.
/// Returns a blank list if nothing was transfered. Never returns null.
/// </returns>
public List <FtpResult> DownloadDirectory(string localFolder, string remoteFolder, FtpFolderSyncMode mode = FtpFolderSyncMode.Update,
FtpLocalExists existsMode = FtpLocalExists.Skip, FtpVerify verifyOptions = FtpVerify.None, List <FtpRule> rules = null, Action <FtpProgress> progress = null)
{
if (localFolder.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "localFolder");
}
if (remoteFolder.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "remoteFolder");
}
LogFunc("DownloadDirectory", new object[] { localFolder, remoteFolder, mode, existsMode, verifyOptions, (rules.IsBlank() ? null : rules.Count + " rules") });
var results = new List <FtpResult>();
// ensure the local path ends with slash
localFolder = localFolder.EnsurePostfix(Path.DirectorySeparatorChar.ToString());
// cleanup the remote path
remoteFolder = remoteFolder.GetFtpPath().EnsurePostfix("/");
// if the dir does not exist, fail fast
if (!DirectoryExists(remoteFolder))
{
return(results);
}
// ensure the local dir exists
localFolder.EnsureDirectory();
// get all the files in the remote directory
var listing = GetListing(remoteFolder, FtpListOption.Recursive | FtpListOption.Size);
// collect paths of the files that should exist (lowercase for CI checks)
var shouldExist = new Dictionary <string, bool>();
// loop thru each file and transfer it
var toDownload = GetFilesToDownload(localFolder, remoteFolder, rules, results, listing, shouldExist);
DownloadServerFiles(toDownload, existsMode, verifyOptions, progress);
// delete the extra local files if in mirror mode
DeleteExtraLocalFiles(localFolder, mode, shouldExist);
return(results);
}
public bool DownloadFile(string localPath, string remotePath, FtpLocalExists existsMode = FtpLocalExists.Overwrite, FtpVerify verifyOptions = FtpVerify.None, Action <FtpProgress> progress = null)
{
Assert.AreEqual(this.IsConnected, true);
if (localPath == "/local/filename.txt" && remotePath == "/filename.txt")
{
return(true);
}
if (localPath == "/local/filename2.txt" && remotePath == "/filename2.txt")
{
throw new DiExample.FtpException("Wrong parameter in test");
}
Assert.Fail($"Wrong parameters: {localPath} {remotePath}");
return(false);
}
/// <summary>
/// Downloads the specified file onto the local file system.
/// High-level API that takes care of various edge cases internally.
/// Supports very large files since it downloads data in chunks.
/// </summary>
/// <param name="localPath">The full or relative path to the file on the local file system</param>
/// <param name="remotePath">The full or relative path to the file on the server</param>
/// <param name="existsMode">If the file exists on disk, should we skip it, resume the download or restart the download?</param>
/// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param>
/// <param name="progress">Provide an implementation of IProgress to track download progress. The value provided is in the range 0 to 100, indicating the percentage of the file transferred. If the progress is indeterminate, -1 is sent.</param>
/// <returns>If true then the file was downloaded, false otherwise.</returns>
/// <remarks>
/// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support
/// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful
/// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically be set to true for subsequent attempts.
/// </remarks>
public bool DownloadFile(string localPath, string remotePath, FtpLocalExists existsMode = FtpLocalExists.Overwrite, FtpVerify verifyOptions = FtpVerify.None, IProgress <FtpProgress> progress = null)
{
// verify args
if (localPath.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "localPath");
}
if (remotePath.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "remotePath");
}
this.LogFunc("DownloadFile", new object[] { localPath, remotePath, existsMode, verifyOptions });
return(DownloadFileToFile(localPath, remotePath, existsMode, verifyOptions, progress));
}
/// <summary>
/// Download all the listed files and folders from the main directory
/// </summary>
private void DownloadServerFiles(List <FtpResult> toDownload, FtpLocalExists existsMode, FtpVerify verifyOptions, Action <FtpProgress> progress)
{
LogFunc("DownloadServerFiles", new object[] { toDownload.Count + " files" });
foreach (var result in toDownload)
{
if (result.Type == FtpFileSystemObjectType.File)
{
// absorb errors
try {
// download the file
var transferred = this.DownloadFile(result.LocalPath, result.RemotePath, existsMode, verifyOptions, progress);
result.IsSuccess = true;
result.IsSkipped = !transferred;
}
catch (Exception ex) {
LogStatus(FtpTraceLevel.Warn, "File failed to download: " + result.RemotePath);
// mark that the file failed to download
result.IsFailed = true;
result.Exception = ex;
}
}
else if (result.Type == FtpFileSystemObjectType.Directory)
{
// absorb errors
try {
// create directory on local filesystem
// to ensure we download the blank remote dirs as well
var created = result.LocalPath.EnsureDirectory();
result.IsSuccess = true;
result.IsSkipped = !created;
}
catch (Exception ex) {
// mark that the file failed to download
result.IsFailed = true;
result.Exception = ex;
}
}
}
}
void IFtpSession.Download(string remotePath, string localPath, FtpLocalExists overwrite, bool recursive)
{
if (string.IsNullOrWhiteSpace(remotePath))
{
throw new ArgumentNullException(nameof(remotePath));
}
if (string.IsNullOrWhiteSpace(localPath))
{
throw new ArgumentNullException(nameof(localPath));
}
FtpObjectType objectType = ((IFtpSession)this).GetObjectType(remotePath);
if (objectType == FtpObjectType.Directory)
{
IEnumerable <Tuple <string, string> > listing = GetRemoteListing(remotePath, localPath, recursive);
foreach (Tuple <string, string> pair in listing)
{
_ftpClient.DownloadFile(pair.Item1, pair.Item2, overwrite);
}
}
else
{
if (objectType == FtpObjectType.File)
{
if (File.Exists(localPath) && overwrite != FtpLocalExists.Overwrite)
{
throw new IOException("File Not Exist");
}
else
{
_ftpClient.DownloadFileAsync(localPath, remotePath, overwrite);
}
}
else
{
throw new NotImplementedException("UnsupportedObjectTypeException");
}
}
}
async Task IFtpSession.DownloadAsync(string remotePath, string localPath, FtpLocalExists overwrite, bool recursive, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(remotePath))
{
throw new ArgumentNullException(nameof(remotePath));
}
if (string.IsNullOrWhiteSpace(localPath))
{
throw new ArgumentNullException(nameof(localPath));
}
FtpObjectType objectType = await((IFtpSession)this).GetObjectTypeAsync(remotePath, cancellationToken);
if (objectType == FtpObjectType.Directory)
{
IEnumerable <Tuple <string, string> > listing = await GetRemoteListingAsync(remotePath, localPath, recursive, cancellationToken);
foreach (Tuple <string, string> pair in listing)
{
cancellationToken.ThrowIfCancellationRequested();
await _ftpClient.DownloadFileAsync(pair.Item1, pair.Item2, overwrite, FtpVerify.None, null, cancellationToken);
}
}
else
{
if (objectType == FtpObjectType.File)
{
if (File.Exists(localPath) && overwrite != FtpLocalExists.Overwrite)
{
throw new IOException("FileExistsException");
}
await _ftpClient.DownloadFileAsync(localPath, remotePath, overwrite, FtpVerify.None, null, cancellationToken);
}
else
{
throw new NotImplementedException("UnsupportedObjectTypeException");
}
}
}
/// <summary>
/// Downloads the specified directory onto the local file system.
/// In Mirror mode, we will download missing files, and delete any extra files from disk that are not present on the server. This is very useful when creating an exact local backup of an FTP directory.
/// In Update mode, we will only download missing files and preserve any extra files on disk. This is useful when you want to simply download missing files from an FTP directory.
/// Only downloads the files and folders matching all the rules provided, if any.
/// All exceptions during downloading are caught, and the exception is stored in the related FtpResult object.
/// </summary>
/// <param name="localFolder">The full path of the local folder on disk to download into. It is created if it does not exist.</param>
/// <param name="remoteFolder">The full path of the remote FTP folder that you want to download. If it does not exist, an empty result list is returned.</param>
/// <param name="mode">Mirror or Update mode, as explained above</param>
/// <param name="existsMode">If the file exists on disk, should we skip it, resume the download or restart the download?</param>
/// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param>
/// <param name="rules">Only files and folders that pass all these rules are downloaded, and the files that don't pass are skipped. In the Mirror mode, the files that fail the rules are also deleted from the local folder.</param>
/// <param name="progress">Provide a callback to track download progress.</param>
/// <remarks>
/// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support
/// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful
/// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically switch to true for subsequent attempts.
/// If <see cref="FtpVerify.Throw"/> is set and <see cref="FtpError.Throw"/> is <i>not set</i>, then individual verification errors will not cause an exception
/// to propagate from this method.
/// </remarks>
/// <returns>
/// Returns a listing of all the remote files, indicating if they were downloaded, skipped or overwritten.
/// Returns a blank list if nothing was transfered. Never returns null.
/// </returns>
public List <FtpResult> DownloadDirectory(string localFolder, string remoteFolder, FtpFolderSyncMode mode = FtpFolderSyncMode.Update, FtpLocalExists existsMode = FtpLocalExists.Skip, FtpVerify verifyOptions = FtpVerify.None, List <FtpRule> rules = null, Action <FtpProgress> progress = null)
{
if (localFolder.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "localFolder");
}
if (remoteFolder.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "remoteFolder");
}
LogFunc("DownloadDirectory", new object[] { localFolder, remoteFolder, mode, existsMode, verifyOptions, (rules.IsBlank() ? null : rules.Count + " rules") });
var results = new List <FtpResult>();
// ensure the local path ends with slash
localFolder = localFolder.EnsurePostfix(Path.DirectorySeparatorChar.ToString());
// cleanup the remote path
remoteFolder = remoteFolder.GetFtpPath().EnsurePostfix("/");
// if the dir does not exist, fail fast
if (!DirectoryExists(remoteFolder))
{
return(results);
}
// ensure the local dir exists
localFolder.EnsureDirectory();
// get all the files in the remote directory
var listing = GetListing(remoteFolder, FtpListOption.Recursive | FtpListOption.Size);
// collect paths of the files that should exist (lowercase for CI checks)
var shouldExist = new Dictionary <string, bool>();
// loop thru each file and transfer it
foreach (var remoteFile in listing)
{
// calculate the local path
var relativePath = remoteFile.FullName.Replace(remoteFolder, "");
var localFile = Path.Combine(localFolder, relativePath);
// create the result object
var result = new FtpResult()
{
Type = remoteFile.Type,
Size = remoteFile.Size,
Name = remoteFile.Name,
RemotePath = remoteFile.FullName,
LocalPath = localFile
};
// only files and folders are processed
if (remoteFile.Type == FtpFileSystemObjectType.File ||
remoteFile.Type == FtpFileSystemObjectType.Directory)
{
// record the file
results.Add(result);
// if the file passes all rules
if (rules != null && rules.Count > 0)
{
var passes = FtpRule.IsAllAllowed(rules, remoteFile);
if (!passes)
{
// mark that the file was skipped due to a rule
result.IsSkipped = true;
result.IsSkippedByRule = true;
// skip downloading the file
continue;
}
}
// record that this file/folder should exist
shouldExist.Add(localFile.ToLower(), true);
// only files are processed
if (remoteFile.Type == FtpFileSystemObjectType.File)
{
// absorb errors
try {
// download the file
var transferred = this.DownloadFile(result.LocalPath, result.RemotePath, existsMode, verifyOptions, progress);
result.IsSuccess = true;
result.IsSkipped = !transferred;
}
catch (Exception ex) {
// mark that the file failed to download
result.IsFailed = true;
result.Exception = ex;
}
}
else if (remoteFile.Type == FtpFileSystemObjectType.Directory)
{
// absorb errors
try {
// create directory on local filesystem
// to ensure we download the blank remote dirs as well
var created = result.LocalPath.EnsureDirectory();
result.IsSuccess = true;
result.IsSkipped = !created;
}
catch (Exception ex) {
// mark that the file failed to download
result.IsFailed = true;
result.Exception = ex;
}
}
}
}
// delete the extra local files if in mirror mode
if (mode == FtpFolderSyncMode.Mirror)
{
// get all the local files
var localListing = Directory.GetFiles(localFolder, "*.*", SearchOption.AllDirectories);
// delete files that are not in listed in shouldExist
foreach (var existingLocalFile in localListing)
{
if (!shouldExist.ContainsKey(existingLocalFile.ToLower()))
{
// delete the file from disk
try {
File.Delete(existingLocalFile);
}
catch (Exception ex) {}
}
}
}
return(results);
}
async Task IFtpSession.DownloadAsync(string remotePath, string localPath, FtpLocalExists overwrite, bool recursive, CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(remotePath))
{
throw new ArgumentNullException(nameof(remotePath));
}
if (string.IsNullOrWhiteSpace(localPath))
{
throw new ArgumentNullException(nameof(localPath));
}
FtpObjectType objectType = ((IFtpSession)this).GetObjectType(remotePath);
if (objectType == FtpObjectType.Directory)
{
IEnumerable <Tuple <string, string> > listing = await GetRemoteListingAsync(remotePath, localPath, recursive, cancellationToken);
foreach (Tuple <string, string> file in listing)
{
if (File.Exists(file.Item1) && overwrite != FtpLocalExists.Overwrite)
{
continue;
}
cancellationToken.ThrowIfCancellationRequested();
string directoryPath = Path.GetDirectoryName(file.Item1);
if (!Directory.Exists(directoryPath))
{
Directory.CreateDirectory(directoryPath);
}
using (Stream fileStream = File.OpenWrite(file.Item1))
{
await Task.Factory.FromAsync(_sftpClient.BeginDownloadFile(file.Item2, fileStream), _sftpClient.EndDownloadFile);
}
}
}
else
{
if (objectType == FtpObjectType.File)
{
if (File.Exists(localPath))
{
if (overwrite == FtpLocalExists.Overwrite)
{
File.Delete(localPath);
}
else
{
throw new IOException("FileExistsException");
}
}
using (Stream fileStream = File.OpenWrite(localPath))
{
await Task.Factory.FromAsync(_sftpClient.BeginDownloadFile(remotePath, fileStream), _sftpClient.EndDownloadFile);
}
}
else
{
throw new NotImplementedException("UnsupportedObjectTypeException");
}
}
}
/// <summary>
/// Downloads the specified files into a local single directory.
/// High-level API that takes care of various edge cases internally.
/// Supports very large files since it downloads data in chunks.
/// Same speed as <see cref="o:DownloadFile"/>.
/// </summary>
/// <param name="localDir">The full or relative path to the directory that files will be downloaded into.</param>
/// <param name="remotePaths">The full or relative paths to the files on the server</param>
/// <param name="existsMode">If the file exists on disk, should we skip it, resume the download or restart the download?</param>
/// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param>
/// <param name="errorHandling">Used to determine how errors are handled</param>
/// <returns>The count of how many files were downloaded successfully. When existing files are skipped, they are not counted.</returns>
/// <remarks>
/// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support
/// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful
/// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically switch to true for subsequent attempts.
/// If <see cref="FtpVerify.Throw"/> is set and <see cref="FtpError.Throw"/> is <i>not set</i>, then individual verification errors will not cause an exception
/// to propagate from this method.
/// </remarks>
public int DownloadFiles(string localDir, IEnumerable <string> remotePaths, FtpLocalExists existsMode = FtpLocalExists.Overwrite, FtpVerify verifyOptions = FtpVerify.None,
FtpError errorHandling = FtpError.None)
{
// verify args
if (!errorHandling.IsValidCombination())
{
throw new ArgumentException("Invalid combination of FtpError flags. Throw & Stop cannot be combined");
}
if (localDir.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "localDir");
}
LogFunc("DownloadFiles", new object[] { localDir, remotePaths, existsMode, verifyOptions });
var errorEncountered = false;
var successfulDownloads = new List <string>();
// ensure ends with slash
localDir = !localDir.EndsWith(Path.DirectorySeparatorChar.ToString()) ? localDir + Path.DirectorySeparatorChar.ToString() : localDir;
foreach (var remotePath in remotePaths)
{
// calc local path
var localPath = localDir + remotePath.GetFtpFileName();
// try to download it
try {
var ok = DownloadFileToFile(localPath, remotePath, existsMode, verifyOptions, null);
if (ok)
{
successfulDownloads.Add(localPath);
}
else if ((int)errorHandling > 1)
{
errorEncountered = true;
break;
}
}
catch (Exception ex) {
LogStatus(FtpTraceLevel.Error, "Failed to download " + remotePath + ". Error: " + ex);
if (errorHandling.HasFlag(FtpError.Stop))
{
errorEncountered = true;
break;
}
if (errorHandling.HasFlag(FtpError.Throw))
{
if (errorHandling.HasFlag(FtpError.DeleteProcessed))
{
PurgeSuccessfulDownloads(successfulDownloads);
}
throw new FtpException("An error occurred downloading file(s). See inner exception for more info.", ex);
}
}
}
if (errorEncountered)
{
//Delete any successful uploads if needed
if (errorHandling.HasFlag(FtpError.DeleteProcessed))
{
PurgeSuccessfulDownloads(successfulDownloads);
successfulDownloads.Clear(); //forces return of 0
}
//Throw generic error because requested
if (errorHandling.HasFlag(FtpError.Throw))
{
throw new FtpException("An error occurred downloading one or more files. Refer to trace output if available.");
}
}
return(successfulDownloads.Count);
}
private async Task <bool> DownloadFileToFileAsync(string localPath, string remotePath, FtpLocalExists existsMode = FtpLocalExists.Append, FtpVerify verifyOptions = FtpVerify.None, IProgress <FtpProgress> progress = null, CancellationToken token = default(CancellationToken))
{
// verify args
if (localPath.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "localPath");
}
if (remotePath.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "remotePath");
}
LogFunc("DownloadFileAsync", new object[] { localPath, remotePath, existsMode, verifyOptions });
var outStreamFileMode = FileMode.Create;
// skip downloading if the local file exists
#if CORE
if (existsMode == FtpLocalExists.Append && await Task.Run(() => File.Exists(localPath), token))
{
if ((await GetFileSizeAsync(remotePath, token)).Equals((await Task.Run(() => new FileInfo(localPath), token)).Length))
{
#else
if (existsMode == FtpLocalExists.Append && File.Exists(localPath))
{
if ((await GetFileSizeAsync(remotePath)).Equals(new FileInfo(localPath).Length))
{
#endif
LogStatus(FtpTraceLevel.Info, "Append is enabled => Local file size matches size on server => skipping");
return(false);
}
else
{
outStreamFileMode = FileMode.Append;
}
}
#if CORE
else if (existsMode == FtpLocalExists.Skip && await Task.Run(() => File.Exists(localPath), token))
{
#else
else if (existsMode == FtpLocalExists.Skip && File.Exists(localPath))
{
#endif
LogStatus(FtpTraceLevel.Info, "Skip is selected => Local file exists => skipping");
return(false);
}
try {
// create the folders
var dirPath = Path.GetDirectoryName(localPath);
#if CORE
if (!string.IsNullOrWhiteSpace(dirPath) && !await Task.Run(() => Directory.Exists(dirPath), token))
{
#else
if (!string.IsNullOrWhiteSpace(dirPath) && !Directory.Exists(dirPath))
{
#endif
Directory.CreateDirectory(dirPath);
}
} catch (Exception ex1) {
// catch errors creating directory
throw new FtpException("Error while crated directories. See InnerException for more info.", ex1);
}
bool downloadSuccess;
var verified = true;
var attemptsLeft = verifyOptions.HasFlag(FtpVerify.Retry) ? m_retryAttempts : 1;
do
{
// download the file from server
using (var outStream = new FileStream(localPath, outStreamFileMode, FileAccess.Write, FileShare.None, 4096, true)) {
// download the file straight to a file stream
downloadSuccess = await DownloadFileInternalAsync(remotePath, outStream, await Task.Run(() => File.Exists(localPath), token)?(await Task.Run(() => new FileInfo(localPath), token)).Length : 0, progress, token);
attemptsLeft--;
}
// if verification is needed
if (downloadSuccess && verifyOptions != FtpVerify.None)
{
verified = await VerifyTransferAsync(localPath, remotePath, token);
LogStatus(FtpTraceLevel.Info, "File Verification: " + (verified ? "PASS" : "FAIL"));
#if DEBUG
if (!verified && attemptsLeft > 0)
{
LogStatus(FtpTraceLevel.Verbose, "Retrying due to failed verification." + (existsMode == FtpLocalExists.Append ? " Overwrite will occur." : "") + " " + attemptsLeft + " attempts remaining");
}
#endif
}
} while (!verified && attemptsLeft > 0);
if (downloadSuccess && !verified && verifyOptions.HasFlag(FtpVerify.Delete))
{
File.Delete(localPath);
}
if (downloadSuccess && !verified && verifyOptions.HasFlag(FtpVerify.Throw))
{
throw new FtpException("Downloaded file checksum value does not match remote file");
}
return(downloadSuccess && verified);
}
/// <summary>
/// Downloads the specified file onto the local file system asynchronously.
/// High-level API that takes care of various edge cases internally.
/// Supports very large files since it downloads data in chunks.
/// </summary>
/// <param name="localPath">The full or relative path to the file on the local file system</param>
/// <param name="remotePath">The full or relative path to the file on the server</param>
/// <param name="existsMode">Overwrite if you want the local file to be overwritten if it already exists. Append will also create a new file if it dosen't exists</param>
/// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param>
/// <param name="progress">Provide an implementation of IProgress to track download progress. The value provided is in the range 0 to 100, indicating the percentage of the file transferred. If the progress is indeterminate, -1 is sent.</param>
/// <param name="token">Cancellation Token</param>
/// <returns>If true then the file was downloaded, false otherwise.</returns>
/// <remarks>
/// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support
/// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful
/// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically be set to true for subsequent attempts.
/// </remarks>
public async Task <bool> DownloadFileAsync(string localPath, string remotePath, FtpLocalExists existsMode = FtpLocalExists.Append, FtpVerify verifyOptions = FtpVerify.None, IProgress <FtpProgress> progress = null, CancellationToken token = default(CancellationToken))
{
// verify args
if (localPath.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "localPath");
}
if (remotePath.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "remotePath");
}
LogFunc("DownloadFileAsync", new object[] { localPath, remotePath, existsMode, verifyOptions });
return(await DownloadFileToFileAsync(localPath, remotePath, existsMode, verifyOptions, progress, token));
}
private bool DownloadFileToFile(string localPath, string remotePath, FtpLocalExists existsMode, FtpVerify verifyOptions, IProgress <FtpProgress> progress)
{
var outStreamFileMode = FileMode.Create;
// skip downloading if local file size matches
if (existsMode == FtpLocalExists.Append && File.Exists(localPath))
{
if (GetFileSize(remotePath).Equals(new FileInfo(localPath).Length))
{
LogStatus(FtpTraceLevel.Info, "Append is selected => Local file size matches size on server => skipping");
return(false);
}
else
{
outStreamFileMode = FileMode.Append;
}
}
else if (existsMode == FtpLocalExists.Skip && File.Exists(localPath))
{
LogStatus(FtpTraceLevel.Info, "Skip is selected => Local file exists => skipping");
return(false);
}
try {
// create the folders
var dirPath = Path.GetDirectoryName(localPath);
if (!FtpExtensions.IsNullOrWhiteSpace(dirPath) && !Directory.Exists(dirPath))
{
Directory.CreateDirectory(dirPath);
}
}
catch (Exception ex1) {
// catch errors creating directory
throw new FtpException("Error while creating directories. See InnerException for more info.", ex1);
}
bool downloadSuccess;
var verified = true;
var attemptsLeft = verifyOptions.HasFlag(FtpVerify.Retry) ? m_retryAttempts : 1;
do
{
// download the file from server
using (var outStream = new FileStream(localPath, outStreamFileMode, FileAccess.Write, FileShare.None)) {
// download the file straight to a file stream
downloadSuccess = DownloadFileInternal(remotePath, outStream, File.Exists(localPath) ? new FileInfo(localPath).Length : 0, progress);
attemptsLeft--;
}
// if verification is needed
if (downloadSuccess && verifyOptions != FtpVerify.None)
{
verified = VerifyTransfer(localPath, remotePath);
LogLine(FtpTraceLevel.Info, "File Verification: " + (verified ? "PASS" : "FAIL"));
#if DEBUG
if (!verified && attemptsLeft > 0)
{
LogStatus(FtpTraceLevel.Verbose, "Retrying due to failed verification." + (existsMode == FtpLocalExists.Overwrite ? " Overwrite will occur." : "") + " " + attemptsLeft + " attempts remaining");
}
#endif
}
} while (!verified && attemptsLeft > 0);
if (downloadSuccess && !verified && verifyOptions.HasFlag(FtpVerify.Delete))
{
File.Delete(localPath);
}
if (downloadSuccess && !verified && verifyOptions.HasFlag(FtpVerify.Throw))
{
throw new FtpException("Downloaded file checksum value does not match remote file");
}
return(downloadSuccess && verified);
}
/// <summary>
/// Downloads the specified files into a local single directory.
/// High-level API that takes care of various edge cases internally.
/// Supports very large files since it downloads data in chunks.
/// Same speed as <see cref="o:DownloadFile"/>.
/// </summary>
/// <param name="localDir">The full or relative path to the directory that files will be downloaded.</param>
/// <param name="remotePaths">The full or relative paths to the files on the server</param>
/// <param name="existsMode">Overwrite if you want the local file to be overwritten if it already exists. Append will also create a new file if it dosen't exists</param>
/// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param>
/// <param name="errorHandling">Used to determine how errors are handled</param>
/// <param name="token">The token to monitor for cancellation requests</param>
/// <returns>The count of how many files were downloaded successfully. When existing files are skipped, they are not counted.</returns>
/// <remarks>
/// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support
/// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful
/// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically be set to true for subsequent attempts.
/// If <see cref="FtpVerify.Throw"/> is set and <see cref="FtpError.Throw"/> is <i>not set</i>, then individual verification errors will not cause an exception
/// to propagate from this method.
/// </remarks>
public async Task <int> DownloadFilesAsync(string localDir, IEnumerable <string> remotePaths, FtpLocalExists existsMode = FtpLocalExists.Overwrite,
FtpVerify verifyOptions = FtpVerify.None, FtpError errorHandling = FtpError.None, CancellationToken token = default(CancellationToken))
{
// verify args
if (!errorHandling.IsValidCombination())
{
throw new ArgumentException("Invalid combination of FtpError flags. Throw & Stop cannot be combined");
}
if (localDir.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "localDir");
}
LogFunc("DownloadFilesAsync", new object[] { localDir, remotePaths, existsMode, verifyOptions });
//check if cancellation was requested and throw to set TaskStatus state to Canceled
token.ThrowIfCancellationRequested();
var errorEncountered = false;
var successfulDownloads = new List <string>();
// ensure ends with slash
localDir = !localDir.EndsWith(Path.DirectorySeparatorChar.ToString()) ? localDir + Path.DirectorySeparatorChar.ToString() : localDir;
foreach (var remotePath in remotePaths)
{
//check if cancellation was requested and throw to set TaskStatus state to Canceled
token.ThrowIfCancellationRequested();
// calc local path
var localPath = localDir + remotePath.GetFtpFileName();
// try to download it
try {
bool ok = await DownloadFileToFileAsync(localPath, remotePath, existsMode, verifyOptions, token : token);
if (ok)
{
successfulDownloads.Add(localPath);
}
else if ((int)errorHandling > 1)
{
errorEncountered = true;
break;
}
}
catch (Exception ex) {
if (ex is OperationCanceledException)
{
LogStatus(FtpTraceLevel.Info, "Download cancellation requested");
//DO NOT SUPPRESS CANCELLATION REQUESTS -- BUBBLE UP!
throw;
}
if (errorHandling.HasFlag(FtpError.Stop))
{
errorEncountered = true;
break;
}
if (errorHandling.HasFlag(FtpError.Throw))
{
if (errorHandling.HasFlag(FtpError.DeleteProcessed))
{
PurgeSuccessfulDownloads(successfulDownloads);
}
throw new FtpException("An error occurred downloading file(s). See inner exception for more info.", ex);
}
}
}
if (errorEncountered)
{
//Delete any successful uploads if needed
if (errorHandling.HasFlag(FtpError.DeleteProcessed))
{
PurgeSuccessfulDownloads(successfulDownloads);
successfulDownloads.Clear(); //forces return of 0
}
//Throw generic error because requested
if (errorHandling.HasFlag(FtpError.Throw))
{
throw new FtpException("An error occurred downloading one or more files. Refer to trace output if available.");
}
}
return(successfulDownloads.Count);
}
public ActionResult Execute(ArgumentCollection arguments)
{
try
{
// Must provide arguments
if (arguments == null)
{
throw new ArgumentNullException(nameof(arguments));
}
if (!arguments.HasArgument(FtpDownloadActionExecutionArgs.RemoteFilesCollection))
{
throw new MissingArgumentException(FtpDownloadActionExecutionArgs.RemoteFilesCollection);
}
if (!(arguments[FtpDownloadActionExecutionArgs.RemoteFilesCollection] is List <string> remotePaths))
{
throw new ArgumentsValidationException(
$"unable to cast argument value into list of string. key({FtpDownloadActionExecutionArgs.RemoteFilesCollection})");
}
FtpLocalExists existMode = Overwrite
? FtpLocalExists.Overwrite
: Append
? FtpLocalExists.Append
: FtpLocalExists.Skip;
LoggingService.Trace($"ExistMode: {existMode:G}");
var directoryPath = Environment.ExpandEnvironmentVariables(DirectoryPath);
List <string> downloadedFiles = new List <string>();
List <string> skippedFiles = new List <string>();
List <string> failedFiles = new List <string>();
if (UseTemporaryName && existMode != FtpLocalExists.Append)
{
LoggingService.Warn($"Cannot use temporary name for download in mode Append.");
}
using (FtpClient ftpClient = new FtpClient(Host, Port, Username, Password))
{
if (!string.IsNullOrWhiteSpace(RemoteWorkingDir))
{
ftpClient.SetWorkingDirectory(RemoteWorkingDir);
}
foreach (var remotePath in remotePaths)
{
var remoteFileName = UriHelper.GetFileName(remotePath);
var localPath = Path.Combine(directoryPath, remoteFileName);
string temporaryPath = string.Empty;
bool useTemporaryName = UseTemporaryName && existMode != FtpLocalExists.Append;
if (useTemporaryName)
{
temporaryPath = Path.ChangeExtension(Path.Combine(directoryPath, Path.GetRandomFileName()),
DEFAULT_TEMP_EXTENSION);
}
var downloadPath = useTemporaryName ? temporaryPath : localPath;
LoggingService.Debug($"Downloading file {remotePath} to temporary path {downloadPath}");
var ftpStatus = ftpClient.DownloadFile(downloadPath, remotePath, existMode);
LoggingService.Debug($"File download completed with status {ftpStatus:G}");
if (ftpStatus == FtpStatus.Success && useTemporaryName)
{
LoggingService.Trace($"Renaming file {downloadPath} to {localPath}");
File.Move(downloadPath, localPath, Overwrite);
}
if (ftpStatus == FtpStatus.Success)
{
downloadedFiles.Add(remotePath);
}
if (ftpStatus == FtpStatus.Skipped)
{
skippedFiles.Add(remotePath);
}
if (ftpStatus == FtpStatus.Failed)
{
failedFiles.Add(remotePath);
}
}
if (MoveDownloaded || DeleteDownloaded || RenameDownloaded)
{
foreach (var downloadedFile in downloadedFiles)
{
if (DeleteDownloaded)
{
ftpClient.DeleteFile(downloadedFile);
}
else if (MoveDownloaded && !string.IsNullOrWhiteSpace(MoveDownloadedPath))
{
ftpClient.MoveFile(downloadedFile, UriHelper.BuildPath(MoveDownloadedPath, Path.GetFileName(downloadedFile)));
}
else if (RenameDownloaded)
{
LoggingService.Debug($"Trying to rename remote file {downloadedFile}");
var remoteParentPath = UriHelper.GetParentUriString(downloadedFile);
if (!string.IsNullOrWhiteSpace(RenameDownloadedNewName))
{
LoggingService.Debug($"Renaming remote file {downloadedFile} to {RenameDownloadedNewName}");
var newName = UriHelper.BuildPath(remoteParentPath, RenameDownloadedNewName);
ftpClient.Rename(downloadedFile, newName);
LoggingService.Debug($"Renamed file {downloadedFile} to {newName}");
}
else
{
LoggingService.Debug($"Trying to apply prefix or extension to remote file {downloadedFile}");
var fileName = Path.GetFileName(downloadedFile);
if (!string.IsNullOrWhiteSpace(RenameDownloadedExtension))
{
LoggingService.Debug($"Changing extension to {RenameDownloadedExtension}");
fileName = Path.ChangeExtension(fileName, RenameDownloadedExtension);
}
if (!string.IsNullOrWhiteSpace(RenameDownloadedPrefix))
{
LoggingService.Debug($"Applying prefix to {RenameDownloadedPrefix}");
fileName = $"{RenameDownloadedPrefix}{fileName}";
}
LoggingService.Debug($"Renaming remote file {downloadedFile} to {fileName}");
var newName = UriHelper.BuildPath(remoteParentPath, fileName);
ftpClient.Rename(downloadedFile, newName);
LoggingService.Debug($"Renamed file {downloadedFile} to {newName}");
}
}
}
}
}
if (downloadedFiles.Any())
{
return(ActionResult.Succeeded()
.WithAdditionInformation(ArgumentCollection.New()
.WithArgument(FtpDownloadActionResultArgs.DownloadedFiles, downloadedFiles)
.WithArgument(FtpDownloadActionResultArgs.SkippedFiles, skippedFiles)
.WithArgument(FtpDownloadActionResultArgs.FailedFiles, failedFiles)
));
}
return(ActionResult.Failed()
.WithAdditionInformation(ArgumentCollection.New()
.WithArgument(FtpDownloadActionResultArgs.DownloadedFiles, downloadedFiles)
.WithArgument(FtpDownloadActionResultArgs.SkippedFiles, skippedFiles)
.WithArgument(FtpDownloadActionResultArgs.FailedFiles, failedFiles)
));
}
catch (Exception exception)
{
LoggingService.Error(exception);
return(ActionResult.Failed(exception)
.WithAdditionInformation(ArgumentCollection.New()
.WithArgument(FtpDownloadActionResultArgs.DownloadedFiles, new List <string>())
.WithArgument(FtpDownloadActionResultArgs.SkippedFiles, new List <string>())
.WithArgument(FtpDownloadActionResultArgs.FailedFiles, new List <string>())
));
}
}
private bool DownloadFile(string localPath, string remotePath, FtpLocalExists localExists = FtpLocalExists.Skip, FtpVerify ftpVerify = FtpVerify.OnlyChecksum)
=> client.DownloadFile(
localPath,
remotePath,
localExists,
ftpVerify);
