public static async Task FtpUploadFile(BotData data, string remoteFileName, string localFileName, FtpRemoteExists existsPolicy = FtpRemoteExists.Overwrite) { data.Logger.LogHeader(); var client = GetClient(data); await client.UploadFileAsync(localFileName, remoteFileName, existsPolicy, true, FtpVerify.None, null, data.CancellationToken).ConfigureAwait(false); data.Logger.Log($"{localFileName} uploaded to {remoteFileName}", LogColors.Maize); }
/// <summary> /// Begins an asynchronous operation to move a directory on the remote file system, from one directory to another. /// Always checks if the source directory exists. Checks if the dest directory exists based on the `existsMode` parameter. /// Only throws exceptions for critical errors. /// </summary> /// <param name="path">The full or relative path to the object</param> /// <param name="dest">The new full or relative path including the new name of the object</param> /// <param name="existsMode">Should we check if the dest directory exists? And if it does should we overwrite/skip the operation?</param> /// <param name="callback">Async callback</param> /// <param name="state">State object</param> /// <returns>IAsyncResult</returns> public IAsyncResult BeginMoveDirectory(string path, string dest, FtpRemoteExists existsMode, AsyncCallback callback, object state) { AsyncMoveDirectory func; IAsyncResult ar; lock (m_asyncmethods) { ar = (func = MoveDirectory).BeginInvoke(path, dest, existsMode, callback, state); m_asyncmethods.Add(ar, func); } return(ar); }
/// <summary> /// Transfer the specified file from the source FTP Server to the destination FTP Server using the FXP protocol. /// High-level API that takes care of various edge cases internally. /// </summary> /// <param name="sourcePath">The full or relative path to the file on the source FTP Server</param> /// <param name="remoteClient">Valid FTP connection to the destination FTP Server</param> /// <param name="remotePath">The full or relative path to destination file on the remote FTP Server</param> /// <param name="createRemoteDir">Indicates if the folder should be created on the remote FTP Server</param> /// <param name="existsMode">If the file exists on disk, should we skip it, resume the download or restart the download?</param> /// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param> /// <param name="progress">Provide a callback to track download progress.</param> /// Returns a FtpStatus indicating if the file was transfered. /// <remarks> /// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support /// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful /// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically be set to true for subsequent attempts. /// </remarks> public FtpStatus TransferFile(string sourcePath, FtpClient remoteClient, string remotePath, bool createRemoteDir = false, FtpRemoteExists existsMode = FtpRemoteExists.Resume, FtpVerify verifyOptions = FtpVerify.None, Action <FtpProgress> progress = null, FtpProgress metaProgress = null) { sourcePath = sourcePath.GetFtpPath(); remotePath = remotePath.GetFtpPath(); LogFunc(nameof(TransferFile), new object[] { sourcePath, remoteClient, remotePath, FXPDataType, createRemoteDir, existsMode, verifyOptions }); // verify input params VerifyTransferFileParams(sourcePath, remoteClient, remotePath, existsMode); // ensure source file exists if (!FileExists(sourcePath)) { throw new FtpException("Source File " + sourcePath + " cannot be found or does not exists!"); } bool fxpSuccess; var verified = true; var attemptsLeft = verifyOptions.HasFlag(FtpVerify.Retry) ? m_retryAttempts : 1; do { fxpSuccess = TransferFileFXPInternal(sourcePath, remoteClient, remotePath, createRemoteDir, existsMode, progress, metaProgress is null ? new FtpProgress(1, 0) : metaProgress); attemptsLeft--; // if verification is needed if (fxpSuccess && verifyOptions != FtpVerify.None) { verified = VerifyFXPTransfer(sourcePath, remoteClient, remotePath); LogStatus(FtpTraceLevel.Info, "File Verification: " + (verified ? "PASS" : "FAIL")); if (!verified && attemptsLeft > 0) { LogStatus(FtpTraceLevel.Verbose, "Retrying due to failed verification." + (existsMode == FtpRemoteExists.Resume ? " Overwrite will occur." : "") + " " + attemptsLeft + " attempts remaining"); // Force overwrite if a retry is required existsMode = FtpRemoteExists.Overwrite; } } } while (!verified && attemptsLeft > 0); if (fxpSuccess && !verified && verifyOptions.HasFlag(FtpVerify.Delete)) { remoteClient.DeleteFile(remotePath); } if (fxpSuccess && !verified && verifyOptions.HasFlag(FtpVerify.Throw)) { throw new FtpException("Destination file checksum value does not match source file"); } return(fxpSuccess && verified ? FtpStatus.Success : FtpStatus.Failed); }
/// <summary> /// Upload all the files within the main directory /// </summary> private async Task UploadDirectoryFilesAsync(List <FtpResult> filesToUpload, FtpRemoteExists existsMode, FtpVerify verifyOptions, IProgress <FtpProgress> progress, FtpListItem[] remoteListing, CancellationToken token) { LogFunc(nameof(UploadDirectoryFilesAsync), new object[] { filesToUpload.Count + " files" }); var r = -1; foreach (var result in filesToUpload) { r++; // absorb errors try { // check if the file already exists on the server var existsModeToUse = existsMode; var fileExists = FtpExtensions.FileExistsInListing(remoteListing, result.RemotePath); // if we want to skip uploaded files and the file already exists, mark its skipped if (existsMode == FtpRemoteExists.Skip && fileExists) { LogStatus(FtpTraceLevel.Info, "Skipped file that already exists: " + result.LocalPath); result.IsSuccess = true; result.IsSkipped = true; continue; } // in any mode if the file does not exist, mark that exists check is not required if (!fileExists) { existsModeToUse = existsMode == FtpRemoteExists.Append ? FtpRemoteExists.AppendNoCheck : FtpRemoteExists.NoCheck; } // create meta progress to store the file progress var metaProgress = new FtpProgress(filesToUpload.Count, r); // upload the file var transferred = await UploadFileFromFileAsync(result.LocalPath, result.RemotePath, false, existsModeToUse, false, false, verifyOptions, token, progress, metaProgress); result.IsSuccess = transferred.IsSuccess(); result.IsSkipped = transferred == FtpStatus.Skipped; } catch (Exception ex) { LogStatus(FtpTraceLevel.Warn, "File failed to upload: " + result.LocalPath); // mark that the file failed to upload result.IsFailed = true; result.Exception = ex; } } }
/// <summary> /// Moves a directory asynchronously on the remote file system from one directory to another. /// Always checks if the source directory exists. Checks if the dest directory exists based on the `existsMode` parameter. /// Only throws exceptions for critical errors. /// </summary> /// <param name="path">The full or relative path to the object</param> /// <param name="dest">The new full or relative path including the new name of the object</param> /// <param name="existsMode">Should we check if the dest directory exists? And if it does should we overwrite/skip the operation?</param> /// <param name="token">The token that can be used to cancel the entire process</param> /// <returns>Whether the directory was moved</returns> public async Task <bool> MoveDirectoryAsync(string path, string dest, FtpRemoteExists existsMode = FtpRemoteExists.Overwrite, CancellationToken token = default(CancellationToken)) { // verify args if (path.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "path"); } if (dest.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "dest"); } path = path.GetFtpPath(); dest = dest.GetFtpPath(); LogFunc(nameof(MoveDirectoryAsync), new object[] { path, dest, existsMode }); if (await DirectoryExistsAsync(path, token)) { // check if dest directory exists and act accordingly if (existsMode != FtpRemoteExists.NoCheck) { bool destExists = await DirectoryExistsAsync(dest, token); if (destExists) { switch (existsMode) { case FtpRemoteExists.Overwrite: await DeleteDirectoryAsync(dest, token); break; case FtpRemoteExists.Skip: return(false); } } } // move the directory await RenameAsync(path, dest, token); return(true); } return(false); }
/// <summary> /// Moves a directory on the remote file system from one directory to another. /// Always checks if the source directory exists. Checks if the dest directory exists based on the `existsMode` parameter. /// Only throws exceptions for critical errors. /// </summary> /// <param name="path">The full or relative path to the object</param> /// <param name="dest">The new full or relative path including the new name of the object</param> /// <param name="existsMode">Should we check if the dest directory exists? And if it does should we overwrite/skip the operation?</param> /// <returns>Whether the directory was moved</returns> public bool MoveDirectory(string path, string dest, FtpRemoteExists existsMode = FtpRemoteExists.Overwrite) { // verify args if (path.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "path"); } if (dest.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "dest"); } path = path.GetFtpPath(); dest = dest.GetFtpPath(); LogFunc(nameof(MoveDirectory), new object[] { path, dest, existsMode }); if (DirectoryExists(path)) { // check if dest directory exists and act accordingly if (existsMode != FtpRemoteExists.NoCheck) { var destExists = DirectoryExists(dest); if (destExists) { switch (existsMode) { case FtpRemoteExists.Overwrite: DeleteDirectory(dest); break; case FtpRemoteExists.Skip: return(false); } } } // move the directory Rename(path, dest); return(true); } return(false); }
/// <summary> /// Upload all the files within the main directory /// </summary> private void UploadDirectoryFiles(List <FtpResult> filesToUpload, FtpRemoteExists existsMode, FtpVerify verifyOptions, Action <FtpProgress> progress, FtpListItem[] remoteListing) { LogFunc("UploadDirectoryFiles", new object[] { filesToUpload.Count + " files" }); foreach (var result in filesToUpload) { // absorb errors try { // check if the file already exists on the server var existsModeToUse = existsMode; var fileExists = FtpExtensions.FileExistsInListing(remoteListing, result.RemotePath); // if we want to skip uploaded files and the file already exists, mark its skipped if (existsMode == FtpRemoteExists.Skip && fileExists) { LogStatus(FtpTraceLevel.Info, "Skipped file that already exists: " + result.LocalPath); result.IsSuccess = true; result.IsSkipped = true; continue; } // in any mode if the file does not exist, mark that exists check is not required if (!fileExists) { existsModeToUse = existsMode == FtpRemoteExists.Append ? FtpRemoteExists.AppendNoCheck : FtpRemoteExists.NoCheck; } // upload the file var transferred = this.UploadFile(result.LocalPath, result.RemotePath, existsModeToUse, false, verifyOptions, progress); result.IsSuccess = true; result.IsSkipped = !transferred; } catch (Exception ex) { LogStatus(FtpTraceLevel.Warn, "File failed to upload: " + result.LocalPath); // mark that the file failed to upload result.IsFailed = true; result.Exception = ex; } } }
/// <summary> /// Check if the file is cleared to be uploaded, taking its existance/filesize and existsMode options into account. /// </summary> private bool CanUploadFile(FtpResult result, FtpListItem[] remoteListing, FtpRemoteExists existsMode, out FtpRemoteExists existsModeToUse) { // check if the file already exists on the server existsModeToUse = existsMode; var fileExists = FtpExtensions.FileExistsInListing(remoteListing, result.RemotePath); // if we want to skip uploaded files and the file already exists, mark its skipped if (existsMode == FtpRemoteExists.Skip && fileExists) { LogStatus(FtpTraceLevel.Info, "Skipped file that already exists: " + result.LocalPath); result.IsSuccess = true; result.IsSkipped = true; return(false); } // in any mode if the file does not exist, mark that exists check is not required if (!fileExists) { existsModeToUse = existsMode == FtpRemoteExists.Append ? FtpRemoteExists.AppendNoCheck : FtpRemoteExists.NoCheck; } return(true); }
/// <summary> /// Upload all the files within the main directory /// </summary> private async Task UploadDirectoryFilesAsync(List <FtpResult> filesToUpload, FtpRemoteExists existsMode, FtpVerify verifyOptions, IProgress <FtpProgress> progress, FtpListItem[] remoteListing, CancellationToken token) { LogFunc(nameof(UploadDirectoryFilesAsync), new object[] { filesToUpload.Count + " files" }); var r = -1; foreach (var result in filesToUpload) { r++; // absorb errors try { // skip uploading if the file already exists on the server FtpRemoteExists existsModeToUse; if (!CanUploadFile(result, remoteListing, existsMode, out existsModeToUse)) { continue; } // create meta progress to store the file progress var metaProgress = new FtpProgress(filesToUpload.Count, r); // upload the file var transferred = await UploadFileFromFileAsync(result.LocalPath, result.RemotePath, false, existsModeToUse, false, false, verifyOptions, token, progress, metaProgress); result.IsSuccess = transferred.IsSuccess(); result.IsSkipped = transferred == FtpStatus.Skipped; } catch (Exception ex) { LogStatus(FtpTraceLevel.Warn, "File failed to upload: " + result.LocalPath); // mark that the file failed to upload result.IsFailed = true; result.Exception = ex; } } }
/// <summary> /// Transfer the specified directory from the source FTP Server onto the remote FTP Server using the FXP protocol. /// You will need to create a valid connection to your remote FTP Server before calling this method. /// In Update mode, we will only transfer missing files and preserve any extra files on the remote FTP Server. This is useful when you want to simply transfer missing files from an FTP directory. /// Currently Mirror mode is not implemented. /// Only transfers the files and folders matching all the rules provided, if any. /// All exceptions during transfer are caught, and the exception is stored in the related FtpResult object. /// </summary> /// <param name="sourceFolder">The full or relative path to the folder on the source FTP Server. If it does not exist, an empty result list is returned.</param> /// <param name="remoteClient">Valid FTP connection to the destination FTP Server</param> /// <param name="remoteFolder">The full or relative path to destination folder on the remote FTP Server</param> /// <param name="mode">Only Update mode is currently implemented</param> /// <param name="existsMode">If the file exists on disk, should we skip it, resume the download or restart the download?</param> /// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param> /// <param name="rules">Only files and folders that pass all these rules are downloaded, and the files that don't pass are skipped. In the Mirror mode, the files that fail the rules are also deleted from the local folder.</param> /// <param name="progress">Provide a callback to track download progress.</param> /// <remarks> /// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support /// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful /// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically switch to true for subsequent attempts. /// If <see cref="FtpVerify.Throw"/> is set and <see cref="FtpError.Throw"/> is <i>not set</i>, then individual verification errors will not cause an exception /// to propagate from this method. /// </remarks> /// <returns> /// Returns a listing of all the remote files, indicating if they were downloaded, skipped or overwritten. /// Returns a blank list if nothing was transfered. Never returns null. /// </returns> public List <FtpResult> TransferDirectory(string sourceFolder, FtpClient remoteClient, string remoteFolder, FtpFolderSyncMode mode = FtpFolderSyncMode.Update, FtpRemoteExists existsMode = FtpRemoteExists.Skip, FtpVerify verifyOptions = FtpVerify.None, List <FtpRule> rules = null, Action <FtpProgress> progress = null) { if (sourceFolder.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "sourceFolder"); } if (remoteFolder.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "remoteFolder"); } LogFunc(nameof(TransferDirectory), new object[] { sourceFolder, remoteClient, remoteFolder, mode, existsMode, verifyOptions, (rules.IsBlank() ? null : rules.Count + " rules") }); var results = new List <FtpResult>(); // cleanup the FTP paths sourceFolder = sourceFolder.GetFtpPath().EnsurePostfix("/"); remoteFolder = remoteFolder.GetFtpPath().EnsurePostfix("/"); // if the source dir does not exist, fail fast if (!DirectoryExists(sourceFolder)) { return(results); } // flag to determine if existence checks are required var checkFileExistence = true; // ensure the remote dir exists if (!remoteClient.DirectoryExists(remoteFolder)) { remoteClient.CreateDirectory(remoteFolder); checkFileExistence = false; } // collect paths of the files that should exist (lowercase for CI checks) var shouldExist = new Dictionary <string, bool>(); // get all the folders in the local directory var dirListing = GetListing(sourceFolder, FtpListOption.Recursive).Where(x => x.Type == FtpFileSystemObjectType.Directory).Select(x => x.FullName).ToArray(); // get all the already existing files var remoteListing = checkFileExistence ? remoteClient.GetListing(remoteFolder, FtpListOption.Recursive) : null; // loop thru each folder and ensure it exists var dirsToUpload = GetSubDirectoriesToTransfer(sourceFolder, remoteFolder, rules, results, dirListing); CreateSubDirectories(remoteClient, dirsToUpload); // get all the files in the local directory var fileListing = GetListing(sourceFolder, FtpListOption.Recursive).Where(x => x.Type == FtpFileSystemObjectType.File).Select(x => x.FullName).ToArray(); // loop thru each file and transfer it var filesToUpload = GetFilesToTransfer(sourceFolder, remoteFolder, rules, results, shouldExist, fileListing); TransferServerFiles(filesToUpload, remoteClient, existsMode, verifyOptions, progress, remoteListing); // delete the extra remote files if in mirror mode and the directory was pre-existing // DeleteExtraServerFiles(mode, shouldExist, remoteListing); return(results); }
private void TransferServerFiles(List <FtpResult> filesToTransfer, FtpClient remoteClient, FtpRemoteExists existsMode, FtpVerify verifyOptions, Action <FtpProgress> progress, FtpListItem[] remoteListing) { LogFunc(nameof(TransferServerFiles), new object[] { filesToTransfer.Count + " files" }); int r = -1; foreach (var result in filesToTransfer) { r++; // absorb errors try { // skip uploading if the file already exists on the server FtpRemoteExists existsModeToUse; if (!CanUploadFile(result, remoteListing, existsMode, out existsModeToUse)) { continue; } // create meta progress to store the file progress var metaProgress = new FtpProgress(filesToTransfer.Count, r); // transfer the file var transferred = TransferFile(result.LocalPath, remoteClient, result.RemotePath, false, existsModeToUse, verifyOptions, progress, metaProgress); result.IsSuccess = transferred.IsSuccess(); result.IsSkipped = transferred == FtpStatus.Skipped; } catch (Exception ex) { LogStatus(FtpTraceLevel.Warn, "File failed to transfer: " + result.LocalPath); // mark that the file failed to upload result.IsFailed = true; result.Exception = ex; } } }
private void VerifyTransferFileParams(string sourcePath, FtpClient remoteClient, string remotePath, FtpRemoteExists existsMode) { if (remoteClient is null) { throw new ArgumentNullException("Destination FXP FtpClient cannot be null!", "remoteClient"); } if (sourcePath.IsBlank()) { throw new ArgumentNullException("FtpListItem must be specified!", "sourceFtpFileItem"); } if (remotePath.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "remotePath"); } if (!remoteClient.IsConnected) { throw new FtpException("The connection must be open before a transfer between servers can be intitiated"); } if (!this.IsConnected) { throw new FtpException("The source FXP FtpClient must be open and connected before a transfer between servers can be intitiated"); } if (existsMode == FtpRemoteExists.AddToEnd || existsMode == FtpRemoteExists.AddToEndNoCheck) { throw new ArgumentException("FXP file transfer does not currently support AddToEnd or AddToEndNoCheck modes. Use another value for existsMode.", "existsMode"); } }
/// <summary> /// Uploads the specified directory onto the server. /// In Mirror mode, we will upload missing files, and delete any extra files from the server that are not present on disk. This is very useful when publishing an exact copy of a local folder onto an FTP server. /// In Update mode, we will only upload missing files and preserve any extra files on the server. This is useful when you want to simply upload missing files to a server. /// Only uploads the files and folders matching all the rules provided, if any. /// All exceptions during uploading are caught, and the exception is stored in the related FtpResult object. /// </summary> /// <param name="localFolder">The full path of the local folder on disk that you want to upload. If it does not exist, an empty result list is returned.</param> /// <param name="remoteFolder">The full path of the remote FTP folder to upload into. It is created if it does not exist.</param> /// <param name="mode">Mirror or Update mode, as explained above</param> /// <param name="existsMode">If the file exists on disk, should we skip it, resume the upload or restart the upload?</param> /// <param name="verifyOptions">Sets if checksum verification is required for a successful upload and what to do if it fails verification (See Remarks)</param> /// <param name="rules">Only files and folders that pass all these rules are downloaded, and the files that don't pass are skipped. In the Mirror mode, the files that fail the rules are also deleted from the local folder.</param> /// <param name="progress">Provide a callback to track upload progress.</param> /// <remarks> /// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support /// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful /// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically switch to true for subsequent attempts. /// If <see cref="FtpVerify.Throw"/> is set and <see cref="FtpError.Throw"/> is <i>not set</i>, then individual verification errors will not cause an exception /// to propagate from this method. /// </remarks> /// <returns> /// Returns a listing of all the remote files, indicating if they were downloaded, skipped or overwritten. /// Returns a blank list if nothing was transfered. Never returns null. /// </returns> public List <FtpResult> UploadDirectory(string localFolder, string remoteFolder, FtpFolderSyncMode mode = FtpFolderSyncMode.Update, FtpRemoteExists existsMode = FtpRemoteExists.Skip, FtpVerify verifyOptions = FtpVerify.None, List <FtpRule> rules = null, Action <FtpProgress> progress = null) { if (localFolder.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "localFolder"); } if (remoteFolder.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "remoteFolder"); } LogFunc("UploadDirectory", new object[] { localFolder, remoteFolder, mode, existsMode, verifyOptions, (rules.IsBlank() ? null : rules.Count + " rules") }); var results = new List <FtpResult>(); // ensure the local path ends with slash localFolder = localFolder.EnsurePostfix(Path.DirectorySeparatorChar.ToString()); // cleanup the remote path remoteFolder = remoteFolder.GetFtpPath().EnsurePostfix("/"); // if the dir does not exist, fail fast if (!Directory.Exists(localFolder)) { return(results); } // ensure the remote dir exists if (!DirectoryExists(remoteFolder)) { CreateDirectory(remoteFolder); } // collect paths of the files that should exist (lowercase for CI checks) var shouldExist = new Dictionary <string, bool>(); // get all the folders in the local directory var dirListing = Directory.GetDirectories(localFolder, "*.*", SearchOption.AllDirectories); // loop thru each folder and ensure it exists foreach (var localFile in dirListing) { // calculate the local path var relativePath = localFile.Replace(localFolder, "").EnsurePostfix(Path.DirectorySeparatorChar.ToString()); var remoteFile = remoteFolder + relativePath.Replace('\\', '/'); // create the result object var result = new FtpResult() { Type = FtpFileSystemObjectType.Directory, Size = 0, Name = Path.GetDirectoryName(localFile), RemotePath = remoteFile, LocalPath = localFile }; // record the folder results.Add(result); // if the folder passes all rules if (rules != null && rules.Count > 0) { var passes = FtpRule.IsAllAllowed(rules, result.ToListItem(true)); if (!passes) { // mark that the file was skipped due to a rule result.IsSkipped = true; result.IsSkippedByRule = true; // skip uploading the file continue; } } // absorb errors try { // create directory on the server // to ensure we upload the blank remote dirs as well if (!DirectoryExists(remoteFile)) { CreateDirectory(remoteFile); result.IsSuccess = true; result.IsSkipped = false; } else { result.IsSkipped = true; } } catch (Exception ex) { // mark that the folder failed to upload result.IsFailed = true; result.Exception = ex; } } // get all the files in the local directory var fileListing = Directory.GetFiles(localFolder, "*.*", SearchOption.AllDirectories); // loop thru each file and transfer it foreach (var localFile in fileListing) { // calculate the local path var relativePath = localFile.Replace(localFolder, ""); var remoteFile = remoteFolder + relativePath.Replace('\\', '/'); // create the result object var result = new FtpResult() { Type = FtpFileSystemObjectType.File, Size = new FileInfo(localFile).Length, Name = Path.GetFileName(localFile), RemotePath = remoteFile, LocalPath = localFile }; // record the file results.Add(result); // if the file passes all rules if (rules != null && rules.Count > 0) { var passes = FtpRule.IsAllAllowed(rules, result.ToListItem(true)); if (!passes) { // mark that the file was skipped due to a rule result.IsSkipped = true; result.IsSkippedByRule = true; // skip uploading the file continue; } } // record that this file should exist shouldExist.Add(remoteFile.ToLower(), true); // absorb errors try { // upload the file var transferred = this.UploadFile(result.LocalPath, result.RemotePath, existsMode, false, verifyOptions, progress); result.IsSuccess = true; result.IsSkipped = !transferred; } catch (Exception ex) { // mark that the file failed to upload result.IsFailed = true; result.Exception = ex; } } // delete the extra remote files if in mirror mode if (mode == FtpFolderSyncMode.Mirror) { // get all the files on the server var remoteListing = GetListing(remoteFolder, FtpListOption.Recursive); // delete files that are not in listed in shouldExist foreach (var existingServerFile in remoteListing) { if (existingServerFile.Type == FtpFileSystemObjectType.File) { if (!shouldExist.ContainsKey(existingServerFile.FullName.ToLower())) { // delete the file from the server try { DeleteFile(existingServerFile.FullName); } catch (Exception ex) { } } } } } return(results); }
/// <summary> /// Transfers a file from the source FTP Server to the destination FTP Server via the FXP protocol asynchronously. /// </summary> private async Task <bool> TransferFileFXPInternalAsync(string sourcePath, FtpClient remoteClient, string remotePath, bool createRemoteDir, FtpRemoteExists existsMode, IProgress <FtpProgress> progress, CancellationToken token, FtpProgress metaProgress) { FtpReply reply; long offset = 0; bool fileExists = false; long fileSize = 0; var ftpFxpSession = await OpenPassiveFXPConnectionAsync(remoteClient, progress != null, token); if (ftpFxpSession != null) { try { ftpFxpSession.SourceServer.ReadTimeout = (int)TimeSpan.FromMinutes(30.0).TotalMilliseconds; ftpFxpSession.TargetServer.ReadTimeout = (int)TimeSpan.FromMinutes(30.0).TotalMilliseconds; // check if the file exists, and skip, overwrite or append if (existsMode == FtpRemoteExists.AppendNoCheck) { offset = await remoteClient.GetFileSizeAsync(remotePath, token); if (offset == -1) { offset = 0; // start from the beginning } } else { fileExists = await remoteClient.FileExistsAsync(remotePath, token); switch (existsMode) { case FtpRemoteExists.Skip: if (fileExists) { LogStatus(FtpTraceLevel.Info, "Skip is selected => Destination file exists => skipping"); //Fix #413 - progress callback isn't called if the file has already been uploaded to the server //send progress reports if (progress != null) { progress.Report(new FtpProgress(100.0, 0, 0, TimeSpan.FromSeconds(0), sourcePath, remotePath, metaProgress)); } return(true); } break; case FtpRemoteExists.Overwrite: if (fileExists) { await remoteClient.DeleteFileAsync(remotePath, token); } break; case FtpRemoteExists.Append: if (fileExists) { offset = await remoteClient.GetFileSizeAsync(remotePath, token); if (offset == -1) { offset = 0; // start from the beginning } } break; } } fileSize = await GetFileSizeAsync(sourcePath, token); // ensure the remote dir exists .. only if the file does not already exist! if (createRemoteDir && !fileExists) { var dirname = remotePath.GetFtpDirectoryName(); if (!await remoteClient.DirectoryExistsAsync(dirname, token)) { await remoteClient.CreateDirectoryAsync(dirname, token); } } if (offset == 0 && existsMode != FtpRemoteExists.AppendNoCheck) { // send command to tell the source server to 'send' the file to the destination server if (!(reply = await ftpFxpSession.SourceServer.ExecuteAsync($"RETR {sourcePath}", token)).Success) { throw new FtpCommandException(reply); } //Instruct destination server to store the file if (!(reply = await ftpFxpSession.TargetServer.ExecuteAsync($"STOR {remotePath}", token)).Success) { throw new FtpCommandException(reply); } } else { //tell source server to restart / resume if (!(reply = await ftpFxpSession.SourceServer.ExecuteAsync($"REST {offset}", token)).Success) { throw new FtpCommandException(reply); } // send command to tell the source server to 'send' the file to the destination server if (!(reply = await ftpFxpSession.SourceServer.ExecuteAsync($"RETR {sourcePath}", token)).Success) { throw new FtpCommandException(reply); } //Instruct destination server to append the file if (!(reply = await ftpFxpSession.TargetServer.ExecuteAsync($"APPE {remotePath}", token)).Success) { throw new FtpCommandException(reply); } } var transferStarted = DateTime.Now; long lastSize = 0; var sourceFXPTransferReply = ftpFxpSession.SourceServer.GetReplyAsync(token); var destinationFXPTransferReply = ftpFxpSession.TargetServer.GetReplyAsync(token); // while the transfer is not complete while (!sourceFXPTransferReply.IsCompleted || !destinationFXPTransferReply.IsCompleted) { // send progress reports every 1 second if (ftpFxpSession.ProgressServer != null) { // send progress reports if (progress != null && fileSize != -1) { offset = await ftpFxpSession.ProgressServer.GetFileSizeAsync(remotePath, token); if (offset != -1 && lastSize <= offset) { long bytesProcessed = offset - lastSize; lastSize = offset; ReportProgress(progress, fileSize, offset, bytesProcessed, DateTime.Now - transferStarted, sourcePath, remotePath, metaProgress); } } } await Task.Delay(FXPProgressInterval); } FtpTrace.WriteLine(FtpTraceLevel.Info, $"FXP transfer of file {sourcePath} has completed"); await NoopAsync(token); await remoteClient.NoopAsync(token); ftpFxpSession.Dispose(); return(true); } // Fix: catch all exceptions and dispose off the FTP clients if one occurs catch (Exception ex) { ftpFxpSession.Dispose(); throw ex; } } else { FtpTrace.WriteLine(FtpTraceLevel.Error, "Failed to open FXP passive Connection"); return(false); } }
/// <summary> /// Transfer the specified directory from the source FTP Server onto the remote FTP Server asynchronously using the FXP protocol. /// You will need to create a valid connection to your remote FTP Server before calling this method. /// In Update mode, we will only transfer missing files and preserve any extra files on the remote FTP Server. This is useful when you want to simply transfer missing files from an FTP directory. /// Currently Mirror mode is not implemented. /// Only transfers the files and folders matching all the rules provided, if any. /// All exceptions during transfer are caught, and the exception is stored in the related FtpResult object. /// </summary> /// <param name="sourceFolder">The full or relative path to the folder on the source FTP Server. If it does not exist, an empty result list is returned.</param> /// <param name="remoteClient">Valid FTP connection to the destination FTP Server</param> /// <param name="remoteFolder">The full or relative path to destination folder on the remote FTP Server</param> /// <param name="mode">Only Update mode is currently implemented</param> /// <param name="existsMode">If the file exists on disk, should we skip it, resume the download or restart the download?</param> /// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param> /// <param name="rules">Only files and folders that pass all these rules are downloaded, and the files that don't pass are skipped. In the Mirror mode, the files that fail the rules are also deleted from the local folder.</param> /// <param name="progress">Provide a callback to track download progress.</param> /// <param name="token">The token that can be used to cancel the entire process</param> /// <remarks> /// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support /// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful /// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically switch to true for subsequent attempts. /// If <see cref="FtpVerify.Throw"/> is set and <see cref="FtpError.Throw"/> is <i>not set</i>, then individual verification errors will not cause an exception /// to propagate from this method. /// </remarks> /// <returns> /// Returns a listing of all the remote files, indicating if they were downloaded, skipped or overwritten. /// Returns a blank list if nothing was transfered. Never returns null. /// </returns> public async Task <List <FtpResult> > TransferDirectoryAsync(string sourceFolder, FtpClient remoteClient, string remoteFolder, FtpFolderSyncMode mode = FtpFolderSyncMode.Update, FtpRemoteExists existsMode = FtpRemoteExists.Skip, FtpVerify verifyOptions = FtpVerify.None, List <FtpRule> rules = null, IProgress <FtpProgress> progress = null, CancellationToken token = default(CancellationToken)) { if (sourceFolder.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "sourceFolder"); } if (remoteFolder.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "remoteFolder"); } LogFunc(nameof(TransferDirectoryAsync), new object[] { sourceFolder, remoteClient, remoteFolder, mode, existsMode, verifyOptions, (rules.IsBlank() ? null : rules.Count + " rules") }); var results = new List <FtpResult>(); // cleanup the FTP paths sourceFolder = sourceFolder.GetFtpPath().EnsurePostfix("/"); remoteFolder = remoteFolder.GetFtpPath().EnsurePostfix("/"); // if the source dir does not exist, fail fast if (!await DirectoryExistsAsync(sourceFolder, token)) { return(results); } // flag to determine if existence checks are required var checkFileExistence = true; // ensure the remote dir exists if (!await remoteClient.DirectoryExistsAsync(remoteFolder, token)) { await remoteClient.CreateDirectoryAsync(remoteFolder, token); checkFileExistence = false; } // break if task is cancelled token.ThrowIfCancellationRequested(); // collect paths of the files that should exist (lowercase for CI checks) var shouldExist = new Dictionary <string, bool>(); // get all the folders in the local directory var dirListing = (await GetListingAsync(sourceFolder, FtpListOption.Recursive, token)).Where(x => x.Type == FtpFileSystemObjectType.Directory).Select(x => x.FullName).ToArray(); // break if task is cancelled token.ThrowIfCancellationRequested(); // get all the already existing files var remoteListing = checkFileExistence ? await remoteClient.GetListingAsync(remoteFolder, FtpListOption.Recursive, token) : null; // break if task is cancelled token.ThrowIfCancellationRequested(); // loop thru each folder and ensure it exists #1 var dirsToUpload = GetSubDirectoriesToTransfer(sourceFolder, remoteFolder, rules, results, dirListing); // break if task is cancelled token.ThrowIfCancellationRequested(); /*-------------------------------------------------------------------------------------/ * Cancelling after this point would leave the FTP server in an inconsistant state * *-------------------------------------------------------------------------------------*/ // loop thru each folder and ensure it exists #2 await CreateSubDirectoriesAsync(remoteClient, dirsToUpload, token); // get all the files in the local directory var fileListing = (await GetListingAsync(sourceFolder, FtpListOption.Recursive, token)).Where(x => x.Type == FtpFileSystemObjectType.File).Select(x => x.FullName).ToArray(); // loop thru each file and transfer it var filesToUpload = GetFilesToTransfer(sourceFolder, remoteFolder, rules, results, shouldExist, fileListing); await TransferServerFilesAsync(filesToUpload, remoteClient, existsMode, verifyOptions, progress, remoteListing, token); // delete the extra remote files if in mirror mode and the directory was pre-existing // DeleteExtraServerFiles(mode, shouldExist, remoteListing); return(results); }
/// <summary> /// Uploads the specified directory onto the server. /// In Mirror mode, we will upload missing files, and delete any extra files from the server that are not present on disk. This is very useful when publishing an exact copy of a local folder onto an FTP server. /// In Update mode, we will only upload missing files and preserve any extra files on the server. This is useful when you want to simply upload missing files to a server. /// Only uploads the files and folders matching all the rules provided, if any. /// All exceptions during uploading are caught, and the exception is stored in the related FtpResult object. /// </summary> /// <param name="localFolder">The full path of the local folder on disk that you want to upload. If it does not exist, an empty result list is returned.</param> /// <param name="remoteFolder">The full path of the remote FTP folder to upload into. It is created if it does not exist.</param> /// <param name="mode">Mirror or Update mode, as explained above</param> /// <param name="existsMode">If the file exists on disk, should we skip it, resume the upload or restart the upload?</param> /// <param name="verifyOptions">Sets if checksum verification is required for a successful upload and what to do if it fails verification (See Remarks)</param> /// <param name="rules">Only files and folders that pass all these rules are downloaded, and the files that don't pass are skipped. In the Mirror mode, the files that fail the rules are also deleted from the local folder.</param> /// <param name="progress">Provide an implementation of IProgress to track upload progress.</param> /// <param name="token">The token that can be used to cancel the entire process</param> /// <remarks> /// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support /// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful /// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically switch to true for subsequent attempts. /// If <see cref="FtpVerify.Throw"/> is set and <see cref="FtpError.Throw"/> is <i>not set</i>, then individual verification errors will not cause an exception /// to propagate from this method. /// </remarks> /// <returns> /// Returns a listing of all the remote files, indicating if they were downloaded, skipped or overwritten. /// Returns a blank list if nothing was transfered. Never returns null. /// </returns> public async Task <List <FtpResult> > UploadDirectoryAsync(string localFolder, string remoteFolder, FtpFolderSyncMode mode = FtpFolderSyncMode.Update, FtpRemoteExists existsMode = FtpRemoteExists.Skip, FtpVerify verifyOptions = FtpVerify.None, List <FtpRule> rules = null, IProgress <FtpProgress> progress = null, CancellationToken token = default(CancellationToken)) { if (localFolder.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "localFolder"); } if (remoteFolder.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "remoteFolder"); } // ensure the local path ends with slash localFolder = localFolder.EnsurePostfix(Path.DirectorySeparatorChar.ToString()); // cleanup the remote path remoteFolder = remoteFolder.GetFtpPath().EnsurePostfix("/"); LogFunc(nameof(UploadDirectoryAsync), new object[] { localFolder, remoteFolder, mode, existsMode, verifyOptions, (rules.IsBlank() ? null : rules.Count + " rules") }); var results = new List <FtpResult>(); // if the dir does not exist, fail fast if (!Directory.Exists(localFolder)) { return(results); } // flag to determine if existence checks are required var checkFileExistence = true; // ensure the remote dir exists if (!await DirectoryExistsAsync(remoteFolder, token)) { await CreateDirectoryAsync(remoteFolder, token); checkFileExistence = false; } // break if task is cancelled token.ThrowIfCancellationRequested(); // collect paths of the files that should exist (lowercase for CI checks) var shouldExist = new Dictionary <string, bool>(); // get all the folders in the local directory var dirListing = Directory.GetDirectories(localFolder, "*.*", SearchOption.AllDirectories); // break if task is cancelled token.ThrowIfCancellationRequested(); // get all the already existing files var remoteListing = checkFileExistence ? GetListing(remoteFolder, FtpListOption.Recursive) : null; // break if task is cancelled token.ThrowIfCancellationRequested(); // loop thru each folder and ensure it exists #1 var dirsToUpload = GetSubDirectoriesToUpload(localFolder, remoteFolder, rules, results, dirListing); // break if task is cancelled token.ThrowIfCancellationRequested(); /*-------------------------------------------------------------------------------------/ * Cancelling after this point would leave the FTP server in an inconsistant state * *-------------------------------------------------------------------------------------*/ // loop thru each folder and ensure it exists #2 await CreateSubDirectoriesAsync(this, dirsToUpload, token); // get all the files in the local directory var fileListing = Directory.GetFiles(localFolder, "*.*", SearchOption.AllDirectories); // loop thru each file and transfer it var filesToUpload = GetFilesToUpload(localFolder, remoteFolder, rules, results, shouldExist, fileListing); await UploadDirectoryFilesAsync(filesToUpload, existsMode, verifyOptions, progress, remoteListing, token); // delete the extra remote files if in mirror mode and the directory was pre-existing await DeleteExtraServerFilesAsync(mode, remoteFolder, shouldExist, remoteListing, rules, token); return(results); }
/// <summary> /// Uploads the specified directory onto the server. /// In Mirror mode, we will upload missing files, and delete any extra files from the server that are not present on disk. This is very useful when publishing an exact copy of a local folder onto an FTP server. /// In Update mode, we will only upload missing files and preserve any extra files on the server. This is useful when you want to simply upload missing files to a server. /// Only uploads the files and folders matching all the rules provided, if any. /// All exceptions during uploading are caught, and the exception is stored in the related FtpResult object. /// </summary> /// <param name="localFolder">The full path of the local folder on disk that you want to upload. If it does not exist, an empty result list is returned.</param> /// <param name="remoteFolder">The full path of the remote FTP folder to upload into. It is created if it does not exist.</param> /// <param name="mode">Mirror or Update mode, as explained above</param> /// <param name="existsMode">If the file exists on disk, should we skip it, resume the upload or restart the upload?</param> /// <param name="verifyOptions">Sets if checksum verification is required for a successful upload and what to do if it fails verification (See Remarks)</param> /// <param name="rules">Only files and folders that pass all these rules are downloaded, and the files that don't pass are skipped. In the Mirror mode, the files that fail the rules are also deleted from the local folder.</param> /// <param name="progress">Provide a callback to track upload progress.</param> /// <remarks> /// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support /// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful /// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically switch to true for subsequent attempts. /// If <see cref="FtpVerify.Throw"/> is set and <see cref="FtpError.Throw"/> is <i>not set</i>, then individual verification errors will not cause an exception /// to propagate from this method. /// </remarks> /// <returns> /// Returns a listing of all the remote files, indicating if they were downloaded, skipped or overwritten. /// Returns a blank list if nothing was transfered. Never returns null. /// </returns> public List <FtpResult> UploadDirectory(string localFolder, string remoteFolder, FtpFolderSyncMode mode = FtpFolderSyncMode.Update, FtpRemoteExists existsMode = FtpRemoteExists.Skip, FtpVerify verifyOptions = FtpVerify.None, List <FtpRule> rules = null, Action <FtpProgress> progress = null) { if (localFolder.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "localFolder"); } if (remoteFolder.IsBlank()) { throw new ArgumentException("Required parameter is null or blank.", "remoteFolder"); } LogFunc(nameof(UploadDirectory), new object[] { localFolder, remoteFolder, mode, existsMode, verifyOptions, (rules.IsBlank() ? null : rules.Count + " rules") }); var results = new List <FtpResult>(); // ensure the local path ends with slash localFolder = localFolder.EnsurePostfix(Path.DirectorySeparatorChar.ToString()); // cleanup the remote path remoteFolder = remoteFolder.GetFtpPath().EnsurePostfix("/"); // if the dir does not exist, fail fast if (!Directory.Exists(localFolder)) { return(results); } // flag to determine if existence checks are required var checkFileExistence = true; // ensure the remote dir exists if (!DirectoryExists(remoteFolder)) { CreateDirectory(remoteFolder); checkFileExistence = false; } // collect paths of the files that should exist (lowercase for CI checks) var shouldExist = new Dictionary <string, bool>(); // get all the folders in the local directory var dirListing = Directory.GetDirectories(localFolder, "*.*", SearchOption.AllDirectories); // get all the already existing files var remoteListing = checkFileExistence ? GetListing(remoteFolder, FtpListOption.Recursive) : null; // loop thru each folder and ensure it exists var dirsToUpload = GetSubDirectoriesToUpload(localFolder, remoteFolder, rules, results, dirListing); CreateSubDirectories(this, dirsToUpload); // get all the files in the local directory var fileListing = Directory.GetFiles(localFolder, "*.*", SearchOption.AllDirectories); // loop thru each file and transfer it var filesToUpload = GetFilesToUpload(localFolder, remoteFolder, rules, results, shouldExist, fileListing); UploadDirectoryFiles(filesToUpload, existsMode, verifyOptions, progress, remoteListing); // delete the extra remote files if in mirror mode and the directory was pre-existing DeleteExtraServerFiles(mode, shouldExist, remoteListing); return(results); }
/// <summary> /// Transfer the specified file from the source FTP Server to the destination FTP Server asynchronously using the FXP protocol. /// High-level API that takes care of various edge cases internally. /// </summary> /// <param name="sourcePath">The full or relative path to the file on the source FTP Server</param> /// <param name="remoteClient">Valid FTP connection to the destination FTP Server</param> /// <param name="remotePath">The full or relative path to destination file on the remote FTP Server</param> /// <param name="createRemoteDir">Indicates if the folder should be created on the remote FTP Server</param> /// <param name="existsMode">If the file exists on disk, should we skip it, resume the download or restart the download?</param> /// <param name="verifyOptions">Sets if checksum verification is required for a successful download and what to do if it fails verification (See Remarks)</param> /// <param name="progress">Provide a callback to track download progress.</param> /// <param name="token">The token that can be used to cancel the entire process</param> /// Returns a FtpStatus indicating if the file was transfered. /// <remarks> /// If verification is enabled (All options other than <see cref="FtpVerify.None"/>) the hash will be checked against the server. If the server does not support /// any hash algorithm, then verification is ignored. If only <see cref="FtpVerify.OnlyChecksum"/> is set then the return of this method depends on both a successful /// upload & verification. Additionally, if any verify option is set and a retry is attempted then overwrite will automatically be set to true for subsequent attempts. /// </remarks> public async Task <FtpStatus> TransferFileAsync(string sourcePath, FtpClient remoteClient, string remotePath, bool createRemoteDir = false, FtpRemoteExists existsMode = FtpRemoteExists.Append, FtpVerify verifyOptions = FtpVerify.None, IProgress <FtpProgress> progress = null, FtpProgress metaProgress = null, CancellationToken token = default(CancellationToken)) { LogFunc(nameof(TransferFileAsync), new object[] { sourcePath, remoteClient, remotePath, FXPDataType, createRemoteDir, existsMode, verifyOptions }); // verify input params VerifyTransferFileParams(sourcePath, remoteClient, remotePath); // ensure source file exists if (!await FileExistsAsync(sourcePath, token)) { throw new FtpException(string.Format("Source File {0} cannot be found or does not exists!", sourcePath)); } bool fxpSuccess; var verified = true; var attemptsLeft = verifyOptions.HasFlag(FtpVerify.Retry) ? m_retryAttempts : 1; do { fxpSuccess = await TransferFileFXPInternalAsync(sourcePath, remoteClient, remotePath, createRemoteDir, existsMode, progress, token, metaProgress is null?new FtpProgress(1, 0) : metaProgress); attemptsLeft--; // if verification is needed if (fxpSuccess && verifyOptions != FtpVerify.None) { verified = await VerifyFXPTransferAsync(sourcePath, remoteClient, remotePath, token); LogStatus(FtpTraceLevel.Info, "File Verification: " + (verified ? "PASS" : "FAIL")); #if DEBUG if (!verified && attemptsLeft > 0) { LogStatus(FtpTraceLevel.Verbose, "Retrying due to failed verification." + (existsMode == FtpRemoteExists.Append ? " Overwrite will occur." : "") + " " + attemptsLeft + " attempts remaining"); } #endif } } while (!verified && attemptsLeft > 0); if (fxpSuccess && !verified && verifyOptions.HasFlag(FtpVerify.Delete)) { await remoteClient.DeleteFileAsync(remotePath, token); } if (fxpSuccess && !verified && verifyOptions.HasFlag(FtpVerify.Throw)) { throw new FtpException("Destination file checksum value does not match source file"); } return(fxpSuccess && verified ? FtpStatus.Success : FtpStatus.Failed); }