/// <summary>
/// Upload all the files within the main directory
/// </summary>
private async Task UploadDirectoryFilesAsync(List <FtpResult> filesToUpload, FtpRemoteExists existsMode, FtpVerify verifyOptions, IProgress <FtpProgress> progress, FtpListItem[] remoteListing, CancellationToken token)
{
LogFunc(nameof(UploadDirectoryFilesAsync), new object[] { filesToUpload.Count + " files" });
var r = -1;
foreach (var result in filesToUpload)
{
r++;
// absorb errors
try {
// check if the file already exists on the server
var existsModeToUse = existsMode;
var fileExists = FtpExtensions.FileExistsInListing(remoteListing, result.RemotePath);
// if we want to skip uploaded files and the file already exists, mark its skipped
if (existsMode == FtpRemoteExists.Skip && fileExists)
{
LogStatus(FtpTraceLevel.Info, "Skipped file that already exists: " + result.LocalPath);
result.IsSuccess = true;
result.IsSkipped = true;
continue;
}
// in any mode if the file does not exist, mark that exists check is not required
if (!fileExists)
{
existsModeToUse = existsMode == FtpRemoteExists.Append ? FtpRemoteExists.AppendNoCheck : FtpRemoteExists.NoCheck;
}
// create meta progress to store the file progress
var metaProgress = new FtpProgress(filesToUpload.Count, r);
// upload the file
var transferred = await UploadFileFromFileAsync(result.LocalPath, result.RemotePath, false, existsModeToUse, false, false, verifyOptions, token, progress, metaProgress);
result.IsSuccess = transferred.IsSuccess();
result.IsSkipped = transferred == FtpStatus.Skipped;
}
catch (Exception ex) {
LogStatus(FtpTraceLevel.Warn, "File failed to upload: " + result.LocalPath);
// mark that the file failed to upload
result.IsFailed = true;
result.Exception = ex;
}
}
}
/// <summary>
/// Upload all the files within the main directory
/// </summary>
private void UploadDirectoryFiles(List <FtpResult> filesToUpload, FtpRemoteExists existsMode, FtpVerify verifyOptions, Action <FtpProgress> progress, FtpListItem[] remoteListing)
{
LogFunc("UploadDirectoryFiles", new object[] { filesToUpload.Count + " files" });
foreach (var result in filesToUpload)
{
// absorb errors
try {
// check if the file already exists on the server
var existsModeToUse = existsMode;
var fileExists = FtpExtensions.FileExistsInListing(remoteListing, result.RemotePath);
// if we want to skip uploaded files and the file already exists, mark its skipped
if (existsMode == FtpRemoteExists.Skip && fileExists)
{
LogStatus(FtpTraceLevel.Info, "Skipped file that already exists: " + result.LocalPath);
result.IsSuccess = true;
result.IsSkipped = true;
continue;
}
// in any mode if the file does not exist, mark that exists check is not required
if (!fileExists)
{
existsModeToUse = existsMode == FtpRemoteExists.Append ? FtpRemoteExists.AppendNoCheck : FtpRemoteExists.NoCheck;
}
// upload the file
var transferred = this.UploadFile(result.LocalPath, result.RemotePath, existsModeToUse, false, verifyOptions, progress);
result.IsSuccess = true;
result.IsSkipped = !transferred;
}
catch (Exception ex) {
LogStatus(FtpTraceLevel.Warn, "File failed to upload: " + result.LocalPath);
// mark that the file failed to upload
result.IsFailed = true;
result.Exception = ex;
}
}
}
/// <summary>
/// Check if the file is cleared to be uploaded, taking its existance/filesize and existsMode options into account.
/// </summary>
private bool CanUploadFile(FtpResult result, FtpListItem[] remoteListing, FtpRemoteExists existsMode, out FtpRemoteExists existsModeToUse)
{
// check if the file already exists on the server
existsModeToUse = existsMode;
var fileExists = FtpExtensions.FileExistsInListing(remoteListing, result.RemotePath);
// if we want to skip uploaded files and the file already exists, mark its skipped
if (existsMode == FtpRemoteExists.Skip && fileExists)
{
LogStatus(FtpTraceLevel.Info, "Skipped file that already exists: " + result.LocalPath);
result.IsSuccess = true;
result.IsSkipped = true;
return(false);
}
// in any mode if the file does not exist, mark that exists check is not required
if (!fileExists)
{
existsModeToUse = existsMode == FtpRemoteExists.Append ? FtpRemoteExists.AppendNoCheck : FtpRemoteExists.NoCheck;
}
return(true);
}
/// <summary>
/// Modify the permissions of the given file/folder.
/// Only works on *NIX systems, and not on Windows/IIS servers.
/// Only works if the FTP server supports the SITE CHMOD command
/// (requires the CHMOD extension to be installed and enabled).
/// Throws FtpCommandException if there is an issue.
/// </summary>
/// <param name="path">The full or relative path to the item</param>
/// <param name="owner">The owner permissions</param>
/// <param name="group">The group permissions</param>
/// <param name="other">The other permissions</param>
/// <param name="token">The token that can be used to cancel the entire process</param>
public Task SetFilePermissionsAsync(string path, FtpPermission owner, FtpPermission group, FtpPermission other, CancellationToken token = default(CancellationToken))
{
return(SetFilePermissionsAsync(path, FtpExtensions.CalcChmod(owner, group, other), token));
}
/// <summary>
/// Modify the permissions of the given file/folder.
/// Only works on *NIX systems, and not on Windows/IIS servers.
/// Only works if the FTP server supports the SITE CHMOD command
/// (requires the CHMOD extension to be installed and enabled).
/// Throws FtpCommandException if there is an issue.
/// </summary>
/// <param name="path">The full or relative path to the item</param>
/// <param name="owner">The owner permissions</param>
/// <param name="group">The group permissions</param>
/// <param name="other">The other permissions</param>
public void SetFilePermissions(string path, FtpPermission owner, FtpPermission group, FtpPermission other)
{
SetFilePermissions(path, FtpExtensions.CalcChmod(owner, group, other));
}
/// <summary>
/// Calculates the CHMOD value from the permissions flags
/// </summary>
public static void CalculateChmod(this FtpListItem item)
{
item.Chmod = FtpExtensions.CalcChmod(item.OwnerPermissions, item.GroupPermissions, item.OthersPermissions);
}
/// <summary>
/// Gets a file listing from the server asynchronously. Each <see cref="FtpListItem"/> object returned
/// contains information about the file that was able to be retrieved.
/// </summary>
/// <remarks>
/// If a <see cref="DateTime"/> property is equal to <see cref="DateTime.MinValue"/> then it means the
/// date in question was not able to be retrieved. If the <see cref="FtpListItem.Size"/> property
/// is equal to 0, then it means the size of the object could also not
/// be retrieved.
/// </remarks>
/// <param name="path">The path to list</param>
/// <param name="options">Options that dictate how the list operation is performed</param>
/// <param name="token">Cancellation Token</param>
/// <returns>An array of items retrieved in the listing</returns>
public async Task <FtpListItem[]> GetListingAsync(string path, FtpListOption options, CancellationToken token = default(CancellationToken))
{
// start recursive process if needed and unsupported by the server
if ((options & FtpListOption.Recursive) == FtpListOption.Recursive && !RecursiveList)
{
return(await GetListingRecursiveAsync(GetAbsolutePath(path), options));
}
this.LogFunc(nameof(GetListingAsync), new object[] { path, options });
FtpListItem item = null;
List <FtpListItem> lst = new List <FtpListItem>();
List <string> rawlisting = new List <string>();
string listcmd = null;
string buf = null;
// read flags
bool isIncludeSelf = (options & FtpListOption.IncludeSelfAndParent) == FtpListOption.IncludeSelfAndParent;
bool isForceList = (options & FtpListOption.ForceList) == FtpListOption.ForceList;
bool isNoPath = (options & FtpListOption.NoPath) == FtpListOption.NoPath;
bool isNameList = (options & FtpListOption.NameList) == FtpListOption.NameList;
bool isUseLS = (options & FtpListOption.UseLS) == FtpListOption.UseLS;
bool isAllFiles = (options & FtpListOption.AllFiles) == FtpListOption.AllFiles;
bool isRecursive = (options & FtpListOption.Recursive) == FtpListOption.Recursive && RecursiveList;
bool isDerefLinks = (options & FtpListOption.DerefLinks) == FtpListOption.DerefLinks;
bool isGetModified = (options & FtpListOption.Modify) == FtpListOption.Modify;
bool isGetSize = (options & FtpListOption.Size) == FtpListOption.Size;
// calc path to request
path = await GetAbsolutePathAsync(path, token);
// MLSD provides a machine readable format with 100% accurate information
// so always prefer MLSD over LIST unless the caller of this method overrides it with the ForceList option
bool machineList = false;
if ((!isForceList || m_parser == FtpParser.Machine) && HasFeature(FtpCapability.MLSD))
{
listcmd = "MLSD";
machineList = true;
}
else
{
if (isUseLS)
{
listcmd = "LS";
}
else if (isNameList)
{
listcmd = "NLST";
}
else
{
string listopts = "";
listcmd = "LIST";
if (isAllFiles)
{
listopts += "a";
}
if (isRecursive)
{
listopts += "R";
}
if (listopts.Length > 0)
{
listcmd += " -" + listopts;
}
}
}
if (!isNoPath)
{
listcmd = (listcmd + " " + path.GetFtpPath());
}
await ExecuteAsync("TYPE I", token);
// read in raw file listing
try
{
using (FtpDataStream stream = await OpenDataStreamAsync(listcmd, 0, token))
{
try
{
this.LogLine(FtpTraceLevel.Verbose, "+---------------------------------------+");
if (this.BulkListing)
{
// increases performance of GetListing by reading multiple lines of the file listing at once
foreach (var line in await stream.ReadAllLinesAsync(Encoding, this.BulkListingLength, token))
{
if (!FtpExtensions.IsNullOrWhiteSpace(line))
{
rawlisting.Add(line);
this.LogLine(FtpTraceLevel.Verbose, "Listing: " + line);
}
}
}
else
{
// GetListing will read file listings line-by-line (actually byte-by-byte)
while ((buf = await stream.ReadLineAsync(Encoding, token)) != null)
{
if (buf.Length > 0)
{
rawlisting.Add(buf);
this.LogLine(FtpTraceLevel.Verbose, "Listing: " + buf);
}
}
}
this.LogLine(FtpTraceLevel.Verbose, "-----------------------------------------");
}
finally
{
stream.Close();
}
}
}
catch (FtpMissingSocketException)
{
// Some FTP server does not send any response when listing an empty directory
// and the authentication fails, if no communication socket is provided by the server
}
for (int i = 0; i < rawlisting.Count; i++)
{
buf = rawlisting[i];
if (isNameList)
{
// if NLST was used we only have a file name so
// there is nothing to parse.
item = new FtpListItem()
{
FullName = buf
};
if (await DirectoryExistsAsync(item.FullName, token))
{
item.Type = FtpFileSystemObjectType.Directory;
}
else
{
item.Type = FtpFileSystemObjectType.File;
}
lst.Add(item);
}
else
{
// if this is a result of LIST -R then the path will be spit out
// before each block of objects
if (listcmd.StartsWith("LIST") && isRecursive)
{
if (buf.StartsWith("/") && buf.EndsWith(":"))
{
path = buf.TrimEnd(':');
continue;
}
}
// if the next line in the listing starts with spaces
// it is assumed to be a continuation of the current line
if (i + 1 < rawlisting.Count && (rawlisting[i + 1].StartsWith("\t") || rawlisting[i + 1].StartsWith(" ")))
{
buf += rawlisting[++i];
}
try
{
item = m_listParser.ParseSingleLine(path, buf, m_caps, machineList);
}
catch (FtpListParser.CriticalListParseException)
{
this.LogStatus(FtpTraceLevel.Verbose, "Restarting parsing from first entry in list");
i = -1;
lst.Clear();
continue;
}
// FtpListItem.Parse() returns null if the line
// could not be parsed
if (item != null)
{
if (isIncludeSelf || !(item.Name == "." || item.Name == ".."))
{
lst.Add(item);
}
else
{
//this.LogStatus(FtpTraceLevel.Verbose, "Skipped self or parent item: " + item.Name);
}
}
else
{
this.LogStatus(FtpTraceLevel.Warn, "Failed to parse file listing: " + buf);
}
}
// load extended information that wasn't available if the list options flags say to do so.
if (item != null)
{
// try to dereference symbolic links if the appropriate list
// option was passed
if (item.Type == FtpFileSystemObjectType.Link && isDerefLinks)
{
item.LinkObject = await DereferenceLinkAsync(item, token);
}
// if need to get file modified date
if (isGetModified && HasFeature(FtpCapability.MDTM))
{
// if the modified date was not loaded or the modified date is more than a day in the future
// and the server supports the MDTM command, load the modified date.
// most servers do not support retrieving the modified date
// of a directory but we try any way.
if (item.Modified == DateTime.MinValue || listcmd.StartsWith("LIST"))
{
DateTime modify;
if (item.Type == FtpFileSystemObjectType.Directory)
{
this.LogStatus(FtpTraceLevel.Verbose, "Trying to retrieve modification time of a directory, some servers don't like this...");
}
if ((modify = await GetModifiedTimeAsync(item.FullName, token: token)) != DateTime.MinValue)
{
item.Modified = modify;
}
}
}
// if need to get file size
if (isGetSize && HasFeature(FtpCapability.SIZE))
{
// if no size was parsed, the object is a file and the server
// supports the SIZE command, then load the file size
if (item.Size == -1)
{
if (item.Type != FtpFileSystemObjectType.Directory)
{
item.Size = await GetFileSizeAsync(item.FullName, token);
}
else
{
item.Size = 0;
}
}
}
}
}
return(lst.ToArray());
}
private bool DownloadFileToFile(string localPath, string remotePath, FtpLocalExists existsMode, FtpVerify verifyOptions, IProgress <FtpProgress> progress)
{
var outStreamFileMode = FileMode.Create;
// skip downloading if local file size matches
if (existsMode == FtpLocalExists.Append && File.Exists(localPath))
{
if (GetFileSize(remotePath).Equals(new FileInfo(localPath).Length))
{
LogStatus(FtpTraceLevel.Info, "Append is selected => Local file size matches size on server => skipping");
return(false);
}
else
{
outStreamFileMode = FileMode.Append;
}
}
else if (existsMode == FtpLocalExists.Skip && File.Exists(localPath))
{
LogStatus(FtpTraceLevel.Info, "Skip is selected => Local file exists => skipping");
return(false);
}
try {
// create the folders
var dirPath = Path.GetDirectoryName(localPath);
if (!FtpExtensions.IsNullOrWhiteSpace(dirPath) && !Directory.Exists(dirPath))
{
Directory.CreateDirectory(dirPath);
}
}
catch (Exception ex1) {
// catch errors creating directory
throw new FtpException("Error while creating directories. See InnerException for more info.", ex1);
}
bool downloadSuccess;
var verified = true;
var attemptsLeft = verifyOptions.HasFlag(FtpVerify.Retry) ? m_retryAttempts : 1;
do
{
// download the file from server
using (var outStream = new FileStream(localPath, outStreamFileMode, FileAccess.Write, FileShare.None)) {
// download the file straight to a file stream
downloadSuccess = DownloadFileInternal(remotePath, outStream, File.Exists(localPath) ? new FileInfo(localPath).Length : 0, progress);
attemptsLeft--;
}
// if verification is needed
if (downloadSuccess && verifyOptions != FtpVerify.None)
{
verified = VerifyTransfer(localPath, remotePath);
LogLine(FtpTraceLevel.Info, "File Verification: " + (verified ? "PASS" : "FAIL"));
#if DEBUG
if (!verified && attemptsLeft > 0)
{
LogStatus(FtpTraceLevel.Verbose, "Retrying due to failed verification." + (existsMode == FtpLocalExists.Overwrite ? " Overwrite will occur." : "") + " " + attemptsLeft + " attempts remaining");
}
#endif
}
} while (!verified && attemptsLeft > 0);
if (downloadSuccess && !verified && verifyOptions.HasFlag(FtpVerify.Delete))
{
File.Delete(localPath);
}
if (downloadSuccess && !verified && verifyOptions.HasFlag(FtpVerify.Throw))
{
throw new FtpException("Downloaded file checksum value does not match remote file");
}
return(downloadSuccess && verified);
}
/// <summary>
/// Get the records of a file listing and retry if temporary failure.
/// </summary>
private async Task <List <string> > GetListingInternalAsync(string listcmd, bool retry, CancellationToken token)
{
List <string> rawlisting = new List <string>();
// always get the file listing in binary to avoid character translation issues with ASCII.
await SetDataTypeNoLockAsync(FtpDataType.Binary, token);
try {
// read in raw file listing
using (FtpDataStream stream = await OpenDataStreamAsync(listcmd, 0, token)) {
try {
this.LogLine(FtpTraceLevel.Verbose, "+---------------------------------------+");
if (this.BulkListing)
{
// increases performance of GetListing by reading multiple lines of the file listing at once
foreach (var line in await stream.ReadAllLinesAsync(Encoding, this.BulkListingLength, token))
{
if (!FtpExtensions.IsNullOrWhiteSpace(line))
{
rawlisting.Add(line);
this.LogLine(FtpTraceLevel.Verbose, "Listing: " + line);
}
}
}
else
{
// GetListing will read file listings line-by-line (actually byte-by-byte)
string buf;
while ((buf = await stream.ReadLineAsync(Encoding, token)) != null)
{
if (buf.Length > 0)
{
rawlisting.Add(buf);
this.LogLine(FtpTraceLevel.Verbose, "Listing: " + buf);
}
}
}
this.LogLine(FtpTraceLevel.Verbose, "-----------------------------------------");
} finally {
stream.Close();
}
}
} catch (FtpMissingSocketException) {
// Some FTP server does not send any response when listing an empty directory
// and the connection fails because no communication socket is provided by the server
} catch (IOException ioEx) {
// Some FTP servers forcibly close the connection, we absorb these errors
// Fix #410: Retry if its a temporary failure ("Received an unexpected EOF or 0 bytes from the transport stream")
if (retry && ioEx.Message.IsKnownError(unexpectedEOFStrings))
{
// retry once more, but do not go into a infinite recursion loop here
return(await GetListingInternalAsync(listcmd, false, token));
}
else
{
// suppress all other types of exceptions
}
}
return(rawlisting);
}
/// <summary>
/// Checks if a file exists on the server asynchronously.
/// </summary>
/// <param name="path">The full or relative path to the file</param>
/// <param name="token">The token that can be used to cancel the entire process</param>
/// <returns>True if the file exists, false otherwise</returns>
public async Task <bool> FileExistsAsync(string path, CancellationToken token = default(CancellationToken))
{
// verify args
if (path.IsBlank())
{
throw new ArgumentException("Required parameter is null or blank.", "path");
}
LogFunc(nameof(FileExistsAsync), new object[] { path });
// calc the absolute filepath
path = await GetAbsolutePathAsync(path.GetFtpPath(), token);
// since FTP does not include a specific command to check if a file exists
// here we check if file exists by attempting to get its filesize (SIZE)
if (HasFeature(FtpCapability.SIZE))
{
// Fix #328: get filesize in ASCII or Binary mode as required by server
FtpSizeReply sizeReply = new FtpSizeReply();
await GetFileSizeInternalAsync(path, token, sizeReply);
// handle known errors to the SIZE command
var sizeKnownError = CheckFileExistsBySize(sizeReply);
if (sizeKnownError.HasValue)
{
return(sizeKnownError.Value);
}
}
// check if file exists by attempting to get its date modified (MDTM)
if (HasFeature(FtpCapability.MDTM))
{
FtpReply reply = await ExecuteAsync("MDTM " + path, token);
var ch = reply.Code[0];
if (ch == '2')
{
return(true);
}
if (ch == '5' && reply.Message.IsKnownError(FtpServerStrings.fileNotFound))
{
return(false);
}
}
// check if file exists by getting a name listing (NLST)
string[] fileList = await GetNameListingAsync(path.GetFtpDirectoryName(), token);
return(FtpExtensions.FileExistsInNameListing(fileList, path));
// check if file exists by attempting to download it (RETR)
/*try {
* Stream stream = OpenRead(path);
* stream.Close();
* return true;
* } catch (FtpException ex) {
* }*/
return(false);
}
