public bool TryDequeue(out FtpListItem item)
{
var result = !_priorityQueue.IsEmpty
? _priorityQueue.TryDequeue(out item)
: _normalQueue.TryDequeue(out item);
OnRemove?.Invoke(this, new ObservedListEventArgs(item));
return result;
}
public bool Contains(FtpListItem checkItem, IEqualityComparer<FtpListItem> comparer = null)
{
if (comparer == null)
{
comparer = _ftpListItemComparer;
}
return
_priorityQueue.Contains(checkItem, comparer) ||
_normalQueue.Contains(checkItem, comparer);
}
public static void Download(this FtpClient conn, FtpListItem file, string localFile)
{
if (file.Type != FtpFileSystemObjectType.File) throw new FtpException("invaild file");
using (var responseStream = conn.OpenRead(file.FullName))
{
// Cache data in memory.
using (var downloadCache = new MemoryStream(MaxCacheSize))
{
int bytesSize = 0;
int cachedSize = 0;
byte[] downloadBuffer = new byte[BufferSize];
// Download the file until the download is completed.
while (true)
{
// Read a buffer of data from the stream.
bytesSize = responseStream.Read(downloadBuffer, 0,
downloadBuffer.Length);
// If the cache is full, or the download is completed, write
// the data in cache to local file.
if (bytesSize == 0
|| MaxCacheSize < cachedSize + bytesSize)
{
// Write the data in cache to local file.
WriteCacheToFile(downloadCache, localFile, cachedSize);
// Stop downloading the file if the download is paused,
// canceled or completed.
if (bytesSize == 0)
{
break;
}
// Reset cache.
downloadCache.Seek(0, SeekOrigin.Begin);
cachedSize = 0;
}
// Write the data from the buffer to the cache in memory.
downloadCache.Write(downloadBuffer, 0, bytesSize);
cachedSize += bytesSize;
}
}
}
}
public void DeleteFileCallsClientDelete()
{
//Arrange
var file = new FtpListItem
{
FullName = ""
};
_ftpClient.Expect(x => x.DeleteFile(file.FullName));
//Act
_ftpWrapper.DeleteFile(file);
//Assert
_ftpClient.VerifyAllExpectations();
}
public void Enqueue(FtpListItem item, bool priority = false)
{
OnAdd?.Invoke(this, new ObservedListEventArgs(item, priority));
if (priority)
{
if (!_priorityQueue.Any(x => _ftpListItemComparer.Equals(x, item)))
{
_priorityQueue.Enqueue(item);
}
}
else
{
if (!_normalQueue.Any(x => _ftpListItemComparer.Equals(x, item)))
{
_normalQueue.Enqueue(item);
}
}
}
public static void BeginDereferenceLinkExample(FtpListItem item) {
// The using statement here is OK _only_ because m_reset.WaitOne()
// causes the code to block until the async process finishes, otherwise
// the connection object would be disposed early. In practice, you
// typically would not wrap the following code with a using statement.
using (FtpClient conn = new FtpClient()) {
m_reset.Reset();
conn.Host = "localhost";
conn.Credentials = new NetworkCredential("ftptest", "ftptest");
conn.Connect();
if (item.Type == FtpFileSystemObjectType.Link && item.LinkTarget != null) {
conn.BeginDereferenceLink(item, new AsyncCallback(DereferenceLinkCallback), conn);
m_reset.WaitOne();
}
conn.Disconnect();
}
}
/// <summary>
/// Parses LIST format listings
/// </summary>
/// <param name="buf">A line from the listing</param>
/// <param name="capabilities">Server capabilities</param>
/// <returns>FtpListItem if the item is able to be parsed</returns>
static FtpListItem ParseUnixList(string buf, FtpCapability capabilities)
{
string regex =
@"(?<permissions>[\w-]{10})\s+" +
@"(?<objectcount>\d+)\s+" +
@"(?<user>[\w\d]+)\s+" +
@"(?<group>[\w\d]+)\s+" +
@"(?<size>\d+)\s+" +
@"(?<modify>\w+\s+\d+\s+\d+:\d+|\w+\s+\d+\s+\d+)\s+" +
@"(?<name>.*)$";
FtpListItem item = new FtpListItem();
Match m;
if (!(m = Regex.Match(buf, regex, RegexOptions.IgnoreCase)).Success)
return null;
if (m.Groups["permissions"].Value.StartsWith("d"))
item.Type = FtpFileSystemObjectType.Directory;
else if (m.Groups["permissions"].Value.StartsWith("-"))
item.Type = FtpFileSystemObjectType.File;
else
return null;
// if we can't determine a file name then
// we are not considering this a successful parsing operation.
if (m.Groups["name"].Value.Length < 1)
return null;
item.Name = m.Groups["name"].Value;
if (item.Type == FtpFileSystemObjectType.Directory && (item.Name == "." || item.Name == ".."))
return null;
////
// Ignore the Modify times sent in LIST format for files
// when the server has support for the MDTM command
// because they will never be as accurate as what can be had
// by using the MDTM command. MDTM does not work on directories
// so if a modify time was parsed from the listing we will try
// to convert it to a DateTime object and use it for directories.
////
if ((!capabilities.HasFlag(FtpCapability.MDTM) || item.Type == FtpFileSystemObjectType.Directory) && m.Groups["modify"].Value.Length > 0)
item.Modified = m.Groups["modify"].Value.GetFtpDate();
if (m.Groups["size"].Value.Length > 0) {
long size;
if (long.TryParse(m.Groups["size"].Value, out size))
item.Size = size;
}
if (m.Groups["permissions"].Value.Length > 0) {
Match perms = Regex.Match(m.Groups["permissions"].Value,
@"[\w-]{1}(?<owner>[\w-]{3})(?<group>[\w-]{3})(?<others>[\w-]{3})",
RegexOptions.IgnoreCase);
if (perms.Success) {
if (perms.Groups["owner"].Value.Length == 3) {
if (perms.Groups["owner"].Value[0] == 'r')
item.OwnerPermissions |= FtpPermission.Read;
if (perms.Groups["owner"].Value[1] == 'w')
item.OwnerPermissions |= FtpPermission.Write;
if (perms.Groups["owner"].Value[2] == 'x' || perms.Groups["owner"].Value[2] == 's')
item.OwnerPermissions |= FtpPermission.Execute;
if (perms.Groups["owner"].Value[2] == 's' || perms.Groups["owner"].Value[2] == 'S')
item.SpecialPermissions |= FtpSpecialPermissions.SetUserID;
}
if (perms.Groups["group"].Value.Length == 3) {
if (perms.Groups["group"].Value[0] == 'r')
item.GroupPermissions |= FtpPermission.Read;
if (perms.Groups["group"].Value[1] == 'w')
item.GroupPermissions |= FtpPermission.Write;
if (perms.Groups["group"].Value[2] == 'x' || perms.Groups["group"].Value[2] == 's')
item.GroupPermissions |= FtpPermission.Execute;
if (perms.Groups["group"].Value[2] == 's' || perms.Groups["group"].Value[2] == 'S')
item.SpecialPermissions |= FtpSpecialPermissions.SetGroupID;
}
if (perms.Groups["others"].Value.Length == 3) {
if (perms.Groups["others"].Value[0] == 'r')
item.OthersPermissions |= FtpPermission.Read;
if (perms.Groups["others"].Value[1] == 'w')
item.OthersPermissions |= FtpPermission.Write;
if (perms.Groups["others"].Value[2] == 'x' || perms.Groups["others"].Value[2] == 't')
item.OthersPermissions |= FtpPermission.Execute;
if (perms.Groups["others"].Value[2] == 't' || perms.Groups["others"].Value[2] == 'T')
item.SpecialPermissions |= FtpSpecialPermissions.Sticky;
}
}
}
return item;
}
private string GetURL(FtpListItem file)
{
if (file != null && file.Type == FtpFileSystemObjectType.File)
{
FTPAccount accountClone = Account.Clone();
accountClone.SubFolderPath = currentDirectory;
accountClone.HttpHomePathAutoAddSubFolderPath = true;
return accountClone.GetUriPath(file.Name);
}
return null;
}
/// <summary>
/// Parses IIS DOS format listings
/// </summary>
/// <param name="buf">A line from the listing</param>
/// <param name="capabilities">Server capabilities</param>
/// <returns>FtpListItem if the item is able to be parsed</returns>
static FtpListItem ParseDosList(string buf, FtpCapability capabilities)
{
FtpListItem item = new FtpListItem();
Match m;
// directory
if ((m = Regex.Match(buf, @"(?<modify>\d+-\d+-\d+\s+\d+:\d+\w+)\s+<DIR>\s+(?<name>.*)$", RegexOptions.IgnoreCase)).Success) {
DateTime modify;
item.Type = FtpFileSystemObjectType.Directory;
item.Name = m.Groups["name"].Value;
if (DateTime.TryParse(m.Groups["modify"].Value, out modify))
item.Modified = modify;
}
// file
else if ((m = Regex.Match(buf, @"(?<modify>\d+-\d+-\d+\s+\d+:\d+\w+)\s+(?<size>\d+)\s+(?<name>.*)$", RegexOptions.IgnoreCase)).Success) {
DateTime modify;
long size;
item.Type = FtpFileSystemObjectType.File;
item.Name = m.Groups["name"].Value;
if (long.TryParse(m.Groups["size"].Value, out size))
item.Size = size;
if (DateTime.TryParse(m.Groups["modify"].Value, out modify))
item.Modified = modify;
}
else
return null;
return item;
}
/// <summary>
/// Parses MLS* format listings
/// </summary>
/// <param name="buf">A line from the listing</param>
/// <param name="capabilities">Server capabilities</param>
/// <returns>FtpListItem if the item is able to be parsed</returns>
static FtpListItem ParseMachineList(string buf, FtpCapability capabilities)
{
FtpListItem item = new FtpListItem();
Match m;
if (!(m = Regex.Match(buf, "^type=(?<type>.+?);", RegexOptions.IgnoreCase)).Success)
return null;
switch (m.Groups["type"].Value.ToLower()) {
case "dir":
item.Type = FtpFileSystemObjectType.Directory;
break;
case "file":
item.Type = FtpFileSystemObjectType.File;
break;
// These are not supported for now.
case "link":
case "device":
default:
return null;
}
if ((m = Regex.Match(buf, "; (?<name>.*)$", RegexOptions.IgnoreCase)).Success)
item.Name = m.Groups["name"].Value;
else // if we can't parse the file name there is a problem.
return null;
if ((m = Regex.Match(buf, "modify=(?<modify>.+?);", RegexOptions.IgnoreCase)).Success)
item.Modified = m.Groups["modify"].Value.GetFtpDate();
if ((m = Regex.Match(buf, "created?=(?<create>.+?);", RegexOptions.IgnoreCase)).Success)
item.Created = m.Groups["create"].Value.GetFtpDate();
if ((m = Regex.Match(buf, @"size=(?<size>\d+);", RegexOptions.IgnoreCase)).Success) {
long size;
if (long.TryParse(m.Groups["size"].Value, out size))
item.Size = size;
}
if((m = Regex.Match(buf, @"unix.mode=(?<mode>\d+);", RegexOptions.IgnoreCase)).Success) {
if (m.Groups["mode"].Value.Length == 4) {
item.SpecialPermissions = (FtpSpecialPermissions)int.Parse(m.Groups["mode"].Value[0].ToString());
item.OwnerPermissions = (FtpPermission)int.Parse(m.Groups["mode"].Value[1].ToString());
item.GroupPermissions = (FtpPermission)int.Parse(m.Groups["mode"].Value[2].ToString());
item.OthersPermissions = (FtpPermission)int.Parse(m.Groups["mode"].Value[3].ToString());
}
else if(m.Groups["mode"].Value.Length == 3) {
item.OwnerPermissions = (FtpPermission)int.Parse(m.Groups["mode"].Value[0].ToString());
item.GroupPermissions = (FtpPermission)int.Parse(m.Groups["mode"].Value[1].ToString());
item.OthersPermissions = (FtpPermission)int.Parse(m.Groups["mode"].Value[2].ToString());
}
}
return item;
}
public void DownloadFiles(int maxPerSession = 0)
{
if (_downloadStatus)
{
return;
}
_downloadStatus = true;
var downloadCount = 0;
if (maxPerSession != 0)
{
Log.Info($"Beginning session of {maxPerSession} downloads.");
}
FtpListItem item;
while ((maxPerSession == 0 || downloadCount < maxPerSession) && DownloadList.TryDequeue(out item))
{
CurrentItem = item;
try
{
if (DownloadFile(item))
{
downloadCount++;
DeleteFile(item);
}
}
catch (Exception e)
{
Log.Error($"{e.Message}");
TrackErrorCount(item);
}
}
_downloadStatus = false;
}
static FtpListItem ParseVaxList(string buf, FtpCapability capabilities) {
string regex =
@"(?<name>.+)\.(?<extension>.+);(?<version>\d+)\s+" +
@"(?<size>\d+)\s+" +
@"(?<modify>\d+-\w+-\d+\s+\d+:\d+)";
Match m;
if ((m = Regex.Match(buf, regex)).Success) {
FtpListItem item = new FtpListItem();
item.m_name = string.Format("{0}.{1};{2}",
m.Groups["name"].Value,
m.Groups["extension"].Value,
m.Groups["version"].Value);
if (m.Groups["extension"].Value.ToUpper() == "DIR")
item.m_type = FtpFileSystemObjectType.Directory;
else
item.m_type = FtpFileSystemObjectType.File;
if (!long.TryParse(m.Groups["size"].Value, out item.m_size))
item.m_size = -1;
if (!DateTime.TryParse(m.Groups["modify"].Value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out item.m_modified))
item.m_modified = DateTime.MinValue;
return item;
}
return null;
}
private static void downloadFile(FtpClient client, FtpListItem file, bool deleteFtpFile)
{
string destinationPath = String.Format(@"{0}\{1}", Directory.GetCurrentDirectory(), file.Name);
using (var ftpStream = client.OpenRead(file.FullName)) {
using (var fileStream = File.Create(destinationPath, (int)ftpStream.Length)) {
var buffer = new byte[8 * 1024];
int count;
while ((count = ftpStream.Read(buffer, 0, buffer.Length)) > 0) {
fileStream.Write(buffer, 0, count);
}
}
}
DownloadedFile temp = new DownloadedFile();
temp.originalPath = file.FullName;
temp.fullOriginalPath = client.Host + file.FullName;
temp.finalPath = destinationPath;
temp.name = file.Name;
temp.extension = ".txt";
minervaFiles.Add(temp);
if (deleteFtpFile) {
client.DeleteFile(file.FullName);
}
}
private static void ftpStepIntoFolder(FtpListItem folder)
{
logFile.WriteLine(String.Format("stepping into ftp folder {0}", folder));
client.SetWorkingDirectory(folder.FullName);
FtpListItem[] files;
files = client.GetListing();
body = body + "\nfolder: " + folder.Name + "\n";
body = body + "filesFound: " + files.Length + "\n";
if (files.Length > 0)
{
for (var i = 0; i < files.Length; i++)
{
setCurrentDirectory(rootDirectory);
logFile.WriteLine(String.Format("resetting current directory {0}", rootDirectory));
System.IO.Stream output;
bool downloadThisFile = true;
bool fileAlreadyExisted = false;
bool fileDownloadSuccess = true;
//if the file already exists, no reason to redownload it if it's valid
if (System.IO.File.Exists(files[i].Name))
{
//Make sure we don't have a partial file...
FileInfo info = new FileInfo(files[i].Name);
if (info.Length == files[i].Size)
{
logFile.WriteLine(String.Format("{0} already exists in directy, must not have been processed correctly", files[i].Name));
downloadThisFile = false;
fileAlreadyExisted = true;
}
else {
logFile.WriteLine(String.Format("{0} already exists in directy, incorrect file size...partial download", files[i].Name));
}
}
//If the file has an invalid naming convention, no point trying to download it
if(!validateFileName(files[i].Name) && downloadThisFile)
{
//We can flag this file as an error
filesInvalidName++;
Console.WriteLine("{0} name is invalid...not downloading", files[i].Name);
logFile.WriteLine(String.Format("{0} is improperly named...skipping it", files[i].Name));
report.WriteLine(String.Format("{0},-,{1},-,- ,Couldn't parse name! did not download file",files[i].Name,folder.Name));
downloadThisFile = false;
fileAlreadyExisted = false;
fileDownloadSuccess = false;
}
//If the file has an invalid account number, no reason to download it
int accountNumber = extractAccountNumber(files[i].Name);
if (!validateAccountNumber(accountNumber) && downloadThisFile)
{
downloadThisFile = false;
fileAlreadyExisted = false;
logFile.WriteLine("{0} is an invalid account number", accountNumber);
}
if (downloadThisFile)
{
output = new System.IO.FileStream(files[i].Name, System.IO.FileMode.Create, System.IO.FileAccess.ReadWrite);
Stream input = client.OpenRead(files[i].FullName);
//represents miliseconds of delay between each chunk of data read
int downloadThrottle = 0;
int attempts = 0;
int attemptsAllowed = 10;
//start reading the chunks and copying them
logFile.WriteLine(String.Format("Begin download of {0}", files[i].Name));
Console.WriteLine("Download begin! {0}", files[i].Name);
downloadFile(input, output, downloadThrottle);
attempts++;
while (output.Length != input.Length)
{
if (attempts < attemptsAllowed)
{
Console.WriteLine("file did not download correctly...");
logFile.WriteLine(String.Format("{0} downloaded incorrectly...{1} of {2}", files[i].Name,output.Length,input.Length));
//Close the output file, so we can create a new one...
output.Close();
//reopen the file we want to download
input = client.OpenRead(files[i].FullName);
//recreate the file in the download location
output = new System.IO.FileStream(files[i].Name, System.IO.FileMode.Create, System.IO.FileAccess.ReadWrite);
//slightly reduce download speed
downloadThrottle = attempts * 10;
//attempt to download
Console.WriteLine("Attempt redownload of {0}", files[i].Name);
logFile.WriteLine(String.Format("attempt redownload of {0}", files[i].Name));
downloadFile(input, output, downloadThrottle);
//Increase the number of attemps we have had
attempts++;
}
else
{
Console.WriteLine("reached max download attempts...");
logFile.WriteLine(String.Format("Max download attemps reached for {0}. Canceling download Attempt...", files[i].Name));
report.WriteLine(String.Format("{0},{1},{2},-,- ,File download failed 10 times - needs to manual download", files[i].Name,extractAccountNumber(files[i].Name), folder.Name));
fileDownloadSuccess = false;
fileAlreadyExisted = false;
break;
}
}
//close the file we just created
output.Close();
}
if (fileAlreadyExisted || fileDownloadSuccess)
{
Console.WriteLine("File successfully exists in our directory!");
logFile.WriteLine(String.Format("{0} exists successfully!",files[0].Name));
//copy file to its home
List<string> directories = new List<string>() { "COMMV_PDFS", extractAccountNumber(files[i].Name).ToString(), folder.Name };
logFile.WriteLine(String.Format("move file to final destination!"));
KeyValuePair<string, int> kvp1,kvp2 = new KeyValuePair<string, int>();
kvp1 = buildDestinationAndMoveFile(files[i].Name, folder.Name, directories);
//go back to the applications directory
logFile.WriteLine(String.Format("reset currentDirectory {0}", rootDirectory));
Directory.SetCurrentDirectory(rootDirectory);
//Copy file to archive
directories = new List<string>() { "PeruvianAutomatedArchive", "Done", folder.Name };
logFile.WriteLine(String.Format("move file to archive destination!"));
kvp2 = buildDestinationAndMoveFile(files[i].Name, folder.Name, directories);
//insert record into database
insertToDatabase(extractAccountNumber(files[i].Name).ToString(), files[i].Name, kvp1.Key, "0", folder.Name);
//record some success/fail data about the files we are pulling down
switch (kvp1.Value)
{
case 0:
filesFail++;
break;
case 1:
filesDuplicate++;
break;
case 2:
filesMoved++;
break;
}
//delete file from ftp
try
{
client.DeleteFile(files[i].FullName);
logFile.WriteLine(String.Format("Deleted {0} from the ftp server",files[i].Name));
}
catch (Exception e) {
logFile.WriteLine(String.Format("failed to delete {0} from the ftp server", files[i].Name));
logFile.WriteLine(String.Format("{0}", e));
}
//delete file from directory
try {
File.Delete(rootDirectory + "\\" + files[i].Name);
logFile.WriteLine(String.Format("deleted {0} from {1}", files[i].Name,rootDirectory));
}
catch (Exception e) {
logFile.WriteLine(String.Format("delete failed...{e}", e));
}
//open record .csv and append
report.WriteLine(String.Format("{0},{1},{2},{3},{4}, ",files[i].Name,extractAccountNumber(files[i].Name),folder.Name,kvp1.Key,kvp2.Key));
}
}
}
body = body + "Successful moves: " + filesMoved + "\n";
body = body + "File Already Existed: " + filesDuplicate + "\n";
body = body + "Failure to move: " + filesFail + "\n";
body = body + "File invalid name: " + filesInvalidName+ "\n";
}
/// <summary>
/// Parses MLS* format listings
/// </summary>
/// <param name="buf">A line from the listing</param>
/// <param name="capabilities">Server capabilities</param>
/// <returns>FtpListItem if the item is able to be parsed</returns>
static FtpListItem ParseMachineList(string buf, FtpCapability capabilities)
{
FtpListItem item = new FtpListItem();
Match m;
if (!(m = Regex.Match(buf, "^type=(?<type>.+?);", RegexOptions.IgnoreCase)).Success)
{
return(null);
}
switch (m.Groups["type"].Value.ToLower())
{
case "dir":
item.Type = FtpFileSystemObjectType.Directory;
break;
case "file":
item.Type = FtpFileSystemObjectType.File;
break;
// These are not supported for now.
case "link":
case "device":
default:
return(null);
}
if ((m = Regex.Match(buf, "; (?<name>.*)$", RegexOptions.IgnoreCase)).Success)
{
item.Name = m.Groups["name"].Value;
}
else // if we can't parse the file name there is a problem.
{
return(null);
}
if ((m = Regex.Match(buf, "modify=(?<modify>.+?);", RegexOptions.IgnoreCase)).Success)
{
item.Modified = m.Groups["modify"].Value.GetFtpDate();
}
if ((m = Regex.Match(buf, "created?=(?<create>.+?);", RegexOptions.IgnoreCase)).Success)
{
item.Created = m.Groups["create"].Value.GetFtpDate();
}
if ((m = Regex.Match(buf, @"size=(?<size>\d+);", RegexOptions.IgnoreCase)).Success)
{
long size;
if (long.TryParse(m.Groups["size"].Value, out size))
{
item.Size = size;
}
}
if ((m = Regex.Match(buf, @"unix.mode=(?<mode>\d+);", RegexOptions.IgnoreCase)).Success)
{
if (m.Groups["mode"].Value.Length == 4)
{
item.SpecialPermissions = (FtpSpecialPermissions)int.Parse(m.Groups["mode"].Value[0].ToString());
item.OwnerPermissions = (FtpPermission)int.Parse(m.Groups["mode"].Value[1].ToString());
item.GroupPermissions = (FtpPermission)int.Parse(m.Groups["mode"].Value[2].ToString());
item.OthersPermissions = (FtpPermission)int.Parse(m.Groups["mode"].Value[3].ToString());
}
else if (m.Groups["mode"].Value.Length == 3)
{
item.OwnerPermissions = (FtpPermission)int.Parse(m.Groups["mode"].Value[0].ToString());
item.GroupPermissions = (FtpPermission)int.Parse(m.Groups["mode"].Value[1].ToString());
item.OthersPermissions = (FtpPermission)int.Parse(m.Groups["mode"].Value[2].ToString());
}
}
return(item);
}
void DownloadFile(FtpListItem item,string localFolderPath)
{
DownloadFile(item.Name, localFolderPath);
}
/// <summary>
/// Parses LIST format listings
/// </summary>
/// <param name="buf">A line from the listing</param>
/// <param name="capabilities">Server capabilities</param>
/// <returns>FtpListItem if the item is able to be parsed</returns>
static FtpListItem ParseUnixList(string buf, FtpCapability capabilities)
{
string regex =
@"(?<permissions>[\w-]{10})\s+" +
@"(?<objectcount>\d+)\s+" +
@"(?<user>[\w\d]+)\s+" +
@"(?<group>[\w\d]+)\s+" +
@"(?<size>\d+)\s+" +
@"(?<modify>\w+\s+\d+\s+\d+:\d+|\w+\s+\d+\s+\d+)\s+" +
@"(?<name>.*)$";
FtpListItem item = new FtpListItem();
Match m;
if (!(m = Regex.Match(buf, regex, RegexOptions.IgnoreCase)).Success)
{
return(null);
}
if (m.Groups["permissions"].Value.StartsWith("d"))
{
item.Type = FtpFileSystemObjectType.Directory;
}
else if (m.Groups["permissions"].Value.StartsWith("-"))
{
item.Type = FtpFileSystemObjectType.File;
}
else
{
return(null);
}
// if we can't determine a file name then
// we are not considering this a successful parsing operation.
if (m.Groups["name"].Value.Length < 1)
{
return(null);
}
item.Name = m.Groups["name"].Value;
if (item.Type == FtpFileSystemObjectType.Directory && (item.Name == "." || item.Name == ".."))
{
return(null);
}
////
// Ignore the Modify times sent in LIST format for files
// when the server has support for the MDTM command
// because they will never be as accurate as what can be had
// by using the MDTM command. MDTM does not work on directories
// so if a modify time was parsed from the listing we will try
// to convert it to a DateTime object and use it for directories.
////
if ((!capabilities.HasFlag(FtpCapability.MDTM) || item.Type == FtpFileSystemObjectType.Directory) && m.Groups["modify"].Value.Length > 0)
{
item.Modified = m.Groups["modify"].Value.GetFtpDate();
}
if (m.Groups["size"].Value.Length > 0)
{
long size;
if (long.TryParse(m.Groups["size"].Value, out size))
{
item.Size = size;
}
}
if (m.Groups["permissions"].Value.Length > 0)
{
Match perms = Regex.Match(m.Groups["permissions"].Value,
@"[\w-]{1}(?<owner>[\w-]{3})(?<group>[\w-]{3})(?<others>[\w-]{3})",
RegexOptions.IgnoreCase);
if (perms.Success)
{
if (perms.Groups["owner"].Value.Length == 3)
{
if (perms.Groups["owner"].Value[0] == 'r')
{
item.OwnerPermissions |= FtpPermission.Read;
}
if (perms.Groups["owner"].Value[1] == 'w')
{
item.OwnerPermissions |= FtpPermission.Write;
}
if (perms.Groups["owner"].Value[2] == 'x' || perms.Groups["owner"].Value[2] == 's')
{
item.OwnerPermissions |= FtpPermission.Execute;
}
if (perms.Groups["owner"].Value[2] == 's' || perms.Groups["owner"].Value[2] == 'S')
{
item.SpecialPermissions |= FtpSpecialPermissions.SetUserID;
}
}
if (perms.Groups["group"].Value.Length == 3)
{
if (perms.Groups["group"].Value[0] == 'r')
{
item.GroupPermissions |= FtpPermission.Read;
}
if (perms.Groups["group"].Value[1] == 'w')
{
item.GroupPermissions |= FtpPermission.Write;
}
if (perms.Groups["group"].Value[2] == 'x' || perms.Groups["group"].Value[2] == 's')
{
item.GroupPermissions |= FtpPermission.Execute;
}
if (perms.Groups["group"].Value[2] == 's' || perms.Groups["group"].Value[2] == 'S')
{
item.SpecialPermissions |= FtpSpecialPermissions.SetGroupID;
}
}
if (perms.Groups["others"].Value.Length == 3)
{
if (perms.Groups["others"].Value[0] == 'r')
{
item.OthersPermissions |= FtpPermission.Read;
}
if (perms.Groups["others"].Value[1] == 'w')
{
item.OthersPermissions |= FtpPermission.Write;
}
if (perms.Groups["others"].Value[2] == 'x' || perms.Groups["others"].Value[2] == 't')
{
item.OthersPermissions |= FtpPermission.Execute;
}
if (perms.Groups["others"].Value[2] == 't' || perms.Groups["others"].Value[2] == 'T')
{
item.SpecialPermissions |= FtpSpecialPermissions.Sticky;
}
}
}
}
return(item);
}
public void DeleteFile(FtpListItem file)
{
_client.DeleteFile(file.FullName);
if (file.FullName == "") return;
var path = Path.GetDirectoryName(file.FullName);
while (path != "\\" + _settings.FtpRootCheckDir && DeleteFolder(path))
{
path = Path.GetDirectoryName(path);
}
}
private void TrackErrorCount(FtpListItem file)
{
if (_currentErrors.ContainsKey(file.Name))
{
_currentErrors[file.Name]++;
}
else
{
_currentErrors.Add(file.Name, 1);
}
if (_currentErrors[file.Name] < _settings.MaxRetryCount)
{
DownloadList.Enqueue(file);
return;
}
_currentErrors.Remove(file.Name);
Log.Warn($"Removed {file.Name} from download list");
}
/// <summary>
/// Convert an FtpItem to a ClientItem
/// </summary>
private ClientItem ConvertItem(FtpListItem f)
{
var fullPath = f.FullName;
if (fullPath.StartsWith("./"))
{
var cwd = WorkingDirectory;
var wd = (controller.Paths.Remote != null && cwd.StartsWithButNotEqual(controller.Paths.Remote) && cwd != "/") ? cwd : controller.GetCommonPath(cwd, false);
fullPath = fullPath.Substring(2);
if (wd != "/")
fullPath = string.Format("/{0}/{1}", wd, fullPath);
fullPath = fullPath.Replace("//", "/");
}
return new ClientItem
{
Name = f.Name,
FullPath = fullPath,
Type = _ItemTypeOf(f.Type),
Size = f.Size,
LastWriteTime = f.Modified
};
}
/// <summary>
/// Parses LIST format listings
/// </summary>
/// <param name="buf">A line from the listing</param>
/// <param name="capabilities">Server capabilities</param>
/// <returns>FtpListItem if the item is able to be parsed</returns>
static FtpListItem ParseUnixList(string buf, FtpCapability capabilities) {
string regex =
@"(?<permissions>.+)\s+" +
@"(?<objectcount>\d+)\s+" +
@"(?<user>.+)\s+" +
@"(?<group>.+)\s+" +
@"(?<size>\d+)\s+" +
@"(?<modify>\w+\s+\d+\s+\d+:\d+|\w+\s+\d+\s+\d+)\s" +
@"(?<name>.*)$";
FtpListItem item = new FtpListItem();
Match m;
if (!(m = Regex.Match(buf, regex, RegexOptions.IgnoreCase)).Success)
return null;
// if this field is missing we can't determine
// what the object is.
if (m.Groups["permissions"].Value.Length == 0)
return null;
switch (m.Groups["permissions"].Value[0]) {
case 'd':
item.Type = FtpFileSystemObjectType.Directory;
break;
case '-':
case 's':
item.Type = FtpFileSystemObjectType.File;
break;
case 'l':
item.Type = FtpFileSystemObjectType.Link;
break;
default:
return null;
}
// if we can't determine a file name then
// we are not considering this a successful parsing operation.
if (m.Groups["name"].Value.Length < 1)
return null;
item.Name = m.Groups["name"].Value;
switch (item.Type) {
case FtpFileSystemObjectType.Directory:
// ignore these...
if (item.Name == "." || item.Name == "..")
return null;
break;
case FtpFileSystemObjectType.Link:
if (!item.Name.Contains(" -> "))
return null;
item.LinkTarget = item.Name.Remove(0, item.Name.IndexOf("-> ") + 3);
item.Name = item.Name.Remove(item.Name.IndexOf(" -> "));
break;
}
// for date parser testing only
//capabilities = ~(capabilities & FtpCapability.MDTM);
////
// Ignore the Modify times sent in LIST format for files
// when the server has support for the MDTM command
// because they will never be as accurate as what can be had
// by using the MDTM command. MDTM does not work on directories
// so if a modify time was parsed from the listing we will try
// to convert it to a DateTime object and use it for directories.
////
if (((capabilities & FtpCapability.MDTM) != FtpCapability.MDTM || item.Type == FtpFileSystemObjectType.Directory) && m.Groups["modify"].Value.Length > 0) {
item.Modified = m.Groups["modify"].Value.GetFtpDate(DateTimeStyles.AssumeLocal);
if (item.Modified == DateTime.MinValue) {
FtpTrace.WriteLine("GetFtpDate() failed on {0}", m.Groups["modify"].Value);
}
}
else {
if (m.Groups["modify"].Value.Length == 0)
FtpTrace.WriteLine("RegEx failed to parse modified date from {0}.", buf);
else if (item.Type == FtpFileSystemObjectType.Directory)
FtpTrace.WriteLine("Modified times of directories are ignored in UNIX long listings.");
else if ((capabilities & FtpCapability.MDTM) == FtpCapability.MDTM)
FtpTrace.WriteLine("Ignoring modified date because MDTM feature is present. If you aren't already, pass FtpListOption.Modify or FtpListOption.SizeModify to GetListing() to retrieve the modification time.");
}
if (m.Groups["size"].Value.Length > 0) {
long size;
if (long.TryParse(m.Groups["size"].Value, out size))
item.Size = size;
}
if (m.Groups["permissions"].Value.Length > 0) {
Match perms = Regex.Match(m.Groups["permissions"].Value,
@"[\w-]{1}(?<owner>[\w-]{3})(?<group>[\w-]{3})(?<others>[\w-]{3})",
RegexOptions.IgnoreCase);
if (perms.Success) {
if (perms.Groups["owner"].Value.Length == 3) {
if (perms.Groups["owner"].Value[0] == 'r')
item.OwnerPermissions |= FtpPermission.Read;
if (perms.Groups["owner"].Value[1] == 'w')
item.OwnerPermissions |= FtpPermission.Write;
if (perms.Groups["owner"].Value[2] == 'x' || perms.Groups["owner"].Value[2] == 's')
item.OwnerPermissions |= FtpPermission.Execute;
if (perms.Groups["owner"].Value[2] == 's' || perms.Groups["owner"].Value[2] == 'S')
item.SpecialPermissions |= FtpSpecialPermissions.SetUserID;
}
if (perms.Groups["group"].Value.Length == 3) {
if (perms.Groups["group"].Value[0] == 'r')
item.GroupPermissions |= FtpPermission.Read;
if (perms.Groups["group"].Value[1] == 'w')
item.GroupPermissions |= FtpPermission.Write;
if (perms.Groups["group"].Value[2] == 'x' || perms.Groups["group"].Value[2] == 's')
item.GroupPermissions |= FtpPermission.Execute;
if (perms.Groups["group"].Value[2] == 's' || perms.Groups["group"].Value[2] == 'S')
item.SpecialPermissions |= FtpSpecialPermissions.SetGroupID;
}
if (perms.Groups["others"].Value.Length == 3) {
if (perms.Groups["others"].Value[0] == 'r')
item.OthersPermissions |= FtpPermission.Read;
if (perms.Groups["others"].Value[1] == 'w')
item.OthersPermissions |= FtpPermission.Write;
if (perms.Groups["others"].Value[2] == 'x' || perms.Groups["others"].Value[2] == 't')
item.OthersPermissions |= FtpPermission.Execute;
if (perms.Groups["others"].Value[2] == 't' || perms.Groups["others"].Value[2] == 'T')
item.SpecialPermissions |= FtpSpecialPermissions.Sticky;
}
}
}
return item;
}
/// <summary>
/// Parses LIST format listings
/// </summary>
/// <param name="buf">A line from the listing</param>
/// <param name="capabilities">Server capabilities</param>
/// <returns>FtpListItem if the item is able to be parsed</returns>
static FtpListItem ParseUnixList(string buf, FtpCapability capabilities)
{
string regex =
@"(?<permissions>.+)\s+" +
@"(?<objectcount>\d+)\s+" +
@"(?<user>.+)\s+" +
@"(?<group>.+)\s+" +
@"(?<size>\d+)\s+" +
@"(?<modify>\w+\s+\d+\s+\d+:\d+|\w+\s+\d+\s+\d+)\s" +
@"(?<name>.*)$";
FtpListItem item = new FtpListItem();
Match m;
if (!(m = Regex.Match(buf, regex, RegexOptions.IgnoreCase)).Success)
{
return(null);
}
// if this field is missing we can't determine
// what the object is.
if (m.Groups["permissions"].Value.Length == 0)
{
return(null);
}
switch (m.Groups["permissions"].Value[0])
{
case 'd':
item.Type = FtpFileSystemObjectType.Directory;
break;
case '-':
case 's':
item.Type = FtpFileSystemObjectType.File;
break;
case 'l':
item.Type = FtpFileSystemObjectType.Link;
break;
default:
return(null);
}
// if we can't determine a file name then
// we are not considering this a successful parsing operation.
if (m.Groups["name"].Value.Length < 1)
{
return(null);
}
item.Name = m.Groups["name"].Value;
switch (item.Type)
{
case FtpFileSystemObjectType.Directory:
// ignore these...
if (item.Name == "." || item.Name == "..")
{
return(null);
}
break;
case FtpFileSystemObjectType.Link:
if (!item.Name.Contains(" -> "))
{
return(null);
}
item.LinkTarget = item.Name.Remove(0, item.Name.IndexOf("-> ") + 3);
item.Name = item.Name.Remove(item.Name.IndexOf(" -> "));
break;
}
// for date parser testing only
//capabilities = ~(capabilities & FtpCapability.MDTM);
////
// Ignore the Modify times sent in LIST format for files
// when the server has support for the MDTM command
// because they will never be as accurate as what can be had
// by using the MDTM command. MDTM does not work on directories
// so if a modify time was parsed from the listing we will try
// to convert it to a DateTime object and use it for directories.
////
if (((capabilities & FtpCapability.MDTM) != FtpCapability.MDTM || item.Type == FtpFileSystemObjectType.Directory) && m.Groups["modify"].Value.Length > 0)
{
item.Modified = m.Groups["modify"].Value.GetFtpDate(DateTimeStyles.AssumeLocal);
if (item.Modified == DateTime.MinValue)
{
FtpTrace.WriteLine("GetFtpDate() failed on {0}", m.Groups["modify"].Value);
}
}
else
{
if (m.Groups["modify"].Value.Length == 0)
{
FtpTrace.WriteLine("RegEx failed to parse modified date from {0}.", buf);
}
else if (item.Type == FtpFileSystemObjectType.Directory)
{
FtpTrace.WriteLine("Modified times of directories are ignored in UNIX long listings.");
}
else if ((capabilities & FtpCapability.MDTM) == FtpCapability.MDTM)
{
FtpTrace.WriteLine("Ignoring modified date because MDTM feature is present. If you aren't already, pass FtpListOption.Modify or FtpListOption.SizeModify to GetListing() to retrieve the modification time.");
}
}
if (m.Groups["size"].Value.Length > 0)
{
long size;
if (long.TryParse(m.Groups["size"].Value, out size))
{
item.Size = size;
}
}
if (m.Groups["permissions"].Value.Length > 0)
{
Match perms = Regex.Match(m.Groups["permissions"].Value,
@"[\w-]{1}(?<owner>[\w-]{3})(?<group>[\w-]{3})(?<others>[\w-]{3})",
RegexOptions.IgnoreCase);
if (perms.Success)
{
if (perms.Groups["owner"].Value.Length == 3)
{
if (perms.Groups["owner"].Value[0] == 'r')
{
item.OwnerPermissions |= FtpPermission.Read;
}
if (perms.Groups["owner"].Value[1] == 'w')
{
item.OwnerPermissions |= FtpPermission.Write;
}
if (perms.Groups["owner"].Value[2] == 'x' || perms.Groups["owner"].Value[2] == 's')
{
item.OwnerPermissions |= FtpPermission.Execute;
}
if (perms.Groups["owner"].Value[2] == 's' || perms.Groups["owner"].Value[2] == 'S')
{
item.SpecialPermissions |= FtpSpecialPermissions.SetUserID;
}
}
if (perms.Groups["group"].Value.Length == 3)
{
if (perms.Groups["group"].Value[0] == 'r')
{
item.GroupPermissions |= FtpPermission.Read;
}
if (perms.Groups["group"].Value[1] == 'w')
{
item.GroupPermissions |= FtpPermission.Write;
}
if (perms.Groups["group"].Value[2] == 'x' || perms.Groups["group"].Value[2] == 's')
{
item.GroupPermissions |= FtpPermission.Execute;
}
if (perms.Groups["group"].Value[2] == 's' || perms.Groups["group"].Value[2] == 'S')
{
item.SpecialPermissions |= FtpSpecialPermissions.SetGroupID;
}
}
if (perms.Groups["others"].Value.Length == 3)
{
if (perms.Groups["others"].Value[0] == 'r')
{
item.OthersPermissions |= FtpPermission.Read;
}
if (perms.Groups["others"].Value[1] == 'w')
{
item.OthersPermissions |= FtpPermission.Write;
}
if (perms.Groups["others"].Value[2] == 'x' || perms.Groups["others"].Value[2] == 't')
{
item.OthersPermissions |= FtpPermission.Execute;
}
if (perms.Groups["others"].Value[2] == 't' || perms.Groups["others"].Value[2] == 'T')
{
item.SpecialPermissions |= FtpSpecialPermissions.Sticky;
}
}
}
}
return(item);
}
/// <summary>
/// Parses IIS DOS format listings
/// </summary>
/// <param name="buf">A line from the listing</param>
/// <param name="capabilities">Server capabilities</param>
/// <returns>FtpListItem if the item is able to be parsed</returns>
static FtpListItem ParseDosList(string buf, FtpCapability capabilities) {
FtpListItem item = new FtpListItem();
string[] datefmt = new string[] {
"MM-dd-yy hh:mmtt",
"MM-dd-yyyy hh:mmtt"
};
Match m;
// directory
if ((m = Regex.Match(buf, @"(?<modify>\d+-\d+-\d+\s+\d+:\d+\w+)\s+<DIR>\s+(?<name>.*)$", RegexOptions.IgnoreCase)).Success) {
DateTime modify;
item.Type = FtpFileSystemObjectType.Directory;
item.Name = m.Groups["name"].Value;
//if (DateTime.TryParse(m.Groups["modify"].Value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out modify))
if (DateTime.TryParseExact(m.Groups["modify"].Value, datefmt, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out modify))
item.Modified = modify;
}
// file
else if ((m = Regex.Match(buf, @"(?<modify>\d+-\d+-\d+\s+\d+:\d+\w+)\s+(?<size>\d+)\s+(?<name>.*)$", RegexOptions.IgnoreCase)).Success) {
DateTime modify;
long size;
item.Type = FtpFileSystemObjectType.File;
item.Name = m.Groups["name"].Value;
if (long.TryParse(m.Groups["size"].Value, out size))
item.Size = size;
//if (DateTime.TryParse(m.Groups["modify"].Value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out modify))
if (DateTime.TryParseExact(m.Groups["modify"].Value, datefmt, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out modify))
item.Modified = modify;
}
else
return null;
return item;
}
private static void printFiles(FtpListItem[] files)
{
foreach (var dir in files)
{
Console.WriteLine(dir.Name);
}
}
public bool DownloadFile(FtpListItem file)
{
var filePath = _settings.TempDownloadPath + file.Name;
try
{
var fileExists = File.Exists(filePath);
var stats = new ProgressStatistic
{
CurrentBytesSampleCount = 50
};
stats.ProgressChanged += OnProgressUpdated;
stats.Finished += OnProgressUpdated;
if (!_client.FileExists(file.FullName))
{
Log.Warn($"Attempted to download {file.Name}, but it no longer exists on the server.");
DownloadError(new DownloadedEventArgs(new DownloadedFile {DownloadLocation = filePath, FtpListItem = file}));
return false;
}
DownloadStarted(new DownloadedEventArgs(new DownloadedFile { DownloadLocation = filePath, FtpListItem = file }, stats));
using (var local = new FileStream(filePath, fileExists ? FileMode.Append : FileMode.Create))
{
using (var stream = _client.OpenRead(file.FullName, local.Position))
{
var args = new CopyFromArguments(stats.ProgressChange, new TimeSpan(0, 0, 1), stream.Length);
Log.Info($"{(fileExists ? "Resuming" : "Downloading")} {file.Name}");
local.CopyFrom(stream, args);
}
if (local.Length != file.Size)
{
throw new Exception("File reported as download, but is not complete.");
}
}
Log.Info("Completed " + file.Name);
DownloadComplete(new DownloadedEventArgs(new DownloadedFile {DownloadLocation = filePath, FtpListItem = file}, stats));
return true;
}
catch (FtpException e)
{
DownloadError(new DownloadedEventArgs(new DownloadedFile { DownloadLocation = filePath, FtpListItem = file }));
Log.Error("FTP error with " + file.Name);
Log.Error(e.Message);
TrackErrorCount(file);
return false;
}
catch (NotSupportedException)
{
//The file exists, but doesn't support seeking to the end for resume.
//Delete the existing file and try again from scratch.
File.Delete(filePath);
return DownloadFile(file);
}
catch (Exception e)
{
DownloadError(new DownloadedEventArgs(new DownloadedFile { DownloadLocation = filePath, FtpListItem = file }));
Log.Error("Error with " + file.Name);
Log.Error(e.Message);
TrackErrorCount(file);
return false;
}
}
