/// <summary>
/// Start listing all files an folders starting with provided Uri
/// </summary>
/// <param name="options">Listing preferences and parameters</param>
/// <returns>All listings</returns>
public IEnumerable <ListingInfo> GetListing(ListingOptions options)
{
toVisit.Enqueue(Uri);
yield return(new ListingInfo()
{
IsDirectory = true,
Parent = Uri,
Uri = Uri
});
while (toVisit.TryDequeue(out Uri uri))
{
IEnumerable <ListingInfo> result = processPage(uri, options);
foreach (var l in result)
{
yield return(l);
if (!l.IsDirectory)
{
continue;
}
enqueueDirectory(l, options);
}
}
}
private void enqueueDirectory(ListingInfo l, ListingOptions options)
{
if (options.ShouldFetch != null)
{
var args = new ShouldFetchEventArgs(new Link(l.Uri, l.Uri));
options.ShouldFetch(this, args);
if (args.Cancel)
{
return;
}
}
toVisit.Enqueue(l.Uri);
}
private IEnumerable <ListingInfo> processPage(Uri uri, ListingOptions options)
{
HtmlAgilityPack.HtmlDocument doc;
var url = uri.ToString();
if (visited.Contains(url))
{
return(new ListingInfo[0]);
}
try
{
doc = FetchHelper.FetchResourceDocument(uri, enableCaching: options.AllowCaching);
}
catch (Exception ex)
{
Console.WriteLine("Err: " + ex.Message);
// send to end
toVisit.Enqueue(Uri);
Thread.Sleep(5000);
return(new ListingInfo[0]);
}
// add to 'seen' list
visited.Add(url);
var rows = doc.DocumentNode.SelectNodes("//table/tr");
var pre = doc.DocumentNode.SelectSingleNode("//pre");
if (rows != null)
{
return(processRows(rows, uri, options));
}
else if (pre != null)
{
var nodes = pre.ChildNodes.ToArray(); // index-based
return(processPreLines(nodes, uri, options));
}
else
{
throw new NotImplementedException("This version of the apache directory listing is not supported yet");
}
}
private IEnumerable <ListingInfo> processPreLines(HtmlAgilityPack.HtmlNode[] nodes, Uri uri, ListingOptions options)
{
// [9] is header's <hr>
for (int i = 10; i < nodes.Length; i += 2)
{
if (nodes[i].Name == "hr")
{
break;
}
var text = nodes[i + 1];
var textParts = text.InnerText.Split(' ', StringSplitOptions.RemoveEmptyEntries);
var href = nodes[i].GetAttributeValue("href", "");
var lnkText = nodes[i].InnerText;
bool isParent = lnkText == "Parent Directory";
if (isParent)
{
textParts = new string[] { " ", " ", "-" }
}
;
bool isDir = href.EndsWith("/");
if (isParent && options.NoParent)
{
continue;
}
DateTime.TryParse(textParts[0] + " " + textParts[1], out DateTime lastModified);
string size = textParts[2];
long numSize = processTextualSize(size);
//bool isDir = dir || isParent;
string fileName = "";
if (!isDir)
{
fileName = href;
}
yield return(new ListingInfo()
{
Parent = uri,
Uri = new Uri(uri, href),
LastModified = lastModified,
Size = size,
FileSize = numSize,
IsDirectory = isDir,
FileName = fileName,
FileExtension = fileName.Split('.')[^ 1]
