static async void FetchUri(Uri uri)
{
Console.WriteLine($"Fetching: {uri}");
var httpClient = new HttpClient();
var urlParser = new UrlParser();
var parser = new Parser();
var response = await httpClient.GetAsync(uri);
try
{
response.EnsureSuccessStatusCode();
}
catch (HttpRequestException)
{
Console.WriteLine($"Error '{response.ReasonPhrase}' when trying to fetch: {uri}");
_inProgress--;
return;
}
var urlResults = urlParser.Parse(uri);
var folder = CreateDirectories(urlResults.Folders, uri.Host);
var filePath = Path.Combine(folder, urlResults.Filename);
if (filePath == uri.Host)
{
filePath = Path.Combine(filePath, "index");
}
filePath = CleanupFileExtensions(response, filePath);
await WriteToDisk(response, uri, filePath);
Console.WriteLine($"Download Complete: {uri}");
if (response.Content.Headers.ContentType.MediaType == "text/html")
{
var parseResults = parser.Parse(filePath, href => href.StartsWith(uri.AbsoluteUri));
var resources = parseResults.Resources.Where(x => !ProcessedList.Contains(x) && !Queue.Contains(x));
foreach (var newUri in resources)
{
Queue.Enqueue(newUri);
}
}
_inProgress--;
}
static async void FetchUri(Uri uri)
{
Console.WriteLine($"Fetching: {uri}");
var httpClient = new HttpClient();
var urlParser = new UrlParser();
var parser = new Parser();
var response = await httpClient.GetAsync(uri);
try
{
response.EnsureSuccessStatusCode();
}
catch (HttpRequestException)
{
Console.WriteLine($"Error '{response.ReasonPhrase}' when trying to fetch: {uri}");
_inProgress--;
return;
}
var urlResults = urlParser.Parse(uri);
var folder = CreateDirectories(urlResults.Folders, uri.Host);
var filePath = Path.Combine(folder, urlResults.Filename);
if (filePath == uri.Host)
{
filePath = Path.Combine(filePath, "index");
}
filePath = CleanupFileExtensions(response, filePath);
await WriteToDisk(response, uri, filePath);
Console.WriteLine($"Download Complete: {uri}");
if (response.Content.Headers.ContentType.MediaType == "text/html")
{
var parseResults = parser.Parse(filePath, href => href.StartsWith(uri.AbsoluteUri));
var resources = parseResults.Resources.Where(x => !ProcessedList.Contains(x) && !Queue.Contains(x));
foreach (var newUri in resources)
{
Queue.Enqueue(newUri);
}
}
_inProgress--;
}
