//Generates a list of valid addresses for a given Internet
public static List <string> GetAddresses(Internet internet)
{
List <string> ValidAdresses = new List <string>();
ValidAdresses = internet.pages.Select(x => x.address).ToList();
return(ValidAdresses);
}
public static InternetResults crawlInternet(Internet internet)
{
InternetResults internetResults = new InternetResults();
internetResults.InternetValid = new List <string>();
internetResults.InternetInvalid = new List <string>();
internetResults.InternetDuplicate = new List <string>();
List <string> ValidAdresses = new List <string>();
//Use this list to determine if link is valid
ValidAdresses = GetAddresses(internet);
foreach (var page in internet.pages)
{
foreach (var link in page.links)
{
//Check to see if Link is valid first
if (ValidAdresses.Contains(link))
{
//Check to see if this site has been visited
if (!internetResults.InternetValid.Contains(link))
{
internetResults.InternetValid.Add(link);
}
//This site does not need to be visited
else
{
internetResults.InternetDuplicate.Add(link);
}
}
//Invalid URL
else
{
internetResults.InternetInvalid.Add(link);
}
}
}
return(internetResults);
}