internal static WebsiteDataModel WsDownloadSycn(string websiteURL)
{
WebsiteDataModel output = new WebsiteDataModel();
WebClient client = new WebClient();
output.WebsiteUrl = websiteURL;
output.WebsiteData = client.DownloadString(websiteURL);
return(output);
}
internal static async Task <WebsiteDataModel> WsDownloadAsycn(string websiteURL)
{
WebsiteDataModel output = new WebsiteDataModel();
WebClient client = new WebClient();
output.WebsiteUrl = websiteURL;
output.WebsiteData = await client.DownloadStringTaskAsync(websiteURL);
return(output);
}
public static List <WebsiteDataModel> RunWebSiteDownloadSync()
{
List <string> websites = GetURLs();
List <WebsiteDataModel> output = new List <WebsiteDataModel>();
Console.WriteLine("download is in progress for site.. ");
foreach (string site in websites)
{
Console.Write($"[{DateTime.UtcNow.Second}.{DateTime.UtcNow.Millisecond}] Download started for {site}");
WebsiteDataModel results = WsDownloadSycn(site);
Console.WriteLine($"..completed [{DateTime.UtcNow.Second}.{DateTime.UtcNow.Millisecond}].");
output.Add(results);
}
return(output);
}
internal static async Task <WebsiteDataModel> WsDownloadPrallelAsycn(string websiteURL)
{
System.Diagnostics.Debug.WriteLine(websiteURL);
Console.WriteLine($"[{DateTime.UtcNow.Second}.{DateTime.UtcNow.Millisecond}] " +
$"Download started for {websiteURL}");
//$"Download started for {websiteURL.Substring(0,websiteURL.IndexOf('/',8))}");
WebsiteDataModel output = new WebsiteDataModel();
WebClient client = new WebClient();
output.WebsiteUrl = websiteURL;
output.WebsiteData = await client.DownloadStringTaskAsync(websiteURL);
//Console.WriteLine($"..completed[{DateTime.UtcNow.Second}.{DateTime.UtcNow.Millisecond}].");
return(output);
}
internal static void ReportWebSiteInfo(WebsiteDataModel dataModel)
{
Console.WriteLine($"{ dataModel.WebsiteUrl } downloaded: { dataModel.WebsiteData.Length } characters long.{ Environment.NewLine }");
}