public void GetData()
{
try
{
List <string> linksToVisit = SearchEngineCore.ParseLinks(BaseUrl);
List <Thread> threads = new List <Thread>();
Dictionary <string, List <Product> > resultDictionary = new Dictionary <string, List <Product> >();
List <LinkStruct> webStructs = new List <LinkStruct>();
linksToVisit.RemoveAll(item => !item.Contains("http") || !item.Contains("https"));
foreach (var link in linksToVisit)
{
Thread subThread = new Thread(new ThreadStart(() => webStructs.Add(new LinkStruct(link, GetSingleHtml(link)))));
subThread.Start();
threads.Add(subThread);
}
foreach (var item in threads)
{
item.Join();
}
threads.Clear();
webStructs.RemoveAll(link => !link.Link.Contains("biedronka") && !link.Link.Contains("lidl") && !link.Link.Contains("kaufland"));
foreach (var webStruct in webStructs)
{
//GetResultsForSingleUrl(resultDictionary, webStruct);
Thread subThread = new Thread(new ThreadStart(() => GetResultsForSingleUrl(resultDictionary, webStruct)));
subThread.Start();
threads.Add(subThread);
}
foreach (var item in threads)
{
item.Join();
}
threads.Clear();
foreach (var products in resultDictionary.Values)
{
//List<Product> distinctProducts = products.GroupBy(p => new { p.Description, p.Provider, p.PriceZl, p.PriceGr }).Select(g => g.First()).ToList();
foreach (var product in products)
{
Products.Add(product);
}
}
Products.TrimExcess();
}
catch (Exception)
{
return;
}
}
