public void GetData()
{
try
{
List <string> linksToVisit = SearchEngineCore.ParseLinks(BaseUrl);
List <Thread> threads = new List <Thread>();
Dictionary <string, List <Product> > resultDictionary = new Dictionary <string, List <Product> >();
List <LinkStruct> webStructs = new List <LinkStruct>();
linksToVisit.RemoveAll(item => !item.Contains("http") || !item.Contains("https"));
foreach (var link in linksToVisit)
{
Thread subThread = new Thread(new ThreadStart(() => webStructs.Add(new LinkStruct(link, GetSingleHtml(link)))));
subThread.Start();
threads.Add(subThread);
}
foreach (var item in threads)
{
item.Join();
}
threads.Clear();
webStructs.RemoveAll(link => !link.Link.Contains("biedronka") && !link.Link.Contains("lidl") && !link.Link.Contains("kaufland"));
foreach (var webStruct in webStructs)
{
//GetResultsForSingleUrl(resultDictionary, webStruct);
Thread subThread = new Thread(new ThreadStart(() => GetResultsForSingleUrl(resultDictionary, webStruct)));
subThread.Start();
threads.Add(subThread);
}
foreach (var item in threads)
{
item.Join();
}
threads.Clear();
foreach (var products in resultDictionary.Values)
{
//List<Product> distinctProducts = products.GroupBy(p => new { p.Description, p.Provider, p.PriceZl, p.PriceGr }).Select(g => g.First()).ToList();
foreach (var product in products)
{
Products.Add(product);
}
}
Products.TrimExcess();
}
catch (Exception)
{
return;
}
}
/// <summary>
/// Initialize result item.
/// </summary>
/// <param name="eventEntity">Event entity.</param>
/// <returns>Result item.</returns>
protected virtual CalendarEventSearchResultItem InitializeItem(CalendarEventEntity eventEntity)
{
CalendarEventSearchResultItem item = new CalendarEventSearchResultItem(this);
string eventUrl = GetEventViewUrl(eventEntity);
item.Title = String.Format("<a href='{0}' title='{1}'>{1}</a>", ClearSearchQueryParams(eventUrl), eventEntity.Title);
item.Url = String.Empty;
string description = GetFormattedEventDescription(eventEntity);
if (ProviderAttributes[BoldSearchPhrasesParam] != null)
{
if (bool.Parse(ProviderAttributes[BoldSearchPhrasesParam]))
{
SearchEngineCore.BoldSearchPhrases(SearchPhrase, ref description);
}
}
item.Description = description;
item.Related = String.Empty;
return(item);
}
