private void GetSourceItemsOnWorker(BackgroundWorker bw, string siteName, string keyword, int pageStart, int pageEnd, DoWorkEventArgs e, HashSet<string> existingIds) { var siteFactory = new SiteFactory(); var relevanceCalculater = new RelevanceCalculator(); var allResults = new List<Tuple<string, string, string>>(); var site = siteFactory.GetByName(siteName); int totalItemCount = 0; for (var page = pageStart; page <= pageEnd; page++) { int pageCount; var results = site.GetItems(keyword, out pageCount, out totalItemCount, page); if (page > pageCount) { break; } if (results == null) { continue; } if (bw.CancellationPending) { e.Cancel = true; break; } allResults.AddRange(results); } if (!allResults.Any()) { if (!e.Cancel) { bw.ReportProgress(100, string.Format("No results found for keyword {0} for pages {1}-{2} for the site {3} ", keyword, pageStart, pageEnd, site.Name)); } return; } bw.ReportProgress(100, allResults.Count); bw.ReportProgress(100, new TotalResultsFoundEventArgs(siteName, allResults.Count, totalItemCount)); var allResultsCount = allResults.Count; var blockIndex = 0; do { var subResults = allResults.Skip(site.BlockSize * blockIndex).Take(site.BlockSize); if (!subResults.Any()) break; blockIndex++; var listViewItems = new List<ListViewItem>(); var tasks = new List<Task<int>>(); foreach (var result in subResults) { var fk = site.Name + "_" + site.GetId(result.Item2); if (existingIds != null) { if (existingIds.Contains(fk)) { continue; } } tasks.Add(new Task<int>(() => { var item = site.GetItem(result.Item1, result.Item2, result.Item3); if (item != null) { var relevance = relevanceCalculater.GetRelevance(item, keyword); string[] row1 = { item.Id.ToString(), item.Url, item.Title ?? "", item.MetaDescription ?? "", item.Content ?? "", item.Price.ToString(CultureInfo.GetCultureInfo("en-US")) , string.Join(",", item.ItemImages.Select( ii => ii.OriginalSource)), string.Join(",", item.Tags), site.Name, item.WordCount.ToString(new CultureInfo("en-US")), item.Created.ToString("dd-MMM-yyyy", new CultureInfo("en-US")), relevance.ToString(), "" }; lock (_lock) { var listViewitem = new ListViewItem(""); listViewitem.SubItems.AddRange(row1); listViewItems.Add(listViewitem); bw.ReportProgress(_itemIndex / allResultsCount * 100, listViewitem); _itemIndex++; } } return 1; })); if (bw.CancellationPending) { e.Cancel = true; break; } } foreach (var task in tasks) { task.Start(); } Task.WaitAll(tasks.ToArray()); lock (_lock) { bw.ReportProgress(_itemIndex / allResultsCount * 100, listViewItems); } if (bw.CancellationPending) { e.Cancel = true; break; } } while (true); }
private void FillSites() { chkSites.Items.Clear(); var siteFactory = new SiteFactory(); foreach (var name in siteFactory.GetNames) { chkSites.Items.Add(name); } chkSites.SetItemChecked(0, true); }