private void GetSourceItemsOnWorker(BackgroundWorker bw, string siteName, string keyword, int pageStart, int pageEnd, DoWorkEventArgs e, HashSet<string> existingIds)
{
var siteFactory = new SiteFactory();
var relevanceCalculater = new RelevanceCalculator();
var allResults = new List<Tuple<string, string, string>>();
var site = siteFactory.GetByName(siteName);
int totalItemCount = 0;
for (var page = pageStart; page <= pageEnd; page++)
{
int pageCount;
var results = site.GetItems(keyword, out pageCount, out totalItemCount, page);
if (page > pageCount)
{
break;
}
if (results == null)
{
continue;
}
if (bw.CancellationPending)
{
e.Cancel = true;
break;
}
allResults.AddRange(results);
}
if (!allResults.Any())
{
if (!e.Cancel)
{
bw.ReportProgress(100, string.Format("No results found for keyword {0} for pages {1}-{2} for the site {3} ", keyword, pageStart, pageEnd, site.Name));
}
return;
}
bw.ReportProgress(100, allResults.Count);
bw.ReportProgress(100, new TotalResultsFoundEventArgs(siteName, allResults.Count, totalItemCount));
var allResultsCount = allResults.Count;
var blockIndex = 0;
do
{
var subResults = allResults.Skip(site.BlockSize * blockIndex).Take(site.BlockSize);
if (!subResults.Any()) break;
blockIndex++;
var listViewItems = new List<ListViewItem>();
var tasks = new List<Task<int>>();
foreach (var result in subResults)
{
var fk = site.Name + "_" + site.GetId(result.Item2);
if (existingIds != null)
{
if (existingIds.Contains(fk))
{
continue;
}
}
tasks.Add(new Task<int>(() =>
{
var item = site.GetItem(result.Item1, result.Item2, result.Item3);
if (item != null)
{
var relevance = relevanceCalculater.GetRelevance(item, keyword);
string[] row1 =
{
item.Id.ToString(), item.Url, item.Title ?? "",
item.MetaDescription ?? "", item.Content ?? "",
item.Price.ToString(CultureInfo.GetCultureInfo("en-US"))
,
string.Join(",",
item.ItemImages.Select(
ii => ii.OriginalSource)),
string.Join(",", item.Tags), site.Name,
item.WordCount.ToString(new CultureInfo("en-US")),
item.Created.ToString("dd-MMM-yyyy",
new CultureInfo("en-US")),
relevance.ToString(),
""
};
lock (_lock)
{
var listViewitem = new ListViewItem("");
listViewitem.SubItems.AddRange(row1);
listViewItems.Add(listViewitem);
bw.ReportProgress(_itemIndex / allResultsCount * 100,
listViewitem);
_itemIndex++;
}
}
return 1;
}));
if (bw.CancellationPending)
{
e.Cancel = true;
break;
}
}
foreach (var task in tasks)
{
task.Start();
}
Task.WaitAll(tasks.ToArray());
lock (_lock)
{
bw.ReportProgress(_itemIndex / allResultsCount * 100, listViewItems);
}
if (bw.CancellationPending)
{
e.Cancel = true;
break;
}
} while (true);
}
private void FillSites()
{
chkSites.Items.Clear();
var siteFactory = new SiteFactory();
foreach (var name in siteFactory.GetNames)
{
chkSites.Items.Add(name);
}
chkSites.SetItemChecked(0, true);
}
