protected static void NagivateBrowserToUrlWithTimeout(BrowserSession.BrowserSession browser, string url, TimeSpan timeout)
{
AutoResetEvent ev = new AutoResetEvent(false);
var thread = new Thread(new ThreadStart(() =>
{
try
{
browser.GoTo(url);
ev.Set();
}
catch (ThreadAbortException)
{
}
}));
thread.Start();
if (!ev.WaitOne(timeout))
{
browser.Stop();
Thread.Sleep(5000);
thread.Abort();
throw new TimeoutException();
}
}
protected override void ScrapeCore(BrowserSession.BrowserSession browser, UpdateStatusDelegate update_status)
{
for (int retry = 0; retry < 30; retry++)
{
try
{
// stop current loading on browser
browser.Stop();
// clear bag
RemoveAllItemsInBag(browser);
// add all items to bag
foreach (var item in this.Items)
{
bool sold_out;
item.GetProductDetailAndAddToBag(browser, update_status, out sold_out);
item.SoldOut = sold_out;
}
// check out
Item.CheckoutAndSelectDomesticDelivery(browser);
// set checkout price
GetCheckoutPriceByRemovingItemOneByOne(browser);
// report status
foreach (var item in this.Items)
{
if (!item.Success.HasValue)
{
item.Success = true;
}
if (item.SoldOut.HasValue && item.SoldOut.Value)
{
continue;
}
item.SoldOut = false;
item.WriteProgress();
update_status(System.Threading.Thread.CurrentThread.ManagedThreadId + ": " + string.Join(" ", item.Title, item.Color, item.OriginalPrice.ToString("f2"), item.CheckoutPrice.ToString("f2")));
}
return;
}
catch (Exception ex)
{
System.Threading.Thread.Sleep(2000);
}
}
foreach (var item in this.Items)
{
item.WriteFailed();
}
update_status("group failed");
}
/// <summary>
/// get url of all items in this category page
/// </summary>
/// <param name="browser"></param>
/// <param name="update_status"></param>
/// <returns></returns>
private string[] GetAllItemUrls(BrowserSession.BrowserSession browser, UpdateStatusDelegate update_status)
{
for (int trial = 0; trial < 10; trial++)
{
try
{
// stop current loading on browser
browser.Stop();
// go to the category page
#if kill
browser.GoTo(this.Url);
#else // kill
NagivateBrowserToUrlWithTimeout(browser, this.Url, TimeSpan.FromMinutes(1));
#endif // kill
var item_urls = new HashSet <string>();
while (true)
{
var product_containers = browser.DefaultFrame.FindElementByClassName("hproduct");
if (product_containers == null)
{
break;
}
bool added = false;
foreach (var product_container in product_containers)
{
var anchors = product_container.FindElementByTagName <BrowserSession.WebElement.Anchor>("a");
if (anchors == null)
{
continue;
}
foreach (var anchor in anchors)
{
if (item_urls.Contains(anchor.Href))
{
continue;
}
item_urls.Add(anchor.Href);
added = true;
}
}
if (!added) // no more
{
break;
}
#if future
// scroll to bottom 5 times to load more items
for (int i = 0; i < 5; i++)
{
browser.ExecuteJavascriptAsync("window.scrollTo(0, document.body.scrollHeight);").Wait();
}
#else // future
break;
#endif // future
}
if (item_urls.Count == 0)
{
throw new Exception("force retry");
}
return(item_urls.ToArray());
}
catch (Exception ex)
{
System.Threading.Thread.Sleep(5000);
}
}
update_status("category failed: " + this.Url);
return(null);
}
protected override void ScrapeCore(BrowserSession.BrowserSession browser, UpdateStatusDelegate update_status)
{
for (int trial = 0; trial < 5; trial++)
{
try
{
// stop current loading on browser
browser.Stop();
bool sold_out;
GetProductDetailAndAddToBag(browser, update_status, out sold_out);
if (sold_out)
{
SoldOut = true;
return;
}
// get checkout price by checking out
{
var original_tab = browser.CurrentTabHandle;
// open a new tab to show cart
try
{
// find the checkout tab
string checkout_tab = null;
{
var tabs = browser.AllTabHandles;
foreach (var tab in tabs)
{
browser.SwitchToTab(tab);
if (browser.Url == URL.CheckoutUrl)
{
checkout_tab = tab;
break;
}
}
}
if (checkout_tab == null)
{
// no checkout tab yet
// create tab
checkout_tab = browser.CreateNewTabAsync().Result;
browser.SwitchToTab(checkout_tab);
}
CheckoutAndSelectDomesticDelivery(browser);
// remove all items in bag except the last one
{
var remove_button_containers = browser.DefaultFrame.FindElementByClassName <BrowserSession.WebElement.Anchor>("remove-consignment-line");
if (remove_button_containers == null || remove_button_containers.Count < 1)
{
throw new Exception("cant find .remove-consignment-line");
}
for (int i = 0; i < remove_button_containers.Count - 1; i++)
{
RemoveItemFromCheckoutPage(browser, 0);
}
}
// get the total price in cart
this.CheckoutPrice = GetTotalPriceOnCheckoutPage(browser);
}
finally
{
// switch back to original tab
browser.SwitchToTab(original_tab);
}
}
WriteProgress();
update_status(string.Join(" ", this.Title, this.Color, this.OriginalPrice.ToString("f2"), this.CheckoutPrice.ToString("f2")));
Success = true;
SoldOut = false;
return;
}
catch (Exception ex)
{
update_status("failed=" + this.Url + " trial=" + trial + " err=" + ex.Message);
}
}
WriteFailed();
Success = false;
}