public static bool TestIPProxy2(IPProxy proxy)
{
try
{
var sw = Stopwatch.StartNew();
proxy.CheckStatus = IPProxy.CheckStatusEnum.Checking;
var content = SeleniumHelper.GrabPage("http://example.com", proxy.AsTuple());
sw.Stop();
proxy.SpeedRate = (int)sw.ElapsedMilliseconds;
proxy.CheckStatus = IPProxy.CheckStatusEnum.Checked;
}
catch (Exception e)
{
proxy.CheckStatus = IPProxy.CheckStatusEnum.CheckedInvalid;
}
return(proxy.CheckStatus == IPProxy.CheckStatusEnum.Checked);
}
/// <summary>
///
/// </summary>
/// <param name="proxy"></param>
public void GrabFirstOrNextPage(IPProxy proxy = null)
{
if (AgentStatus != IPProxyAgentStatusEnum.Idle)
{
return;
}
var pgNo = PageNo.ToString();
if (PageInstruction != null)
{
pgNo = PageInstruction.PageNo(PageNo);
}
TargetPgUrl = TargetPageUrlPattern.Replace("{PAGENO}", pgNo);
AgentStatus = IPProxyAgentStatusEnum.Reading;
InvokeEventFreeIPProxiesReading(new EventHandlers.FreeIPProxiesReadingEventArgs(TargetPgUrl));
// Get the html response called from the url
var contentDoc = ScraperBoxHelper.GrabPage(_driver as PhantomJSDriver, TargetPgUrl, proxy?.AsTuple());
ValidatePage(contentDoc);
if (!PageIsValid)
{
AgentStatus = IPProxyAgentStatusEnum.Completed;
InvokeEventFreeIPProviderSourceCompleted(new EventHandlers.FreeIPProviderSourceCompletedEventArgs(PageNo));
base.Shutdown();
return;
}
// have the inheriting class parse the html result
ParseProxyPage(contentDoc);
TotalPagesScraped++;
// increase the page no for next use.
PageNo++;
}