Example #1
0
 public static bool TestIPProxy2(IPProxy proxy)
 {
     try
     {
         var sw = Stopwatch.StartNew();
         proxy.CheckStatus = IPProxy.CheckStatusEnum.Checking;
         var content = SeleniumHelper.GrabPage("http://example.com", proxy.AsTuple());
         sw.Stop();
         proxy.SpeedRate   = (int)sw.ElapsedMilliseconds;
         proxy.CheckStatus = IPProxy.CheckStatusEnum.Checked;
     }
     catch (Exception e)
     {
         proxy.CheckStatus = IPProxy.CheckStatusEnum.CheckedInvalid;
     }
     return(proxy.CheckStatus == IPProxy.CheckStatusEnum.Checked);
 }
        /// <summary>
        ///
        /// </summary>
        /// <param name="proxy"></param>
        public void GrabFirstOrNextPage(IPProxy proxy = null)
        {
            if (AgentStatus != IPProxyAgentStatusEnum.Idle)
            {
                return;
            }

            var pgNo = PageNo.ToString();

            if (PageInstruction != null)
            {
                pgNo = PageInstruction.PageNo(PageNo);
            }
            TargetPgUrl = TargetPageUrlPattern.Replace("{PAGENO}", pgNo);

            AgentStatus = IPProxyAgentStatusEnum.Reading;
            InvokeEventFreeIPProxiesReading(new EventHandlers.FreeIPProxiesReadingEventArgs(TargetPgUrl));

            // Get the html response called from the url
            var contentDoc = ScraperBoxHelper.GrabPage(_driver as PhantomJSDriver, TargetPgUrl, proxy?.AsTuple());

            ValidatePage(contentDoc);
            if (!PageIsValid)
            {
                AgentStatus = IPProxyAgentStatusEnum.Completed;
                InvokeEventFreeIPProviderSourceCompleted(new EventHandlers.FreeIPProviderSourceCompletedEventArgs(PageNo));
                base.Shutdown();
                return;
            }

            // have the inheriting class parse the html result
            ParseProxyPage(contentDoc);
            TotalPagesScraped++;

            // increase the page no for next use.
            PageNo++;
        }