private void cbProxyServers_SelectedIndexChanged(object sender, EventArgs e) { try { string proxyServer = cbProxyServers.SelectedItem.ToString().TrimEnd('/'); if (!proxyServer.Contains("127.0.0.1")) { ConnectionProxy.SetConnectionProxy(proxyServer); } else { ConnectionProxy.RestoreSystemProxy(); } if (wbWhatIsMyIP.Document != null) { wbWhatIsMyIP.Document.Cookie = null; } wbWhatIsMyIP.Navigate("http://www.infobyip.com/detectproxy.php"); } catch (Exception exception) { MessageBox.Show(exception.Message); } }
public Renderer() { try { InitializeComponent(); /**/ //remove limits from service point manager ServicePointManager.MaxServicePoints = 10000; ServicePointManager.DefaultConnectionLimit = 10000; ServicePointManager.CheckCertificateRevocationList = true; ServicePointManager.Expect100Continue = false; ServicePointManager.MaxServicePointIdleTime = 1000 * 30; ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls; ServicePointManager.UseNagleAlgorithm = false; //Use if you encounter certificate errors... ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(delegate { return true; }); /**/ ApplicationSettings applicationSettings = new ApplicationSettings(); _arachnodeDAO = new ArachnodeDAO(applicationSettings.ConnectionString); _htmlRenderer = new HtmlRenderer(_arachnodeDAO); Closed += Renderer_Closed; if (_useAxWebBrowser && !DesignMode) { object o = axWebBrowser1.GetOcx(); IOleObject oleObject = o as IOleObject; oleObject.SetClientSite(this); } axWebBrowser1.Silent = true; if (_useAxWebBrowser) { Thread thread = new Thread(() => { while (true) { Thread.Sleep(1000 * 60 * 1); if (_stopwatch.Elapsed.TotalMinutes > 1) { _stopwatch.Reset(); _stopwatch.Start(); axWebBrowser1.Stop(); axWebBrowser1_DocumentComplete(this, null); } } }); thread.SetApartmentState(ApartmentState.STA); thread.Start(); } /**/ //uncomment these to use... //_rendererActions.Add(new IFrames()); //_rendererActions.Add(new Hrefs()); //_rendererActions.Add(new Inputs()); _htmlRenderer.DocumentComplete += _htmlParser_DocumentComplete; /**/ if (_debugSingleAbsoluteUri || _debugMultipleAbsoluteUris) { return; } /**/ #region Default Crawling Thread //both should be set to 'false' for default crawling execution... if (!_debugSingleAbsoluteUri && !_debugMultipleAbsoluteUris) { _stopwatchTotal.Reset(); _stopwatchTotal.Start(); _thread = new Thread(delegate() { try { MessageQueue rendererMessageQueue = new MessageQueue(".\\private$\\Renderer_Renderers:" + 0); rendererMessageQueue.Formatter = new XmlMessageFormatter(new[] { typeof(RendererMessage) }); while (rendererMessageQueue.Peek() == null) { Thread.Sleep(10); } Message message = rendererMessageQueue.Receive(); _rendererMessage = (RendererMessage)message.Body; /**/ rendererMessageQueue = new MessageQueue(".\\private$\\Renderer_Renderers:" + _rendererMessage.ThreadNumber); rendererMessageQueue.Formatter = new XmlMessageFormatter(new[] { typeof(RendererMessage) }); _engineMessageQueue = new MessageQueue(".\\private$\\Renderer_Engine:" + _rendererMessage.ThreadNumber); /**/ //remoting code for Marshalling the HTMLDocumentClass... BinaryClientFormatterSinkProvider clientProvider = null; BinaryServerFormatterSinkProvider serverProvider = new BinaryServerFormatterSinkProvider(); serverProvider.TypeFilterLevel = TypeFilterLevel.Full; Hashtable props = new Hashtable(); props["name"] = "Renderer" + _rendererMessage.ThreadNumber; props["portName"] = "Renderer" + _rendererMessage.ThreadNumber; props["authorizedGroup"] = WindowsIdentity.GetCurrent().Name; //props["typeFilterLevel"] = TypeFilterLevel.Full; IpcChannel channel = new IpcChannel(props, clientProvider, serverProvider); ChannelServices.RegisterChannel(channel, false); RemotingConfiguration.RegisterWellKnownServiceType(typeof(Renderer), "Renderer" + _rendererMessage.ThreadNumber, WellKnownObjectMode.SingleCall); RemotingServices.Marshal(this, "Renderer" + _rendererMessage.ThreadNumber); /**/ tsslStatus.Text = ".\\private$\\Renderer_Engine:" + _rendererMessage.ThreadNumber + " Awaiting CrawlRequests..."; while (true && !_abortThread) { try { message = rendererMessageQueue.Receive(); _stopwatch.Reset(); _stopwatch.Start(); _rendererMessage = (RendererMessage)message.Body; _htmlRenderer.CrawlRequestTimeoutInMinutes = _rendererMessage.CrawlRequestTimeoutInMinutes; tsslStatus.Text = DateTime.Now.ToLongTimeString() + " .\\private$\\Renderer_Engine:" + _rendererMessage.ThreadNumber + " " + _rendererMessage.AbsoluteUri + " TimeTakenToReceiveMessage: " + _stopwatch.Elapsed.TotalSeconds; if (!_rendererMessage.Kill) { switch (_rendererMessage.RenderAction) { case RenderAction.Render: if (!string.IsNullOrEmpty(_rendererMessage.ProxyServer)) { ConnectionProxy.SetConnectionProxy(_rendererMessage.ProxyServer.TrimEnd('/')); } else { ConnectionProxy.RestoreSystemProxy(); } if (!string.IsNullOrEmpty(_rendererMessage.Cookie)) { //key1=value1;key2=value2; if (!string.IsNullOrEmpty(_rendererMessage.Cookie)) { string[] cookieSplit = _rendererMessage.Cookie.Split(";".ToCharArray()); foreach (string cookieSplit2 in cookieSplit) { string[] cookieSplit3 = cookieSplit2.Split("=".ToCharArray()); if (cookieSplit3.Length >= 2) { StringBuilder stringBuilder = new StringBuilder(); for (int i = 1; i < cookieSplit3.Length; i++) { stringBuilder.Append(cookieSplit3[i] + "="); } string value = stringBuilder.ToString().TrimEnd("=".ToCharArray()); InternetSetCookie(_rendererMessage.AbsoluteUri, cookieSplit3[0], cookieSplit3[1]); } } } } if (_useAxWebBrowser) { object userAgent = "User-Agent: " + _rendererMessage.UserAgent; object o1 = null; object o2 = null; object o3 = null; DateTime startTime = DateTime.Now; axWebBrowser1.Navigate(_rendererMessage.AbsoluteUri, ref o1, ref o2, ref o3, ref userAgent); if (_modifyDOM) { bool wasDOMModified = false; while (axWebBrowser1.ReadyState != tagREADYSTATE.READYSTATE_COMPLETE && DateTime.Now.Subtract(startTime).Duration().TotalMinutes < _rendererMessage.CrawlRequestTimeoutInMinutes) { Thread.Sleep(100); if (axWebBrowser1.ReadyState == tagREADYSTATE.READYSTATE_INTERACTIVE) { if (!wasDOMModified) { _htmlRenderer.ModifyDOM((IHTMLDocument2)axWebBrowser1.Document, false); wasDOMModified = true; } } } } } else { _htmlRenderer.Render(_rendererMessage.AbsoluteUri); } break; case RenderAction.Back: axWebBrowser1.GoBack(); break; case RenderAction.Forward: axWebBrowser1.GoForward(); break; } try { foreach (Process process in Process.GetProcesses()) { if (process.ProcessName.ToLowerInvariant() == "iexplore" || process.ProcessName.ToLowerInvariant() == "chrome" || process.ProcessName.ToLowerInvariant() == "vsjitdebugger" || process.MainWindowTitle.ToLowerInvariant() == "web browser" || process.MainWindowTitle.ToLowerInvariant() == "renderer" || process.MainWindowTitle.ToLowerInvariant() == "visual studio just-in-time debugger") { //if (MessageBox.Show("Close? 1", "Arachnode.Renderer", MessageBoxButtons.YesNo) == DialogResult.Yes) //{ // process.Kill(); //} } } IntPtr window = WinApis.FindWindowByCaption(IntPtr.Zero, "Web Browser"); if (window != IntPtr.Zero) { WinApis.CloseWindow(window); } window = WinApis.FindWindowByCaption(IntPtr.Zero, "Message from webpage"); if (window != IntPtr.Zero) { WinApis.CloseWindow(window); } } catch (Exception exception) { //MessageBox.Show(exception.Message); //MessageBox.Show(exception.StackTrace); _arachnodeDAO.InsertException(null, null, exception, false); } } else { //if (MessageBox.Show("Close? 2", "Arachnode.Renderer", MessageBoxButtons.YesNo) == DialogResult.Yes) //{ // Process.GetCurrentProcess().Kill(); //} } } catch (Exception exception) { //MessageBox.Show(exception.Message); //MessageBox.Show(exception.StackTrace); _arachnodeDAO.InsertException(null, null, exception, false); } } } catch (Exception exception) { //MessageBox.Show(exception.Message); //MessageBox.Show(exception.StackTrace); _arachnodeDAO.InsertException(null, null, exception, false); } });