public static WebClient GetTorWebClient() { WebClient wc = new WebClient(); if (NetworkAdapter.OpenVpnAdapter != null && NetworkAdapter.OpenVpnAdapter.IsConnected) { return(wc); } try { var proxyConfig = new ProxyConfig( //This is an internal http->socks proxy that runs in process IPAddress.Parse(Settings.BrowserProxy), //This is the port your in process http->socks proxy will run on 53549, //This could be an address to a local socks proxy (ex: Tor / Tor Browser, If Tor is running it will be on 127.0.0.1) IPAddress.Parse(Settings.BrowserProxy), //This is the port that the socks proxy lives on (ex: Tor / Tor Browser, Tor is 9150) 9150, //This Can be Socks4 or Socks5 ProxyConfig.SocksVersion.Five ); var proxy = new SocksWebProxy(proxyConfig); wc.Proxy = proxy;// new WebProxy(Settings.BrowserProxy, true); } catch (Exception ex) { Logging.Log("Error: not able to establish Tor proxy."); } return(wc); }
public static EpsData GetEpsData(string symbol, SocksWebProxy proxy) { //ProxyConfig pc = new ProxyConfig(IPAddress.Parse("127.0.0.1"), 1080, ip, 1080, ProxyConfig.SocksVersion.Five, "x7212591", "c4gkjs4rSg"); //var proxy = new SocksWebProxy(pc); WebClient client = new WebClient(); client.Headers.Add("Cache-Control", "no-cache"); client.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache); client.Proxy = proxy; //ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3; string html = ""; int tryCount = 0; while (html.Equals("") && tryCount < 6) { try { html = client.DownloadString("https://ycharts.com/companies/" + symbol + "/eps"); } catch (Exception ex) { Trading.Debug.Nlog(symbol + "\n\t" + ex.Message + " \n\t" + ex.InnerException.Message); tryCount++; System.Threading.Thread.Sleep(500); continue; } break; } if (html.Equals("")) { return(new EpsData()); } HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument(); doc.LoadHtml(html); var page = doc.DocumentNode.SelectSingleNode("//body"); List <HtmlNode> dateNodes = doc.DocumentNode.CssSelect("td.col1").ToList(); List <HtmlNode> epsNodes = doc.DocumentNode.CssSelect("td.col2").ToList(); List <string> dateStrings = dateNodes.Select(x => x.InnerHtml).ToList(); List <string> epsStrings = epsNodes.Select(x => x.InnerHtml.Replace(" ", "").Replace("\n", "")).ToList(); EpsData epsData = new EpsData(dateStrings, epsStrings); return(epsData); }
public static ITelegramBotClient GetBotWithSocksProxy(string apiToken) { var wp = new SocksWebProxy( new ProxyConfig( IPAddress.Parse("127.0.0.1"), GetNextFreePort(), IPAddress.Parse("185.20.184.217"), 3693, ProxyConfig.SocksVersion.Five, "userid66n9", "pSnEA7M"), false); var bc = new TelegramBotClient(apiToken, wp); Console.WriteLine("Created BotClient with SOCKS5 Proxy."); return(bc); }
private static string RunParallel(string url, bool proxyActived = true) { try { var proxy = new SocksWebProxy(new ProxyConfig (IPAddress.Parse("127.0.0.1"), 12345, IPAddress.Parse("127.0.0.1"), 9150, ProxyConfig.SocksVersion.Five)); var client = new WebClient { Proxy = proxyActived ? proxy : null }; var doc = new HtmlAgilityPack.HtmlDocument(); var html = client.DownloadString(url); doc.LoadHtml(html); return(html); } catch { MessageBox.Show("Connection Error"); return(null); } }
public HttpWebClient2(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc) : base(p: p, l: l, c: c, sc: sc) { cookies = new CookieContainer(); var useProxy = false; IWebProxy proxyServer = null; var proxyUrl = serverConfig.GetProxyUrl(); if (!string.IsNullOrWhiteSpace(proxyUrl)) { useProxy = true; NetworkCredential creds = null; if (!serverConfig.ProxyIsAnonymous) { var username = serverConfig.ProxyUsername; var password = serverConfig.ProxyPassword; creds = new NetworkCredential(username, password); } if (serverConfig.ProxyType != ProxyType.Http) { var addresses = Dns.GetHostAddressesAsync(serverConfig.ProxyUrl).Result; var socksConfig = new ProxyConfig { SocksAddress = addresses.FirstOrDefault(), Username = serverConfig.ProxyUsername, Password = serverConfig.ProxyPassword, Version = serverConfig.ProxyType == ProxyType.Socks4 ? ProxyConfig.SocksVersion.Four : ProxyConfig.SocksVersion.Five }; if (serverConfig.ProxyPort.HasValue) { socksConfig.SocksPort = serverConfig.ProxyPort.Value; } proxyServer = new SocksWebProxy(socksConfig, false); } else { proxyServer = new WebProxy(proxyUrl) { BypassProxyOnLocal = false, Credentials = creds }; } } clearanceHandlr = new ClearanceHandler(); clientHandlr = new HttpClientHandler { CookieContainer = cookies, AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more. UseCookies = true, Proxy = proxyServer, UseProxy = useProxy, AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate }; clearanceHandlr.InnerHandler = clientHandlr; client = new HttpClient(clearanceHandlr); }
private static void RunParallel(int count, string url) { var locker = new object(); var proxy = new SocksWebProxy(new ProxyConfig( //This is an internal http->socks proxy that runs in process IPAddress.Parse("127.0.0.1"), //This is the port your in process http->socks proxy will run on 12345, //This could be an address to a local socks proxy (ex: Tor / Tor Browser, If Tor is running it will be on 127.0.0.1) IPAddress.Parse("127.0.0.1"), //This is the port that the socks proxy lives on (ex: Tor / Tor Browser, Tor is 9150) 9150, //This Can be Socks4 or Socks5 ProxyConfig.SocksVersion.Five )); Enumerable.Range(0, count).ToList().ForEach(new Action <int>(x => { if (x != 0) { Thread.Sleep(6000); } WebClient client = new WebClient(); //client.Proxy = proxy.IsActive() ? proxy : null; client.Proxy = proxy; var doc = new HtmlAgilityPack.HtmlDocument(); var html = client.DownloadString(url); doc.LoadHtml(html); var nodes = doc.DocumentNode.SelectNodes("//p/strong"); IPAddress ip; foreach (var node in nodes) { if (IPAddress.TryParse(node.InnerText, out ip)) { lock (locker) { Console.WriteLine(x + ":::::::::::::::::::::"); Console.WriteLine(""); if (html.Contains("Congratulations. This browser is configured to use Tor.")) { Console.WriteLine("Connected through Tor with IP: " + ip.ToString()); } else { Console.Write("Not connected through Tor with IP: " + ip.ToString()); } Console.WriteLine(""); Console.WriteLine(x + ":::::::::::::::::::::"); } return; } } lock (locker) { Console.WriteLine(x + ":::::::::::::::::::::"); Console.WriteLine(""); Console.Write("IP not found"); Console.WriteLine(""); Console.WriteLine(x + ":::::::::::::::::::::"); } })); }
private static void RunParallel(int count, string url) { var locker = new object(); for (int a = 0; a < count; a++) { //if (a != 0) //{ // Thread.Sleep(2000); //} var proxy = new SocksWebProxy(new ProxyConfig( //This is an internal http->socks proxy that runs in process IPAddress.Parse("127.0.0.1"), //This is the port your in process http->socks proxy will run on GetNextFreePort(), //This could be an address to a local socks proxy (ex: Tor / Tor Browser, If Tor is running it will be on 127.0.0.1) IPAddress.Parse("127.0.0.1"), //This is the port that the socks proxy lives on (ex: Tor / Tor Browser, Tor is 9150) 9150, //This Can be Socks4 or Socks5 ProxyConfig.SocksVersion.Five )); proxy = null; GC.Collect(); int counter = 0; WebClient client = new WebClient(); client.Headers.Add("Cache-Control", "no-cache"); client.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache); //client.Proxy = proxy.IsActive() ? proxy : null; client.Proxy = proxy; var doc = new HtmlAgilityPack.HtmlDocument(); var html = client.DownloadString(url); doc.LoadHtml(html); var nodes = doc.DocumentNode.SelectNodes("//p/strong"); IPAddress ip; foreach (var node in nodes) { try { if (IPAddress.TryParse(node.InnerText, out ip)) { //lock (locker) //{ if (oldIp != null) { { while (oldIp.ToString() != ip.ToString()) { counter++; html = client.DownloadString(url + "?random=" + counter); doc.LoadHtml(html); nodes = doc.DocumentNode.SelectNodes("//p/strong"); int index = nodes.FirstOrDefault().InnerText.IndexOf("<"); if (index != -1) { IPAddress.TryParse(nodes.FirstOrDefault().InnerText.Substring(0, index), out ip); } else { IPAddress.TryParse(nodes.FirstOrDefault().InnerText, out ip); } } } } oldIp = ip; Console.WriteLine(a + ":::::::::::::::::::::"); Console.WriteLine(""); if (html.Contains("Congratulations. This browser is configured to use Tor.")) { Console.WriteLine("Connected through Tor with IP: " + ip.ToString()); // Connect to tor, get a new identity and drop existing circuits Socket server = null; try { //Authenticate using control password IPEndPoint endPoint = new IPEndPoint(IPAddress.Parse("127.0.0.1"), 9151); server = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); server.Connect(endPoint); server.Send( Encoding.ASCII.GetBytes("AUTHENTICATE \"password\"" + Environment.NewLine)); byte[] data = new byte[1024]; int receivedDataLength = server.Receive(data); string stringData = Encoding.ASCII.GetString(data, 0, receivedDataLength); //Request a new Identity server.Send(Encoding.ASCII.GetBytes("SIGNAL NEWNYM" + Environment.NewLine)); data = new byte[1024]; receivedDataLength = server.Receive(data); stringData = Encoding.ASCII.GetString(data, 0, receivedDataLength); if (!stringData.Contains("250")) { Console.WriteLine("Unable to signal new user to server."); server.Shutdown(SocketShutdown.Both); server.Close(); } else { Console.WriteLine("SIGNAL NEWNYM sent successfully"); } //Enable circuit events to enable console output server.Send(Encoding.ASCII.GetBytes("setevents circ" + Environment.NewLine)); data = new byte[1024]; receivedDataLength = server.Receive(data); stringData = Encoding.ASCII.GetString(data, 0, receivedDataLength); //Get circuit information server.Send(Encoding.ASCII.GetBytes("getinfo circuit-status" + Environment.NewLine)); data = new byte[16384]; receivedDataLength = server.Receive(data); stringData = Encoding.ASCII.GetString(data, 0, receivedDataLength); stringData = stringData.Substring(stringData.IndexOf("250+circuit-status"), stringData.IndexOf("250 OK") - stringData.IndexOf("250+circuit-status")); var stringArray = stringData.Split(new[] { "\r\n" }, StringSplitOptions.None); foreach (var s in stringArray) { if (s.Contains("BUILT")) { //Close any existing circuit in order to get a new IP var circuit = s.Substring(0, s.IndexOf("BUILT")).Trim(); server.Send( Encoding.ASCII.GetBytes($"closecircuit {circuit}" + Environment.NewLine)); data = new byte[1024]; receivedDataLength = server.Receive(data); stringData = Encoding.ASCII.GetString(data, 0, receivedDataLength); } } } finally { server.Shutdown(SocketShutdown.Both); server.Close(); } } else { Console.Write("Not connected through Tor with IP: " + ip.ToString()); } Console.WriteLine(""); Console.WriteLine(a + ":::::::::::::::::::::"); } //} else { //lock (locker) //{ Console.WriteLine(a + ":::::::::::::::::::::"); Console.WriteLine(""); Console.Write("IP not found"); Console.WriteLine(""); Console.WriteLine(a + ":::::::::::::::::::::"); //} } } catch { } } } }
private static void RunParallel(int count, string url) { var locker = new object(); for (int i = 0; i < count; i++) { if (i != 0) { Thread.Sleep(2000); } var proxy = new SocksWebProxy(new ProxyConfig( //This is an internal http->socks proxy that runs in process IPAddress.Parse("127.0.0.1"), //This is the port your in process http->socks proxy will run on GetNextFreePort(), //This could be an address to a local socks proxy (ex: Tor / Tor Browser, If Tor is running it will be on 127.0.0.1) IPAddress.Parse("127.0.0.1"), //This is the port that the socks proxy lives on (ex: Tor / Tor Browser, Tor is 9150) 9150, //This Can be Socks4 or Socks5 ProxyConfig.SocksVersion.Five )); WebClient client = new WebClient(); client.Headers.Add("Cache-Control", "no-cache"); client.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache); //client.Proxy = proxy.IsActive() ? proxy : null; client.Proxy = proxy; var doc = new HtmlAgilityPack.HtmlDocument(); var html = client.DownloadString(url); doc.LoadHtml(html); var nodes = doc.DocumentNode.SelectNodes("//p/strong"); foreach (var node in nodes) { try { if (IPAddress.TryParse(node.InnerText, out IPAddress ip)) { Console.WriteLine(i + ":::::::::::::::::::::"); Console.WriteLine(""); if (html.Contains("Congratulations. This browser is configured to use Tor.")) { Console.WriteLine("Connected through Tor with IP: " + ip.ToString()); } else { Console.Write("Not connected through Tor with IP: " + ip.ToString()); } Console.WriteLine(""); Console.WriteLine(i + ":::::::::::::::::::::"); } else { Console.WriteLine(i + ":::::::::::::::::::::"); Console.WriteLine(""); Console.Write("IP not found"); Console.WriteLine(""); Console.WriteLine(i + ":::::::::::::::::::::"); } GetNewTorIdentity(); } catch { } } } }
override protected async Task <WebClientByteResult> Run(WebRequest webRequest) { ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072; var cookies = new CookieContainer(); if (!string.IsNullOrEmpty(webRequest.Cookies)) { var uri = new Uri(webRequest.Url); var cookieUrl = new Uri(uri.Scheme + "://" + uri.Host); // don't include the path, Scheme is needed for mono compatibility foreach (var c in webRequest.Cookies.Split(';')) { try { cookies.SetCookies(cookieUrl, c.Trim()); } catch (CookieException ex) { logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message); } } } var useProxy = false; using (ClearanceHandler clearanceHandlr = new ClearanceHandler()) { IWebProxy proxyServer = null; var proxyUrl = serverConfig.GetProxyUrl(); if (!string.IsNullOrWhiteSpace(proxyUrl)) { useProxy = true; NetworkCredential creds = null; if (!serverConfig.ProxyIsAnonymous) { var username = serverConfig.ProxyUsername; var password = serverConfig.ProxyPassword; creds = new NetworkCredential(username, password); } if (serverConfig.ProxyType != ProxyType.Http) { var addresses = await Dns.GetHostAddressesAsync(serverConfig.ProxyUrl); var socksConfig = new ProxyConfig { SocksAddress = addresses.FirstOrDefault(), Username = serverConfig.ProxyUsername, Password = serverConfig.ProxyPassword, Version = serverConfig.ProxyType == ProxyType.Socks4 ? ProxyConfig.SocksVersion.Four : ProxyConfig.SocksVersion.Five }; if (serverConfig.ProxyPort.HasValue) { socksConfig.SocksPort = serverConfig.ProxyPort.Value; } proxyServer = new SocksWebProxy(socksConfig, false); } else { proxyServer = new WebProxy(proxyUrl) { BypassProxyOnLocal = false, Credentials = creds }; } } using (HttpClientHandler clientHandlr = new HttpClientHandler { CookieContainer = cookies, AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more. UseCookies = true, Proxy = proxyServer, UseProxy = useProxy, AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate }) { clearanceHandlr.InnerHandler = clientHandlr; using (var client = new HttpClient(clearanceHandlr)) { if (webRequest.EmulateBrowser == true) { client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent); } else { client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion()); } HttpResponseMessage response = null; using (var request = new HttpRequestMessage()) { request.Headers.ExpectContinue = false; request.RequestUri = new Uri(webRequest.Url); if (webRequest.Headers != null) { foreach (var header in webRequest.Headers) { if (header.Key != "Content-Type") { request.Headers.TryAddWithoutValidation(header.Key, header.Value); } } } if (!string.IsNullOrEmpty(webRequest.Referer)) { request.Headers.Referrer = new Uri(webRequest.Referer); } if (!string.IsNullOrEmpty(webRequest.RawBody)) { var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault(); if (type.HasValue) { var str = new StringContent(webRequest.RawBody); str.Headers.Remove("Content-Type"); str.Headers.Add("Content-Type", type.Value.Value); request.Content = str; } else { request.Content = new StringContent(webRequest.RawBody); } request.Method = HttpMethod.Post; } else if (webRequest.Type == RequestType.POST) { if (webRequest.PostData != null) { request.Content = new FormUrlEncodedContent(webRequest.PostData); } request.Method = HttpMethod.Post; } else { request.Method = HttpMethod.Get; } using (response = await client.SendAsync(request)) { var result = new WebClientByteResult { Content = await response.Content.ReadAsByteArrayAsync() }; foreach (var header in response.Headers) { IEnumerable <string> value = header.Value; result.Headers[header.Key.ToLowerInvariant()] = value.ToArray(); } // some cloudflare clients are using a refresh header // Pull it out manually if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh")) { var refreshHeaders = response.Headers.GetValues("Refresh"); var redirval = ""; var redirtime = 0; if (refreshHeaders != null) { foreach (var value in refreshHeaders) { var start = value.IndexOf("="); var end = value.IndexOf(";"); var len = value.Length; if (start > -1) { redirval = value.Substring(start + 1); result.RedirectingTo = redirval; // normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature // of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally // it shoudln't include service unavailable..only if we have this redirect header. response.StatusCode = System.Net.HttpStatusCode.Redirect; redirtime = Int32.Parse(value.Substring(0, end)); System.Threading.Thread.Sleep(redirtime * 1000); } } } } if (response.Headers.Location != null) { result.RedirectingTo = response.Headers.Location.ToString(); } // Mono won't add the baseurl to relative redirects. // e.g. a "Location: /index.php" header will result in the Uri "file:///index.php" // See issue #1200 if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://")) { var newRedirectingTo = result.RedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host); logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo); result.RedirectingTo = newRedirectingTo; } result.Status = response.StatusCode; // Compatiblity issue between the cookie format and httpclient // Pull it out manually ignoring the expiry date then set it manually // http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer IEnumerable <string> cookieHeaders; var responseCookies = new List <Tuple <string, string> >(); if (response.Headers.TryGetValues("set-cookie", out cookieHeaders)) { foreach (var value in cookieHeaders) { var nameSplit = value.IndexOf('='); if (nameSplit > -1) { responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";")); } } var cookieBuilder = new StringBuilder(); foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1)) { cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2); } result.Cookies = cookieBuilder.ToString().Trim(); } ServerUtil.ResureRedirectIsFullyQualified(webRequest, result); return(result); } } } } } }