public static WebClient GetTorWebClient()
{
WebClient wc = new WebClient();
if (NetworkAdapter.OpenVpnAdapter != null && NetworkAdapter.OpenVpnAdapter.IsConnected)
{
return(wc);
}
try
{
var proxyConfig = new ProxyConfig(
//This is an internal http->socks proxy that runs in process
IPAddress.Parse(Settings.BrowserProxy),
//This is the port your in process http->socks proxy will run on
53549,
//This could be an address to a local socks proxy (ex: Tor / Tor Browser, If Tor is running it will be on 127.0.0.1)
IPAddress.Parse(Settings.BrowserProxy),
//This is the port that the socks proxy lives on (ex: Tor / Tor Browser, Tor is 9150)
9150,
//This Can be Socks4 or Socks5
ProxyConfig.SocksVersion.Five
);
var proxy = new SocksWebProxy(proxyConfig);
wc.Proxy = proxy;// new WebProxy(Settings.BrowserProxy, true);
}
catch (Exception ex)
{
Logging.Log("Error: not able to establish Tor proxy.");
}
return(wc);
}
public static EpsData GetEpsData(string symbol, SocksWebProxy proxy)
{
//ProxyConfig pc = new ProxyConfig(IPAddress.Parse("127.0.0.1"), 1080, ip, 1080, ProxyConfig.SocksVersion.Five, "x7212591", "c4gkjs4rSg");
//var proxy = new SocksWebProxy(pc);
WebClient client = new WebClient();
client.Headers.Add("Cache-Control", "no-cache");
client.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache);
client.Proxy = proxy;
//ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3;
string html = "";
int tryCount = 0;
while (html.Equals("") && tryCount < 6)
{
try
{
html = client.DownloadString("https://ycharts.com/companies/" + symbol + "/eps");
}
catch (Exception ex)
{
Trading.Debug.Nlog(symbol + "\n\t" + ex.Message + " \n\t" + ex.InnerException.Message);
tryCount++;
System.Threading.Thread.Sleep(500);
continue;
}
break;
}
if (html.Equals(""))
{
return(new EpsData());
}
HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
doc.LoadHtml(html);
var page = doc.DocumentNode.SelectSingleNode("//body");
List <HtmlNode> dateNodes = doc.DocumentNode.CssSelect("td.col1").ToList();
List <HtmlNode> epsNodes = doc.DocumentNode.CssSelect("td.col2").ToList();
List <string> dateStrings = dateNodes.Select(x => x.InnerHtml).ToList();
List <string> epsStrings = epsNodes.Select(x => x.InnerHtml.Replace(" ", "").Replace("\n", "")).ToList();
EpsData epsData = new EpsData(dateStrings, epsStrings);
return(epsData);
}
public static ITelegramBotClient GetBotWithSocksProxy(string apiToken)
{
var wp = new SocksWebProxy(
new ProxyConfig(
IPAddress.Parse("127.0.0.1"),
GetNextFreePort(),
IPAddress.Parse("185.20.184.217"),
3693,
ProxyConfig.SocksVersion.Five,
"userid66n9",
"pSnEA7M"),
false);
var bc = new TelegramBotClient(apiToken, wp);
Console.WriteLine("Created BotClient with SOCKS5 Proxy.");
return(bc);
}
private static string RunParallel(string url, bool proxyActived = true)
{
try
{
var proxy = new SocksWebProxy(new ProxyConfig
(IPAddress.Parse("127.0.0.1"), 12345, IPAddress.Parse("127.0.0.1"), 9150, ProxyConfig.SocksVersion.Five));
var client = new WebClient {
Proxy = proxyActived ? proxy : null
};
var doc = new HtmlAgilityPack.HtmlDocument();
var html = client.DownloadString(url);
doc.LoadHtml(html);
return(html);
}
catch
{
MessageBox.Show("Connection Error");
return(null);
}
}
public HttpWebClient2(IProcessService p, Logger l, IConfigurationService c, ServerConfig sc)
: base(p: p,
l: l,
c: c,
sc: sc)
{
cookies = new CookieContainer();
var useProxy = false;
IWebProxy proxyServer = null;
var proxyUrl = serverConfig.GetProxyUrl();
if (!string.IsNullOrWhiteSpace(proxyUrl))
{
useProxy = true;
NetworkCredential creds = null;
if (!serverConfig.ProxyIsAnonymous)
{
var username = serverConfig.ProxyUsername;
var password = serverConfig.ProxyPassword;
creds = new NetworkCredential(username, password);
}
if (serverConfig.ProxyType != ProxyType.Http)
{
var addresses = Dns.GetHostAddressesAsync(serverConfig.ProxyUrl).Result;
var socksConfig = new ProxyConfig
{
SocksAddress = addresses.FirstOrDefault(),
Username = serverConfig.ProxyUsername,
Password = serverConfig.ProxyPassword,
Version = serverConfig.ProxyType == ProxyType.Socks4 ?
ProxyConfig.SocksVersion.Four :
ProxyConfig.SocksVersion.Five
};
if (serverConfig.ProxyPort.HasValue)
{
socksConfig.SocksPort = serverConfig.ProxyPort.Value;
}
proxyServer = new SocksWebProxy(socksConfig, false);
}
else
{
proxyServer = new WebProxy(proxyUrl)
{
BypassProxyOnLocal = false,
Credentials = creds
};
}
}
clearanceHandlr = new ClearanceHandler();
clientHandlr = new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
Proxy = proxyServer,
UseProxy = useProxy,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
};
clearanceHandlr.InnerHandler = clientHandlr;
client = new HttpClient(clearanceHandlr);
}
private static void RunParallel(int count, string url)
{
var locker = new object();
var proxy = new SocksWebProxy(new ProxyConfig(
//This is an internal http->socks proxy that runs in process
IPAddress.Parse("127.0.0.1"),
//This is the port your in process http->socks proxy will run on
12345,
//This could be an address to a local socks proxy (ex: Tor / Tor Browser, If Tor is running it will be on 127.0.0.1)
IPAddress.Parse("127.0.0.1"),
//This is the port that the socks proxy lives on (ex: Tor / Tor Browser, Tor is 9150)
9150,
//This Can be Socks4 or Socks5
ProxyConfig.SocksVersion.Five
));
Enumerable.Range(0, count).ToList().ForEach(new Action <int>(x =>
{
if (x != 0)
{
Thread.Sleep(6000);
}
WebClient client = new WebClient();
//client.Proxy = proxy.IsActive() ? proxy : null;
client.Proxy = proxy;
var doc = new HtmlAgilityPack.HtmlDocument();
var html = client.DownloadString(url);
doc.LoadHtml(html);
var nodes = doc.DocumentNode.SelectNodes("//p/strong");
IPAddress ip;
foreach (var node in nodes)
{
if (IPAddress.TryParse(node.InnerText, out ip))
{
lock (locker)
{
Console.WriteLine(x + ":::::::::::::::::::::");
Console.WriteLine("");
if (html.Contains("Congratulations. This browser is configured to use Tor."))
{
Console.WriteLine("Connected through Tor with IP: " + ip.ToString());
}
else
{
Console.Write("Not connected through Tor with IP: " + ip.ToString());
}
Console.WriteLine("");
Console.WriteLine(x + ":::::::::::::::::::::");
}
return;
}
}
lock (locker)
{
Console.WriteLine(x + ":::::::::::::::::::::");
Console.WriteLine("");
Console.Write("IP not found");
Console.WriteLine("");
Console.WriteLine(x + ":::::::::::::::::::::");
}
}));
}
private static void RunParallel(int count, string url)
{
var locker = new object();
for (int a = 0; a < count; a++)
{
//if (a != 0)
//{
// Thread.Sleep(2000);
//}
var proxy = new SocksWebProxy(new ProxyConfig(
//This is an internal http->socks proxy that runs in process
IPAddress.Parse("127.0.0.1"),
//This is the port your in process http->socks proxy will run on
GetNextFreePort(),
//This could be an address to a local socks proxy (ex: Tor / Tor Browser, If Tor is running it will be on 127.0.0.1)
IPAddress.Parse("127.0.0.1"),
//This is the port that the socks proxy lives on (ex: Tor / Tor Browser, Tor is 9150)
9150,
//This Can be Socks4 or Socks5
ProxyConfig.SocksVersion.Five
));
proxy = null;
GC.Collect();
int counter = 0;
WebClient client = new WebClient();
client.Headers.Add("Cache-Control", "no-cache");
client.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache);
//client.Proxy = proxy.IsActive() ? proxy : null;
client.Proxy = proxy;
var doc = new HtmlAgilityPack.HtmlDocument();
var html = client.DownloadString(url);
doc.LoadHtml(html);
var nodes = doc.DocumentNode.SelectNodes("//p/strong");
IPAddress ip;
foreach (var node in nodes)
{
try
{
if (IPAddress.TryParse(node.InnerText, out ip))
{
//lock (locker)
//{
if (oldIp != null)
{
{
while (oldIp.ToString() != ip.ToString())
{
counter++;
html = client.DownloadString(url + "?random=" + counter);
doc.LoadHtml(html);
nodes = doc.DocumentNode.SelectNodes("//p/strong");
int index = nodes.FirstOrDefault().InnerText.IndexOf("<");
if (index != -1)
{
IPAddress.TryParse(nodes.FirstOrDefault().InnerText.Substring(0, index),
out ip);
}
else
{
IPAddress.TryParse(nodes.FirstOrDefault().InnerText, out ip);
}
}
}
}
oldIp = ip;
Console.WriteLine(a + ":::::::::::::::::::::");
Console.WriteLine("");
if (html.Contains("Congratulations. This browser is configured to use Tor."))
{
Console.WriteLine("Connected through Tor with IP: " + ip.ToString());
// Connect to tor, get a new identity and drop existing circuits
Socket server = null;
try
{
//Authenticate using control password
IPEndPoint endPoint = new IPEndPoint(IPAddress.Parse("127.0.0.1"), 9151);
server = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
server.Connect(endPoint);
server.Send(
Encoding.ASCII.GetBytes("AUTHENTICATE \"password\"" + Environment.NewLine));
byte[] data = new byte[1024];
int receivedDataLength = server.Receive(data);
string stringData = Encoding.ASCII.GetString(data, 0, receivedDataLength);
//Request a new Identity
server.Send(Encoding.ASCII.GetBytes("SIGNAL NEWNYM" + Environment.NewLine));
data = new byte[1024];
receivedDataLength = server.Receive(data);
stringData = Encoding.ASCII.GetString(data, 0, receivedDataLength);
if (!stringData.Contains("250"))
{
Console.WriteLine("Unable to signal new user to server.");
server.Shutdown(SocketShutdown.Both);
server.Close();
}
else
{
Console.WriteLine("SIGNAL NEWNYM sent successfully");
}
//Enable circuit events to enable console output
server.Send(Encoding.ASCII.GetBytes("setevents circ" + Environment.NewLine));
data = new byte[1024];
receivedDataLength = server.Receive(data);
stringData = Encoding.ASCII.GetString(data, 0, receivedDataLength);
//Get circuit information
server.Send(Encoding.ASCII.GetBytes("getinfo circuit-status" + Environment.NewLine));
data = new byte[16384];
receivedDataLength = server.Receive(data);
stringData = Encoding.ASCII.GetString(data, 0, receivedDataLength);
stringData = stringData.Substring(stringData.IndexOf("250+circuit-status"),
stringData.IndexOf("250 OK") - stringData.IndexOf("250+circuit-status"));
var stringArray = stringData.Split(new[] { "\r\n" }, StringSplitOptions.None);
foreach (var s in stringArray)
{
if (s.Contains("BUILT"))
{
//Close any existing circuit in order to get a new IP
var circuit = s.Substring(0, s.IndexOf("BUILT")).Trim();
server.Send(
Encoding.ASCII.GetBytes($"closecircuit {circuit}" + Environment.NewLine));
data = new byte[1024];
receivedDataLength = server.Receive(data);
stringData = Encoding.ASCII.GetString(data, 0, receivedDataLength);
}
}
}
finally
{
server.Shutdown(SocketShutdown.Both);
server.Close();
}
}
else
{
Console.Write("Not connected through Tor with IP: " + ip.ToString());
}
Console.WriteLine("");
Console.WriteLine(a + ":::::::::::::::::::::");
}
//}
else
{
//lock (locker)
//{
Console.WriteLine(a + ":::::::::::::::::::::");
Console.WriteLine("");
Console.Write("IP not found");
Console.WriteLine("");
Console.WriteLine(a + ":::::::::::::::::::::");
//}
}
} catch { }
}
}
}
private static void RunParallel(int count, string url)
{
var locker = new object();
for (int i = 0; i < count; i++)
{
if (i != 0)
{
Thread.Sleep(2000);
}
var proxy = new SocksWebProxy(new ProxyConfig(
//This is an internal http->socks proxy that runs in process
IPAddress.Parse("127.0.0.1"),
//This is the port your in process http->socks proxy will run on
GetNextFreePort(),
//This could be an address to a local socks proxy (ex: Tor / Tor Browser, If Tor is running it will be on 127.0.0.1)
IPAddress.Parse("127.0.0.1"),
//This is the port that the socks proxy lives on (ex: Tor / Tor Browser, Tor is 9150)
9150,
//This Can be Socks4 or Socks5
ProxyConfig.SocksVersion.Five
));
WebClient client = new WebClient();
client.Headers.Add("Cache-Control", "no-cache");
client.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache);
//client.Proxy = proxy.IsActive() ? proxy : null;
client.Proxy = proxy;
var doc = new HtmlAgilityPack.HtmlDocument();
var html = client.DownloadString(url);
doc.LoadHtml(html);
var nodes = doc.DocumentNode.SelectNodes("//p/strong");
foreach (var node in nodes)
{
try
{
if (IPAddress.TryParse(node.InnerText, out IPAddress ip))
{
Console.WriteLine(i + ":::::::::::::::::::::");
Console.WriteLine("");
if (html.Contains("Congratulations. This browser is configured to use Tor."))
{
Console.WriteLine("Connected through Tor with IP: " + ip.ToString());
}
else
{
Console.Write("Not connected through Tor with IP: " + ip.ToString());
}
Console.WriteLine("");
Console.WriteLine(i + ":::::::::::::::::::::");
}
else
{
Console.WriteLine(i + ":::::::::::::::::::::");
Console.WriteLine("");
Console.Write("IP not found");
Console.WriteLine("");
Console.WriteLine(i + ":::::::::::::::::::::");
}
GetNewTorIdentity();
}
catch { }
}
}
}
override protected async Task <WebClientByteResult> Run(WebRequest webRequest)
{
ServicePointManager.SecurityProtocol = (SecurityProtocolType)192 | (SecurityProtocolType)768 | (SecurityProtocolType)3072;
var cookies = new CookieContainer();
if (!string.IsNullOrEmpty(webRequest.Cookies))
{
var uri = new Uri(webRequest.Url);
var cookieUrl = new Uri(uri.Scheme + "://" + uri.Host); // don't include the path, Scheme is needed for mono compatibility
foreach (var c in webRequest.Cookies.Split(';'))
{
try
{
cookies.SetCookies(cookieUrl, c.Trim());
}
catch (CookieException ex)
{
logger.Info("(Non-critical) Problem loading cookie {0}, {1}, {2}", uri, c, ex.Message);
}
}
}
var useProxy = false;
using (ClearanceHandler clearanceHandlr = new ClearanceHandler())
{
IWebProxy proxyServer = null;
var proxyUrl = serverConfig.GetProxyUrl();
if (!string.IsNullOrWhiteSpace(proxyUrl))
{
useProxy = true;
NetworkCredential creds = null;
if (!serverConfig.ProxyIsAnonymous)
{
var username = serverConfig.ProxyUsername;
var password = serverConfig.ProxyPassword;
creds = new NetworkCredential(username, password);
}
if (serverConfig.ProxyType != ProxyType.Http)
{
var addresses = await Dns.GetHostAddressesAsync(serverConfig.ProxyUrl);
var socksConfig = new ProxyConfig
{
SocksAddress = addresses.FirstOrDefault(),
Username = serverConfig.ProxyUsername,
Password = serverConfig.ProxyPassword,
Version = serverConfig.ProxyType == ProxyType.Socks4 ?
ProxyConfig.SocksVersion.Four :
ProxyConfig.SocksVersion.Five
};
if (serverConfig.ProxyPort.HasValue)
{
socksConfig.SocksPort = serverConfig.ProxyPort.Value;
}
proxyServer = new SocksWebProxy(socksConfig, false);
}
else
{
proxyServer = new WebProxy(proxyUrl)
{
BypassProxyOnLocal = false,
Credentials = creds
};
}
}
using (HttpClientHandler clientHandlr = new HttpClientHandler
{
CookieContainer = cookies,
AllowAutoRedirect = false, // Do not use this - Bugs ahoy! Lost cookies and more.
UseCookies = true,
Proxy = proxyServer,
UseProxy = useProxy,
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
})
{
clearanceHandlr.InnerHandler = clientHandlr;
using (var client = new HttpClient(clearanceHandlr))
{
if (webRequest.EmulateBrowser == true)
{
client.DefaultRequestHeaders.Add("User-Agent", BrowserUtil.ChromeUserAgent);
}
else
{
client.DefaultRequestHeaders.Add("User-Agent", "Jackett/" + configService.GetVersion());
}
HttpResponseMessage response = null;
using (var request = new HttpRequestMessage())
{
request.Headers.ExpectContinue = false;
request.RequestUri = new Uri(webRequest.Url);
if (webRequest.Headers != null)
{
foreach (var header in webRequest.Headers)
{
if (header.Key != "Content-Type")
{
request.Headers.TryAddWithoutValidation(header.Key, header.Value);
}
}
}
if (!string.IsNullOrEmpty(webRequest.Referer))
{
request.Headers.Referrer = new Uri(webRequest.Referer);
}
if (!string.IsNullOrEmpty(webRequest.RawBody))
{
var type = webRequest.Headers.Where(h => h.Key == "Content-Type").Cast <KeyValuePair <string, string>?>().FirstOrDefault();
if (type.HasValue)
{
var str = new StringContent(webRequest.RawBody);
str.Headers.Remove("Content-Type");
str.Headers.Add("Content-Type", type.Value.Value);
request.Content = str;
}
else
{
request.Content = new StringContent(webRequest.RawBody);
}
request.Method = HttpMethod.Post;
}
else if (webRequest.Type == RequestType.POST)
{
if (webRequest.PostData != null)
{
request.Content = new FormUrlEncodedContent(webRequest.PostData);
}
request.Method = HttpMethod.Post;
}
else
{
request.Method = HttpMethod.Get;
}
using (response = await client.SendAsync(request))
{
var result = new WebClientByteResult
{
Content = await response.Content.ReadAsByteArrayAsync()
};
foreach (var header in response.Headers)
{
IEnumerable <string> value = header.Value;
result.Headers[header.Key.ToLowerInvariant()] = value.ToArray();
}
// some cloudflare clients are using a refresh header
// Pull it out manually
if (response.StatusCode == HttpStatusCode.ServiceUnavailable && response.Headers.Contains("Refresh"))
{
var refreshHeaders = response.Headers.GetValues("Refresh");
var redirval = "";
var redirtime = 0;
if (refreshHeaders != null)
{
foreach (var value in refreshHeaders)
{
var start = value.IndexOf("=");
var end = value.IndexOf(";");
var len = value.Length;
if (start > -1)
{
redirval = value.Substring(start + 1);
result.RedirectingTo = redirval;
// normally we don't want a serviceunavailable (503) to be a redirect, but that's the nature
// of this cloudflare approach..don't want to alter BaseWebResult.IsRedirect because normally
// it shoudln't include service unavailable..only if we have this redirect header.
response.StatusCode = System.Net.HttpStatusCode.Redirect;
redirtime = Int32.Parse(value.Substring(0, end));
System.Threading.Thread.Sleep(redirtime * 1000);
}
}
}
}
if (response.Headers.Location != null)
{
result.RedirectingTo = response.Headers.Location.ToString();
}
// Mono won't add the baseurl to relative redirects.
// e.g. a "Location: /index.php" header will result in the Uri "file:///index.php"
// See issue #1200
if (result.RedirectingTo != null && result.RedirectingTo.StartsWith("file://"))
{
var newRedirectingTo = result.RedirectingTo.Replace("file://", request.RequestUri.Scheme + "://" + request.RequestUri.Host);
logger.Debug("[MONO relative redirect bug] Rewriting relative redirect URL from " + result.RedirectingTo + " to " + newRedirectingTo);
result.RedirectingTo = newRedirectingTo;
}
result.Status = response.StatusCode;
// Compatiblity issue between the cookie format and httpclient
// Pull it out manually ignoring the expiry date then set it manually
// http://stackoverflow.com/questions/14681144/httpclient-not-storing-cookies-in-cookiecontainer
IEnumerable <string> cookieHeaders;
var responseCookies = new List <Tuple <string, string> >();
if (response.Headers.TryGetValues("set-cookie", out cookieHeaders))
{
foreach (var value in cookieHeaders)
{
var nameSplit = value.IndexOf('=');
if (nameSplit > -1)
{
responseCookies.Add(new Tuple <string, string>(value.Substring(0, nameSplit), value.Substring(0, value.IndexOf(';') == -1 ? value.Length : (value.IndexOf(';'))) + ";"));
}
}
var cookieBuilder = new StringBuilder();
foreach (var cookieGroup in responseCookies.GroupBy(c => c.Item1))
{
cookieBuilder.AppendFormat("{0} ", cookieGroup.Last().Item2);
}
result.Cookies = cookieBuilder.ToString().Trim();
}
ServerUtil.ResureRedirectIsFullyQualified(webRequest, result);
return(result);
}
}
}
}
}
}