private async Task ExecuteEndToEndTestAsync(TorSharpSettings settings) { // Arrange using (var proxy = new TorSharpProxy(settings)) { _output.WriteLine($"Initialized proxy with tool runner type {settings.ToolRunnerType}"); // Act await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync(); _output.WriteLine("The tools have been fetched"); await proxy.ConfigureAndStartAsync(); _output.WriteLine("The proxy has been started"); // get the first identity var ipA = await GetCurrentIpAddressAsync(settings); await proxy.GetNewIdentityAsync(); _output.WriteLine("Get new identity succeeded"); var ipB = await GetCurrentIpAddressAsync(settings); // Assert Assert.Equal(AddressFamily.InterNetwork, ipA.AddressFamily); Assert.Equal(AddressFamily.InterNetwork, ipB.AddressFamily); } }
public static async Task DeployTorClients(int count) { var taskWaitingConfigurationEnd = new List <Task>(); for (var i = 0; i < count; i++) { var offset = i * 3; var extractedToolsDirectory = Path.Combine(Path.GetTempPath(), "Tor", "TorExtracted" + i); var settings = new TorSharpSettings { ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "Tor", "TorZipped"), ExtractedToolsDirectory = extractedToolsDirectory, PrivoxyPort = 1337 + offset, TorSocksPort = 1338 + offset, TorControlPort = 1339 + offset, TorControlPassword = "******" + i, TorDataDirectory = Path.Combine(extractedToolsDirectory, "DataDirectory") }; var torSharpProxy = new TorSharpProxy(settings); await torSharpProxy.ConfigureAndStartAsync(); ProxyList.Add($"http://localhost:{settings.PrivoxyPort}"); taskWaitingConfigurationEnd.Add(torSharpProxy.WaitForConnectionAsync()); } Task.WaitAll(taskWaitingConfigurationEnd.ToArray()); }
private static async Task MainAsync() { // configure var settings = new TorSharpSettings { ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"), ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"), PrivoxyPort = 1337, TorSocksPort = 1338, TorControlPort = 1339, TorControlPassword = "******", ToolRunnerType = ToolRunnerType.VirtualDesktop, WaitForTestEndpoint = true }; // download tools await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync(); // execute using (var proxy = new TorSharpProxy(settings)) using (var handler = proxy.CreateHttpClientHandler()) using (var httpClient = new HttpClient(handler)) { await proxy.ConfigureAndStartAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com")); await proxy.GetNewIdentityAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com")); } }
private static async Task MainAsync() { // configure var settings = new TorSharpSettings { ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"), ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"), PrivoxyPort = 1337, TorSocksPort = 1338, TorControlPort = 1339, TorControlPassword = "******", ToolRunnerType = ToolRunnerType.Simple }; // download tools await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync(); // execute var proxy = new TorSharpProxy(settings); var handler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxyPort)) }; var httpClient = new HttpClient(handler); await proxy.ConfigureAndStartAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com")); await proxy.GetNewIdentityAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com")); proxy.Stop(); }
private static async Task MainAsync() { // configure var settings = new TorSharpSettings { ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"), ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"), PrivoxySettings = { Port = 1337, }, TorSettings = { SocksPort = 1338, ControlPort = 1339, ControlPassword = "******", }, }; // download tools using (var httpClient = new HttpClient()) { var fetcher = new TorSharpToolFetcher(settings, httpClient); var updates = await fetcher.CheckForUpdatesAsync(); Console.WriteLine($"Current Privoxy: {updates.Privoxy.LocalVersion?.ToString() ?? "(none)"}"); Console.WriteLine($" Latest Privoxy: {updates.Privoxy.LatestDownload.Version}"); Console.WriteLine(); Console.WriteLine($"Current Tor: {updates.Tor.LocalVersion?.ToString() ?? "(none)"}"); Console.WriteLine($" Latest Tor: {updates.Tor.LatestDownload.Version}"); Console.WriteLine(); if (updates.HasUpdate) { await fetcher.FetchAsync(updates); } } // execute using (var proxy = new TorSharpProxy(settings)) { var handler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port)) }; using (handler) using (var httpClient = new HttpClient(handler)) { await proxy.ConfigureAndStartAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org")); await proxy.GetNewIdentityAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org")); } proxy.Stop(); } }
private static async Task StartTorAndPrivoxyAsync() { var settings = new TorSharpSettings { TorSocksPort = SocksPort, PrivoxyPort = HttpConnectPort }; var fetcher = new TorSharpToolFetcher(settings, new HttpClient()); await fetcher.FetchAsync(); var proxy = new TorSharpProxy(settings); await proxy.ConfigureAndStartAsync(); }
public static async Task <string> MakeTorRequest(string url) { string result = String.Empty; using (var proxy = new TorSharpProxy(_torSettings)) using (var handler = proxy.CreateHttpClientHandler()) using (var httpClient = new HttpClient(handler)) { await proxy.ConfigureAndStartAsync(); result = await httpClient.GetStringAsync(url); } return(result); }
private async Task ExecuteEndToEndTestAsync(TorSharpSettings settings) { // Arrange using (var httpClient = new HttpClient()) using (var proxy = new TorSharpProxy(settings)) { _output.WriteLine(settings); // Act var fetcher = _httpFixture.GetTorSharpToolFetcher(settings, httpClient); await fetcher.FetchAsync(); _output.WriteLine("The tools have been fetched"); await proxy.ConfigureAndStartAsync(); _output.WriteLine("The proxy has been started"); // get the first identity var ipA = await GetCurrentIpAddressAsync(proxy, settings); await proxy.GetNewIdentityAsync(); _output.WriteLine("Get new identity succeeded"); var ipB = await GetCurrentIpAddressAsync(proxy, settings); // Assert Assert.Equal(AddressFamily.InterNetwork, ipA.AddressFamily); Assert.Equal(AddressFamily.InterNetwork, ipB.AddressFamily); var zippedDir = new DirectoryInfo(settings.ZippedToolsDirectory); Assert.True(zippedDir.Exists, "The zipped tools directory should exist."); Assert.Empty(zippedDir.EnumerateDirectories()); Assert.Equal(2, zippedDir.EnumerateFiles().Count()); var extractedDir = new DirectoryInfo(settings.ExtractedToolsDirectory); Assert.True(extractedDir.Exists, "The extracted tools directory should exist."); Assert.Equal(2, extractedDir.EnumerateDirectories().Count()); Assert.Empty(extractedDir.EnumerateFiles()); } }
public async void StartProxyAsync() { var settings = new TorSharpSettings { ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"), ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"), PrivoxyPort = 1337, TorSocksPort = 1338, TorControlPort = 1339, TorControlPassword = "******", TorrcLoc = Application.StartupPath + "\\torrc" }; await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync(); proxy = new TorSharpProxy(settings); var handler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxyPort)) }; var httpClient = new HttpClient(handler); await proxy.ConfigureAndStartAsync(); }
private static async Task MainAsync() { // configure var settings = new TorSharpSettings { ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"), ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"), PrivoxySettings = { Port = 18118, }, TorSettings = { SocksPort = 19050, ControlPort = 19051, ControlPassword = "******", }, }; // output runtime information var message = new StringBuilder(); message.Append($"Running the sample on {settings.OSPlatform} OS and {settings.Architecture} architecture."); #if NETCOREAPP message.Append($" OS description: {RuntimeInformation.OSDescription}."); #endif Console.WriteLine(message.ToString()); Console.WriteLine(); // download tools using (var httpClient = new HttpClient()) { var fetcher = new TorSharpToolFetcher(settings, httpClient); var updates = await fetcher.CheckForUpdatesAsync(); Console.WriteLine($"Current Privoxy: {updates.Privoxy.LocalVersion?.ToString() ?? "(none)"}"); Console.WriteLine($" Latest Privoxy: {updates.Privoxy.LatestDownload.Version}"); Console.WriteLine(); Console.WriteLine($"Current Tor: {updates.Tor.LocalVersion?.ToString() ?? "(none)"}"); Console.WriteLine($" Latest Tor: {updates.Tor.LatestDownload.Version}"); Console.WriteLine(); if (updates.HasUpdate) { await fetcher.FetchAsync(updates); } } // execute using (var proxy = new TorSharpProxy(settings)) { var handler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port)) }; using (handler) using (var httpClient = new HttpClient(handler)) { await proxy.ConfigureAndStartAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org")); await proxy.GetNewIdentityAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org")); using (var controlClient = await proxy.GetControlClientAsync()) { var read = await controlClient.GetTrafficReadAsync(); Console.WriteLine("Bytes read : {0}", read); var written = await controlClient.GetTrafficWrittenAsync(); Console.WriteLine("Bytes written : {0}", written); } } proxy.Stop(); } }
public static async Task Run(CancellationToken cancelToken, string store) { Timer.Tick += new EventHandler(Timer_Tick); Timer.Interval = 1000; if (!Settings.Get <bool>("ScanNew") && !Settings.Get <bool>("ScanUsed")) { mf.textBox4.AppendText("Crawler wird gestoppt - Reason: Neu & Gebraucht sind Deaktiviert" + Environment.NewLine); mf.metroButton3.PerformClick(); return; } mf.textBox4.AppendText("Übernehme Neu & Gebraucht Einstellungen" + Environment.NewLine); var sellerForNew = new List <string>(); if (!String.IsNullOrWhiteSpace(Settings.Get <string>("SellerForNew"))) { sellerForNew = Settings.Get <string>("SellerForNew").Split(',').Select(s => s.Trim()).ToList(); } else { sellerForNew.Add("Amazon"); } var sellerForUsed = new List <string>(); if (!String.IsNullOrWhiteSpace(Settings.Get <string>("SellerForUsed"))) { sellerForUsed = Settings.Get <string>("SellerForUsed").Split(',').Select(s => s.Trim()).ToList(); } else { sellerForUsed.Add("Amazon"); } TorSharpSettings torSettings = null; TorSharpProxy torProxy = null; if (Settings.Get <bool>("UseTorProxies")) { mf.textBox4.AppendText("Initialisiere TOR Proxy Funktion" + Environment.NewLine); try { torSettings = new TorSharpSettings { ReloadTools = true, ZippedToolsDirectory = Path.Combine(FoldersFilesAndPaths.Temp, "TorZipped"), ExtractedToolsDirectory = Path.Combine(FoldersFilesAndPaths.Temp, "TorExtracted"), PrivoxySettings = { Port = 1337, }, TorSettings = { SocksPort = 1338, ControlPort = 1339, ControlPassword = "******", }, }; await new TorSharpToolFetcher(torSettings, new HttpClient()).FetchAsync(); torProxy = new TorSharpProxy(torSettings); await torProxy.ConfigureAndStartAsync(); } catch (Exception ex) { mf.textBox4.AppendText("Fehler beim Initialisiere der TOR Proxy Funktion ;-(" + Environment.NewLine + "TOR Proxy Funktion wird deaktiviert!" + Environment.NewLine); mf.textBox4.AppendText(ex.Message); mf.metroToggle5.Checked = false; } if (Settings.IsPremium && Settings.Get <bool>("UseTorProxies") && Settings.Get <bool>("ProxyAlwaysActive")) { mf.textBox4.AppendText("Set TOR Proxy - Reason: P. always Active" + Environment.NewLine); mf.metroLabel45.Text = Convert.ToString(await WebUtils.GetCurrentIpAddressAsync(torSettings)); Proxies.TorProxyActive = true; } } string sourcecode; int sites, loops; while (true) { Database.OpenConnection(); SQLiteCommand getEntry; switch (store) { case "ALLE": getEntry = new SQLiteCommand("SELECT * FROM Products WHERE Status = '0' ORDER BY [Letzter Check] ASC LIMIT 0,1", Database.Connection); break; default: getEntry = new SQLiteCommand("SELECT * FROM Products WHERE Status = '0' AND Store = @store ORDER BY [Letzter Check] ASC LIMIT 0,1", Database.Connection); getEntry.Parameters.AddWithValue("@store", store); break; } SQLiteDataReader entry = getEntry.ExecuteReader(); if (!entry.HasRows) { mf.metroButton3.PerformClick(); Timer.Stop(); torProxy.Stop(); return; } while (entry.Read()) { if (cancelToken.IsCancellationRequested) { Timer.Stop(); torProxy.Stop(); return; } mf.metroLabel10.Text = Convert.ToString(entry["Store"]); mf.metroLabel11.Text = Convert.ToString(entry["ASIN / ISBN"]); mf.metroLabel12.Text = Convert.ToString(entry["Name"]); mf.metroLabel13.Text = Convert.ToString(entry["Preis: Neu"]); mf.metroLabel14.Text = Convert.ToString(entry["Preis: Wie Neu"]); mf.metroLabel15.Text = Convert.ToString(entry["Preis: Sehr Gut"]); mf.metroLabel16.Text = Convert.ToString(entry["Preis: Gut"]); mf.metroLabel17.Text = Convert.ToString(entry["Preis: Akzeptabel"]); mf.metroLabel18.Text = Convert.ToString(entry["Letzter Check"]); mf.metroLabel24.Text = mf.metroLabel25.Text = mf.metroLabel26.Text = mf.metroLabel27.Text = mf.metroLabel28.Text = null; mf.metroLabel41.Text = null; mf.webBrowser1.DocumentText = mf.webBrowser2.DocumentText = null; await Task.Delay(Tools.RandomNumber(250, 500), cancelToken).ContinueWith(tsk => { }); string priceNew = null; if (Settings.Get <bool>("ScanNew")) { List <string> priceNewList = new List <string>(); mf.tabControl2.SelectedTab = mf.TabPage4; sites = 1; loops = 0; do { if (cancelToken.IsCancellationRequested) { Timer.Stop(); torProxy.Stop(); return; } try { string url = "https://www.amazon." + Amazon.GetTld(Convert.ToString(entry["Store"])) + "/gp/aw/ol/" + entry["ASIN / ISBN"] + "/ref=mw_dp_olp?ca=" + entry["ASIN / ISBN"] + "&o=New&op=" + (loops + 1) + "&vs=1"; var handler = new HttpClientHandler(); if (Settings.Get <bool>("UseTorProxies")) { if (Settings.IsPremium && Settings.Get <bool>("ProxyAlwaysActive") || Proxies.TorProxyActive && Proxies.RealIPnexttime > DateTime.Now) { Debug.WriteLine("Add TOR Proxy to HttpClientHandler..."); handler.Proxy = new WebProxy(new Uri("http://*****:*****@"<title> (Tut uns Leid!|Toutes nos excuses)</title>"); Match errordetection1 = Regex.Match(sourcecode.RemoveLineEndings(), @"<title>(503 - Service Unavailable Error)</title>"); if (errordetection.Success || errordetection1.Success) { Debug.WriteLine("Issue Found..."); continue; } //CAPTCHA Detection Match captchadetection = Regex.Match(sourcecode, "<title dir=\"ltr\">(Amazon CAPTCHA|Bot Check|Robot Check)</title>"); if (captchadetection.Success) { Debug.WriteLine("Captcha detected..."); if (Settings.Get <bool>("UseTorProxies")) { await torProxy.GetNewIdentityAsync(); mf.metroLabel45.Text = Convert.ToString(await WebUtils.GetCurrentIpAddressAsync(torSettings)); Proxies.TorProxyActive = true; if (!Settings.Get <bool>("ProxyAlwaysActive") && Proxies.RealIPnexttime < DateTime.Now) { Proxies.RealIPnexttime = DateTime.Now.AddMinutes(15); } } continue; } //How many Sites? Match sitesNumber = Regex.Match(sourcecode, "<a name=\"New\">([^\"]*) / ([^\"]*)</a>"); if (sitesNumber.Success && loops == 0 && Settings.Get <int>("ScanMethod") == 0) { sites = Convert.ToInt32( Math.Ceiling(Convert.ToDecimal(sitesNumber.Groups[2].Value) / 10)); } mf.metroLabel41.Text = $@"{loops + 1} / {sites} (NEU)"; //Get Price for NEW if (sellerForNew.Contains("Amazon")) { Match newama = Regex.Match(sourcecode, "<a href=(.*)ca=([a-zA-Z0-9_]*)&vs=1(.*)>(Neu|Nuovo|Neuf|Nuevo|New) - (EUR |£)(.*)</a>"); if (newama.Success) { priceNewList.Add(await CurrencyConverter.ConvertToEuro(newama.Groups[6].Value, Convert.ToString(entry["Store"]))); } } if (sellerForNew.Contains("Drittanbieter")) { if (sellerForNew.Contains("Versand per Amazon")) { Regex new3rd = new Regex( "(Versand durch Amazon.de|Spedito da Amazon|EXPÉDIÉ PAR AMAZON|Distribuido por Amazon|Fulfilled by Amazon)(\r\n|\r|\n)</font>(\r\n|\r|\n)(\r\n|\r|\n)<br />(\r\n|\r|\n)(\r\n|\r|\n)(\r\n|\r|\n)" + "<a href=(.*)ca=([a-zA-Z0-9_]*)&eid=(.*)>(Neu|Nuovo|Neuf|Nuevo|New) - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in new3rd.Matches(sourcecode)) { priceNewList.Add( await CurrencyConverter.ConvertToEuro(itemMatch.Groups[13].Value, Convert.ToString(entry["Store"]))); } } else { Regex new3rd = new Regex( @"<a href=(.*)ca=([a-zA-Z0-9_]*)&eid=(.*)>(Neu|Nuovo|Neuf|Nuevo|New) - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in new3rd.Matches(sourcecode)) { priceNewList.Add( await CurrencyConverter.ConvertToEuro(itemMatch.Groups[6].Value, Convert.ToString(entry["Store"]))); } } } loops++; if (sites > loops) { await Task.Delay( Tools.RandomNumber(Convert.ToInt32(mf.numericUpDown3.Value) * 1000 - 500, Convert.ToInt32(mf.numericUpDown3.Value) * 1000 + 500), cancelToken) .ContinueWith(tsk => { }); } } while (sites > loops); RemoveInvalidEntrys(priceNewList); priceNew = priceNewList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceNew)) { mf.metroLabel24.Text = priceNew; ComparePrice(entry["Preis: Neu"], priceNew, mf.metroLabel24); } } await Task.Delay(Tools.RandomNumber(500, 1000), cancelToken).ContinueWith(tsk => { }); string priceLikeNew = null; string priceVeryGood = null; string priceGood = null; string priceAcceptable = null; if (Settings.Get <bool>("ScanUsed")) { List <string> priceLikeNewList = new List <string>(); List <string> priceVeryGoodList = new List <string>(); List <string> priceGoodList = new List <string>(); List <string> priceAcceptableList = new List <string>(); mf.tabControl2.SelectedTab = mf.TabPage5; var conditionList = new List <Tuple <List <string>, string> > { new Tuple <List <string>, string>(priceLikeNewList, "Wie neu|Condizioni pari al nuovo|Comme neuf|Como nuevo|Mint"), new Tuple <List <string>, string>(priceVeryGoodList, "Sehr gut|Ottimo|Très bon état|Muy bueno|Very good"), new Tuple <List <string>, string>(priceGoodList, "Gut|Buono|D'occasion - très bon état|Bueno|Good"), new Tuple <List <string>, string>(priceAcceptableList, "Akzeptabel|Accettabile|Acceptable|Aceptable|Acceptable") }; sites = 1; loops = 0; do { if (cancelToken.IsCancellationRequested) { Timer.Stop(); torProxy.Stop(); return; } try { string url = "https://www.amazon." + Amazon.GetTld(Convert.ToString(entry["Store"])) + "/gp/aw/ol/" + entry["ASIN / ISBN"] + "/ref=mw_dp_olp?o=Used&op=" + (loops + 1); var handler = new HttpClientHandler(); if (Settings.Get <bool>("UseTorProxies")) { if (Settings.IsPremium && Settings.Get <bool>("ProxyAlwaysActive") || Proxies.TorProxyActive && Proxies.RealIPnexttime > DateTime.Now) { Debug.WriteLine("Add TOR Proxy to HttpClientHandler..."); handler.Proxy = new WebProxy(new Uri("http://*****:*****@"<title> (Tut uns Leid!|Toutes nos excuses)</title>"); Match errordetection1 = Regex.Match(sourcecode.RemoveLineEndings(), @"<title>(503 - Service Unavailable Error)</title>"); if (errordetection.Success || errordetection1.Success) { Debug.WriteLine("Issue Found..."); continue; } //CAPTCHA Detection Match captchadetection = Regex.Match(sourcecode, "<title dir=\"ltr\">(Amazon CAPTCHA|Bot Check|Robot Check)</title>"); if (captchadetection.Success) { Debug.WriteLine("Captcha detected..."); if (Settings.Get <bool>("UseTorProxies")) { await torProxy.GetNewIdentityAsync(); mf.metroLabel45.Text = Convert.ToString(await WebUtils.GetCurrentIpAddressAsync(torSettings)); Proxies.TorProxyActive = true; if (!Settings.Get <bool>("ProxyAlwaysActive") && Proxies.RealIPnexttime < DateTime.Now) { Proxies.RealIPnexttime = DateTime.Now.AddMinutes(15); } } continue; } //How many WHD Sites? Match sitesNumber = Regex.Match(sourcecode, "<a name=\"Used\">([^\"]*) / ([^\"]*)</a>"); if (sitesNumber.Success && loops == 0 && Settings.Get <int>("ScanMethod") == 0) { sites = Convert.ToInt32(Math.Ceiling(Convert.ToDecimal(sitesNumber.Groups[2].Value) / 10)); } mf.metroLabel41.Text = $@"{loops + 1} / {sites} (GEBRAUCHT)"; //Get Price for WHDs //.de WHD Seller ID: A8KICS1PHF7ZO //.it WHD Seller ID: A1HO9729ND375Y //.fr WHD Seller ID: A2CVHYRTWLQO9T //.es WHD Seller ID: A6T89FGPU3U0Q //.co.uk WHD Seller ID: A2OAJ7377F756P if (sellerForUsed.Contains("Amazon")) { foreach (var condition in conditionList) { Regex search = new Regex( @"<a href=(.*)(A8KICS1PHF7ZO|A1HO9729ND375Y|A2CVHYRTWLQO9T|A6T89FGPU3U0Q|A2OAJ7377F756P)(.*)>(" + condition.Item2 + ") - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in search.Matches(sourcecode)) { condition.Item1.Add(await CurrencyConverter.ConvertToEuro(itemMatch.Groups[6].Value, Convert.ToString(entry["Store"]))); } } } if (sellerForUsed.Contains("Drittanbieter")) { if (sellerForUsed.Contains("Versand per Amazon")) { foreach (var condition in conditionList) { Regex search = new Regex( "(Versand durch Amazon.de|Spedito da Amazon|EXPÉDIÉ PAR AMAZON|Distribuido por Amazon|Fulfilled by Amazon)(\r\n|\r|\n)</font>(\r\n|\r|\n)(\r\n|\r|\n)<br />(\r\n|\r|\n)(\r\n|\r|\n)(\r\n|\r|\n)" + "<a href=(.*)&me=(?!(A8KICS1PHF7ZO|A1HO9729ND375Y|A2CVHYRTWLQO9T|A6T89FGPU3U0Q|A2OAJ7377F756P).)(.*)>(" + condition.Item2 + ") - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in search.Matches(sourcecode)) { condition.Item1.Add(await CurrencyConverter.ConvertToEuro(itemMatch.Groups[13].Value, Convert.ToString(entry["Store"]))); } } } else { foreach (var condition in conditionList) { Regex search = new Regex( @"<a href=(.*)&me=(?!(A8KICS1PHF7ZO|A1HO9729ND375Y|A2CVHYRTWLQO9T|A6T89FGPU3U0Q|A2OAJ7377F756P).)(.*)>(" + condition.Item2 + ") - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in search.Matches(sourcecode)) { condition.Item1.Add(await CurrencyConverter.ConvertToEuro(itemMatch.Groups[6].Value, Convert.ToString(entry["Store"]))); } } } } loops++; if (sites > loops) { await Task.Delay(Tools.RandomNumber(Convert.ToInt32(mf.numericUpDown3.Value) * 1000 - 500, Convert.ToInt32(mf.numericUpDown3.Value) * 1000 + 500), cancelToken).ContinueWith(tsk => { }); } } while (sites > loops); foreach (var condition in conditionList) { RemoveInvalidEntrys(condition.Item1); } priceLikeNew = priceLikeNewList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceLikeNew)) { mf.metroLabel25.Text = priceLikeNew; ComparePrice(entry["Preis: Wie Neu"], priceLikeNew, mf.metroLabel25); } priceVeryGood = priceVeryGoodList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceVeryGood)) { mf.metroLabel26.Text = priceVeryGood; ComparePrice(entry["Preis: Sehr Gut"], priceVeryGood, mf.metroLabel26); } priceGood = priceGoodList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceGood)) { mf.metroLabel27.Text = priceGood; ComparePrice(entry["Preis: Gut"], priceGood, mf.metroLabel27); } priceAcceptable = priceAcceptableList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceAcceptable)) { mf.metroLabel28.Text = priceAcceptable; ComparePrice(entry["Preis: Akzeptabel"], priceAcceptable, mf.metroLabel28); } } await Task.Delay(Tools.RandomNumber(500, 1000), cancelToken).ContinueWith(tsk => { }); Database.OpenConnection(); SQLiteCommand updateEntry = new SQLiteCommand( "UPDATE Products SET [Preis: Neu] = @priceNew, [Preis: Wie Neu] = @priceLikenew, [Preis: Sehr Gut] = @priceVerygood, [Preis: Gut] = @priceGood, [Preis: Akzeptabel] = @priceAcceptable, [Letzter Check] = @lastcheck WHERE ID = @id", Database.Connection); updateEntry.Parameters.AddWithValue("@id", entry["ID"]); updateEntry.Parameters.AddWithValue("@priceNew", !String.IsNullOrEmpty(priceNew) ? priceNew : null); updateEntry.Parameters.AddWithValue("@priceLikenew", !String.IsNullOrEmpty(priceLikeNew) ? priceLikeNew : null); updateEntry.Parameters.AddWithValue("@priceVerygood", !String.IsNullOrEmpty(priceVeryGood) ? priceVeryGood : null); updateEntry.Parameters.AddWithValue("@priceGood", !String.IsNullOrEmpty(priceGood) ? priceGood : null); updateEntry.Parameters.AddWithValue("@priceAcceptable", !String.IsNullOrEmpty(priceAcceptable) ? priceAcceptable : null); updateEntry.Parameters.AddWithValue("@lastcheck", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); updateEntry.ExecuteNonQuery(); await Task.Delay(250, cancelToken).ContinueWith(tsk => { }); if (mf.metroComboBox2.SelectedIndex == -1 || mf.metroComboBox2.Text == "ALLE" || store == mf.metroComboBox2.Text) { ProductDatabase.Display(mf.metroComboBox2.SelectedIndex == -1 ? "ALLE" : mf.metroComboBox2.Text); } await MyReminder.DoRemindWhenPossible(Convert.ToString(entry["ID"])); _randomWait = Tools.RandomNumber(Convert.ToInt32(mf.numericUpDown2.Value) - 1, Convert.ToInt32(mf.numericUpDown2.Value) + 1); mf.metroProgressBar1.Value = 0; mf.metroProgressBar1.Maximum = _randomWait * 1000; Timer.Start(); mf.metroLabel29.Text = _dt.AddSeconds(_randomWait).ToString("mm:ss"); await Task.Delay((_randomWait + 1) * 1000, cancelToken).ContinueWith(tsk => { }); } } }
// PRIVATE CLASS MEMBERS // MAIN =================================================================================== /// <param name="args"> Command line arguements passed to executable. </param> static async Task Main(string[] args) { // Creates a Logger object from Serilog. Writes up to Debug level prints. SetupLogger(); Log.Logger.Information("Darkweb Data Scraper start..."); // Parses command line arguements and stores them in "parsedArgs" SetupParser(args); // I made this function to move this setup out of main. // Returns a TorSharpSettings object for use with TorSharp. var settings = SetupTorSharpSettings(); // Idk exactly how this works but like... its for torsharp // its uh... setting up torsharp "tools"...also its asyncronous await SetupTorSharpTools(settings); // * starts tor proxy ----------------------------------------------------------------- using (var proxy = new TorSharpProxy(settings)) { var waiting = true; while (waiting) { // block untill we wait for TorSharp Proxy to be configured await proxy.ConfigureAndStartAsync(); waiting = false; } // * SETUP AND EXECUTE CRAWLER ================================================ // Setup Crawler configuration CrawlConfigurationX crawlConfig = new CrawlConfigurationX { MaxPagesToCrawl = 30, // Max total urls this crawler should crawl MaxCrawlDepth = 1, // Depth for crawler to traverse urls IsJavascriptRenderingEnabled = false, // Should crawler render JS? JavascriptRenderingWaitTimeInMilliseconds = 2000, // How long to wait for js to process MaxConcurrentSiteCrawls = 1, // Only crawl a single site at a time MaxRetryCount = 3 // Retries to connect and crawl site 'x' times }; if (parsedArgs.InputFile == null) // THIS IS "-s" or "--single" { var handler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port)) }; // Crawl await DataScraper.Crawl(crawlConfig, handler, parsedArgs.handlerType, parsedArgs.StartingUri); BuildBsonDocument(DataScraper.allParsedText, parsedArgs.StartingUri); //reset vals for next crawl DataScraper.allParsedText = new List <string>(); DataScraper.siteTitle = ""; DataScraper.dataDocuments = new List <BsonDocument>(); if (MONGO_URI == "") { Log.Logger.Information("Database information is no longer accessible or available." + "You will need to provide your own Mongo details in \"Crawler.cs\"."); } } else // THIS IS "-m" or "--multi" { string inputFilePath = @parsedArgs.InputFile; var sitesToCrawl = GenerateSiteList(inputFilePath); for (int i = 0; i < sitesToCrawl.Count; i++) { var handler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port)) }; // Crawl await DataScraper.Crawl(crawlConfig, handler, parsedArgs.handlerType, sitesToCrawl[i]); BuildBsonDocument(DataScraper.allParsedText, sitesToCrawl[i]); if (MONGO_URI == "") { Log.Logger.Information("Database information is no longer accessible or available." + "You will need to provide your own Mongo details in \"Crawler.cs\"."); } if (MONGO_URI != "" && MONGO_COLLECTION_NAME != "" && MONGO_DATABASE_NAME != "") { var client = new MongoClient(MONGO_URI); var database = client.GetDatabase(MONGO_DATABASE_NAME); var collection = database.GetCollection <BsonDocument>(MONGO_COLLECTION_NAME); collection.InsertMany(DataScraper.dataDocuments); } //reset vals for next crawl DataScraper.allParsedText = new List <string>(); DataScraper.siteTitle = ""; DataScraper.dataDocuments = new List <BsonDocument>(); } } // * ========================================================================== // Stop the TorSharp tools so that the proxy is no longer listening on the configured port. proxy.Stop(); } // * ---------------------------------------------------------------------------------- }
public async Task TrafficReadAndWritten() { using (var te = TestEnvironment.Initialize(_output)) { // Arrange var settings = te.BuildSettings(); settings.ToolRunnerType = ToolRunnerType.Simple; using (var httpClient = new HttpClient()) using (var proxy = new TorSharpProxy(settings)) { _output.WriteLine(settings); var fetcher = _httpFixture.GetTorSharpToolFetcher(settings, httpClient); await fetcher.FetchAsync(); _output.WriteLine("The tools have been fetched"); await proxy.ConfigureAndStartAsync(); _output.WriteLine("The proxy has been started"); await GetCurrentIpAddressAsync(proxy, settings); _output.WriteLine("The first request has succeeded"); // Act using (var controlClient = await proxy.GetControlClientAsync()) { var readBefore = await controlClient.GetTrafficReadAsync(); _output.WriteLine($"Read before: {readBefore}"); var writtenBefore = await controlClient.GetTrafficWrittenAsync(); _output.WriteLine($"Written before: {writtenBefore}"); // Upload and download some bytes var bytes = 10000; await SendProxiedRequestAsync( proxy, settings, $"uploading and downloading {bytes} bytes", async proxiedHttpClient => { using (var content = new ByteArrayContent(new byte[bytes])) using (var response = await proxiedHttpClient.PostAsync("https://httpbin.org/anything", content)) { } return(true); }); var readAfter = await controlClient.GetTrafficReadAsync(); _output.WriteLine($"Read after: {readBefore}"); var writtenAfter = await controlClient.GetTrafficWrittenAsync(); _output.WriteLine($"Written after: {writtenBefore}"); // Assert Assert.True(readBefore + bytes <= readAfter, $"At least {bytes} more bytes should have been read."); Assert.True(writtenBefore + bytes <= writtenAfter, $"At least {bytes} more bytes should have been written."); } } } }
public async Task Run() { Console.WriteLine("===============================>>Processando<<=================================="); Console.WriteLine("Obs.: Os arquivos de saída conterão as seguintes colunas:"); Console.WriteLine("Accension | Collection Date | Segment | Region | Strain/Isolate"); Console.WriteLine(); var arquivos = FindFiles(); Console.WriteLine($"{arquivos.Count()} arquivos encontrados:"); await proxy.ConfigureAndStartAsync(); foreach (var arquivo in arquivos) { Console.WriteLine($"* {arquivo}"); } Console.WriteLine(); foreach (var arquivo in arquivos) { Console.WriteLine("================================================================================"); Console.WriteLine($"Processando arquvo '{arquivo}'"); var linhas = CarregarLinhasPendentes(arquivo); var arquivoSaida = ObterNomeArquivoSaida(arquivo); streamFiles.Add(arquivoSaida, new FileStream(arquivoSaida, FileMode.Append)); var gruposAccNum = AgruparPorQuantidade(ret_max, linhas); var items = new List <Item>(); var total = gruposAccNum.SelectMany(g => g).Count(); Console.WriteLine($"{total} registros encontrados.."); for (int i = 0; i < gruposAccNum.Count(); i++) { var grupoAccNum = gruposAccNum.ElementAt(i); var index = (i * ret_max) + grupoAccNum.Count(); items.AddRange(await RequestAndTranslateGINumber(grupoAccNum)); var perc = (((decimal)index) / total) * 100m; perc = Math.Truncate(perc); WriteSameLine($"Obtidos GIs dos accession numbers: {index}/{total} ({perc}%)"); } Console.WriteLine(); var itemsGroups = AgruparPorQuantidade(quantidadeRequestsDados, items); total = items.Count; var step = 0; foreach (var grupo in itemsGroups) { grupo.AsParallel().ForAll(item => { try { RequestAndSaveItem(item, arquivoSaida).GetAwaiter().GetResult(); } catch (Exception ex) { Console.WriteLine(ex.Message); Console.WriteLine(ex.InnerException?.Message); throw ex; } finally { ShowGetPercent(ref total, ref step, item.Accension); } }); } Console.WriteLine(); Console.WriteLine($"Processamento do arquivo '{arquivo}' finalizado!"); } }