private async Task <IPAddress> GetCurrentIpAddressAsync(TorSharpProxy proxy, TorSharpSettings settings) { const int maxAttempts = 3; for (var attempt = 1; attempt <= maxAttempts; attempt++) { try { var handler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port)) }; using (handler) using (var httpClient = new HttpClient(handler)) { httpClient.Timeout = TimeSpan.FromSeconds(60); var ip = (await httpClient.GetStringAsync("https://api.ipify.org")).Trim(); _output.WriteLine($"Get IP succeeded: {ip}"); return(IPAddress.Parse(ip)); } } catch (Exception ex) when(attempt < maxAttempts) { _output.WriteLine($"[Attempt {attempt}] An exception was thrown while fetching an IP. Retrying." + Environment.NewLine + ex); await proxy.GetNewIdentityAsync(); } } throw new NotImplementedException(); }
private static async Task MainAsync() { // configure var settings = new TorSharpSettings { ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"), ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"), PrivoxyPort = 1337, TorSocksPort = 1338, TorControlPort = 1339, TorControlPassword = "******", ToolRunnerType = ToolRunnerType.Simple }; // download tools await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync(); // execute var proxy = new TorSharpProxy(settings); var handler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxyPort)) }; var httpClient = new HttpClient(handler); await proxy.ConfigureAndStartAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com")); await proxy.GetNewIdentityAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com")); proxy.Stop(); }
private static async Task MainAsync() { // configure var settings = new TorSharpSettings { ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"), ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"), PrivoxyPort = 1337, TorSocksPort = 1338, TorControlPort = 1339, TorControlPassword = "******", ToolRunnerType = ToolRunnerType.VirtualDesktop, WaitForTestEndpoint = true }; // download tools await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync(); // execute using (var proxy = new TorSharpProxy(settings)) using (var handler = proxy.CreateHttpClientHandler()) using (var httpClient = new HttpClient(handler)) { await proxy.ConfigureAndStartAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com")); await proxy.GetNewIdentityAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com")); } }
private async Task <T> SendProxiedRequestAsync <T>( TorSharpProxy proxy, TorSharpSettings settings, string operation, Func <HttpClient, Task <T> > executeAsync) { const int maxAttempts = 3; for (var attempt = 1; attempt <= maxAttempts; attempt++) { try { var handler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port)) }; using (handler) using (var httpClient = new HttpClient(handler)) { httpClient.Timeout = TimeSpan.FromSeconds(60); return(await executeAsync(httpClient)); } } catch (Exception ex) when(attempt < maxAttempts) { _output.WriteLine($"[Attempt {attempt}] An exception was thrown while {operation}. Retrying." + Environment.NewLine + ex); await proxy.GetNewIdentityAsync(); } } throw new NotImplementedException(); }
private async Task ExecuteEndToEndTestAsync(TorSharpSettings settings) { // Arrange using (var proxy = new TorSharpProxy(settings)) { _output.WriteLine($"Initialized proxy with tool runner type {settings.ToolRunnerType}"); // Act await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync(); _output.WriteLine("The tools have been fetched"); await proxy.ConfigureAndStartAsync(); _output.WriteLine("The proxy has been started"); // get the first identity var ipA = await GetCurrentIpAddressAsync(settings); await proxy.GetNewIdentityAsync(); _output.WriteLine("Get new identity succeeded"); var ipB = await GetCurrentIpAddressAsync(settings); // Assert Assert.Equal(AddressFamily.InterNetwork, ipA.AddressFamily); Assert.Equal(AddressFamily.InterNetwork, ipB.AddressFamily); } }
public async Task TorSharpToolFetch_AllResultsAreWorking(TorSharpOSPlatform osPlatform, TorSharpArchitecture architecture) { using (var te = TestEnvironment.Initialize(_output)) { // Arrange var settings = te.BuildSettings(); settings.OSPlatform = osPlatform; settings.Architecture = architecture; settings.ToolDownloadStrategy = ToolDownloadStrategy.All; using (var httpClientHandler = new HttpClientHandler()) using (var loggingHandler = new LoggingHandler(_output) { InnerHandler = httpClientHandler }) using (var httpClient = new HttpClient(loggingHandler)) using (var proxy = new TorSharpProxy(settings)) { _output.WriteLine(settings); var fetcher = _httpFixture.GetTorSharpToolFetcher(settings, httpClient); // Act var updates = await fetcher.CheckForUpdatesAsync(); // Assert Assert.NotNull(updates); _output.WriteLine("Privoxy URL: " + updates.Privoxy.LatestDownload.Url.AbsoluteUri); _output.WriteLine("Tor URL: " + updates.Tor.LatestDownload.Url.AbsoluteUri); } } }
public static async Task DeployTorClients(int count) { var taskWaitingConfigurationEnd = new List <Task>(); for (var i = 0; i < count; i++) { var offset = i * 3; var extractedToolsDirectory = Path.Combine(Path.GetTempPath(), "Tor", "TorExtracted" + i); var settings = new TorSharpSettings { ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "Tor", "TorZipped"), ExtractedToolsDirectory = extractedToolsDirectory, PrivoxyPort = 1337 + offset, TorSocksPort = 1338 + offset, TorControlPort = 1339 + offset, TorControlPassword = "******" + i, TorDataDirectory = Path.Combine(extractedToolsDirectory, "DataDirectory") }; var torSharpProxy = new TorSharpProxy(settings); await torSharpProxy.ConfigureAndStartAsync(); ProxyList.Add($"http://localhost:{settings.PrivoxyPort}"); taskWaitingConfigurationEnd.Add(torSharpProxy.WaitForConnectionAsync()); } Task.WaitAll(taskWaitingConfigurationEnd.ToArray()); }
private static async Task MainAsync() { // configure var settings = new TorSharpSettings { ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"), ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"), PrivoxySettings = { Port = 1337, }, TorSettings = { SocksPort = 1338, ControlPort = 1339, ControlPassword = "******", }, }; // download tools using (var httpClient = new HttpClient()) { var fetcher = new TorSharpToolFetcher(settings, httpClient); var updates = await fetcher.CheckForUpdatesAsync(); Console.WriteLine($"Current Privoxy: {updates.Privoxy.LocalVersion?.ToString() ?? "(none)"}"); Console.WriteLine($" Latest Privoxy: {updates.Privoxy.LatestDownload.Version}"); Console.WriteLine(); Console.WriteLine($"Current Tor: {updates.Tor.LocalVersion?.ToString() ?? "(none)"}"); Console.WriteLine($" Latest Tor: {updates.Tor.LatestDownload.Version}"); Console.WriteLine(); if (updates.HasUpdate) { await fetcher.FetchAsync(updates); } } // execute using (var proxy = new TorSharpProxy(settings)) { var handler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port)) }; using (handler) using (var httpClient = new HttpClient(handler)) { await proxy.ConfigureAndStartAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org")); await proxy.GetNewIdentityAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org")); } proxy.Stop(); } }
private static async Task StartTorAndPrivoxyAsync() { var settings = new TorSharpSettings { TorSocksPort = SocksPort, PrivoxyPort = HttpConnectPort }; var fetcher = new TorSharpToolFetcher(settings, new HttpClient()); await fetcher.FetchAsync(); var proxy = new TorSharpProxy(settings); await proxy.ConfigureAndStartAsync(); }
private async Task <IPAddress> GetCurrentIpAddressAsync(TorSharpProxy proxy, TorSharpSettings settings) { return(await SendProxiedRequestAsync( proxy, settings, "fetching an IP", async proxiedHttpClient => { var ip = (await proxiedHttpClient.GetStringAsync("https://api.ipify.org")).Trim(); _output.WriteLine($"Get IP succeeded: {ip}"); return IPAddress.Parse(ip); })); }
public async Task TorSharpToolFetcher_CheckForUpdates(ToolDownloadStrategy strategy) { using (var te = TestEnvironment.Initialize(_output)) { // Arrange var settings = te.BuildSettings(); settings.ToolDownloadStrategy = strategy; using (var httpClientHandler = new HttpClientHandler()) using (var loggingHandler = new LoggingHandler(_output) { InnerHandler = httpClientHandler }) using (var httpClient = new HttpClient(loggingHandler)) using (var proxy = new TorSharpProxy(settings)) { _output.WriteLine(settings); var fetcher = _httpFixture.GetTorSharpToolFetcher(settings, httpClient); var initial = await fetcher.CheckForUpdatesAsync(); await fetcher.FetchAsync(initial); var prefix = ToolUtility.GetPrivoxyToolSettings(settings).Prefix; var extension = Path.GetExtension(initial.Privoxy.DestinationPath); var fakeOldPrivoxy = Path.Combine(settings.ZippedToolsDirectory, $"{prefix}0.0.1{extension}"); File.Move(initial.Privoxy.DestinationPath, fakeOldPrivoxy); // Act var newerVersion = await fetcher.CheckForUpdatesAsync(); await fetcher.FetchAsync(newerVersion); var upToDate = await fetcher.CheckForUpdatesAsync(); // Assert Assert.True(initial.HasUpdate); Assert.Equal(ToolUpdateStatus.NoLocalVersion, initial.Privoxy.Status); Assert.Equal(ToolUpdateStatus.NoLocalVersion, initial.Tor.Status); Assert.True(newerVersion.HasUpdate); Assert.Equal(ToolUpdateStatus.NewerVersionAvailable, newerVersion.Privoxy.Status); Assert.Equal(ToolUpdateStatus.NoUpdateAvailable, newerVersion.Tor.Status); Assert.False(upToDate.HasUpdate); Assert.Equal(ToolUpdateStatus.NoUpdateAvailable, upToDate.Privoxy.Status); Assert.Equal(ToolUpdateStatus.NoUpdateAvailable, upToDate.Tor.Status); } } }
public static async Task <string> MakeTorRequest(string url) { string result = String.Empty; using (var proxy = new TorSharpProxy(_torSettings)) using (var handler = proxy.CreateHttpClientHandler()) using (var httpClient = new HttpClient(handler)) { await proxy.ConfigureAndStartAsync(); result = await httpClient.GetStringAsync(url); } return(result); }
private async Task ExecuteEndToEndTestAsync(TorSharpSettings settings) { // Arrange using (var httpClient = new HttpClient()) using (var proxy = new TorSharpProxy(settings)) { _output.WriteLine(settings); // Act var fetcher = _httpFixture.GetTorSharpToolFetcher(settings, httpClient); await fetcher.FetchAsync(); _output.WriteLine("The tools have been fetched"); await proxy.ConfigureAndStartAsync(); _output.WriteLine("The proxy has been started"); // get the first identity var ipA = await GetCurrentIpAddressAsync(proxy, settings); await proxy.GetNewIdentityAsync(); _output.WriteLine("Get new identity succeeded"); var ipB = await GetCurrentIpAddressAsync(proxy, settings); // Assert Assert.Equal(AddressFamily.InterNetwork, ipA.AddressFamily); Assert.Equal(AddressFamily.InterNetwork, ipB.AddressFamily); var zippedDir = new DirectoryInfo(settings.ZippedToolsDirectory); Assert.True(zippedDir.Exists, "The zipped tools directory should exist."); Assert.Empty(zippedDir.EnumerateDirectories()); Assert.Equal(2, zippedDir.EnumerateFiles().Count()); var extractedDir = new DirectoryInfo(settings.ExtractedToolsDirectory); Assert.True(extractedDir.Exists, "The extracted tools directory should exist."); Assert.Equal(2, extractedDir.EnumerateDirectories().Count()); Assert.Empty(extractedDir.EnumerateFiles()); } }
private async Task InitializeTor() { Settings = new TorSharpSettings { ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"), ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"), PrivoxyPort = 1337, TorSocksPort = 1338, TorControlPort = 1339, TorControlPassword = "******" }; // download tools await new TorSharpToolFetcher(Settings, new HttpClient()).FetchAsync(); // execute Proxy = new TorSharpProxy(Settings); TorHandler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + Settings.PrivoxyPort)) }; }
public async void StartProxyAsync() { var settings = new TorSharpSettings { ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"), ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"), PrivoxyPort = 1337, TorSocksPort = 1338, TorControlPort = 1339, TorControlPassword = "******", TorrcLoc = Application.StartupPath + "\\torrc" }; await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync(); proxy = new TorSharpProxy(settings); var handler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxyPort)) }; var httpClient = new HttpClient(handler); await proxy.ConfigureAndStartAsync(); }
public async Task TorSharpToolFetcher_UseExistingTools() { using (var te = TestEnvironment.Initialize(_output)) { // Arrange var settings = te.BuildSettings(); settings.ReloadTools = true; settings.UseExistingTools = true; using (var httpClientHandler = new HttpClientHandler()) using (var requestCountHandler = new RequestCountHandler { InnerHandler = httpClientHandler }) using (var loggingHandler = new LoggingHandler(_output) { InnerHandler = requestCountHandler }) using (var httpClient = new HttpClient(requestCountHandler)) using (var proxy = new TorSharpProxy(settings)) { _output.WriteLine(settings); var fetcherA = _httpFixture.GetTorSharpToolFetcher(settings, httpClient); await fetcherA.FetchAsync(); var requestCount = requestCountHandler.RequestCount; // Act var fetcherB = _httpFixture.GetTorSharpToolFetcher(settings, httpClient); await fetcherB.FetchAsync(); // Assert Assert.True(requestCount > 0, "The should be at least one request."); Assert.Equal(requestCount, requestCountHandler.RequestCount); Assert.NotNull(ToolUtility.GetLatestToolOrNull(settings, ToolUtility.GetPrivoxyToolSettings(settings))); Assert.NotNull(ToolUtility.GetLatestToolOrNull(settings, ToolUtility.GetTorToolSettings(settings))); } } }
public async Task TrafficReadAndWritten() { using (var te = TestEnvironment.Initialize(_output)) { // Arrange var settings = te.BuildSettings(); settings.ToolRunnerType = ToolRunnerType.Simple; using (var httpClient = new HttpClient()) using (var proxy = new TorSharpProxy(settings)) { _output.WriteLine(settings); var fetcher = _httpFixture.GetTorSharpToolFetcher(settings, httpClient); await fetcher.FetchAsync(); _output.WriteLine("The tools have been fetched"); await proxy.ConfigureAndStartAsync(); _output.WriteLine("The proxy has been started"); await GetCurrentIpAddressAsync(proxy, settings); _output.WriteLine("The first request has succeeded"); // Act using (var controlClient = await proxy.GetControlClientAsync()) { var readBefore = await controlClient.GetTrafficReadAsync(); _output.WriteLine($"Read before: {readBefore}"); var writtenBefore = await controlClient.GetTrafficWrittenAsync(); _output.WriteLine($"Written before: {writtenBefore}"); // Upload and download some bytes var bytes = 10000; await SendProxiedRequestAsync( proxy, settings, $"uploading and downloading {bytes} bytes", async proxiedHttpClient => { using (var content = new ByteArrayContent(new byte[bytes])) using (var response = await proxiedHttpClient.PostAsync("https://httpbin.org/anything", content)) { } return(true); }); var readAfter = await controlClient.GetTrafficReadAsync(); _output.WriteLine($"Read after: {readBefore}"); var writtenAfter = await controlClient.GetTrafficWrittenAsync(); _output.WriteLine($"Written after: {writtenBefore}"); // Assert Assert.True(readBefore + bytes <= readAfter, $"At least {bytes} more bytes should have been read."); Assert.True(writtenBefore + bytes <= writtenAfter, $"At least {bytes} more bytes should have been written."); } } } }
private static async Task MainAsync() { // configure var settings = new TorSharpSettings { ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"), ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"), PrivoxySettings = { Port = 18118, }, TorSettings = { SocksPort = 19050, ControlPort = 19051, ControlPassword = "******", }, }; // output runtime information var message = new StringBuilder(); message.Append($"Running the sample on {settings.OSPlatform} OS and {settings.Architecture} architecture."); #if NETCOREAPP message.Append($" OS description: {RuntimeInformation.OSDescription}."); #endif Console.WriteLine(message.ToString()); Console.WriteLine(); // download tools using (var httpClient = new HttpClient()) { var fetcher = new TorSharpToolFetcher(settings, httpClient); var updates = await fetcher.CheckForUpdatesAsync(); Console.WriteLine($"Current Privoxy: {updates.Privoxy.LocalVersion?.ToString() ?? "(none)"}"); Console.WriteLine($" Latest Privoxy: {updates.Privoxy.LatestDownload.Version}"); Console.WriteLine(); Console.WriteLine($"Current Tor: {updates.Tor.LocalVersion?.ToString() ?? "(none)"}"); Console.WriteLine($" Latest Tor: {updates.Tor.LatestDownload.Version}"); Console.WriteLine(); if (updates.HasUpdate) { await fetcher.FetchAsync(updates); } } // execute using (var proxy = new TorSharpProxy(settings)) { var handler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port)) }; using (handler) using (var httpClient = new HttpClient(handler)) { await proxy.ConfigureAndStartAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org")); await proxy.GetNewIdentityAsync(); Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org")); using (var controlClient = await proxy.GetControlClientAsync()) { var read = await controlClient.GetTrafficReadAsync(); Console.WriteLine("Bytes read : {0}", read); var written = await controlClient.GetTrafficWrittenAsync(); Console.WriteLine("Bytes written : {0}", written); } } proxy.Stop(); } }
// PRIVATE CLASS MEMBERS // MAIN =================================================================================== /// <param name="args"> Command line arguements passed to executable. </param> static async Task Main(string[] args) { // Creates a Logger object from Serilog. Writes up to Debug level prints. SetupLogger(); Log.Logger.Information("Darkweb Data Scraper start..."); // Parses command line arguements and stores them in "parsedArgs" SetupParser(args); // I made this function to move this setup out of main. // Returns a TorSharpSettings object for use with TorSharp. var settings = SetupTorSharpSettings(); // Idk exactly how this works but like... its for torsharp // its uh... setting up torsharp "tools"...also its asyncronous await SetupTorSharpTools(settings); // * starts tor proxy ----------------------------------------------------------------- using (var proxy = new TorSharpProxy(settings)) { var waiting = true; while (waiting) { // block untill we wait for TorSharp Proxy to be configured await proxy.ConfigureAndStartAsync(); waiting = false; } // * SETUP AND EXECUTE CRAWLER ================================================ // Setup Crawler configuration CrawlConfigurationX crawlConfig = new CrawlConfigurationX { MaxPagesToCrawl = 30, // Max total urls this crawler should crawl MaxCrawlDepth = 1, // Depth for crawler to traverse urls IsJavascriptRenderingEnabled = false, // Should crawler render JS? JavascriptRenderingWaitTimeInMilliseconds = 2000, // How long to wait for js to process MaxConcurrentSiteCrawls = 1, // Only crawl a single site at a time MaxRetryCount = 3 // Retries to connect and crawl site 'x' times }; if (parsedArgs.InputFile == null) // THIS IS "-s" or "--single" { var handler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port)) }; // Crawl await DataScraper.Crawl(crawlConfig, handler, parsedArgs.handlerType, parsedArgs.StartingUri); BuildBsonDocument(DataScraper.allParsedText, parsedArgs.StartingUri); //reset vals for next crawl DataScraper.allParsedText = new List <string>(); DataScraper.siteTitle = ""; DataScraper.dataDocuments = new List <BsonDocument>(); if (MONGO_URI == "") { Log.Logger.Information("Database information is no longer accessible or available." + "You will need to provide your own Mongo details in \"Crawler.cs\"."); } } else // THIS IS "-m" or "--multi" { string inputFilePath = @parsedArgs.InputFile; var sitesToCrawl = GenerateSiteList(inputFilePath); for (int i = 0; i < sitesToCrawl.Count; i++) { var handler = new HttpClientHandler { Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port)) }; // Crawl await DataScraper.Crawl(crawlConfig, handler, parsedArgs.handlerType, sitesToCrawl[i]); BuildBsonDocument(DataScraper.allParsedText, sitesToCrawl[i]); if (MONGO_URI == "") { Log.Logger.Information("Database information is no longer accessible or available." + "You will need to provide your own Mongo details in \"Crawler.cs\"."); } if (MONGO_URI != "" && MONGO_COLLECTION_NAME != "" && MONGO_DATABASE_NAME != "") { var client = new MongoClient(MONGO_URI); var database = client.GetDatabase(MONGO_DATABASE_NAME); var collection = database.GetCollection <BsonDocument>(MONGO_COLLECTION_NAME); collection.InsertMany(DataScraper.dataDocuments); } //reset vals for next crawl DataScraper.allParsedText = new List <string>(); DataScraper.siteTitle = ""; DataScraper.dataDocuments = new List <BsonDocument>(); } } // * ========================================================================== // Stop the TorSharp tools so that the proxy is no longer listening on the configured port. proxy.Stop(); } // * ---------------------------------------------------------------------------------- }
public static async Task Run(CancellationToken cancelToken, string store) { Timer.Tick += new EventHandler(Timer_Tick); Timer.Interval = 1000; if (!Settings.Get <bool>("ScanNew") && !Settings.Get <bool>("ScanUsed")) { mf.textBox4.AppendText("Crawler wird gestoppt - Reason: Neu & Gebraucht sind Deaktiviert" + Environment.NewLine); mf.metroButton3.PerformClick(); return; } mf.textBox4.AppendText("Übernehme Neu & Gebraucht Einstellungen" + Environment.NewLine); var sellerForNew = new List <string>(); if (!String.IsNullOrWhiteSpace(Settings.Get <string>("SellerForNew"))) { sellerForNew = Settings.Get <string>("SellerForNew").Split(',').Select(s => s.Trim()).ToList(); } else { sellerForNew.Add("Amazon"); } var sellerForUsed = new List <string>(); if (!String.IsNullOrWhiteSpace(Settings.Get <string>("SellerForUsed"))) { sellerForUsed = Settings.Get <string>("SellerForUsed").Split(',').Select(s => s.Trim()).ToList(); } else { sellerForUsed.Add("Amazon"); } TorSharpSettings torSettings = null; TorSharpProxy torProxy = null; if (Settings.Get <bool>("UseTorProxies")) { mf.textBox4.AppendText("Initialisiere TOR Proxy Funktion" + Environment.NewLine); try { torSettings = new TorSharpSettings { ReloadTools = true, ZippedToolsDirectory = Path.Combine(FoldersFilesAndPaths.Temp, "TorZipped"), ExtractedToolsDirectory = Path.Combine(FoldersFilesAndPaths.Temp, "TorExtracted"), PrivoxySettings = { Port = 1337, }, TorSettings = { SocksPort = 1338, ControlPort = 1339, ControlPassword = "******", }, }; await new TorSharpToolFetcher(torSettings, new HttpClient()).FetchAsync(); torProxy = new TorSharpProxy(torSettings); await torProxy.ConfigureAndStartAsync(); } catch (Exception ex) { mf.textBox4.AppendText("Fehler beim Initialisiere der TOR Proxy Funktion ;-(" + Environment.NewLine + "TOR Proxy Funktion wird deaktiviert!" + Environment.NewLine); mf.textBox4.AppendText(ex.Message); mf.metroToggle5.Checked = false; } if (Settings.IsPremium && Settings.Get <bool>("UseTorProxies") && Settings.Get <bool>("ProxyAlwaysActive")) { mf.textBox4.AppendText("Set TOR Proxy - Reason: P. always Active" + Environment.NewLine); mf.metroLabel45.Text = Convert.ToString(await WebUtils.GetCurrentIpAddressAsync(torSettings)); Proxies.TorProxyActive = true; } } string sourcecode; int sites, loops; while (true) { Database.OpenConnection(); SQLiteCommand getEntry; switch (store) { case "ALLE": getEntry = new SQLiteCommand("SELECT * FROM Products WHERE Status = '0' ORDER BY [Letzter Check] ASC LIMIT 0,1", Database.Connection); break; default: getEntry = new SQLiteCommand("SELECT * FROM Products WHERE Status = '0' AND Store = @store ORDER BY [Letzter Check] ASC LIMIT 0,1", Database.Connection); getEntry.Parameters.AddWithValue("@store", store); break; } SQLiteDataReader entry = getEntry.ExecuteReader(); if (!entry.HasRows) { mf.metroButton3.PerformClick(); Timer.Stop(); torProxy.Stop(); return; } while (entry.Read()) { if (cancelToken.IsCancellationRequested) { Timer.Stop(); torProxy.Stop(); return; } mf.metroLabel10.Text = Convert.ToString(entry["Store"]); mf.metroLabel11.Text = Convert.ToString(entry["ASIN / ISBN"]); mf.metroLabel12.Text = Convert.ToString(entry["Name"]); mf.metroLabel13.Text = Convert.ToString(entry["Preis: Neu"]); mf.metroLabel14.Text = Convert.ToString(entry["Preis: Wie Neu"]); mf.metroLabel15.Text = Convert.ToString(entry["Preis: Sehr Gut"]); mf.metroLabel16.Text = Convert.ToString(entry["Preis: Gut"]); mf.metroLabel17.Text = Convert.ToString(entry["Preis: Akzeptabel"]); mf.metroLabel18.Text = Convert.ToString(entry["Letzter Check"]); mf.metroLabel24.Text = mf.metroLabel25.Text = mf.metroLabel26.Text = mf.metroLabel27.Text = mf.metroLabel28.Text = null; mf.metroLabel41.Text = null; mf.webBrowser1.DocumentText = mf.webBrowser2.DocumentText = null; await Task.Delay(Tools.RandomNumber(250, 500), cancelToken).ContinueWith(tsk => { }); string priceNew = null; if (Settings.Get <bool>("ScanNew")) { List <string> priceNewList = new List <string>(); mf.tabControl2.SelectedTab = mf.TabPage4; sites = 1; loops = 0; do { if (cancelToken.IsCancellationRequested) { Timer.Stop(); torProxy.Stop(); return; } try { string url = "https://www.amazon." + Amazon.GetTld(Convert.ToString(entry["Store"])) + "/gp/aw/ol/" + entry["ASIN / ISBN"] + "/ref=mw_dp_olp?ca=" + entry["ASIN / ISBN"] + "&o=New&op=" + (loops + 1) + "&vs=1"; var handler = new HttpClientHandler(); if (Settings.Get <bool>("UseTorProxies")) { if (Settings.IsPremium && Settings.Get <bool>("ProxyAlwaysActive") || Proxies.TorProxyActive && Proxies.RealIPnexttime > DateTime.Now) { Debug.WriteLine("Add TOR Proxy to HttpClientHandler..."); handler.Proxy = new WebProxy(new Uri("http://*****:*****@"<title> (Tut uns Leid!|Toutes nos excuses)</title>"); Match errordetection1 = Regex.Match(sourcecode.RemoveLineEndings(), @"<title>(503 - Service Unavailable Error)</title>"); if (errordetection.Success || errordetection1.Success) { Debug.WriteLine("Issue Found..."); continue; } //CAPTCHA Detection Match captchadetection = Regex.Match(sourcecode, "<title dir=\"ltr\">(Amazon CAPTCHA|Bot Check|Robot Check)</title>"); if (captchadetection.Success) { Debug.WriteLine("Captcha detected..."); if (Settings.Get <bool>("UseTorProxies")) { await torProxy.GetNewIdentityAsync(); mf.metroLabel45.Text = Convert.ToString(await WebUtils.GetCurrentIpAddressAsync(torSettings)); Proxies.TorProxyActive = true; if (!Settings.Get <bool>("ProxyAlwaysActive") && Proxies.RealIPnexttime < DateTime.Now) { Proxies.RealIPnexttime = DateTime.Now.AddMinutes(15); } } continue; } //How many Sites? Match sitesNumber = Regex.Match(sourcecode, "<a name=\"New\">([^\"]*) / ([^\"]*)</a>"); if (sitesNumber.Success && loops == 0 && Settings.Get <int>("ScanMethod") == 0) { sites = Convert.ToInt32( Math.Ceiling(Convert.ToDecimal(sitesNumber.Groups[2].Value) / 10)); } mf.metroLabel41.Text = $@"{loops + 1} / {sites} (NEU)"; //Get Price for NEW if (sellerForNew.Contains("Amazon")) { Match newama = Regex.Match(sourcecode, "<a href=(.*)ca=([a-zA-Z0-9_]*)&vs=1(.*)>(Neu|Nuovo|Neuf|Nuevo|New) - (EUR |£)(.*)</a>"); if (newama.Success) { priceNewList.Add(await CurrencyConverter.ConvertToEuro(newama.Groups[6].Value, Convert.ToString(entry["Store"]))); } } if (sellerForNew.Contains("Drittanbieter")) { if (sellerForNew.Contains("Versand per Amazon")) { Regex new3rd = new Regex( "(Versand durch Amazon.de|Spedito da Amazon|EXPÉDIÉ PAR AMAZON|Distribuido por Amazon|Fulfilled by Amazon)(\r\n|\r|\n)</font>(\r\n|\r|\n)(\r\n|\r|\n)<br />(\r\n|\r|\n)(\r\n|\r|\n)(\r\n|\r|\n)" + "<a href=(.*)ca=([a-zA-Z0-9_]*)&eid=(.*)>(Neu|Nuovo|Neuf|Nuevo|New) - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in new3rd.Matches(sourcecode)) { priceNewList.Add( await CurrencyConverter.ConvertToEuro(itemMatch.Groups[13].Value, Convert.ToString(entry["Store"]))); } } else { Regex new3rd = new Regex( @"<a href=(.*)ca=([a-zA-Z0-9_]*)&eid=(.*)>(Neu|Nuovo|Neuf|Nuevo|New) - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in new3rd.Matches(sourcecode)) { priceNewList.Add( await CurrencyConverter.ConvertToEuro(itemMatch.Groups[6].Value, Convert.ToString(entry["Store"]))); } } } loops++; if (sites > loops) { await Task.Delay( Tools.RandomNumber(Convert.ToInt32(mf.numericUpDown3.Value) * 1000 - 500, Convert.ToInt32(mf.numericUpDown3.Value) * 1000 + 500), cancelToken) .ContinueWith(tsk => { }); } } while (sites > loops); RemoveInvalidEntrys(priceNewList); priceNew = priceNewList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceNew)) { mf.metroLabel24.Text = priceNew; ComparePrice(entry["Preis: Neu"], priceNew, mf.metroLabel24); } } await Task.Delay(Tools.RandomNumber(500, 1000), cancelToken).ContinueWith(tsk => { }); string priceLikeNew = null; string priceVeryGood = null; string priceGood = null; string priceAcceptable = null; if (Settings.Get <bool>("ScanUsed")) { List <string> priceLikeNewList = new List <string>(); List <string> priceVeryGoodList = new List <string>(); List <string> priceGoodList = new List <string>(); List <string> priceAcceptableList = new List <string>(); mf.tabControl2.SelectedTab = mf.TabPage5; var conditionList = new List <Tuple <List <string>, string> > { new Tuple <List <string>, string>(priceLikeNewList, "Wie neu|Condizioni pari al nuovo|Comme neuf|Como nuevo|Mint"), new Tuple <List <string>, string>(priceVeryGoodList, "Sehr gut|Ottimo|Très bon état|Muy bueno|Very good"), new Tuple <List <string>, string>(priceGoodList, "Gut|Buono|D'occasion - très bon état|Bueno|Good"), new Tuple <List <string>, string>(priceAcceptableList, "Akzeptabel|Accettabile|Acceptable|Aceptable|Acceptable") }; sites = 1; loops = 0; do { if (cancelToken.IsCancellationRequested) { Timer.Stop(); torProxy.Stop(); return; } try { string url = "https://www.amazon." + Amazon.GetTld(Convert.ToString(entry["Store"])) + "/gp/aw/ol/" + entry["ASIN / ISBN"] + "/ref=mw_dp_olp?o=Used&op=" + (loops + 1); var handler = new HttpClientHandler(); if (Settings.Get <bool>("UseTorProxies")) { if (Settings.IsPremium && Settings.Get <bool>("ProxyAlwaysActive") || Proxies.TorProxyActive && Proxies.RealIPnexttime > DateTime.Now) { Debug.WriteLine("Add TOR Proxy to HttpClientHandler..."); handler.Proxy = new WebProxy(new Uri("http://*****:*****@"<title> (Tut uns Leid!|Toutes nos excuses)</title>"); Match errordetection1 = Regex.Match(sourcecode.RemoveLineEndings(), @"<title>(503 - Service Unavailable Error)</title>"); if (errordetection.Success || errordetection1.Success) { Debug.WriteLine("Issue Found..."); continue; } //CAPTCHA Detection Match captchadetection = Regex.Match(sourcecode, "<title dir=\"ltr\">(Amazon CAPTCHA|Bot Check|Robot Check)</title>"); if (captchadetection.Success) { Debug.WriteLine("Captcha detected..."); if (Settings.Get <bool>("UseTorProxies")) { await torProxy.GetNewIdentityAsync(); mf.metroLabel45.Text = Convert.ToString(await WebUtils.GetCurrentIpAddressAsync(torSettings)); Proxies.TorProxyActive = true; if (!Settings.Get <bool>("ProxyAlwaysActive") && Proxies.RealIPnexttime < DateTime.Now) { Proxies.RealIPnexttime = DateTime.Now.AddMinutes(15); } } continue; } //How many WHD Sites? Match sitesNumber = Regex.Match(sourcecode, "<a name=\"Used\">([^\"]*) / ([^\"]*)</a>"); if (sitesNumber.Success && loops == 0 && Settings.Get <int>("ScanMethod") == 0) { sites = Convert.ToInt32(Math.Ceiling(Convert.ToDecimal(sitesNumber.Groups[2].Value) / 10)); } mf.metroLabel41.Text = $@"{loops + 1} / {sites} (GEBRAUCHT)"; //Get Price for WHDs //.de WHD Seller ID: A8KICS1PHF7ZO //.it WHD Seller ID: A1HO9729ND375Y //.fr WHD Seller ID: A2CVHYRTWLQO9T //.es WHD Seller ID: A6T89FGPU3U0Q //.co.uk WHD Seller ID: A2OAJ7377F756P if (sellerForUsed.Contains("Amazon")) { foreach (var condition in conditionList) { Regex search = new Regex( @"<a href=(.*)(A8KICS1PHF7ZO|A1HO9729ND375Y|A2CVHYRTWLQO9T|A6T89FGPU3U0Q|A2OAJ7377F756P)(.*)>(" + condition.Item2 + ") - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in search.Matches(sourcecode)) { condition.Item1.Add(await CurrencyConverter.ConvertToEuro(itemMatch.Groups[6].Value, Convert.ToString(entry["Store"]))); } } } if (sellerForUsed.Contains("Drittanbieter")) { if (sellerForUsed.Contains("Versand per Amazon")) { foreach (var condition in conditionList) { Regex search = new Regex( "(Versand durch Amazon.de|Spedito da Amazon|EXPÉDIÉ PAR AMAZON|Distribuido por Amazon|Fulfilled by Amazon)(\r\n|\r|\n)</font>(\r\n|\r|\n)(\r\n|\r|\n)<br />(\r\n|\r|\n)(\r\n|\r|\n)(\r\n|\r|\n)" + "<a href=(.*)&me=(?!(A8KICS1PHF7ZO|A1HO9729ND375Y|A2CVHYRTWLQO9T|A6T89FGPU3U0Q|A2OAJ7377F756P).)(.*)>(" + condition.Item2 + ") - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in search.Matches(sourcecode)) { condition.Item1.Add(await CurrencyConverter.ConvertToEuro(itemMatch.Groups[13].Value, Convert.ToString(entry["Store"]))); } } } else { foreach (var condition in conditionList) { Regex search = new Regex( @"<a href=(.*)&me=(?!(A8KICS1PHF7ZO|A1HO9729ND375Y|A2CVHYRTWLQO9T|A6T89FGPU3U0Q|A2OAJ7377F756P).)(.*)>(" + condition.Item2 + ") - (EUR |£)(.*)</a>", RegexOptions.Compiled); foreach (Match itemMatch in search.Matches(sourcecode)) { condition.Item1.Add(await CurrencyConverter.ConvertToEuro(itemMatch.Groups[6].Value, Convert.ToString(entry["Store"]))); } } } } loops++; if (sites > loops) { await Task.Delay(Tools.RandomNumber(Convert.ToInt32(mf.numericUpDown3.Value) * 1000 - 500, Convert.ToInt32(mf.numericUpDown3.Value) * 1000 + 500), cancelToken).ContinueWith(tsk => { }); } } while (sites > loops); foreach (var condition in conditionList) { RemoveInvalidEntrys(condition.Item1); } priceLikeNew = priceLikeNewList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceLikeNew)) { mf.metroLabel25.Text = priceLikeNew; ComparePrice(entry["Preis: Wie Neu"], priceLikeNew, mf.metroLabel25); } priceVeryGood = priceVeryGoodList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceVeryGood)) { mf.metroLabel26.Text = priceVeryGood; ComparePrice(entry["Preis: Sehr Gut"], priceVeryGood, mf.metroLabel26); } priceGood = priceGoodList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceGood)) { mf.metroLabel27.Text = priceGood; ComparePrice(entry["Preis: Gut"], priceGood, mf.metroLabel27); } priceAcceptable = priceAcceptableList.OrderBy(price => price).FirstOrDefault(); if (!String.IsNullOrEmpty(priceAcceptable)) { mf.metroLabel28.Text = priceAcceptable; ComparePrice(entry["Preis: Akzeptabel"], priceAcceptable, mf.metroLabel28); } } await Task.Delay(Tools.RandomNumber(500, 1000), cancelToken).ContinueWith(tsk => { }); Database.OpenConnection(); SQLiteCommand updateEntry = new SQLiteCommand( "UPDATE Products SET [Preis: Neu] = @priceNew, [Preis: Wie Neu] = @priceLikenew, [Preis: Sehr Gut] = @priceVerygood, [Preis: Gut] = @priceGood, [Preis: Akzeptabel] = @priceAcceptable, [Letzter Check] = @lastcheck WHERE ID = @id", Database.Connection); updateEntry.Parameters.AddWithValue("@id", entry["ID"]); updateEntry.Parameters.AddWithValue("@priceNew", !String.IsNullOrEmpty(priceNew) ? priceNew : null); updateEntry.Parameters.AddWithValue("@priceLikenew", !String.IsNullOrEmpty(priceLikeNew) ? priceLikeNew : null); updateEntry.Parameters.AddWithValue("@priceVerygood", !String.IsNullOrEmpty(priceVeryGood) ? priceVeryGood : null); updateEntry.Parameters.AddWithValue("@priceGood", !String.IsNullOrEmpty(priceGood) ? priceGood : null); updateEntry.Parameters.AddWithValue("@priceAcceptable", !String.IsNullOrEmpty(priceAcceptable) ? priceAcceptable : null); updateEntry.Parameters.AddWithValue("@lastcheck", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); updateEntry.ExecuteNonQuery(); await Task.Delay(250, cancelToken).ContinueWith(tsk => { }); if (mf.metroComboBox2.SelectedIndex == -1 || mf.metroComboBox2.Text == "ALLE" || store == mf.metroComboBox2.Text) { ProductDatabase.Display(mf.metroComboBox2.SelectedIndex == -1 ? "ALLE" : mf.metroComboBox2.Text); } await MyReminder.DoRemindWhenPossible(Convert.ToString(entry["ID"])); _randomWait = Tools.RandomNumber(Convert.ToInt32(mf.numericUpDown2.Value) - 1, Convert.ToInt32(mf.numericUpDown2.Value) + 1); mf.metroProgressBar1.Value = 0; mf.metroProgressBar1.Maximum = _randomWait * 1000; Timer.Start(); mf.metroLabel29.Text = _dt.AddSeconds(_randomWait).ToString("mm:ss"); await Task.Delay((_randomWait + 1) * 1000, cancelToken).ContinueWith(tsk => { }); } } }