private async Task <IPAddress> GetCurrentIpAddressAsync(TorSharpProxy proxy, TorSharpSettings settings)
{
const int maxAttempts = 3;
for (var attempt = 1; attempt <= maxAttempts; attempt++)
{
try
{
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port))
};
using (handler)
using (var httpClient = new HttpClient(handler))
{
httpClient.Timeout = TimeSpan.FromSeconds(60);
var ip = (await httpClient.GetStringAsync("https://api.ipify.org")).Trim();
_output.WriteLine($"Get IP succeeded: {ip}");
return(IPAddress.Parse(ip));
}
}
catch (Exception ex) when(attempt < maxAttempts)
{
_output.WriteLine($"[Attempt {attempt}] An exception was thrown while fetching an IP. Retrying." + Environment.NewLine + ex);
await proxy.GetNewIdentityAsync();
}
}
throw new NotImplementedException();
}
private static async Task MainAsync()
{
// configure
var settings = new TorSharpSettings
{
ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"),
ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"),
PrivoxyPort = 1337,
TorSocksPort = 1338,
TorControlPort = 1339,
TorControlPassword = "******",
ToolRunnerType = ToolRunnerType.Simple
};
// download tools
await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync();
// execute
var proxy = new TorSharpProxy(settings);
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxyPort))
};
var httpClient = new HttpClient(handler);
await proxy.ConfigureAndStartAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com"));
await proxy.GetNewIdentityAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com"));
proxy.Stop();
}
private static async Task MainAsync()
{
// configure
var settings = new TorSharpSettings
{
ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"),
ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"),
PrivoxyPort = 1337,
TorSocksPort = 1338,
TorControlPort = 1339,
TorControlPassword = "******",
ToolRunnerType = ToolRunnerType.VirtualDesktop,
WaitForTestEndpoint = true
};
// download tools
await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync();
// execute
using (var proxy = new TorSharpProxy(settings))
using (var handler = proxy.CreateHttpClientHandler())
using (var httpClient = new HttpClient(handler))
{
await proxy.ConfigureAndStartAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com"));
await proxy.GetNewIdentityAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com"));
}
}
private async Task <T> SendProxiedRequestAsync <T>(
TorSharpProxy proxy,
TorSharpSettings settings,
string operation,
Func <HttpClient, Task <T> > executeAsync)
{
const int maxAttempts = 3;
for (var attempt = 1; attempt <= maxAttempts; attempt++)
{
try
{
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port))
};
using (handler)
using (var httpClient = new HttpClient(handler))
{
httpClient.Timeout = TimeSpan.FromSeconds(60);
return(await executeAsync(httpClient));
}
}
catch (Exception ex) when(attempt < maxAttempts)
{
_output.WriteLine($"[Attempt {attempt}] An exception was thrown while {operation}. Retrying." + Environment.NewLine + ex);
await proxy.GetNewIdentityAsync();
}
}
throw new NotImplementedException();
}
private async Task ExecuteEndToEndTestAsync(TorSharpSettings settings)
{
// Arrange
using (var proxy = new TorSharpProxy(settings))
{
_output.WriteLine($"Initialized proxy with tool runner type {settings.ToolRunnerType}");
// Act
await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync();
_output.WriteLine("The tools have been fetched");
await proxy.ConfigureAndStartAsync();
_output.WriteLine("The proxy has been started");
// get the first identity
var ipA = await GetCurrentIpAddressAsync(settings);
await proxy.GetNewIdentityAsync();
_output.WriteLine("Get new identity succeeded");
var ipB = await GetCurrentIpAddressAsync(settings);
// Assert
Assert.Equal(AddressFamily.InterNetwork, ipA.AddressFamily);
Assert.Equal(AddressFamily.InterNetwork, ipB.AddressFamily);
}
}
public async Task TorSharpToolFetch_AllResultsAreWorking(TorSharpOSPlatform osPlatform, TorSharpArchitecture architecture)
{
using (var te = TestEnvironment.Initialize(_output))
{
// Arrange
var settings = te.BuildSettings();
settings.OSPlatform = osPlatform;
settings.Architecture = architecture;
settings.ToolDownloadStrategy = ToolDownloadStrategy.All;
using (var httpClientHandler = new HttpClientHandler())
using (var loggingHandler = new LoggingHandler(_output)
{
InnerHandler = httpClientHandler
})
using (var httpClient = new HttpClient(loggingHandler))
using (var proxy = new TorSharpProxy(settings))
{
_output.WriteLine(settings);
var fetcher = _httpFixture.GetTorSharpToolFetcher(settings, httpClient);
// Act
var updates = await fetcher.CheckForUpdatesAsync();
// Assert
Assert.NotNull(updates);
_output.WriteLine("Privoxy URL: " + updates.Privoxy.LatestDownload.Url.AbsoluteUri);
_output.WriteLine("Tor URL: " + updates.Tor.LatestDownload.Url.AbsoluteUri);
}
}
}
public static async Task DeployTorClients(int count)
{
var taskWaitingConfigurationEnd = new List <Task>();
for (var i = 0; i < count; i++)
{
var offset = i * 3;
var extractedToolsDirectory = Path.Combine(Path.GetTempPath(), "Tor", "TorExtracted" + i);
var settings = new TorSharpSettings
{
ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "Tor", "TorZipped"),
ExtractedToolsDirectory = extractedToolsDirectory,
PrivoxyPort = 1337 + offset,
TorSocksPort = 1338 + offset,
TorControlPort = 1339 + offset,
TorControlPassword = "******" + i,
TorDataDirectory = Path.Combine(extractedToolsDirectory, "DataDirectory")
};
var torSharpProxy = new TorSharpProxy(settings);
await torSharpProxy.ConfigureAndStartAsync();
ProxyList.Add($"http://localhost:{settings.PrivoxyPort}");
taskWaitingConfigurationEnd.Add(torSharpProxy.WaitForConnectionAsync());
}
Task.WaitAll(taskWaitingConfigurationEnd.ToArray());
}
private static async Task MainAsync()
{
// configure
var settings = new TorSharpSettings
{
ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"),
ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"),
PrivoxyPort = 1337,
TorSocksPort = 1338,
TorControlPort = 1339,
TorControlPassword = "******",
ToolRunnerType = ToolRunnerType.Simple
};
// download tools
await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync();
// execute
var proxy = new TorSharpProxy(settings);
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxyPort))
};
var httpClient = new HttpClient(handler);
await proxy.ConfigureAndStartAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com"));
await proxy.GetNewIdentityAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com"));
proxy.Stop();
}
private static async Task MainAsync()
{
// configure
var settings = new TorSharpSettings
{
ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"),
ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"),
PrivoxySettings =
{
Port = 1337,
},
TorSettings =
{
SocksPort = 1338,
ControlPort = 1339,
ControlPassword = "******",
},
};
// download tools
using (var httpClient = new HttpClient())
{
var fetcher = new TorSharpToolFetcher(settings, httpClient);
var updates = await fetcher.CheckForUpdatesAsync();
Console.WriteLine($"Current Privoxy: {updates.Privoxy.LocalVersion?.ToString() ?? "(none)"}");
Console.WriteLine($" Latest Privoxy: {updates.Privoxy.LatestDownload.Version}");
Console.WriteLine();
Console.WriteLine($"Current Tor: {updates.Tor.LocalVersion?.ToString() ?? "(none)"}");
Console.WriteLine($" Latest Tor: {updates.Tor.LatestDownload.Version}");
Console.WriteLine();
if (updates.HasUpdate)
{
await fetcher.FetchAsync(updates);
}
}
// execute
using (var proxy = new TorSharpProxy(settings))
{
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port))
};
using (handler)
using (var httpClient = new HttpClient(handler))
{
await proxy.ConfigureAndStartAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org"));
await proxy.GetNewIdentityAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org"));
}
proxy.Stop();
}
}
private static async Task StartTorAndPrivoxyAsync()
{
var settings = new TorSharpSettings {
TorSocksPort = SocksPort, PrivoxyPort = HttpConnectPort
};
var fetcher = new TorSharpToolFetcher(settings, new HttpClient());
await fetcher.FetchAsync();
var proxy = new TorSharpProxy(settings);
await proxy.ConfigureAndStartAsync();
}
private async Task <IPAddress> GetCurrentIpAddressAsync(TorSharpProxy proxy, TorSharpSettings settings)
{
return(await SendProxiedRequestAsync(
proxy,
settings,
"fetching an IP",
async proxiedHttpClient =>
{
var ip = (await proxiedHttpClient.GetStringAsync("https://api.ipify.org")).Trim();
_output.WriteLine($"Get IP succeeded: {ip}");
return IPAddress.Parse(ip);
}));
}
public async Task TorSharpToolFetcher_CheckForUpdates(ToolDownloadStrategy strategy)
{
using (var te = TestEnvironment.Initialize(_output))
{
// Arrange
var settings = te.BuildSettings();
settings.ToolDownloadStrategy = strategy;
using (var httpClientHandler = new HttpClientHandler())
using (var loggingHandler = new LoggingHandler(_output)
{
InnerHandler = httpClientHandler
})
using (var httpClient = new HttpClient(loggingHandler))
using (var proxy = new TorSharpProxy(settings))
{
_output.WriteLine(settings);
var fetcher = _httpFixture.GetTorSharpToolFetcher(settings, httpClient);
var initial = await fetcher.CheckForUpdatesAsync();
await fetcher.FetchAsync(initial);
var prefix = ToolUtility.GetPrivoxyToolSettings(settings).Prefix;
var extension = Path.GetExtension(initial.Privoxy.DestinationPath);
var fakeOldPrivoxy = Path.Combine(settings.ZippedToolsDirectory, $"{prefix}0.0.1{extension}");
File.Move(initial.Privoxy.DestinationPath, fakeOldPrivoxy);
// Act
var newerVersion = await fetcher.CheckForUpdatesAsync();
await fetcher.FetchAsync(newerVersion);
var upToDate = await fetcher.CheckForUpdatesAsync();
// Assert
Assert.True(initial.HasUpdate);
Assert.Equal(ToolUpdateStatus.NoLocalVersion, initial.Privoxy.Status);
Assert.Equal(ToolUpdateStatus.NoLocalVersion, initial.Tor.Status);
Assert.True(newerVersion.HasUpdate);
Assert.Equal(ToolUpdateStatus.NewerVersionAvailable, newerVersion.Privoxy.Status);
Assert.Equal(ToolUpdateStatus.NoUpdateAvailable, newerVersion.Tor.Status);
Assert.False(upToDate.HasUpdate);
Assert.Equal(ToolUpdateStatus.NoUpdateAvailable, upToDate.Privoxy.Status);
Assert.Equal(ToolUpdateStatus.NoUpdateAvailable, upToDate.Tor.Status);
}
}
}
public static async Task <string> MakeTorRequest(string url)
{
string result = String.Empty;
using (var proxy = new TorSharpProxy(_torSettings))
using (var handler = proxy.CreateHttpClientHandler())
using (var httpClient = new HttpClient(handler))
{
await proxy.ConfigureAndStartAsync();
result = await httpClient.GetStringAsync(url);
}
return(result);
}
private async Task ExecuteEndToEndTestAsync(TorSharpSettings settings)
{
// Arrange
using (var httpClient = new HttpClient())
using (var proxy = new TorSharpProxy(settings))
{
_output.WriteLine(settings);
// Act
var fetcher = _httpFixture.GetTorSharpToolFetcher(settings, httpClient);
await fetcher.FetchAsync();
_output.WriteLine("The tools have been fetched");
await proxy.ConfigureAndStartAsync();
_output.WriteLine("The proxy has been started");
// get the first identity
var ipA = await GetCurrentIpAddressAsync(proxy, settings);
await proxy.GetNewIdentityAsync();
_output.WriteLine("Get new identity succeeded");
var ipB = await GetCurrentIpAddressAsync(proxy, settings);
// Assert
Assert.Equal(AddressFamily.InterNetwork, ipA.AddressFamily);
Assert.Equal(AddressFamily.InterNetwork, ipB.AddressFamily);
var zippedDir = new DirectoryInfo(settings.ZippedToolsDirectory);
Assert.True(zippedDir.Exists, "The zipped tools directory should exist.");
Assert.Empty(zippedDir.EnumerateDirectories());
Assert.Equal(2, zippedDir.EnumerateFiles().Count());
var extractedDir = new DirectoryInfo(settings.ExtractedToolsDirectory);
Assert.True(extractedDir.Exists, "The extracted tools directory should exist.");
Assert.Equal(2, extractedDir.EnumerateDirectories().Count());
Assert.Empty(extractedDir.EnumerateFiles());
}
}
private async Task InitializeTor()
{
Settings = new TorSharpSettings
{
ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"),
ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"),
PrivoxyPort = 1337,
TorSocksPort = 1338,
TorControlPort = 1339,
TorControlPassword = "******"
};
// download tools
await new TorSharpToolFetcher(Settings, new HttpClient()).FetchAsync();
// execute
Proxy = new TorSharpProxy(Settings);
TorHandler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + Settings.PrivoxyPort))
};
}
public async void StartProxyAsync()
{
var settings = new TorSharpSettings
{
ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"),
ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"),
PrivoxyPort = 1337,
TorSocksPort = 1338,
TorControlPort = 1339,
TorControlPassword = "******",
TorrcLoc = Application.StartupPath + "\\torrc"
};
await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync();
proxy = new TorSharpProxy(settings);
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxyPort))
};
var httpClient = new HttpClient(handler);
await proxy.ConfigureAndStartAsync();
}
public async Task TorSharpToolFetcher_UseExistingTools()
{
using (var te = TestEnvironment.Initialize(_output))
{
// Arrange
var settings = te.BuildSettings();
settings.ReloadTools = true;
settings.UseExistingTools = true;
using (var httpClientHandler = new HttpClientHandler())
using (var requestCountHandler = new RequestCountHandler {
InnerHandler = httpClientHandler
})
using (var loggingHandler = new LoggingHandler(_output)
{
InnerHandler = requestCountHandler
})
using (var httpClient = new HttpClient(requestCountHandler))
using (var proxy = new TorSharpProxy(settings))
{
_output.WriteLine(settings);
var fetcherA = _httpFixture.GetTorSharpToolFetcher(settings, httpClient);
await fetcherA.FetchAsync();
var requestCount = requestCountHandler.RequestCount;
// Act
var fetcherB = _httpFixture.GetTorSharpToolFetcher(settings, httpClient);
await fetcherB.FetchAsync();
// Assert
Assert.True(requestCount > 0, "The should be at least one request.");
Assert.Equal(requestCount, requestCountHandler.RequestCount);
Assert.NotNull(ToolUtility.GetLatestToolOrNull(settings, ToolUtility.GetPrivoxyToolSettings(settings)));
Assert.NotNull(ToolUtility.GetLatestToolOrNull(settings, ToolUtility.GetTorToolSettings(settings)));
}
}
}
private async Task ExecuteEndToEndTestAsync(TorSharpSettings settings)
{
// Arrange
using (var proxy = new TorSharpProxy(settings))
{
_output.WriteLine($"Initialized proxy with tool runner type {settings.ToolRunnerType}");
// Act
await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync();
_output.WriteLine("The tools have been fetched");
await proxy.ConfigureAndStartAsync();
_output.WriteLine("The proxy has been started");
// get the first identity
var ipA = await GetCurrentIpAddressAsync(settings);
await proxy.GetNewIdentityAsync();
_output.WriteLine("Get new identity succeeded");
var ipB = await GetCurrentIpAddressAsync(settings);
// Assert
Assert.Equal(AddressFamily.InterNetwork, ipA.AddressFamily);
Assert.Equal(AddressFamily.InterNetwork, ipB.AddressFamily);
}
}
public async Task TrafficReadAndWritten()
{
using (var te = TestEnvironment.Initialize(_output))
{
// Arrange
var settings = te.BuildSettings();
settings.ToolRunnerType = ToolRunnerType.Simple;
using (var httpClient = new HttpClient())
using (var proxy = new TorSharpProxy(settings))
{
_output.WriteLine(settings);
var fetcher = _httpFixture.GetTorSharpToolFetcher(settings, httpClient);
await fetcher.FetchAsync();
_output.WriteLine("The tools have been fetched");
await proxy.ConfigureAndStartAsync();
_output.WriteLine("The proxy has been started");
await GetCurrentIpAddressAsync(proxy, settings);
_output.WriteLine("The first request has succeeded");
// Act
using (var controlClient = await proxy.GetControlClientAsync())
{
var readBefore = await controlClient.GetTrafficReadAsync();
_output.WriteLine($"Read before: {readBefore}");
var writtenBefore = await controlClient.GetTrafficWrittenAsync();
_output.WriteLine($"Written before: {writtenBefore}");
// Upload and download some bytes
var bytes = 10000;
await SendProxiedRequestAsync(
proxy,
settings,
$"uploading and downloading {bytes} bytes",
async proxiedHttpClient =>
{
using (var content = new ByteArrayContent(new byte[bytes]))
using (var response = await proxiedHttpClient.PostAsync("https://httpbin.org/anything", content))
{
}
return(true);
});
var readAfter = await controlClient.GetTrafficReadAsync();
_output.WriteLine($"Read after: {readBefore}");
var writtenAfter = await controlClient.GetTrafficWrittenAsync();
_output.WriteLine($"Written after: {writtenBefore}");
// Assert
Assert.True(readBefore + bytes <= readAfter, $"At least {bytes} more bytes should have been read.");
Assert.True(writtenBefore + bytes <= writtenAfter, $"At least {bytes} more bytes should have been written.");
}
}
}
}
private static async Task MainAsync()
{
// configure
var settings = new TorSharpSettings
{
ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"),
ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"),
PrivoxySettings =
{
Port = 18118,
},
TorSettings =
{
SocksPort = 19050,
ControlPort = 19051,
ControlPassword = "******",
},
};
// output runtime information
var message = new StringBuilder();
message.Append($"Running the sample on {settings.OSPlatform} OS and {settings.Architecture} architecture.");
#if NETCOREAPP
message.Append($" OS description: {RuntimeInformation.OSDescription}.");
#endif
Console.WriteLine(message.ToString());
Console.WriteLine();
// download tools
using (var httpClient = new HttpClient())
{
var fetcher = new TorSharpToolFetcher(settings, httpClient);
var updates = await fetcher.CheckForUpdatesAsync();
Console.WriteLine($"Current Privoxy: {updates.Privoxy.LocalVersion?.ToString() ?? "(none)"}");
Console.WriteLine($" Latest Privoxy: {updates.Privoxy.LatestDownload.Version}");
Console.WriteLine();
Console.WriteLine($"Current Tor: {updates.Tor.LocalVersion?.ToString() ?? "(none)"}");
Console.WriteLine($" Latest Tor: {updates.Tor.LatestDownload.Version}");
Console.WriteLine();
if (updates.HasUpdate)
{
await fetcher.FetchAsync(updates);
}
}
// execute
using (var proxy = new TorSharpProxy(settings))
{
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port))
};
using (handler)
using (var httpClient = new HttpClient(handler))
{
await proxy.ConfigureAndStartAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org"));
await proxy.GetNewIdentityAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org"));
using (var controlClient = await proxy.GetControlClientAsync())
{
var read = await controlClient.GetTrafficReadAsync();
Console.WriteLine("Bytes read : {0}", read);
var written = await controlClient.GetTrafficWrittenAsync();
Console.WriteLine("Bytes written : {0}", written);
}
}
proxy.Stop();
}
}
// PRIVATE CLASS MEMBERS
// MAIN ===================================================================================
/// <param name="args"> Command line arguements passed to executable. </param>
static async Task Main(string[] args)
{
// Creates a Logger object from Serilog. Writes up to Debug level prints.
SetupLogger();
Log.Logger.Information("Darkweb Data Scraper start...");
// Parses command line arguements and stores them in "parsedArgs"
SetupParser(args);
// I made this function to move this setup out of main.
// Returns a TorSharpSettings object for use with TorSharp.
var settings = SetupTorSharpSettings();
// Idk exactly how this works but like... its for torsharp
// its uh... setting up torsharp "tools"...also its asyncronous
await SetupTorSharpTools(settings);
// * starts tor proxy -----------------------------------------------------------------
using (var proxy = new TorSharpProxy(settings))
{
var waiting = true;
while (waiting)
{
// block untill we wait for TorSharp Proxy to be configured
await proxy.ConfigureAndStartAsync();
waiting = false;
}
// * SETUP AND EXECUTE CRAWLER ================================================
// Setup Crawler configuration
CrawlConfigurationX crawlConfig = new CrawlConfigurationX
{
MaxPagesToCrawl = 30, // Max total urls this crawler should crawl
MaxCrawlDepth = 1, // Depth for crawler to traverse urls
IsJavascriptRenderingEnabled = false, // Should crawler render JS?
JavascriptRenderingWaitTimeInMilliseconds = 2000, // How long to wait for js to process
MaxConcurrentSiteCrawls = 1, // Only crawl a single site at a time
MaxRetryCount = 3 // Retries to connect and crawl site 'x' times
};
if (parsedArgs.InputFile == null) // THIS IS "-s" or "--single"
{
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port))
};
// Crawl
await DataScraper.Crawl(crawlConfig, handler, parsedArgs.handlerType, parsedArgs.StartingUri);
BuildBsonDocument(DataScraper.allParsedText, parsedArgs.StartingUri);
//reset vals for next crawl
DataScraper.allParsedText = new List <string>();
DataScraper.siteTitle = "";
DataScraper.dataDocuments = new List <BsonDocument>();
if (MONGO_URI == "")
{
Log.Logger.Information("Database information is no longer accessible or available." +
"You will need to provide your own Mongo details in \"Crawler.cs\".");
}
}
else // THIS IS "-m" or "--multi"
{
string inputFilePath = @parsedArgs.InputFile;
var sitesToCrawl = GenerateSiteList(inputFilePath);
for (int i = 0; i < sitesToCrawl.Count; i++)
{
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port))
};
// Crawl
await DataScraper.Crawl(crawlConfig, handler, parsedArgs.handlerType, sitesToCrawl[i]);
BuildBsonDocument(DataScraper.allParsedText, sitesToCrawl[i]);
if (MONGO_URI == "")
{
Log.Logger.Information("Database information is no longer accessible or available." +
"You will need to provide your own Mongo details in \"Crawler.cs\".");
}
if (MONGO_URI != "" && MONGO_COLLECTION_NAME != "" && MONGO_DATABASE_NAME != "")
{
var client = new MongoClient(MONGO_URI);
var database = client.GetDatabase(MONGO_DATABASE_NAME);
var collection = database.GetCollection <BsonDocument>(MONGO_COLLECTION_NAME);
collection.InsertMany(DataScraper.dataDocuments);
}
//reset vals for next crawl
DataScraper.allParsedText = new List <string>();
DataScraper.siteTitle = "";
DataScraper.dataDocuments = new List <BsonDocument>();
}
}
// * ==========================================================================
// Stop the TorSharp tools so that the proxy is no longer listening on the configured port.
proxy.Stop();
}
// * ----------------------------------------------------------------------------------
}
public static async Task Run(CancellationToken cancelToken, string store)
{
Timer.Tick += new EventHandler(Timer_Tick);
Timer.Interval = 1000;
if (!Settings.Get <bool>("ScanNew") && !Settings.Get <bool>("ScanUsed"))
{
mf.textBox4.AppendText("Crawler wird gestoppt - Reason: Neu & Gebraucht sind Deaktiviert" + Environment.NewLine);
mf.metroButton3.PerformClick();
return;
}
mf.textBox4.AppendText("Übernehme Neu & Gebraucht Einstellungen" + Environment.NewLine);
var sellerForNew = new List <string>();
if (!String.IsNullOrWhiteSpace(Settings.Get <string>("SellerForNew")))
{
sellerForNew = Settings.Get <string>("SellerForNew").Split(',').Select(s => s.Trim()).ToList();
}
else
{
sellerForNew.Add("Amazon");
}
var sellerForUsed = new List <string>();
if (!String.IsNullOrWhiteSpace(Settings.Get <string>("SellerForUsed")))
{
sellerForUsed = Settings.Get <string>("SellerForUsed").Split(',').Select(s => s.Trim()).ToList();
}
else
{
sellerForUsed.Add("Amazon");
}
TorSharpSettings torSettings = null;
TorSharpProxy torProxy = null;
if (Settings.Get <bool>("UseTorProxies"))
{
mf.textBox4.AppendText("Initialisiere TOR Proxy Funktion" + Environment.NewLine);
try
{
torSettings = new TorSharpSettings
{
ReloadTools = true,
ZippedToolsDirectory = Path.Combine(FoldersFilesAndPaths.Temp, "TorZipped"),
ExtractedToolsDirectory = Path.Combine(FoldersFilesAndPaths.Temp, "TorExtracted"),
PrivoxySettings =
{
Port = 1337,
},
TorSettings =
{
SocksPort = 1338,
ControlPort = 1339,
ControlPassword = "******",
},
};
await new TorSharpToolFetcher(torSettings, new HttpClient()).FetchAsync();
torProxy = new TorSharpProxy(torSettings);
await torProxy.ConfigureAndStartAsync();
}
catch (Exception ex)
{
mf.textBox4.AppendText("Fehler beim Initialisiere der TOR Proxy Funktion ;-(" + Environment.NewLine + "TOR Proxy Funktion wird deaktiviert!" + Environment.NewLine);
mf.textBox4.AppendText(ex.Message);
mf.metroToggle5.Checked = false;
}
if (Settings.IsPremium && Settings.Get <bool>("UseTorProxies") && Settings.Get <bool>("ProxyAlwaysActive"))
{
mf.textBox4.AppendText("Set TOR Proxy - Reason: P. always Active" + Environment.NewLine);
mf.metroLabel45.Text = Convert.ToString(await WebUtils.GetCurrentIpAddressAsync(torSettings));
Proxies.TorProxyActive = true;
}
}
string sourcecode;
int sites, loops;
while (true)
{
Database.OpenConnection();
SQLiteCommand getEntry;
switch (store)
{
case "ALLE":
getEntry = new SQLiteCommand("SELECT * FROM Products WHERE Status = '0' ORDER BY [Letzter Check] ASC LIMIT 0,1",
Database.Connection);
break;
default:
getEntry =
new SQLiteCommand("SELECT * FROM Products WHERE Status = '0' AND Store = @store ORDER BY [Letzter Check] ASC LIMIT 0,1",
Database.Connection);
getEntry.Parameters.AddWithValue("@store", store);
break;
}
SQLiteDataReader entry = getEntry.ExecuteReader();
if (!entry.HasRows)
{
mf.metroButton3.PerformClick();
Timer.Stop();
torProxy.Stop();
return;
}
while (entry.Read())
{
if (cancelToken.IsCancellationRequested)
{
Timer.Stop();
torProxy.Stop();
return;
}
mf.metroLabel10.Text = Convert.ToString(entry["Store"]);
mf.metroLabel11.Text = Convert.ToString(entry["ASIN / ISBN"]);
mf.metroLabel12.Text = Convert.ToString(entry["Name"]);
mf.metroLabel13.Text = Convert.ToString(entry["Preis: Neu"]);
mf.metroLabel14.Text = Convert.ToString(entry["Preis: Wie Neu"]);
mf.metroLabel15.Text = Convert.ToString(entry["Preis: Sehr Gut"]);
mf.metroLabel16.Text = Convert.ToString(entry["Preis: Gut"]);
mf.metroLabel17.Text = Convert.ToString(entry["Preis: Akzeptabel"]);
mf.metroLabel18.Text = Convert.ToString(entry["Letzter Check"]);
mf.metroLabel24.Text = mf.metroLabel25.Text = mf.metroLabel26.Text = mf.metroLabel27.Text = mf.metroLabel28.Text = null;
mf.metroLabel41.Text = null;
mf.webBrowser1.DocumentText = mf.webBrowser2.DocumentText = null;
await Task.Delay(Tools.RandomNumber(250, 500), cancelToken).ContinueWith(tsk => { });
string priceNew = null;
if (Settings.Get <bool>("ScanNew"))
{
List <string> priceNewList = new List <string>();
mf.tabControl2.SelectedTab = mf.TabPage4;
sites = 1;
loops = 0;
do
{
if (cancelToken.IsCancellationRequested)
{
Timer.Stop();
torProxy.Stop();
return;
}
try
{
string url = "https://www.amazon." + Amazon.GetTld(Convert.ToString(entry["Store"])) +
"/gp/aw/ol/" + entry["ASIN / ISBN"] +
"/ref=mw_dp_olp?ca=" + entry["ASIN / ISBN"] + "&o=New&op=" + (loops + 1) +
"&vs=1";
var handler = new HttpClientHandler();
if (Settings.Get <bool>("UseTorProxies"))
{
if (Settings.IsPremium && Settings.Get <bool>("ProxyAlwaysActive") ||
Proxies.TorProxyActive && Proxies.RealIPnexttime > DateTime.Now)
{
Debug.WriteLine("Add TOR Proxy to HttpClientHandler...");
handler.Proxy =
new WebProxy(new Uri("http://*****:*****@"<title> (Tut uns Leid!|Toutes nos excuses)</title>");
Match errordetection1 = Regex.Match(sourcecode.RemoveLineEndings(),
@"<title>(503 - Service Unavailable Error)</title>");
if (errordetection.Success || errordetection1.Success)
{
Debug.WriteLine("Issue Found...");
continue;
}
//CAPTCHA Detection
Match captchadetection =
Regex.Match(sourcecode,
"<title dir=\"ltr\">(Amazon CAPTCHA|Bot Check|Robot Check)</title>");
if (captchadetection.Success)
{
Debug.WriteLine("Captcha detected...");
if (Settings.Get <bool>("UseTorProxies"))
{
await torProxy.GetNewIdentityAsync();
mf.metroLabel45.Text =
Convert.ToString(await WebUtils.GetCurrentIpAddressAsync(torSettings));
Proxies.TorProxyActive = true;
if (!Settings.Get <bool>("ProxyAlwaysActive") &&
Proxies.RealIPnexttime < DateTime.Now)
{
Proxies.RealIPnexttime = DateTime.Now.AddMinutes(15);
}
}
continue;
}
//How many Sites?
Match sitesNumber = Regex.Match(sourcecode, "<a name=\"New\">([^\"]*) / ([^\"]*)</a>");
if (sitesNumber.Success && loops == 0 && Settings.Get <int>("ScanMethod") == 0)
{
sites = Convert.ToInt32(
Math.Ceiling(Convert.ToDecimal(sitesNumber.Groups[2].Value) / 10));
}
mf.metroLabel41.Text = $@"{loops + 1} / {sites} (NEU)";
//Get Price for NEW
if (sellerForNew.Contains("Amazon"))
{
Match newama = Regex.Match(sourcecode,
"<a href=(.*)ca=([a-zA-Z0-9_]*)&vs=1(.*)>(Neu|Nuovo|Neuf|Nuevo|New) - (EUR |£)(.*)</a>");
if (newama.Success)
{
priceNewList.Add(await CurrencyConverter.ConvertToEuro(newama.Groups[6].Value,
Convert.ToString(entry["Store"])));
}
}
if (sellerForNew.Contains("Drittanbieter"))
{
if (sellerForNew.Contains("Versand per Amazon"))
{
Regex new3rd =
new Regex(
"(Versand durch Amazon.de|Spedito da Amazon|EXPÉDIÉ PAR AMAZON|Distribuido por Amazon|Fulfilled by Amazon)(\r\n|\r|\n)</font>(\r\n|\r|\n)(\r\n|\r|\n)<br />(\r\n|\r|\n)(\r\n|\r|\n)(\r\n|\r|\n)" +
"<a href=(.*)ca=([a-zA-Z0-9_]*)&eid=(.*)>(Neu|Nuovo|Neuf|Nuevo|New) - (EUR |£)(.*)</a>",
RegexOptions.Compiled);
foreach (Match itemMatch in new3rd.Matches(sourcecode))
{
priceNewList.Add(
await CurrencyConverter.ConvertToEuro(itemMatch.Groups[13].Value,
Convert.ToString(entry["Store"])));
}
}
else
{
Regex new3rd =
new Regex(
@"<a href=(.*)ca=([a-zA-Z0-9_]*)&eid=(.*)>(Neu|Nuovo|Neuf|Nuevo|New) - (EUR |£)(.*)</a>",
RegexOptions.Compiled);
foreach (Match itemMatch in new3rd.Matches(sourcecode))
{
priceNewList.Add(
await CurrencyConverter.ConvertToEuro(itemMatch.Groups[6].Value,
Convert.ToString(entry["Store"])));
}
}
}
loops++;
if (sites > loops)
{
await Task.Delay(
Tools.RandomNumber(Convert.ToInt32(mf.numericUpDown3.Value) * 1000 - 500,
Convert.ToInt32(mf.numericUpDown3.Value) * 1000 + 500), cancelToken)
.ContinueWith(tsk => { });
}
} while (sites > loops);
RemoveInvalidEntrys(priceNewList);
priceNew = priceNewList.OrderBy(price => price).FirstOrDefault();
if (!String.IsNullOrEmpty(priceNew))
{
mf.metroLabel24.Text = priceNew;
ComparePrice(entry["Preis: Neu"], priceNew, mf.metroLabel24);
}
}
await Task.Delay(Tools.RandomNumber(500, 1000), cancelToken).ContinueWith(tsk => { });
string priceLikeNew = null;
string priceVeryGood = null;
string priceGood = null;
string priceAcceptable = null;
if (Settings.Get <bool>("ScanUsed"))
{
List <string> priceLikeNewList = new List <string>();
List <string> priceVeryGoodList = new List <string>();
List <string> priceGoodList = new List <string>();
List <string> priceAcceptableList = new List <string>();
mf.tabControl2.SelectedTab = mf.TabPage5;
var conditionList = new List <Tuple <List <string>, string> >
{
new Tuple <List <string>, string>(priceLikeNewList,
"Wie neu|Condizioni pari al nuovo|Comme neuf|Como nuevo|Mint"),
new Tuple <List <string>, string>(priceVeryGoodList,
"Sehr gut|Ottimo|Très bon état|Muy bueno|Very good"),
new Tuple <List <string>, string>(priceGoodList,
"Gut|Buono|D'occasion - très bon état|Bueno|Good"),
new Tuple <List <string>, string>(priceAcceptableList,
"Akzeptabel|Accettabile|Acceptable|Aceptable|Acceptable")
};
sites = 1;
loops = 0;
do
{
if (cancelToken.IsCancellationRequested)
{
Timer.Stop();
torProxy.Stop();
return;
}
try
{
string url = "https://www.amazon." + Amazon.GetTld(Convert.ToString(entry["Store"])) +
"/gp/aw/ol/" + entry["ASIN / ISBN"] +
"/ref=mw_dp_olp?o=Used&op=" + (loops + 1);
var handler = new HttpClientHandler();
if (Settings.Get <bool>("UseTorProxies"))
{
if (Settings.IsPremium && Settings.Get <bool>("ProxyAlwaysActive") || Proxies.TorProxyActive && Proxies.RealIPnexttime > DateTime.Now)
{
Debug.WriteLine("Add TOR Proxy to HttpClientHandler...");
handler.Proxy = new WebProxy(new Uri("http://*****:*****@"<title> (Tut uns Leid!|Toutes nos excuses)</title>");
Match errordetection1 = Regex.Match(sourcecode.RemoveLineEndings(), @"<title>(503 - Service Unavailable Error)</title>");
if (errordetection.Success || errordetection1.Success)
{
Debug.WriteLine("Issue Found...");
continue;
}
//CAPTCHA Detection
Match captchadetection = Regex.Match(sourcecode, "<title dir=\"ltr\">(Amazon CAPTCHA|Bot Check|Robot Check)</title>");
if (captchadetection.Success)
{
Debug.WriteLine("Captcha detected...");
if (Settings.Get <bool>("UseTorProxies"))
{
await torProxy.GetNewIdentityAsync();
mf.metroLabel45.Text = Convert.ToString(await WebUtils.GetCurrentIpAddressAsync(torSettings));
Proxies.TorProxyActive = true;
if (!Settings.Get <bool>("ProxyAlwaysActive") && Proxies.RealIPnexttime < DateTime.Now)
{
Proxies.RealIPnexttime = DateTime.Now.AddMinutes(15);
}
}
continue;
}
//How many WHD Sites?
Match sitesNumber = Regex.Match(sourcecode, "<a name=\"Used\">([^\"]*) / ([^\"]*)</a>");
if (sitesNumber.Success && loops == 0 && Settings.Get <int>("ScanMethod") == 0)
{
sites = Convert.ToInt32(Math.Ceiling(Convert.ToDecimal(sitesNumber.Groups[2].Value) / 10));
}
mf.metroLabel41.Text = $@"{loops + 1} / {sites} (GEBRAUCHT)";
//Get Price for WHDs
//.de WHD Seller ID: A8KICS1PHF7ZO
//.it WHD Seller ID: A1HO9729ND375Y
//.fr WHD Seller ID: A2CVHYRTWLQO9T
//.es WHD Seller ID: A6T89FGPU3U0Q
//.co.uk WHD Seller ID: A2OAJ7377F756P
if (sellerForUsed.Contains("Amazon"))
{
foreach (var condition in conditionList)
{
Regex search =
new Regex(
@"<a href=(.*)(A8KICS1PHF7ZO|A1HO9729ND375Y|A2CVHYRTWLQO9T|A6T89FGPU3U0Q|A2OAJ7377F756P)(.*)>(" + condition.Item2 + ") - (EUR |£)(.*)</a>",
RegexOptions.Compiled);
foreach (Match itemMatch in search.Matches(sourcecode))
{
condition.Item1.Add(await CurrencyConverter.ConvertToEuro(itemMatch.Groups[6].Value, Convert.ToString(entry["Store"])));
}
}
}
if (sellerForUsed.Contains("Drittanbieter"))
{
if (sellerForUsed.Contains("Versand per Amazon"))
{
foreach (var condition in conditionList)
{
Regex search =
new Regex(
"(Versand durch Amazon.de|Spedito da Amazon|EXPÉDIÉ PAR AMAZON|Distribuido por Amazon|Fulfilled by Amazon)(\r\n|\r|\n)</font>(\r\n|\r|\n)(\r\n|\r|\n)<br />(\r\n|\r|\n)(\r\n|\r|\n)(\r\n|\r|\n)" +
"<a href=(.*)&me=(?!(A8KICS1PHF7ZO|A1HO9729ND375Y|A2CVHYRTWLQO9T|A6T89FGPU3U0Q|A2OAJ7377F756P).)(.*)>(" + condition.Item2 + ") - (EUR |£)(.*)</a>",
RegexOptions.Compiled);
foreach (Match itemMatch in search.Matches(sourcecode))
{
condition.Item1.Add(await CurrencyConverter.ConvertToEuro(itemMatch.Groups[13].Value, Convert.ToString(entry["Store"])));
}
}
}
else
{
foreach (var condition in conditionList)
{
Regex search =
new Regex(
@"<a href=(.*)&me=(?!(A8KICS1PHF7ZO|A1HO9729ND375Y|A2CVHYRTWLQO9T|A6T89FGPU3U0Q|A2OAJ7377F756P).)(.*)>(" + condition.Item2 + ") - (EUR |£)(.*)</a>",
RegexOptions.Compiled);
foreach (Match itemMatch in search.Matches(sourcecode))
{
condition.Item1.Add(await CurrencyConverter.ConvertToEuro(itemMatch.Groups[6].Value, Convert.ToString(entry["Store"])));
}
}
}
}
loops++;
if (sites > loops)
{
await Task.Delay(Tools.RandomNumber(Convert.ToInt32(mf.numericUpDown3.Value) * 1000 - 500, Convert.ToInt32(mf.numericUpDown3.Value) * 1000 + 500), cancelToken).ContinueWith(tsk => { });
}
} while (sites > loops);
foreach (var condition in conditionList)
{
RemoveInvalidEntrys(condition.Item1);
}
priceLikeNew = priceLikeNewList.OrderBy(price => price).FirstOrDefault();
if (!String.IsNullOrEmpty(priceLikeNew))
{
mf.metroLabel25.Text = priceLikeNew;
ComparePrice(entry["Preis: Wie Neu"], priceLikeNew, mf.metroLabel25);
}
priceVeryGood = priceVeryGoodList.OrderBy(price => price).FirstOrDefault();
if (!String.IsNullOrEmpty(priceVeryGood))
{
mf.metroLabel26.Text = priceVeryGood;
ComparePrice(entry["Preis: Sehr Gut"], priceVeryGood, mf.metroLabel26);
}
priceGood = priceGoodList.OrderBy(price => price).FirstOrDefault();
if (!String.IsNullOrEmpty(priceGood))
{
mf.metroLabel27.Text = priceGood;
ComparePrice(entry["Preis: Gut"], priceGood, mf.metroLabel27);
}
priceAcceptable = priceAcceptableList.OrderBy(price => price).FirstOrDefault();
if (!String.IsNullOrEmpty(priceAcceptable))
{
mf.metroLabel28.Text = priceAcceptable;
ComparePrice(entry["Preis: Akzeptabel"], priceAcceptable, mf.metroLabel28);
}
}
await Task.Delay(Tools.RandomNumber(500, 1000), cancelToken).ContinueWith(tsk => { });
Database.OpenConnection();
SQLiteCommand updateEntry =
new SQLiteCommand(
"UPDATE Products SET [Preis: Neu] = @priceNew, [Preis: Wie Neu] = @priceLikenew, [Preis: Sehr Gut] = @priceVerygood, [Preis: Gut] = @priceGood, [Preis: Akzeptabel] = @priceAcceptable, [Letzter Check] = @lastcheck WHERE ID = @id",
Database.Connection);
updateEntry.Parameters.AddWithValue("@id", entry["ID"]);
updateEntry.Parameters.AddWithValue("@priceNew", !String.IsNullOrEmpty(priceNew) ? priceNew : null);
updateEntry.Parameters.AddWithValue("@priceLikenew", !String.IsNullOrEmpty(priceLikeNew) ? priceLikeNew : null);
updateEntry.Parameters.AddWithValue("@priceVerygood", !String.IsNullOrEmpty(priceVeryGood) ? priceVeryGood : null);
updateEntry.Parameters.AddWithValue("@priceGood", !String.IsNullOrEmpty(priceGood) ? priceGood : null);
updateEntry.Parameters.AddWithValue("@priceAcceptable", !String.IsNullOrEmpty(priceAcceptable) ? priceAcceptable : null);
updateEntry.Parameters.AddWithValue("@lastcheck", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
updateEntry.ExecuteNonQuery();
await Task.Delay(250, cancelToken).ContinueWith(tsk => { });
if (mf.metroComboBox2.SelectedIndex == -1 || mf.metroComboBox2.Text == "ALLE" || store == mf.metroComboBox2.Text)
{
ProductDatabase.Display(mf.metroComboBox2.SelectedIndex == -1 ? "ALLE" : mf.metroComboBox2.Text);
}
await MyReminder.DoRemindWhenPossible(Convert.ToString(entry["ID"]));
_randomWait = Tools.RandomNumber(Convert.ToInt32(mf.numericUpDown2.Value) - 1, Convert.ToInt32(mf.numericUpDown2.Value) + 1);
mf.metroProgressBar1.Value = 0;
mf.metroProgressBar1.Maximum = _randomWait * 1000;
Timer.Start();
mf.metroLabel29.Text = _dt.AddSeconds(_randomWait).ToString("mm:ss");
await Task.Delay((_randomWait + 1) * 1000, cancelToken).ContinueWith(tsk => { });
}
}
}
