private static async Task MainAsync()
{
// configure
var settings = new TorSharpSettings
{
ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"),
ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"),
PrivoxyPort = 1337,
TorSocksPort = 1338,
TorControlPort = 1339,
TorControlPassword = "******",
ToolRunnerType = ToolRunnerType.Simple
};
// download tools
await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync();
// execute
var proxy = new TorSharpProxy(settings);
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxyPort))
};
var httpClient = new HttpClient(handler);
await proxy.ConfigureAndStartAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com"));
await proxy.GetNewIdentityAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com"));
proxy.Stop();
}
private static async Task MainAsync()
{
// configure
var settings = new TorSharpSettings
{
ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"),
ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"),
PrivoxyPort = 1337,
TorSocksPort = 1338,
TorControlPort = 1339,
TorControlPassword = "******",
ToolRunnerType = ToolRunnerType.Simple
};
// download tools
await new TorSharpToolFetcher(settings, new HttpClient()).FetchAsync();
// execute
var proxy = new TorSharpProxy(settings);
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxyPort))
};
var httpClient = new HttpClient(handler);
await proxy.ConfigureAndStartAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com"));
await proxy.GetNewIdentityAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://icanhazip.com"));
proxy.Stop();
}
private static async Task MainAsync()
{
// configure
var settings = new TorSharpSettings
{
ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"),
ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"),
PrivoxySettings =
{
Port = 1337,
},
TorSettings =
{
SocksPort = 1338,
ControlPort = 1339,
ControlPassword = "******",
},
};
// download tools
using (var httpClient = new HttpClient())
{
var fetcher = new TorSharpToolFetcher(settings, httpClient);
var updates = await fetcher.CheckForUpdatesAsync();
Console.WriteLine($"Current Privoxy: {updates.Privoxy.LocalVersion?.ToString() ?? "(none)"}");
Console.WriteLine($" Latest Privoxy: {updates.Privoxy.LatestDownload.Version}");
Console.WriteLine();
Console.WriteLine($"Current Tor: {updates.Tor.LocalVersion?.ToString() ?? "(none)"}");
Console.WriteLine($" Latest Tor: {updates.Tor.LatestDownload.Version}");
Console.WriteLine();
if (updates.HasUpdate)
{
await fetcher.FetchAsync(updates);
}
}
// execute
using (var proxy = new TorSharpProxy(settings))
{
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port))
};
using (handler)
using (var httpClient = new HttpClient(handler))
{
await proxy.ConfigureAndStartAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org"));
await proxy.GetNewIdentityAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org"));
}
proxy.Stop();
}
}
public void StopProxy()
{
proxy.Stop();
}
private static async Task MainAsync()
{
// configure
var settings = new TorSharpSettings
{
ZippedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorZipped"),
ExtractedToolsDirectory = Path.Combine(Path.GetTempPath(), "TorExtracted"),
PrivoxySettings =
{
Port = 18118,
},
TorSettings =
{
SocksPort = 19050,
ControlPort = 19051,
ControlPassword = "******",
},
};
// output runtime information
var message = new StringBuilder();
message.Append($"Running the sample on {settings.OSPlatform} OS and {settings.Architecture} architecture.");
#if NETCOREAPP
message.Append($" OS description: {RuntimeInformation.OSDescription}.");
#endif
Console.WriteLine(message.ToString());
Console.WriteLine();
// download tools
using (var httpClient = new HttpClient())
{
var fetcher = new TorSharpToolFetcher(settings, httpClient);
var updates = await fetcher.CheckForUpdatesAsync();
Console.WriteLine($"Current Privoxy: {updates.Privoxy.LocalVersion?.ToString() ?? "(none)"}");
Console.WriteLine($" Latest Privoxy: {updates.Privoxy.LatestDownload.Version}");
Console.WriteLine();
Console.WriteLine($"Current Tor: {updates.Tor.LocalVersion?.ToString() ?? "(none)"}");
Console.WriteLine($" Latest Tor: {updates.Tor.LatestDownload.Version}");
Console.WriteLine();
if (updates.HasUpdate)
{
await fetcher.FetchAsync(updates);
}
}
// execute
using (var proxy = new TorSharpProxy(settings))
{
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port))
};
using (handler)
using (var httpClient = new HttpClient(handler))
{
await proxy.ConfigureAndStartAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org"));
await proxy.GetNewIdentityAsync();
Console.WriteLine(await httpClient.GetStringAsync("http://api.ipify.org"));
using (var controlClient = await proxy.GetControlClientAsync())
{
var read = await controlClient.GetTrafficReadAsync();
Console.WriteLine("Bytes read : {0}", read);
var written = await controlClient.GetTrafficWrittenAsync();
Console.WriteLine("Bytes written : {0}", written);
}
}
proxy.Stop();
}
}
// PRIVATE CLASS MEMBERS
// MAIN ===================================================================================
/// <param name="args"> Command line arguements passed to executable. </param>
static async Task Main(string[] args)
{
// Creates a Logger object from Serilog. Writes up to Debug level prints.
SetupLogger();
Log.Logger.Information("Darkweb Data Scraper start...");
// Parses command line arguements and stores them in "parsedArgs"
SetupParser(args);
// I made this function to move this setup out of main.
// Returns a TorSharpSettings object for use with TorSharp.
var settings = SetupTorSharpSettings();
// Idk exactly how this works but like... its for torsharp
// its uh... setting up torsharp "tools"...also its asyncronous
await SetupTorSharpTools(settings);
// * starts tor proxy -----------------------------------------------------------------
using (var proxy = new TorSharpProxy(settings))
{
var waiting = true;
while (waiting)
{
// block untill we wait for TorSharp Proxy to be configured
await proxy.ConfigureAndStartAsync();
waiting = false;
}
// * SETUP AND EXECUTE CRAWLER ================================================
// Setup Crawler configuration
CrawlConfigurationX crawlConfig = new CrawlConfigurationX
{
MaxPagesToCrawl = 30, // Max total urls this crawler should crawl
MaxCrawlDepth = 1, // Depth for crawler to traverse urls
IsJavascriptRenderingEnabled = false, // Should crawler render JS?
JavascriptRenderingWaitTimeInMilliseconds = 2000, // How long to wait for js to process
MaxConcurrentSiteCrawls = 1, // Only crawl a single site at a time
MaxRetryCount = 3 // Retries to connect and crawl site 'x' times
};
if (parsedArgs.InputFile == null) // THIS IS "-s" or "--single"
{
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port))
};
// Crawl
await DataScraper.Crawl(crawlConfig, handler, parsedArgs.handlerType, parsedArgs.StartingUri);
BuildBsonDocument(DataScraper.allParsedText, parsedArgs.StartingUri);
//reset vals for next crawl
DataScraper.allParsedText = new List <string>();
DataScraper.siteTitle = "";
DataScraper.dataDocuments = new List <BsonDocument>();
if (MONGO_URI == "")
{
Log.Logger.Information("Database information is no longer accessible or available." +
"You will need to provide your own Mongo details in \"Crawler.cs\".");
}
}
else // THIS IS "-m" or "--multi"
{
string inputFilePath = @parsedArgs.InputFile;
var sitesToCrawl = GenerateSiteList(inputFilePath);
for (int i = 0; i < sitesToCrawl.Count; i++)
{
var handler = new HttpClientHandler
{
Proxy = new WebProxy(new Uri("http://localhost:" + settings.PrivoxySettings.Port))
};
// Crawl
await DataScraper.Crawl(crawlConfig, handler, parsedArgs.handlerType, sitesToCrawl[i]);
BuildBsonDocument(DataScraper.allParsedText, sitesToCrawl[i]);
if (MONGO_URI == "")
{
Log.Logger.Information("Database information is no longer accessible or available." +
"You will need to provide your own Mongo details in \"Crawler.cs\".");
}
if (MONGO_URI != "" && MONGO_COLLECTION_NAME != "" && MONGO_DATABASE_NAME != "")
{
var client = new MongoClient(MONGO_URI);
var database = client.GetDatabase(MONGO_DATABASE_NAME);
var collection = database.GetCollection <BsonDocument>(MONGO_COLLECTION_NAME);
collection.InsertMany(DataScraper.dataDocuments);
}
//reset vals for next crawl
DataScraper.allParsedText = new List <string>();
DataScraper.siteTitle = "";
DataScraper.dataDocuments = new List <BsonDocument>();
}
}
// * ==========================================================================
// Stop the TorSharp tools so that the proxy is no longer listening on the configured port.
proxy.Stop();
}
// * ----------------------------------------------------------------------------------
}