private static string UserAgent(HttpSystem system, HttpBrowser browser)
{
string systemString = string.Empty;
string browserString = string.Empty;
switch (system)
{
case HttpSystem.Windows:
systemString = $"(Windows NT {GenerateVersionOS(system)}; Win64; x64)";
break;
case HttpSystem.Linux:
systemString = $"(X11; Linux {GenerateVersionOS(system)})";
break;
case HttpSystem.Mac:
systemString = $"(Macintosh; Intel Mac OS X {GenerateVersionOS(system)})";
break;
case HttpSystem.ChromeOS:
systemString = $"(X11; CrOS x86_64 {GenerateVersionOS(system)})";
break;
}
switch (browser)
{
case HttpBrowser.Chrome:
browserString = $"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{GenerateBrowserVersion(browser)} Safari/605.1.15";
break;
case HttpBrowser.Firefox:
browserString = $"Gecko/20100101 Firefox/{GenerateBrowserVersion(browser)}";
break;
case HttpBrowser.Opera:
browserString = $"AppleWebKit/537.36 (KHTML, like Gecko) {GenerateBrowserVersion(browser)}";
break;
case HttpBrowser.Safari:
browserString = $"AppleWebKit/533.20.25 (KHTML, like Gecko) Version/{GenerateBrowserVersion(browser)} Safari/605.1.15";
break;
case HttpBrowser.Edge:
browserString = $"AppleWebKit/537.36 (KHTML, like Gecko) {GenerateBrowserVersion(browser)}";
break;
}
return($"Mozilla/5.0 {systemString} {browserString}");
}
private static string GenerateBrowserVersion(HttpBrowser browser)
{
string[] chrome = new[]
{
"70.0.3538", "71.0", "72.0", "69.0.3497.32", "70.0.3538", "71.0.3578", "72.0.3626", "73.0.3683",
"74.0.3729", "75.0.3770", "76.0.3809", "77.0.3865", "78.0.3904", "79.0.3945", "80.0.3987", "81.0.4044",
"83.0.4103", "84.0.4147", "85.0.4183", "86.0.4240", "87.0.4280", "88.0"
};
string[] opera = new[]
{
"57.0.3098.14", "56.0.3051.52", "58.0.3111.0", "56.0.3051.43", "55.0.2994.34",
"57.0.3098.1", "54.0.2952.64", "53.0.2907.68", "54.0.2952.60", "53.0.2907.99"
};
string[] safari = new[]
{
"14.0", "13.1.2", "12.1.2"
};
string[] Edge = new[]
{
"17.17134", "16.16299", "15.15063", "14.14393", "13.10586", "12.10240"
};
switch (browser)
{
case HttpBrowser.Chrome:
return(chrome[new Random().Next(0, chrome.Count())]);
case HttpBrowser.Firefox:
return($"{new Random().Next(49, 70)}.0");
case HttpBrowser.Opera:
return($"Chrome/{chrome[new Random().Next(0, chrome.Count())]} Safari/537.36 OPR/{opera[new Random().Next(0, opera.Count())]}");
case HttpBrowser.Safari:
return(safari[new Random().Next(0, safari.Count())]);
case HttpBrowser.Edge:
return($"Chrome/{chrome[new Random().Next(0, chrome.Count())]} Safari/537.36 Edge/{Edge[new Random().Next(0, Edge.Count())]}");
default:
return(null);
}
}
static async Task Main(string[] args)
{
var running = true;
var startUri = new Uri("https://tretton37.com");
var browser = new HttpBrowser();
var storage = new FileStorage(new Uri(Environment.CurrentDirectory + "/downloaded/"));
var crawler = new Crawler(browser, storage);
var session = crawler.CreateSession(startUri);
Console.WriteLine("Press any key to start crawling...");
Console.ReadKey();
Console.Clear();
_ = Task.Run(() =>
{
while (running)
{
Console.SetCursorPosition(0, 0);
Console.WriteLine($"Currently downloading: {session.Downloading.Count()} ");
Console.WriteLine($"Completed: {session.Completed.Count()} ");
Task.Delay(500);
}
});
await storage.Clear();
await session.Crawl();
running = false;
Console.WriteLine();
Console.WriteLine("Crawl finished succesfully. Press any key to exit...");
Console.ReadKey();
}
public static string GenerateUserAgent(HttpBrowser browser)
{
HttpSystem[] systemList = Enum.GetValues(typeof(HttpSystem)).Cast <HttpSystem>().ToArray();
return(UserAgent(systemList[new Random().Next(0, systemList.Count())], browser));
}
public static string GenerateUserAgent(HttpSystem system, HttpBrowser browser)
{
return(UserAgent(system, browser));
}
