private static string UserAgent(HttpSystem system, HttpBrowser browser) { string systemString = string.Empty; string browserString = string.Empty; switch (system) { case HttpSystem.Windows: systemString = $"(Windows NT {GenerateVersionOS(system)}; Win64; x64)"; break; case HttpSystem.Linux: systemString = $"(X11; Linux {GenerateVersionOS(system)})"; break; case HttpSystem.Mac: systemString = $"(Macintosh; Intel Mac OS X {GenerateVersionOS(system)})"; break; case HttpSystem.ChromeOS: systemString = $"(X11; CrOS x86_64 {GenerateVersionOS(system)})"; break; } switch (browser) { case HttpBrowser.Chrome: browserString = $"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{GenerateBrowserVersion(browser)} Safari/605.1.15"; break; case HttpBrowser.Firefox: browserString = $"Gecko/20100101 Firefox/{GenerateBrowserVersion(browser)}"; break; case HttpBrowser.Opera: browserString = $"AppleWebKit/537.36 (KHTML, like Gecko) {GenerateBrowserVersion(browser)}"; break; case HttpBrowser.Safari: browserString = $"AppleWebKit/533.20.25 (KHTML, like Gecko) Version/{GenerateBrowserVersion(browser)} Safari/605.1.15"; break; case HttpBrowser.Edge: browserString = $"AppleWebKit/537.36 (KHTML, like Gecko) {GenerateBrowserVersion(browser)}"; break; } return($"Mozilla/5.0 {systemString} {browserString}"); }
private static string GenerateBrowserVersion(HttpBrowser browser) { string[] chrome = new[] { "70.0.3538", "71.0", "72.0", "69.0.3497.32", "70.0.3538", "71.0.3578", "72.0.3626", "73.0.3683", "74.0.3729", "75.0.3770", "76.0.3809", "77.0.3865", "78.0.3904", "79.0.3945", "80.0.3987", "81.0.4044", "83.0.4103", "84.0.4147", "85.0.4183", "86.0.4240", "87.0.4280", "88.0" }; string[] opera = new[] { "57.0.3098.14", "56.0.3051.52", "58.0.3111.0", "56.0.3051.43", "55.0.2994.34", "57.0.3098.1", "54.0.2952.64", "53.0.2907.68", "54.0.2952.60", "53.0.2907.99" }; string[] safari = new[] { "14.0", "13.1.2", "12.1.2" }; string[] Edge = new[] { "17.17134", "16.16299", "15.15063", "14.14393", "13.10586", "12.10240" }; switch (browser) { case HttpBrowser.Chrome: return(chrome[new Random().Next(0, chrome.Count())]); case HttpBrowser.Firefox: return($"{new Random().Next(49, 70)}.0"); case HttpBrowser.Opera: return($"Chrome/{chrome[new Random().Next(0, chrome.Count())]} Safari/537.36 OPR/{opera[new Random().Next(0, opera.Count())]}"); case HttpBrowser.Safari: return(safari[new Random().Next(0, safari.Count())]); case HttpBrowser.Edge: return($"Chrome/{chrome[new Random().Next(0, chrome.Count())]} Safari/537.36 Edge/{Edge[new Random().Next(0, Edge.Count())]}"); default: return(null); } }
static async Task Main(string[] args) { var running = true; var startUri = new Uri("https://tretton37.com"); var browser = new HttpBrowser(); var storage = new FileStorage(new Uri(Environment.CurrentDirectory + "/downloaded/")); var crawler = new Crawler(browser, storage); var session = crawler.CreateSession(startUri); Console.WriteLine("Press any key to start crawling..."); Console.ReadKey(); Console.Clear(); _ = Task.Run(() => { while (running) { Console.SetCursorPosition(0, 0); Console.WriteLine($"Currently downloading: {session.Downloading.Count()} "); Console.WriteLine($"Completed: {session.Completed.Count()} "); Task.Delay(500); } }); await storage.Clear(); await session.Crawl(); running = false; Console.WriteLine(); Console.WriteLine("Crawl finished succesfully. Press any key to exit..."); Console.ReadKey(); }
public static string GenerateUserAgent(HttpBrowser browser) { HttpSystem[] systemList = Enum.GetValues(typeof(HttpSystem)).Cast <HttpSystem>().ToArray(); return(UserAgent(systemList[new Random().Next(0, systemList.Count())], browser)); }
public static string GenerateUserAgent(HttpSystem system, HttpBrowser browser) { return(UserAgent(system, browser)); }