private void InitializeEnvironment() { // var executablePath = PuppeteerSharp.Launcher.GetExecutablePath(); UpdateLog($"正在检测浏览器环境"); try { var bf = new BrowserFetcher() { //WebProxy = new WebProxy("http://127.0.0.1:10809") }; bf.DownloadProgressChanged += (s, e) => { UpdateLog($"正在下载 {BrowserFetcher.DefaultRevision},已完成 {e.ProgressPercentage} %"); }; var revisionInfo = bf.DownloadAsync(BrowserFetcher.DefaultRevision).GetAwaiter().GetResult(); BuildBrowser(); //var page = _browser.NewPageAsync().GetAwaiter().GetResult(); UpdateLog($"环境初始化完成,Downloaded={revisionInfo.Downloaded} ExecutablePath={revisionInfo.ExecutablePath} FolderPath={revisionInfo.FolderPath} {revisionInfo.Local} Platform={revisionInfo.Platform} Revision={revisionInfo.Revision}"); } catch (Exception ex) { XtraMessageBox.Show(this, $"初始化环境出错,{ex.Message}"); Environment.Exit(-1); } }
private static void DownloadBrowser() { var browserFetcher = new BrowserFetcher(); browserFetcher.DownloadProgressChanged += (sender, e) => { Logger.Debug( $"Browser download progress {e.ProgressPercentage}% - {e.BytesReceived}/{e.TotalBytesToReceive}bytes"); }; Logger.Debug("Download browser"); AsyncHelper.RunSync(() => browserFetcher.DownloadAsync(BrowserFetcher.DefaultRevision)); if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { var executablePath = browserFetcher.GetExecutablePath(BrowserFetcher.DefaultRevision); const int _0755 = S_IRUSR | S_IXUSR | S_IWUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; chmod(executablePath, _0755); } Logger.Debug("Browser downloaded"); }
public static async Task <LaunchOptions> StartPuppeteer() { var cctPth = Path.Join(".\\WaveExtension", "3.1.3_0"); cctPth = Path.GetFullPath(cctPth); LaunchOptions chromeOpts = new LaunchOptions { Headless = false, Args = new string[] { $"--load-extension={cctPth}", $"--disable-extensions-except={cctPth}", "--disable-web-security", "--disable-features=IsolateOrigins,site-per-process", "--disable-features=IsolateOrigins", "--disable-site-isolation-trials", "--disable-oor-cors", "--unsafely-treat-insecure-origin-as-secure=chrome-extension://jbbplnpkjmmeebjpijfedlgcdilocofh/sidebar.html" } }; AppConfig.WriteOut(">> downloading chrome if not found"); var dlTask = new BrowserFetcher().DownloadAsync(); while (dlTask.IsCompleted != true) { if (!AppConfig.silent) { AppConfig.ConsoleSpinner(); } } AppConfig.WriteOut(">> done"); //await using var browser = await Puppeteer.LaunchAsync(chromeOpts); return(chromeOpts); }
public async Task ShouldCloseTheBrowserWhenTheConnectedProcessCloses() { var browserClosedTaskWrapper = new TaskCompletionSource <bool>(); using var browserFetcher = new BrowserFetcher(Product.Chrome); var ChromiumLauncher = new ChromiumLauncher( (await browserFetcher.GetRevisionInfoAsync()).ExecutablePath, new LaunchOptions { Headless = true }); await ChromiumLauncher.StartAsync().ConfigureAwait(false); var browser = await Puppeteer.ConnectAsync(new ConnectOptions { BrowserWSEndpoint = ChromiumLauncher.EndPoint }); browser.Disconnected += (_, _) => { browserClosedTaskWrapper.SetResult(true); }; KillProcess(ChromiumLauncher.Process.Id); await browserClosedTaskWrapper.Task; Assert.True(browser.IsClosed); }
public async Task <byte[]> ConvertHtmlToPdf(string html, string cssPath) { var pdfOptions = new PdfOptions(); var browserFetcher = new BrowserFetcher(new BrowserFetcherOptions { Path = _chromiumPath }); await browserFetcher.DownloadAsync(BrowserFetcher.DefaultRevision); using var browser = await Puppeteer.LaunchAsync(new LaunchOptions { ExecutablePath = browserFetcher.RevisionInfo(BrowserFetcher.DefaultRevision).ExecutablePath, Headless = true }); var page = await browser.NewPageAsync(); await page.SetContentAsync(html); await page.AddStyleTagAsync(new AddTagOptions { Path = cssPath }); var stream = await page.PdfStreamAsync(pdfOptions); using var ms = new MemoryStream(); await stream.CopyToAsync(ms); return(ms.ToArray()); }
public static IApplicationBuilder UseBrowserFetcher(this IApplicationBuilder app) { BrowserFetcher browserFetcher = new BrowserFetcher(); browserFetcher.DownloadAsync(BrowserFetcher.DefaultRevision).GetAwaiter().GetResult(); if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { var path = Launcher.GetExecutablePath(); Bash($"chmod 777 {path}"); } return(app); void Bash(string cmd) { var escapedArgs = cmd.Replace("\"", "\\\""); var process = new Process() { StartInfo = new ProcessStartInfo { FileName = "/bin/bash", Arguments = $"-c \"{escapedArgs}\"", RedirectStandardOutput = true, UseShellExecute = false, CreateNoWindow = true, } }; process.Start(); string result = process.StandardOutput.ReadToEnd(); process.WaitForExit(); } }
private static async Task <FileInfo> DownloadAsync(DirectoryInfo directory, int revision) { await _semaphoreSlim.WaitAsync(); if (!directory.Exists) { directory.Create(); } var fetchOpt = new BrowserFetcherOptions() { Host = Environment.GetEnvironmentVariable("PUPPETEER_CHROMIUM_DOWNLOADHOST"), Path = directory.FullName }; var fetcher = new BrowserFetcher(fetchOpt); if (revision < 1) { string revisionStr = Environment.GetEnvironmentVariable("PUPPETEER_CHROMIUM_REVISION"); if (!int.TryParse(revisionStr, out revision) || revision < 1) { revision = BrowserFetcher.DefaultRevision; } } var info = await fetcher.DownloadAsync(revision).ConfigureAwait(false); _semaphoreSlim.Release(); if (!string.IsNullOrWhiteSpace(info.ExecutablePath)) { return(new FileInfo(info.ExecutablePath)); } TryGetChromeExePath(directory, out var exeFile); return(exeFile); }
public async Task GetActivity(string mssv) { var fetcher = new BrowserFetcher(); if (await fetcher.CanDownloadAsync(BrowserFetcher.DefaultRevision)) { await fetcher.DownloadAsync(BrowserFetcher.DefaultRevision); } var browser = await Puppeteer.LaunchAsync(new LaunchOptions { Headless = true }); var page = await browser.NewPageAsync(); await page.GoToAsync("http://ctct.ueh.edu.vn/modules.php?name=xemdiemmacle", WaitUntilNavigation.DOMContentLoaded); await page.TypeAsync("input[id='madoi']", mssv); await page.Keyboard.PressAsync("Enter"); await page.WaitForNavigationAsync(); string res = await page.GetContentAsync(); Debug.WriteLine(res); System.IO.File.WriteAllText(@"D:\demo.txt", res); await page.CloseAsync(); await browser.CloseAsync(); }
public async Task Usage() { var outputFile = Path.Combine(BaseDirectory, "Usage.pdf"); var fileInfo = new FileInfo(outputFile); if (fileInfo.Exists) { fileInfo.Delete(); } #region PdfAsync using var browserFetcher = new BrowserFetcher(); await browserFetcher.DownloadAsync(); await using var browser = await Puppeteer.LaunchAsync(new LaunchOptions { Headless = true }); await using var page = await browser.NewPageAsync(); await page.GoToAsync("http://www.google.com"); // In case of fonts being loaded from a CDN, use WaitUntilNavigation.Networkidle0 as a second param. await page.EvaluateExpressionHandleAsync("document.fonts.ready"); // Wait for fonts to be loaded. Omitting this might result in no text rendered in pdf. await page.PdfAsync(outputFile); #endregion Assert.True(File.Exists(outputFile)); }
static PicGen() { var downloadsFolder = Path.GetTempPath(); var option = new BrowserFetcherOptions { Path = downloadsFolder }; var fetcher = new BrowserFetcher(option); fetcher.DownloadAsync(BrowserFetcher.DefaultRevision).Wait(); var browser = Puppeteer.LaunchAsync(new LaunchOptions { ExecutablePath = fetcher.RevisionInfo(BrowserFetcher.DefaultRevision).ExecutablePath, Headless = true, Args = new[] { !isProd ? "--proxy-server=winip:1080" : "" } }).Result; page = browser.NewPageAsync().Result; page.SetViewportAsync(new ViewPortOptions { Height = 800, Width = 1300 }).Wait(); }
public static async Task Main(string[] args) { var options = new LaunchOptions { Headless = true }; Console.WriteLine("Downloading chromium"); using var browserFetcher = new BrowserFetcher(); await browserFetcher.DownloadAsync(); Console.WriteLine("Navigating google"); using (var browser = await Puppeteer.LaunchAsync(options)) using (var page = await browser.NewPageAsync()) { await page.GoToAsync("http://www.google.com"); Console.WriteLine("Generating PDF"); await page.PdfAsync(Path.Combine(Directory.GetCurrentDirectory(), "google.pdf")); Console.WriteLine("Export completed"); if (!args.Any(arg => arg == "auto-exit")) { Console.ReadLine(); } } }
public async Task Should_render_properly(string name, string url) { using (var browserFetcher = new BrowserFetcher()) { await browserFetcher.DownloadAsync(); } await using var browser = await Puppeteer.LaunchAsync(new LaunchOptions { Headless = true, DefaultViewport = new ViewPortOptions { Height = 800, IsLandscape = true, IsMobile = false, Width = 1000 }, Args = new string[] { "--no-sandbox" } }); await using var page = await browser.NewPageAsync(); await page.GoToAsync(_.ClientManager.Options.Url + url + "?skip-setup"); await page.ScreenshotAsync($"__{name}.jpg"); var diff = ImageSharpCompare.CalcDiff($"__{name}.jpg", $"Assets/{name}.jpg"); Assert.InRange(diff.MeanError, 0, 10); }
private static async Task RunPuppeteer(int externalSpoofingEnginePort) { var _ = new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision).Result; var browser = await Puppeteer.LaunchAsync(new LaunchOptions { Headless = false, Args = new[] { ("--proxy-server=http://127.0.0.1:" + externalSpoofingEnginePort) } }); try { var page = await browser.NewPageAsync(); await page.GoToAsync("https://help.kameleo.io"); Thread.Sleep(10000); } catch (Exception ex) { Console.WriteLine(ex); } finally { if (browser != null) { await browser.CloseAsync(); } } }
public static bool InitializePuppeteer() { if (!InitializingPuppeteer) { if (DefaultBrowser != null && DefaultBrowser.IsClosed) { DefaultBrowser.Dispose(); DefaultBrowser = null; } if (DefaultBrowser == null) { InitializingPuppeteer = true; try { MainWindow.mainWindowCM.SetStatusBar(MainWindowComponentModel.StatusBarEnum.PreparingChromium); RevisionInfo _revisionInfo = new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision).Result; if (_revisionInfo.Downloaded) { MainWindow.mainWindowCM.SetStatusBar(MainWindowComponentModel.StatusBarEnum.Launching); DefaultBrowser = Puppeteer.LaunchAsync(new LaunchOptions { Args = new string[] { "--disable-blink-features", "--disable-blink-features=AutomationControlled" }, IgnoredDefaultArgs = new string[] { "--enable-automation", "--headless" }, IgnoreHTTPSErrors = true, UserDataDir = "./tmp", }).Result; IList <Page> _p = DefaultBrowser.PagesAsync().Result; CurrentPage = _p[0]; if (File.Exists("./cookies.json")) { // If they exist, load cookies string _cookies = File.ReadAllText("./cookies.json"); CurrentPage.SetCookieAsync(JsonConvert.DeserializeObject <CookieParam[]>(_cookies)); } MainWindow.mainWindowCM.SetStatusBar(MainWindowComponentModel.StatusBarEnum.Ready); return(true); } else { MainWindow.mainWindowCM.SetStatusBar(MainWindowComponentModel.StatusBarEnum.Error); return(false); } } catch { MainWindow.mainWindowCM.SetStatusBar(MainWindowComponentModel.StatusBarEnum.Error); } InitializingPuppeteer = false; } } return(true); }
public async Task <Browser> BuildBrowser() { var downloadPath = Path.Combine(Path.GetTempPath(), "puppeteer"); var browserFetcherOptions = new BrowserFetcherOptions { Path = downloadPath }; var browserFetcher = new BrowserFetcher(browserFetcherOptions); await browserFetcher.DownloadAsync(BrowserFetcher.DefaultChromiumRevision); var executableDirectory = Directory.EnumerateDirectories( Directory.EnumerateDirectories(downloadPath) .First()) .First(); var executableFilename = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "chrome.exe" : "chrome"; var executablePath = Path.Combine(executableDirectory, executableFilename); var options = new LaunchOptions { Headless = true, ExecutablePath = executablePath }; var browser = await Puppeteer.LaunchAsync(options); return(browser); }
public async Task Usage() { var outputFile = Path.Combine(BaseDirectory, "Usage.pdf"); var fileInfo = new FileInfo(outputFile); if (fileInfo.Exists) { fileInfo.Delete(); } #region PdfAsync var browserFetcher = new BrowserFetcher(); await browserFetcher.DownloadAsync(); await using var browser = await Puppeteer.LaunchAsync(new LaunchOptions { Headless = true }); await using var page = await browser.NewPageAsync(); await page.GoToAsync("http://www.google.com"); await page.PdfAsync(outputFile); #endregion Assert.True(File.Exists(outputFile)); }
private async void FormLoad() { var smsServiceItems = SmsServiceItem.GetSmsServiceItems(); cmbSmsService.DataSource = smsServiceItems; cmbSmsService.DisplayMember = "Text"; cmbSmsService.SelectedIndex = 0; var browserProfileItems = BrowserProfileItem.GetItems(_browserProfileService); cmbBrowserProfile.DataSource = browserProfileItems; cmbBrowserProfile.DisplayMember = "Text"; cmbBrowserProfile.SelectedIndex = 0; var browserFetcher = new BrowserFetcher(); browserFetcher.DownloadProgressChanged += OnDownloadProgressChanged; GetBrowserLastVersion(browserFetcher); //init ServiceInfoList await _smsServices.GetServiceInfoList(ServiceCode.MailRu); foreach (SmsServiceCode smsServiceCode in Enum.GetValues(typeof(SmsServiceCode))) { var balance = await _smsServices.GetSmsService(smsServiceCode).GetBalance(); if (balance < 5) { textBox1.AppendText($@"Low balance {smsServiceCode} {balance} - {DateTime.Now} {Environment.NewLine}"); _smsServices.RemoveSmsServiceLowBalance(smsServiceCode); } } }
public async Task <int> ScrapeAndAddProducts() { int result = 0; var browserFetcher = new BrowserFetcher(); await browserFetcher.DownloadAsync(); await using var browser = await Puppeteer.LaunchAsync( new LaunchOptions { Headless = false, DefaultViewport = new ViewPortOptions { Width = 1920, Height = 1024 } }); foreach (var categoryUrl in _categoryUrlsToScrape) { await using var page = await browser.NewPageAsync(); await page.GoToAsync(categoryUrl); await ScrollUntilEnd(page); var products = await ExtractProducts(page); await AddProductsIfMissing(products); result += products.Count; } return(result); }
/// <summary> /// Load the browser fetcher from file /// </summary> /// <returns></returns> protected virtual BrowserFetcher GetBrowserFetcher() { if (BrowserFetcher != null) { return(BrowserFetcher); } var downloadPath = _settingRepository.GetSettingValue <string>("SmartScrapper.Chromium.DownloadPath"); _loggingService.Log(LogEventLevel.Information, $"Attempting to set up puppeteer to use Chromium found under directory {downloadPath}"); if (!Directory.Exists(downloadPath)) { _loggingService.Log(LogEventLevel.Information, "Custom directory not found. Creating directory"); Directory.CreateDirectory(downloadPath); } _loggingService.Log(LogEventLevel.Information, "Downloading Chromium"); var browserFetcherOptions = new BrowserFetcherOptions { Path = downloadPath }; BrowserFetcher = new BrowserFetcher(browserFetcherOptions); return(BrowserFetcher); }
private void Form1_Load(object sender, EventArgs e) { toolStripProgressBar1.Maximum = 100; toolStripProgressBar1.Value = 0; Task.Run(async() => { var browserFetcher = new BrowserFetcher(new BrowserFetcherOptions() { Host = "http://cdn.npm.taobao.org/dist" }); try { var cp = Process.GetProcessesByName("chrome"); if (cp.Length > 0) { foreach (var item in cp) { if (!item.HasExited && item.MainModule.FileName == browserFetcher.GetExecutablePath(BrowserFetcher.DefaultRevision)) { item.Kill(); } } } } catch (Exception) { } browserFetcher.DownloadProgressChanged += Bf_DownloadProgressChanged; await browserFetcher.DownloadAsync(BrowserFetcher.DefaultRevision); }); }
static async Task Browser(bool headless) { Console.WriteLine("Loading chromium"); var fetcher = new BrowserFetcher(); await fetcher.DownloadAsync(BrowserFetcher.DefaultRevision); using var browser = await Puppeteer.LaunchAsync(new LaunchOptions { UserDataDir = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "Puppeteer"), Headless = headless, DefaultViewport = new ViewPortOptions { IsMobile = false, HasTouch = false, IsLandscape = true, Width = 1600, Height = 800 }, }); Console.WriteLine("Loading codesignal.com"); var pages = await browser.PagesAsync(); var page = pages.Length > 0 ? pages[0] : await browser.NewPageAsync(); var areas = (await Scraping.ScanAreas(page)); var tasks = (await Scraping.ScanTasks(page, areas)); Console.WriteLine($"Located {tasks.Count} tasks where {tasks.Count(t => t.Solved)} is solved"); Console.WriteLine("Please note that only solved tasks and first new task and unlocked tasks are found!"); tasks = Source.FilterTasks(tasks).ToList(); Console.WriteLine($"Retrieving {tasks.Count} task to update source and tests."); int failed = 0; // var tasks = new TaskInfo[] { new TaskInfo("https://app.codesignal.com/arcade/code-arcade", "The Core", "Cliffs Of Pain", "https://app.codesignal.com/arcade/code-arcade/cliffs-of-pain/iGBDQE3KjqbYyF8DH", "timeASCIIRepresentation", false) }; foreach (var task in tasks) { await Scraping.RetrieveTask(page, task); if (task.Failed) { failed++; continue; } Source.WriteTask(task); task.Description = null; task.Source = null; task.Tests = null; } Console.WriteLine("Finished, press space to end the program."); if (failed != 0) { Console.WriteLine(); Console.WriteLine($"However, failed to import {failed} tasks."); Console.WriteLine($"Please, rerun to try again."); } Console.ReadKey(); await page.CloseAsync(); await browser.CloseAsync(); }
public static async Task DownloadRevision(RevisionInfo revision) { if (Directory.Exists(revision.FolderPath) && !IsValidRevision(revision)) { Directory.Delete(revision.FolderPath, true); } await BrowserFetcher.DownloadAsync(revision.Revision); }
public static IServiceCollection AddPuppeteerSharpPdfConverter(this IServiceCollection services) { var browserFetcher = new BrowserFetcher(); browserFetcher.DownloadAsync().GetAwaiter().GetResult(); services.AddSingleton <IPdfConverter, PuppeteerSharpConverter>(); return(services); }
private static async Task DownloadBrowserAsync() { var browserFetcher = new BrowserFetcher(new BrowserFetcherOptions() { Host = "https://npm.taobao.org/mirrors", }); browserFetcher.DownloadProgressChanged += new MyHandler().DownloadProgressChangedHandler; await browserFetcher.DownloadAsync(BrowserFetcher.DefaultRevision); }
public static async Task Main(string[] args) { Console.WriteLine("This example downloads the default version of Chromium to a custom location"); var currentDirectory = Directory.GetCurrentDirectory(); var downloadPath = Path.Combine(currentDirectory, "..", "..", "CustomChromium"); Console.WriteLine($"Attemping to set up puppeteer to use Chromium found under directory {downloadPath} "); if (!Directory.Exists(downloadPath)) { Console.WriteLine("Custom directory not found. Creating directory"); Directory.CreateDirectory(downloadPath); } Console.WriteLine("Downloading Chromium"); var browserFetcherOptions = new BrowserFetcherOptions { Path = downloadPath }; var browserFetcher = new BrowserFetcher(browserFetcherOptions); await browserFetcher.DownloadAsync(BrowserFetcher.DefaultRevision); var executablePath = browserFetcher.GetExecutablePath(BrowserFetcher.DefaultRevision); if (string.IsNullOrEmpty(executablePath)) { Console.WriteLine("Custom Chromium location is empty. Unable to start Chromium. Exiting.\n Press any key to continue"); Console.ReadLine(); return; } Console.WriteLine($"Attemping to start Chromium using executable path: {executablePath}"); var options = new LaunchOptions { Headless = true, ExecutablePath = executablePath }; using (var browser = await Puppeteer.LaunchAsync(options)) using (var page = await browser.NewPageAsync()) { await page.GoToAsync("http://www.google.com"); var jsSelectAllAnchors = @"Array.from(document.querySelectorAll('a')).map(a => a.href);"; var urls = await page.EvaluateExpressionAsync <string[]>(jsSelectAllAnchors); foreach (string url in urls) { Console.WriteLine($"Url: {url}"); } Console.WriteLine("Press any key to continue..."); Console.ReadLine(); } return; }
static async Task <Browser> FetchAndLaunchBrowser() { var browserFetcher = new BrowserFetcher(); var revision = await browserFetcher.DownloadAsync(); var launchOptions = new LaunchOptions { ExecutablePath = revision.ExecutablePath }; return(await Puppeteer.LaunchAsync(launchOptions)); }
private async Task <string> DownloadChrome(string path) { BrowserFetcherOptions options = new BrowserFetcherOptions() { Path = path, }; BrowserFetcher fetcher = new BrowserFetcher(options); RevisionInfo info = await fetcher.DownloadAsync(BrowserFetcher.DefaultRevision); return(info.ExecutablePath); }
protected override void Load(ContainerBuilder builder) { //Download browser required to scrape //DOM generated from JS will be available var result = new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision).Result; builder.RegisterType <GoogleSearchScraperStrategy>().As <IScrapeRequestStrategy>(); builder.RegisterType <BingSearchScraperStrategy>().As <IScrapeRequestStrategy>(); builder.RegisterType <TimedScrapeRequestHandler>().As <IScrapeHandler <TimedScrapeRequest> >(); builder.RegisterType <ScrapeRequestHandler>().As <IScrapeHandler <ScrapeRequest> >(); base.Load(builder); }
private async Task SetupAsync() { var downloaderTask = new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision); Server = SimpleServer.Create(TestConstants.Port, TestUtils.FindParentDirectory("PuppeteerSharp.TestServer")); HttpsServer = SimpleServer.CreateHttps(TestConstants.HttpsPort, TestUtils.FindParentDirectory("PuppeteerSharp.TestServer")); var serverStart = Server.StartAsync(); var httpsServerStart = HttpsServer.StartAsync(); await Task.WhenAll(downloaderTask, serverStart, httpsServerStart); }
private async Task SetupAsync() { using var browserFetcher = new BrowserFetcher(TestConstants.IsChrome ? Product.Chrome : Product.Firefox); var downloaderTask = browserFetcher.DownloadAsync(); Server = SimpleServer.Create(TestConstants.Port, TestUtils.FindParentDirectory("PuppeteerSharp.TestServer")); HttpsServer = SimpleServer.CreateHttps(TestConstants.HttpsPort, TestUtils.FindParentDirectory("PuppeteerSharp.TestServer")); var serverStart = Server.StartAsync(); var httpsServerStart = HttpsServer.StartAsync(); await Task.WhenAll(downloaderTask, serverStart, httpsServerStart); }