Exemple #1
0
 private void InitializeEnvironment()
 {
     // var executablePath = PuppeteerSharp.Launcher.GetExecutablePath();
     UpdateLog($"正在检测浏览器环境");
     try
     {
         var bf = new BrowserFetcher()
         {
             //WebProxy = new WebProxy("http://127.0.0.1:10809")
         };
         bf.DownloadProgressChanged += (s, e) =>
         {
             UpdateLog($"正在下载 {BrowserFetcher.DefaultRevision},已完成 {e.ProgressPercentage} %");
         };
         var revisionInfo = bf.DownloadAsync(BrowserFetcher.DefaultRevision).GetAwaiter().GetResult();
         BuildBrowser();
         //var page = _browser.NewPageAsync().GetAwaiter().GetResult();
         UpdateLog($"环境初始化完成,Downloaded={revisionInfo.Downloaded} ExecutablePath={revisionInfo.ExecutablePath} FolderPath={revisionInfo.FolderPath} {revisionInfo.Local} Platform={revisionInfo.Platform} Revision={revisionInfo.Revision}");
     }
     catch (Exception ex)
     {
         XtraMessageBox.Show(this, $"初始化环境出错,{ex.Message}");
         Environment.Exit(-1);
     }
 }
Exemple #2
0
        private static void DownloadBrowser()
        {
            var browserFetcher = new BrowserFetcher();

            browserFetcher.DownloadProgressChanged += (sender, e) =>
            {
                Logger.Debug(
                    $"Browser download progress {e.ProgressPercentage}% - {e.BytesReceived}/{e.TotalBytesToReceive}bytes");
            };

            Logger.Debug("Download browser");
            AsyncHelper.RunSync(() => browserFetcher.DownloadAsync(BrowserFetcher.DefaultRevision));

            if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
            {
                var       executablePath = browserFetcher.GetExecutablePath(BrowserFetcher.DefaultRevision);
                const int _0755          =
                    S_IRUSR | S_IXUSR | S_IWUSR
                    | S_IRGRP | S_IXGRP
                    | S_IROTH | S_IXOTH;
                chmod(executablePath, _0755);
            }

            Logger.Debug("Browser downloaded");
        }
Exemple #3
0
        public static async Task <LaunchOptions> StartPuppeteer()
        {
            var cctPth = Path.Join(".\\WaveExtension", "3.1.3_0");

            cctPth = Path.GetFullPath(cctPth);
            LaunchOptions chromeOpts = new LaunchOptions {
                Headless = false,
                Args     = new string[]
                {
                    $"--load-extension={cctPth}",
                    $"--disable-extensions-except={cctPth}",
                    "--disable-web-security",
                    "--disable-features=IsolateOrigins,site-per-process",
                    "--disable-features=IsolateOrigins",
                    "--disable-site-isolation-trials",
                    "--disable-oor-cors",
                    "--unsafely-treat-insecure-origin-as-secure=chrome-extension://jbbplnpkjmmeebjpijfedlgcdilocofh/sidebar.html"
                }
            };

            AppConfig.WriteOut(">> downloading chrome if not found");
            var dlTask = new BrowserFetcher().DownloadAsync();

            while (dlTask.IsCompleted != true)
            {
                if (!AppConfig.silent)
                {
                    AppConfig.ConsoleSpinner();
                }
            }
            AppConfig.WriteOut(">> done");
            //await using var browser = await Puppeteer.LaunchAsync(chromeOpts);
            return(chromeOpts);
        }
        public async Task ShouldCloseTheBrowserWhenTheConnectedProcessCloses()
        {
            var browserClosedTaskWrapper = new TaskCompletionSource <bool>();

            using var browserFetcher = new BrowserFetcher(Product.Chrome);
            var ChromiumLauncher = new ChromiumLauncher(
                (await browserFetcher.GetRevisionInfoAsync()).ExecutablePath,
                new LaunchOptions {
                Headless = true
            });

            await ChromiumLauncher.StartAsync().ConfigureAwait(false);

            var browser = await Puppeteer.ConnectAsync(new ConnectOptions
            {
                BrowserWSEndpoint = ChromiumLauncher.EndPoint
            });

            browser.Disconnected += (_, _) =>
            {
                browserClosedTaskWrapper.SetResult(true);
            };

            KillProcess(ChromiumLauncher.Process.Id);

            await browserClosedTaskWrapper.Task;

            Assert.True(browser.IsClosed);
        }
Exemple #5
0
        public async Task <byte[]> ConvertHtmlToPdf(string html, string cssPath)
        {
            var pdfOptions = new PdfOptions();

            var browserFetcher = new BrowserFetcher(new BrowserFetcherOptions
            {
                Path = _chromiumPath
            });

            await browserFetcher.DownloadAsync(BrowserFetcher.DefaultRevision);

            using var browser = await Puppeteer.LaunchAsync(new LaunchOptions
            {
                ExecutablePath = browserFetcher.RevisionInfo(BrowserFetcher.DefaultRevision).ExecutablePath,
                Headless       = true
            });

            var page = await browser.NewPageAsync();

            await page.SetContentAsync(html);

            await page.AddStyleTagAsync(new AddTagOptions { Path = cssPath });

            var stream = await page.PdfStreamAsync(pdfOptions);

            using var ms = new MemoryStream();
            await stream.CopyToAsync(ms);

            return(ms.ToArray());
        }
Exemple #6
0
        public static IApplicationBuilder UseBrowserFetcher(this IApplicationBuilder app)
        {
            BrowserFetcher browserFetcher = new BrowserFetcher();

            browserFetcher.DownloadAsync(BrowserFetcher.DefaultRevision).GetAwaiter().GetResult();

            if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
            {
                var path = Launcher.GetExecutablePath();
                Bash($"chmod 777 {path}");
            }
            return(app);

            void Bash(string cmd)
            {
                var escapedArgs = cmd.Replace("\"", "\\\"");

                var process = new Process()
                {
                    StartInfo = new ProcessStartInfo
                    {
                        FileName  = "/bin/bash",
                        Arguments = $"-c \"{escapedArgs}\"",
                        RedirectStandardOutput = true,
                        UseShellExecute        = false,
                        CreateNoWindow         = true,
                    }
                };

                process.Start();
                string result = process.StandardOutput.ReadToEnd();

                process.WaitForExit();
            }
        }
Exemple #7
0
        private static async Task <FileInfo> DownloadAsync(DirectoryInfo directory, int revision)
        {
            await _semaphoreSlim.WaitAsync();

            if (!directory.Exists)
            {
                directory.Create();
            }
            var fetchOpt = new BrowserFetcherOptions()
            {
                Host = Environment.GetEnvironmentVariable("PUPPETEER_CHROMIUM_DOWNLOADHOST"),
                Path = directory.FullName
            };
            var fetcher = new BrowserFetcher(fetchOpt);

            if (revision < 1)
            {
                string revisionStr = Environment.GetEnvironmentVariable("PUPPETEER_CHROMIUM_REVISION");
                if (!int.TryParse(revisionStr, out revision) || revision < 1)
                {
                    revision = BrowserFetcher.DefaultRevision;
                }
            }
            var info = await fetcher.DownloadAsync(revision).ConfigureAwait(false);

            _semaphoreSlim.Release();
            if (!string.IsNullOrWhiteSpace(info.ExecutablePath))
            {
                return(new FileInfo(info.ExecutablePath));
            }
            TryGetChromeExePath(directory, out var exeFile);
            return(exeFile);
        }
Exemple #8
0
        public async Task GetActivity(string mssv)
        {
            var fetcher = new BrowserFetcher();

            if (await fetcher.CanDownloadAsync(BrowserFetcher.DefaultRevision))
            {
                await fetcher.DownloadAsync(BrowserFetcher.DefaultRevision);
            }

            var browser = await Puppeteer.LaunchAsync(new LaunchOptions
            {
                Headless = true
            });

            var page = await browser.NewPageAsync();

            await page.GoToAsync("http://ctct.ueh.edu.vn/modules.php?name=xemdiemmacle", WaitUntilNavigation.DOMContentLoaded);

            await page.TypeAsync("input[id='madoi']", mssv);

            await page.Keyboard.PressAsync("Enter");

            await page.WaitForNavigationAsync();

            string res = await page.GetContentAsync();

            Debug.WriteLine(res);

            System.IO.File.WriteAllText(@"D:\demo.txt", res);

            await page.CloseAsync();

            await browser.CloseAsync();
        }
Exemple #9
0
        public async Task Usage()
        {
            var outputFile = Path.Combine(BaseDirectory, "Usage.pdf");
            var fileInfo   = new FileInfo(outputFile);

            if (fileInfo.Exists)
            {
                fileInfo.Delete();
            }

            #region PdfAsync

            using var browserFetcher = new BrowserFetcher();
            await browserFetcher.DownloadAsync();

            await using var browser = await Puppeteer.LaunchAsync(new LaunchOptions { Headless = true });

            await using var page = await browser.NewPageAsync();

            await page.GoToAsync("http://www.google.com");                    // In case of fonts being loaded from a CDN, use WaitUntilNavigation.Networkidle0 as a second param.

            await page.EvaluateExpressionHandleAsync("document.fonts.ready"); // Wait for fonts to be loaded. Omitting this might result in no text rendered in pdf.

            await page.PdfAsync(outputFile);

            #endregion

            Assert.True(File.Exists(outputFile));
        }
Exemple #10
0
        static PicGen()
        {
            var downloadsFolder = Path.GetTempPath();
            var option          = new BrowserFetcherOptions
            {
                Path = downloadsFolder
            };
            var fetcher = new BrowserFetcher(option);

            fetcher.DownloadAsync(BrowserFetcher.DefaultRevision).Wait();
            var browser = Puppeteer.LaunchAsync(new LaunchOptions
            {
                ExecutablePath = fetcher.RevisionInfo(BrowserFetcher.DefaultRevision).ExecutablePath,
                Headless       = true,
                Args           = new[]
                {
                    !isProd ? "--proxy-server=winip:1080" : ""
                }
            }).Result;

            page = browser.NewPageAsync().Result;
            page.SetViewportAsync(new ViewPortOptions
            {
                Height = 800,
                Width  = 1300
            }).Wait();
        }
Exemple #11
0
        public static async Task Main(string[] args)
        {
            var options = new LaunchOptions
            {
                Headless = true
            };

            Console.WriteLine("Downloading chromium");
            using var browserFetcher = new BrowserFetcher();
            await browserFetcher.DownloadAsync();

            Console.WriteLine("Navigating google");
            using (var browser = await Puppeteer.LaunchAsync(options))
                using (var page = await browser.NewPageAsync())
                {
                    await page.GoToAsync("http://www.google.com");

                    Console.WriteLine("Generating PDF");
                    await page.PdfAsync(Path.Combine(Directory.GetCurrentDirectory(), "google.pdf"));

                    Console.WriteLine("Export completed");

                    if (!args.Any(arg => arg == "auto-exit"))
                    {
                        Console.ReadLine();
                    }
                }
        }
Exemple #12
0
        public async Task Should_render_properly(string name, string url)
        {
            using (var browserFetcher = new BrowserFetcher())
            {
                await browserFetcher.DownloadAsync();
            }

            await using var browser = await Puppeteer.LaunchAsync(new LaunchOptions
            {
                Headless        = true,
                DefaultViewport = new ViewPortOptions
                {
                    Height      = 800,
                    IsLandscape = true,
                    IsMobile    = false,
                    Width       = 1000
                },
                Args = new string[]
                {
                    "--no-sandbox"
                }
            });

            await using var page = await browser.NewPageAsync();

            await page.GoToAsync(_.ClientManager.Options.Url + url + "?skip-setup");

            await page.ScreenshotAsync($"__{name}.jpg");

            var diff = ImageSharpCompare.CalcDiff($"__{name}.jpg", $"Assets/{name}.jpg");

            Assert.InRange(diff.MeanError, 0, 10);
        }
Exemple #13
0
        private static async Task RunPuppeteer(int externalSpoofingEnginePort)
        {
            var _ = new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision).Result;

            var browser = await Puppeteer.LaunchAsync(new LaunchOptions
            {
                Headless = false,
                Args     = new[]
                {
                    ("--proxy-server=http://127.0.0.1:" + externalSpoofingEnginePort)
                }
            });

            try
            {
                var page = await browser.NewPageAsync();

                await page.GoToAsync("https://help.kameleo.io");

                Thread.Sleep(10000);
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex);
            }
            finally
            {
                if (browser != null)
                {
                    await browser.CloseAsync();
                }
            }
        }
        public static bool InitializePuppeteer()
        {
            if (!InitializingPuppeteer)
            {
                if (DefaultBrowser != null && DefaultBrowser.IsClosed)
                {
                    DefaultBrowser.Dispose();
                    DefaultBrowser = null;
                }

                if (DefaultBrowser == null)
                {
                    InitializingPuppeteer = true;

                    try
                    {
                        MainWindow.mainWindowCM.SetStatusBar(MainWindowComponentModel.StatusBarEnum.PreparingChromium);

                        RevisionInfo _revisionInfo = new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision).Result;

                        if (_revisionInfo.Downloaded)
                        {
                            MainWindow.mainWindowCM.SetStatusBar(MainWindowComponentModel.StatusBarEnum.Launching);

                            DefaultBrowser = Puppeteer.LaunchAsync(new LaunchOptions
                            {
                                Args = new string[] { "--disable-blink-features", "--disable-blink-features=AutomationControlled" }, IgnoredDefaultArgs = new string[] { "--enable-automation", "--headless" },
                                IgnoreHTTPSErrors = true,
                                UserDataDir       = "./tmp",
                            }).Result;


                            IList <Page> _p = DefaultBrowser.PagesAsync().Result;
                            CurrentPage = _p[0];

                            if (File.Exists("./cookies.json"))
                            {
                                // If they exist, load cookies
                                string _cookies = File.ReadAllText("./cookies.json");
                                CurrentPage.SetCookieAsync(JsonConvert.DeserializeObject <CookieParam[]>(_cookies));
                            }

                            MainWindow.mainWindowCM.SetStatusBar(MainWindowComponentModel.StatusBarEnum.Ready);
                            return(true);
                        }
                        else
                        {
                            MainWindow.mainWindowCM.SetStatusBar(MainWindowComponentModel.StatusBarEnum.Error);
                            return(false);
                        }
                    }
                    catch {
                        MainWindow.mainWindowCM.SetStatusBar(MainWindowComponentModel.StatusBarEnum.Error);
                    }

                    InitializingPuppeteer = false;
                }
            }
            return(true);
        }
        public async Task <Browser> BuildBrowser()
        {
            var downloadPath = Path.Combine(Path.GetTempPath(), "puppeteer");

            var browserFetcherOptions = new BrowserFetcherOptions {
                Path = downloadPath
            };
            var browserFetcher = new BrowserFetcher(browserFetcherOptions);
            await browserFetcher.DownloadAsync(BrowserFetcher.DefaultChromiumRevision);

            var executableDirectory = Directory.EnumerateDirectories(
                Directory.EnumerateDirectories(downloadPath)
                .First())
                                      .First();

            var executableFilename = RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
                                    ? "chrome.exe"
                                    : "chrome";

            var executablePath = Path.Combine(executableDirectory, executableFilename);

            var options = new LaunchOptions {
                Headless = true, ExecutablePath = executablePath
            };

            var browser = await Puppeteer.LaunchAsync(options);

            return(browser);
        }
        public async Task Usage()
        {
            var outputFile = Path.Combine(BaseDirectory, "Usage.pdf");
            var fileInfo   = new FileInfo(outputFile);

            if (fileInfo.Exists)
            {
                fileInfo.Delete();
            }

            #region PdfAsync

            var browserFetcher = new BrowserFetcher();
            await browserFetcher.DownloadAsync();

            await using var browser = await Puppeteer.LaunchAsync(new LaunchOptions { Headless = true });

            await using var page = await browser.NewPageAsync();

            await page.GoToAsync("http://www.google.com");

            await page.PdfAsync(outputFile);

            #endregion

            Assert.True(File.Exists(outputFile));
        }
Exemple #17
0
        private async void FormLoad()
        {
            var smsServiceItems = SmsServiceItem.GetSmsServiceItems();

            cmbSmsService.DataSource    = smsServiceItems;
            cmbSmsService.DisplayMember = "Text";
            cmbSmsService.SelectedIndex = 0;

            var browserProfileItems = BrowserProfileItem.GetItems(_browserProfileService);

            cmbBrowserProfile.DataSource    = browserProfileItems;
            cmbBrowserProfile.DisplayMember = "Text";
            cmbBrowserProfile.SelectedIndex = 0;

            var browserFetcher = new BrowserFetcher();

            browserFetcher.DownloadProgressChanged += OnDownloadProgressChanged;
            GetBrowserLastVersion(browserFetcher);
            //init ServiceInfoList
            await _smsServices.GetServiceInfoList(ServiceCode.MailRu);

            foreach (SmsServiceCode smsServiceCode in Enum.GetValues(typeof(SmsServiceCode)))
            {
                var balance = await _smsServices.GetSmsService(smsServiceCode).GetBalance();

                if (balance < 5)
                {
                    textBox1.AppendText($@"Low balance {smsServiceCode} {balance} - {DateTime.Now} {Environment.NewLine}");
                    _smsServices.RemoveSmsServiceLowBalance(smsServiceCode);
                }
            }
        }
        public async Task <int> ScrapeAndAddProducts()
        {
            int result = 0;

            var browserFetcher = new BrowserFetcher();
            await browserFetcher.DownloadAsync();

            await using var browser = await Puppeteer.LaunchAsync(
                            new LaunchOptions { Headless = false, DefaultViewport = new ViewPortOptions { Width = 1920, Height = 1024 } });

            foreach (var categoryUrl in _categoryUrlsToScrape)
            {
                await using var page = await browser.NewPageAsync();

                await page.GoToAsync(categoryUrl);
                await ScrollUntilEnd(page);

                var products = await ExtractProducts(page);
                await AddProductsIfMissing(products);

                result += products.Count;
            }

            return(result);
        }
Exemple #19
0
        /// <summary>
        /// Load the browser fetcher from file
        /// </summary>
        /// <returns></returns>
        protected virtual BrowserFetcher GetBrowserFetcher()
        {
            if (BrowserFetcher != null)
            {
                return(BrowserFetcher);
            }

            var downloadPath = _settingRepository.GetSettingValue <string>("SmartScrapper.Chromium.DownloadPath");

            _loggingService.Log(LogEventLevel.Information, $"Attempting to set up puppeteer to use Chromium found under directory {downloadPath}");

            if (!Directory.Exists(downloadPath))
            {
                _loggingService.Log(LogEventLevel.Information, "Custom directory not found. Creating directory");
                Directory.CreateDirectory(downloadPath);
            }

            _loggingService.Log(LogEventLevel.Information, "Downloading Chromium");

            var browserFetcherOptions = new BrowserFetcherOptions {
                Path = downloadPath
            };

            BrowserFetcher = new BrowserFetcher(browserFetcherOptions);

            return(BrowserFetcher);
        }
Exemple #20
0
        private void Form1_Load(object sender, EventArgs e)
        {
            toolStripProgressBar1.Maximum = 100;
            toolStripProgressBar1.Value   = 0;
            Task.Run(async() =>
            {
                var browserFetcher = new BrowserFetcher(new BrowserFetcherOptions()
                {
                    Host = "http://cdn.npm.taobao.org/dist"
                });

                try
                {
                    var cp = Process.GetProcessesByName("chrome");
                    if (cp.Length > 0)
                    {
                        foreach (var item in cp)
                        {
                            if (!item.HasExited && item.MainModule.FileName == browserFetcher.GetExecutablePath(BrowserFetcher.DefaultRevision))
                            {
                                item.Kill();
                            }
                        }
                    }
                }
                catch (Exception)
                {
                }
                browserFetcher.DownloadProgressChanged += Bf_DownloadProgressChanged;
                await browserFetcher.DownloadAsync(BrowserFetcher.DefaultRevision);
            });
        }
        static async Task Browser(bool headless)
        {
            Console.WriteLine("Loading chromium");
            var fetcher = new BrowserFetcher();
            await fetcher.DownloadAsync(BrowserFetcher.DefaultRevision);

            using var browser = await Puppeteer.LaunchAsync(new LaunchOptions
            {
                UserDataDir     = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ApplicationData), "Puppeteer"),
                Headless        = headless,
                DefaultViewport = new ViewPortOptions { IsMobile = false, HasTouch = false, IsLandscape = true, Width = 1600, Height = 800 },
            });

            Console.WriteLine("Loading codesignal.com");
            var pages = await browser.PagesAsync();

            var page = pages.Length > 0 ? pages[0] : await browser.NewPageAsync();

            var areas = (await Scraping.ScanAreas(page));
            var tasks = (await Scraping.ScanTasks(page, areas));

            Console.WriteLine($"Located {tasks.Count} tasks where {tasks.Count(t => t.Solved)} is solved");
            Console.WriteLine("Please note that only solved tasks and first new task and unlocked tasks are found!");

            tasks = Source.FilterTasks(tasks).ToList();
            Console.WriteLine($"Retrieving {tasks.Count} task to update source and tests.");

            int failed = 0;

            //            var tasks = new TaskInfo[] { new TaskInfo("https://app.codesignal.com/arcade/code-arcade", "The Core", "Cliffs Of Pain", "https://app.codesignal.com/arcade/code-arcade/cliffs-of-pain/iGBDQE3KjqbYyF8DH", "timeASCIIRepresentation", false) };
            foreach (var task in tasks)
            {
                await Scraping.RetrieveTask(page, task);

                if (task.Failed)
                {
                    failed++;
                    continue;
                }
                Source.WriteTask(task);
                task.Description = null;
                task.Source      = null;
                task.Tests       = null;
            }

            Console.WriteLine("Finished, press space to end the program.");
            if (failed != 0)
            {
                Console.WriteLine();
                Console.WriteLine($"However, failed to import {failed} tasks.");
                Console.WriteLine($"Please, rerun to try again.");
            }

            Console.ReadKey();
            await page.CloseAsync();

            await browser.CloseAsync();
        }
        public static async Task DownloadRevision(RevisionInfo revision)
        {
            if (Directory.Exists(revision.FolderPath) && !IsValidRevision(revision))
            {
                Directory.Delete(revision.FolderPath, true);
            }

            await BrowserFetcher.DownloadAsync(revision.Revision);
        }
        public static IServiceCollection AddPuppeteerSharpPdfConverter(this IServiceCollection services)
        {
            var browserFetcher = new BrowserFetcher();

            browserFetcher.DownloadAsync().GetAwaiter().GetResult();

            services.AddSingleton <IPdfConverter, PuppeteerSharpConverter>();

            return(services);
        }
Exemple #24
0
        private static async Task DownloadBrowserAsync()
        {
            var browserFetcher = new BrowserFetcher(new BrowserFetcherOptions()
            {
                Host = "https://npm.taobao.org/mirrors",
            });

            browserFetcher.DownloadProgressChanged += new MyHandler().DownloadProgressChangedHandler;
            await browserFetcher.DownloadAsync(BrowserFetcher.DefaultRevision);
        }
        public static async Task Main(string[] args)
        {
            Console.WriteLine("This example downloads the default version of Chromium to a custom location");

            var currentDirectory = Directory.GetCurrentDirectory();
            var downloadPath     = Path.Combine(currentDirectory, "..", "..", "CustomChromium");

            Console.WriteLine($"Attemping to set up puppeteer to use Chromium found under directory {downloadPath} ");

            if (!Directory.Exists(downloadPath))
            {
                Console.WriteLine("Custom directory not found. Creating directory");
                Directory.CreateDirectory(downloadPath);
            }

            Console.WriteLine("Downloading Chromium");

            var browserFetcherOptions = new BrowserFetcherOptions {
                Path = downloadPath
            };
            var browserFetcher = new BrowserFetcher(browserFetcherOptions);
            await browserFetcher.DownloadAsync(BrowserFetcher.DefaultRevision);

            var executablePath = browserFetcher.GetExecutablePath(BrowserFetcher.DefaultRevision);

            if (string.IsNullOrEmpty(executablePath))
            {
                Console.WriteLine("Custom Chromium location is empty. Unable to start Chromium. Exiting.\n Press any key to continue");
                Console.ReadLine();
                return;
            }

            Console.WriteLine($"Attemping to start Chromium using executable path: {executablePath}");

            var options = new LaunchOptions {
                Headless = true, ExecutablePath = executablePath
            };

            using (var browser = await Puppeteer.LaunchAsync(options))
                using (var page = await browser.NewPageAsync())
                {
                    await page.GoToAsync("http://www.google.com");

                    var jsSelectAllAnchors = @"Array.from(document.querySelectorAll('a')).map(a => a.href);";
                    var urls = await page.EvaluateExpressionAsync <string[]>(jsSelectAllAnchors);

                    foreach (string url in urls)
                    {
                        Console.WriteLine($"Url: {url}");
                    }
                    Console.WriteLine("Press any key to continue...");
                    Console.ReadLine();
                }
            return;
        }
Exemple #26
0
        static async Task <Browser> FetchAndLaunchBrowser()
        {
            var browserFetcher = new BrowserFetcher();
            var revision       = await browserFetcher.DownloadAsync();

            var launchOptions = new LaunchOptions {
                ExecutablePath = revision.ExecutablePath
            };

            return(await Puppeteer.LaunchAsync(launchOptions));
        }
        private async Task <string> DownloadChrome(string path)
        {
            BrowserFetcherOptions options = new BrowserFetcherOptions()
            {
                Path = path,
            };
            BrowserFetcher fetcher = new BrowserFetcher(options);
            RevisionInfo   info    = await fetcher.DownloadAsync(BrowserFetcher.DefaultRevision);

            return(info.ExecutablePath);
        }
        protected override void Load(ContainerBuilder builder)
        {
            //Download browser required to scrape
            //DOM generated from JS will be available
            var result = new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision).Result;

            builder.RegisterType <GoogleSearchScraperStrategy>().As <IScrapeRequestStrategy>();
            builder.RegisterType <BingSearchScraperStrategy>().As <IScrapeRequestStrategy>();
            builder.RegisterType <TimedScrapeRequestHandler>().As <IScrapeHandler <TimedScrapeRequest> >();
            builder.RegisterType <ScrapeRequestHandler>().As <IScrapeHandler <ScrapeRequest> >();
            base.Load(builder);
        }
Exemple #29
0
        private async Task SetupAsync()
        {
            var downloaderTask = new BrowserFetcher().DownloadAsync(BrowserFetcher.DefaultRevision);

            Server      = SimpleServer.Create(TestConstants.Port, TestUtils.FindParentDirectory("PuppeteerSharp.TestServer"));
            HttpsServer = SimpleServer.CreateHttps(TestConstants.HttpsPort, TestUtils.FindParentDirectory("PuppeteerSharp.TestServer"));

            var serverStart      = Server.StartAsync();
            var httpsServerStart = HttpsServer.StartAsync();

            await Task.WhenAll(downloaderTask, serverStart, httpsServerStart);
        }
Exemple #30
0
        private async Task SetupAsync()
        {
            using var browserFetcher = new BrowserFetcher(TestConstants.IsChrome ? Product.Chrome : Product.Firefox);
            var downloaderTask = browserFetcher.DownloadAsync();

            Server      = SimpleServer.Create(TestConstants.Port, TestUtils.FindParentDirectory("PuppeteerSharp.TestServer"));
            HttpsServer = SimpleServer.CreateHttps(TestConstants.HttpsPort, TestUtils.FindParentDirectory("PuppeteerSharp.TestServer"));

            var serverStart      = Server.StartAsync();
            var httpsServerStart = HttpsServer.StartAsync();

            await Task.WhenAll(downloaderTask, serverStart, httpsServerStart);
        }