public async Task <CaptureResponseEventArgs> CapturePage(object sender, UserRequestEventArgs e) { var ret = new CaptureResponseEventArgs() { Request = e, Url = e.Url, Title = "", }; var result = Uri.TryCreate(e.Url, UriKind.Absolute, out var uriResult); if (!result) { ret.StatusText += $"Not a valid URL: \"{e.Url}\""; return(ret); } var attachments = new List <string>(); try { var wc = new WebClientWithTimeout(Timeout); var expectedFileName = Path.ConcentrateFilename(uriResult.Segments.Last(), e.Id.ToString()); await wc.DownloadFileTaskAsync(e.Url, expectedFileName); attachments.Append(expectedFileName); } catch (WebException we) { Logger.Warn("Something happened.\n" + we); ret.StatusText += we; } ret.Attachments = attachments.ToArray(); return(ret); }
private async void QueueUserRequest(object sender, UserRequestEventArgs e) { var s = sender as IConnector; if (s == null) { Logger.Warn("sender is not a IConnector, ignoring"); return; } e.Requester = s; // add this request to the request queue _requestQueue.Put(e, e.IsPriority); // if user configured aggressive GC, perform it now if (Globals.GlobalConfig.AggressiveGc) { Logger.Debug("GC requested"); await Task.Run(() => { GC.Collect(); }); } RuntimeInformation.QueuedRequests += 1; // notify for new request ProcessRequests(); }
public async Task <CaptureResponseEventArgs> CapturePage(object sender, UserRequestEventArgs e) { Logger.Debug($"Enter CapturePage() for session {e.Id}"); var ret = new CaptureResponseEventArgs { Request = e, StatusText = "", Attachments = null, HasPotentialUnfinishedDownloads = true }; var currentSessionBrowser = await NewBrowser(); // create a new session Logger.Debug("Create incognito session"); var context = await currentSessionBrowser.CreateIncognitoBrowserContextAsync(); Logger.Debug("Create new tab"); var page = await context.NewPageAsync(); await page.SetViewportAsync(new ViewPortOptions { Width = WindowWidth, Height = WindowHeight }); // load document Logger.Debug($"Navigate to \"{e.Url}\""); try { await page.GoToAsync( e.Url, PageDownloadTimeout, new[] { WaitUntilNavigation.Networkidle0 } ); } catch (WaitTaskTimeoutException) { // TODO: time exceeded Logger.Warn($"Page loading time exceeded for url \"{e.Url}\""); ret.StatusText += $"Page loading time exceeded for url \"{e.Url}\"\n"; ret.HasPotentialUnfinishedDownloads = false; } catch (NavigationException) { Logger.Warn($"Document download time exceeded for url \"{e.Url}\""); ret.StatusText += $"Document download time exceeded for url \"{e.Url}\"\n"; ret.HasPotentialUnfinishedDownloads = false; } await page.WaitForTimeoutAsync(DocumentLoadDelay); ret.Url = page.Url; ret.Title = await page.GetTitleAsync(); var prefix = Path.Escape(ret.Title.Substring(0, Math.Min(MaxTitlePrependLength, ret.Title.Length)) + "-" + e.Id); // scroll through the page to load any lazy-loading elements Logger.Debug("Trying to load lazy-loading elements"); try { // https://www.screenshotbin.com/blog/handling-lazy-loaded-webpages-puppeteer var bodyHandle = await page.QuerySelectorAsync("body"); var boundingBox = await bodyHandle.BoundingBoxAsync(); var viewportHeight = page.Viewport.Height; var viewportIncr = 0; // scroll down while (viewportIncr + viewportHeight < boundingBox.Height) { await page.EvaluateExpressionAsync($"window.scrollBy(0, {viewportHeight})"); await page.WaitForTimeoutAsync(PageScrollActionWaitDelay); viewportIncr += viewportHeight; } // scroll up await page.EvaluateExpressionAsync("window.scrollTo(0, 0)"); await page.WaitForTimeoutAsync(ExtraDownloadWaitDelay); } catch (NullReferenceException) { // body cannot be downloaded Logger.Error("Body is missing, either your URL contains a redirection or there is a network issue"); ret.StatusText += "Failed to download web page\n"; ret.HasPotentialUnfinishedDownloads = true; return(ret); } var attachments = new List <string>(); if (e.RequestTypes.Contains(UserRequestType.Pdf)) { // capture PDF try { Logger.Debug("Saving PDF"); await page.PdfAsync($"{prefix}.pdf", new PdfOptions { // TODO: header and footer is not in the correct position, ignore for now // HeaderTemplate = "<title /> - <date />", // FooterTemplate = // "<url /> - Captured by ScreenShooter - https://github.com/Jamesits/ScreenShooter - <pageNumber />/<totalPages />", DisplayHeaderFooter = true, PrintBackground = true, Format = PaperFormat.A4, MarginOptions = new MarginOptions { Bottom = "0.5in", Top = "0.5in", Left = "0.3in", Right = "0.3in" } }); attachments.Add($"{prefix}.pdf"); } catch (TargetClosedException ex) { // possibility out of memory, see https://github.com/Jamesits/ScreenShooter/issues/1 ret.StatusText += "Possible out of memory when requesting PDF\n"; Logger.Error($"Something happened on requesting PDF. \n\nException:\n{ex}\n\nInnerException:{ex.InnerException}"); _requireNewBrowserInstance = true; } } if (e.RequestTypes.Contains(UserRequestType.Png)) { // capture screenshot try { Logger.Debug("Taking screenshot"); await page.ScreenshotAsync($"{prefix}.png", new ScreenshotOptions { FullPage = true }); attachments.Add($"{prefix}.png"); } catch (TargetClosedException ex) { // possibility out of memory, see https://github.com/Jamesits/ScreenShooter/issues/1 ret.StatusText += "Possible out of memory when requesting screenshot\n"; Logger.Error($"Something happened on requesting screenshot. \n\nException:\n{ex}\n\nInnerException:{ex.InnerException}"); _requireNewBrowserInstance = true; } } ret.Attachments = attachments.ToArray(); // clean up try { Logger.Debug("Close tab"); await page.CloseAsync(); Logger.Debug("Kill context"); await context.CloseAsync(); Logger.Debug("Closing browser"); await currentSessionBrowser.CloseAsync(); } catch (Exception ex) { Logger.Error($"Something happened on cleaning up. \n\nException:\n{ex}\n\nInnerException:{ex.InnerException}"); } Logger.Debug("Exit CapturePage()"); return(ret); }
// ReSharper disable once UnusedMember.Global public async Task OnExecuteAsync() { Logger.Debug("Entering OnExecute()"); #region Set up basic events AppDomain.CurrentDomain.UnhandledException += CurrentDomainUnhandledException; AppDomain.CurrentDomain.ProcessExit += (sender, e) => { AsyncHelper.RunSync(CleanUp); }; Console.CancelKeyPress += (sender, e) => { Logger.Info("SIGINT received, cleaning up..."); AsyncHelper.RunSync(CleanUp); }; #endregion #region Global initialization _staticSelf = this; Globals.ProgramIdentifier = $"{Assembly.GetExecutingAssembly().GetName().Name} {Assembly.GetExecutingAssembly().GetName().Version}"; Console.Title = Globals.ProgramIdentifier; Logger.Info(Globals.ProgramIdentifier); if (ConfigPath != null) { _config = Toml.ReadFile(ConfigPath); } // parse config file Globals.GlobalConfig = _config.Get <GlobalConfig>("GlobalConfig"); #endregion #region Apply config // set up GC if (Globals.GlobalConfig.LowMemoryAddMemoryPressure > 0) { // Ask @hjc4869 why there is a *2 Logger.Debug($"Adding memory pressure {Globals.GlobalConfig.LowMemoryAddMemoryPressure * 2 / 1048576}MiB"); GC.AddMemoryPressure(Globals.GlobalConfig.LowMemoryAddMemoryPressure * 2); } #endregion if (ConfigPath != null && Address == null) { #region Entering daemon mode Logger.Debug("Entering daemon mode"); Logger.Trace("Enumerating actuators"); var actuators = _config.Get <TomlTable>("Actuator"); CreateObjects(actuators, "ScreenShooter.Actuator", _actuators); Logger.Trace("Enumerating Connectors"); var connectors = _config.Get <TomlTable>("Connector"); CreateObjects(connectors, "ScreenShooter.IO", _connectors); if (_connectors.Count == 0) { Logger.Fatal("No valid connector created, check your config"); Environment.Exit(-1); } if (_actuators.Count == 0) { Logger.Fatal("No valid actuators found, check your config"); Environment.Exit(-1); } Logger.Debug("Starting connectors"); foreach (var connector in _connectors) { await connector.CreateSession(); _connectorTasks.Add(connector.EventLoop()); connector.NewRequest += QueueUserRequest; } Logger.Info("Entered running state"); await Task.WhenAll(_connectorTasks); await CleanUp(); Logger.Debug("All connectors have quit, daemon quitting"); #endregion } else if (Address != null) { #region enter one-shot mode Logger.Debug("Entering one-shot mode"); var request = new UserRequestEventArgs { Url = Address, Requester = new NullConnector() }; var actuator = new HeadlessChromeActuator(); Logger.Debug("Capturing page"); var ret = await actuator.CapturePage(Address, request); Logger.Info(ret); #endregion } else { Logger.Fatal("Confused by the provided command line arguments"); Environment.Exit(-1); } Logger.Debug("Exiting OnExecute()"); }