public async Task <CaptureResponseEventArgs> CapturePage(object sender, UserRequestEventArgs e)
{
var ret = new CaptureResponseEventArgs()
{
Request = e,
Url = e.Url,
Title = "",
};
var result = Uri.TryCreate(e.Url, UriKind.Absolute, out var uriResult);
if (!result)
{
ret.StatusText += $"Not a valid URL: \"{e.Url}\"";
return(ret);
}
var attachments = new List <string>();
try
{
var wc = new WebClientWithTimeout(Timeout);
var expectedFileName = Path.ConcentrateFilename(uriResult.Segments.Last(), e.Id.ToString());
await wc.DownloadFileTaskAsync(e.Url, expectedFileName);
attachments.Append(expectedFileName);
}
catch (WebException we)
{
Logger.Warn("Something happened.\n" + we);
ret.StatusText += we;
}
ret.Attachments = attachments.ToArray();
return(ret);
}
private async void QueueUserRequest(object sender, UserRequestEventArgs e)
{
var s = sender as IConnector;
if (s == null)
{
Logger.Warn("sender is not a IConnector, ignoring");
return;
}
e.Requester = s;
// add this request to the request queue
_requestQueue.Put(e, e.IsPriority);
// if user configured aggressive GC, perform it now
if (Globals.GlobalConfig.AggressiveGc)
{
Logger.Debug("GC requested");
await Task.Run(() => { GC.Collect(); });
}
RuntimeInformation.QueuedRequests += 1;
// notify for new request
ProcessRequests();
}
public async Task <CaptureResponseEventArgs> CapturePage(object sender, UserRequestEventArgs e)
{
Logger.Debug($"Enter CapturePage() for session {e.Id}");
var ret = new CaptureResponseEventArgs
{
Request = e,
StatusText = "",
Attachments = null,
HasPotentialUnfinishedDownloads = true
};
var currentSessionBrowser = await NewBrowser();
// create a new session
Logger.Debug("Create incognito session");
var context = await currentSessionBrowser.CreateIncognitoBrowserContextAsync();
Logger.Debug("Create new tab");
var page = await context.NewPageAsync();
await page.SetViewportAsync(new ViewPortOptions
{
Width = WindowWidth,
Height = WindowHeight
});
// load document
Logger.Debug($"Navigate to \"{e.Url}\"");
try
{
await page.GoToAsync(
e.Url,
PageDownloadTimeout,
new[] { WaitUntilNavigation.Networkidle0 }
);
}
catch (WaitTaskTimeoutException)
{
// TODO: time exceeded
Logger.Warn($"Page loading time exceeded for url \"{e.Url}\"");
ret.StatusText += $"Page loading time exceeded for url \"{e.Url}\"\n";
ret.HasPotentialUnfinishedDownloads = false;
}
catch (NavigationException)
{
Logger.Warn($"Document download time exceeded for url \"{e.Url}\"");
ret.StatusText += $"Document download time exceeded for url \"{e.Url}\"\n";
ret.HasPotentialUnfinishedDownloads = false;
}
await page.WaitForTimeoutAsync(DocumentLoadDelay);
ret.Url = page.Url;
ret.Title = await page.GetTitleAsync();
var prefix =
Path.Escape(ret.Title.Substring(0, Math.Min(MaxTitlePrependLength, ret.Title.Length)) + "-" + e.Id);
// scroll through the page to load any lazy-loading elements
Logger.Debug("Trying to load lazy-loading elements");
try
{
// https://www.screenshotbin.com/blog/handling-lazy-loaded-webpages-puppeteer
var bodyHandle = await page.QuerySelectorAsync("body");
var boundingBox = await bodyHandle.BoundingBoxAsync();
var viewportHeight = page.Viewport.Height;
var viewportIncr = 0;
// scroll down
while (viewportIncr + viewportHeight < boundingBox.Height)
{
await page.EvaluateExpressionAsync($"window.scrollBy(0, {viewportHeight})");
await page.WaitForTimeoutAsync(PageScrollActionWaitDelay);
viewportIncr += viewportHeight;
}
// scroll up
await page.EvaluateExpressionAsync("window.scrollTo(0, 0)");
await page.WaitForTimeoutAsync(ExtraDownloadWaitDelay);
}
catch (NullReferenceException)
{
// body cannot be downloaded
Logger.Error("Body is missing, either your URL contains a redirection or there is a network issue");
ret.StatusText += "Failed to download web page\n";
ret.HasPotentialUnfinishedDownloads = true;
return(ret);
}
var attachments = new List <string>();
if (e.RequestTypes.Contains(UserRequestType.Pdf))
{ // capture PDF
try
{
Logger.Debug("Saving PDF");
await page.PdfAsync($"{prefix}.pdf", new PdfOptions
{
// TODO: header and footer is not in the correct position, ignore for now
// HeaderTemplate = "<title /> - <date />",
// FooterTemplate =
// "<url /> - Captured by ScreenShooter - https://github.com/Jamesits/ScreenShooter - <pageNumber />/<totalPages />",
DisplayHeaderFooter = true,
PrintBackground = true,
Format = PaperFormat.A4,
MarginOptions = new MarginOptions
{
Bottom = "0.5in",
Top = "0.5in",
Left = "0.3in",
Right = "0.3in"
}
});
attachments.Add($"{prefix}.pdf");
}
catch (TargetClosedException ex)
{
// possibility out of memory, see https://github.com/Jamesits/ScreenShooter/issues/1
ret.StatusText += "Possible out of memory when requesting PDF\n";
Logger.Error($"Something happened on requesting PDF. \n\nException:\n{ex}\n\nInnerException:{ex.InnerException}");
_requireNewBrowserInstance = true;
}
}
if (e.RequestTypes.Contains(UserRequestType.Png))
{ // capture screenshot
try
{
Logger.Debug("Taking screenshot");
await page.ScreenshotAsync($"{prefix}.png", new ScreenshotOptions
{
FullPage = true
});
attachments.Add($"{prefix}.png");
}
catch (TargetClosedException ex)
{
// possibility out of memory, see https://github.com/Jamesits/ScreenShooter/issues/1
ret.StatusText += "Possible out of memory when requesting screenshot\n";
Logger.Error($"Something happened on requesting screenshot. \n\nException:\n{ex}\n\nInnerException:{ex.InnerException}");
_requireNewBrowserInstance = true;
}
}
ret.Attachments = attachments.ToArray();
// clean up
try
{
Logger.Debug("Close tab");
await page.CloseAsync();
Logger.Debug("Kill context");
await context.CloseAsync();
Logger.Debug("Closing browser");
await currentSessionBrowser.CloseAsync();
}
catch (Exception ex)
{
Logger.Error($"Something happened on cleaning up. \n\nException:\n{ex}\n\nInnerException:{ex.InnerException}");
}
Logger.Debug("Exit CapturePage()");
return(ret);
}
// ReSharper disable once UnusedMember.Global
public async Task OnExecuteAsync()
{
Logger.Debug("Entering OnExecute()");
#region Set up basic events
AppDomain.CurrentDomain.UnhandledException += CurrentDomainUnhandledException;
AppDomain.CurrentDomain.ProcessExit += (sender, e) => { AsyncHelper.RunSync(CleanUp); };
Console.CancelKeyPress += (sender, e) =>
{
Logger.Info("SIGINT received, cleaning up...");
AsyncHelper.RunSync(CleanUp);
};
#endregion
#region Global initialization
_staticSelf = this;
Globals.ProgramIdentifier =
$"{Assembly.GetExecutingAssembly().GetName().Name} {Assembly.GetExecutingAssembly().GetName().Version}";
Console.Title = Globals.ProgramIdentifier;
Logger.Info(Globals.ProgramIdentifier);
if (ConfigPath != null)
{
_config = Toml.ReadFile(ConfigPath);
}
// parse config file
Globals.GlobalConfig = _config.Get <GlobalConfig>("GlobalConfig");
#endregion
#region Apply config
// set up GC
if (Globals.GlobalConfig.LowMemoryAddMemoryPressure > 0)
{
// Ask @hjc4869 why there is a *2
Logger.Debug($"Adding memory pressure {Globals.GlobalConfig.LowMemoryAddMemoryPressure * 2 / 1048576}MiB");
GC.AddMemoryPressure(Globals.GlobalConfig.LowMemoryAddMemoryPressure * 2);
}
#endregion
if (ConfigPath != null && Address == null)
{
#region Entering daemon mode
Logger.Debug("Entering daemon mode");
Logger.Trace("Enumerating actuators");
var actuators = _config.Get <TomlTable>("Actuator");
CreateObjects(actuators, "ScreenShooter.Actuator", _actuators);
Logger.Trace("Enumerating Connectors");
var connectors = _config.Get <TomlTable>("Connector");
CreateObjects(connectors, "ScreenShooter.IO", _connectors);
if (_connectors.Count == 0)
{
Logger.Fatal("No valid connector created, check your config");
Environment.Exit(-1);
}
if (_actuators.Count == 0)
{
Logger.Fatal("No valid actuators found, check your config");
Environment.Exit(-1);
}
Logger.Debug("Starting connectors");
foreach (var connector in _connectors)
{
await connector.CreateSession();
_connectorTasks.Add(connector.EventLoop());
connector.NewRequest += QueueUserRequest;
}
Logger.Info("Entered running state");
await Task.WhenAll(_connectorTasks);
await CleanUp();
Logger.Debug("All connectors have quit, daemon quitting");
#endregion
}
else if (Address != null)
{
#region enter one-shot mode
Logger.Debug("Entering one-shot mode");
var request = new UserRequestEventArgs
{
Url = Address,
Requester = new NullConnector()
};
var actuator = new HeadlessChromeActuator();
Logger.Debug("Capturing page");
var ret = await actuator.CapturePage(Address, request);
Logger.Info(ret);
#endregion
}
else
{
Logger.Fatal("Confused by the provided command line arguments");
Environment.Exit(-1);
}
Logger.Debug("Exiting OnExecute()");
}