public async Task SendResult(object sender, CaptureResponseEventArgs e) { if (e == null) { Logger.Warn("result is null, ignoring"); return; } var message = e.Request.RequestContext as Message; if (message == null) { Logger.Error("Returned RequestContext is not a valid Telegram UserRequest object"); return; } if (e.Attachments != null) { foreach (var filePath in e.Attachments) { using (var fs = File.OpenRead(filePath)) { var trial = 0; var succeed = false; var fileName = Path.GetFileName(filePath); while (!succeed && trial < MaxUploadRetries) { try { Logger.Debug($"(retry {trial}/{MaxUploadRetries}) Uploading file \"{fileName}\""); var inputOnlineFile = new InputOnlineFile(fs, fileName); await _bot.SendDocumentAsync(message.Chat, inputOnlineFile, replyToMessageId : message.MessageId); succeed = true; } catch (ApiRequestException) { Logger.Warn("Telegram API timeout"); trial += 1; } } if (!succeed) { Logger.Error("Unable to upload file \"{fileName}\""); e.StatusText += "Unable to upload file \"{fileName}\".\n"; } } } } Logger.Debug("Sending session information"); await _bot.SendTextMessageAsync( message.Chat, $"<pre>{e}</pre>", replyToMessageId : message.MessageId, parseMode : ParseMode.Html ); }
public async Task <CaptureResponseEventArgs> CapturePage(object sender, UserRequestEventArgs e) { var ret = new CaptureResponseEventArgs() { Request = e, Url = e.Url, Title = "", }; var result = Uri.TryCreate(e.Url, UriKind.Absolute, out var uriResult); if (!result) { ret.StatusText += $"Not a valid URL: \"{e.Url}\""; return(ret); } var attachments = new List <string>(); try { var wc = new WebClientWithTimeout(Timeout); var expectedFileName = Path.ConcentrateFilename(uriResult.Segments.Last(), e.Id.ToString()); await wc.DownloadFileTaskAsync(e.Url, expectedFileName); attachments.Append(expectedFileName); } catch (WebException we) { Logger.Warn("Something happened.\n" + we); ret.StatusText += we; } ret.Attachments = attachments.ToArray(); return(ret); }
public async Task SendResult(object sender, CaptureResponseEventArgs e) { Logger.Info("Request finished."); if (e.Attachments != null) { foreach (var item in e.Attachments) { Logger.Info($"Attachment: {item}"); } } }
public async Task <CaptureResponseEventArgs> CapturePage(object sender, UserRequestEventArgs e) { Logger.Debug($"Enter CapturePage() for session {e.Id}"); var ret = new CaptureResponseEventArgs { Request = e, StatusText = "", Attachments = null, HasPotentialUnfinishedDownloads = true }; var currentSessionBrowser = await NewBrowser(); // create a new session Logger.Debug("Create incognito session"); var context = await currentSessionBrowser.CreateIncognitoBrowserContextAsync(); Logger.Debug("Create new tab"); var page = await context.NewPageAsync(); await page.SetViewportAsync(new ViewPortOptions { Width = WindowWidth, Height = WindowHeight }); // load document Logger.Debug($"Navigate to \"{e.Url}\""); try { await page.GoToAsync( e.Url, PageDownloadTimeout, new[] { WaitUntilNavigation.Networkidle0 } ); } catch (WaitTaskTimeoutException) { // TODO: time exceeded Logger.Warn($"Page loading time exceeded for url \"{e.Url}\""); ret.StatusText += $"Page loading time exceeded for url \"{e.Url}\"\n"; ret.HasPotentialUnfinishedDownloads = false; } catch (NavigationException) { Logger.Warn($"Document download time exceeded for url \"{e.Url}\""); ret.StatusText += $"Document download time exceeded for url \"{e.Url}\"\n"; ret.HasPotentialUnfinishedDownloads = false; } await page.WaitForTimeoutAsync(DocumentLoadDelay); ret.Url = page.Url; ret.Title = await page.GetTitleAsync(); var prefix = Path.Escape(ret.Title.Substring(0, Math.Min(MaxTitlePrependLength, ret.Title.Length)) + "-" + e.Id); // scroll through the page to load any lazy-loading elements Logger.Debug("Trying to load lazy-loading elements"); try { // https://www.screenshotbin.com/blog/handling-lazy-loaded-webpages-puppeteer var bodyHandle = await page.QuerySelectorAsync("body"); var boundingBox = await bodyHandle.BoundingBoxAsync(); var viewportHeight = page.Viewport.Height; var viewportIncr = 0; // scroll down while (viewportIncr + viewportHeight < boundingBox.Height) { await page.EvaluateExpressionAsync($"window.scrollBy(0, {viewportHeight})"); await page.WaitForTimeoutAsync(PageScrollActionWaitDelay); viewportIncr += viewportHeight; } // scroll up await page.EvaluateExpressionAsync("window.scrollTo(0, 0)"); await page.WaitForTimeoutAsync(ExtraDownloadWaitDelay); } catch (NullReferenceException) { // body cannot be downloaded Logger.Error("Body is missing, either your URL contains a redirection or there is a network issue"); ret.StatusText += "Failed to download web page\n"; ret.HasPotentialUnfinishedDownloads = true; return(ret); } var attachments = new List <string>(); if (e.RequestTypes.Contains(UserRequestType.Pdf)) { // capture PDF try { Logger.Debug("Saving PDF"); await page.PdfAsync($"{prefix}.pdf", new PdfOptions { // TODO: header and footer is not in the correct position, ignore for now // HeaderTemplate = "<title /> - <date />", // FooterTemplate = // "<url /> - Captured by ScreenShooter - https://github.com/Jamesits/ScreenShooter - <pageNumber />/<totalPages />", DisplayHeaderFooter = true, PrintBackground = true, Format = PaperFormat.A4, MarginOptions = new MarginOptions { Bottom = "0.5in", Top = "0.5in", Left = "0.3in", Right = "0.3in" } }); attachments.Add($"{prefix}.pdf"); } catch (TargetClosedException ex) { // possibility out of memory, see https://github.com/Jamesits/ScreenShooter/issues/1 ret.StatusText += "Possible out of memory when requesting PDF\n"; Logger.Error($"Something happened on requesting PDF. \n\nException:\n{ex}\n\nInnerException:{ex.InnerException}"); _requireNewBrowserInstance = true; } } if (e.RequestTypes.Contains(UserRequestType.Png)) { // capture screenshot try { Logger.Debug("Taking screenshot"); await page.ScreenshotAsync($"{prefix}.png", new ScreenshotOptions { FullPage = true }); attachments.Add($"{prefix}.png"); } catch (TargetClosedException ex) { // possibility out of memory, see https://github.com/Jamesits/ScreenShooter/issues/1 ret.StatusText += "Possible out of memory when requesting screenshot\n"; Logger.Error($"Something happened on requesting screenshot. \n\nException:\n{ex}\n\nInnerException:{ex.InnerException}"); _requireNewBrowserInstance = true; } } ret.Attachments = attachments.ToArray(); // clean up try { Logger.Debug("Close tab"); await page.CloseAsync(); Logger.Debug("Kill context"); await context.CloseAsync(); Logger.Debug("Closing browser"); await currentSessionBrowser.CloseAsync(); } catch (Exception ex) { Logger.Error($"Something happened on cleaning up. \n\nException:\n{ex}\n\nInnerException:{ex.InnerException}"); } Logger.Debug("Exit CapturePage()"); return(ret); }
private async Task NextRequest() { var currentRequest = _requestQueue.Get(); var requester = currentRequest.Requester as IConnector; if (requester == null) { Logger.Error("Requester is not a IConnector"); return; } RuntimeInformation.QueuedRequests -= 1; RuntimeInformation.OnGoingRequests += 1; Logger.Debug($"Processing request {currentRequest.Id}"); // get all actuators with specific capability // TODO: what if a request needs 2 actuators w/ different capabilities to fulfill? var availableActuators = _actuators.Where(x => x.Capability.IsSubsetOf(currentRequest.RequestTypes)).ToArray(); if (availableActuators.Length <= 0) { Logger.Error("No actuator matches user request"); await requester.SendResult(this, new CaptureResponseEventArgs { Request = currentRequest, HasPotentialUnfinishedDownloads = true, StatusText = $"Sorry, no actuators can fulfill your request." }); RuntimeInformation.FailedRequests += 1; return; } // randomly select a actuator var r = Rnd.Next(availableActuators.Length); var a = availableActuators[r]; CaptureResponseEventArgs ret = null; try { // try get a result from actuator ret = await a.CapturePage(this, currentRequest); Logger.Info(ret); Logger.Debug("Sending result"); await requester.SendResult(this, ret); RuntimeInformation.FinishedRequests += 1; } catch (Exception exception) { // if failed, we make a result ourselves CurrentDomainUnhandledException(this, new UnhandledExceptionEventArgs(exception, false)); await requester.SendResult(this, new CaptureResponseEventArgs { Request = currentRequest, HasPotentialUnfinishedDownloads = true, StatusText = $"Something happened. \nException: {exception}" }); RuntimeInformation.FailedRequests += 1; } finally { if (Globals.GlobalConfig.RemoveLocalFile && ret != null) { foreach (var attachment in ret.Attachments) { try { File.Delete(attachment); } catch (Exception e) { Logger.Error($"Cannot delete file {attachment}, Exception: \n{e}"); } } } RuntimeInformation.OnGoingRequests -= 1; Logger.Debug($"Finished request {currentRequest.Id}"); } }