public async Task SendResult(object sender, CaptureResponseEventArgs e)
{
if (e == null)
{
Logger.Warn("result is null, ignoring");
return;
}
var message = e.Request.RequestContext as Message;
if (message == null)
{
Logger.Error("Returned RequestContext is not a valid Telegram UserRequest object");
return;
}
if (e.Attachments != null)
{
foreach (var filePath in e.Attachments)
{
using (var fs = File.OpenRead(filePath))
{
var trial = 0;
var succeed = false;
var fileName = Path.GetFileName(filePath);
while (!succeed && trial < MaxUploadRetries)
{
try
{
Logger.Debug($"(retry {trial}/{MaxUploadRetries}) Uploading file \"{fileName}\"");
var inputOnlineFile = new InputOnlineFile(fs, fileName);
await _bot.SendDocumentAsync(message.Chat, inputOnlineFile,
replyToMessageId : message.MessageId);
succeed = true;
}
catch (ApiRequestException)
{
Logger.Warn("Telegram API timeout");
trial += 1;
}
}
if (!succeed)
{
Logger.Error("Unable to upload file \"{fileName}\"");
e.StatusText += "Unable to upload file \"{fileName}\".\n";
}
}
}
}
Logger.Debug("Sending session information");
await _bot.SendTextMessageAsync(
message.Chat,
$"<pre>{e}</pre>",
replyToMessageId : message.MessageId,
parseMode : ParseMode.Html
);
}
public async Task <CaptureResponseEventArgs> CapturePage(object sender, UserRequestEventArgs e)
{
var ret = new CaptureResponseEventArgs()
{
Request = e,
Url = e.Url,
Title = "",
};
var result = Uri.TryCreate(e.Url, UriKind.Absolute, out var uriResult);
if (!result)
{
ret.StatusText += $"Not a valid URL: \"{e.Url}\"";
return(ret);
}
var attachments = new List <string>();
try
{
var wc = new WebClientWithTimeout(Timeout);
var expectedFileName = Path.ConcentrateFilename(uriResult.Segments.Last(), e.Id.ToString());
await wc.DownloadFileTaskAsync(e.Url, expectedFileName);
attachments.Append(expectedFileName);
}
catch (WebException we)
{
Logger.Warn("Something happened.\n" + we);
ret.StatusText += we;
}
ret.Attachments = attachments.ToArray();
return(ret);
}
public async Task SendResult(object sender, CaptureResponseEventArgs e)
{
Logger.Info("Request finished.");
if (e.Attachments != null)
{
foreach (var item in e.Attachments)
{
Logger.Info($"Attachment: {item}");
}
}
}
public async Task <CaptureResponseEventArgs> CapturePage(object sender, UserRequestEventArgs e)
{
Logger.Debug($"Enter CapturePage() for session {e.Id}");
var ret = new CaptureResponseEventArgs
{
Request = e,
StatusText = "",
Attachments = null,
HasPotentialUnfinishedDownloads = true
};
var currentSessionBrowser = await NewBrowser();
// create a new session
Logger.Debug("Create incognito session");
var context = await currentSessionBrowser.CreateIncognitoBrowserContextAsync();
Logger.Debug("Create new tab");
var page = await context.NewPageAsync();
await page.SetViewportAsync(new ViewPortOptions
{
Width = WindowWidth,
Height = WindowHeight
});
// load document
Logger.Debug($"Navigate to \"{e.Url}\"");
try
{
await page.GoToAsync(
e.Url,
PageDownloadTimeout,
new[] { WaitUntilNavigation.Networkidle0 }
);
}
catch (WaitTaskTimeoutException)
{
// TODO: time exceeded
Logger.Warn($"Page loading time exceeded for url \"{e.Url}\"");
ret.StatusText += $"Page loading time exceeded for url \"{e.Url}\"\n";
ret.HasPotentialUnfinishedDownloads = false;
}
catch (NavigationException)
{
Logger.Warn($"Document download time exceeded for url \"{e.Url}\"");
ret.StatusText += $"Document download time exceeded for url \"{e.Url}\"\n";
ret.HasPotentialUnfinishedDownloads = false;
}
await page.WaitForTimeoutAsync(DocumentLoadDelay);
ret.Url = page.Url;
ret.Title = await page.GetTitleAsync();
var prefix =
Path.Escape(ret.Title.Substring(0, Math.Min(MaxTitlePrependLength, ret.Title.Length)) + "-" + e.Id);
// scroll through the page to load any lazy-loading elements
Logger.Debug("Trying to load lazy-loading elements");
try
{
// https://www.screenshotbin.com/blog/handling-lazy-loaded-webpages-puppeteer
var bodyHandle = await page.QuerySelectorAsync("body");
var boundingBox = await bodyHandle.BoundingBoxAsync();
var viewportHeight = page.Viewport.Height;
var viewportIncr = 0;
// scroll down
while (viewportIncr + viewportHeight < boundingBox.Height)
{
await page.EvaluateExpressionAsync($"window.scrollBy(0, {viewportHeight})");
await page.WaitForTimeoutAsync(PageScrollActionWaitDelay);
viewportIncr += viewportHeight;
}
// scroll up
await page.EvaluateExpressionAsync("window.scrollTo(0, 0)");
await page.WaitForTimeoutAsync(ExtraDownloadWaitDelay);
}
catch (NullReferenceException)
{
// body cannot be downloaded
Logger.Error("Body is missing, either your URL contains a redirection or there is a network issue");
ret.StatusText += "Failed to download web page\n";
ret.HasPotentialUnfinishedDownloads = true;
return(ret);
}
var attachments = new List <string>();
if (e.RequestTypes.Contains(UserRequestType.Pdf))
{ // capture PDF
try
{
Logger.Debug("Saving PDF");
await page.PdfAsync($"{prefix}.pdf", new PdfOptions
{
// TODO: header and footer is not in the correct position, ignore for now
// HeaderTemplate = "<title /> - <date />",
// FooterTemplate =
// "<url /> - Captured by ScreenShooter - https://github.com/Jamesits/ScreenShooter - <pageNumber />/<totalPages />",
DisplayHeaderFooter = true,
PrintBackground = true,
Format = PaperFormat.A4,
MarginOptions = new MarginOptions
{
Bottom = "0.5in",
Top = "0.5in",
Left = "0.3in",
Right = "0.3in"
}
});
attachments.Add($"{prefix}.pdf");
}
catch (TargetClosedException ex)
{
// possibility out of memory, see https://github.com/Jamesits/ScreenShooter/issues/1
ret.StatusText += "Possible out of memory when requesting PDF\n";
Logger.Error($"Something happened on requesting PDF. \n\nException:\n{ex}\n\nInnerException:{ex.InnerException}");
_requireNewBrowserInstance = true;
}
}
if (e.RequestTypes.Contains(UserRequestType.Png))
{ // capture screenshot
try
{
Logger.Debug("Taking screenshot");
await page.ScreenshotAsync($"{prefix}.png", new ScreenshotOptions
{
FullPage = true
});
attachments.Add($"{prefix}.png");
}
catch (TargetClosedException ex)
{
// possibility out of memory, see https://github.com/Jamesits/ScreenShooter/issues/1
ret.StatusText += "Possible out of memory when requesting screenshot\n";
Logger.Error($"Something happened on requesting screenshot. \n\nException:\n{ex}\n\nInnerException:{ex.InnerException}");
_requireNewBrowserInstance = true;
}
}
ret.Attachments = attachments.ToArray();
// clean up
try
{
Logger.Debug("Close tab");
await page.CloseAsync();
Logger.Debug("Kill context");
await context.CloseAsync();
Logger.Debug("Closing browser");
await currentSessionBrowser.CloseAsync();
}
catch (Exception ex)
{
Logger.Error($"Something happened on cleaning up. \n\nException:\n{ex}\n\nInnerException:{ex.InnerException}");
}
Logger.Debug("Exit CapturePage()");
return(ret);
}
private async Task NextRequest()
{
var currentRequest = _requestQueue.Get();
var requester = currentRequest.Requester as IConnector;
if (requester == null)
{
Logger.Error("Requester is not a IConnector");
return;
}
RuntimeInformation.QueuedRequests -= 1;
RuntimeInformation.OnGoingRequests += 1;
Logger.Debug($"Processing request {currentRequest.Id}");
// get all actuators with specific capability
// TODO: what if a request needs 2 actuators w/ different capabilities to fulfill?
var availableActuators =
_actuators.Where(x => x.Capability.IsSubsetOf(currentRequest.RequestTypes)).ToArray();
if (availableActuators.Length <= 0)
{
Logger.Error("No actuator matches user request");
await requester.SendResult(this, new CaptureResponseEventArgs
{
Request = currentRequest,
HasPotentialUnfinishedDownloads = true,
StatusText = $"Sorry, no actuators can fulfill your request."
});
RuntimeInformation.FailedRequests += 1;
return;
}
// randomly select a actuator
var r = Rnd.Next(availableActuators.Length);
var a = availableActuators[r];
CaptureResponseEventArgs ret = null;
try
{ // try get a result from actuator
ret = await a.CapturePage(this, currentRequest);
Logger.Info(ret);
Logger.Debug("Sending result");
await requester.SendResult(this, ret);
RuntimeInformation.FinishedRequests += 1;
}
catch (Exception exception)
{ // if failed, we make a result ourselves
CurrentDomainUnhandledException(this, new UnhandledExceptionEventArgs(exception, false));
await requester.SendResult(this, new CaptureResponseEventArgs
{
Request = currentRequest,
HasPotentialUnfinishedDownloads = true,
StatusText = $"Something happened. \nException: {exception}"
});
RuntimeInformation.FailedRequests += 1;
}
finally
{
if (Globals.GlobalConfig.RemoveLocalFile && ret != null)
{
foreach (var attachment in ret.Attachments)
{
try
{
File.Delete(attachment);
}
catch (Exception e)
{
Logger.Error($"Cannot delete file {attachment}, Exception: \n{e}");
}
}
}
RuntimeInformation.OnGoingRequests -= 1;
Logger.Debug($"Finished request {currentRequest.Id}");
}
}
