public void Process(String content) { if (Engine != null) { _response = VirtualResponse.Create(res => res.Content(content).Address(_script.BaseUri)); } }
public void ShouldChangeSelectedCookiesOnRedirect_Issue548() { var requester = new MockRequester(); var mcp = new MemoryCookieProvider(); var config = Configuration.Default.With(mcp).With(requester).WithDefaultLoader(); var context = BrowsingContext.New(config); var receivedCookieHeader = "THECOOKIE=value1"; var url = new Url("http://example.com/path1"); //request 1: /path1, set a cookie THECOOKIE=value1 requester.BuildResponse(req => VirtualResponse.Create(r => r .Address("http://example.com/path1") .Content("") .Header(HeaderNames.SetCookie, receivedCookieHeader))); context.OpenAsync("http://example.com/path1"); //request 2: /path1/somefile.jsp redirects to /path2/file2.jsp requester.BuildResponses(new Func <Request, IResponse>[] { req => VirtualResponse.Create(r => r .Address("http://example.com/path1/somefile.jsp") .Content("") .Status(System.Net.HttpStatusCode.Redirect) .Header(HeaderNames.Location, "http://example.com/path2/file2.jsp")), req => { receivedCookieHeader = req.Headers.GetOrDefault(HeaderNames.Cookie, String.Empty); return(VirtualResponse.Create(r => r .Address("http://example.com/path2/file2.jsp") .Content(""))); } }); context.OpenAsync("http://example.com/path1/somefile.jsp"); Assert.AreEqual(String.Empty, receivedCookieHeader); }
protected static void RegisterEventCallback <TElement>(String eventName) where TElement : Element { var name = "on" + eventName; RegisterCallback <TElement>(name, (element, value) => { var document = element.Owner; var configuration = document.Options; var engine = configuration.GetScriptEngine(MimeTypeNames.DefaultJavaScript); if (engine != null) { var function = "undefined"; if (!String.IsNullOrEmpty(value)) { function = String.Concat("function () { ", value, " }"); } var source = String.Format("element.{0} = {1};", name, function); var response = VirtualResponse.Create(res => res.Content(source)); var options = new ScriptOptions(document); engine.EvaluateScriptAsync(response, options, CancellationToken.None); } }); }
/// <summary> /// Opens a new document loaded from a virtual response that can be /// filled via the provided callback. /// </summary> /// <param name="context">The browsing context to use.</param> /// <param name="request">Callback with the response to setup.</param> /// <param name="cancel">The cancellation token.</param> /// <returns>The task that creates the document.</returns> public static async Task <IDocument> OpenAsync(this IBrowsingContext context, Action <VirtualResponse> request, CancellationToken cancel = default) { request = request ?? throw new ArgumentNullException(nameof(request)); using (var response = VirtualResponse.Create(request)) { return(await context.OpenAsync(response, cancel).ConfigureAwait(false)); } }
// Trick to be able to share the same parsed Html document between Abot and HtmlDocumentConverter // We need to activate Css dependencies loading to enable this private CrawlDecision WebCrawler_ShouldCrawlPageLinks(CrawledPage crawledPage, CrawlContext crawlContext) { try { // Add the page already downloaded by Abot in the document cache var htmlDocumentUri = crawledPage.HttpWebResponse.ResponseUri; if (!context.ResponseCache.ContainsKey(htmlDocumentUri.AbsoluteUri)) { var response = VirtualResponse.Create(r => { r.Address(new Url(htmlDocumentUri.AbsoluteUri)) .Status(crawledPage.HttpWebResponse.StatusCode) .Content(crawledPage.Content.Text, crawledPage.Content.Charset); foreach (var header in crawledPage.HttpWebResponse.Headers.AllKeys) { r.Header(header, crawledPage.HttpWebResponse.Headers[header]); } }); context.ResponseCache.Add(htmlDocumentUri.AbsoluteUri, response); } // Parse the page and its Css dependencies whith Anglesharp // in the right context, initialized in the constructor Stopwatch timer = Stopwatch.StartNew(); crawledPage.AngleSharpHtmlDocument = context.OpenAsync(htmlDocumentUri.AbsoluteUri).Result as IHtmlDocument; timer.Stop(); Perfs.AddParseTime(timer.ElapsedMilliseconds); // Remove page which was just parsed from document cache (not useful anymore) context.ResponseCache.Remove(htmlDocumentUri.AbsoluteUri); // Don't impact the crawl decision return(new CrawlDecision() { Allow = true }); } catch (Exception e) { if (e is ArgumentException) { // Do nothing if the key already exists : // - one exception every 15 minutes is better than a systematic lock on each call // - the crawl decision below will properly avoid analyzing the page twice } else { WriteError("Error while parsing the page " + crawledPage.HttpWebResponse.ResponseUri.AbsoluteUri, e); } // Don't crawl return(new CrawlDecision() { Allow = false }); } }
private IResponse Request(IRequest angleSharpRequest, CancellationToken cancel) { IResponse angleSharpResponse = null; string content = this.GetFileContent(angleSharpRequest.Address); angleSharpResponse = VirtualResponse.Create(r => r .Address(angleSharpRequest.Address) .Content(content) ); return(angleSharpResponse); }
async Task CreateSheetAsync(IStyleEngine engine, IBrowsingContext context) { var cancel = CancellationToken.None; var response = VirtualResponse.Create(res => res.Content(TextContent).Address(default(Url))); var options = new StyleOptions(context) { Element = this, IsDisabled = IsDisabled, IsAlternate = false }; var task = engine.ParseStylesheetAsync(response, options, cancel); _sheet = await task.ConfigureAwait(false); }
public async Task CookieWithUTCTimeStampVariant2() { var content = "<!doctype html>"; var cookieValue = "ct0=cf2c3d61837dc0513fe9dfa8019a3af8; Expires=Wed, 03 Jan 2018 16:54:34 UTC; Path=/; Domain=.twitter.com; Secure"; var requestCount = 0; var initial = VirtualResponse.Create(m => m.Content(content).Address("http://www.twitter.com").Header(HeaderNames.SetCookie, cookieValue)); var document = await LoadDocumentWithFakeRequesterAndCookie(initial, req => { var res = VirtualResponse.Create(m => m.Content(String.Empty).Address(req.Address)); requestCount++; return(res); }); Assert.AreEqual(0, requestCount); }
public async Task SettingNoCookieInSubsequentRequestLeavesCookieSituationUnchanged() { var config = Configuration.Default. WithCookies(). WithVirtualRequester(req => VirtualResponse.Create( res => res.Address("http://localhost/mockpage.html"). Content("<div></div>"))); var context = BrowsingContext.New(config); var document = await context.OpenAsync(res => res.Content("<a href=mockpage.html></a>"). Address("http://localhost/"). Header(HeaderNames.SetCookie, "UserID=Foo")); document = await document.QuerySelector <IHtmlAnchorElement>("a").NavigateAsync(); Assert.AreEqual("UserID=Foo", document.Cookie); }
public async Task SendingRequestToLocalResourceSendsLocalCookie() { var content = "<!doctype html><img src=http://www.local.com/foo.png />"; var cookieValue = "test=true"; var requestCount = 0; var imgCookie = String.Empty; var initial = VirtualResponse.Create(m => m.Content(content).Address("http://www.local.com").Header(HeaderNames.SetCookie, cookieValue)); var document = await LoadDocumentWithFakeRequesterAndCookie(initial, req => { var res = VirtualResponse.Create(m => m.Content(String.Empty).Address(req.Address)); imgCookie = req.Headers.GetOrDefault(HeaderNames.Cookie, String.Empty); requestCount++; return(res); }); Assert.AreEqual(1, requestCount); Assert.AreEqual(cookieValue, imgCookie); }
public async Task SettingNewCookieInSubsequentRequestDoesNotExpirePreviousCookies() { var config = Configuration.Default.WithCookies().WithVirtualRequester(req => VirtualResponse.Create( res => res.Address("http://localhost/mockpage.html").Content("<div></div>") .Cookie("Auth=Bar; Path=/"))); var context = BrowsingContext.New(config); var document = await context.OpenAsync(res => res.Content("<a href=mockpage.html></a>").Address("http://localhost/").Cookie("UserID=Foo; Path=/")); document = await document.QuerySelector <IHtmlAnchorElement>("a").NavigateAsync(); Assert.AreEqual("UserID=Foo; Auth=Bar", document.Cookie); }