private IVirtualChangeSet <TObject, TKey> Virtualise(ISortedChangeSet <TObject, TKey> updates = null)
{
if (_isLoaded == false)
{
return(null);
}
var previous = _current;
var virualised = _all.Skip(_parameters.StartIndex)
.Take(_parameters.Size)
.ToList();
_current = new KeyValueCollection <TObject, TKey>(virualised, _all.Comparer, updates?.SortedItems.SortReason ?? SortReason.DataChanged, _all.Optimisations);
////check for changes within the current virtualised page. Notify if there have been changes or if the overall count has changed
var notifications = _changedCalculator.Calculate(_current, previous, updates);
if (notifications.Count == 0 && (previous.Count != _current.Count))
{
return(null);
}
var response = new VirtualResponse(_parameters.Size, _parameters.StartIndex, _all.Count);
return(new VirtualChangeSet <TObject, TKey>(notifications, _current, response));
}
public void Process(String content)
{
if (Engine != null)
{
_response = VirtualResponse.Create(res => res.Content(content).Address(_script.BaseUri));
}
}
public void ShouldChangeSelectedCookiesOnRedirect_Issue548()
{
var requester = new MockRequester();
var mcp = new MemoryCookieProvider();
var config = Configuration.Default.With(mcp).With(requester).WithDefaultLoader();
var context = BrowsingContext.New(config);
var receivedCookieHeader = "THECOOKIE=value1";
var url = new Url("http://example.com/path1");
//request 1: /path1, set a cookie THECOOKIE=value1
requester.BuildResponse(req => VirtualResponse.Create(r => r
.Address("http://example.com/path1")
.Content("")
.Header(HeaderNames.SetCookie, receivedCookieHeader)));
context.OpenAsync("http://example.com/path1");
//request 2: /path1/somefile.jsp redirects to /path2/file2.jsp
requester.BuildResponses(new Func <Request, IResponse>[] {
req => VirtualResponse.Create(r => r
.Address("http://example.com/path1/somefile.jsp")
.Content("")
.Status(System.Net.HttpStatusCode.Redirect)
.Header(HeaderNames.Location, "http://example.com/path2/file2.jsp")),
req => {
receivedCookieHeader = req.Headers.GetOrDefault(HeaderNames.Cookie, String.Empty);
return(VirtualResponse.Create(r => r
.Address("http://example.com/path2/file2.jsp")
.Content("")));
}
});
context.OpenAsync("http://example.com/path1/somefile.jsp");
Assert.AreEqual(String.Empty, receivedCookieHeader);
}
protected static void RegisterEventCallback <TElement>(String eventName)
where TElement : Element
{
var name = "on" + eventName;
RegisterCallback <TElement>(name, (element, value) =>
{
var document = element.Owner;
var configuration = document.Options;
var engine = configuration.GetScriptEngine(MimeTypeNames.DefaultJavaScript);
if (engine != null)
{
var function = "undefined";
if (!String.IsNullOrEmpty(value))
{
function = String.Concat("function () { ", value, " }");
}
var source = String.Format("element.{0} = {1};", name, function);
var response = VirtualResponse.Create(res => res.Content(source));
var options = new ScriptOptions(document);
engine.EvaluateScriptAsync(response, options, CancellationToken.None);
}
});
}
/// <summary>
/// Opens a new document loaded from a virtual response that can be
/// filled via the provided callback.
/// </summary>
/// <param name="context">The browsing context to use.</param>
/// <param name="request">Callback with the response to setup.</param>
/// <param name="cancel">The cancellation token.</param>
/// <returns>The task that creates the document.</returns>
public static async Task <IDocument> OpenAsync(this IBrowsingContext context, Action <VirtualResponse> request, CancellationToken cancel = default)
{
request = request ?? throw new ArgumentNullException(nameof(request));
using (var response = VirtualResponse.Create(request))
{
return(await context.OpenAsync(response, cancel).ConfigureAwait(false));
}
}
// Trick to be able to share the same parsed Html document between Abot and HtmlDocumentConverter
// We need to activate Css dependencies loading to enable this
private CrawlDecision WebCrawler_ShouldCrawlPageLinks(CrawledPage crawledPage, CrawlContext crawlContext)
{
try
{
// Add the page already downloaded by Abot in the document cache
var htmlDocumentUri = crawledPage.HttpWebResponse.ResponseUri;
if (!context.ResponseCache.ContainsKey(htmlDocumentUri.AbsoluteUri))
{
var response = VirtualResponse.Create(r =>
{
r.Address(new Url(htmlDocumentUri.AbsoluteUri))
.Status(crawledPage.HttpWebResponse.StatusCode)
.Content(crawledPage.Content.Text, crawledPage.Content.Charset);
foreach (var header in crawledPage.HttpWebResponse.Headers.AllKeys)
{
r.Header(header, crawledPage.HttpWebResponse.Headers[header]);
}
});
context.ResponseCache.Add(htmlDocumentUri.AbsoluteUri, response);
}
// Parse the page and its Css dependencies whith Anglesharp
// in the right context, initialized in the constructor
Stopwatch timer = Stopwatch.StartNew();
crawledPage.AngleSharpHtmlDocument = context.OpenAsync(htmlDocumentUri.AbsoluteUri).Result as IHtmlDocument;
timer.Stop();
Perfs.AddParseTime(timer.ElapsedMilliseconds);
// Remove page which was just parsed from document cache (not useful anymore)
context.ResponseCache.Remove(htmlDocumentUri.AbsoluteUri);
// Don't impact the crawl decision
return(new CrawlDecision()
{
Allow = true
});
}
catch (Exception e)
{
if (e is ArgumentException)
{
// Do nothing if the key already exists :
// - one exception every 15 minutes is better than a systematic lock on each call
// - the crawl decision below will properly avoid analyzing the page twice
}
else
{
WriteError("Error while parsing the page " + crawledPage.HttpWebResponse.ResponseUri.AbsoluteUri, e);
}
// Don't crawl
return(new CrawlDecision()
{
Allow = false
});
}
}
private static void MapHeaders(VirtualResponse virtualResponse, HttpHeaders headers)
{
foreach (var header in headers)
{
foreach (var value in header.Value)
{
virtualResponse.Header(header.Key, value);
}
}
}
private static async void ResponseFactory(VirtualResponse virtualResponse, HttpResponseMessage response)
{
virtualResponse
.Address(response.RequestMessage.RequestUri)
.Status(response.StatusCode)
.Content(await response.Content.ReadAsStringAsync());
MapHeaders(virtualResponse, response.Headers);
MapHeaders(virtualResponse, response.Content.Headers);
}
private IResponse Request(IRequest angleSharpRequest, CancellationToken cancel)
{
IResponse angleSharpResponse = null;
string content = this.GetFileContent(angleSharpRequest.Address);
angleSharpResponse = VirtualResponse.Create(r => r
.Address(angleSharpRequest.Address)
.Content(content)
);
return(angleSharpResponse);
}
async Task CreateSheetAsync(IStyleEngine engine, IBrowsingContext context)
{
var cancel = CancellationToken.None;
var response = VirtualResponse.Create(res => res.Content(TextContent).Address(default(Url)));
var options = new StyleOptions(context)
{
Element = this,
IsDisabled = IsDisabled,
IsAlternate = false
};
var task = engine.ParseStylesheetAsync(response, options, cancel);
_sheet = await task.ConfigureAwait(false);
}
public async Task CookieWithUTCTimeStampVariant2()
{
var content = "<!doctype html>";
var cookieValue = "ct0=cf2c3d61837dc0513fe9dfa8019a3af8; Expires=Wed, 03 Jan 2018 16:54:34 UTC; Path=/; Domain=.twitter.com; Secure";
var requestCount = 0;
var initial = VirtualResponse.Create(m => m.Content(content).Address("http://www.twitter.com").Header(HeaderNames.SetCookie, cookieValue));
var document = await LoadDocumentWithFakeRequesterAndCookie(initial, req =>
{
var res = VirtualResponse.Create(m => m.Content(String.Empty).Address(req.Address));
requestCount++;
return(res);
});
Assert.AreEqual(0, requestCount);
}
public async Task SettingNoCookieInSubsequentRequestLeavesCookieSituationUnchanged()
{
var config = Configuration.Default.
WithCookies().
WithVirtualRequester(req => VirtualResponse.Create(
res => res.Address("http://localhost/mockpage.html").
Content("<div></div>")));
var context = BrowsingContext.New(config);
var document = await context.OpenAsync(res =>
res.Content("<a href=mockpage.html></a>").
Address("http://localhost/").
Header(HeaderNames.SetCookie, "UserID=Foo"));
document = await document.QuerySelector <IHtmlAnchorElement>("a").NavigateAsync();
Assert.AreEqual("UserID=Foo", document.Cookie);
}
public async Task SendingRequestToLocalResourceSendsLocalCookie()
{
var content = "<!doctype html><img src=http://www.local.com/foo.png />";
var cookieValue = "test=true";
var requestCount = 0;
var imgCookie = String.Empty;
var initial = VirtualResponse.Create(m => m.Content(content).Address("http://www.local.com").Header(HeaderNames.SetCookie, cookieValue));
var document = await LoadDocumentWithFakeRequesterAndCookie(initial, req =>
{
var res = VirtualResponse.Create(m => m.Content(String.Empty).Address(req.Address));
imgCookie = req.Headers.GetOrDefault(HeaderNames.Cookie, String.Empty);
requestCount++;
return(res);
});
Assert.AreEqual(1, requestCount);
Assert.AreEqual(cookieValue, imgCookie);
}
/// <summary>
/// Opens a new document loaded from a virtual response that can be
/// filled via the provided callback.
/// </summary>
/// <param name="context">The browsing context to use.</param>
/// <param name="request">Callback with the response to setup.</param>
/// <param name="cancel">The cancellation token.</param>
/// <returns>The task that creates the document.</returns>
public static Task <IDocument> OpenAsync(this IBrowsingContext context, Action <VirtualResponse> request, CancellationToken cancel)
{
if (request == null)
{
throw new ArgumentNullException("request");
}
if (context == null)
{
context = BrowsingContext.New();
}
using (var response = new VirtualResponse())
{
request(response);
var source = response.CreateSourceFor(context.Configuration);
return(context.LoadDocumentAsync(response, source, cancel));
}
}
/// <summary>
/// Opens a new document loaded from a virtual response that can be
/// filled via the provided callback.
/// </summary>
/// <param name="context">The browsing context to use.</param>
/// <param name="request">Callback with the response to setup.</param>
/// <param name="cancel">The cancellation token.</param>
/// <returns>The task that creates the document.</returns>
public static Task <IDocument> OpenAsync(this IBrowsingContext context, Action <VirtualResponse> request, CancellationToken cancel)
{
if (request == null)
{
throw new ArgumentNullException("request");
}
if (context == null)
{
context = BrowsingContext.New();
}
using (var response = new VirtualResponse())
{
request(response);
var contentType = response.GetContentType(MimeTypeNames.Html);
var source = response.CreateSourceFor(context.Configuration);
var options = new CreateDocumentOptions(response, source);
return(context.OpenAsync(options, cancel));
}
}
public async Task SettingNewCookieInSubsequentRequestDoesNotExpirePreviousCookies()
{
var config = Configuration.Default.WithCookies().WithVirtualRequester(req => VirtualResponse.Create(
res => res.Address("http://localhost/mockpage.html").Content("<div></div>")
.Cookie("Auth=Bar; Path=/")));
var context = BrowsingContext.New(config);
var document = await context.OpenAsync(res =>
res.Content("<a href=mockpage.html></a>").Address("http://localhost/").Cookie("UserID=Foo; Path=/"));
document = await document.QuerySelector <IHtmlAnchorElement>("a").NavigateAsync();
Assert.AreEqual("UserID=Foo; Auth=Bar", document.Cookie);
}
