public void GetHtmlTimeout() { var creator = new Mocks.MockWebRequestCreator(); creator.ResponseTime = 1000; creator.ResponseStream = GetMemoryStream("<div>Test</div>"); var request = new CsqWebRequest("http://test.com", creator); request.Options = new ServerConfig { Timeout = TimeSpan.FromMilliseconds(500), UserAgent = "test" }; var httpRequest = request.GetWebRequest(); IHttpWebResponse response; Assert.Throws<System.Net.WebException>(() => { response = httpRequest.GetResponse(); }); request.Timeout = 1500; httpRequest = request.GetWebRequest(); response = httpRequest.GetResponse(); var responseStream = response.GetResponseStream(); var encoding = CsqWebRequest.GetEncoding(response); var dom = CQ.CreateDocument(responseStream, encoding); Assert.AreEqual(1, dom["div"].Length); }
/// <summary> /// Creates a new DOM from an HTML file. /// </summary> /// /// <param name="url"> /// The URL of the remote server. /// </param> /// <param name="options"> /// The options to use when creating the reqest. /// </param> /// /// <returns> /// A CQ object composed from the HTML response from the server. /// </returns> public static CQ CreateFromUrl(string url, ServerConfig options=null) { CsqWebRequest request = new CsqWebRequest(url); request.Options = options; var httpRequest = request.GetWebRequest(); var response = httpRequest.GetResponse(); var responseStream = response.GetResponseStream(); var encoding = CsqWebRequest.GetEncoding(response); return CQ.CreateDocument(responseStream,encoding); }
/// <summary> /// Process the mock web request synchronously using same rules as CreateFromUrl /// </summary> /// /// <param name="request"> /// The request. /// </param> /// /// <returns> /// . /// </returns> private CQ ProcessMockWebRequestSync(CsqWebRequest request) { var httpRequest = request.GetWebRequest(); var response = httpRequest.GetResponse(); var responseStream = response.GetResponseStream(); var encoding = CsqWebRequest.GetEncoding(response); return CQ.CreateDocument(responseStream, encoding); }
public void Utf8NoContentType() { var creator = new Mocks.MockWebRequestCreator(); creator.CharacterSet = "ISO-8859-1"; creator.ResponseStream = GetMemoryStream(TestHtml("arabic"), new UTF8Encoding(false)); CsqWebRequest request = new CsqWebRequest("http://test.com", creator); // remove the content type header var html = ReplaceCharacterSet(request.Get()); var dom = CQ.CreateDocument(html); Assert.AreNotEqual(arabicExpected, dom["h1"].Text()); //test synchronous: this is the code that CreateFromURL uses creator.CharacterSet = null; request = new CsqWebRequest("http://test.com", creator); var httpRequest = request.GetWebRequest(); var response = httpRequest.GetResponse(); var responseStream = response.GetResponseStream(); var encoding = CsqWebRequest.GetEncoding(response); var dom2 = CQ.CreateDocument(responseStream, encoding); Assert.AreEqual(arabicExpected, dom2["h1"].Text()); // Test async version now request = new CsqWebRequest("http://test.com", creator); bool? done=null; CQ dom3=null; request.GetAsync((r) => { dom3 = r.Dom; done = true; }, (r)=>{ done = false; }); while (done == null) ; Assert.IsTrue((bool)done); Assert.AreEqual(arabicExpected, dom3["h1"].Text()); }
public void ContentTypeHeader() { var creator = new Mocks.MockWebRequestCreator(); creator.CharacterSet = "windows-1255"; creator.ResponseStream = GetMemoryStream(htmlStart + htmlStart3 + hebrewChar + htmlEnd, Encoding.GetEncoding("windows-1255")); CsqWebRequest request = new CsqWebRequest("http://test.com", creator); var dom1 = CQ.Create(request.Get()); creator.CharacterSet = ""; request = new CsqWebRequest("http://test.com", creator); var dom2 = CQ.Create(request.Get()); var output = dom1.Render(OutputFormatters.HtmlEncodingMinimum); // The characters should be encoded differently. var outputHebrewChar = dom1["#test"].Text(); var outputUTF8Char = dom2["#test"].Text(); Assert.AreNotEqual(outputHebrewChar, outputUTF8Char); // try it again, using the meta tag creator.CharacterSet = "windows-1255"; creator.ResponseStream = GetMemoryStream(htmlStart + htmlStartMeta + htmlStart3 + hebrewChar + htmlEnd, Encoding.GetEncoding("windows-1255")); /// CreateFromUrl process request = new CsqWebRequest("http://test.com", creator); var httpRequest = request.GetWebRequest(); var response = httpRequest.GetResponse(); var responseStream = response.GetResponseStream(); var encoding = CsqWebRequest.GetEncoding(response); var dom3 = CQ.CreateDocument(responseStream, encoding); var outputHebrewChar2 = dom3["#test"].Text(); Assert.AreEqual(outputHebrewChar, outputHebrewChar2); }
public void Utf8NoContentType() { var creator = new Mocks.MockWebRequestCreator(); creator.CharacterSet = "ISO-8859-1"; creator.ResponseStream = GetMemoryStream(TestHtml("arabic"), new UTF8Encoding(false)); CsqWebRequest request = new CsqWebRequest("http://test.com", creator); // remove the content type header var html = request.Get(); var start = html.IndexOf(@"<meta http-equiv=""Content-Type"""); var end = html.IndexOf(">",start); html = html.Substring(0,start)+html.Substring(end+1); var dom = CQ.CreateDocument(html); var expected = @"البابا: اوقفوا ""المجزرة"" في سوريا قبل ان تتحول البلاد الى ""أطلال"""; Assert.AreNotEqual(expected, dom["h1"].Text()); //test synchronous: this is the code that CreateFromURL uses request = new CsqWebRequest("http://test.com", creator); var httpRequest = request.GetWebRequest(); var response = httpRequest.GetResponse(); var responseStream = response.GetResponseStream(); var encoding = CsqWebRequest.GetEncoding(response); var dom2 = CQ.CreateDocument(responseStream, encoding); Assert.AreEqual(expected, dom2["h1"].Text()); // Test async version now request = new CsqWebRequest("http://test.com", creator); bool? done=null; CQ dom3=null; request.GetAsync((r) => { dom3 = r.Dom; done = true; }, (r)=>{ done = false; }); while (done == null) ; Assert.IsTrue((bool)done); Assert.AreEqual(expected, dom3["h1"].Text()); }