public void XmlEncodingDeclaration() { var creator = new Mocks.MockWebRequestCreator(); creator.CharacterSet = null; var html = ReplaceCharacterSet(TestHtml("arabic"), "ISO-8859-1"); creator.ResponseStream = GetMemoryStream(html, null); CsqWebRequest request = new CsqWebRequest("http://test.com", creator); var dom = ProcessMockWebRequestSync(request); Assert.AreNotEqual(arabicExpected, dom["h1"].Text()); // contains xml UTF8 and inline ISO encoding html = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + html; creator.ResponseStream = GetMemoryStream(html, null); request = new CsqWebRequest("http://test.com", creator); dom = ProcessMockWebRequestSync(request); Assert.AreEqual(arabicExpected, dom["h1"].Text()); }
public static CQ CreateFromUrl(string url) { CsqWebRequest con = new CsqWebRequest(url); con.Get(); con.UserAgent = UserAgent; return(CQ.Create(con.Html)); }
/// <summary> /// Process the mock web request synchronously using same rules as CreateFromUrl /// </summary> /// /// <param name="request"> /// The request. /// </param> /// /// <returns> /// . /// </returns> private CQ ProcessMockWebRequestSync(CsqWebRequest request) { var httpRequest = request.GetWebRequest(); var response = httpRequest.GetResponse(); var responseStream = response.GetResponseStream(); var encoding = CsqWebRequest.GetEncoding(response); return(CQ.CreateDocument(responseStream, encoding)); }
/// <summary> /// Creates a new DOM from an HTML file. /// </summary> /// /// <param name="url"> /// The URL of the remote server. /// </param> /// <param name="options"> /// The options to use when creating the reqest. /// </param> /// /// <returns> /// A CQ object composed from the HTML response from the server. /// </returns> public static CQ CreateFromUrl(string url, ServerConfig options = null) { CsqWebRequest request = new CsqWebRequest(url); ServerConfig.Apply(options, request); request.Get(); return(CQ.CreateDocument(request.Html)); }
public static void StartAsyncWebRequest(string url, Action <ICsqWebResponse> callback, object id = null) { var request = new CsqWebRequest(url); request.Id = id; request.Async = true; var mrEvent = request.GetAsync(callback); AsyncEvents.Add(mrEvent); }
/// <summary> /// Creates a new DOM from an HTML file. /// </summary> /// /// <param name="url"> /// The URL of the remote server. /// </param> /// <param name="options"> /// The options to use when creating the reqest. /// </param> /// /// <returns> /// A CQ object composed from the HTML response from the server. /// </returns> public static CQ CreateFromUrl(string url, ServerConfig options = null) { CsqWebRequest request = new CsqWebRequest(url); request.Options = options; var httpRequest = request.GetWebRequest(); var response = httpRequest.GetResponse(); var responseStream = response.GetResponseStream(); var encoding = CsqWebRequest.GetEncoding(response); return(CQ.CreateDocument(responseStream, encoding)); }
public void Utf8NoContentType() { var creator = new Mocks.MockWebRequestCreator(); creator.CharacterSet = "ISO-8859-1"; creator.ResponseStream = GetMemoryStream(TestHtml("arabic"), new UTF8Encoding(false)); CsqWebRequest request = new CsqWebRequest("http://test.com", creator); // remove the content type header var html = ReplaceCharacterSet(request.Get()); var dom = CQ.CreateDocument(html); Assert.AreNotEqual(arabicExpected, dom["h1"].Text()); //test synchronous: this is the code that CreateFromURL uses creator.CharacterSet = null; request = new CsqWebRequest("http://test.com", creator); var httpRequest = request.GetWebRequest(); var response = httpRequest.GetResponse(); var responseStream = response.GetResponseStream(); var encoding = CsqWebRequest.GetEncoding(response); var dom2 = CQ.CreateDocument(responseStream, encoding); Assert.AreEqual(arabicExpected, dom2["h1"].Text()); // Test async version now request = new CsqWebRequest("http://test.com", creator); bool?done = null; CQ dom3 = null; request.GetAsync((r) => { dom3 = r.Dom; done = true; }, (r) => { done = false; }); while (done == null) { ; } Assert.IsTrue((bool)done); Assert.AreEqual(arabicExpected, dom3["h1"].Text()); }
public void GetHtmlAsyncTimeout() { var creator = new Mocks.MockWebRequestCreator(); creator.ResponseTime = 1000; creator.ResponseStream = GetMemoryStream("<div>Test</div>"); var request = new CsqWebRequest("http://test.com", creator); request.Options = new ServerConfig { Timeout = TimeSpan.FromMilliseconds(500), UserAgent = "test" }; bool?done = null; request.GetAsync((r) => { done = true; }, (r) => { done = false; }); while (done == null) { ; } Assert.IsFalse((bool)done); creator.ResponseTime = 300; request = new CsqWebRequest("http://test.com", creator); done = null; request.GetAsync((r) => { done = true; }, (r) => { done = false; }); while (done == null) { ; } Assert.IsTrue((bool)done); }
public void MismatchedContentTypeHeaderAndBOM() { var creator = new Mocks.MockWebRequestCreator(); creator.CharacterSet = "ISO-8859-1"; // response stream has UTF8 BOM; the latin encoding should be ignored. var html = ReplaceCharacterSet(TestHtml("arabic")); creator.ResponseStream = GetMemoryStream(html, new UTF8Encoding(true)); CsqWebRequest request = new CsqWebRequest("http://test.com", creator); var dom = ProcessMockWebRequestSync(request); Assert.AreEqual(arabicExpected, dom["h1"].Text()); }
public void ContentTypeHeader() { var creator = new Mocks.MockWebRequestCreator(); creator.CharacterSet = "windows-1255"; creator.ResponseStream = GetMemoryStream(htmlStart + htmlStart3 + hebrewChar + htmlEnd, Encoding.GetEncoding("windows-1255")); CsqWebRequest request = new CsqWebRequest("http://test.com", creator); var dom1 = CQ.Create(request.Get()); creator.CharacterSet = ""; request = new CsqWebRequest("http://test.com", creator); var dom2 = CQ.Create(request.Get()); var output = dom1.Render(OutputFormatters.HtmlEncodingMinimum); // The characters should be encoded differently. var outputHebrewChar = dom1["#test"].Text(); var outputUTF8Char = dom2["#test"].Text(); Assert.AreNotEqual(outputHebrewChar, outputUTF8Char); // try it again, using the meta tag creator.CharacterSet = "windows-1255"; creator.ResponseStream = GetMemoryStream(htmlStart + htmlStartMeta + htmlStart3 + hebrewChar + htmlEnd, Encoding.GetEncoding("windows-1255")); /// CreateFromUrl process request = new CsqWebRequest("http://test.com", creator); var httpRequest = request.GetWebRequest(); var response = httpRequest.GetResponse(); var responseStream = response.GetResponseStream(); var encoding = CsqWebRequest.GetEncoding(response); var dom3 = CQ.CreateDocument(responseStream, encoding); var outputHebrewChar2 = dom3["#test"].Text(); Assert.AreEqual(outputHebrewChar, outputHebrewChar2); }
public void GetHtmlTimeout() { var creator = new Mocks.MockWebRequestCreator(); creator.ResponseTime = 1000; creator.ResponseStream = GetMemoryStream("<div>Test</div>"); var request = new CsqWebRequest("http://test.com", creator); request.Options = new ServerConfig { Timeout = TimeSpan.FromMilliseconds(500), UserAgent = "test" }; var httpRequest = request.GetWebRequest(); IHttpWebResponse response; Assert.Throws <System.Net.WebException>(() => { response = httpRequest.GetResponse(); }); request.Timeout = 1500; httpRequest = request.GetWebRequest(); response = httpRequest.GetResponse(); var responseStream = response.GetResponseStream(); var encoding = CsqWebRequest.GetEncoding(response); var dom = CQ.CreateDocument(responseStream, encoding); Assert.AreEqual(1, dom["div"].Length); }