public void GetHtmlTimeout()
{
var creator = new Mocks.MockWebRequestCreator();
creator.ResponseTime = 1000;
creator.ResponseStream = GetMemoryStream("<div>Test</div>");
var request = new CsqWebRequest("http://test.com", creator);
request.Options = new ServerConfig
{
Timeout = TimeSpan.FromMilliseconds(500),
UserAgent = "test"
};
var httpRequest = request.GetWebRequest();
IHttpWebResponse response;
Assert.Throws<System.Net.WebException>(() =>
{
response = httpRequest.GetResponse();
});
request.Timeout = 1500;
httpRequest = request.GetWebRequest();
response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom = CQ.CreateDocument(responseStream, encoding);
Assert.AreEqual(1, dom["div"].Length);
}
/// <summary>
/// Creates a new DOM from an HTML file.
/// </summary>
///
/// <param name="url">
/// The URL of the remote server.
/// </param>
/// <param name="options">
/// The options to use when creating the reqest.
/// </param>
///
/// <returns>
/// A CQ object composed from the HTML response from the server.
/// </returns>
public static CQ CreateFromUrl(string url, ServerConfig options=null)
{
CsqWebRequest request = new CsqWebRequest(url);
request.Options = options;
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
return CQ.CreateDocument(responseStream,encoding);
}
/// <summary>
/// Process the mock web request synchronously using same rules as CreateFromUrl
/// </summary>
///
/// <param name="request">
/// The request.
/// </param>
///
/// <returns>
/// .
/// </returns>
private CQ ProcessMockWebRequestSync(CsqWebRequest request)
{
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
return CQ.CreateDocument(responseStream, encoding);
}
public void Utf8NoContentType()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "ISO-8859-1";
creator.ResponseStream = GetMemoryStream(TestHtml("arabic"), new UTF8Encoding(false));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
// remove the content type header
var html = ReplaceCharacterSet(request.Get());
var dom = CQ.CreateDocument(html);
Assert.AreNotEqual(arabicExpected, dom["h1"].Text());
//test synchronous: this is the code that CreateFromURL uses
creator.CharacterSet = null;
request = new CsqWebRequest("http://test.com", creator);
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom2 = CQ.CreateDocument(responseStream, encoding);
Assert.AreEqual(arabicExpected, dom2["h1"].Text());
// Test async version now
request = new CsqWebRequest("http://test.com", creator);
bool? done=null;
CQ dom3=null;
request.GetAsync((r) =>
{
dom3 = r.Dom;
done = true;
}, (r)=>{
done = false;
});
while (done == null) ;
Assert.IsTrue((bool)done);
Assert.AreEqual(arabicExpected, dom3["h1"].Text());
}
public void ContentTypeHeader()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "windows-1255";
creator.ResponseStream = GetMemoryStream(htmlStart + htmlStart3 + hebrewChar + htmlEnd, Encoding.GetEncoding("windows-1255"));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom1 = CQ.Create(request.Get());
creator.CharacterSet = "";
request = new CsqWebRequest("http://test.com", creator);
var dom2 = CQ.Create(request.Get());
var output = dom1.Render(OutputFormatters.HtmlEncodingMinimum);
// The characters should be encoded differently.
var outputHebrewChar = dom1["#test"].Text();
var outputUTF8Char = dom2["#test"].Text();
Assert.AreNotEqual(outputHebrewChar, outputUTF8Char);
// try it again, using the meta tag
creator.CharacterSet = "windows-1255";
creator.ResponseStream = GetMemoryStream(htmlStart + htmlStartMeta + htmlStart3 + hebrewChar + htmlEnd, Encoding.GetEncoding("windows-1255"));
/// CreateFromUrl process
request = new CsqWebRequest("http://test.com", creator);
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom3 = CQ.CreateDocument(responseStream, encoding);
var outputHebrewChar2 = dom3["#test"].Text();
Assert.AreEqual(outputHebrewChar, outputHebrewChar2);
}
public void Utf8NoContentType()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "ISO-8859-1";
creator.ResponseStream = GetMemoryStream(TestHtml("arabic"), new UTF8Encoding(false));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
// remove the content type header
var html = request.Get();
var start = html.IndexOf(@"<meta http-equiv=""Content-Type""");
var end = html.IndexOf(">",start);
html = html.Substring(0,start)+html.Substring(end+1);
var dom = CQ.CreateDocument(html);
var expected = @"البابا: اوقفوا ""المجزرة"" في سوريا قبل ان تتحول البلاد الى ""أطلال""";
Assert.AreNotEqual(expected, dom["h1"].Text());
//test synchronous: this is the code that CreateFromURL uses
request = new CsqWebRequest("http://test.com", creator);
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom2 = CQ.CreateDocument(responseStream, encoding);
Assert.AreEqual(expected, dom2["h1"].Text());
// Test async version now
request = new CsqWebRequest("http://test.com", creator);
bool? done=null;
CQ dom3=null;
request.GetAsync((r) =>
{
dom3 = r.Dom;
done = true;
}, (r)=>{
done = false;
});
while (done == null) ;
Assert.IsTrue((bool)done);
Assert.AreEqual(expected, dom3["h1"].Text());
}
