public void GetHtmlTimeout()
{
var creator = new Mocks.MockWebRequestCreator();
creator.ResponseTime = 1000;
creator.ResponseStream = GetMemoryStream("<div>Test</div>");
var request = new CsqWebRequest("http://test.com", creator);
request.Options = new ServerConfig
{
Timeout = TimeSpan.FromMilliseconds(500),
UserAgent = "test"
};
var httpRequest = request.GetWebRequest();
IHttpWebResponse response;
Assert.Throws<System.Net.WebException>(() =>
{
response = httpRequest.GetResponse();
});
request.Timeout = 1500;
httpRequest = request.GetWebRequest();
response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom = CQ.CreateDocument(responseStream, encoding);
Assert.AreEqual(1, dom["div"].Length);
}
public void XmlEncodingDeclaration()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = null;
var html = ReplaceCharacterSet(TestHtml("arabic"), "ISO-8859-1");
creator.ResponseStream = GetMemoryStream(html, null);
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom = ProcessMockWebRequestSync(request);
Assert.AreNotEqual(arabicExpected, dom["h1"].Text());
// contains xml UTF8 and inline ISO encoding
html = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + html;
creator.ResponseStream = GetMemoryStream(html, null);
request = new CsqWebRequest("http://test.com", creator);
dom = ProcessMockWebRequestSync(request);
Assert.AreEqual(arabicExpected, dom["h1"].Text());
}
public void Utf8NoContentType()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "ISO-8859-1";
creator.ResponseStream = GetMemoryStream(TestHtml("arabic"), new UTF8Encoding(false));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
// remove the content type header
var html = ReplaceCharacterSet(request.Get());
var dom = CQ.CreateDocument(html);
Assert.AreNotEqual(arabicExpected, dom["h1"].Text());
//test synchronous: this is the code that CreateFromURL uses
creator.CharacterSet = null;
request = new CsqWebRequest("http://test.com", creator);
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom2 = CQ.CreateDocument(responseStream, encoding);
Assert.AreEqual(arabicExpected, dom2["h1"].Text());
// Test async version now
request = new CsqWebRequest("http://test.com", creator);
bool?done = null;
CQ dom3 = null;
request.GetAsync((r) =>
{
dom3 = r.Dom;
done = true;
}, (r) => {
done = false;
});
while (done == null)
{
;
}
Assert.IsTrue((bool)done);
Assert.AreEqual(arabicExpected, dom3["h1"].Text());
}
public void GetHtmlAsyncTimeout()
{
var creator = new Mocks.MockWebRequestCreator();
creator.ResponseTime = 1000;
creator.ResponseStream = GetMemoryStream("<div>Test</div>");
var request = new CsqWebRequest("http://test.com", creator);
request.Options = new ServerConfig
{
Timeout = TimeSpan.FromMilliseconds(500),
UserAgent = "test"
};
bool?done = null;
request.GetAsync((r) =>
{
done = true;
}, (r) =>
{
done = false;
});
while (done == null)
{
;
}
Assert.IsFalse((bool)done);
creator.ResponseTime = 300;
request = new CsqWebRequest("http://test.com", creator);
done = null;
request.GetAsync((r) =>
{
done = true;
}, (r) =>
{
done = false;
});
while (done == null)
{
;
}
Assert.IsTrue((bool)done);
}
public void MismatchedContentTypeHeaderAndBOM()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "ISO-8859-1";
// response stream has UTF8 BOM; the latin encoding should be ignored.
var html = ReplaceCharacterSet(TestHtml("arabic"));
creator.ResponseStream = GetMemoryStream(html, new UTF8Encoding(true));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom = ProcessMockWebRequestSync(request);
Assert.AreEqual(arabicExpected, dom["h1"].Text());
}
public void GetHtmlAsyncTimeout()
{
var creator = new Mocks.MockWebRequestCreator();
creator.ResponseTime = 1000;
creator.ResponseStream = GetMemoryStream("<div>Test</div>");
var request = new CsqWebRequest("http://test.com", creator);
request.Options = new ServerConfig
{
Timeout = TimeSpan.FromMilliseconds(500),
UserAgent = "test"
};
bool? done = null;
request.GetAsync((r) =>
{
done = true;
}, (r) =>
{
done = false;
});
while (done == null) ;
Assert.IsFalse((bool)done);
creator.ResponseTime = 300;
request = new CsqWebRequest("http://test.com", creator);
done = null;
request.GetAsync((r) =>
{
done = true;
}, (r) =>
{
done = false;
});
while (done == null) ;
Assert.IsTrue((bool)done);
}
public void ContentTypeHeader()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "windows-1255";
creator.ResponseStream = GetMemoryStream(htmlStart + htmlStart3 + hebrewChar + htmlEnd, Encoding.GetEncoding("windows-1255"));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom1 = CQ.Create(request.Get());
creator.CharacterSet = "";
request = new CsqWebRequest("http://test.com", creator);
var dom2 = CQ.Create(request.Get());
var output = dom1.Render(OutputFormatters.HtmlEncodingMinimum);
// The characters should be encoded differently.
var outputHebrewChar = dom1["#test"].Text();
var outputUTF8Char = dom2["#test"].Text();
Assert.AreNotEqual(outputHebrewChar, outputUTF8Char);
// try it again, using the meta tag
creator.CharacterSet = "windows-1255";
creator.ResponseStream = GetMemoryStream(htmlStart + htmlStartMeta + htmlStart3 + hebrewChar + htmlEnd, Encoding.GetEncoding("windows-1255"));
/// CreateFromUrl process
request = new CsqWebRequest("http://test.com", creator);
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom3 = CQ.CreateDocument(responseStream, encoding);
var outputHebrewChar2 = dom3["#test"].Text();
Assert.AreEqual(outputHebrewChar, outputHebrewChar2);
}
public void GetHtmlTimeout()
{
var creator = new Mocks.MockWebRequestCreator();
creator.ResponseTime = 1000;
creator.ResponseStream = GetMemoryStream("<div>Test</div>");
var request = new CsqWebRequest("http://test.com", creator);
request.Options = new ServerConfig
{
Timeout = TimeSpan.FromMilliseconds(500),
UserAgent = "test"
};
var httpRequest = request.GetWebRequest();
IHttpWebResponse response;
Assert.Throws <System.Net.WebException>(() =>
{
response = httpRequest.GetResponse();
});
request.Timeout = 1500;
httpRequest = request.GetWebRequest();
response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom = CQ.CreateDocument(responseStream, encoding);
Assert.AreEqual(1, dom["div"].Length);
}
public void ContentTypeHeader()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "windows-1255";
creator.ResponseHTML = htmlStart + htmlStart3 + hebrewChar + htmlEnd;
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom1 = CQ.Create(request.Get());
creator.CharacterSet = "";
request = new CsqWebRequest("http://test.com", creator);
var dom2 = CQ.Create(request.Get());
var output = dom1.Render(OutputFormatters.HtmlEncodingMinimum);
// The characters should be encoded differently.
var outputHebrewChar = dom1["#test"].Text();
var outputUTF8Char = dom2["#test"].Text();
Assert.AreNotEqual(outputHebrewChar, outputUTF8Char);
// try it again, using the meta tag
creator.CharacterSet = "windows-1255";
creator.ResponseHTML = htmlStart + htmlStartMeta + htmlStart3 + hebrewChar + htmlEnd;
request = new CsqWebRequest("http://test.com", creator);
var dom3 = CQ.Create(request.Get());
var outputHebrewChar2 = dom3["#test"].Text();
Assert.AreEqual(outputHebrewChar, outputHebrewChar2);
}
public void XmlEncodingDeclaration()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = null;
var html = ReplaceCharacterSet(TestHtml("arabic"),"ISO-8859-1");
creator.ResponseStream = GetMemoryStream(html, null);
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom = ProcessMockWebRequestSync(request);
Assert.AreNotEqual(arabicExpected, dom["h1"].Text());
// contains xml UTF8 and inline ISO encoding
html = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + html;
creator.ResponseStream = GetMemoryStream(html, null);
request = new CsqWebRequest("http://test.com", creator);
dom = ProcessMockWebRequestSync(request);
Assert.AreEqual(arabicExpected, dom["h1"].Text());
}
public void MismatchedContentTypeHeaderAndBOM()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "ISO-8859-1";
// response stream has UTF8 BOM; the latin encoding should be ignored.
var html = ReplaceCharacterSet(TestHtml("arabic"));
creator.ResponseStream = GetMemoryStream(html, new UTF8Encoding(true));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom = ProcessMockWebRequestSync(request);
Assert.AreEqual(arabicExpected, dom["h1"].Text());
}
public void Utf8NoContentType()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "ISO-8859-1";
creator.ResponseStream = GetMemoryStream(TestHtml("arabic"), new UTF8Encoding(false));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
// remove the content type header
var html = ReplaceCharacterSet(request.Get());
var dom = CQ.CreateDocument(html);
Assert.AreNotEqual(arabicExpected, dom["h1"].Text());
//test synchronous: this is the code that CreateFromURL uses
creator.CharacterSet = null;
request = new CsqWebRequest("http://test.com", creator);
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom2 = CQ.CreateDocument(responseStream, encoding);
Assert.AreEqual(arabicExpected, dom2["h1"].Text());
// Test async version now
request = new CsqWebRequest("http://test.com", creator);
bool? done=null;
CQ dom3=null;
request.GetAsync((r) =>
{
dom3 = r.Dom;
done = true;
}, (r)=>{
done = false;
});
while (done == null) ;
Assert.IsTrue((bool)done);
Assert.AreEqual(arabicExpected, dom3["h1"].Text());
}
public void ContentTypeHeader()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "windows-1255";
creator.ResponseStream = GetMemoryStream(htmlStart + htmlStart3 + hebrewChar + htmlEnd, Encoding.GetEncoding("windows-1255"));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom1 = CQ.Create(request.Get());
creator.CharacterSet = "";
request = new CsqWebRequest("http://test.com", creator);
var dom2 = CQ.Create(request.Get());
var output = dom1.Render(OutputFormatters.HtmlEncodingMinimum);
// The characters should be encoded differently.
var outputHebrewChar = dom1["#test"].Text();
var outputUTF8Char = dom2["#test"].Text();
Assert.AreNotEqual(outputHebrewChar, outputUTF8Char);
// try it again, using the meta tag
creator.CharacterSet = "windows-1255";
creator.ResponseStream = GetMemoryStream(htmlStart + htmlStartMeta + htmlStart3 + hebrewChar + htmlEnd, Encoding.GetEncoding("windows-1255"));
/// CreateFromUrl process
request = new CsqWebRequest("http://test.com", creator);
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom3 = CQ.CreateDocument(responseStream, encoding);
var outputHebrewChar2 = dom3["#test"].Text();
Assert.AreEqual(outputHebrewChar, outputHebrewChar2);
}
public void Utf8NoContentType()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "ISO-8859-1";
creator.ResponseStream = GetMemoryStream(TestHtml("arabic"), new UTF8Encoding(false));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
// remove the content type header
var html = request.Get();
var start = html.IndexOf(@"<meta http-equiv=""Content-Type""");
var end = html.IndexOf(">",start);
html = html.Substring(0,start)+html.Substring(end+1);
var dom = CQ.CreateDocument(html);
var expected = @"البابا: اوقفوا ""المجزرة"" في سوريا قبل ان تتحول البلاد الى ""أطلال""";
Assert.AreNotEqual(expected, dom["h1"].Text());
//test synchronous: this is the code that CreateFromURL uses
request = new CsqWebRequest("http://test.com", creator);
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom2 = CQ.CreateDocument(responseStream, encoding);
Assert.AreEqual(expected, dom2["h1"].Text());
// Test async version now
request = new CsqWebRequest("http://test.com", creator);
bool? done=null;
CQ dom3=null;
request.GetAsync((r) =>
{
dom3 = r.Dom;
done = true;
}, (r)=>{
done = false;
});
while (done == null) ;
Assert.IsTrue((bool)done);
Assert.AreEqual(expected, dom3["h1"].Text());
}