public void XmlEncodingDeclaration()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = null;
var html = ReplaceCharacterSet(TestHtml("arabic"), "ISO-8859-1");
creator.ResponseStream = GetMemoryStream(html, null);
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom = ProcessMockWebRequestSync(request);
Assert.AreNotEqual(arabicExpected, dom["h1"].Text());
// contains xml UTF8 and inline ISO encoding
html = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + html;
creator.ResponseStream = GetMemoryStream(html, null);
request = new CsqWebRequest("http://test.com", creator);
dom = ProcessMockWebRequestSync(request);
Assert.AreEqual(arabicExpected, dom["h1"].Text());
}
public static CQ CreateFromUrl(string url)
{
CsqWebRequest con = new CsqWebRequest(url);
con.Get();
con.UserAgent = UserAgent;
return(CQ.Create(con.Html));
}
/// <summary>
/// Process the mock web request synchronously using same rules as CreateFromUrl
/// </summary>
///
/// <param name="request">
/// The request.
/// </param>
///
/// <returns>
/// .
/// </returns>
private CQ ProcessMockWebRequestSync(CsqWebRequest request)
{
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
return(CQ.CreateDocument(responseStream, encoding));
}
/// <summary>
/// Creates a new DOM from an HTML file.
/// </summary>
///
/// <param name="url">
/// The URL of the remote server.
/// </param>
/// <param name="options">
/// The options to use when creating the reqest.
/// </param>
///
/// <returns>
/// A CQ object composed from the HTML response from the server.
/// </returns>
public static CQ CreateFromUrl(string url, ServerConfig options = null)
{
CsqWebRequest request = new CsqWebRequest(url);
ServerConfig.Apply(options, request);
request.Get();
return(CQ.CreateDocument(request.Html));
}
public static void StartAsyncWebRequest(string url, Action <ICsqWebResponse> callback, object id = null)
{
var request = new CsqWebRequest(url);
request.Id = id;
request.Async = true;
var mrEvent = request.GetAsync(callback);
AsyncEvents.Add(mrEvent);
}
/// <summary>
/// Creates a new DOM from an HTML file.
/// </summary>
///
/// <param name="url">
/// The URL of the remote server.
/// </param>
/// <param name="options">
/// The options to use when creating the reqest.
/// </param>
///
/// <returns>
/// A CQ object composed from the HTML response from the server.
/// </returns>
public static CQ CreateFromUrl(string url, ServerConfig options = null)
{
CsqWebRequest request = new CsqWebRequest(url);
request.Options = options;
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
return(CQ.CreateDocument(responseStream, encoding));
}
public void Utf8NoContentType()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "ISO-8859-1";
creator.ResponseStream = GetMemoryStream(TestHtml("arabic"), new UTF8Encoding(false));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
// remove the content type header
var html = ReplaceCharacterSet(request.Get());
var dom = CQ.CreateDocument(html);
Assert.AreNotEqual(arabicExpected, dom["h1"].Text());
//test synchronous: this is the code that CreateFromURL uses
creator.CharacterSet = null;
request = new CsqWebRequest("http://test.com", creator);
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom2 = CQ.CreateDocument(responseStream, encoding);
Assert.AreEqual(arabicExpected, dom2["h1"].Text());
// Test async version now
request = new CsqWebRequest("http://test.com", creator);
bool?done = null;
CQ dom3 = null;
request.GetAsync((r) =>
{
dom3 = r.Dom;
done = true;
}, (r) => {
done = false;
});
while (done == null)
{
;
}
Assert.IsTrue((bool)done);
Assert.AreEqual(arabicExpected, dom3["h1"].Text());
}
public void GetHtmlAsyncTimeout()
{
var creator = new Mocks.MockWebRequestCreator();
creator.ResponseTime = 1000;
creator.ResponseStream = GetMemoryStream("<div>Test</div>");
var request = new CsqWebRequest("http://test.com", creator);
request.Options = new ServerConfig
{
Timeout = TimeSpan.FromMilliseconds(500),
UserAgent = "test"
};
bool?done = null;
request.GetAsync((r) =>
{
done = true;
}, (r) =>
{
done = false;
});
while (done == null)
{
;
}
Assert.IsFalse((bool)done);
creator.ResponseTime = 300;
request = new CsqWebRequest("http://test.com", creator);
done = null;
request.GetAsync((r) =>
{
done = true;
}, (r) =>
{
done = false;
});
while (done == null)
{
;
}
Assert.IsTrue((bool)done);
}
public void MismatchedContentTypeHeaderAndBOM()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "ISO-8859-1";
// response stream has UTF8 BOM; the latin encoding should be ignored.
var html = ReplaceCharacterSet(TestHtml("arabic"));
creator.ResponseStream = GetMemoryStream(html, new UTF8Encoding(true));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom = ProcessMockWebRequestSync(request);
Assert.AreEqual(arabicExpected, dom["h1"].Text());
}
public void ContentTypeHeader()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "windows-1255";
creator.ResponseStream = GetMemoryStream(htmlStart + htmlStart3 + hebrewChar + htmlEnd, Encoding.GetEncoding("windows-1255"));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom1 = CQ.Create(request.Get());
creator.CharacterSet = "";
request = new CsqWebRequest("http://test.com", creator);
var dom2 = CQ.Create(request.Get());
var output = dom1.Render(OutputFormatters.HtmlEncodingMinimum);
// The characters should be encoded differently.
var outputHebrewChar = dom1["#test"].Text();
var outputUTF8Char = dom2["#test"].Text();
Assert.AreNotEqual(outputHebrewChar, outputUTF8Char);
// try it again, using the meta tag
creator.CharacterSet = "windows-1255";
creator.ResponseStream = GetMemoryStream(htmlStart + htmlStartMeta + htmlStart3 + hebrewChar + htmlEnd, Encoding.GetEncoding("windows-1255"));
/// CreateFromUrl process
request = new CsqWebRequest("http://test.com", creator);
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom3 = CQ.CreateDocument(responseStream, encoding);
var outputHebrewChar2 = dom3["#test"].Text();
Assert.AreEqual(outputHebrewChar, outputHebrewChar2);
}
public void GetHtmlTimeout()
{
var creator = new Mocks.MockWebRequestCreator();
creator.ResponseTime = 1000;
creator.ResponseStream = GetMemoryStream("<div>Test</div>");
var request = new CsqWebRequest("http://test.com", creator);
request.Options = new ServerConfig
{
Timeout = TimeSpan.FromMilliseconds(500),
UserAgent = "test"
};
var httpRequest = request.GetWebRequest();
IHttpWebResponse response;
Assert.Throws <System.Net.WebException>(() =>
{
response = httpRequest.GetResponse();
});
request.Timeout = 1500;
httpRequest = request.GetWebRequest();
response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom = CQ.CreateDocument(responseStream, encoding);
Assert.AreEqual(1, dom["div"].Length);
}