public void GetHtmlTimeout()
{
var creator = new Mocks.MockWebRequestCreator();
creator.ResponseTime = 1000;
creator.ResponseStream = GetMemoryStream("<div>Test</div>");
var request = new CsqWebRequest("http://test.com", creator);
request.Options = new ServerConfig
{
Timeout = TimeSpan.FromMilliseconds(500),
UserAgent = "test"
};
var httpRequest = request.GetWebRequest();
IHttpWebResponse response;
Assert.Throws<System.Net.WebException>(() =>
{
response = httpRequest.GetResponse();
});
request.Timeout = 1500;
httpRequest = request.GetWebRequest();
response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom = CQ.CreateDocument(responseStream, encoding);
Assert.AreEqual(1, dom["div"].Length);
}
public static CQ CreateFromUrl(string url)
{
CsqWebRequest con = new CsqWebRequest(url);
con.Get();
con.UserAgent = UserAgent;
return CQ.Create(con.Html);
}
/// <summary>
/// Creates a new DOM from an HTML file.
/// </summary>
///
/// <param name="url">
/// The URL of the remote server.
/// </param>
/// <param name="options">
/// The options to use when creating the reqest.
/// </param>
///
/// <returns>
/// A CQ object composed from the HTML response from the server.
/// </returns>
public static CQ CreateFromUrl(string url, ServerConfig options=null)
{
CsqWebRequest request = new CsqWebRequest(url);
ServerConfig.Apply(options, request);
request.Get();
return CQ.CreateDocument(request.Html);
}
/// <summary>
/// Apply these options to a web request.
/// </summary>
///
/// <param name="options">
/// The options.
/// </param>
/// <param name="request">
/// The CsqWebRequest object to apply the options to.
/// </param>
public static void Apply(ServerConfig options, CsqWebRequest request)
{
var opts = Merge(options);
if (opts.Timeout != null)
{
request.Timeout = (int)Math.Floor(opts.Timeout.TotalMilliseconds);
}
if (opts.UserAgent != null)
{
request.UserAgent = opts.UserAgent;
}
}
/// <summary>
/// Apply these options to a web request.
/// </summary>
///
/// <param name="options">
/// The options.
/// </param>
/// <param name="request">
/// The CsqWebRequest object to apply the options to.
/// </param>
public static void Apply(ServerConfig options, CsqWebRequest request)
{
var opts = Merge(options);
if (opts.Timeout != null)
{
request.Timeout = (int)Math.Floor(opts.Timeout.TotalMilliseconds);
}
if (opts.UserAgent != null)
{
request.UserAgent = opts.UserAgent;
}
}
/// <summary>
/// Creates a new DOM from an HTML file.
/// </summary>
///
/// <param name="url">
/// The URL of the remote server.
/// </param>
/// <param name="options">
/// The options to use when creating the reqest.
/// </param>
///
/// <returns>
/// A CQ object composed from the HTML response from the server.
/// </returns>
public static CQ CreateFromUrl(string url, ServerConfig options=null)
{
CsqWebRequest request = new CsqWebRequest(url);
request.Options = options;
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
return CQ.CreateDocument(responseStream,encoding);
}
/// <summary>
/// Start an async request, and return a unique ID that identifies it.
/// </summary>
///
/// <param name="url">
/// The URL of the remote server.
/// </param>
/// <param name="success">
/// A delegate to invoke upon successful completion of the request.
/// </param>
/// <param name="fail">
/// A delegate to invoke when a request fails.
/// </param>
/// <param name="id">
/// The identifier.
/// </param>
/// <param name="options">
/// Options to be used when creating this request. If not provided, the default options will be
/// used.
/// </param>
public static void StartAsyncWebRequest(string url, Action <ICsqWebResponse> success, Action <ICsqWebResponse> fail, int id, ServerConfig options = null)
{
var request = new CsqWebRequest(url);
ServerConfig.Apply(options, request);
request.Id = id;
request.Async = true;
var mrEvent = request.GetAsync(success, fail);
AsyncEvents.Add(mrEvent);
}
private void RespCallback(IAsyncResult ar)
{
// Get the RequestState object from the async result.
WebRequestState rs = (WebRequestState)ar.AsyncState;
// Get the WebRequest from RequestState.
IHttpWebRequest req = rs.Request;
req.Timeout = Timeout;
req.Headers["UserAgent"] = UserAgent;
try
{
// Call EndGetResponse, which produces the WebResponse object
// that came from the request issued above.
Response = (IHttpWebResponse)req.EndGetResponse(ar);
}
catch (WebException e)
{
Complete = true;
Success = false;
WebException = e;
if (CallbackFail != null)
{
CallbackFail(this);
}
allDone.Set();
return;
}
ResponseEncoding = CsqWebRequest.GetEncoding(Response);
// Start reading data from the response stream.
Stream responseStream = Response.GetResponseStream();
// Store the response stream in RequestState to read
// the stream asynchronously.
rs.ResponseStreamAsync = responseStream;
ResponseStream = new MemoryStream();
// Pass rs.BufferRead to BeginRead. Read data into rs.BufferRead
IAsyncResult iarRead = responseStream.BeginRead(rs.BufferRead, 0,
BUFFER_SIZE, new AsyncCallback(ReadCallBack), rs);
}
public void GetHtmlAsyncTimeout()
{
var creator = new Mocks.MockWebRequestCreator();
creator.ResponseTime = 1000;
creator.ResponseStream = GetMemoryStream("<div>Test</div>");
var request = new CsqWebRequest("http://test.com", creator);
request.Options = new ServerConfig
{
Timeout = TimeSpan.FromMilliseconds(500),
UserAgent = "test"
};
bool? done = null;
request.GetAsync((r) =>
{
done = true;
}, (r) =>
{
done = false;
});
while (done == null) ;
Assert.IsFalse((bool)done);
creator.ResponseTime = 300;
request = new CsqWebRequest("http://test.com", creator);
done = null;
request.GetAsync((r) =>
{
done = true;
}, (r) =>
{
done = false;
});
while (done == null) ;
Assert.IsTrue((bool)done);
}
public void Utf8NoContentType()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "ISO-8859-1";
creator.ResponseStream = GetMemoryStream(TestHtml("arabic"), new UTF8Encoding(false));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
// remove the content type header
var html = request.Get();
var start = html.IndexOf(@"<meta http-equiv=""Content-Type""");
var end = html.IndexOf(">",start);
html = html.Substring(0,start)+html.Substring(end+1);
var dom = CQ.CreateDocument(html);
var expected = @"البابا: اوقفوا ""المجزرة"" في سوريا قبل ان تتحول البلاد الى ""أطلال""";
Assert.AreNotEqual(expected, dom["h1"].Text());
//test synchronous: this is the code that CreateFromURL uses
request = new CsqWebRequest("http://test.com", creator);
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom2 = CQ.CreateDocument(responseStream, encoding);
Assert.AreEqual(expected, dom2["h1"].Text());
// Test async version now
request = new CsqWebRequest("http://test.com", creator);
bool? done=null;
CQ dom3=null;
request.GetAsync((r) =>
{
dom3 = r.Dom;
done = true;
}, (r)=>{
done = false;
});
while (done == null) ;
Assert.IsTrue((bool)done);
Assert.AreEqual(expected, dom3["h1"].Text());
}
public void ContentTypeHeader()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "windows-1255";
creator.ResponseHTML = htmlStart + htmlStart3 + hebrewChar + htmlEnd;
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom1 = CQ.Create(request.Get());
creator.CharacterSet = "";
request = new CsqWebRequest("http://test.com", creator);
var dom2 = CQ.Create(request.Get());
var output = dom1.Render(OutputFormatters.HtmlEncodingMinimum);
// The characters should be encoded differently.
var outputHebrewChar = dom1["#test"].Text();
var outputUTF8Char = dom2["#test"].Text();
Assert.AreNotEqual(outputHebrewChar, outputUTF8Char);
// try it again, using the meta tag
creator.CharacterSet = "windows-1255";
creator.ResponseHTML = htmlStart + htmlStartMeta + htmlStart3 + hebrewChar + htmlEnd;
request = new CsqWebRequest("http://test.com", creator);
var dom3 = CQ.Create(request.Get());
var outputHebrewChar2 = dom3["#test"].Text();
Assert.AreEqual(outputHebrewChar, outputHebrewChar2);
}
/// <summary>
/// Start an async request, and return a unique ID that identifies it.
/// </summary>
///
/// <param name="url">
/// The URL of the remote server.
/// </param>
/// <param name="success">
/// A delegate to invoke upon successful completion of the request.
/// </param>
/// <param name="fail">
/// A delegate to invoke when a request fails.
/// </param>
/// <param name="id">
/// The identifier.
/// </param>
/// <param name="options">
/// Options to be used when creating this request. If not provided, the default options will be
/// used.
/// </param>
public static void StartAsyncWebRequest(string url, Action<ICsqWebResponse> success, Action<ICsqWebResponse> fail, int id, ServerConfig options = null)
{
var request = new CsqWebRequest(url);
ServerConfig.Apply(options, request);
request.Id = id;
request.Async = true;
var mrEvent = request.GetAsync(success, fail);
AsyncEvents.Add(mrEvent);
}
public static void StartAsyncWebRequest(string url, Action<ICsqWebResponse> callback, object id=null)
{
var request = new CsqWebRequest(url);
request.Id = id;
request.Async = true;
var mrEvent = request.GetAsync(callback);
AsyncEvents.Add(mrEvent);
}
public void ContentTypeHeader()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "windows-1255";
creator.ResponseStream = GetMemoryStream(htmlStart + htmlStart3 + hebrewChar + htmlEnd, Encoding.GetEncoding("windows-1255"));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom1 = CQ.Create(request.Get());
creator.CharacterSet = "";
request = new CsqWebRequest("http://test.com", creator);
var dom2 = CQ.Create(request.Get());
var output = dom1.Render(OutputFormatters.HtmlEncodingMinimum);
// The characters should be encoded differently.
var outputHebrewChar = dom1["#test"].Text();
var outputUTF8Char = dom2["#test"].Text();
Assert.AreNotEqual(outputHebrewChar, outputUTF8Char);
// try it again, using the meta tag
creator.CharacterSet = "windows-1255";
creator.ResponseStream = GetMemoryStream(htmlStart + htmlStartMeta + htmlStart3 + hebrewChar + htmlEnd, Encoding.GetEncoding("windows-1255"));
/// CreateFromUrl process
request = new CsqWebRequest("http://test.com", creator);
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom3 = CQ.CreateDocument(responseStream, encoding);
var outputHebrewChar2 = dom3["#test"].Text();
Assert.AreEqual(outputHebrewChar, outputHebrewChar2);
}
public void Utf8NoContentType()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "ISO-8859-1";
creator.ResponseStream = GetMemoryStream(TestHtml("arabic"), new UTF8Encoding(false));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
// remove the content type header
var html = ReplaceCharacterSet(request.Get());
var dom = CQ.CreateDocument(html);
Assert.AreNotEqual(arabicExpected, dom["h1"].Text());
//test synchronous: this is the code that CreateFromURL uses
creator.CharacterSet = null;
request = new CsqWebRequest("http://test.com", creator);
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
var dom2 = CQ.CreateDocument(responseStream, encoding);
Assert.AreEqual(arabicExpected, dom2["h1"].Text());
// Test async version now
request = new CsqWebRequest("http://test.com", creator);
bool? done=null;
CQ dom3=null;
request.GetAsync((r) =>
{
dom3 = r.Dom;
done = true;
}, (r)=>{
done = false;
});
while (done == null) ;
Assert.IsTrue((bool)done);
Assert.AreEqual(arabicExpected, dom3["h1"].Text());
}
public void MismatchedContentTypeHeaderAndBOM()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = "ISO-8859-1";
// response stream has UTF8 BOM; the latin encoding should be ignored.
var html = ReplaceCharacterSet(TestHtml("arabic"));
creator.ResponseStream = GetMemoryStream(html, new UTF8Encoding(true));
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom = ProcessMockWebRequestSync(request);
Assert.AreEqual(arabicExpected, dom["h1"].Text());
}
public void XmlEncodingDeclaration()
{
var creator = new Mocks.MockWebRequestCreator();
creator.CharacterSet = null;
var html = ReplaceCharacterSet(TestHtml("arabic"),"ISO-8859-1");
creator.ResponseStream = GetMemoryStream(html, null);
CsqWebRequest request = new CsqWebRequest("http://test.com", creator);
var dom = ProcessMockWebRequestSync(request);
Assert.AreNotEqual(arabicExpected, dom["h1"].Text());
// contains xml UTF8 and inline ISO encoding
html = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>" + html;
creator.ResponseStream = GetMemoryStream(html, null);
request = new CsqWebRequest("http://test.com", creator);
dom = ProcessMockWebRequestSync(request);
Assert.AreEqual(arabicExpected, dom["h1"].Text());
}
/// <summary>
/// Process the mock web request synchronously using same rules as CreateFromUrl
/// </summary>
///
/// <param name="request">
/// The request.
/// </param>
///
/// <returns>
/// .
/// </returns>
private CQ ProcessMockWebRequestSync(CsqWebRequest request)
{
var httpRequest = request.GetWebRequest();
var response = httpRequest.GetResponse();
var responseStream = response.GetResponseStream();
var encoding = CsqWebRequest.GetEncoding(response);
return CQ.CreateDocument(responseStream, encoding);
}
