public void GetHtmlAsyncPromises()
{
bool p1resolved = false;
bool p2resolved = false;
var promise1 = CQ.CreateFromUrlAsync("http://www.microsoft.com/en/us/default.aspx?redir=true")
.Then(new Action <ICsqWebResponse>((resp) => {
Assert.IsTrue(resp.Dom.Find(".hpMst_Stage").Length == 1, "I found an expected content container - if MS changed their web site this could fail.");
p1resolved = true;
}));
var promise2 = CQ.CreateFromUrlAsync("http://www.cnn.com/").Then(new Action <ICsqWebResponse>((resp) =>
{
Assert.IsTrue(resp.Dom.Find("#cnn_hdr").Length == 1, "I found an expected content container - if CNN changed their web site this could fail.");
p2resolved = true;
}));
bool complete = false;
CQ.WhenAll(promise1, promise2).Then(new Action <ICsqWebResponse>((response) =>
{
Assert.IsTrue(p1resolved, "Promise 1 is resolved");
Assert.IsTrue(p2resolved, "Promise 1 is resolved");
complete = true;
}), new Action(() =>
{
Assert.Fail("The web requests were rejected.");
}));
// if we don't do this the test will exit before finishing
CQ.WaitForAsyncEvents(10000);
Assert.IsTrue(complete);
}
public void GetHtmlAsyncPromises()
{
bool p1resolved = false;
bool p2resolved = false;
var promise1 = CQ.CreateFromUrlAsync(urls[0].Key)
.Then(new Action <ICsqWebResponse>((resp) => {
Assert.IsTrue(resp.Dom.Find(urls[0].Value).Length > 0, "I found an expected content container - if MS changed their web site this could fail.");
p1resolved = true;
}));
var promise2 = CQ.CreateFromUrlAsync(urls[1].Key).Then(new Action <ICsqWebResponse>((resp) =>
{
Assert.IsTrue(resp.Dom.Find(urls[1].Value).Length > 0, "I found an expected content container - if CNN changed their web site this could fail.");
p2resolved = true;
}));
bool complete = false;
CQ.WhenAll(promise1, promise2).Then(new Action <ICsqWebResponse>((response) =>
{
Assert.IsTrue(p1resolved, "Promise 1 is resolved");
Assert.IsTrue(p2resolved, "Promise 1 is resolved");
complete = true;
}), new Action(() =>
{
Assert.Fail("The web requests were rejected.");
}));
// if we don't do this the test will exit before finishing
CQ.WaitForAsyncEvents(10000);
Assert.IsTrue(complete);
}
static void Main(string[] args)
{
var stringBuilder = new StringBuilder();
var url = "http://24.173.220.131/carter/currentinmates.aspx";
CQ.CreateFromUrlAsync(url)
.Then(response =>
{
var dom = response.Dom;
var trs = dom.Select("#dgrdLandRecords tr").Elements;
foreach (var row in trs)
{
stringBuilder.AppendLine();
var tds = row.ChildElements.ToList();
for (int i = 1; i < tds.Count; i++)
{
stringBuilder.Append(tds[i].Cq().Text());
stringBuilder.Append("|");
}
}
var result = stringBuilder.ToString();
Console.Write(result);
});
Console.WriteLine("Press any key to exit.");
Console.ReadKey();
}
public static IObservable <CQ> CreateCrawlerForUri(this ICrawler source, Uri uri, int retry = 3, IScheduler scheduler = null)
{
var crawl = Observable.Defer(() => CQ.CreateFromUrlAsync(uri.AbsoluteUri).ToObservable(scheduler))
.Retry(retry)
.Select(webResponse => webResponse.Dom);
return(crawl);
}
public static void Test()
{
var r = CQ.CreateFromUrlAsync("http://www.xxia.com/site/flight/departFlights1/0/T0",
success =>
{
},
fail =>
{
});
}
public void RandomUrls()
{
ServerConfig options = new ServerConfig {
TimeoutSeconds = 5
};
int outer = totalTests / simultaneousThreads;
var list = GetRandomUrls(totalTests);
List <string> successList = new List <string>();
List <string> failList = new List <string>();
int cur = 0;
for (int i = 0; i < outer; i++)
{
IPromise[] promises = new IPromise[simultaneousThreads];
List <string> active = new List <string>();
for (int j = 0; j < simultaneousThreads; j++)
{
string url = list[cur++];
active.Add(url);
promises[j] = CQ.CreateFromUrlAsync(url, options).Then((success) =>
{
successList.Add(FormatResult(success));
active.Remove(success.Url);
}, (fail) => {
failList.Add(FormatResult(fail));
active.Remove(fail.Url);
});
}
if (!AsyncWebRequestManager.WaitForAsyncEvents(10000))
{
AsyncWebRequestManager.CancelAsyncEvents();
foreach (var item in active)
{
failList.Add(item + ": aborted");
}
}
}
Debug.WriteLine(FormatTestOutput(successList, failList));
}
public void GetHtmlAsync()
{
CQ.CreateFromUrlAsync(urls[0].Key, 1, FinishRequest);
Debug.WriteLine("Started Async Request 1 @" + DateTime.Now);
AsyncStep |= 1;
CQ.CreateFromUrlAsync(urls[1].Key, 2, FinishRequest);
Debug.WriteLine("Started Async Request 2 @" + DateTime.Now);
AsyncStep |= 2;
// Time out after 15 seconds
CQ.WaitForAsyncEvents(15000);
AsyncStep |= 4;
Debug.WriteLine("Finished Test @" + DateTime.Now);
Assert.AreEqual(31, AsyncStep, "All async steps finished before exiting.");
}
public void GetHtmlAsync()
{
CQ.CreateFromUrlAsync("http://www.microsoft.com/en/us/default.aspx?redir=true", 1, FinishRequest);
Debug.WriteLine("Started Async Request 1 @" + DateTime.Now);
AsyncStep |= 1;
CQ.CreateFromUrlAsync("http://www.cnn.com/", 2, FinishRequest);
Debug.WriteLine("Started Async Request 2 @" + DateTime.Now);
AsyncStep |= 2;
// Time out after 15 seconds
CQ.WaitForAsyncEvents(15000);
AsyncStep |= 4;
Debug.WriteLine("Finished Test @" + DateTime.Now);
Assert.AreEqual(31, AsyncStep, "All async steps finished before exiting.");
}
public void GetHtmlAsyncPromisesFailed()
{
bool p1resolved = false;
bool p2rejected = false;
var promise1 = CQ.CreateFromUrlAsync(urls[0].Key)
.Then(resp =>
{
Assert.IsTrue(resp.Dom.Find(urls[0].Value).Length > 0, "I found an expected content container - if MS changed their web site this could fail.");
p1resolved = true;
});
var promise2 = CQ.CreateFromUrlAsync("http://www.bad-domain.zzyzx/").Then(null,
resp =>
{
p2rejected = true;
});
bool complete = false;
CQ.WhenAll(promise1, promise2).Then(response =>
{
Assert.Fail("This should have been rejected");
}, response =>
{
Assert.IsTrue(p1resolved, "Promise 1 is resolved");
Assert.IsTrue(p2rejected, "Promise 2 is rejected");
complete = true;
});
// if we don't do this the test will exit before finishing
CQ.WaitForAsyncEvents(10000);
Assert.IsTrue(complete, "Complete flag was set properly.");
}