private static async Task TestCrawler()
{
Dictionary <string, string> dicParams = new Dictionary <string, string>
{
{ "date", "2018-01-10" },
{ "exchange", "sz" },
{ "stockcode", "300666" }
};
CommonCrawler commonCrawler = new CommonCrawler("CJMX", dicParams);
CommonCrawler commonCrawler2 = new CommonCrawler("CJMX");
var result = commonCrawler.ProcessCrawling();
for (int i = 0; i < 1000; i++)
{
CommonCrawler loopCrawler = new CommonCrawler("CJMX", dicParams);
Task <Stream> resultAsync = loopCrawler.ProcessCrawlingAsync();
Console.WriteLine(string.Format("current loop number:{1}, Crawling:{0} , ", loopCrawler.CrawlerRequestConfig.RequestUrl, i.ToString()));
var ret = resultAsync.Result;
Console.WriteLine(string.Format("current loop number:{1}, Crawling:{0} , ", loopCrawler.CrawlerRequestConfig.RequestUrl, i.ToString()));
if (ret != null)
{
Console.WriteLine(string.Format("current loop number:{0}, Response is NOT Null. ", i.ToString()));
Console.WriteLine(commonCrawler.GetResponseContent(ret as Stream, "text/html").ToString());
}
else
{
Console.WriteLine(string.Format("current loop number:{0}, Response is Null. ", i.ToString()));
}
}
}
public async Task CommonCrawlerTest()
{
Dictionary <string, string> dicParams = new Dictionary <string, string>
{
{ "date", "2018-01-10" },
{ "exchange", "sz" },
{ "stockcode", "300666" }
};
CommonCrawler commonCrawler = new CommonCrawler("CJMX", dicParams);
Assert.IsNotNull(commonCrawler);
CommonCrawler commonCrawler2 = new CommonCrawler("CJMX");
Assert.IsNotNull(commonCrawler2);
var result = commonCrawler.ProcessCrawling();
Assert.IsNotNull(result);
for (int i = 0; i < 10; i++)
{
CommonCrawler loopCrawler = new CommonCrawler("CJMX", dicParams);
Task <Stream> resultAsync = loopCrawler.ProcessCrawlingAsync();
Console.WriteLine(string.Format("current loop number:{1}, Crawling:{0} , ", loopCrawler.CrawlerRequestConfig.RequestUrl, i.ToString()));
var ret = await resultAsync;
Console.WriteLine(string.Format("current loop number:{1}, Crawling:{0} , ", loopCrawler.CrawlerRequestConfig.RequestUrl, i.ToString()));
if (ret != null)
{
Console.WriteLine(string.Format("current loop number:{0}, Response is not Null.", i.ToString()));
// string content = commonCrawler.GetResponseContent(ret as Stream, "text/html").ToString();
using (StreamReader sr = new StreamReader(ret))
{
string ss = sr.ReadToEnd();
Console.Write(sr);
}
}
else
{
Console.WriteLine(string.Format("current loop number:{0}, Response is Null.", i.ToString()));
}
}
}
