public void RedisSchedulerTest()
{
Shell.NScrapy scrapy = NScrapy.Shell.NScrapy.GetInstance();
NScrapyContext.CurrentContext.RefreshConfigFile("appsettingRedis.json");
scrapy.ConfigSerivces();
Shell.NScrapy.GetInstance().Crawl("JobSpider");
}
public void NScrapySuccessfullyExitTest()
{
var logPath = Path.Combine(Directory.GetCurrentDirectory(), "log-file.txt");
if (File.Exists(logPath))
{
File.Delete(logPath);
}
Shell.NScrapy scrapy = NScrapy.Shell.NScrapy.GetInstance();
scrapy.Crawl("UrlFilterTestSpider");
//Make a copy of log file since the original log file is still in using by log4net
var copiedLogFile = Path.Combine(Directory.GetCurrentDirectory(), "log-file.test.txt");
File.Copy(logPath, copiedLogFile);
var logFileContent = string.Empty;
using (FileStream stream = File.OpenRead(copiedLogFile))
{
using (var ms = new MemoryStream())
{
stream.CopyTo(ms);
logFileContent = Encoding.UTF8.GetString(ms.ToArray());
}
}
File.Delete(copiedLogFile);
Assert.IsTrue(logFileContent.Contains("https://www.liepin.com/zhaopin/?d_sfrom=search_fp_nvbar&init=1"));
}
public void PipelineTestItemLoaderValueSetTest2()
{
this.DeleteLog();
Shell.NScrapy scrapy = NScrapy.Shell.NScrapy.GetInstance();
Shell.NScrapy.GetInstance().Crawl("PipelineTestSpider");
var logContent = this.GetLogContent(Path.Combine(Directory.GetCurrentDirectory(), "log-file.txt"));
Assert.IsTrue(logContent.Contains("MockValue2 Mapped!"));
}
public void PipelineTest()
{
this.DeleteLog();
Shell.NScrapy scrapy = NScrapy.Shell.NScrapy.GetInstance();
Shell.NScrapy.GetInstance().Crawl("PipelineTestSpider");
var logContent = this.GetLogContent(Path.Combine(Directory.GetCurrentDirectory(), "log-file.txt"));
Assert.IsTrue(logContent.Contains("Mock Pipeline Processed, Mock Value=Hello World"));
Assert.IsTrue(logContent.Contains("MockValue Mapped!"));
}
public void DistributedDownloaderTest()
{
Shell.NScrapy scrapy = NScrapy.Shell.NScrapy.GetInstance();
NScrapyContext.CurrentContext.RefreshConfigFile("appsettingRedis.json");
Thread t = new Thread(() => DownloaderShell.Program.Main(null));
t.Start();
Shell.NScrapy.GetInstance().Crawl("JobSpider2");
//Let's first start the individual Downloader by a thread
}
public void NScrapySuccessfullyExitTest()
{
string logPath = DeleteLog();
Shell.NScrapy scrapy = NScrapy.Shell.NScrapy.GetInstance();
scrapy.Crawl("UrlFilterTestSpider");
//Make a copy of log file since the original log file is still in using by log4net
string logFileContent = GetLogContent(logPath);
Assert.IsTrue(logFileContent.Contains("https://www.liepin.com/zhaopin/?d_sfrom=search_fp_nvbar&init=1"));
}
public void UserAgentMiddlewareTest()
{
Shell.NScrapy scrapy = NScrapy.Shell.NScrapy.GetInstance();
NScrapyContext.CurrentContext.RefreshConfigFile("appsettingUserAgent.json");
var request = new HttpRequest()
{
URL = "http://www.sina.com"
};
var response = Downloader.Downloader.SendRequestAsync(request);
Assert.IsTrue(response.Result.ResponsePlanText != null);
}
public void ConstructDownloader()
{
var sw = Stopwatch.StartNew();
sw.Start();
//init context
Shell.NScrapy scrapy = NScrapy.Shell.NScrapy.GetInstance();
var request = new HttpRequest()
{
URL = "http://www.sina.com"
};
var request2 = new HttpRequest()
{
URL = "http://www.sina.com"
};
var request3 = new HttpRequest()
{
URL = "http://www.sina.com"
};
var request4 = new HttpRequest()
{
URL = "http://www.sina.com"
};
var request5 = new HttpRequest()
{
URL = "http://www.sina.com"
};
var response = Downloader.Downloader.SendRequestAsync(request);
var response2 = Downloader.Downloader.SendRequestAsync(request2);
var response3 = Downloader.Downloader.SendRequestAsync(request3);
var response4 = Downloader.Downloader.SendRequestAsync(request4);
var response5 = Downloader.Downloader.SendRequestAsync(request5);
var result = response5.Result.ResponsePlanText;
//var html= Encoding.UTF8.GetString(Encoding.GetEncoding("ISO-8859-1").GetBytes(result.ResponseMessage.Content.ReadAsStringAsync().Result));
sw.Stop();
var timeCost = sw.ElapsedMilliseconds;
return;
}
public void UrlFilterTest()
{
Shell.NScrapy scrapy = NScrapy.Shell.NScrapy.GetInstance();
scrapy.Crawl("UrlFilterTestSpider");
}