public void Extract()
{
ModelExtractor <Product> extractor = new ModelExtractor <Product>();
var mode = new ModelDefine <Product>();
var results = extractor.Extract(new Page(new Request("http://list.jd.com/list.html?cat=9987,653,655&page=2&JL=6_0_0&ms=5#J_main", new Dictionary <string, dynamic>
{
{ "cat", "手机" },
{ "cat3", "110" }
})
{
Site = new Site()
})
{
Content = File.ReadAllText(Path.Combine(Env.BaseDirectory, "Jd.html"))
}, mode);
Assert.Equal(60, results.Count());
Assert.Equal("手机", results.First().CategoryName);
Assert.Equal(110, results.First().CategoryId);
Assert.Equal("http://item.jd.com/3031737.html", results.First().Url);
Assert.Equal("3031737", results.First().Sku);
Assert.Equal("荣耀官方旗舰店", results.First().ShopName);
Assert.Equal("荣耀 NOTE 8 4GB+32GB 全网通版 冰河银", results.First().Name);
Assert.Equal("1000000904", results.First().VenderId);
Assert.Equal("1000000904", results.First().JdzyShopId);
Assert.Equal(DateTime.Now.ToString("yyyy-MM-dd"), results.First().RunId.ToString("yyyy-MM-dd"));
}
public void Extract()
{
ModelExtractor <Product> extractor = new ModelExtractor <Product>();
var model = new ModelDefinition <Product>();
var selectable = new Selectable(
File.ReadAllText(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Jd.html")), "http://item.jd.com/3031737.htm");
selectable.Properties = new Dictionary <string, dynamic> {
{ "cat", "手机" },
{ "cat3", "110" }
};
var results = extractor.Extract(selectable, model);
Assert.Equal(60, results.Count());
Assert.Equal("手机", results.First().CategoryName);
Assert.Equal(110, results.First().CategoryId);
Assert.Equal("http://item.jd.com/3031737.html", results.First().Url);
Assert.Equal("3031737", results.First().Sku);
Assert.Equal("荣耀官方旗舰店", results.First().ShopName);
Assert.Equal("荣耀 NOTE 8 4GB+32GB 全网通版 冰河银", results.First().Name);
Assert.Equal("1000000904", results.First().VenderId);
Assert.Equal("1000000904", results.First().JdzyShopId);
Assert.Equal(DateTime.Now.ToString("yyyy-MM-dd"), results.First().RunId.ToString("yyyy-MM-dd"));
}
public void NullEntityModelSelector()
{
ModelExtractor extractor = new ModelExtractor();
IModel model = new ModelDefinition <N>();
var result = extractor.Extract(CreatePage(), model).First() as Dictionary <string, dynamic>;
Assert.Equal("i am title", result["title"]);
Assert.Equal("i am dotnetspider", result["dotnetspider"]);
}
public void EntityModelSelector()
{
ModelExtractor extractor = new ModelExtractor();
IModel model = new ModelDefinition <E>();
var results = extractor.Extract(CreatePage(), model).ToList();
Assert.Equal(2, results.Count());
Assert.Equal("a", results[0]["title"]);
Assert.Equal("b", results[1]["title"]);
}
public void NullModelSelector()
{
var fields = new[]
{
new FieldSelector("./div[@class='title']", "title"),
new FieldSelector("./div[@class='dotnetspider']", "dotnetspider"),
};
var model = new ModelDefinition(null, fields);
ModelExtractor extractor = new ModelExtractor();
var result = extractor.Extract(CreatePage(), model).First() as Dictionary <string, dynamic>;
Assert.Equal("i am title", result["title"]);
Assert.Equal("i am dotnetspider", result["dotnetspider"]);
}
public void ModelSelector()
{
var entitySelector = new Selector("//div[@class='aaaa']");
var fields = new[]
{
new FieldSelector(".", "title"),
};
var model = new ModelDefinition(entitySelector, fields);
ModelExtractor extractor = new ModelExtractor();
var results = extractor.Extract(CreatePage(), model).ToList();
Assert.Equal(2, results.Count());
Assert.Equal("a", results[0]["title"]);
Assert.Equal("b", results[1]["title"]);
}