public void Extract()
        {
            ModelExtractor <Product> extractor = new ModelExtractor <Product>();
            var mode    = new ModelDefine <Product>();
            var results = extractor.Extract(new Page(new Request("http://list.jd.com/list.html?cat=9987,653,655&page=2&JL=6_0_0&ms=5#J_main", new Dictionary <string, dynamic>
            {
                { "cat", "手机" },
                { "cat3", "110" }
            })
            {
                Site = new Site()
            })
            {
                Content = File.ReadAllText(Path.Combine(Env.BaseDirectory, "Jd.html"))
            }, mode);

            Assert.Equal(60, results.Count());
            Assert.Equal("手机", results.First().CategoryName);
            Assert.Equal(110, results.First().CategoryId);
            Assert.Equal("http://item.jd.com/3031737.html", results.First().Url);
            Assert.Equal("3031737", results.First().Sku);
            Assert.Equal("荣耀官方旗舰店", results.First().ShopName);
            Assert.Equal("荣耀 NOTE 8 4GB+32GB 全网通版 冰河银", results.First().Name);
            Assert.Equal("1000000904", results.First().VenderId);
            Assert.Equal("1000000904", results.First().JdzyShopId);
            Assert.Equal(DateTime.Now.ToString("yyyy-MM-dd"), results.First().RunId.ToString("yyyy-MM-dd"));
        }
        public void Extract()
        {
            ModelExtractor <Product> extractor = new ModelExtractor <Product>();
            var model      = new ModelDefinition <Product>();
            var selectable = new Selectable(
                File.ReadAllText(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Jd.html")), "http://item.jd.com/3031737.htm");

            selectable.Properties = new Dictionary <string, dynamic> {
                { "cat", "手机" },
                { "cat3", "110" }
            };

            var results = extractor.Extract(selectable, model);

            Assert.Equal(60, results.Count());
            Assert.Equal("手机", results.First().CategoryName);
            Assert.Equal(110, results.First().CategoryId);
            Assert.Equal("http://item.jd.com/3031737.html", results.First().Url);
            Assert.Equal("3031737", results.First().Sku);
            Assert.Equal("荣耀官方旗舰店", results.First().ShopName);
            Assert.Equal("荣耀 NOTE 8 4GB+32GB 全网通版 冰河银", results.First().Name);
            Assert.Equal("1000000904", results.First().VenderId);
            Assert.Equal("1000000904", results.First().JdzyShopId);
            Assert.Equal(DateTime.Now.ToString("yyyy-MM-dd"), results.First().RunId.ToString("yyyy-MM-dd"));
        }
Example #3
0
        public void NullEntityModelSelector()
        {
            ModelExtractor extractor = new ModelExtractor();
            IModel         model     = new ModelDefinition <N>();

            var result = extractor.Extract(CreatePage(), model).First() as Dictionary <string, dynamic>;

            Assert.Equal("i am title", result["title"]);
            Assert.Equal("i am dotnetspider", result["dotnetspider"]);
        }
Example #4
0
        public void EntityModelSelector()
        {
            ModelExtractor extractor = new ModelExtractor();
            IModel         model     = new ModelDefinition <E>();

            var results = extractor.Extract(CreatePage(), model).ToList();

            Assert.Equal(2, results.Count());

            Assert.Equal("a", results[0]["title"]);
            Assert.Equal("b", results[1]["title"]);
        }
Example #5
0
        public void NullModelSelector()
        {
            var fields = new[]
            {
                new FieldSelector("./div[@class='title']", "title"),
                new FieldSelector("./div[@class='dotnetspider']", "dotnetspider"),
            };
            var            model     = new ModelDefinition(null, fields);
            ModelExtractor extractor = new ModelExtractor();

            var result = extractor.Extract(CreatePage(), model).First() as Dictionary <string, dynamic>;

            Assert.Equal("i am title", result["title"]);
            Assert.Equal("i am dotnetspider", result["dotnetspider"]);
        }
Example #6
0
        public void ModelSelector()
        {
            var entitySelector = new Selector("//div[@class='aaaa']");
            var fields         = new[]
            {
                new FieldSelector(".", "title"),
            };
            var            model     = new ModelDefinition(entitySelector, fields);
            ModelExtractor extractor = new ModelExtractor();

            var results = extractor.Extract(CreatePage(), model).ToList();

            Assert.Equal(2, results.Count());

            Assert.Equal("a", results[0]["title"]);
            Assert.Equal("b", results[1]["title"]);
        }