override public void PROCESSOR(BotCycle bc)
            {
                CustomBot cb = (CustomBot)bc.Bot;

                if (!cb.HR.Get(Url))
                {
                    throw new ProcessorException(ProcessorExceptionType.RESTORE_AS_NEW, "Could not get: " + Url);
                }

                DataSifter.Capture gc = cb.product.Parse(cb.HR.HtmlResult);

                Fhr.CrawlerHost.Product product = new Fhr.CrawlerHost.Product(
                    id: gc.ValueOf("Id"),
                    url: Url,
                    name: gc.ValueOf("Name"),
                    sku: gc.ValueOf("Sku"),
                    price: gc.ValueOf("Price"),
                    category_branch: gc.ValuesOf("Category"),
                    image_urls: Spider.GetAbsoluteUrls(gc.ValuesOf("ImageUrl"), Url, cb.HR.HtmlResult),
                    stock: gc.ValueOf("Stock") != null ? (decimal)Fhr.CrawlerHost.Product.StockValue.IN_STOCK : (decimal)Fhr.CrawlerHost.Product.StockValue.NOT_IN_STOCK,
                    description: gc.ValueOf("Description")
                    );
                if (!Cliver.Fhr.CrawlerHost.CrawlerApi.SaveProductAsJson(product))
                {
                    throw new ProcessorException(ProcessorExceptionType.ERROR, "Product was not saved.");
                }
            }
            override public void PROCESSOR(BotCycle bc)
            {
                CustomBot cb = (CustomBot)bc.Bot;

                if (!cb.HR.Get(Url))
                {
                    throw new ProcessorException(ProcessorExceptionType.RESTORE_AS_NEW, "Could not get: " + Url);
                }

                DataSifter.Capture gc = cb.list.Parse(cb.HR.HtmlResult);

                {
                    string url = gc.ValueOf("NextPageUrl");
                    if (url != null)
                    {
                        cb.BotCycle.Add(new ListItem(Spider.GetAbsoluteUrl(url, cb.HR.ResponseUrl)));
                    }
                }

                string[] urls = Spider.GetAbsoluteUrls(gc.ValuesOf("ProductUrl"), cb.HR.ResponseUrl, cb.HR.HtmlResult);
                foreach (string url in urls)
                {
                    cb.BotCycle.Add(new ProductItem(url));
                }
            }
            override public void PROCESSOR(BotCycle bc)
            {
                CustomBot cb = (CustomBot)bc.Bot;

                if (!cb.HR.Get(Url))
                {
                    throw new ProcessorException(ProcessorExceptionType.RESTORE_AS_NEW, "Could not get: " + Url);
                }

                DataSifter.Capture gc   = cb.category.Parse(cb.HR.HtmlResult);
                string[]           urls = Spider.GetAbsoluteUrls(gc.ValuesOf("CategoryUrl"), cb.HR.ResponseUrl, cb.HR.HtmlResult);
                foreach (string url in urls)
                {
                    cb.BotCycle.Add(new CategoryItem(url));
                }
            }
Ejemplo n.º 4
0
            override public void PROCESSOR(BotCycle bc)
            {
                CustomBot cb  = (CustomBot)bc.Bot;
                string    url = "http://www.rent.com/" + Regex.Replace(State, @"\s", "-");

                if (!cb.HR.GetPage(url))
                {
                    throw new ProcessorException(ProcessorExceptionType.RESTORE_AS_NEW, "Could not get: " + url);
                }

                DataSifter.Capture c = cities.Parse(cb.HR.HtmlResult);

                string[] us = c.ValuesOf("Url");
                for (int i = 0; i < us.Length; i++)
                {
                    bc.Add(new SearchItem("http://www.rent.com" + us[i]));
                }
            }
Ejemplo n.º 5
0
            void search_processor(string url)
            {
                if (!HR.GetPage(url))
                {
                    throw new ProcessorException(ProcessorExceptionType.RESTORE_AS_NEW, "Could not get: " + url);
                }

                DataSifter.Capture c = yp.Parse(HR.HtmlResult);

                string npu = c.ValueOf("NextPageUrl");

                if (npu != null)
                {
                    Add(new SearchNextPageItem(Cliver.BotWeb.Spider.GetAbsoluteUrl(npu, url)));
                }

                foreach (string u in Cliver.BotWeb.Spider.GetAbsoluteUrls(c.ValuesOf("Company/Url"), url, HR.HtmlResult))
                {
                    Add(new CompanyItem(u));
                }
            }