コード例 #1
0
ファイル: TaimyrScrapper.cs プロジェクト: poolsar/LotCreator
        public void DownloadProducts()
        {
            try
            {
                WriteOutput("Загрузка запущена");

                var context = Context.Inst;

                var supplier = context.SupplierSet.FirstOrDefault(s => s.Title == SupplierName);
                if (supplier == null)
                {
                    supplier = new Supplier();
                    supplier.Title = "ЭкоТаймыр";
                    supplier.Uri = SupplierUri;
                    supplier.Discount = SupplierDiscount;

                    context.SupplierSet.Add(supplier);
                    context.SaveChanges();
                }

                WriteOutput("Загрузка прайса");

                var timyrPriceRows = DownloadPrice();

                WriteOutput("Прайса загружен");

                SupplierProduct product = null;

                //var tpRow = timyrPriceRows.First(r => r.Uri == "catalog.php?id=60");
                //var product = DownloadProduct(context, tpRow, supplier);
                //context.SupplierProductSet.Add(product);
                //context.SaveChanges();
                //context.SaveHistory();

                var count = timyrPriceRows.Count;
                int counter = 0;

                foreach (var pRow in timyrPriceRows)
                {
                    WriteOutput("Загрузка продукта {0}", pRow.Title);

                    product = DownloadProduct(context, pRow, supplier);
                    context.SaveHistory();

                    counter++;
                    var message = string.Format("Продукт загружен {0} из {1}", counter, count);
                    WriteOutput(message);
                }

            }
            catch (Exception ex)
            {
                throw ex;
            }
        }
コード例 #2
0
ファイル: Scrapper24AuRu.cs プロジェクト: poolsar/LotCreator
        private static SupplierCategory ParseCategory(HtmlDocument doc, Supplier supplier)
        {
            SupplierCategory parentCategory = null;

            var categoryLinks = doc.DocumentNode.NodeByXpath("//*[@id=\"lot-content-col\"]/div[1]/div").SelectNodes("a");

            string catalogRooUri = "http://krsk.24au.ru/auction/";
            foreach (var categoryLink in categoryLinks)
            {
                var tCatHref = categoryLink.GetAttributeValue("href", null);

                // проверка, что ссылка на категорию
                if (tCatHref == null) continue;
                if (tCatHref == catalogRooUri) continue;
                if (!tCatHref.Contains(catalogRooUri)) continue;

                var catUrlParts = tCatHref.Split(new[] { catalogRooUri, @"/", @"\" }, StringSplitOptions.RemoveEmptyEntries);
                if (catUrlParts.Count() == 0) continue;

                // парсим

                var tCat = supplier.Categories.FirstOrDefault(c => c.UriOnSource == tCatHref);
                if (tCat == null)
                {
                    tCat = new SupplierCategory();
                    tCat.Set("UriOnSource", tCatHref);
                    tCat.Set("Supplier", supplier);
                }

                tCat.Set("Title", categoryLink.InnerText);

                if (parentCategory != null)
                {
                    tCat.Set("Parent", parentCategory);
                }

                parentCategory = tCat;
            }
            return parentCategory;
        }
コード例 #3
0
ファイル: TaimyrScrapper.cs プロジェクト: poolsar/LotCreator
        //public class TaimyrCategory
        //{
        //    public int Id { get; set; }
        //    public string Title { get; set; }
        //    public int ParentCategoryId { get; set; }
        //    public TaimyrCategory()
        //    {
        //        ParentCategoryId = -1;
        //        _allCategories.Add(this);
        //    }
        //    public TaimyrCategory Parent
        //    {
        //        get
        //        {
        //            var result = _allCategories.FirstOrDefault(c => c.Id == this.ParentCategoryId);
        //            return result;
        //        }
        //    }
        //    public List<TaimyrCategory> Childs
        //    {
        //        get
        //        {
        //            var result = _allCategories.Where(c => c.ParentCategoryId == this.Id).ToList();
        //            return result;
        //        }
        //    }
        //    public string Uri { get; set; }
        //    static List<TaimyrCategory> _allCategories = new List<TaimyrCategory>();
        //}
        //private T IfModified<T>(SupplierProduct prod, T oval, T nval)
        //{
        //    var modified = !oval.Equals(nval);
        //    if (modified && prod.Status == ScrapeStatus.Stable)
        //    {
        //        prod.Status = ScrapeStatus.Modified;
        //        return nval;
        //    }
        //    return oval;
        //}
        private SupplierProduct ParseTaimyrProduct(ShopEntities context, TimyrPriceRow pRow, string html, Supplier supplier)
        {
            var doc = new HtmlDocument();

            doc.LoadHtml(html);

            var idOnSource = pRow.Uri.Split(new[] { "id=" }, StringSplitOptions.RemoveEmptyEntries)[1];

            var product = supplier.Products.FirstOrDefault(p => p.IdOnSource == idOnSource);

            if (product == null)
            {
                product = new SupplierProduct();

                context.SupplierProductSet.Add(product);

                product.Set("IdOnSource", idOnSource);
                product.Set("Supplier", supplier);
            }

            product.Set("Title", pRow.Title);
            product.Set("UriOnSource", pRow.Uri);

            var infoDiv = doc.DocumentNode.NodeByXpath("/html/body/div[3]/div[2]");

            //------------------------------------------------
            // категории
            //
            var categoryLinks = infoDiv.NodeByXpath("p[1]").SelectNodes("a");

            SupplierCategory parentCategory = null;
            foreach (var categoryLink in categoryLinks)
            {
                var tCatHref = categoryLink.GetAttributeValue("href", null);

                if (!tCatHref.Contains("?sid=")) continue;

                var sid = tCatHref.Split(new[] { "sid=" }, StringSplitOptions.RemoveEmptyEntries)[1];

                var tCat = supplier.Categories.FirstOrDefault(c => c.IdOnSource == sid);

                if (tCat == null)
                {
                    tCat = new SupplierCategory();
                    tCat.Set("IdOnSource", sid);
                    tCat.Set("Supplier", supplier);
                }

                tCat.Set("Title", categoryLink.InnerText);
                tCat.Set("UriOnSource", tCatHref);

                if (parentCategory != null)
                {
                    tCat.Set("Parent", parentCategory);
                }

                parentCategory = tCat;
            }

            product.Set("Category", parentCategory);

            //------------------------------------------------
            // цена без акции (если есть такая цена - значит текущая цена по акции)
            //

            var priceB = infoDiv.PriceByXpath("p[3]/font[2]/s", "руб.");
            if (priceB != null)
            {
                product.Set("IsSale", true);
            }

            var discountPrice = pRow.CurrentPrice;
            var price = priceB ?? discountPrice;
            var costPrice = discountPrice * (100m - supplier.Discount) / 100m;

            product.Set("Price", price);
            product.Set("DiscountPrice", pRow.CurrentPrice);
            product.Set("CostPrice", costPrice);

            //--------------------------------------------------
            // описание
            //
            var descriptionPs = infoDiv.SelectNodes("p").Skip(3).ToList();

            StringBuilder sbDescription = new StringBuilder();

            foreach (var descriptionP in descriptionPs)
            {
                sbDescription.AppendLine(descriptionP.OuterHtml);
            }

            product.Set("Description", sbDescription.ToString());

            context.SaveChanges();

            var pageLoader = new PageLoader();
            var imageTd = infoDiv.NodeByXpath("div[1]/table/tr/td");
            var imageLinks = imageTd.SelectNodes("a");

            for (int i = 0; i < imageLinks.Count; i++)
            {
                var imageLink = imageLinks[i];
                var imageUri = imageLink.GetAttributeValue("href", null);
                if (imageUri == null) continue;

                //imageUri = "plugins/resize.php?f=../uploads/ct60-0.jpg&amp;w=800";

                var imageName = imageUri.Split(new[] { "plugins/resize.php?f=../uploads/", "&" }, StringSplitOptions.RemoveEmptyEntries)[0];
                imageUri += "&s=0"; // убираем логотип

                //var ext = new[] {".jpg", "png"}.FirstOrDefault(e => imageUri.Contains(e));

                var sourceUri = SupplierUri + imageUri;
                var savePath = string.Format("img/taimyr/{0}/{1}", product.IdOnSource, imageName);

                var image = product.Images.FirstOrDefault(im => im.LocalPath == savePath);
                if (image == null)
                {
                    image = new Image();

                    image.Set("UriOnSupplier", imageUri);
                    image.Set("LocalPath", savePath);
                    FileSystemUtils.GetFolder(savePath);

                    if (!File.Exists(savePath))
                    {
                        pageLoader.RequestImage(sourceUri, savePath);
                    }
                }

                if (product.DefaultImage == null) product.DefaultImage = image;
                product.Images.Add(image);
            }

            context.SaveChanges();

            return product;
        }
コード例 #4
0
ファイル: TaimyrScrapper.cs プロジェクト: poolsar/LotCreator
 private SupplierProduct DownloadProduct(ShopEntities context, TimyrPriceRow pRow, Supplier supplier)
 {
     var pageLoader = new PageLoader();
     string uri = SupplierUri + pRow.Uri;
     var response = pageLoader.RequestsHtml(uri);
     var product = ParseTaimyrProduct(context, pRow, response.Html, supplier);
     return product;
 }
コード例 #5
0
ファイル: Scrapper24AuRu.cs プロジェクト: poolsar/LotCreator
        private static Supplier ParseSupplier(HtmlDocument doc)
        {
            // Поставщик
            var userLink = doc.DocumentNode.NodeByXpath("//*[@id=\"card_user_form\"]/div/div/div[1]/div/a[1]");
            var supplierName = userLink.InnerText;
            var supplierUri = userLink.GetAttributeValue("href", null);

            Supplier supplier = Context.Inst.SupplierSet.FirstOrDefault(sup => sup.Uri == supplierUri);
            if (supplier == null)
            {
                supplier = new Supplier();
                supplier.Title = string.Format("{0} ({1})", supplierName, BaseName);
                supplier.Uri = supplierUri;
            }
            return supplier;
        }
コード例 #6
0
ファイル: Scrapper24AuRu.cs プロジェクト: poolsar/LotCreator
        private static SupplierProduct ParseProduct(HtmlDocument doc, string uri, Supplier supplier, SupplierCategory сategory)
        {
            var idOnSource = uri.Split(new[] { BaseUri, @"/", @"\" }, StringSplitOptions.RemoveEmptyEntries)[0];

            SupplierProduct product = supplier.Products.FirstOrDefault(p => p.IdOnSource == idOnSource);

            if (product == null)
            {
                product = new SupplierProduct();

                Context.Inst.SupplierProductSet.Add(product);

                product.Set("IdOnSource", idOnSource);
                product.Set("UriOnSource", uri);

                product.Set("Supplier", supplier);
                product.Set("Category", сategory);
            }

            string title = doc.DocumentNode.TextByXpath("//*[@id=\"lot-postinfo-cnt\"]");
            product.Set("Title", title);

            var price = doc.DocumentNode.PriceByXpath("//*[@id=\"bid_form\"]/div/div/div[1]/div[1]/span[2]", "");
            product.Set("Price", price);
            product.Set("DiscountPrice", price);
            product.Set("CostPrice", price);

            var description = doc.DocumentNode.NodeByXpath("//*[@id=\"lot-content-col\"]/div[5]/div").InnerHtml;
            description = description.Replace("\r\n", "").Replace("\n\r", "").Trim();
            product.Set("Description", description);

            Context.Save();
            return product;
        }
コード例 #7
0
ファイル: SupNode.cs プロジェクト: poolsar/LotCreator
 public SupNode(Supplier @base)
 {
     Base = @base;
 }