Пример #1
0
        public object Query(FreeTask tsk, string recordId)
        {
            SnapSearchResult result = null;

            result = new SnapSearchResult();


            result = GetLinks(result, tsk, recordId);

            return(result);
        }
Пример #2
0
        void Snapshot(FreeTask task, string conn, string recordId)
        {
            int             itemCount = 0;
            List <XListing> xListings = new List <XListing>();

            // TaoBaoSnapshotQuery tbq = new TaoBaoSnapshotQuery();
            TaoBaoQuery      tbq    = new TaoBaoQuery(task.TaskName);
            SnapSearchResult result = null;

            result    = tbq.Query(task, recordId) as SnapSearchResult;
            xListings = result.Listings;
            //int pageSize = 2000;
            //xListings = xListings.OrderByDescending(x => x.ItemSold30Days).ToList();
            //for (int page = 0; page * pageSize < xListings.Count; page++)
            //{
            //    SaveResult(xListings.Skip(page * pageSize).Take(pageSize).ToList(), conn, BotTypes.ItemSnapshot);
            //}
            // SaveKeyRecord(task, xListings);
        }
Пример #3
0
        SnapSearchResult GetLinks(SnapSearchResult result, FreeTask tsk, string recordId)
        {
            List <XListing> xListings = new List <XListing>();

            string link = "https://s.taobao.com/search?q={0}&ie=utf8&sort=default".FormatStr(tsk.TaskName);
            //   string link = "https://s.taobao.com/search?q={0}&ie=utf8&sort=default".FormatStr("连衣裙冬");
            int nohist_pages = 0;
            int quried_pages = 1;
            int Position     = 1;

            //最多搜索20页
            while (!string.IsNullOrEmpty(link) && quried_pages <= 20)
            {
                log(link);
                var html = TaobaoWebHelper.GetSnapshotHtml(link);;
                try
                {
                    if (html != null)
                    {
                        var tagslist = html.SubAfter("itemlist").SubBefore("recommendAuctions");
                        var tags     = tagslist.SubAfter("p4pTags").SplitWith("p4pTags");

                        if (tags == null || tags.Length == 0)
                        {
                            log("BLOCKED " + tsk);
                            break;
                        }
                        bool nohit = true;

                        foreach (var tag in tags)
                        {
                            try
                            {
                                if (!tag.Contains("raw_title"))
                                {
                                    Console.WriteLine(DateTime.Now);
                                    break;
                                }
                                if (tag == null || tag == "" || tag.Trim().Length == 0)
                                {
                                    Console.WriteLine(DateTime.Now);
                                    break;
                                }
                                var    a          = tag.SubAfter("raw_title").SubBefore("pic_url");
                                string title      = RemoveChar(a.GetLower());
                                string nid        = RemoveChar(tag.SubAfter("nid").SubBefore("category"));
                                string pic_url    = RemoveChar(tag.SubAfter("pic_url").SubBefore("detail_url"));
                                string detail_url = "https:" + RemoveChar(tag.SubAfter("detail_url").SubBefore("view_price"));
                                detail_url = Regex.Unescape(detail_url);
                                var    view_price = RemoveChar(tag.SubAfter("view_price").SubBefore("view_fee"));
                                double price      = 0;
                                if (!string.IsNullOrEmpty(view_price))
                                {
                                    price = Convert.ToDouble(view_price);
                                }
                                string item_loc   = RemoveChar(tag.SubAfter("item_loc").SubBefore("reserve_price"));
                                string view_sales = RemoveChar(tag.SubAfter("view_sales").SubBefore("comment_count")).Replace("人收货", "").Replace("人付款", "");
                                int    days30     = 0;
                                if (!string.IsNullOrEmpty(view_sales))
                                {
                                    days30 = Convert.ToInt32(view_sales);
                                }
                                string comment_count = RemoveChar(tag.SubAfter("comment_count").SubBefore("user_id"));
                                int    commentcount  = 0;
                                if (!string.IsNullOrEmpty(comment_count))
                                {
                                    commentcount = Convert.ToInt32(comment_count);
                                }
                                //shop
                                string user_id      = RemoveChar(tag.SubAfter("user_id").SubBefore("nick"));
                                string nick         = RemoveChar(tag.SubAfter("nick").SubBefore("shopcard"));
                                string isTmall      = RemoveChar(tag.SubAfter("isTmall").SubBefore("delivery"));
                                string delivery     = RemoveChar(tag.SubAfter("delivery").SubBefore("description"));
                                string description  = RemoveChar(tag.SubAfter("description").SubBefore("service"));
                                string sellerCredit = RemoveChar(tag.SubAfter("sellerCredit").SubBefore("totalRate"));
                                string siteName     = "taobao";
                                Guid   siteId       = Guid.Parse("A00A672B-DD05-65FB-4EE0-CFA26EBF2ED5");
                                var    totalRate    = RemoveChar(tag.SubAfter("totalRate").SubBefore("icon").GetLower());
                                var    shopLink     = RemoveChar(tag.SubAfter("shopLink").SubBefore("}"));
                                shopLink = "https:" + shopLink;
                                shopLink = Regex.Unescape(shopLink);
                                XListing listing = new XListing
                                {
                                    ShopContactUrl        = shopLink,
                                    ItemDetailUrl         = detail_url,
                                    ItemPrice             = price,
                                    ItemName              = title,
                                    ItemID                = nid,
                                    ItemLocation          = item_loc,
                                    ItemSold30Days        = days30,
                                    Itempic               = pic_url,
                                    ItemTotalCommentCount = commentcount,
                                    UId              = tsk.UId,
                                    ShopID           = user_id,
                                    ShopName         = nick,
                                    ShopLocation     = item_loc,
                                    ShopIsTmall      = isTmall == "true" ? true : false,
                                    taskid           = tsk._id,
                                    taskName         = tsk.TaskName,
                                    SiteName         = siteName,
                                    SiteID           = siteId,
                                    usrid            = tsk.UsrId,
                                    ShopIsAuthorized = false,
                                    Position         = Position,
                                    PageNum          = quried_pages,
                                    ProjectId        = tsk.ProjectId
                                };
                                if (listing.ItemDetailUrl != null && listing.ItemName != null)
                                {
                                    listing.ItemBotStatus = BotStatus.Ok;
                                }
                                result.Listings.Add(listing);
                                xListings.Add(listing);
                                nohit        = false;
                                nohist_pages = 0;
                                Position++;
                            }
                            catch (Exception we)
                            {
                                Console.WriteLine(DateTime.Now + "错误:" + we.Message);
                                break;
                            }
                        }

                        if (nohit)
                        {
                            nohist_pages++;
                        }
                        //如果连续3页都没有结果,就跳出
                        if (nohist_pages > 3)
                        {
                            break;
                        }

                        quried_pages++;
                        pages++;
                        NextPage = NextPage + 44;

                        //  link = "https://s.taobao.com/search?q={0}&sort=sale-desc&s={1}".FormatStr(tsk.TaskName, NextPage);
                        //
                        link = "https://s.taobao.com/search?q={0}&ie=utf8&sort=default&s={1}".FormatStr(tsk.TaskName, NextPage);
                        Console.WriteLine(DateTime.Now + "任务名:" + tsk.TaskName + ";开始搜索第" + quried_pages + "页");

                        SaveResult(xListings, BotTypes.ItemSnapshot, recordId, tsk);
                        xListings.Clear();

                        int n = new Random().Next(3000, 6000);
                        Thread.Sleep(n);
                    }
                    else
                    {
                        return(result);
                    }
                }
                catch (Exception ex)
                {
                    Console.WriteLine(DateTime.Now + "错误:" + ex.Message);
                    break;
                }
            }
            return(result);
        }