public void AddTest()
        {
            HtmlElement root = new HtmlElement("root");
            HtmlNodeCollection target = new HtmlNodeCollection(root);
            HtmlElement child = new HtmlElement("child");
            int index = target.Add(child);
            Assert.AreEqual(root, child.Parent);
            Assert.AreEqual(index, 0);

            target.Add(null);
        }
Exemplo n.º 2
0
 static void ListAllHyperlinks(HtmlNodeCollection nodes)
 {
     foreach (HtmlNode node in nodes)
     {
         HtmlElement element = node as HtmlElement;
         if (null != element)
         {
             if ((element.Name.ToLower() == "a") && element.Attributes.Contains("href"))
             {
                 System.Console.WriteLine(element.ToString());
             }
             ListAllHyperlinks((node as HtmlElement).Nodes);
         }
     }
 }
        /// <summary>
        /// Selects a list of nodes matching the <see cref="XPath"/> expression.
        /// </summary>
        /// <param name="xpath">The XPath expression.</param>
        /// <returns>An <see cref="HtmlNodeCollection"/> containing a collection of nodes matching the <see cref="XPath"/> query, or <c>null</c> if no node matched the XPath expression.</returns>
        public HtmlNodeCollection SelectNodes(string xpath)
        {
            HtmlNodeCollection list = new HtmlNodeCollection(null);

            HtmlNodeNavigator nav = new HtmlNodeNavigator(OwnerDocument, this);
            XPathNodeIterator it = nav.Select(xpath);
            while (it.MoveNext())
            {
                HtmlNodeNavigator n = (HtmlNodeNavigator)it.Current;
                list.Add(n.CurrentNode);
            }
            if (list.Count == 0)
            {
                return null;
            }
            return list;
        }
        public void FindByAttributeNameTest()
        {
            HtmlElement root = new HtmlElement("root");
            HtmlNodeCollection target = new HtmlNodeCollection(root);
            target.Add(new HtmlElement("first"));
            target.Add(new HtmlElement("second"));
            target.Add(new HtmlElement("third"));
            ((HtmlElement)target[0]).Nodes.Add(new HtmlElement("secondchild"));

            ((HtmlElement)target[1]).Attributes.Add(new HtmlAttribute("firstattribute"));
            ((HtmlElement)target[1]).Attributes.Add(new HtmlAttribute("secondattribute"));
            ((HtmlElement)target[2]).Attributes.Add(new HtmlAttribute("firstattribute"));

            Assert.AreEqual(target.FindByAttributeName("firstattribute").Count, 2);

            ((HtmlElement)((HtmlElement)target[0]).Nodes[0]).Attributes.Add(new HtmlAttribute("firstattribute"));
            Assert.AreEqual(target.FindByAttributeName("firstattribute", false).Count, 2);
            Assert.AreEqual(target.FindByAttributeName("firstattribute", true).Count, 3);
        }
 public void HtmlNodeCollectionConstructorTest()
 {
     HtmlElement root = new HtmlElement("root");
     HtmlNodeCollection target = new HtmlNodeCollection(root);
     HtmlElement child = new HtmlElement("child");
     target.Add(child);
     Assert.AreEqual(root, child.Parent);
 }
Exemplo n.º 6
0
        private async Task LoadMarkedTask(string uri)
        {
            MarkedList.Clear();
            int o = 0;

            while (true)
            {
                string reqUri         = uri + o;
                string responseString = await SendRequest.GET(reqUri);

                HtmlDocument doc = new HtmlDocument();
                doc.LoadHtml(responseString);

                HtmlNodeCollection nodes = doc.DocumentNode.SelectNodes(@"//form[@name='frm_mis']/tr");

                int count = 0;

                Regex regex = new Regex(@"# \d+");
                for (int i = 0; i < nodes.Count; i++)
                {
                    if (i == 0 || i == nodes.Count - 1)
                    {
                        continue;
                    }

                    string complete = nodes[i].SelectSingleNode(@"./td[1]/table/tr[@class='m2']//a").GetAttributeValue("href", "");
                    string mark     = ""; // nodes[i].SelectSingleNode(@"./td[1]/table/tr[2]//a").GetAttributeValue("href", "");
                    string targetIP = nodes[i].SelectSingleNode(@"./td[2]/table/tr[1]//a/span[@class='green']").InnerText;
                    string type     = nodes[i].SelectSingleNode(@"./td[3]/a/span").InnerText;
                    string detail   = StringHelper.RemoveSpecial(nodes[i].SelectSingleNode(@"./td[4]//td[@align='justify']").InnerText);
                    string reward   = nodes[i].SelectSingleNode(@"./td[5]").InnerText;
                    string fileID   = "";
                    var    match    = regex.Match(detail);
                    if (match.Success)
                    {
                        fileID = match.Value.Replace("#", "").Trim();
                    }

                    MissionModel newData = new MissionModel()
                    {
                        Complete = complete,
                        Mark     = mark,
                        TargetIP = targetIP,
                        Type     = type,
                        Details  = detail,
                        Reward   = reward,
                        FileID   = fileID
                    };
                    MarkedList.Add(newData);

                    count++;
                }

                if (count == 0)
                {
                    break;
                }

                o += 50;
            }
        }
Exemplo n.º 7
0
 /// <summary>
 /// This constructs a new Html element with the specified tag attributeName.
 /// </summary>
 public HtmlElement(string name)
 {
     this.nodes = new HtmlNodeCollection(this);
     this.attributes = new HtmlAttributeCollection(this);
     this.name = name;
 }
Exemplo n.º 8
0
        /// <summary>
        /// Get friend helper method. This method support for GetFriend
        /// </summary>
        /// <param name="firstFriendPageUrl">Url</param>
        /// <param name="type">
        /// Type = 1 if you want to get other friends.
        /// Type = 2 if you want to get your friend.
        /// </param>
        /// <returns>List of facebook user id</returns>
        List <string> GetFriends(string firstFriendPageUrl)
        {
            // Declare list string to store user id
            var friends = new List <string>();

            // using queue to remove recursive search over pages
            var friendPages = new Queue <string>();

            friendPages.Enqueue(firstFriendPageUrl);

            // we will loop to the end of friend list
            while (friendPages.Count > 0)
            {
                string   currentFriendPageUrl = friendPages.Dequeue();
                string   currentHtmlContent   = _http.DownloadContent(currentFriendPageUrl);
                HtmlNode docNode = HtmlHelper.BuildDom(currentHtmlContent);

                // select root node which is div element with id is root, does not real root node
                HtmlNode rootNode = docNode.SelectSingleNode("//div[@id='root']");

                // Because selectNodes and SelectSingleNode method working incorrect.
                // Both method working in entire document node rather than current node when we pass relative xpath expression.
                // So, we maybe get more element than expect.

                // E.g : Trying select table element with role=presentation in rootNode will search in entire document
                // var friendTables1 = rootNode.SelectNodes("//table[@role='presentation']");

                // If we only want to search in InnerHtml of current node, just load it to another HtmlDocument object
                // Maybe isn't the best way to do our job. But at the moment, i think it still good.
                docNode = HtmlHelper.BuildDom(rootNode.InnerHtml);

                // Now search table in new document
                HtmlNodeCollection friendAnchors =
                    docNode.SelectNodes("//table[@role='presentation']/tr/td[2]/a") ??
                    docNode.SelectNodes("//table[@role='presentation']/tbody/tr/td[2]/a");
                if (friendAnchors == null)
                {
                    return(friends);
                }

                // Loop through all node and trying to get user alias or id
                foreach (HtmlNode friendAnchor in friendAnchors)
                {
                    string id = string.Empty;
                    string userProfileHref = friendAnchor.GetAttributeValue("href", null);

                    if (userProfileHref != null)
                    {
                        if (!userProfileHref.Contains("profile.php"))
                        {
                            // if userProfileHref does't contain "profile.php", userProfileHref contain user alias.
                            // E.g : https://m.facebook.com:443/user.alias.here?fref=fr_tab&amp;refid=17/about
                            int questionMarkIndex = userProfileHref.IndexOf("?");

                            if (questionMarkIndex > -1)
                            {
                                userProfileHref = userProfileHref.Substring(1, questionMarkIndex - 1);
                            }
                            else
                            {
                                userProfileHref = userProfileHref.Substring(1);
                            }

                            friends.Add(userProfileHref);
                        }
                        else
                        {
                            // Extract user id from href profile.php?id=user_id&fre...
                            // If extract not success then we need to log this error
                            Match match = CompiledRegex.Match("UserId", userProfileHref);
                            if (match.Success)
                            {
                                friends.Add(match.Groups["id"].Value);
                            }
                            else
                            {
                                _logger.WriteLine("Match user id by CompiledRege.Match(UserId) is fail. Addition info : url=" + firstFriendPageUrl + " and user profile is " + userProfileHref);
                            }
                        }
                    }
                    else
                    {
                        // If we go to this code block, there are some case happend :
                        // - Our bot has been block by this user or facebook.
                        // - This is deleted user.
                        // - We need provide more pattern to detect user id

                        // now i will log it for later fix
                        _logger.WriteLine("Maybe " + friendAnchor.InnerText + " has been banned. Access this link from browser to check again.");
                    }
                }

                HtmlNode moreFriend = rootNode.SelectSingleNode("//div[@id='m_more_friends']/a");
                if (moreFriend == null)
                {
                    continue;
                }

                var nextUrl = WebUtility.HtmlDecode(moreFriend.GetAttributeValue("href", null));

                if (nextUrl != null)
                {
                    friendPages.Enqueue("https://m.facebook.com" + nextUrl);
                }
                else
                {
                    _logger.WriteLine("This is last page.");
                }
            }

            return(friends);
        }
Exemplo n.º 9
0
        private void ParseAndInsertIntoList(string homePage, HtmlDocument htmlDoc)
        {
            HtmlNodeCollection nodeCollectionLinks = htmlDoc.DocumentNode.SelectNodes("//article/div/div/div/div/a");

            foreach (HtmlNode link in nodeCollectionLinks)
            {
                ImmobilienProperties = new Dictionary <string, string>();

                string propertiesLink = link.Attributes["href"].Value;

                if (!propertiesLink.Contains(homePage))
                {
                    propertiesLink = homePage + link.Attributes["href"].Value;
                }

                ImmobilienProperties["link"] = propertiesLink;

                HtmlWeb propertyWeb = new HtmlWeb();
                try
                {
                    HtmlDocument htmlPropertyDoc = propertyWeb.Load(propertiesLink);

                    ImmobilienProperties["html"] = htmlPropertyDoc.DocumentNode.InnerHtml;

                    //title
                    HtmlNodeCollection titles = htmlPropertyDoc.DocumentNode.SelectNodes("//div/div/div/div/div/h1");
                    ImmobilienProperties[Constants.Db.title] = titles[0].InnerText;

                    //etage
                    string             dbType = "";
                    HtmlNodeCollection types  = htmlPropertyDoc.DocumentNode.SelectNodes("//div/div/div/div/div/div/div/div/dl");

                    HtmlNodeCollection address = htmlPropertyDoc.DocumentNode.SelectNodes("//div/div/div/div/div/span/div");

                    string strAddress = " ";

                    foreach (HtmlNode ndAddress in address)
                    {
                        strAddress = strAddress + " " + ndAddress.InnerText;
                    }

                    ImmobilienProperties[Constants.Db.address] = strAddress;

                    if (!(types is null))
                    {
                        foreach (HtmlNode type in types)
                        {
                            //Etage
                            ParseHtml(type, Constants.Html.etage, Constants.Db.typ, new int[] { 2 }, ImmobilienProperties);

                            //Typ
                            ParseHtml(type, Constants.Html.typ, Constants.Db.typ, new int[] { 2 }, ImmobilienProperties);

                            //Wohnflaeche
                            ParseHtml(type, Constants.Html.wohnflaeche, Constants.Db.wohnflaeche, new int[] { 3 }, ImmobilienProperties);

                            //Bezugsfrei
                            ParseHtml(type, Constants.Html.bezugsfrei, Constants.Db.bezugsfrei, new int[] { 3 }, ImmobilienProperties);

                            //Bonitätsauskunft
                            ParseHtml(type, Constants.Html.bonitaetsauskunft, Constants.Db.bonitaetsauskunft, new int[] { 2 }, ImmobilienProperties);

                            //Zimmer
                            ParseHtml(type, Constants.Html.zimmer, Constants.Db.zimmer, new int[] { 2 }, ImmobilienProperties);

                            //Kaltmiete
                            ParseHtml(type, Constants.Html.kaltmiete, Constants.Db.kaltmiete, new int[] { 2 }, ImmobilienProperties);

                            //Nebenkosten
                            ParseHtml(type, Constants.Html.nebenkosten, Constants.Db.nebenkosten, new int[] { 3 }, ImmobilienProperties);

                            //Gesamtmiete
                            ParseHtml(type, Constants.Html.gesamtmiete, Constants.Db.gesamtmiete, new int[] { 2 }, ImmobilienProperties);

                            dbType = type.InnerText;
                        }
                    }
                    //ListOfImmobilienProperties.Add(ImmobilienProperties);

                    DbManagement dbManagement = new DbManagement();

                    dbManagement.UpdateOrInsert(ImmobilienProperties);
                }
                catch (Exception e)
                {
                    Console.WriteLine(DateTime.Now + " Link: " + propertiesLink + ", Exception: " + e.Message);
                }
            }
        }
Exemplo n.º 10
0
 private HtmlDocument(StreamReader reader)
 {
     this.nodes = HtmlParser.Parse(reader);
 }
Exemplo n.º 11
0
        public override async Task <List <Book> > GetBestsellersAsync()
        {
            var    bookList = new List <Book>();
            string url      = "https://aros.pl/";
            string htmlCode = "";

            using (WebClient client = new WebClient()
            {
                Encoding = System.Text.Encoding.UTF8
            })
            {
                try{
                    await Task.Run(() =>
                    {
                        htmlCode = client.DownloadString(url);
                    });
                }
                catch (Exception ex)
                {
                    Log.Error($"Network Error in Aros: {ex.Message}");
                }
            }

            var htmlDoc = new HtmlDocument();

            if (!string.IsNullOrEmpty(htmlCode))
            {
                HtmlNodeCollection bestList = null;
                try {
                    await Task.Run(() =>
                    {
                        htmlDoc.LoadHtml(htmlCode);
                    });

                    var bestSel      = htmlDoc.DocumentNode.SelectNodes("//b").FirstOrDefault(x => x.InnerText == "Bestsellery");
                    var bestListhtml = bestSel.ParentNode.ParentNode.InnerHtml;
                    htmlDoc.LoadHtml(bestListhtml);
                    bestList = htmlDoc.DocumentNode.SelectNodes("//tr");
                }
                catch (Exception ex)
                {
                    Log.Error($"Html Error in Aros: {ex.Message}");
                    Log.Error(ex.StackTrace);
                }

                if (bestList != null)
                {
                    foreach (var bestNode in bestList)
                    {
                        try
                        {
                            var htmlDocSingle = new HtmlDocument();

                            htmlDocSingle.LoadHtml(bestNode.InnerHtml);
                            var href = htmlDocSingle.DocumentNode.SelectSingleNode("//a");

                            var urlAddress = href.Attributes["href"].Value;
                            urlAddress = "https://www.aros.pl" + urlAddress;

                            if (bestNode.InnerHtml.Contains("autor"))
                            {
                                string htmlBook = "";



                                using (WebClient client = new WebClient())
                                {
                                    var htmlData = client.DownloadData(urlAddress);
                                    htmlBook = Encoding.UTF8.GetString(htmlData);
                                }

                                var htmlDocBook = new HtmlDocument();
                                htmlDocBook.LoadHtml(htmlBook);

                                var titleNode = htmlDocBook.DocumentNode.SelectSingleNode("//h1");
                                var title     = titleNode.InnerHtml.Trim();
                                var mainNode  = titleNode.ParentNode.ParentNode.ParentNode;

                                var authorNode = mainNode.InnerHtml;
                                var authorDoc  = new HtmlDocument();
                                authorDoc.LoadHtml(authorNode);
                                var author = authorDoc.DocumentNode.SelectSingleNode("//b").InnerText;
                                var imgSrc = authorDoc.DocumentNode.SelectNodes("//img")
                                             .FirstOrDefault(x => x.Attributes["alt"].Value == title).Attributes["src"].Value;

                                //Attributes["alt"].Value;
                                imgSrc = "https:" + imgSrc;


                                var book = new Book(title, author, imgSrc, "Aros");
                                await book.SetSizeAsync();

                                bookList.Add(book);
                                //Console.WriteLine(book.Title);
                            }
                        }
                        catch (Exception e)
                        {
                            Log.Error($"BOOK error in Aros: {e.Message}");
                            Log.Error(e.StackTrace);
                        }
                    }
                }
            }

            return(bookList);
        }
Exemplo n.º 12
0
        public static List <Post> RunScrubOnADoc(HtmlDocument doc, List <Player> players, List <Replacement> replacements, List <string> moderatorNames)
        {
            List <Post> postsOnThisPage = new List <Post>();

            try
            {
                HtmlNodeCollection divs = doc.DocumentNode.SelectNodes("//div");
                Regex reg = new Regex("^p[0-9]", RegexOptions.IgnoreCase);



                foreach (HtmlNode div in divs)
                {
                    string id = div.GetAttributeValue("id", string.Empty);
                    if (reg.IsMatch(id))
                    {
                        HtmlNode postProfile = div.SelectSingleNode(".//dl[contains(@class,'postprofile')]");
                        string   postAuthor  = postProfile.SelectSingleNode(".//a").InnerHtml;
                        //displayString += postAuthor;
                        //We don't care about votes the mod has made.

                        bool wasPostByModerator = false;
                        foreach (string moderatorName in moderatorNames)
                        {
                            if (Player.makeNameFriendly(postAuthor).Equals(Player.makeNameFriendly(moderatorName)))
                            {
                                wasPostByModerator = true;
                                break;
                            }
                        }

                        if (wasPostByModerator == true)
                        {
                            continue;
                        }



                        HtmlNode postbody    = div.SelectSingleNode(".//div[contains(@class,'postbody')]");
                        string   pbInnerText = postbody.InnerText;
                        string   pbInnerHtml = postbody.InnerHtml;
                        int      postNumber  = Int32.Parse(postbody.SelectSingleNode(".//strong").InnerHtml.Replace("#", ""));


                        Player playerVoting = Player.FindPlayerByNameUserAid(players, postAuthor);
                        if (playerVoting == null)
                        {
                            Player.FindPlayerByNameUserAidReplacementsLoop(players, postAuthor, replacements);
                        }
                        if (playerVoting == null)
                        {
                            playerVoting = PlayerNameWordSearch(players, replacements, Player.makeNameFriendly(postAuthor));
                        }
                        //Required for UTF8 issues. For some reason the encoding/decoding on the byte array f***s everything to hell. Wasn't an issue before.
                        if (playerVoting == null)
                        {
                            playerVoting = players.returnClosestPlayerLevensheinDistance(postAuthor);
                        }

                        // if (playerVoting != null)
                        //{
                        // playerVoting.addPostNumber(postNumber);
                        // }


                        HtmlNode authorNode         = postbody.SelectSingleNode(".//p[contains(@class,'author')]");
                        HtmlNode firstURL           = authorNode.SelectSingleNode(".//a");
                        string   hrefValue          = firstURL.Attributes["href"].Value;
                        string   directPostLocation = hrefValue.Substring(hrefValue.IndexOf("#")).Replace("#p", "#");
                        string   bbCode             = "[url=https://forum.mafiascum.net/viewtopic.php?p=" + directPostLocation.Replace("#", "") + directPostLocation + "]" + postNumber + "[/url]";

                        string dateTextStart = authorNode.InnerText;
                        dateTextStart = dateTextStart.Replace("Post&nbsp;#" + postNumber + "&nbsp;\n\t\t\t\t    \n\t\t\t\t\t(ISO)&nbsp;\n\t\t\t\t    \n\t\t\t      &raquo; ", "").Trim();

                        DateTime timeOfPost;
                        bool     isValidDate = DateTime.TryParse(dateTextStart, out timeOfPost);
                        if (!isValidDate)
                        {
                            throw new ArgumentException("Bad date provided for post: " + postNumber);
                        }

                        Post post = null;
                        if (playerVoting != null)
                        {
                            post = new Post(postNumber, playerVoting, timeOfPost, bbCode);
                        }

                        HtmlNode content = postbody.SelectSingleNode(".//div[contains(@class,'content')]");

                        var quotes = content.SelectNodes(".//blockquote");

                        HtmlNode contentMinusQuotes = content;
                        if (quotes != null)
                        {
                            foreach (var quote in quotes)
                            {
                                try
                                {
                                    quote.Remove();
                                }
                                catch
                                {
                                }
                            }
                        }

                        HtmlNodeCollection spoilerTags        = contentMinusQuotes.SelectNodes(".//div[contains(@class,'quotetitle')]");
                        HtmlNodeCollection spoilerContentTags = contentMinusQuotes.SelectNodes(".//div[contains(@class,'quotecontent')]");
                        if (spoilerTags != null)
                        {
                            try
                            {
                                foreach (HtmlNode spoiler in spoilerTags)
                                {
                                    spoiler.Remove();
                                }
                                foreach (HtmlNode spoilerContent in spoilerContentTags)
                                {
                                    spoilerContent.Remove();
                                }
                            }
                            catch
                            {
                            }
                        }


                        try
                        {
                            post.ExtractVoteFromPost(players, contentMinusQuotes, replacements);
                            postsOnThisPage.Add(post);
                        }
                        catch (Exception e)
                        {
                            List <string> playerNames = new List <string>();
                            foreach (Player player in players)
                            {
                                playerNames.Add(player.Name);
                            }

                            UnityEngine.Debug.Log(playerVoting == null ? "Unable to get original player posting. Check for typos in settings." : "Some other error occurred in getting the vote. ");
                        }
                    }
                }
            }
            catch (Exception e2)
            {
                UnityEngine.Debug.Log("Major unknown error occurred. Run me in debug. ");
            }

            return(postsOnThisPage);
        }
Exemplo n.º 13
0
        public static string ScrubPostForSettings(string url, int postNumberInput)
        {
            try
            {
                url = "https://forum.mafiascum.net/viewtopic.php?" + url + "&start=" + postNumberInput;
                //return "Url: " + url + " Post Number Input: " + postNumberInput;

                Regex        reg = new Regex("^p[0-9]", RegexOptions.IgnoreCase);
                HtmlDocument doc = URLReadLogic.URLReadLogic.GetHTMLDocumentFromURL(url);
                if ((doc == null) || doc.DocumentNode == null || doc.DocumentNode.SelectSingleNode("//div") == null)
                {
                    throw new ArgumentNullException("Could not connect with mafiascum. Try again later.");
                }
                HtmlNode div      = doc.DocumentNode.SelectSingleNode("//div");
                HtmlNode postbody = div.SelectSingleNode(".//div[contains(@class,'postbody')]");

                string             divInnerHtml       = div.InnerHtml;
                HtmlNode           content            = postbody.SelectSingleNode(".//div[contains(@class,'content')]");
                HtmlNodeCollection spoilerTags        = content.SelectNodes(".//div[contains(@class,'quotetitle')]");
                HtmlNodeCollection spoilerContentTags = content.SelectNodes(".//div[contains(@class,'quotecontent')]");

                //string displayString = "";
                int i = 0;
                //bool settingsFound = false;
                if (spoilerTags == null)
                {
                    throw new ArgumentNullException("No spoiler tags in given post. Please verify URL and post number.");
                }
                foreach (HtmlNode spoilerNode in spoilerTags)
                {
                    HtmlNode spoilerHeaderTextNode = spoilerNode.SelectSingleNode(".//b");
                    if (spoilerHeaderTextNode != null)
                    {
                        if (spoilerNode.SelectSingleNode(".//b").InnerText.Trim().Equals(VOTE_COUNT_SPOILER_NAME))
                        {
                            break;
                        }
                        else
                        {
                            i++;
                        }
                    }
                    else
                    {
                        i++;
                    }
                }

                if (i < spoilerContentTags.Count)
                {
                    return(spoilerContentTags[i].SelectSingleNode(".//div").InnerHtml.Trim());
                }
                else
                {
                    throw new ArgumentNullException("Could not find vote counter settings. Check spoiler tag name.");
                }
            }
            catch (Exception e)
            {
                throw new ArgumentNullException("Could load settings data. Check input.");
            }
        }
Exemplo n.º 14
0
        public override void ParseMatchPageHtml(HtmlDocument doc, string url)
        {
            Sport sport = GetSport(doc);

            if (sport == Sport.NotSupported)
            {
                return;
            }

            MatchName matchName = GetFullMatchName(doc); // повне ім'я (only for tennis cuz all event names were  writed like (Coppejans K.)  )

            if (matchName == null)
            {
                return;
            }
            string BetUrl = url;

            HtmlNodeCollection betsNodes = doc.DocumentNode.SelectNodes("//td");

            Bet result = null;

            if (betsNodes == null)
            {
                return;
            }
            foreach (var node in betsNodes)
            {
                result = null;
                //if (node.Attributes["data-market-type"] != null) continue;
                HtmlAttribute attribute = node.Attributes["data-sel"];
                if (attribute == null)
                {
                    continue;
                }
                string value = attribute.Value.Replace("\"", string.Empty).Replace("&quot;", string.Empty);

                JavaSelectCode =
                    "(function() { var element = document.evaluate( '" + node.XPath + "' ,document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null ).singleNodeValue;  element.click(); })();";


                string coeff = value.Split(new string[] { ",epr:", ",prices:{" }, StringSplitOptions.RemoveEmptyEntries)[1];

                double Probability = Convert.ToDouble(coeff.Replace(".", ","));
                string type        = value.Split(new string[] { "sn:", ",mn:" }, StringSplitOptions.RemoveEmptyEntries)[1];

                string TotalorHand = value.Split(new string[] { "mn:", ",ewc:" }, StringSplitOptions.RemoveEmptyEntries)[1];
                if (TotalorHand.Contains(" + "))
                {
                    continue;
                }
                if (TotalorHand.ToLower().Contains("min."))
                {
                    continue;
                }
                Team team = GetTeam(TotalorHand, matchName);
                Time time = GetTime(TotalorHand);
                #region main bets
                if (TotalorHand.Contains("Match Result") || TotalorHand == "Result" || TotalorHand.Contains("Match Winner Including All OT") || (TotalorHand.Contains("Result") && TotalorHand.Contains("Set")) || TotalorHand.Contains("Normal Time Result") || TotalorHand.Contains("To Win Match") && !TotalorHand.Contains("Handicap"))
                {
                    if (TotalorHand.Contains("Match Winner Including All OT"))
                    {
                        if (type == matchName.FirstTeam)
                        {
                            if (node.Attributes["data-market-type"] != null && node.Attributes["data-market-type"].Value == "RESULT_2WAY")
                            {
                                result = new ResultBet(ResultBetType.P1, time, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                            }
                        }
                        else
                        if (type == matchName.SecondTeam)
                        {
                            if (node.Attributes["data-market-type"] != null && node.Attributes["data-market-type"].Value == "RESULT_2WAY")
                            {
                                result = new ResultBet(ResultBetType.P2, time, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                            }
                        }
                    }
                    if (TotalorHand.Contains("To Win Match"))
                    {
                        if (type == matchName.FirstTeam)
                        {
                            if (node.Attributes["data-market-type"] != null && node.Attributes["data-market-type"].Value == "RESULT_2WAY")
                            {
                                result = new ResultBet(ResultBetType.P1, time, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                            }
                        }
                        else
                        if (type == matchName.SecondTeam)
                        {
                            if (node.Attributes["data-market-type"] != null && node.Attributes["data-market-type"].Value == "RESULT_2WAY")
                            {
                                result = new ResultBet(ResultBetType.P2, time, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                            }
                        }
                    }
                    if (type == matchName.FirstTeam + " To Win")
                    {
                        if (node.Attributes["data-market-type"] != null && node.Attributes["data-market-type"].Value == "RESULT_2WAY")
                        {
                            result = new ResultBet(ResultBetType.P1, time, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                        }
                        else
                        {
                            result = new ResultBet(ResultBetType.First, time, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                        }
                    }
                    else
                    if (type == "Draw")
                    {
                        result = new ResultBet(ResultBetType.Draw, time, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                    }
                    else
                    if (type == matchName.SecondTeam + " To Win")
                    {
                        if (node.Attributes["data-market-type"] != null && node.Attributes["data-market-type"].Value == "RESULT_2WAY")
                        {
                            result = new ResultBet(ResultBetType.P2, time, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                        }
                        else
                        {
                            result = new ResultBet(ResultBetType.Second, time, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                        }
                    }
                    else
                    if (type == matchName.FirstTeam + " To Win or Draw")
                    {
                        result = new ResultBet(ResultBetType.FirstOrDraw, time, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                    }
                    else
                    if (type == matchName.FirstTeam + " To Win or " + matchName.SecondTeam + " To Win")
                    {
                        result = new ResultBet(ResultBetType.FirstOrSecond, time, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                    }
                    else
                    if (type == matchName.SecondTeam + " To Win or Draw")
                    {
                        result = new ResultBet(ResultBetType.SecondOrDraw, time, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                    }
                }
                #endregion
                else
                #region Totals for All Team
                // Totals
                if (TotalorHand.Contains("Total") && !TotalorHand.Contains("Sets") && !TotalorHand.Contains("Innings"))
                {
                    if (type.Contains("Under"))
                    {
                        if (TotalorHand.Contains("Asian"))
                        {
                            try
                            {
                                double first_param  = Convert.ToDouble(type.Split(new string[] { "Under ", "," }, StringSplitOptions.RemoveEmptyEntries)[0].Replace(".", ","));
                                double second_param = Convert.ToDouble(type.Split(new string[] { "Under ", "," }, StringSplitOptions.RemoveEmptyEntries)[1].Replace(".", ","));

                                double param = (first_param + second_param) / 2;
                                result = new TotalBet(TotalBetType.Under, param, time, team, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                            }
                            catch { }
                        }
                        else
                        {
                            try
                            {
                                double param = Convert.ToDouble(type.Split(new string[] { "Under ", "\0" }, StringSplitOptions.RemoveEmptyEntries)[0].Replace(".", ","));

                                result = new TotalBet(TotalBetType.Under, param, time, team, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                            }
                            catch { }
                        }
                    }
                    if (type.Contains("Over"))
                    {
                        if (TotalorHand.Contains("Asian"))
                        {
                            try
                            {
                                double first_param  = Convert.ToDouble(type.Split(new string[] { "Over ", "," }, StringSplitOptions.RemoveEmptyEntries)[0].Replace(".", ","));
                                double second_param = Convert.ToDouble(type.Split(new string[] { "Over ", "," }, StringSplitOptions.RemoveEmptyEntries)[1].Replace(".", ","));

                                double param = (first_param + second_param) / 2;
                                result = new TotalBet(TotalBetType.Over, param, time, team, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                            }
                            catch { }
                        }
                        else
                        {
                            try
                            {
                                double param = Convert.ToDouble(type.Split(new string[] { "Over ", "\0" }, StringSplitOptions.RemoveEmptyEntries)[0].Replace(".", ","));

                                result = new TotalBet(TotalBetType.Over, param, time, team, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                            }
                            catch { }
                        }
                    }
                }
                #endregion
                else
                #region All Handicaps
                if (TotalorHand.Contains("Handicap") || TotalorHand.Contains("Draw No Bet"))     // переробити. draw no bet = 0 все інше нормально. з ханд робити
                {
                    if (type.Contains(matchName.FirstTeam))
                    {
                        double param = 0;
                        string test;
                        if (TotalorHand.Contains("Draw No Bet"))
                        {
                            param = 0;
                        }
                        else
                        {
                            if (TotalorHand.Contains("Asian"))
                            {
                                try
                                {
                                    double first_param  = Convert.ToDouble(type.Split(new string[] { matchName.FirstTeam + " (", ")", "," }, StringSplitOptions.RemoveEmptyEntries)[0].Replace(".", ","));
                                    double second_param = Convert.ToDouble(type.Split(new string[] { matchName.FirstTeam + " (", ")", "," }, StringSplitOptions.RemoveEmptyEntries)[1].Replace(".", ","));

                                    param = (first_param + second_param) / 2;
                                }
                                catch { }
                            }
                            else
                            {
                                try
                                {
                                    test  = type.Split(new string[] { matchName.FirstTeam + " (", ")" }, StringSplitOptions.RemoveEmptyEntries)[0].Replace(".", ",");
                                    param = Convert.ToDouble(test);
                                }
                                catch
                                {
                                    continue;
                                }
                            }
                        }

                        result = new HandicapBet(HandicapBetType.F1, param, time, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                    }
                    if (type.Contains(matchName.SecondTeam))
                    {
                        double param = 0;
                        string test;
                        if (TotalorHand.Contains("Draw No Bet"))
                        {
                            param = 0;
                        }
                        else
                        {
                            if (TotalorHand.Contains("Asian"))
                            {
                                try
                                {
                                    double first_param  = Convert.ToDouble(type.Split(new string[] { matchName.SecondTeam + " (", ")", "," }, StringSplitOptions.RemoveEmptyEntries)[0].Replace(".", ","));
                                    double second_param = Convert.ToDouble(type.Split(new string[] { matchName.SecondTeam + " (", ")", "," }, StringSplitOptions.RemoveEmptyEntries)[1].Replace(".", ","));

                                    param = (first_param + second_param) / 2;
                                }
                                catch { }
                            }
                            else
                            {
                                try
                                {
                                    test  = type.Split(new string[] { matchName.SecondTeam + " (", ")" }, StringSplitOptions.RemoveEmptyEntries)[0].Replace(".", ",");
                                    param = Convert.ToDouble(test);
                                }
                                catch
                                {
                                    continue;
                                }
                            }
                        }
                        result = new HandicapBet(HandicapBetType.F2, param, time, Probability, matchName, BetUrl, JavaSelectCode, sport, Maker);
                    }
                }
                #endregion

                if (result != null)
                {
                    int index = BetList.IndexOf(result);
                    if (index != -1)
                    {
                        BetList[index].ChangeOdds(result.Odds);
                    }
                    else
                    {
                        BetList.Add(result);
                    }
                }
            }

            System.Threading.Thread.Sleep(50);
        }
Exemplo n.º 15
0
        /// <summary>
        /// 影片信息获取
        /// </summary>
        /// <param name="childPageId"></param>
        public Video GetVideoInfo(int videoId)
        {
            try
            {
                string   videoUrl  = baseStr + "?m=vod-detail-id-" + videoId + ".html";
                Video    videoInfo = new Video();
                HtmlNode rootnode1 = GetHtmlDoc(videoUrl);

                //

                HtmlNode vodImg  = rootnode1.SelectSingleNode("//div[@class='vodImg']");
                HtmlNode vodInfo = rootnode1.SelectSingleNode("//div[@class='vodInfo']");


                HtmlNode vodImgSrc = vodImg.SelectSingleNode("//img[@class='lazy']/@src");

                videoInfo.videoImgUrl = vodImgSrc.Attributes["src"].Value; //影片图片地址
                videoInfo.videoName   = vodImgSrc.Attributes["alt"].Value; //影片名

                HtmlNode vodInfoVodh = vodInfo.SelectSingleNode("//div[@class='vodh']");
                videoInfo.videoType    = vodInfoVodh.SelectSingleNode("span").InnerText;
                videoInfo.videoQuality = vodInfoVodh.SelectSingleNode("label").InnerText;


                //影片详情
                try
                {
                    var           htmlVodinfobox = vodInfo.SelectNodes("//div[@class='vodinfobox']/ul/li").Where(FuncVodinfobox);
                    List <string> tset           = (from a in htmlVodinfobox select a.InnerHtml).ToList();

                    List <VideoInfo> strResult = new List <VideoInfo>();
                    foreach (string item in tset)
                    {
                        if (!string.IsNullOrEmpty(item) && item.Contains(":"))
                        {
                            //别名:<span></span>
                            string[] items  = item.Split(':');
                            string   _value = items[1].Trim();

                            HtmlDocument docs = new HtmlDocument();
                            docs.LoadHtml(_value);

                            strResult.Add(new VideoInfo
                            {
                                key   = items[0].Trim(),
                                value = docs.DocumentNode.InnerText.Replace("&nbsp;", " ").Trim()
                            });
                        }
                        else
                        {
                        }
                    }
                    videoInfo.videoTotalInfo = strResult;
                }
                catch { }

                //string vodin = "//div[@class='vodinfobox']/ul/li";
                //var htmle = rootnode1.SelectNodes(vodin);

                //剧情介绍
                videoInfo.videoSynopsis = Compress(rootnode1.SelectSingleNode("//div[@class='vodplayinfo']").InnerText);

                //地址
                videoInfo.playInformation = new List <PlayAddress>();



                HtmlNodeCollection playType_ = rootnode1.SelectNodes("//div[@class='vodplayinfo']/div");
                if (playType_ != null)
                {
                    foreach (HtmlNode playTypeOne in playType_)
                    {
                        var playType = playTypeOne.SelectNodes("//span[@class='suf']");
                        if (playType != null)
                        {
                            foreach (HtmlNode item in playType)
                            {
                                PlayAddress routing = new PlayAddress();
                                routing.playType = item.InnerText;
                                routing.videoUrl = new List <VideoUrl>();
                                var Urls = item.SelectNodes("../../ul/li");
                                foreach (var item1 in Urls)
                                {
                                    VideoUrl videourl = new VideoUrl();
                                    string[] ars      = item1.InnerText.Split('$');
                                    videourl.playName = ars[0] != null ? ars[0] : " ";
                                    videourl.playURL  = ars[1] != null ? ars[1] : " ";
                                    routing.videoUrl.Add(videourl);
                                }
                                videoInfo.playInformation.Add(routing);
                            }
                            if (videoInfo.playInformation.Count > 0)
                            {
                                return(videoInfo);
                            }
                        }
                        else
                        {
                            var playTypes = playTypeOne.SelectNodes("h3");
                            if (playTypes != null)
                            {
                                foreach (var item in playTypes)
                                {
                                    //播放类型:
                                    PlayAddress routing = new PlayAddress();
                                    routing.playType = item.InnerText;
                                    if (routing.playType.Contains(":"))
                                    {
                                        routing.playType = routing.playType.Split(':')[1].Trim();
                                    }
                                    routing.videoUrl = new List <VideoUrl>();
                                    var Urls = item.SelectNodes("../ul/li");
                                    if (Urls != null)
                                    {
                                        foreach (var item1 in Urls)
                                        {
                                            VideoUrl videourl = new VideoUrl();
                                            string[] ars      = item1.InnerText.Split('$');
                                            videourl.playName = ars[0] != null ? ars[0] : " ";
                                            videourl.playURL  = ars[1] != null ? ars[1] : " ";
                                            routing.videoUrl.Add(videourl);
                                        }
                                        videoInfo.playInformation.Add(routing);
                                    }
                                }
                            }
                        }
                    }
                }
                return(videoInfo);
            }
            catch { return(null); }
        }
Exemplo n.º 16
0
    protected void Page_Load(object sender, EventArgs e)
    {
        try
        {
            string aStcokNumber = Request["Number"].ToString();

            StockData aData = new StockData();

            ////下載 Yahoo 奇摩股市資料
            HtmlWeb aWeb = new HtmlWeb();
            aWeb.OverrideEncoding = Encoding.GetEncoding("big5");

            ////使用預設編碼讀入 HTML
            HtmlDocument doc = new HtmlDocument();
            doc = aWeb.Load("http://tw.stock.yahoo.com/q/q?s=" + aStcokNumber);


            //裝載第一層查詢結果
            HtmlDocument docStockContext = new HtmlDocument();

            docStockContext.LoadHtml(doc.DocumentNode.SelectSingleNode("/html[1]/body[1]/center[1]/table[2]/tr[1]/td[1]/table[1]").InnerHtml);

            //取得個股標頭
            HtmlNodeCollection nodeHeaders = docStockContext.DocumentNode.SelectNodes("./tr[1]/th");

            //取得個股數值
            string[] aStockValues = docStockContext.DocumentNode.SelectSingleNode("./tr[2]").InnerText.Trim().Split('\n');

            for (int i = 0; i < aStockValues.Length; i++)
            {
                aStockValues[i] = aStockValues[i].Replace(" ", "");
            }

            List <string> aTitleData = new List <string>();

            //輸出資料
            foreach (HtmlNode nodeHeader in nodeHeaders)
            {
                if (nodeHeader.InnerText.Trim() != "")
                {
                    aTitleData.Add(HttpUtility.HtmlDecode(nodeHeader.InnerText));
                }
            }
            //0.代碼 1.時間(不需要) 2.成交價 3.買進價 4.賣出價 5.漲跌 6.張數 7.昨收 8.開盤 9.最高 10.最低
            aData.StockValue    = aStockValues;
            aData.StockValue[0] = aData.StockValue[0].Replace("加到投資組合", "");
            //取得股票資料標頭
            aData.StockTitle = aTitleData.ToArray();
            if (CheckStockData(aStcokNumber) == false)
            {
                InsertToSQL(aStockValues, aStcokNumber);
            }
            else
            {
                UpdateSQLData(aStockValues, aStcokNumber);
            }

            //取得新聞標題
            aData.NewsTitle = GetNewsTitle(doc);
            //取得新聞URL
            aData.NewsURL = GetNewsURL(doc);

            //aData.IsFavourite = CheckIsFavourite(aStcokNumber);

            doc             = null;
            docStockContext = null;

            aData.Result = "0";

            string aJsonStr = JsonConvert.SerializeObject(aData, new JsonSerializerSettings()
            {
                StringEscapeHandling = StringEscapeHandling.EscapeNonAscii
            });

            Response.Write(aJsonStr);
        }
        catch (Exception ex)
        {
            Response.Write(ex);
        }
    }
        private static void ProcessVehicleListOnPage(HtmlNodeCollection vehicleListItems, string DealerName, ref List <VEHICLE> PreviouslyFoundVehicles, string Market)
        {
            if (vehicleListItems == null)
            {
                return;
            }

            char[] killcomma = { ',' };
            foreach (HtmlNode vehicle in vehicleListItems)
            {
                HtmlNode VehicleNode      = vehicle.SelectSingleNode("div[@data-vin]");
                string   VehicleVIN       = VehicleNode.Attributes["data-vin"].Value.ToString();
                string   VehicleBodyStyle = VehicleNode.Attributes["data-bodyStyle"].Value.ToString();
                string   VehicleMake      = VehicleNode.Attributes["data-make"].Value.ToString();
                string   VehicleYear      = VehicleNode.Attributes["data-year"].Value.ToString();
                string   VehicleModel     = VehicleNode.Attributes["data-model"].Value.ToString();
                string   VehicleTrim      = VehicleNode.Attributes["data-trim"].Value.ToString();
                string   VehicleExtColor  = VehicleNode.Attributes["data-exteriorcolor"].Value.ToString();

                VEHICLE ExistingVehicle = VehicleDBManager.GetVehicleByVIN(VehicleVIN);
                List <VehiclePriceHistory> PriceHistory = null;
                bool Existing = false;

                // Check this vehicle against the list of previously found vehicles.
                // If it is found, remove it from the list, which ultimately removes it
                // from the "sold vehicle" list.
                if (ExistingVehicle != null && PreviouslyFoundVehicles != null)
                {
                    Existing     = true;
                    PriceHistory = RetrieveHistory(ExistingVehicle.VEHICLE_HISTORY);
                    var FoundThisOne = (from v in PreviouslyFoundVehicles
                                        where v.VIN == VehicleVIN
                                        select v).SingleOrDefault <VEHICLE>();
                    if (!ReferenceEquals(null, FoundThisOne))
                    {
                        PreviouslyFoundVehicles.Remove((VEHICLE)FoundThisOne);
                    }
                }
                else
                {
                    PriceHistory = new List <VehiclePriceHistory>();
                }

                HtmlNodeCollection VehicleNodeChildren = vehicle.ChildNodes;

                HtmlNode LinkNode    = VehicleNodeChildren[1].SelectSingleNode("div/div[@class='media']/a[@href]");
                string   VehicleLink = BaseURI + LinkNode.Attributes["href"].Value.ToString();

                HtmlNode           DescriptionNode  = VehicleNodeChildren[1].SelectSingleNode("div/div[@class='description']");
                HtmlNodeCollection DescriptionLists = VehicleNodeChildren[1].SelectNodes("div/div[@class='description']/dl");

                string VehicleEngine    = string.Empty;
                string VehicleTrans     = string.Empty;
                string IntColor         = string.Empty;
                string Mileage          = string.Empty;
                string StockNumber      = string.Empty;
                string ModelCode        = string.Empty;
                string VehicleDriveType = string.Empty;

                foreach (HtmlNode DList in DescriptionLists)
                {
                    string ItemType = string.Empty;
                    foreach (HtmlNode Item in DList.ChildNodes)
                    {
                        string Data = string.Empty;

                        if (Item.Name == "dt")
                        {
                            ItemType = Item.InnerText;
                            continue;
                        }
                        else if (Item.Name == "dd")
                        {
                            Data = Item.InnerText.Trim(killcomma);
                        }
                        else
                        {
                            continue;
                        }

                        switch (ItemType)
                        {
                        case "Engine:":
                            VehicleEngine = Data;
                            break;

                        case "Transmission:":
                            if (VehicleTrans.Length <= Data.Length)
                            {
                                VehicleTrans = Data;
                            }
                            break;

                        case "Exterior Color:":
                            if (Data != VehicleBodyStyle)
                            {
                                VehicleExtColor = Data;
                            }
                            break;

                        case "Interior Color:":
                            IntColor = Data;
                            break;

                        case "Stock #:":
                            StockNumber = Data;
                            break;

                        case "Mileage:":
                            Mileage = Data.Trim();
                            Mileage = Mileage.Replace(",", "");
                            Mileage = Mileage.Replace("miles", "");
                            if (Mileage.Length == 0 || Mileage == null || Mileage == " ")
                            {
                                Mileage = "0";
                            }
                            break;

                        case "Model Code:":
                            ModelCode = Data;
                            break;

                        case "Drive Line:":
                            VehicleDriveType = Data;
                            break;
                        }

                        ItemType = string.Empty;
                    }
                }

                // Stuff we might have to go to vehicle detail pages to retrieve: Mileage, Drive type, price

                // Mileage not on search page? (Snethkamp)
                if (Mileage == string.Empty || Mileage == "UNKNOWN" || Mileage == "0")
                {
                    Mileage = GetVehicleDetail(VehicleLink, "ddc-span6", "Mileage", "UNKNOWN");
                    Mileage = Mileage.Trim();
                    Mileage = Mileage.Replace(",", "");
                    Mileage = Mileage.Replace("miles", "");
                    Mileage = Mileage.Replace(@"\n", "");
                }

                // Vehicle Drive Type not on search page?
                if (VehicleDriveType == string.Empty || VehicleDriveType == "UNKNOWN")
                {
                    VehicleDriveType = GetVehicleDetail(VehicleLink, "item powertrain", "Drive type", "UNKNOWN");
                }
                if (VehicleDriveType == string.Empty || VehicleDriveType == "UNKNOWN")
                {
                    VehicleDriveType = GetVehicleDetail(VehicleLink, "powertrain", "Drive type", "UNKNOWN");
                }

                // Wheel Size
                string WheelSizeInches = GetVehicleDetail(VehicleLink, "item suspension-handling", "Wheel size", "UNKNOWN");
                if (WheelSizeInches == "UNKNOWN")
                {
                    WheelSizeInches = GetVehicleDetail(VehicleLink, "suspension-handling", "Wheel size", "UNKNOWN");
                }
                if (WheelSizeInches == "UNKNOWN")
                {
                    WheelSizeInches = GetVehicleDetail(VehicleLink, "standard-features", "Wheel Diameter", "UNKNOWN");
                }

                // Vehicle Price
                string   Price     = string.Empty;
                HtmlNode PriceNode = VehicleNodeChildren[1].SelectSingleNode("div/div[@class='pricing-area has-buttons']");
                HtmlNode ValueNode = PriceNode.SelectSingleNode("ul/li/span/span[@class='value']");
                if (ValueNode != null)
                {
                    Price = PriceNode.SelectSingleNode("ul/li/span/span[@class='value']").InnerText;
                    Price = Price.Replace("$", "");
                    Price = Price.Replace(",", "");
                    if (Price.Contains("/mo") || Price.Contains("month"))
                    {
                        Price = "0.00";
                    }
                }
                VehiclePriceHistory vph = new VehiclePriceHistory();
                vph.VIN           = VehicleVIN;
                vph.Date_Recorded = DateTime.Now.ToLocalTime();
                vph.Price         = Price;
                PriceHistory.Add(vph);

                // Carfax
                string   CarfaxURL  = string.Empty;
                HtmlNode CarfaxNode = DescriptionNode.SelectSingleNode("div[@class='calloutDetails']/ul/li[@class='carfax']");
                if (CarfaxNode != null)
                {
                    CarfaxURL = CarfaxNode.SelectSingleNode("a").Attributes["href"].Value.ToString();
                }

                // Images
                // List<int> ImageIds = GetandStoreVehicleImages(VehicleLink);

                //var strImageIdListcsv = string.Join(", ", ImageIds);
                // Console.WriteLine(strImageIdListcsv);

                if (ExistingVehicle != null)
                {
                    ExistingVehicle.VEHICLE_HISTORY   = SaveHistory(PriceHistory);
                    ExistingVehicle.BODY_STYLE        = VehicleBodyStyle;
                    ExistingVehicle.CARFAX_URL        = CarfaxURL;
                    ExistingVehicle.COLOR_EXTERIOR    = VehicleExtColor;
                    ExistingVehicle.COLOR_INTERIOR    = IntColor;
                    ExistingVehicle.CURRENT_PRICE     = Price;
                    ExistingVehicle.DATE_LAST_SEEN    = DateTime.Now;
                    ExistingVehicle.DEALERSHIP_NAME   = DealerName;
                    ExistingVehicle.DEALER_DETAIL_URL = VehicleDetailURL;
                    ExistingVehicle.DRIVE_TRAIN       = VehicleDriveType;
                    ExistingVehicle.ENGINE            = VehicleEngine;
                    ExistingVehicle.MAKE   = VehicleMake;
                    ExistingVehicle.MARKET = Market;
                    //ExistingVehicle.IMAGEIDCSV = strImageIdListcsv;
                    ExistingVehicle.MILEAGE         = Mileage;
                    ExistingVehicle.MODEL           = VehicleModel;
                    ExistingVehicle.MODEL_CODE      = ModelCode;
                    ExistingVehicle.STOCK_NUMBER    = StockNumber;
                    ExistingVehicle.TRANSMISSION    = VehicleTrans;
                    ExistingVehicle.TRIM            = VehicleTrim;
                    ExistingVehicle.VEHICLE_HISTORY = SaveHistory(PriceHistory);
                    ExistingVehicle.VIN             = VehicleVIN;
                    ExistingVehicle.WHEEL_SIZE      = WheelSizeInches;
                    ExistingVehicle.YEAR            = VehicleYear;
                    VehicleDBManager.UpdateVehicleRecord(ExistingVehicle);
                }
                else
                {
                    VEHICLE foundvehicle = new VEHICLE();
                    foundvehicle.BODY_STYLE        = VehicleBodyStyle;
                    foundvehicle.CARFAX_URL        = CarfaxURL;
                    foundvehicle.COLOR_EXTERIOR    = VehicleExtColor;
                    foundvehicle.COLOR_INTERIOR    = IntColor;
                    foundvehicle.CURRENT_PRICE     = Price;
                    foundvehicle.DATE_LAST_SEEN    = DateTime.Now;
                    foundvehicle.DEALERSHIP_NAME   = DealerName;
                    foundvehicle.DEALER_DETAIL_URL = VehicleDetailURL;
                    foundvehicle.DRIVE_TRAIN       = VehicleDriveType;
                    foundvehicle.ENGINE            = VehicleEngine;
                    foundvehicle.MAKE   = VehicleMake;
                    foundvehicle.MARKET = Market;
                    //foundvehicle.IMAGEIDCSV = strImageIdListcsv;
                    foundvehicle.MILEAGE         = Mileage;
                    foundvehicle.MODEL           = VehicleModel;
                    foundvehicle.MODEL_CODE      = ModelCode;
                    foundvehicle.STOCK_NUMBER    = StockNumber;
                    foundvehicle.TRANSMISSION    = VehicleTrans;
                    foundvehicle.TRIM            = VehicleTrim;
                    foundvehicle.VEHICLE_HISTORY = SaveHistory(PriceHistory);
                    foundvehicle.VIN             = VehicleVIN;
                    foundvehicle.WHEEL_SIZE      = WheelSizeInches;
                    foundvehicle.YEAR            = VehicleYear;
                    VehicleDBManager.InsertVehicle(foundvehicle);
                }

                Console.WriteLine($@"
        VIN : {VehicleVIN}  Found Previously? {Existing}
            {VehicleYear} {VehicleMake} {VehicleModel} {VehicleTrim} ({VehicleBodyStyle})
            Drivetrain: Engine: {VehicleEngine} | Transmission: {VehicleTrans} 
                        Drive type: {VehicleDriveType} | Wheel size: {WheelSizeInches}
            Color:      Exterior: {VehicleExtColor} | Interior: {IntColor} 
            Mileage:    {Mileage}
            Price:      {Price}
                        Stock #: {StockNumber} | Model Code: {ModelCode}  
            Direct URL: {VehicleLink}
            Carfax URL: {CarfaxURL}");

                VehicleVIN               = StockNumber = VehicleYear = VehicleMake = VehicleModel = ModelCode =
                    VehicleTrim          = VehicleEngine = VehicleTrans = VehicleDriveType = WheelSizeInches =
                        VehicleBodyStyle = VehicleExtColor = Mileage = IntColor = Price = CarfaxURL = VehicleLink = string.Empty;
            }
        }
        public static void ProcessAllDealers()
        {
            WebClient client = new System.Net.WebClient();
            HtmlWeb   webGet = new HtmlWeb();

            List <DEALERSHIP> dealers = VehicleDBManager.GetDealerships("Dealer.Com");

            if (dealers == null)
            {
                return;
            }

            foreach (DEALERSHIP dealer in dealers)
            {
                string DealerName = dealer.DEALER_NAME;
                string MainUrl    = dealer.DEALER_URL;
                Console.WriteLine($"************\r\n*** Processing Dealer: {DealerName}\r\n************");
                HtmlDocument mainpage       = webGet.Load(MainUrl);
                bool         KeepProcessing = (mainpage != null);
                int          PageCount      = 0;

                BaseURI = new Uri(MainUrl).GetLeftPart(UriPartial.Authority);
                Console.WriteLine($"Base URL for dealership: {BaseURI}");
                List <VEHICLE> PreviouslyFoundAtDealer = VehicleDBManager.GetAllUnsoldVehiclesForDealer(DealerName);

                while (KeepProcessing)
                {
                    Console.WriteLine($"\r\n\r\bRetrieving page #{++PageCount} of vehicles for dealer {DealerName} in the {dealer.MARKET_AREA_NAME} Market Area\r\n");

                    // Note: if the items are "shared", then they are listings from affiliated dealerships.
                    HtmlNodeCollection VehicleListItems = mainpage.DocumentNode.SelectNodes("//li[substring(@class, 1, 14)='item notshared']");
                    ProcessVehicleListOnPage(VehicleListItems, DealerName, ref PreviouslyFoundAtDealer, dealer.MARKET_AREA_NAME);

                    HtmlNode NextLink = mainpage.DocumentNode.SelectSingleNode("//a[@class='ddc-btn ddc-btn-link ddc-btn-xsmall'][@rel='next']");
                    string   Next     = string.Empty;
                    if (NextLink != null)
                    {
                        Next = NextLink.Attributes["href"].Value.ToString();
                        string NextURL = MainUrl + Next;
                        mainpage       = webGet.Load(NextURL);
                        KeepProcessing = (mainpage != null);
                    }
                    else
                    {
                        KeepProcessing = false;
                    }
                }

                if (!KeepProcessing && PreviouslyFoundAtDealer != null)
                {
                    // All the vehicles currently at this dealer have been screen-scraped.
                    // Now go through the complete list of "STILL_FOR_SALE" vehicles from the previous run
                    // and update any that have disappeared from their website as being sold or auctioned.

                    Console.WriteLine($"Found {PreviouslyFoundAtDealer.Count} vehicle(s) removed from dealer inventory since last run.");
                    foreach (VEHICLE SoldOrAuctionedVehicle in PreviouslyFoundAtDealer)
                    {
                        SoldOrAuctionedVehicle.STILL_FOR_SALE = "NO";
                        List <VehiclePriceHistory> PriceHistory = RetrieveHistory(SoldOrAuctionedVehicle.VEHICLE_HISTORY);
                        var MinDate           = (from pricehistories in PriceHistory select pricehistories.Date_Recorded).Min();
                        var MaxDate           = (from pricehistories in PriceHistory select pricehistories.Date_Recorded).Max();
                        var DaysOnMarket      = MaxDate - MinDate;
                        var FinalPriceHistory = (from pricehistories in PriceHistory
                                                 where pricehistories.Date_Recorded == MaxDate
                                                 select pricehistories).SingleOrDefault <VehiclePriceHistory>();
                        FinalPriceHistory.WasFinalPrice        = "YES";
                        SoldOrAuctionedVehicle.VEHICLE_HISTORY = SaveHistory(PriceHistory);
                        Console.WriteLine($@"Updating VIN {SoldOrAuctionedVehicle.VIN} : {SoldOrAuctionedVehicle.YEAR} {SoldOrAuctionedVehicle.MAKE} {SoldOrAuctionedVehicle.MODEL} (Stock number {SoldOrAuctionedVehicle.STOCK_NUMBER}) as sold.  Final price was {FinalPriceHistory.Price}, Mileage was {SoldOrAuctionedVehicle.MILEAGE}, Days on market was {DaysOnMarket.Days}");
                        VehicleDBManager.UpdateVehicleRecord(SoldOrAuctionedVehicle);
                    }
                }
            }
        }
Exemplo n.º 19
0
        public void ItemByIndexTest()
        {
            HtmlElement root = new HtmlElement("root");
            HtmlNodeCollection target = new HtmlNodeCollection(root);
            target.Add(new HtmlElement("first"));
            target.Add(new HtmlElement("second"));
            target.Add(new HtmlElement("third"));

            Assert.AreEqual(target[1], target["second"]);

            target[2] = new HtmlElement("another");
            target[0] = null;

            StringAssert.Contains(target[2].ToString(), "another");
        }
Exemplo n.º 20
0
        /// <summary>
        /// Lấy thêm thông tin (Link Stream, Size, PictureCover)vào đối tượng song của lớp Media.NowPlayingSong
        /// </summary>
        /// <param name="song"> đối tượng được bổ sung thông tin</param>
        public static List <Song> GetMoreSongInfo(string linkPage, out string linkStream, out string lyric, out string linkPicture,
                                                  out System.Windows.Media.Imaging.BitmapImage pictureCover, string quality,
                                                  out string size, string albumCover, bool isGetSize = false)
        {
            lyric       = "";
            size        = "";
            linkPicture = "";
            HtmlDocument wap = new HtmlDocument();

            wap.LoadHtml(SourceWeb.GetWebSource(linkPage.Replace("http://www", "http://m")));
            //Lấy lyrics
            try
            {
                HtmlNode lyricNode = wap.DocumentNode.SelectSingleNode("//div[@class='lyric']");
                foreach (HtmlNode node in lyricNode.ChildNodes)
                {
                    lyric += WebUtility.HtmlDecode(node.InnerText);
                }
            }
            catch { }
            //Lấy Stream
            try
            {
                HtmlNode stream = wap.DocumentNode.SelectSingleNode("//div[@class='download']");
                if (quality == null)
                {
                    linkStream = stream.SelectSingleNode(".//a").GetAttributeValue("href", "");
                    if (isGetSize)
                    {
                        WebRequest  request;
                        WebResponse reponse;
                        request        = WebRequest.Create(linkStream);
                        request.Method = "HEAD";
                        reponse        = request.GetResponse();
                        size           = Math.Round(reponse.ContentLength * 1.0 / 1024 / 1024, 2).ToString() + "MB";
                        reponse.Close();
                    }
                }
                else if (quality.Contains("128"))
                {
                    linkStream = stream.SelectSingleNode(".//a").GetAttributeValue("href", "");
                    if (isGetSize)
                    {
                        WebRequest  request;
                        WebResponse reponse;
                        request        = WebRequest.Create(linkStream);
                        request.Method = "HEAD";
                        reponse        = request.GetResponse();
                        size           = Math.Round(reponse.ContentLength * 1.0 / 1024 / 1024, 2).ToString() + "MB";
                        reponse.Close();
                    }
                }
                else
                {
                    WebRequest  request;
                    WebResponse reponse;
                    HtmlNode    pdlikeNode = wap.DocumentNode.SelectSingleNode("//div[@class='pdlike']");
                    HtmlNode    _blankNode = pdlikeNode.SelectSingleNode(".//a[@target='_blank']");
                    request    = WebRequest.Create(_blankNode.GetAttributeValue("href", ""));
                    reponse    = request.GetResponse();
                    linkStream = reponse.ResponseUri.ToString();
                    if (linkStream.Contains("login"))
                    {
                        linkStream = stream.SelectSingleNode(".//a").GetAttributeValue("href", "");
                    }
                    //size
                    if (isGetSize)
                    {
                        size = Math.Round(reponse.ContentLength * 1.0 / 1024 / 1024, 2).ToString() + "MB";
                    }
                    reponse.Close();
                }
            }
            catch { linkStream = null; }
            if (albumCover == null)
            {
                pictureCover = new System.Windows.Media.Imaging.BitmapImage(new Uri(/*MY_PACK_URIS.ALBUM_COVER_PICTURE.NHAC_CUA_TUI*/ null));
            }
            else
            {
                pictureCover = new System.Windows.Media.Imaging.BitmapImage(new Uri(albumCover));
                linkPicture  = albumCover;
            }
            //Lấy các bài hát gợi ý
            HtmlNodeCollection nodeSuggestSongs = wap.DocumentNode.SelectNodes("//div[@id='relatedSong']/div[@class='row bgmusic ']");

            if (nodeSuggestSongs == null)
            {
                return(null);
            }
            List <Song> listSuggestSong = new List <Song>();

            foreach (HtmlNode nodeSong in nodeSuggestSongs)
            {
                Song     tempSong   = new Song();
                HtmlNode nodeArtist = nodeSong.SelectSingleNode(".//p");
                tempSong.Name_Artist = nodeArtist.ChildNodes[1].InnerText;
                HtmlNode nodeTitle = nodeSong.SelectSingleNode(".//h3/a");
                tempSong.Name_Song  = nodeTitle.InnerText;
                tempSong.URL        = nodeTitle.GetAttributeValue("href", null).Replace("http://m", "http://www");
                tempSong.Name_Album = "Nhac Cua Tui";
                if (tempSong.URL != null)
                {
                    listSuggestSong.Add(tempSong);
                }
            }
            return(listSuggestSong);
        }
Exemplo n.º 21
0
        public override void FindItems(out List <Product> listOfProducts, SearchSettingsBase settings,
                                       CancellationToken token)
        {
            listOfProducts = new List <Product>();

            string searchUrl = UrlPrefix + string.Format(Keywords, settings.KeyWords) + PageSizeSuffix;

            //int genderId = 2; // 0 - all, 1 - mens, 2 - womens, 3 - boys, 4 - girls
            //int colorId = 9;
            int genderId = 0;
            int colorId  = 0;

            if (genderId != 0 || colorId != 0)
            {
                Color  color  = null;
                Gender gender = null;

                string genderString = "";
                string crumbs       = "";
                string cmREF;
                if (genderId != 0)
                {
                    gender       = genders[genderId];
                    genderString = "/" + gender.name;
                }

                if (colorId != 0)
                {
                    color = colors[colorId];
                }


                string colorGenderId = "";

                if (color != null && gender != null)
                {
                    colorGenderId = gender.id + "Z" + color.id;
                    crumbs        = gender.crumbs + " " + color.crumbs;
                    cmREF         = color.cmRef;
                }
                else if (color != null)
                {
                    colorGenderId = color.id;
                    crumbs        = color.crumbs;
                    cmREF         = color.cmRef;
                }
                else
                {
                    colorGenderId = gender.id;
                    crumbs        = gender.crumbs;
                    cmREF         = gender.cmRef;
                }


                searchUrl = WebsiteBaseUrl + string.Format(GenderColorPrefix, genderString, colorGenderId) + string.Format(Keywords, settings.KeyWords) + PageSizeSuffix + string.Format(GenderPostfix, crumbs, cmREF);
            }



            HtmlNode container = null;

            while (container == null)
            {
                HtmlNode node = InitialNavigation(searchUrl, token);
                container = node.SelectSingleNode(UlXpath);
            }

            HtmlNodeCollection children = container.SelectNodes("./li");

            foreach (HtmlNode child in children)
            {
                token.ThrowIfCancellationRequested();
#if DEBUG
                LoadSingleProduct(listOfProducts, child);
#else
                LoadSingleProductTryCatchWraper(listOfProducts, child);
#endif
            }
        }
Exemplo n.º 22
0
        public override List <Img> GetImages(string pageString, System.Net.IWebProxy proxy)
        {
            List <Img> imgs = new List <Img>();

            HtmlDocument doc = new HtmlDocument();

            doc.LoadHtml(pageString);

            //retrieve all elements via xpath
            HtmlNodeCollection nodes = null;

            if (srcType == PixivSrcType.Tag)
            {
                nodes = doc.DocumentNode.SelectSingleNode("//ul[@class='_image-items autopagerize_page_element']").SelectNodes("li");
                //nodes = nodes[nodes.Count - 1].SelectNodes("li");
            }
            else if (srcType == PixivSrcType.Author)
            {
                nodes = doc.DocumentNode.SelectSingleNode("//ul[@class='_image-items']").SelectNodes("li");
            }
            //else if (srcType == PixivSrcType.Day || srcType == PixivSrcType.Month || srcType == PixivSrcType.Week) //ranking
            //nodes = doc.DocumentNode.SelectSingleNode("//section[@class='ranking-items autopagerize_page_element']").SelectNodes("div");
            else
            {
                //ranking
                nodes = doc.DocumentNode.SelectNodes("//section[@class='ranking-item']");
            }

            if (nodes == null)
            {
                return(imgs);
            }

            foreach (HtmlNode imgNode in nodes)
            {
                try
                {
                    HtmlNode anode = imgNode.SelectSingleNode("a");
                    if (srcType == PixivSrcType.Day || srcType == PixivSrcType.Month || srcType == PixivSrcType.Week)
                    {
                        anode = imgNode.SelectSingleNode(".//div[@class='ranking-image-item']").SelectSingleNode("a");
                    }
                    //details will be extracted from here
                    //eg. member_illust.php?mode=medium&illust_id=29561307&ref=rn-b-5-thumbnail
                    string detailUrl  = anode.Attributes["href"].Value.Replace("amp;", "");
                    string previewUrl = null;
                    if (srcType == PixivSrcType.Tag || srcType == PixivSrcType.Author)
                    {
                        previewUrl = anode.SelectSingleNode(".//img").Attributes["src"].Value;
                    }
                    else
                    {
                        previewUrl = anode.SelectSingleNode(".//img").Attributes["data-src"].Value;
                    }

                    if (previewUrl.Contains('?'))
                    {
                        previewUrl = previewUrl.Substring(0, previewUrl.IndexOf('?'));
                    }

                    //extract id from detail url
                    //string id = detailUrl.Substring(detailUrl.LastIndexOf('=') + 1);
                    string id = System.Text.RegularExpressions.Regex.Match(detailUrl, @"illust_id=\d+").Value;
                    id = id.Substring(id.IndexOf('=') + 1);

                    Img img = GenerateImg(detailUrl, previewUrl, id);
                    if (img != null)
                    {
                        imgs.Add(img);
                    }
                }
                catch
                {
                    //int i = 0;
                }
            }

            return(imgs);
        }
Exemplo n.º 23
0
        /// <summary>
        /// This will move all the nodes from the specified index to the new parent.
        /// </summary>
        private static void MoveNodesDown(ref HtmlNodeCollection nodes, int index, HtmlElement newParent)
        {
            int count = nodes.Count;

            for (int i = index; i < count; i++)
            {
                ((HtmlElement)newParent).Nodes.Add(nodes[i]);
                nodes[i].SetParent(newParent);
            }

            for (int i = index; i < count; i++)
            {
                nodes.RemoveAt(index);
            }
            newParent.IsExplicitlyTerminated = true;
        }
 private static string GetMetaProperty(HtmlNodeCollection nodes, string prop)
 => HttpUtility.HtmlDecode(nodes.FirstOrDefault(x => x.Attributes["property"]?.Value == prop)?.Attributes["content"]?.Value);
Exemplo n.º 25
0
        /// <summary>
        /// Get friend's id of someone -- heavy method -- need refactor
        /// </summary>
        /// <param name="userIdOrAlias">
        /// If userId passed is blank then you will get your friend list.
        /// Else you will get friend list of this id.
        /// </param>
        /// <returns>List id of friends</returns>
        public UserInfo GetUserInfo(string userIdOrAlias, bool includeUserAbout = false, bool includedFriendList = false)
        {
            // TODO : messy method - we need refactor it.

            if (string.IsNullOrWhiteSpace(userIdOrAlias))
            {
                throw new ArgumentException("userIdOrAlias must not null or empty.");
            }

            UserInfo userInfo = new UserInfo();

            // TODO : Reduce check
            // The first time we passed userIdOrAlias
            // We don't know it is userIdOrAlias so we need to check
            // The second time when we known passed param is user id or alias
            // But we still need check again. it make perf decrease.

            string userAboutUrl = string.Empty;
            bool   isUserId     = !CompiledRegex.Match("NonDigit", userIdOrAlias).Success;

            if (isUserId)
            {
                userAboutUrl   = "https://m.facebook.com/profile.php?v=info&id=" + userIdOrAlias;
                userInfo.Id    = userIdOrAlias;
                userInfo.Alias = string.Empty;
            }
            else
            {
                userAboutUrl   = "https://m.facebook.com/" + userIdOrAlias + "/about";
                userInfo.Id    = string.Empty;
                userInfo.Alias = userIdOrAlias;
            }

            HtmlNode htmlDom = this.BuildDom(userAboutUrl);

            // Get avatar anchor tag :

            // avatarAnchorElem contain avatar image source, user display name and maybe contain id.
            // if userIdOrAlias is current user or other user with animated avatar then 1st xpath is wrong
            // pick another anchor
            HtmlNode avatarAnchor = htmlDom.SelectSingleNode("//div[@id='root']/div/div/div/div/div/a");

            if (avatarAnchor == null ||
                avatarAnchor.InnerText == ConstString.EditProfilePicture || // for current user
                avatarAnchor.InnerText == ConstString.AddProfilePicture)    // for animate avatar
            {
                avatarAnchor = htmlDom.SelectSingleNode("//div[@id='root']/div/div/div/div/div/div/a");
            }
            else if (avatarAnchor.SelectSingleNode("div/a/img") != null ||
                     (avatarAnchor.PreviousSibling != null && avatarAnchor.PreviousSibling.SelectSingleNode("/a/img") != null))
            {
                HtmlNodeCollection anchors = avatarAnchor.SelectNodes("div/a");
                if (anchors != null)
                {
                    foreach (HtmlNode anchor in anchors)
                    {
                        if (anchor.SelectSingleNode("img") != null)
                        {
                            avatarAnchor = anchor;
                            break;
                        }
                    }
                }
            }
            else
            {
                // Support another xpath to get user id
            }

            // get user id
            if (!isUserId && avatarAnchor != null)
            {
                Match idMatch = Match.Empty;

                // trying get id from avatar href
                var avatarHref = avatarAnchor.GetAttributeValue("href", null);

                // If we found avatar href, we might using it to detect user id
                if (avatarHref != null)
                {
                    // There is 3 pattern to detect user id
                    // If we get another url format, return this url for detect later.
                    // /photo.php?fbid=704517456378829&id=100004617430839&...
                    // /profile/picture/view/?profile_id=100003877944061&...
                    // /story.php\?story_fbid=\d+&amp;id=(?<id>\d+) for animate avatar
                    if ((idMatch = CompiledRegex.Match("UserIdFromAvatar1", avatarHref)).Success ||
                        (idMatch = CompiledRegex.Match("UserIdFromAvatar2", avatarHref)).Success ||
                        (idMatch = CompiledRegex.Match("UserIdFromAvatar3", avatarHref)).Success)
                    {
                        userInfo.Id = idMatch.Groups["id"].Value;
                    }
                }

                // try another way
                if (string.IsNullOrEmpty(userInfo.Id))
                {
                    // Trying to detect user id from hyperlink :
                    // Timeline · Friends · Photos · Likes · Followers · Following · [Activity Log]
                    // NOTE :
                    //      - Activity log only show in current user about page
                    //
                    // Important : /div/div/div/a must select before /div/div/a. Do not swap SelectNodes order.
                    HtmlNodeCollection anchors = htmlDom.SelectNodes("//div[@id='root']/div/div/div/a")
                                                 ?? htmlDom.SelectNodes("//div[@id='root']/div/div/a");

                    if (anchors != null && anchors.Count > 0)
                    {
                        foreach (HtmlNode anchor in anchors)
                        {
                            // Get and check hrefAttr and innerText
                            // If both of them have value then we can detect it using compiled pattern
                            // NOTE : Check pattern if you think it's incorrect.
                            string hrefAttr  = anchor.GetAttributeValue("href", string.Empty);
                            string innerText = anchor.InnerText;

                            if (!string.IsNullOrWhiteSpace(innerText) &&
                                (idMatch = CompiledRegex.Match(innerText, hrefAttr)).Success)
                            {
                                userInfo.Id = idMatch.Groups["id"].Value;
                                break;
                            }
                        }
                    }
                }

                // Try another way if id still empty
                if (string.IsNullOrEmpty(userInfo.Id))
                {
                    // Step 3 :
                    // trying get uid from action button : Add Friend, Message, Follow, More
                    // I only select the 1st xpath,the second xpath will be check in the future.
                    HtmlNodeCollection btnHrefts = htmlDom.SelectNodes("//div[@id='root']/div/div/div/table/tr/td/a");
                    // ??htmlDom.SelectNodes("//div[@id='root']/div/div/div");

                    // If we found some button nodes :
                    //      - Add Friend node if we does not add friend with this user
                    //      - Message if this user allow we can send message to him/her
                    //      - Follow if this user allow we can follow him/her and we not follow him/her before
                    //      - More if we can see more about user - i think that, maybe is incorrect.
                    if (btnHrefts != null && btnHrefts.Count > 0)
                    {
                        foreach (var btnHreft in btnHrefts)
                        {
                            // if href and innertext not null then we can trying detect user id by compiled regex
                            // NOTE :
                            //      - Check CompiledRegex if you think it not correct anymore
                            //      - Edit Key if you use another Language rather than English to access FB
                            string hrefAttr  = btnHreft.GetAttributeValue("href", string.Empty);
                            string innerText = btnHreft.InnerText;

                            if (!string.IsNullOrWhiteSpace(innerText) &&
                                (idMatch = CompiledRegex.Match(innerText, hrefAttr)).Success)
                            {
                                userInfo.Id = idMatch.Groups["id"].Value;
                                break;
                            }
                        }
                    }
                }
            }

            if (string.IsNullOrEmpty(userInfo.Id))
            {
                _logger.WriteLine("Could not detect id from " + userAboutUrl);
                return(null);
            }

            // if user id has been detected, check if we want to get user about too
            if (includeUserAbout && avatarAnchor != null)
            {
                HtmlNode avatar = avatarAnchor.SelectSingleNode("img");
                if (avatar != null)
                {
                    // Get name and avatar
                    userInfo.DispayName = WebUtility.HtmlDecode(avatar.GetAttributeValue("alt", string.Empty));
                    userInfo.AvatarUrl  = WebUtility.HtmlDecode(avatar.GetAttributeValue("src", string.Empty));
                }
                else
                {
                    _logger.WriteLine("Img tag in avatar is null. Addition info : " + userAboutUrl);
                }
            }

            // get user friend list if included
            // at this step we do not need to check user id anymore
            // if user id is null then we had return before.
            if (includedFriendList)
            {
                var firstFriendPageUrl = "https://m.facebook.com/profile.php?v=friends&startindex=0&id=" + userInfo.Id;
                userInfo.Friends = GetFriends(firstFriendPageUrl);
            }

            // all step have done
            return(userInfo);
        }
Exemplo n.º 26
0
        static void Main(string[] args)
        {
            string url       = "http://businessdirectory.esdlife.com/wedding/CH-TC/Wedding2CompInfo.php?pages=1&catID={0}";
            string detailurl = "http://businessdirectory.esdlife.com/wedding/CH-TC/{0}";
            string sqlresult = "";



            try
            {
                string urlfilepath      = rootDir + mappingfile;
                IEnumerable <string> ls = File.ReadLines(urlfilepath);
                string detailId         = "";


                foreach (string cats in ls.ToList())
                {
                    //HtmlWeb website = new HtmlWeb();
                    //HtmlAgilityPack.HtmlDocument doc = website.Load(path);
                    //dynamic data = Newtonsoft.Json.JsonConvert.DeserializeObject(doc.);
                    if (string.IsNullOrEmpty(cats))
                    {
                        continue;
                    }
                    string[] catgroup = cats.Split('\t');
                    string   cat      = catgroup[catgroup.Length - 1];
                    string   cat_88   = catgroup[1];
                    if (string.IsNullOrEmpty(cat_88))
                    {
                        cat_88 = cat;
                    }
                    string url_result = string.Format(url, cat);
                    int    page       = getPageParameter(url_result);
                    string pathurl    = url;
                    for (int pageindex = page; pageindex < 30; pageindex++)
                    {
                        pathurl = SetPageParameter(url_result, pageindex);
                        HtmlWeb websiteList = new HtmlWeb();
                        HtmlAgilityPack.HtmlDocument maindoc = websiteList.Load(pathurl);
                        HtmlNodeCollection           nodes   = maindoc.DocumentNode.SelectNodes("//div[@class='cn']/h3/a[@class='orangetxt']");

                        for (int i = 0; i < nodes.Count; i++)
                        {
                            string urlpath = string.Format(detailurl, nodes[i].Attributes["href"].Value);
                            Console.WriteLine("urlpath: " + urlpath);
                            detailId = nodes[i].Attributes["href"].Value.Replace("Wedding2ShowCompInfo.php?id=", "");
                            DataTable dt = SQLHelper.ExecuteDt("SELECT [temppostid] FROM [grabpost] with(nolock) where sourceRefId=" + detailId + " and sourceSite='" + sourceSite + "'");
                            if (dt.Rows.Count > 0)
                            {
                                continue;
                            }


                            HtmlWeb website = new HtmlWeb();
                            HtmlAgilityPack.HtmlDocument doc = website.Load(urlpath);
                            long categoryid = long.Parse(cat);



                            string membermobile   = "";
                            string mebercontactno = "";
                            string memberaddress  = "";
                            string memberemail    = "";
                            string memberwebsite  = "";
                            string memberavator   = "";
                            string membername     = "";
                            string district       = "";
                            string keyword        = "";
                            string postcontent    = "";
                            string summary        = "";
                            string title          = "";
                            string lat            = "";
                            string lng            = "";
                            float  flat           = 0;
                            float  flng           = 0;
                            string Catstring      = "";
                            string postcontactno1 = "";
                            string postcontactno2 = "";
                            string postaddress    = "";
                            string contactemail   = "";
                            //string keyword="";

                            HtmlNode titleNode   = doc.DocumentNode.SelectSingleNode("//h1[@class='d_title']");                                   //A
                            HtmlNode mainNode    = doc.DocumentNode.SelectSingleNode("//div[@class='text']");                                     //B,C,D,E,F,I
                            HtmlNode regionNode  = doc.DocumentNode.SelectSingleNode("//div[@id='guide_search']");                                //E
                            HtmlNode keywrodNode = doc.DocumentNode.SelectSingleNode("//div[@class='ser']/a");                                    //J
                            HtmlNode contentNode = doc.DocumentNode.SelectSingleNode("//div[@class='d_content']");                                //G  簡介:

                            HtmlNode lat_lng = doc.DocumentNode.SelectSingleNode("//div[@id='wedding_content']").PreviousSibling.PreviousSibling; //H
                            //HtmlNode memberavatorNode = doc.DocumentNode.SelectSingleNode("//div[@class='companyLogo']/img");



                            if (titleNode == null)
                            {
                                continue;
                            }
                            if (titleNode != null)
                            {
                                title      = titleNode.InnerHtml.Replace("\n", "").Trim();
                                membername = title;
                            }
                            if (mainNode != null)
                            {
                                foreach (HtmlNode node in mainNode.ChildNodes)
                                {
                                    if (node.InnerText.IndexOf("地址:") != -1)
                                    {
                                        postaddress   = node.InnerText.Replace("\n", "").Replace("\t", "").Replace(" ", "");
                                        memberaddress = postaddress;
                                    }

                                    if (node.InnerText.IndexOf("電話:") != -1)
                                    {
                                        string[] tels = node.InnerText.Replace("電話:", "").Replace("\n", "").Replace("\t", "").Replace(" ", "").Split('/');
                                        if (tels.Length > 1)
                                        {
                                            postcontactno1 = tels[0];
                                            postcontactno2 = tels[1];
                                        }
                                        else
                                        {
                                            postcontactno1 = tels[0];
                                            postcontactno2 = tels[0];
                                        }
                                    }

                                    if (node.InnerText.IndexOf("電郵:") != -1)
                                    {
                                        string[] emails = node.NextSibling.InnerText.Split('/');

                                        memberemail = emails[0].Trim();
                                        if (emails.Length > 1)
                                        {
                                            contactemail = emails[1].Trim();
                                        }
                                    }
                                    if (node.InnerText.IndexOf("網址:") != -1)
                                    {
                                        memberwebsite = node.NextSibling.InnerText;
                                    }
                                    if (node.InnerText.IndexOf("備註:") != -1)
                                    {
                                        summary = node.InnerText.Replace("備註:", "").Trim();
                                    }
                                }
                                //postcontent = desNode.InnerHtml;
                                //summary = desNode.InnerHtml;
                            }



                            if (keywrodNode != null)
                            {
                                keyword = keywrodNode.InnerHtml.Replace("\n", "").Trim();
                                string[] separators = { ",", "/", "|", ">", "." };
                                keyword = string.Join(",", keyword.Replace("\n", "").Replace("\t", "").Replace(" ", "").Split(separators, StringSplitOptions.RemoveEmptyEntries));
                                //keyword =  Catstring;
                            }

                            if (lat_lng != null)
                            {
                                string[] lat_lngs = lat_lng.InnerText.Split('\n');
                                foreach (string str in lat_lngs)
                                {
                                    string row = str.Trim().Replace(";", "");
                                    if (row.IndexOf("var MerchantLatitude") != -1)
                                    {
                                        string[] separators = { "=" };
                                        string[] lats       = row.Split(separators, StringSplitOptions.RemoveEmptyEntries);
                                        if (lats.Length >= 2)
                                        {
                                            lat = lats[1].Replace("'", "");
                                            float.TryParse(lat, out flat);
                                            continue;
                                        }
                                    }
                                    if (row.IndexOf("var MerchantLongitude") != -1)
                                    {
                                        string[] separators = { "=" };
                                        string[] lngs       = row.Split(separators, StringSplitOptions.RemoveEmptyEntries);
                                        if (lngs.Length >= 2)
                                        {
                                            lng = lngs[1].Replace("'", "");
                                            float.TryParse(lng, out flng);
                                            break;
                                        }
                                    }
                                }
                            }

                            if (contentNode != null)
                            {
                                postcontent = contentNode.InnerText.Replace("簡介:", "").Trim();
                            }

                            if (regionNode != null)
                            {
                                //regionNode.LastChild.
                            }



                            /*
                             *                          exec [dbo].insertgrabpost
                             * @categoryid = 1197,    --->cat_88
                             * @title =N'物流tile2333', --> title
                             * @summary ='summary',--->desNode
                             * @postcontent = 'content', --->postcontent
                             * @keyword ='keyword', --->keyword
                             * @district =N'香港島', --->district
                             * @membername =N'abc', --->membername
                             * @memberemail ='*****@*****.**', ---->memberemail
                             * @memberavator ='http://avator', ---->memberavator
                             * @membermobile ='+86-187238273', --->membermobile
                             * @membercontactno ='932742',   ---->mebercontactno
                             * @memberaddress =N'adlfjeosdfdsfsd',   --->memberaddress
                             * @memberwebsite ='http://www.88db.com',  ---->memberwebsite
                             * @postcontactno1 ='22222',   ---->postcontactno1
                             * @postcontactno2 =N'33333你好',--->postcontactno2
                             * @postaddress =N'广告地址', --->postaddress
                             * @postlatitude = '12323',   //lat
                             * @postlongitude ='23232',   //lng
                             * @contactemail =N'33333你好',  --->contactemail
                             * @memberdesc =N'成员描述111111' --->desNode
                             */
                            SQLHelper      sqlHelper  = new SQLHelper();
                            SqlParameter[] parameters =
                            {
                                new SqlParameter("@categoryid",      SqlDbType.BigInt)
                                {
                                    Value = cat_88
                                },
                                new SqlParameter("@title",           SqlDbType.NVarChar, 512)
                                {
                                    Value = title
                                },
                                new SqlParameter("@summary",         SqlDbType.NVarChar, 512)
                                {
                                    Value = summary
                                },
                                new SqlParameter("@postcontent",     SqlDbType.NVarChar, -1)
                                {
                                    Value = postcontent
                                },

                                new SqlParameter("@keyword",         SqlDbType.NVarChar, 512)
                                {
                                    Value = keyword
                                },

                                new SqlParameter("@district",        SqlDbType.NVarChar, 64)
                                {
                                    Value = district
                                },

                                new SqlParameter("@membername",      SqlDbType.NVarChar, 256)
                                {
                                    Value = membername
                                },

                                new SqlParameter("@memberemail",     SqlDbType.VarChar, 100)
                                {
                                    Value = memberemail
                                },

                                new SqlParameter("@memberavator",    SqlDbType.VarChar, 512)
                                {
                                    Value = memberavator
                                },

                                new SqlParameter("@membermobile",    SqlDbType.VarChar, 30)
                                {
                                    Value = membermobile
                                },

                                new SqlParameter("@membercontactno", SqlDbType.VarChar, 30)
                                {
                                    Value = ""
                                },

                                new SqlParameter("@memberaddress",   SqlDbType.NVarChar, 512)
                                {
                                    Value = memberaddress
                                },

                                new SqlParameter("@memberwebsite",   SqlDbType.NVarChar, 512)
                                {
                                    Value = memberwebsite
                                },
                                //new
                                new SqlParameter("@postcontactno1",  SqlDbType.NVarChar, 30)
                                {
                                    Value = postcontactno1
                                },
                                new SqlParameter("@postcontactno2",  SqlDbType.NVarChar, 30)
                                {
                                    Value = postcontactno2
                                },
                                new SqlParameter("@postaddress",     SqlDbType.NVarChar, 512)
                                {
                                    Value = postaddress
                                },
                                new SqlParameter("@postlatitude",    SqlDbType.Float)
                                {
                                    Value = flat
                                },
                                new SqlParameter("@postlongitude",   SqlDbType.Float)
                                {
                                    Value = flng
                                },
                                new SqlParameter("@contactemail",    SqlDbType.NVarChar, 100)
                                {
                                    Value = contactemail
                                },
                                new SqlParameter("@memberdesc",      SqlDbType.NVarChar, 1000)
                                {
                                    Value = summary
                                },
                                new SqlParameter("@sourceSite",      SqlDbType.NVarChar, 100)
                                {
                                    Value = sourceSite
                                },
                                new SqlParameter("@sourceRefID",     SqlDbType.NVarChar, 100)
                                {
                                    Value = detailId
                                }
                            };
                            StringBuilder paramstr = new StringBuilder();
                            paramstr.Append(" exec [dbo].insertgrabpost{");
                            //paramstr=parameters.t
                            foreach (SqlParameter p in parameters)
                            {
                                paramstr.Append(p.ParameterName + "=N'" + p.Value + "',").AppendLine(); //paramstr + p.ParameterName + "=" + p.Value+"||||";
                            }
                            paramstr.Append("}");
                            sqlresult = paramstr.ToString();
                            sqlHelper.RunProc("[dbo].insertgrabpost", parameters);
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                File.AppendAllText(rootDir + logfile, ex.StackTrace + sqlresult);
            }
        }
Exemplo n.º 27
0
        private static string ParseHtmlBrsBodytoJson(string input)
        {
            string        output     = null;
            var           isTD       = true;
            StringBuilder jsonstring = new StringBuilder();

            jsonstring.Append("{");
            bool   isPreviousProperty = false;
            string property           = null;
            string value = null;

            if (!String.IsNullOrEmpty(input))
            {
                var document = new HtmlAgilityPack.HtmlDocument();
                document.LoadHtml(input);
                var metaTags = document.DocumentNode.SelectNodes("//meta");

                if (metaTags != null)
                {
                    foreach (var tag in metaTags)
                    {
                        if (tag.Attributes["name"] != null && tag.Attributes["content"] != null)
                        {
                            if (tag.Attributes["name"].Value == "Generator")

                            {
                                isTD = false;
                            }
                        }
                    }
                }
                else
                {
                    isTD = true;
                }
                if (!isTD)
                {
                    HtmlNodeCollection brs = document.DocumentNode.SelectNodes("//body");
                    if (brs != null && brs.Count > 0)
                    {
                        HtmlNode br     = brs[0];
                        string   brtext = br.InnerText;
                        string[] lines  = brtext.Split('\r');
                        foreach (string line in lines)
                        {
                            string[] propertyvalue = line.Split(':');
                            if (propertyvalue.Length == 2)
                            {
                                if (jsonstring.Length > 1)
                                {
                                    jsonstring.Append(",");
                                }
                                //property = propertyvalue[0].Replace(" ", string.Empty).Replace(":", string.Empty).Replace("&nbsp;", string.Empty).Replace("\r\n", string.Empty).Trim();
                                property = propertyvalue[0].Replace(" ", string.Empty).Replace(":", string.Empty).Replace("&nbsp;", string.Empty).Replace("?", string.Empty).Replace("/", string.Empty).Replace("(", string.Empty).Replace(")", string.Empty).Replace("-", string.Empty).Replace("\r\n", string.Empty).Trim();
                                jsonstring.AppendFormat("'{0}':", property);
                                value = propertyvalue[1].Replace(" ", string.Empty).Replace(":", string.Empty).Replace("&nbsp;", string.Empty).Replace("\r\n", string.Empty).Trim();
                                jsonstring.AppendFormat("'{0}'", value);
                            }
                        }
                    }
                }
                else
                {
                    HtmlNodeCollection tds = document.DocumentNode.SelectNodes("//td");
                    if (tds != null && tds.Count > 0)
                    {
                        foreach (HtmlNode td in tds)
                        {
                            string tdtext = td.InnerText;
                            if (tdtext.Contains("BusinessLocationCity1"))
                            {
                                Console.WriteLine(tdtext.ToString());
                                //MessageBox.Show(tdtext.ToString());
                            }

                            if ((tdtext.Contains(":") && !tdtext.Contains("Last Modified") && !tdtext.Contains(":/")) || (tdtext.Contains("-") && !tdtext.Any(c => char.IsDigit(c))) || (tdtext.Contains("=") && !tdtext.Contains("?"))) //TODO:  IDENTIFY PROPERTY VS VALUE BETTER
                            {
                                if (jsonstring.Length > 1)
                                {
                                    jsonstring.Append(",");
                                }
                                isPreviousProperty = true;
                                property           = td.InnerText.Replace(" ", string.Empty).Replace(":", string.Empty).Replace("&nbsp;", string.Empty).Replace("\r\n", string.Empty).Replace("(", string.Empty).Replace(")", string.Empty).Replace("?", string.Empty).Replace(".", string.Empty).Trim();
                                jsonstring.AppendFormat("'{0}':", property);
                            }
                            else if (isPreviousProperty)
                            {
                                isPreviousProperty = false;
                                value = td.InnerText.Replace("&nbsp;", string.Empty).Replace("\r\n", string.Empty).Replace("'", string.Empty).Trim();
                                jsonstring.AppendFormat("'{0}'", value);
                            }
                        }
                    }
                }
            }
            jsonstring.Append("}");
            var deserialized = JsonConvert.DeserializeObject(jsonstring.ToString());

            output = JsonConvert.SerializeObject(deserialized);
            if (output == "{}")
            {
                output = null;
            }
            return(output);
        }
Exemplo n.º 28
0
        public static async Task <string> GetSubtitle(Movie movie)
        {
            //get movie info from open-movie-db-api
            var client = new HttpClient();
            HttpResponseMessage response = await client.GetAsync($"http://www.omdbapi.com/?&apikey={omdbapikey}={movie.Name}&y={movie.Year}&r=xml");

            string result = await response.Content.ReadAsStringAsync();

            XmlDocument doc = new XmlDocument();

            doc.LoadXml(result);
            string id = string.Empty;

            foreach (XmlNode node in doc.DocumentElement)
            {
                id = node.Attributes[17].Value;
            }

            //get subtitle list page
            client   = new HttpClient();
            response = await client.GetAsync($"https://yts-subs.com/movie-imdb/{id}");

            result = await response.Content.ReadAsStringAsync();

            //filter data and find English subtitle
            var sublistpage = new HtmlDocument();

            sublistpage.LoadHtml(result);
            HtmlNodeCollection nodes = sublistpage.DocumentNode.SelectNodes("//table[@class='table other-subs']/tbody/tr");
            string             link  = string.Empty;

            foreach (var node in nodes)
            {
                var cell = node.SelectNodes("./td");
                if (cell[1].InnerText.Trim().Equals("English"))
                {
                    //get download page link
                    link = cell[4].SelectSingleNode("./a").Attributes["href"].Value;
                }
            }

            if (string.IsNullOrWhiteSpace(link))
            {
                return(string.Empty);
            }

            //get download page
            client   = new HttpClient();
            response = await client.GetAsync($"https://yts-subs.com{link}");

            result = await response.Content.ReadAsStringAsync();

            var downloadpage = new HtmlDocument();

            downloadpage.LoadHtml(result);
            var divs         = downloadpage.DocumentNode.SelectNodes("//div[@class='col-xs-12']");
            var downloadlink = divs[1].SelectSingleNode("./a").Attributes["href"].Value;

            if (string.IsNullOrWhiteSpace(downloadlink))
            {
                return(string.Empty);
            }

            Uri uri = new Uri(downloadlink);

            tempfilepath = Path.Combine(Path.GetTempPath(), Path.GetFileName(uri.LocalPath));


            using (var wc = new WebClient())
            {
                wc.DownloadFileCompleted += Wc_DownloadFileCompleted;
                wc.DownloadFileAsync(uri, tempfilepath);
            }

            client.Dispose();

            return(downloadlink);
        }
Exemplo n.º 29
0
        public override List <Img> GetImages(string pageString, IWebProxy proxy)
        {
            List <Img> list = new List <Img>();

            //API
            if (APImode)
            {
                list = booru.GetImages(pageString, proxy);
                if (list.Count > 0)
                {
                    return(list);
                }
            }

            //Html
            HtmlDocument document = new HtmlDocument();

            document.LoadHtml(pageString);
            HtmlNodeCollection previewNodes = document.DocumentNode.SelectNodes("//div[@class=\"thumbnail-preview\"]");

            if (previewNodes == null)
            {
                return(list);
            }
            foreach (HtmlNode node in previewNodes)
            {
                HtmlNode node1     = node.SelectSingleNode("./span/a");
                HtmlNode node2     = node1.SelectSingleNode("./img");
                string   detailUrl = FormattedImgUrl(node1.Attributes["href"].Value);
                string   desc      = Regex.Match(node1.InnerHtml, "(?<=title=\" ).*?(?=  score)").Value;
                Img      item      = new Img()
                {
                    Desc       = desc,
                    Tags       = desc,
                    Id         = Convert.ToInt32(Regex.Match(node1.Attributes["id"].Value, @"\d+").Value),
                    DetailUrl  = detailUrl,
                    PreviewUrl = node2.Attributes["data-original"].Value
                                 //PreviewUrl = node1.InnerHtml.Substring(node1.InnerHtml.IndexOf("original=\"") + 10,
                                 //        node1.InnerHtml.IndexOf("\" src") - node1.InnerHtml.IndexOf("original=\"") - 10)
                };
                item.DownloadDetail = (i, p) =>
                {
                    string html = new MyWebClient {
                        Proxy = p, Encoding = Encoding.UTF8
                    }.DownloadString(i.DetailUrl);
                    HtmlDocument doc = new HtmlDocument();
                    doc.LoadHtml(html);
                    HtmlNodeCollection liNodes = doc.DocumentNode.SelectNodes("//li");
                    HtmlNode           imgData = doc.DocumentNode.SelectSingleNode("//*[@id=\"image\"]");
                    if (imgData != null)
                    {
                        i.Width     = Convert.ToInt32(imgData.Attributes["data-original-width"].Value);
                        i.Height    = Convert.ToInt32(imgData.Attributes["data-original-height"].Value);
                        i.SampleUrl = imgData.Attributes["src"].Value;
                    }
                    foreach (HtmlNode n in liNodes)
                    {
                        if (n.InnerText.Contains("Posted"))
                        {
                            i.Date = n.InnerText.Substring(n.InnerText.IndexOf("ed: ") + 3, n.InnerText.IndexOf(" by") - n.InnerText.IndexOf("d: ") - 3);
                        }
                        if (n.InnerHtml.Contains("by"))
                        {
                            i.Author = n.InnerText.Substring(n.InnerText.LastIndexOf(' ') + 1, n.InnerText.Length - n.InnerText.LastIndexOf(' ') - 1);
                        }
                        if (n.InnerText.Contains("Source"))
                        {
                            i.Source = n.SelectSingleNode("//*[@rel=\"nofollow\"]").Attributes["href"].Value;
                        }
                        if (n.InnerText.Contains("Rating") && n.InnerText.Contains("Safe"))
                        {
                            i.IsExplicit = false;
                        }
                        else if (n.InnerText.Contains("Rating"))
                        {
                            i.IsExplicit = true;
                        }
                        if (n.InnerText.Contains("Rating") && n.InnerText.Contains("Safe"))
                        {
                            i.IsExplicit = false;
                        }
                        else if (n.InnerText.Contains("Rating"))
                        {
                            i.IsExplicit = true;
                        }

                        if (n.InnerText.Contains("Score"))
                        {
                            i.Score = Convert.ToInt32(n.SelectSingleNode("./span").InnerText);
                        }
                        if (n.InnerHtml.Contains("Original"))
                        {
                            i.OriginalUrl = n.SelectSingleNode("./a").Attributes["href"].Value;
                            i.JpegUrl     = n.SelectSingleNode("./a").Attributes["href"].Value;
                        }
                    }
                };
                list.Add(item);
            }
            return(list);
        }
Exemplo n.º 30
0
        public override void readTablesFromWeb(HtmlDocument page)
        {
            string cvss = "";
            string content;

            if (page.DocumentNode != null)
            {
                if (page.DocumentNode.SelectNodes("//table[@class='dataTable']") != null)
                {
                    foreach (HtmlNode table in page.DocumentNode.SelectNodes("//table[@class='dataTable']"))
                    {
                        if (table.SelectNodes("thead") != null)
                        {
                            HtmlNode head = table.SelectSingleNode("thead");
                            if (head.SelectNodes("tr") != null)
                            {
                                HtmlNode           headRow  = head.SelectSingleNode("tr");
                                HtmlNodeCollection headCols = headRow.SelectNodes("th");
                                if ((headCols.Count == 4) || (headCols.Count == 5) && (scap.myTable.isValidHeader(headCols)))//This table have 4 or 5 colums, so it is a valid table
                                {
                                    scap.myTable.setHeaderSupporteTrue();
                                    scap.myTable.setSystemOrComponent(headCols[0].InnerText);
                                    if (table.SelectNodes("tbody") != null)
                                    {
                                        foreach (HtmlNode body in table.SelectNodes("tbody"))
                                        {
                                            if (body.SelectNodes("tr") != null)
                                            {
                                                HtmlNode bodyRows = body.SelectSingleNode("tr");
                                                if ((bodyRows.SelectNodes("th") != null))
                                                {
                                                    HtmlNode bodyHead = bodyRows.SelectSingleNode("th");  // we are assuming that there is gonna be only on <th> tag
                                                    content = bodyHead.InnerText;
                                                    scap.myTable.setSupportedHeader(content);
                                                }
                                                else
                                                {
                                                    foreach (HtmlNode row in body.SelectNodes("tr"))
                                                    {
                                                        if (row.SelectNodes("td") != null)
                                                        {
                                                            HtmlNodeCollection cols = row.SelectNodes("td");
                                                            if (cols.Count == 4)   // This are the only rows that we are interested in
                                                            {
                                                                processRowFourCols(cols);
                                                            }

                                                            if (cols.Count == 5)
                                                            {
                                                                processRowFiveCols(cols);
                                                            }
                                                        }
                                                    }
                                                }
                                            }
                                        }
                                    }
                                }
                                if (table.SelectNodes("caption") != null)
                                {
                                    HtmlNode caption = table.SelectSingleNode("caption");
                                    if (scap.myTable.isCVEHeader(caption.InnerText))
                                    {
                                        cvss += scap.myTable.getCVS(headCols);
                                        break;
                                    }
                                }
                            }
                        }
                    }
                }
                scap.set_Cvss(cvss);
            }

            /*if (document.Tables.Count > 0)
             * {
             *  foreach (Novacode.Table table in document.Tables)
             *  {
             *      if (myTable.containsAffectedSoftware(table.Rows[0]))
             *      {
             *          extractTableContent(table);
             *      }
             *      else
             *      {
             *          if (myTable.tableContainsCVEs(table.Rows[0]))
             *          {
             *              cvss += myTable.getCVS(table);
             *              break;
             *          }
             *      }
             *  }
             * }
             * set_Cvss(cvss);            */
        }
Exemplo n.º 31
0
 //
 //====================================================================================================
 /// <summary>
 /// Process and return an htmlagility object. Any processing errors are returned in userMessageList. If UserMessageList.count is 0, there wre no errors.
 /// </summary>
 /// <param name="cp"></param>
 /// <param name="htmlDoc"></param>
 /// <param name="importTypeId"></param>
 /// <param name="userMessageList"></param>
 /// <returns></returns>
 private static bool processHtmlDoc(CPBaseClass cp, HtmlDocument htmlDoc, ImporttypeEnum importTypeId, ref List <string> userMessageList)
 {
     //
     // -- get body (except email template because it uses the full html document
     if (importTypeId != ImporttypeEnum.EmailTemplate)
     {
         //
         // -- find the data-body or body tag
         {
             string             xPath    = "//*[@data-body]";
             HtmlNodeCollection nodeList = htmlDoc.DocumentNode.SelectNodes(xPath);
             if (nodeList != null)
             {
                 //
                 // -- import data-body
                 userMessageList.Add("Body set by data-body attribute.");
                 htmlDoc.LoadHtml(nodeList.First().InnerHtml);
             }
             else
             {
                 HtmlNode bodyNode = htmlDoc.DocumentNode.SelectSingleNode("//body");
                 if (bodyNode == null)
                 {
                     //
                     // -- no body found, use entire document
                     userMessageList.Add("No Body found, entire document imported.");
                 }
                 else
                 {
                     //
                     // -- use body
                     string body = bodyNode.InnerHtml;
                     if (string.IsNullOrWhiteSpace(body))
                     {
                         //
                         // -- body tag not found, import the whole document
                         userMessageList.Add("The content does not include a data-body attribute and the body tag is empty.");
                         return(false);
                     }
                     //
                     // -- body found, set the htmlDoc to the body
                     userMessageList.Add("Html Body imported.");
                     htmlDoc.LoadHtml(body);
                 }
             }
         }
     }
     //
     // -- process data-layout nodes. Within each node found, run all other controllers inddividually then save
     DataLayoutController.process(cp, htmlDoc, ref userMessageList);
     //
     // -- process the body
     DataDeleteController.process(htmlDoc);
     MustacheVariableController.process(htmlDoc);
     MustacheSectionController.process(htmlDoc);
     MustacheTruthyController.process(htmlDoc);
     MustacheInvertedSectionController.process(htmlDoc);
     MustacheValueController.process(htmlDoc);
     DataAddonController.process(cp, htmlDoc);
     DataHrefController.process(htmlDoc);
     DataSrcController.process(htmlDoc);
     //
     return(true);
 }
Exemplo n.º 32
0
        public void DownloadCouncilPdfFiles()
        {
            var docs    = this.LoadDocumentsDoneSQL();
            var queries = this.LoadQueriesDoneSQL();
            // var docs = new List<Documents>();
            // var queries = new List<QueryResult>();
            WebClient c   = new WebClient();
            HtmlWeb   web = new HtmlWeb();
            Dictionary <Regex, string> dateRegFormatDic = new Dictionary <Regex, string>();

            dateRegFormatDic.Add(new Regex("[A-Za-z]+[\\s]{0,1}[0-9]{1,2},[\\s]{0,1}[0-9]{4}"), " ");
            dateRegFormatDic.Add(new Regex("[A-Za-z]+[\\s]{0,1}[0-9]{1,2}"), "Month");
            dateRegFormatDic.Add(new Regex("[0-9]{2}-[0-9]{2}-[0-9]{2}"), "MM-dd-yy");
            dateRegFormatDic.Add(new Regex("[0-9]{2}-[0-9]{1}-[0-9]{2}"), "MM-d-yy");
            dateRegFormatDic.Add(new Regex("[0-9]{1}-[0-9]{2}-[0-9]{2}"), "M-dd-yy");
            dateRegFormatDic.Add(new Regex("[0-9]{1}-[0-9]{1}-[0-9]{2}"), "M-d-yy");
            foreach (string url in this.docUrls)
            {
                var                subUrl   = url.Split('*')[1];
                var                category = url.Split('*')[0];
                HtmlDocument       doc      = web.Load(subUrl);
                HtmlNodeCollection list     = doc.DocumentNode.SelectNodes("//a[contains(@href,'/LinkClick.aspx')]");
                foreach (var r in list)
                {
                    var      dateConvert = false;
                    DateTime meetingDate = DateTime.MinValue;

                    foreach (var dateRegKey in dateRegFormatDic.Keys)
                    {
                        string format          = dateRegFormatDic[dateRegKey];
                        string meetingDateText = dateRegKey.Match(r.InnerText).ToString();
                        if (string.IsNullOrWhiteSpace(format))
                        {
                            if (DateTime.TryParse(meetingDateText, out meetingDate))
                            {
                                dateConvert = true;
                                break;
                            }
                        }
                        if (format == "Month" && !string.IsNullOrWhiteSpace(meetingDateText))
                        {
                            try
                            {
                                var year    = r.ParentNode.ParentNode.ParentNode.ParentNode.ParentNode.PreviousSibling.PreviousSibling.InnerText;
                                var dateStr = meetingDateText + ", " + year;
                                if (DateTime.TryParse(dateStr, out meetingDate))
                                {
                                    dateConvert = true;
                                    break;
                                }
                            }
                            catch
                            {
                                continue;
                            }
                        }
                        if (DateTime.TryParseExact(meetingDateText, format, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.None, out meetingDate))
                        {
                            dateConvert = true;
                            break;
                        }
                    }
                    if (!dateConvert)
                    {
                        Console.WriteLine(r.InnerText);
                        Console.WriteLine("date format incorrect...");
                        continue;
                    }

                    if (meetingDate < this.dtStartFrom)
                    {
                        Console.WriteLine("Early...");
                        continue;
                    }
                    // Console.WriteLine(string.Format("datestr:{0},meeting:{1}", r.InnerText, meetingDate.ToString("yyyy-MM-dd")));
                    this.ExtractADoc(c, this.cityEntity.CityUrl + r.Attributes["href"].Value, category, "pdf", meetingDate, ref docs, ref queries);
                }
            }
            Console.WriteLine("docs:" + docs.Count + "--- query:" + queries.Count);
            // Console.ReadKey();
        }
Exemplo n.º 33
0
        public void GetByNameTest()
        {
            HtmlElement root = new HtmlElement("root");
            HtmlNodeCollection target = new HtmlNodeCollection(root);
            target.Add(new HtmlElement("first"));
            target.Add(new HtmlElement("second"));
            target.Add(new HtmlElement("second"));

            Assert.AreEqual(target.GetByName("second").Count, 2);

            ((HtmlElement)target[0]).Nodes.Add(new HtmlElement("second"));

            Assert.AreEqual(target.GetByName("second", false).Count, 2);

            Assert.AreEqual(target.GetByName("second").Count, 3);
        }
Exemplo n.º 34
0
        private void Main_Load(object sender, EventArgs e)
        {
            groupBox1.Hide();
            groupBox2.Hide();

            if (!Data.Login_1)
            {
                label1.Text = "欢迎回来!";
            }
            dataGridView1.AutoSizeRowsMode = DataGridViewAutoSizeRowsMode.AllCellsExceptHeaders;

            #region//加载天气
            try
            {
                HtmlWeb htmlWeb = new HtmlWeb();
                HtmlAgilityPack.HtmlDocument doc = htmlWeb.Load("http://i.tianqi.com/index.php?c=code&id=34&icon=1&num=2");

                ///显示所在地
                HtmlNodeCollection htmlNodes = doc.DocumentNode.SelectNodes("//a");
                toolStripStatusLabel2.Text = htmlNodes[0].InnerText.Trim();
                ///插入图片
                ////*[@id="mobile280"]/div/a[2]/span[1]/img
                htmlNodes = doc.DocumentNode.SelectNodes("//*[@id='mobile280']/div/a[2]/span[1]/img");
                string url = "http:" + htmlNodes[0].Attributes["src"].Value;
                // 图片测试
                System.Net.WebRequest  webreq = System.Net.WebRequest.Create(url);
                System.Net.WebResponse webres = webreq.GetResponse();
                using (System.IO.Stream stream = webres.GetResponseStream())
                {
                    toolStripStatusLabel3.Image = Image.FromStream(stream);
                }

                ///获取天气情况
                htmlNodes = doc.DocumentNode.SelectNodes("//span");
                String tianqi = "|温度:";
                foreach (var iteam in htmlNodes)
                {
                    tianqi += iteam.InnerText.Trim();
                }
                doc       = htmlWeb.Load("http://www.tianqi.com/");
                htmlNodes = doc.DocumentNode.SelectNodes("//p[@class='p_2']");
                tianqi   += "|" + htmlNodes[0].InnerText;

                htmlNodes = doc.DocumentNode.SelectNodes("//p[@class='p_3']");
                tianqi   += "|" + htmlNodes[0].InnerText;

                htmlNodes = doc.DocumentNode.SelectNodes("//p[@class='p_1']");
                tianqi    = htmlNodes[0].InnerText + tianqi;


                toolStripStatusLabel4.Text = tianqi + "|";
            }
            catch (Exception e1)
            {
                MessageBox.Show("天气加载失败!", "提示");
                Data.WriteLog("天气加载失败!" + e1.Message.ToString(), 1);
                statusStrip1.Hide();
            }

            #endregion
        }
Exemplo n.º 35
0
        public void InsertTest()
        {
            HtmlElement root = new HtmlElement("root");
            HtmlNodeCollection target = new HtmlNodeCollection(root);
            HtmlElement child = new HtmlElement("child");
            target.Add(child);

            child = new HtmlElement("second");
            target.Insert(0, child);
            Assert.AreEqual(root, child.Parent);
            Assert.AreEqual(target.IndexOf(child), 0);

            target.Insert(0, null);
        }
Exemplo n.º 36
0
        /// <summary>
        /// Adds the specified node to the end of the list of children of this node.
        /// </summary>
        /// <param name="newChildren">The node list to add. May not be null.</param>
        public void AppendChildren(HtmlNodeCollection newChildren)
        {
            if (newChildren == null)
                throw new ArgumentNullException("newChildren");

            foreach (HtmlNode newChild in newChildren)
            {
                AppendChild(newChild);
            }
        }
Exemplo n.º 37
0
        public void ItemByNameTest()
        {
            HtmlElement root = new HtmlElement("root");
            HtmlNodeCollection target = new HtmlNodeCollection(root);
            target.Add(new HtmlElement("first"));
            target.Add(new HtmlElement("second"));
            target.Add(new HtmlElement("second"));

            Assert.IsNotNull(target["second"]);

            Assert.IsNull(target["anyname"]);
        }
Exemplo n.º 38
0
        private static void Entitize(HtmlNodeCollection collection)
        {
            foreach (HtmlNode node in collection)
            {
                if (node.HasAttributes)
                    Entitize(node.Attributes);

                if (node.HasChildNodes)
                {
                    Entitize(node.ChildNodes);
                }
                else
                {
                    if (node.NodeType == HtmlNodeType.Text)
                    {
                        ((HtmlTextNode)node).Text = Entitize(((HtmlTextNode)node).Text, true, true);
                    }
                }
            }
        }
Exemplo n.º 39
0
    public string insert_event_data_new(string html)
    {
        StringBuilder sb = new StringBuilder();

        html = html.Replace("<thead=\"\"", "");
        HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
        doc.LoadHtml(html);

        HtmlNodeCollection nodes_all = doc.DocumentNode.SelectNodes(@"//*");
        List <HtmlNode>    nodes     = new List <HtmlNode>();

        ArrayList    list_lg    = new ArrayList();
        string       league     = "";
        DateTime     dt_server  = new DateTime();
        string       start_time = "";
        string       host       = "";
        string       client     = "";
        string       win        = "";
        string       draw       = "";
        string       lose       = "";
        string       date       = "";
        string       time       = "";
        string       zone       = "";
        string       event_id   = "";
        HtmlNode     node_timer = doc.DocumentNode.SELECT_NODE("/html[1]/head[1]/script[2]");
        BsonDocument doc_timer  = MongoHelper.get_doc_from_str(node_timer.InnerText.Replace("//<![CDATA[", "").Replace("//]]>>", "").Replace("initData =", ""));

        string server_time = doc_timer["serverTime"].ToString().E_TRIM();
        string zone_gmt    = doc_timer["tzPrefix"].ToString().E_TRIM();

        zone_gmt = zone_gmt.Replace("GMT", "");
        zone     = string.IsNullOrEmpty(zone_gmt) ? "0" : zone_gmt;
        string[] times = server_time.E_SPLIT(",");
        dt_server = new DateTime(Convert.ToInt16(times[0]), Convert.ToInt16(times[1]), Convert.ToInt16(times[2]), Convert.ToInt16(times[3]), Convert.ToInt16(times[4]), Convert.ToInt16(times[5]));
        //TimeSpan span = DateTime.Now - dt_server;
        //zone = (8 - Math.Round(span.TotalHours)).ToString();

        foreach (HtmlNode node in nodes_all)
        {
            if (node.Id == "container_EVENTS")
            {
                HtmlNodeCollection nodes_div = node.SELECT_NODES("/div");
                foreach (HtmlNode node_div in nodes_div)
                {
                    if (node_div.Id.Contains("container"))
                    {
                        league = node_div.SELECT_NODE("div[1]/h2[1]").InnerText;

                        HtmlNode           test        = node_div.SELECT_NODE("div[2]/div[1]/table[1]");
                        HtmlNodeCollection nodes_table = node_div.SELECT_NODES("div[2]/div[1]/table[1]/tbody");
                        foreach (HtmlNode node_table in nodes_table)
                        {
                            if (node_table.Id.Contains("event"))
                            {
                                event_id = node_table.Id.Replace("event_", "");
                                date     = node_table.SELECT_NODE("/tr[1]/td[1]/table[1]/tr[1]/td[2]").InnerText.E_TRIM();
                                date     = date.Replace("2015", "");
                                if (date.Length == 10)
                                {
                                    start_time = dt_server.ToString("yyyy-") + Tool.get_12m_from_eng(date.Substring(2, 3)) + "-" + date.Substring(0, 2) + " " + date.Substring(5, 5);
                                }
                                if (date.Length == 5)
                                {
                                    start_time = dt_server.ToString("yyyy-") + dt_server.ToString("MM-dd") + " " + date;
                                }
                                DateTime dt_time = Convert.ToDateTime(start_time);
                                dt_time = dt_time.AddHours(Convert.ToInt16(zone) * (-1));

                                host   = node_table.SELECT_NODE("/tr[1]/td[1]/table[1]/tr[1]/td[1]/span[1]/div[1]").InnerText;
                                client = node_table.SELECT_NODE("/tr[1]/td[1]/table[1]/tr[1]/td[1]/span[1]/div[2]").InnerText;

                                win  = node_table.SELECT_NODE("/tr[1]/td[2]").InnerText.E_REMOVE();
                                draw = node_table.SELECT_NODE("/tr[1]/td[3]").InnerText.E_REMOVE();
                                lose = node_table.SELECT_NODE("/tr[1]/td[4]").InnerText.E_REMOVE();
                                if (win.Contains("/"))
                                {
                                    win  = AnalyseTool.convert_english_odd(win);
                                    draw = AnalyseTool.convert_english_odd(draw);
                                    lose = AnalyseTool.convert_english_odd(lose);
                                }

                                if (!string.IsNullOrEmpty(win.E_TRIM()) && !string.IsNullOrEmpty(draw.E_TRIM()) && !string.IsNullOrEmpty(lose.E_TRIM()))
                                {
                                    sb.AppendLine(event_id.PR(10) + league.PR(50) + dt_time.ToString("yyyy-MM-dd HH:mm:ss").PR(20) + host.PR(30) + client.PR(30) + win.PR(10) + draw.PR(10) + lose.PR(10));
                                    Match100Helper.insert_data("marathonbet", league, start_time, host, client, win, draw, lose, "0", zone);
                                    MbSQL.insert_events(event_id, league.E_TRIM(), dt_time.ToString("yyyy-MM-dd HH:mm:ss"), host, client);
                                    ;
                                }
                            }
                        }
                    }
                }
            }
        }
        return(sb.ToString());
    }
Exemplo n.º 40
0
 /// <summary>
 /// This will create a new document object by parsing the Html specified.
 /// </summary>
 /// <param attributeName="html">The Html to parse.</param>
 private HtmlDocument(Stream stream)
 {
     this.nodes = HtmlParser.Parse(stream);
 }
Exemplo n.º 41
0
        /// <summary>
        /// Selects a list of nodes matching the <see cref="XPath"/> expression.
        /// </summary>
        /// <param name="xpath">The XPath expression.</param>
        /// <returns>An <see cref="HtmlNodeCollection"/> containing a collection of nodes matching the <see cref="XPath"/> query</returns>
        public HtmlNodeCollection SelectNodes(string xpath, XmlNamespaceManager xmgr)
        {
            HtmlNodeCollection list = new HtmlNodeCollection(null);

            HtmlNodeNavigator nav = new HtmlNodeNavigator(_ownerdocument, this);
            XPathNodeIterator it = nav.Select(xpath, xmgr);
            while (it.MoveNext())
            {
                HtmlNodeNavigator n = (HtmlNodeNavigator)it.Current;
                list.Add(n.CurrentNode);
            }
            if (list.Count == 0)
            {
                return list;
            }
            return list;
        }
Exemplo n.º 42
0
    public string get_event_with_detail(string html)
    {
        StringBuilder sb = new StringBuilder();

        //-------------------------------------------
        html = html.Replace("<thead=\"\"", "");

        HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
        doc.LoadHtml(html);

        HtmlNodeCollection nodes_all = doc.DocumentNode.SelectNodes(@"//*");
        List <HtmlNode>    nodes     = new List <HtmlNode>();

        ArrayList list_lg = new ArrayList();
        string    league  = "";

        string   start_time = "";
        string   host       = "";
        string   client     = "";
        string   win        = "";
        string   draw       = "";
        string   lose       = "";
        string   date       = "";
        string   time       = "";
        string   zone       = "";
        string   event_id   = "";
        DateTime dt_time    = DateTime.Now;

        foreach (HtmlNode node in nodes_all)
        {
            zone = "8";
            if (node.Id == "timer")
            {
                //02.09.14, 14:47 (GMT+1)
                string   timer    = node.InnerText.E_TRIM();
                DateTime dt_timer = new DateTime(Convert.ToInt16("20" + timer.Substring(6, 2)), Convert.ToInt16(timer.Substring(3, 2)), Convert.ToInt16(timer.Substring(0, 2)),
                                                 Convert.ToInt16(timer.Substring(9, 2)), Convert.ToInt16(timer.Substring(12, 2)), 0);
                TimeSpan span = DateTime.Now - dt_timer;
                zone = (8 - Math.Round(span.TotalHours)).ToString();
            }

            if (node.Id == "container_EVENTS")
            {
                HtmlNodeCollection nodes_div = node.SELECT_NODES("/div");
                foreach (HtmlNode node_div in nodes_div)
                {
                    if (node_div.Id.Contains("container"))
                    {
                        league = node_div.SELECT_NODE("div[1]/h2[1]").InnerText.E_REMOVE();

                        HtmlNode           test        = node_div.SELECT_NODE("div[2]/div[1]/table[1]");
                        HtmlNodeCollection nodes_table = node_div.SELECT_NODES("div[2]/div[1]/table[1]/tbody");
                        if (nodes_table == null)
                        {
                            continue;
                        }
                        foreach (HtmlNode node_table in nodes_table)
                        {
                            if (node_table.Id.Contains("event"))
                            {
                                event_id = node_table.Id.Replace("event_", "");
                                date     = node_table.SELECT_NODE("/tr[1]/td[1]/table[1]/tbody[1]/tr[1]/td[2]").InnerText.E_TRIM();
                                date     = date.Replace("2015", "");
                                if (date.Length == 10)
                                {
                                    start_time = Tool.get_12m_from_eng(date.Substring(2, 3)) + "-" + date.Substring(0, 2) + "●" + date.Substring(5, 5);
                                }
                                if (date.Length == 5)
                                {
                                    start_time = dt_time.ToString("MM-dd") + "●" + date;
                                }
                                host   = node_table.SELECT_NODE("/tr[1]/td[1]/table[1]/tbody[1]/tr[1]/td[1]/span[1]/div[1]").InnerText.E_REMOVE();
                                client = node_table.SELECT_NODE("/tr[1]/td[1]/table[1]/tbody[1]/tr[1]/td[1]/span[1]/div[2]").InnerText.E_REMOVE();

                                win  = node_table.SELECT_NODE("/tr[1]/td[2]").InnerText.E_REMOVE();
                                draw = node_table.SELECT_NODE("/tr[1]/td[3]").InnerText.E_REMOVE();
                                lose = node_table.SELECT_NODE("/tr[1]/td[4]").InnerText.E_REMOVE();
                                if (win.Contains("/"))
                                {
                                    win  = AnalyseTool.convert_english_odd(win);
                                    draw = AnalyseTool.convert_english_odd(draw);
                                    lose = AnalyseTool.convert_english_odd(lose);
                                }

                                if (!string.IsNullOrEmpty(win.E_TRIM()) && !string.IsNullOrEmpty(draw.E_TRIM()) && !string.IsNullOrEmpty(lose.E_TRIM()))
                                {
                                    sb.AppendLine(league.PR(50) + start_time.PR(20) + host.PR(30) + client.PR(30) + win.PR(10) + draw.PR(10) + lose.PR(10));
                                    //Match100Helper.insert_data("marathonbet", league, start_time, host, client, win, draw, lose, "1", zone);
                                }
                                MbSQL.insert_events(event_id, league, dt_time.ToString("yyyy-MM-dd HH:mm:ss"), host, client);

                                //get the detail information
                                HtmlNodeCollection nodes_tr = node_table.SELECT_NODES("tr");
                                foreach (HtmlNode node_tr in nodes_tr)
                                {
                                    if (node_tr.CLASS().Contains("market-details"))
                                    {
                                        sb.AppendLine("------------------------------------------------------------------------------------");
                                        HtmlNodeCollection nodes_block = node_tr.SELECT_NODES("/td[1]/div[2]/div[1]/div"); //block-market-wrapper
                                        foreach (HtmlNode node_block in nodes_block)
                                        {
                                            HtmlNodeCollection nodes_div1 = node_block.SELECT_NODES("/div[4]/div");
                                            if (nodes_div1 != null)
                                            {
                                                foreach (HtmlNode node_div1 in nodes_div1)
                                                {
                                                    string odd_type = node_div1.SELECT_NODE("/div[1]").InnerText.E_REMOVE();
                                                    sb.AppendLine(odd_type);
                                                    sb.AppendLine("--------------------");
                                                    HtmlNodeCollection nodes_tr1 = node_div1.SELECT_NODES("/table[1]/tbody[1]/tr");

                                                    foreach (HtmlNode node_tr1 in nodes_tr1)
                                                    {
                                                        //sb.AppendLine(node_tr1.InnerText.E_REMOVE());
                                                        HtmlNodeCollection nodes_td1 = node_tr1.SELECT_NODES("/td");
                                                        ArrayList          list      = new ArrayList();
                                                        if (nodes_td1 != null)
                                                        {
                                                            for (int i = 0; i < nodes_td1.Count; i++)
                                                            {
                                                                HtmlNodeCollection nodes_div2 = nodes_td1[i].SELECT_NODES("/div[1]/div");
                                                                if (nodes_div2 != null)
                                                                {
                                                                    for (int j = 0; j < nodes_div2.Count; j++)
                                                                    {
                                                                        sb.Append(nodes_div2[j].InnerText.E_REMOVE().PR(10));
                                                                        list.Add(nodes_div2[j].InnerText.E_REMOVE().Replace("(", "").Replace(")", "").Replace("+", ""));
                                                                    }
                                                                }
                                                            }
                                                            sb.Append(M.N);
                                                        }


                                                        if (odd_type == "To Win Match With Handicap" && list.Count >= 4)
                                                        {
                                                            MbSQL.insert_odds(event_id, "2", "To Win Match With Handicap",
                                                                              "FULL", "", "", "", "", "",
                                                                              list[0].ToString(), list[2].ToString(), "", "", "", "",
                                                                              list[1].ToString(), list[3].ToString(), "", "", "", "");
                                                        }
                                                        if (odd_type == "To Win 1st Half With Handicap" && list.Count >= 4)
                                                        {
                                                            MbSQL.insert_odds(event_id, "2", "To Win 1st Half With Handicap",
                                                                              "1-HALF", "", "", "", "", "",
                                                                              list[0].ToString(), list[2].ToString(), "", "", "", "",
                                                                              list[1].ToString(), list[3].ToString(), "", "", "", "");
                                                        }
                                                        if (odd_type == "To Win 2nd Half With Handicap" && list.Count >= 4)
                                                        {
                                                            MbSQL.insert_odds(event_id, "2", "To Win 2nd Half With Handicap",
                                                                              "2-HALF", "", "", "", "", "",
                                                                              list[0].ToString(), list[2].ToString(), "", "", "", "",
                                                                              list[1].ToString(), list[3].ToString(), "", "", "", "");
                                                        }
                                                        if (odd_type == "Total Goals" && list.Count >= 4)
                                                        {
                                                            MbSQL.insert_odds(event_id, "3", "Total Goals",
                                                                              "FULL", "", "", "", "", "",
                                                                              list[0].ToString(), list[2].ToString(), "", "", "", "",
                                                                              list[1].ToString(), list[3].ToString(), "", "", "", "");
                                                        }
                                                        if (odd_type == "Total Goals - 1st Half" && list.Count >= 4)
                                                        {
                                                            MbSQL.insert_odds(event_id, "3", "Total Goals - 1st Half",
                                                                              "1-HALF", "", "", "", "", "",
                                                                              list[0].ToString(), list[2].ToString(), "", "", "", "",
                                                                              list[1].ToString(), list[3].ToString(), "", "", "", "");
                                                        }
                                                        if (odd_type == "Total Goals - 2nd Half" && list.Count >= 4)
                                                        {
                                                            MbSQL.insert_odds(event_id, "3", "Total Goals - 2nd Half",
                                                                              "2-HALF", "", "", "", "", "",
                                                                              list[0].ToString(), list[2].ToString(), "", "", "", "",
                                                                              list[1].ToString(), list[3].ToString(), "", "", "", "");
                                                        }
                                                        if (odd_type.Contains("Total Goals") && odd_type.Contains(host) && !odd_type.Contains("+") && list.Count >= 4)
                                                        {
                                                            MbSQL.insert_odds(event_id, "4", "Total Goals",
                                                                              "FULL", "HOME", "", "", "", "",
                                                                              list[0].ToString(), list[2].ToString(), "", "", "", "",
                                                                              list[1].ToString(), list[3].ToString(), "", "", "", "");
                                                        }
                                                        if (odd_type.Contains("Total Goals") && odd_type.Contains(client) && !odd_type.Contains("+") && list.Count >= 4)
                                                        {
                                                            MbSQL.insert_odds(event_id, "4", "Total Goals",
                                                                              "FULL", "AWAY", "", "", "", "",
                                                                              list[0].ToString(), list[2].ToString(), "", "", "", "",
                                                                              list[1].ToString(), list[3].ToString(), "", "", "", "");
                                                        }
                                                    }
                                                    sb.AppendLine("--------------------");
                                                }
                                            }
                                        }
                                        sb.AppendLine("------------------------------------------------------------------------------------");
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
        //------------------------------------------------------
        return(sb.ToString());
    }
Exemplo n.º 43
0
    public string insert_event_data(string html)
    {
        StringBuilder sb = new StringBuilder();

        //-------------------------------------------
        html = html.Replace("<thead=\"\"", "");

        HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
        doc.LoadHtml(html);

        HtmlNodeCollection nodes_all = doc.DocumentNode.SelectNodes(@"//*");
        List <HtmlNode>    nodes     = new List <HtmlNode>();

        ArrayList list_lg = new ArrayList();
        string    league  = "";

        string   start_time = "";
        string   host       = "";
        string   client     = "";
        string   win        = "";
        string   draw       = "";
        string   lose       = "";
        string   date       = "";
        string   time       = "";
        string   zone       = "";
        DateTime dt_time    = DateTime.Now;

        foreach (HtmlNode node in nodes_all)
        {
            zone = "8";
            if (node.Id == "timer")
            {
                //02.09.14, 14:47 (GMT+1)
                string   timer    = node.InnerText.E_TRIM();
                DateTime dt_timer = new DateTime(Convert.ToInt16("20" + timer.Substring(6, 2)), Convert.ToInt16(timer.Substring(3, 2)), Convert.ToInt16(timer.Substring(0, 2)),
                                                 Convert.ToInt16(timer.Substring(9, 2)), Convert.ToInt16(timer.Substring(12, 2)), 0);
                TimeSpan span = DateTime.Now - dt_timer;
                zone = (8 - Math.Round(span.TotalHours)).ToString();
            }

            if (node.Id == "container_EVENTS")
            {
                HtmlNodeCollection nodes_div = node.SELECT_NODES("/div");
                foreach (HtmlNode node_div in nodes_div)
                {
                    if (node_div.Id.Contains("container"))
                    {
                        league = node_div.SELECT_NODE("div[1]/h2[1]").InnerText;

                        HtmlNode           test        = node_div.SELECT_NODE("div[2]/div[1]/table[1]");
                        HtmlNodeCollection nodes_table = node_div.SELECT_NODES("div[2]/div[1]/table[1]/tbody");
                        if (nodes_table == null)
                        {
                            continue;
                        }
                        foreach (HtmlNode node_table in nodes_table)
                        {
                            if (node_table.Id.Contains("event"))
                            {
                                date = node_table.SELECT_NODE("/tr[1]/td[1]/table[1]/tbody[1]/tr[1]/td[2]").InnerText.E_TRIM();
                                date = date.Replace("2015", "");
                                if (date.Length == 10)
                                {
                                    start_time = Tool.get_12m_from_eng(date.Substring(2, 3)) + "-" + date.Substring(0, 2) + "●" + date.Substring(5, 5);
                                }
                                if (date.Length == 5)
                                {
                                    start_time = dt_time.ToString("MM-dd") + "●" + date;
                                }
                                host   = node_table.SELECT_NODE("/tr[1]/td[1]/table[1]/tbody[1]/tr[1]/td[1]/span[1]/div[1]").InnerText;
                                client = node_table.SELECT_NODE("/tr[1]/td[1]/table[1]/tbody[1]/tr[1]/td[1]/span[1]/div[2]").InnerText;

                                win  = node_table.SELECT_NODE("/tr[1]/td[2]").InnerText.E_REMOVE();
                                draw = node_table.SELECT_NODE("/tr[1]/td[3]").InnerText.E_REMOVE();
                                lose = node_table.SELECT_NODE("/tr[1]/td[4]").InnerText.E_REMOVE();
                                if (win.Contains("/"))
                                {
                                    win  = AnalyseTool.convert_english_odd(win);
                                    draw = AnalyseTool.convert_english_odd(draw);
                                    lose = AnalyseTool.convert_english_odd(lose);
                                }

                                if (!string.IsNullOrEmpty(win.E_TRIM()) && !string.IsNullOrEmpty(draw.E_TRIM()) && !string.IsNullOrEmpty(lose.E_TRIM()))
                                {
                                    sb.AppendLine(league.PR(50) + start_time.PR(20) + host.PR(30) + client.PR(30) + win.PR(10) + draw.PR(10) + lose.PR(10));
                                    Match100Helper.insert_data("marathonbet", league, start_time, host, client, win, draw, lose, "1", zone);
                                }
                            }
                        }
                    }
                }
            }
        }
        //------------------------------------------------------
        return(sb.ToString());
    }
Exemplo n.º 44
0
        private async Task CheckPage(HtmlDocument htmlDocument)
        {
            if (htmlDocument == null)
            {
                Bot.ArchiLogger.LogNullError(nameof(htmlDocument));
                return;
            }

            HtmlNodeCollection htmlNodes = htmlDocument.DocumentNode.SelectNodes("//div[@class='badge_row_inner']");

            if (htmlNodes == null)
            {
                // No eligible badges whatsoever
                return;
            }

            HashSet <Task> backgroundTasks = new HashSet <Task>();

            foreach (HtmlNode htmlNode in htmlNodes)
            {
                HtmlNode statsNode = htmlNode.SelectSingleNode(".//div[@class='badge_title_stats_content']");

                HtmlNode appIDNode = statsNode?.SelectSingleNode(".//div[@class='card_drop_info_dialog']");
                if (appIDNode == null)
                {
                    // It's just a badge, nothing more
                    continue;
                }

                string appIDText = appIDNode.GetAttributeValue("id", null);
                if (string.IsNullOrEmpty(appIDText))
                {
                    Bot.ArchiLogger.LogNullError(nameof(appIDText));
                    continue;
                }

                string[] appIDSplitted = appIDText.Split('_');
                if (appIDSplitted.Length < 5)
                {
                    Bot.ArchiLogger.LogNullError(nameof(appIDSplitted));
                    continue;
                }

                appIDText = appIDSplitted[4];

                if (!uint.TryParse(appIDText, out uint appID) || (appID == 0))
                {
                    Bot.ArchiLogger.LogNullError(nameof(appID));
                    continue;
                }

                if (GlobalConfig.SalesBlacklist.Contains(appID) || Program.GlobalConfig.Blacklist.Contains(appID) || Bot.IsBlacklistedFromIdling(appID) || (Bot.BotConfig.IdlePriorityQueueOnly && !Bot.IsPriorityIdling(appID)))
                {
                    // We're configured to ignore this appID, so skip it
                    continue;
                }

                if (IgnoredAppIDs.TryGetValue(appID, out DateTime ignoredUntil))
                {
                    if (ignoredUntil < DateTime.UtcNow)
                    {
                        // This game served its time as being ignored
                        IgnoredAppIDs.TryRemove(appID, out _);
                    }
                    else
                    {
                        // This game is still ignored
                        continue;
                    }
                }

                // Cards
                HtmlNode progressNode = statsNode.SelectSingleNode(".//span[@class='progress_info_bold']");
                if (progressNode == null)
                {
                    Bot.ArchiLogger.LogNullError(nameof(progressNode));
                    continue;
                }

                string progressText = progressNode.InnerText;
                if (string.IsNullOrEmpty(progressText))
                {
                    Bot.ArchiLogger.LogNullError(nameof(progressText));
                    continue;
                }

                ushort cardsRemaining = 0;
                Match  progressMatch  = Regex.Match(progressText, @"\d+");

                // This might fail if we have no card drops remaining, 0 is not printed in this case - that's fine
                if (progressMatch.Success)
                {
                    if (!ushort.TryParse(progressMatch.Value, out cardsRemaining) || (cardsRemaining == 0))
                    {
                        Bot.ArchiLogger.LogNullError(nameof(cardsRemaining));
                        continue;
                    }
                }

                if (cardsRemaining == 0)
                {
                    // Normally we'd trust this information and simply skip the rest
                    // However, Steam is so f****d up that we can't simply assume that it's correct
                    // It's entirely possible that actual game page has different info, and badge page lied to us
                    // We can't check every single game though, as this will literally kill people with cards from games they don't own
                    // Luckily for us, it seems to happen only with some specific games
                    if (!UntrustedAppIDs.Contains(appID))
                    {
                        continue;
                    }

                    // To save us on extra work, check cards earned so far first
                    HtmlNode cardsEarnedNode = statsNode.SelectSingleNode(".//div[@class='card_drop_info_header']");
                    if (cardsEarnedNode == null)
                    {
                        Bot.ArchiLogger.LogNullError(nameof(cardsEarnedNode));
                        continue;
                    }

                    string cardsEarnedText = cardsEarnedNode.InnerText;
                    if (string.IsNullOrEmpty(cardsEarnedText))
                    {
                        Bot.ArchiLogger.LogNullError(nameof(cardsEarnedText));
                        continue;
                    }

                    Match cardsEarnedMatch = Regex.Match(cardsEarnedText, @"\d+");
                    if (!cardsEarnedMatch.Success)
                    {
                        Bot.ArchiLogger.LogNullError(nameof(cardsEarnedMatch));
                        continue;
                    }

                    if (!ushort.TryParse(cardsEarnedMatch.Value, out ushort cardsEarned))
                    {
                        Bot.ArchiLogger.LogNullError(nameof(cardsEarned));
                        continue;
                    }

                    if (cardsEarned > 0)
                    {
                        // If we already earned some cards for this game, it's very likely that it's done
                        // Let's hope that trusting cardsRemaining AND cardsEarned is enough
                        // If I ever hear that it's not, I'll most likely need a doctor
                        continue;
                    }

                    // If we have no cardsRemaining and no cardsEarned, it's either:
                    // - A game we don't own physically, but we have cards from it in inventory
                    // - F2P game that we didn't spend any money in, but we have cards from it in inventory
                    // - Steam issue
                    // As you can guess, we must follow the rest of the logic in case of Steam issue
                }

                // Hours
                HtmlNode timeNode = statsNode.SelectSingleNode(".//div[@class='badge_title_stats_playtime']");
                if (timeNode == null)
                {
                    Bot.ArchiLogger.LogNullError(nameof(timeNode));
                    continue;
                }

                string hoursText = timeNode.InnerText;
                if (string.IsNullOrEmpty(hoursText))
                {
                    Bot.ArchiLogger.LogNullError(nameof(hoursText));
                    continue;
                }

                float hours      = 0.0F;
                Match hoursMatch = Regex.Match(hoursText, @"[0-9\.,]+");

                // This might fail if we have exactly 0.0 hours played, as it's not printed in that case - that's fine
                if (hoursMatch.Success)
                {
                    if (!float.TryParse(hoursMatch.Value, NumberStyles.Number, CultureInfo.InvariantCulture, out hours) || (hours <= 0.0F))
                    {
                        Bot.ArchiLogger.LogNullError(nameof(hours));
                        continue;
                    }
                }

                // Names
                HtmlNode nameNode = statsNode.SelectSingleNode("(.//div[@class='card_drop_info_body'])[last()]");
                if (nameNode == null)
                {
                    Bot.ArchiLogger.LogNullError(nameof(nameNode));
                    continue;
                }

                string name = nameNode.InnerText;
                if (string.IsNullOrEmpty(name))
                {
                    Bot.ArchiLogger.LogNullError(nameof(name));
                    continue;
                }

                // We handle two cases here - normal one, and no card drops remaining
                int nameStartIndex = name.IndexOf(" by playing ", StringComparison.Ordinal);
                if (nameStartIndex <= 0)
                {
                    nameStartIndex = name.IndexOf("You don't have any more drops remaining for ", StringComparison.Ordinal);
                    if (nameStartIndex <= 0)
                    {
                        Bot.ArchiLogger.LogNullError(nameof(nameStartIndex));
                        continue;
                    }

                    nameStartIndex += 32;                     // + 12 below
                }

                nameStartIndex += 12;

                int nameEndIndex = name.LastIndexOf('.');
                if (nameEndIndex <= nameStartIndex)
                {
                    Bot.ArchiLogger.LogNullError(nameof(nameEndIndex));
                    continue;
                }

                name = WebUtility.HtmlDecode(name.Substring(nameStartIndex, nameEndIndex - nameStartIndex));

                // Levels
                byte badgeLevel = 0;

                HtmlNode levelNode = htmlNode.SelectSingleNode(".//div[@class='badge_info_description']/div[2]");
                if (levelNode != null)
                {
                    // There is no levelNode if we didn't craft that badge yet (level 0)
                    string levelText = levelNode.InnerText;
                    if (string.IsNullOrEmpty(levelText))
                    {
                        Bot.ArchiLogger.LogNullError(nameof(levelText));
                        continue;
                    }

                    int levelIndex = levelText.IndexOf("Level ", StringComparison.OrdinalIgnoreCase);
                    if (levelIndex < 0)
                    {
                        Bot.ArchiLogger.LogNullError(nameof(levelIndex));
                        continue;
                    }

                    levelIndex += 6;
                    if (levelText.Length <= levelIndex)
                    {
                        Bot.ArchiLogger.LogNullError(nameof(levelIndex));
                        continue;
                    }

                    levelText = levelText.Substring(levelIndex, 1);
                    if (!byte.TryParse(levelText, out badgeLevel) || (badgeLevel == 0) || (badgeLevel > 5))
                    {
                        Bot.ArchiLogger.LogNullError(nameof(badgeLevel));
                        continue;
                    }
                }

                // Done with parsing, we have two possible cases here
                // Either we have decent info about appID, name, hours, cardsRemaining (cardsRemaining > 0) and level
                // OR we strongly believe that Steam lied to us, in this case we will need to check game individually (cardsRemaining == 0)
                if (cardsRemaining > 0)
                {
                    GamesToFarm.Add(new Game(appID, name, hours, cardsRemaining, badgeLevel));
                }
                else
                {
                    Task task = CheckGame(appID, name, hours, badgeLevel);
                    switch (Program.GlobalConfig.OptimizationMode)
                    {
                    case GlobalConfig.EOptimizationMode.MinMemoryUsage:
                        await task.ConfigureAwait(false);

                        break;

                    default:
                        backgroundTasks.Add(task);
                        break;
                    }
                }
            }

            // If we have any background tasks, wait for them
            if (backgroundTasks.Count > 0)
            {
                await Task.WhenAll(backgroundTasks).ConfigureAwait(false);
            }
        }
Exemplo n.º 45
0
 /// <summary>
 /// This will find the corresponding opening tag for the named one. This is identified as
 /// the most recently read nodes with the same attributeName, but with no child nodes.
 /// </summary>
 private static int FindTagOpenNodeIndex(HtmlNodeCollection nodes, string name)
 {
     for (int index = nodes.Count - 1; index >= 0; index--)
     {
         HtmlElement element = nodes[index] as HtmlElement;
         if (null != element && (string.Compare(element.Name, name, StringComparison.OrdinalIgnoreCase) == 0)
             && element.Nodes.Count == 0 && !element.IsTerminated)
         {
             return index;
         }
     }
     return -1;
 }
Exemplo n.º 46
0
        /// <summary>
        /// Get page reviews
        /// </summary>
        /// <param name="pageIdOrAlias"></param>
        /// <returns></returns>
        public PageReviewInfo GetPageReviewInfo(string pageIdOrAlias)
        {
            var pageId = string.Empty;

            // Checking if pageIdOrAlias is pageAlias or PageId
            MatchCollection matches = Regex.Matches(pageIdOrAlias, "\\d+");

            // param passed is page alias, we need to get page id from it.
            if (matches.Count != 1)
            {
                pageId = GetPageId(pageIdOrAlias);
            }

            if (pageId.Length == 0)
            {
                throw new ArgumentException("Can not detect page id of " + pageIdOrAlias);
            }

            HtmlNode htmlNode = this.BuildDom("https://m.facebook.com/page/reviews.php?id=" + pageIdOrAlias);

            if (ConstString.PageNotFound.Any(text => htmlNode.InnerHtml.Contains(text)))
            {
                throw new ReviewPageNotFoundException(pageIdOrAlias + " not contains review page");
            }

            // get review nodes -- review node contain user's review
            // TODO : Replace with more safely xpath
            HtmlNodeCollection reviewNodes = htmlNode.SelectNodes("/html/body/div/div/div[2]/div[2]/div[1]/div/div[3]/div/div/div/div");

            // Create page review
            var pageReview = new PageReviewInfo();

            // loop through DOM reviewNodes
            foreach (var reviewNode in reviewNodes)
            {
                // create new instance of review info
                var reviewInfo = new PageReview();

                // Get avatar
                HtmlNode imgAvatarNode = reviewNode.SelectSingleNode("div/div/div[1]/a/div/img");
                if (imgAvatarNode != null)
                {
                    reviewInfo.UserAvatarUrl = WebUtility.HtmlDecode(imgAvatarNode.GetAttributeValue("src", string.Empty));
                }

                // User name and id
                HtmlNode userNameIdNode = reviewNode.SelectSingleNode("div/div/div[2]/div/div[1]/a[1]");
                if (userNameIdNode != null)
                {
                    // Get urlink and parse
                    string urlLink = userNameIdNode.GetAttributeValue("href", null);
                    if (urlLink != null)
                    {
                        if (urlLink.Contains("/profile.php?id="))
                        {
                            reviewInfo.UserId = urlLink.Substring(16); // /profile.php?id=100012141183155
                        }
                        else
                        {
                            reviewInfo.UserId = urlLink.Substring(1); // /kakarotto.pham.9
                        }
                    }

                    HtmlNode nameNode = userNameIdNode.SelectSingleNode("span");
                    if (nameNode != null)
                    {
                        reviewInfo.UserDisplayName = WebUtility.HtmlDecode(nameNode.InnerText + string.Empty);
                    }
                }

                // Get rate score
                HtmlNode rateScoreNode = reviewNode.SelectSingleNode("div/div/div[2]/div/div[1]/a[2]/img");
                // return -1 indicate that can not recognize value
                if (rateScoreNode != null)
                {
                    reviewInfo.RateScore = int.Parse(rateScoreNode.GetAttributeValue("alt", "-1"), CultureInfo.CurrentCulture);
                }

                // Get fully rate content page
                HtmlNode rateContentNode = reviewNode.SelectSingleNode("div/div/div[2]/div/div[1]/a[2]");
                if (rateContentNode != null)
                {
                    string rateContentAnchorLink = rateContentNode.GetAttributeValue("href", null);
                    if (rateContentAnchorLink != null)
                    {
                        HtmlNode htmlRateContentNode = this.BuildDom("https://m.facebook.com" + rateContentAnchorLink);
                        // TODO : Replace with more safely
                        HtmlNode contentNode = htmlRateContentNode.SelectSingleNode("html/body/div/div/div[2]/div/div[1]/div/div[1]/div/div[1]/div[2]/p");

                        if (contentNode != null)
                        {
                            reviewInfo.Content = contentNode.InnerText;
                        }
                    }
                }

                pageReview.Reviews.Add(reviewInfo);
            }

            return(pageReview);
        }
Exemplo n.º 47
0
        private static HtmlNodeCollection BuildNodeCollection(Queue<string> tokens)
        {
            HtmlNodeCollection nodes = new HtmlNodeCollection(null);
            HtmlElement element = null;
            string current;

            while (tokens.Count > 0)
            {
                current = tokens.Dequeue();
                switch (current)
                {
                    case ("<"):
                        // Read open tag

                        if (tokens.Count == 0)
                            break;

                        current = tokens.Dequeue();
                        element = new HtmlElement(current);

                        // read the attributes and values
                        while (tokens.Count > 0 && (current = tokens.Dequeue()) != ">" && current != "/>")
                        {
                            string attribute_name = current;
                            if (tokens.Count > 0 && tokens.Peek() == "=")
                            {
                                tokens.Dequeue();
                                current = (tokens.Count > 0) ? tokens.Dequeue() : null;
                                HtmlAttribute attribute = new HtmlAttribute(attribute_name, HttpUtility.HtmlDecode(current));
                                element.Attributes.Add(attribute);
                            }
                            else //if (tokens.Count == 0)
                            {
                                // Null-attributeValue attribute
                                HtmlAttribute attribute = new HtmlAttribute(attribute_name);
                                element.Attributes.Add(attribute);
                            }
                        }
                        nodes.Add(element);

                        if (current == "/>")
                        {
                            element.IsTerminated = true;
                            element = null; //could not have any sub elements
                        }
                        else if (current == ">")
                        {
                            continue;
                        }
                        break;
                    case (">"):
                        continue;
                    case ("</"):
                        // Read close tag

                        if (tokens.Count == 0)
                            break;

                        current = tokens.Dequeue();

                        int open_index = FindTagOpenNodeIndex(nodes, current);
                        if (open_index != -1)
                        {
                            MoveNodesDown(ref nodes, open_index + 1, (HtmlElement)nodes[open_index]);
                        }

                        // Skip to the end of this tag
                        while (tokens.Count > 0 && (current = tokens.Dequeue()) != ">")
                        {
                            //shouldn't happen
                        }
                        element = null;
                        break;
                    default:
                        HtmlText node = new HtmlText(current);
                        nodes.Add(node);
                        break;
                }
            }
            return nodes;
        }
Exemplo n.º 48
0
        private void GetList(IListSheet listSheet)
        {
            String exportDir     = this.RunPage.GetExportDir();
            string pageSourceDir = this.RunPage.GetDetailSourceFileDir();

            Dictionary <string, int> resultColumnDic = new Dictionary <string, int>();

            resultColumnDic.Add("cate1", 0);
            resultColumnDic.Add("cateId1", 1);
            resultColumnDic.Add("cate2", 2);
            resultColumnDic.Add("cateId2", 3);
            resultColumnDic.Add("cate3", 4);
            resultColumnDic.Add("cateId3", 5);
            resultColumnDic.Add("name", 6);
            resultColumnDic.Add("wordCount", 7);
            string      resultFilePath = Path.Combine(exportDir, "教育_术语_scel详情信息.xlsx");
            ExcelWriter resultEW       = new ExcelWriter(resultFilePath, "List", resultColumnDic, null);

            for (int i = 0; i < listSheet.RowCount; i++)
            {
                Dictionary <string, string> row = listSheet.GetRow(i);
                string detailUrl = row["detailPageUrl"];

                string cate1   = row["cate1"];
                string cate2   = row["cate2"];
                string cateId1 = row["cateId1"];
                string cateId2 = row["cateId2"];
                bool   giveUp  = "Y".Equals(row[SysConfig.GiveUpGrabFieldName]);
                if (!giveUp)
                {
                    string localFilePath = this.RunPage.GetFilePath(detailUrl, pageSourceDir);

                    try
                    {
                        HtmlAgilityPack.HtmlDocument htmlDoc   = this.RunPage.GetLocalHtmlDocument(listSheet, i);
                        HtmlNodeCollection           itemNodes = htmlDoc.DocumentNode.SelectNodes("//div[@class=\"dict_info_list\"]/ul/li");

                        HtmlNode itemNode        = itemNodes[0];
                        string   text            = itemNode.InnerText.Trim();
                        int      splitBeginIndex = text.IndexOf(":");
                        int      splitEndIndex   = text.IndexOf("个");
                        int      wordCount       = int.Parse(text.Substring(splitBeginIndex + 1, splitEndIndex - splitBeginIndex - 1));

                        Dictionary <string, string> f2vs = new Dictionary <string, string>();
                        f2vs.Add("cate1", row["cate1"]);
                        f2vs.Add("cateId1", row["cateId1"]);
                        f2vs.Add("cate2", row["cate2"]);
                        f2vs.Add("cateId2", row["cateId2"]);
                        f2vs.Add("cate3", row["cate3"]);
                        f2vs.Add("cateId3", row["cateId3"]);
                        f2vs.Add("name", row["name"]);
                        f2vs.Add("wordCount", wordCount.ToString());
                        resultEW.AddRow(f2vs);
                    }
                    catch (Exception ex)
                    {
                        throw ex;
                    }
                }
            }
            resultEW.SaveToDisk();
        }