Esempio n. 1
0
        public static void SafeDataSQLDatabaseIGN(IGNGame _game)
        {
            try
            {
                var db = new UpdateModelII();
                db.IGNGames.Add(_game);
                db.SaveChanges();
            }
            catch (DbUpdateException e)
            {
                //foreach (var validationErrors in e.GetBaseException)
                //{
                //    foreach (var validationError in validationErrors.ValidationErrors)
                //    {
                //        System.Console.WriteLine("Property: {0} Error: {1}", validationError.PropertyName, validationError.ErrorMessage);
                //    }
                //}

                string h = e.GetBaseException().ToString();
                h = h;
            }
        }
Esempio n. 2
0
        public static void ScrapIGNWebsite()
        {
            for (int pageNumber = 4600; pageNumber <= 4950; pageNumber += 50)
            {
                Thread.Sleep(1000);
                var html = @"http://www.ign.com/games?startIndex=" + pageNumber + "&sortBy=popularity&sortOrder=desc";

                HtmlWeb web = new HtmlWeb();

                var htmlDoc = web.Load(html);

                var htmlNodes = htmlDoc.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("id") && d.Attributes["id"].Value.Contains("item-list"));

                var htmlNodesNextLevel = htmlNodes.First().ChildNodes[3].Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("clear itemList-itemShort"));

                int loop = 0;

                foreach (var node in htmlNodesNextLevel)
                {
                    Thread.Sleep(1000);
                    var _IGNGameName            = node.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Equals("item-title"));
                    var _IGNGameNameSecondLevel = _IGNGameName.First().ChildNodes[1];
                    var _IGNGameNameInner       = _IGNGameNameSecondLevel.InnerText;
                    var _IGNGamePage            = _IGNGameNameSecondLevel.Attributes["href"].Value;
                    var _IGNScore = node.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Equals("grid_3"));

                    var _IGNScoreInner = "";

                    try
                    {
                        _IGNScoreInner = _IGNScore.First().InnerText;
                    }
                    catch
                    {
                        _IGNScoreInner = "NOT PROVIDED";
                    }


                    int retry = 0;
AGAIN:

                    Thread.Sleep(1000);

                    var rootHtml = @"http://www.ign.com";
                    rootHtml = @rootHtml + _IGNGamePage;

                    var _IGNWiki            = "";
                    var htmlDocSpecificGame = new HtmlDocument();
                    htmlDocSpecificGame = web.Load(@rootHtml);


                    try
                    {
                        var children         = htmlDocSpecificGame.DocumentNode.Descendants("a");
                        var _htmlNodeIGNWiki = htmlDocSpecificGame.DocumentNode.Descendants("a").Where(d => d.Attributes.Contains("title") && d.Attributes["title"].Value.Equals("wiki-guide"));
                        _IGNWiki = @_htmlNodeIGNWiki.First().Attributes["href"].Value;
                    }
                    catch
                    {
                        //var _htmlNodeIGNWiki = htmlDocSpecificGame.DocumentNode.Descendants("a").Where(d => d.Attributes.Contains("title") && d.Attributes["title"].Value.Equals("wiki-guide"));
                        //_IGNWiki = @_htmlNodeIGNWiki.First().Attributes["href"].Value;

                        if (retry == 1 || retry == 2)
                        {
                            _IGNWiki = "NOT PROVIDED";
                        }
                        else
                        {
                            retry++;
                            goto AGAIN;
                        }
                    }


                    var _IGNUserScoreInner = "";
                    try
                    {
                        var _IGNUserScore = htmlDocSpecificGame.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Equals("ratingValue"));
                        _IGNUserScoreInner = _IGNUserScore.ElementAt(1).InnerText;
                    }
                    catch (Exception E)
                    {
                        //var _IGNUserScore = htmlDocSpecificGame.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Equals("ratingValue"));
                        //_IGNUserScoreInner = _IGNUserScore.First().ChildNodes.ElementAt(1).InnerText;

                        if (retry == 1 || retry == 2)
                        {
                            _IGNUserScoreInner = "0";
                        }
                        else
                        {
                            retry++;
                            goto AGAIN;
                        }
                    }

                    try
                    {
                        _IGNGameNameInner = _IGNGameNameInner.Replace("&amp", "&");
                        _IGNGameNameInner = _IGNGameNameInner.Replace("&#039", "'");

                        Regex rgx = new Regex("[^a-zA-Z0-9 &]");
                        _IGNGameNameInner = rgx.Replace(_IGNGameNameInner, "");
                        _IGNGameNameInner = Regex.Replace(_IGNGameNameInner, @"\t|\n|\r", "");

                        RegexOptions options = RegexOptions.None;
                        Regex        regex   = new Regex("[ ]{2,}", options);
                        _IGNGameNameInner = regex.Replace(_IGNGameNameInner, " ");
                    }
                    catch
                    {
                        _IGNGameNameInner = "NOT PROVIDED";
                    }

                    //Score
                    try
                    {
                        _IGNUserScoreInner = Regex.Replace(_IGNUserScoreInner, @"\t|\n|\r|,|\s", "");
                    }
                    catch
                    {
                        _IGNUserScoreInner = "0";
                    }

                    try
                    {
                        _IGNScoreInner = Regex.Replace(_IGNScoreInner, @"\t|\n|\r|,|\s", "");

                        RegexOptions options = RegexOptions.None;
                        Regex        regex   = new Regex("[ ]{2,}", options);
                        _IGNScoreInner = regex.Replace(_IGNScoreInner, " ");
                    }
                    catch
                    {
                        _IGNScoreInner = "0";
                    }


                    double _scoreParseResult     = 0;
                    double _userScoreParseResult = 0;

                    Double.TryParse(_IGNScoreInner, out _scoreParseResult);
                    Double.TryParse(_IGNUserScoreInner, out _userScoreParseResult);

                    IGNGame _game = new IGNGame
                    {
                        Name      = _IGNGameNameInner,
                        Score     = _scoreParseResult,
                        UserScore = _userScoreParseResult,
                        Wiki      = _IGNWiki
                    };

                    SaveData.SafeDataSQLDatabaseIGN(_game);


                    Console.WriteLine("IGN Game Name: " + _IGNGameNameInner);
                    Console.WriteLine("IGN Score: " + _IGNScoreInner);
                    Console.WriteLine("IGN User Score: " + _IGNUserScoreInner);
                    Console.WriteLine("IGN Wiki: " + _IGNWiki);

                    Console.WriteLine("//");

                    loop++;
                }
            }

            Console.WriteLine("FINISHED IGN!");
            Console.Beep(3000, 1000);
            Console.ReadLine();
        }