public static void SafeDataSQLDatabaseIGN(IGNGame _game) { try { var db = new UpdateModelII(); db.IGNGames.Add(_game); db.SaveChanges(); } catch (DbUpdateException e) { //foreach (var validationErrors in e.GetBaseException) //{ // foreach (var validationError in validationErrors.ValidationErrors) // { // System.Console.WriteLine("Property: {0} Error: {1}", validationError.PropertyName, validationError.ErrorMessage); // } //} string h = e.GetBaseException().ToString(); h = h; } }
public static void ScrapIGNWebsite() { for (int pageNumber = 4600; pageNumber <= 4950; pageNumber += 50) { Thread.Sleep(1000); var html = @"http://www.ign.com/games?startIndex=" + pageNumber + "&sortBy=popularity&sortOrder=desc"; HtmlWeb web = new HtmlWeb(); var htmlDoc = web.Load(html); var htmlNodes = htmlDoc.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("id") && d.Attributes["id"].Value.Contains("item-list")); var htmlNodesNextLevel = htmlNodes.First().ChildNodes[3].Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("clear itemList-itemShort")); int loop = 0; foreach (var node in htmlNodesNextLevel) { Thread.Sleep(1000); var _IGNGameName = node.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Equals("item-title")); var _IGNGameNameSecondLevel = _IGNGameName.First().ChildNodes[1]; var _IGNGameNameInner = _IGNGameNameSecondLevel.InnerText; var _IGNGamePage = _IGNGameNameSecondLevel.Attributes["href"].Value; var _IGNScore = node.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Equals("grid_3")); var _IGNScoreInner = ""; try { _IGNScoreInner = _IGNScore.First().InnerText; } catch { _IGNScoreInner = "NOT PROVIDED"; } int retry = 0; AGAIN: Thread.Sleep(1000); var rootHtml = @"http://www.ign.com"; rootHtml = @rootHtml + _IGNGamePage; var _IGNWiki = ""; var htmlDocSpecificGame = new HtmlDocument(); htmlDocSpecificGame = web.Load(@rootHtml); try { var children = htmlDocSpecificGame.DocumentNode.Descendants("a"); var _htmlNodeIGNWiki = htmlDocSpecificGame.DocumentNode.Descendants("a").Where(d => d.Attributes.Contains("title") && d.Attributes["title"].Value.Equals("wiki-guide")); _IGNWiki = @_htmlNodeIGNWiki.First().Attributes["href"].Value; } catch { //var _htmlNodeIGNWiki = htmlDocSpecificGame.DocumentNode.Descendants("a").Where(d => d.Attributes.Contains("title") && d.Attributes["title"].Value.Equals("wiki-guide")); //_IGNWiki = @_htmlNodeIGNWiki.First().Attributes["href"].Value; if (retry == 1 || retry == 2) { _IGNWiki = "NOT PROVIDED"; } else { retry++; goto AGAIN; } } var _IGNUserScoreInner = ""; try { var _IGNUserScore = htmlDocSpecificGame.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Equals("ratingValue")); _IGNUserScoreInner = _IGNUserScore.ElementAt(1).InnerText; } catch (Exception E) { //var _IGNUserScore = htmlDocSpecificGame.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Equals("ratingValue")); //_IGNUserScoreInner = _IGNUserScore.First().ChildNodes.ElementAt(1).InnerText; if (retry == 1 || retry == 2) { _IGNUserScoreInner = "0"; } else { retry++; goto AGAIN; } } try { _IGNGameNameInner = _IGNGameNameInner.Replace("&", "&"); _IGNGameNameInner = _IGNGameNameInner.Replace("'", "'"); Regex rgx = new Regex("[^a-zA-Z0-9 &]"); _IGNGameNameInner = rgx.Replace(_IGNGameNameInner, ""); _IGNGameNameInner = Regex.Replace(_IGNGameNameInner, @"\t|\n|\r", ""); RegexOptions options = RegexOptions.None; Regex regex = new Regex("[ ]{2,}", options); _IGNGameNameInner = regex.Replace(_IGNGameNameInner, " "); } catch { _IGNGameNameInner = "NOT PROVIDED"; } //Score try { _IGNUserScoreInner = Regex.Replace(_IGNUserScoreInner, @"\t|\n|\r|,|\s", ""); } catch { _IGNUserScoreInner = "0"; } try { _IGNScoreInner = Regex.Replace(_IGNScoreInner, @"\t|\n|\r|,|\s", ""); RegexOptions options = RegexOptions.None; Regex regex = new Regex("[ ]{2,}", options); _IGNScoreInner = regex.Replace(_IGNScoreInner, " "); } catch { _IGNScoreInner = "0"; } double _scoreParseResult = 0; double _userScoreParseResult = 0; Double.TryParse(_IGNScoreInner, out _scoreParseResult); Double.TryParse(_IGNUserScoreInner, out _userScoreParseResult); IGNGame _game = new IGNGame { Name = _IGNGameNameInner, Score = _scoreParseResult, UserScore = _userScoreParseResult, Wiki = _IGNWiki }; SaveData.SafeDataSQLDatabaseIGN(_game); Console.WriteLine("IGN Game Name: " + _IGNGameNameInner); Console.WriteLine("IGN Score: " + _IGNScoreInner); Console.WriteLine("IGN User Score: " + _IGNUserScoreInner); Console.WriteLine("IGN Wiki: " + _IGNWiki); Console.WriteLine("//"); loop++; } } Console.WriteLine("FINISHED IGN!"); Console.Beep(3000, 1000); Console.ReadLine(); }