public static void SafeDataSQLDatabaseGameSpot(GameSpotGame _game) { try { var db = new UpdateModelII(); db.GameSpotGames.Add(_game); db.SaveChanges(); } catch (Exception e) { //foreach (var validationErrors in e.EntityValidationErrors) //{ // foreach (var validationError in validationErrors.ValidationErrors) // { // System.Console.WriteLine("Property: {0} Error: {1}", validationError.PropertyName, validationError.ErrorMessage); // } //} string h = e.ToString(); } }
public static void ScrapGameSpotWebsite() { for (int pageNumber = 101; pageNumber <= 200; pageNumber++) { var html = @"https://www.gamespot.com/new-games/?sort=score&game_filter_type%5Bplatform%5D=&game_filter_type%5BminRating%5D=&game_filter_type%5BtimeFrame%5D=&game_filter_type%5BstartDate%5D=&game_filter_type%5BendDate%5D=&game_filter_type%5Btheme%5D=&game_filter_type%5Bregion%5D=1&game_filter_type%5Bletter%5D=&page=" + pageNumber; HtmlWeb web = new HtmlWeb(); var htmlDoc = web.Load(html); var htmlNodes = htmlDoc.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("media media-game media--small")); foreach (var node in htmlNodes) { var _gameSpotGameName = node.Descendants("h3").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("media-title")); var _gameSpotScore = node.Descendants("strong").Where(d => d.Attributes.Contains("title") && d.Attributes["title"].Value.Contains("GameSpot Score")); var _gameSpotScoreInner = ""; try { _gameSpotScoreInner = _gameSpotScore.First().InnerHtml; } catch { _gameSpotScoreInner = "0"; } var _gameSpotRecommendation = node.Descendants("span").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("score-word")); var _gameSpotRecommendationInner = ""; try { _gameSpotRecommendationInner = _gameSpotRecommendation.First().InnerText; } catch { _gameSpotRecommendationInner = "NOT PROVIDED"; } var _gameSpotComments = node.Descendants("p").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("media-deck")); var _innerGameSpotComments = _gameSpotComments.First().InnerText; _innerGameSpotComments = _innerGameSpotComments.Replace("&", "&"); _innerGameSpotComments = _innerGameSpotComments.Replace("'", "'"); var _userScoreGameSpot = node.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("media-well--review-user")); var _userScoreGameSpotInner = ""; try { _userScoreGameSpotInner = _userScoreGameSpot.First().SelectSingleNode("strong").InnerText; } catch { _userScoreGameSpotInner = "NOT PROVIDED"; } var _releaseDateGameSpot = node.Descendants("time").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("media-date")); var _releaseDateGameSpotInner = _releaseDateGameSpot.First().ChildAttributes("datetime").First().Value; var _platFormsAvailable = node.Descendants("ul").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("system-list")); var _platFormsAvailableInner = _platFormsAvailable.First().ChildNodes.Descendants("span").Where(d => d.Attributes.Contains("itemprop") && d.Attributes["itemprop"].Value.Contains("device")); var _platFormsAvailableString = ""; int count = 0; foreach (var child in _platFormsAvailableInner) { if (count == 0) { _platFormsAvailableString = _platFormsAvailableString + child.InnerText; } else { _platFormsAvailableString = _platFormsAvailableString + " " + child.InnerText; } count++; } var _urlBaseGameSpot = @"https://www.gamespot.com"; var _gameSpotWiki = ""; try { var _gameSpotGameWikiAttribute = node.ChildNodes[0].Attributes[1].Value; _gameSpotWiki = _urlBaseGameSpot + _gameSpotGameWikiAttribute; } catch { _gameSpotWiki = "NOT PROVIDED"; } double _scoreParseResult = 0; double _userScoreParseResult = 0; Double.TryParse(_gameSpotScoreInner, out _scoreParseResult); Double.TryParse(_userScoreGameSpotInner, out _userScoreParseResult); var _innerGameSpotGameName = _gameSpotGameName.First().InnerHtml; try { _innerGameSpotGameName = _innerGameSpotGameName.Replace("&", "&"); _innerGameSpotGameName = _innerGameSpotGameName.Replace("'", "'"); Regex rgx = new Regex("[^a-zA-Z0-9 &]"); _innerGameSpotGameName = rgx.Replace(_innerGameSpotGameName, ""); _innerGameSpotGameName = Regex.Replace(_innerGameSpotGameName, @"\t|\n|\r", ""); } catch { _innerGameSpotGameName = "NOT PROVIDED"; } GameSpotGame _game = new GameSpotGame { Name = _innerGameSpotGameName, Score = _scoreParseResult, Recommendation = _gameSpotRecommendationInner, Comments = _innerGameSpotComments, UserScore = _userScoreParseResult, Wiki = _gameSpotWiki, ReleaseDate = _releaseDateGameSpotInner, Platforms = _platFormsAvailableString }; SaveData.SafeDataSQLDatabaseGameSpot(_game); Console.WriteLine("GameSpot Game Name: " + _gameSpotGameName.First().InnerHtml); Console.WriteLine("GameSpot Score: " + _gameSpotScoreInner); Console.WriteLine("GameSpot Reccommendation: " + _gameSpotRecommendationInner); Console.WriteLine("GameSpot Comments: " + _innerGameSpotComments); Console.WriteLine("GameSpot User Score: " + _userScoreGameSpotInner); Console.WriteLine("GameSpot Wiki: " + _gameSpotWiki); Console.WriteLine("Release Date: " + _releaseDateGameSpotInner); Console.WriteLine("Platforms Available: " + _platFormsAvailableString); Console.WriteLine("//"); } } Console.WriteLine("FINISHED GAMESPOT!"); Console.Beep(3000, 1000); Console.ReadLine(); }