public static void SafeDataSQLDatabaseMetaCritics(MetaCriticsGame _game) { try { var db = new UpdatedModelGames(); db.MetaCriticsGames.Add(_game); db.SaveChanges(); } catch (DbUpdateException e) { //foreach (var validationErrors in e.GetBaseException) //{ // foreach (var validationError in validationErrors.ValidationErrors) // { // System.Console.WriteLine("Property: {0} Error: {1}", validationError.PropertyName, validationError.ErrorMessage); // } //} string h = e.GetBaseException().ToString(); h = h; } }
public static void ScrapMetaCriticsWebsite() { for (int pageNumber = 31; pageNumber <= 60; pageNumber++) { var html = @"http://www.metacritic.com/browse/games/score/metascore/all/all/filtered?view=detailed&sort=desc&page=" + pageNumber; HtmlWeb web = new HtmlWeb(); var htmlDoc = web.Load(html); var htmlNodes = htmlDoc.DocumentNode.Descendants("ol").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("list_products list_product_summaries")); foreach (var node in htmlNodes.First().ChildNodes) { //Name var _metaCriticsGameName = ""; try { _metaCriticsGameName = node.Descendants("h3").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("product_title")).First().InnerText; _metaCriticsGameName = _metaCriticsGameName.Replace("&", "&"); _metaCriticsGameName = _metaCriticsGameName.Replace("'", "'"); Regex rgx = new Regex("[^a-zA-Z0-9 &]"); _metaCriticsGameName = rgx.Replace(_metaCriticsGameName, ""); _metaCriticsGameName = Regex.Replace(_metaCriticsGameName, @"\t|\n|\r", ""); } catch { _metaCriticsGameName = "NOT PROVIDED"; } var _metaCriticsScore = node.Descendants("a").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("basic_stat product_score")); //Score var _metaCriticsScoreInner = ""; try { _metaCriticsScoreInner = _metaCriticsScore.First().InnerText; _metaCriticsScoreInner = Regex.Replace(_metaCriticsScoreInner, @"\t|\n|\r|,|\s", ""); } catch { _metaCriticsScoreInner = "0"; } //User Score var _metaCriticsUserScore = node.Descendants("li").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("stat product_avguserscore")); var _metaCriticsUserScoreInner = ""; try { _metaCriticsUserScoreInner = _metaCriticsUserScore.First().ChildNodes[3].InnerText; _metaCriticsUserScoreInner = Regex.Replace(_metaCriticsUserScoreInner, @"\t|\n|\r|\s", ""); } catch { _metaCriticsUserScoreInner = "NOT PROVIDED"; } //Genre var _metaCriticsGenreWords = node.Descendants("li").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("stat genre")); var _metaCriticsGenreWordsInner = ""; try { _metaCriticsGenreWordsInner = _metaCriticsGenreWords.First().ChildNodes[3].InnerText; } catch { try { _metaCriticsGenreWords = node.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("product_basics product_image small_image")); var _relativeUrlGame = _metaCriticsGenreWords.First().ChildNodes[0].Attributes["href"].Value; var rootHtml = @"http://www.metacritic.com"; rootHtml = rootHtml + _relativeUrlGame; var htmlDocSpecificGame = web.Load(@rootHtml); var _htmlNodeMetaCriticsGenreWords = htmlDocSpecificGame.DocumentNode.Descendants("li").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("summary_detail product_genre")); var _htmlNodeMetaCriticsGenreWordsInner = _htmlNodeMetaCriticsGenreWords.First().ChildNodes.Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("data")); _metaCriticsGenreWordsInner = ""; int count = 0; foreach (var child in _htmlNodeMetaCriticsGenreWordsInner) { if (count == 0 && child.InnerText != "Genre(s):") { _metaCriticsGenreWordsInner = _metaCriticsGenreWordsInner + child.InnerText; } else if (child.InnerText != "Genre(s):") { _metaCriticsGenreWordsInner = _metaCriticsGenreWordsInner + " " + child.InnerText; } count++; } _metaCriticsGenreWordsInner = _htmlNodeMetaCriticsGenreWords.First().InnerText; } catch (Exception e) { var z = e; _metaCriticsGenreWordsInner = "NOT PROVIDED"; } } _metaCriticsGenreWordsInner = Regex.Replace(_metaCriticsGenreWordsInner, @"\t|\n|\r", ""); _metaCriticsGenreWordsInner = Regex.Replace(_metaCriticsGenreWordsInner, @",", " "); RegexOptions options = RegexOptions.None; Regex regex = new Regex("[ ]{2,}", options); _metaCriticsGenreWordsInner = regex.Replace(_metaCriticsGenreWordsInner, " "); _metaCriticsGenreWordsInner = _metaCriticsGenreWordsInner.Replace("Genre(s):", ""); //Publisher var _metaCriticsPublisher = node.Descendants("li").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("stat publisher")); var _metaCriticsPublisherInner = ""; try { _metaCriticsPublisherInner = _metaCriticsPublisher.First().ChildNodes[3].InnerText; _metaCriticsPublisherInner = Regex.Replace(_metaCriticsPublisherInner, @"\t|\n|\r", ""); _metaCriticsPublisherInner = Regex.Replace(_metaCriticsPublisherInner, @",", " "); RegexOptions optionsII = RegexOptions.None; Regex regexII = new Regex("[ ]{2,}", options); _metaCriticsPublisherInner = regex.Replace(_metaCriticsPublisherInner, " "); } catch { _metaCriticsPublisherInner = "NOT PROVIDED"; } //Rating var _metaCriticsRating = node.Descendants("li").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("stat maturity_rating")); var _metaCriticsRatingInner = ""; try { _metaCriticsRatingInner = _metaCriticsRating.First().ChildNodes[3].InnerText; _metaCriticsRatingInner = Regex.Replace(_metaCriticsRatingInner, @"\t|\n|\r|\s", ""); } catch { _metaCriticsRatingInner = "NOT PROVIDED"; } Console.WriteLine("MetaCritics Game Name: " + _metaCriticsGameName); Console.WriteLine("MetaCritics Score: " + _metaCriticsScoreInner); Console.WriteLine("MetaCritics User Score: " + _metaCriticsUserScoreInner); Console.WriteLine("MetaCritics Genre: " + _metaCriticsGenreWordsInner); Console.WriteLine("MetaCritics Rating: " + _metaCriticsRatingInner); Console.WriteLine("MetaCritics Publisher: " + _metaCriticsPublisherInner); Console.WriteLine("//"); double _scoreParseResult = 0; double _userScoreParseResult = 0; Double.TryParse(_metaCriticsScoreInner, out _scoreParseResult); Double.TryParse(_metaCriticsUserScoreInner, out _userScoreParseResult); MetaCriticsGame _game = new MetaCriticsGame { Name = _metaCriticsGameName, Score = _scoreParseResult, UserScore = _userScoreParseResult, GenreKeyWords = _metaCriticsGenreWordsInner, Rating = _metaCriticsRatingInner, Publisher = _metaCriticsPublisherInner }; SaveData.SafeDataSQLDatabaseMetaCritics(_game); } } Console.WriteLine("FINISHED METACRITICS!!"); Console.Beep(3000, 1000); Console.ReadLine(); }