public static void SafeDataSQLDatabaseMetaCritics(MetaCriticsGame _game)
{
try
{
var db = new UpdatedModelGames();
db.MetaCriticsGames.Add(_game);
db.SaveChanges();
}
catch (DbUpdateException e)
{
//foreach (var validationErrors in e.GetBaseException)
//{
// foreach (var validationError in validationErrors.ValidationErrors)
// {
// System.Console.WriteLine("Property: {0} Error: {1}", validationError.PropertyName, validationError.ErrorMessage);
// }
//}
string h = e.GetBaseException().ToString();
h = h;
}
}
public static void ScrapMetaCriticsWebsite()
{
for (int pageNumber = 31; pageNumber <= 60; pageNumber++)
{
var html = @"http://www.metacritic.com/browse/games/score/metascore/all/all/filtered?view=detailed&sort=desc&page=" + pageNumber;
HtmlWeb web = new HtmlWeb();
var htmlDoc = web.Load(html);
var htmlNodes = htmlDoc.DocumentNode.Descendants("ol").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("list_products list_product_summaries"));
foreach (var node in htmlNodes.First().ChildNodes)
{
//Name
var _metaCriticsGameName = "";
try
{
_metaCriticsGameName = node.Descendants("h3").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("product_title")).First().InnerText;
_metaCriticsGameName = _metaCriticsGameName.Replace("&", "&");
_metaCriticsGameName = _metaCriticsGameName.Replace("'", "'");
Regex rgx = new Regex("[^a-zA-Z0-9 &]");
_metaCriticsGameName = rgx.Replace(_metaCriticsGameName, "");
_metaCriticsGameName = Regex.Replace(_metaCriticsGameName, @"\t|\n|\r", "");
}
catch
{
_metaCriticsGameName = "NOT PROVIDED";
}
var _metaCriticsScore = node.Descendants("a").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("basic_stat product_score"));
//Score
var _metaCriticsScoreInner = "";
try
{
_metaCriticsScoreInner = _metaCriticsScore.First().InnerText;
_metaCriticsScoreInner = Regex.Replace(_metaCriticsScoreInner, @"\t|\n|\r|,|\s", "");
}
catch
{
_metaCriticsScoreInner = "0";
}
//User Score
var _metaCriticsUserScore = node.Descendants("li").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("stat product_avguserscore"));
var _metaCriticsUserScoreInner = "";
try
{
_metaCriticsUserScoreInner = _metaCriticsUserScore.First().ChildNodes[3].InnerText;
_metaCriticsUserScoreInner = Regex.Replace(_metaCriticsUserScoreInner, @"\t|\n|\r|\s", "");
}
catch
{
_metaCriticsUserScoreInner = "NOT PROVIDED";
}
//Genre
var _metaCriticsGenreWords = node.Descendants("li").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("stat genre"));
var _metaCriticsGenreWordsInner = "";
try
{
_metaCriticsGenreWordsInner = _metaCriticsGenreWords.First().ChildNodes[3].InnerText;
}
catch
{
try
{
_metaCriticsGenreWords = node.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("product_basics product_image small_image"));
var _relativeUrlGame = _metaCriticsGenreWords.First().ChildNodes[0].Attributes["href"].Value;
var rootHtml = @"http://www.metacritic.com";
rootHtml = rootHtml + _relativeUrlGame;
var htmlDocSpecificGame = web.Load(@rootHtml);
var _htmlNodeMetaCriticsGenreWords = htmlDocSpecificGame.DocumentNode.Descendants("li").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("summary_detail product_genre"));
var _htmlNodeMetaCriticsGenreWordsInner = _htmlNodeMetaCriticsGenreWords.First().ChildNodes.Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("data"));
_metaCriticsGenreWordsInner = "";
int count = 0;
foreach (var child in _htmlNodeMetaCriticsGenreWordsInner)
{
if (count == 0 && child.InnerText != "Genre(s):")
{
_metaCriticsGenreWordsInner = _metaCriticsGenreWordsInner + child.InnerText;
}
else if (child.InnerText != "Genre(s):")
{
_metaCriticsGenreWordsInner = _metaCriticsGenreWordsInner + " " + child.InnerText;
}
count++;
}
_metaCriticsGenreWordsInner = _htmlNodeMetaCriticsGenreWords.First().InnerText;
}
catch (Exception e)
{
var z = e;
_metaCriticsGenreWordsInner = "NOT PROVIDED";
}
}
_metaCriticsGenreWordsInner = Regex.Replace(_metaCriticsGenreWordsInner, @"\t|\n|\r", "");
_metaCriticsGenreWordsInner = Regex.Replace(_metaCriticsGenreWordsInner, @",", " ");
RegexOptions options = RegexOptions.None;
Regex regex = new Regex("[ ]{2,}", options);
_metaCriticsGenreWordsInner = regex.Replace(_metaCriticsGenreWordsInner, " ");
_metaCriticsGenreWordsInner = _metaCriticsGenreWordsInner.Replace("Genre(s):", "");
//Publisher
var _metaCriticsPublisher = node.Descendants("li").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("stat publisher"));
var _metaCriticsPublisherInner = "";
try
{
_metaCriticsPublisherInner = _metaCriticsPublisher.First().ChildNodes[3].InnerText;
_metaCriticsPublisherInner = Regex.Replace(_metaCriticsPublisherInner, @"\t|\n|\r", "");
_metaCriticsPublisherInner = Regex.Replace(_metaCriticsPublisherInner, @",", " ");
RegexOptions optionsII = RegexOptions.None;
Regex regexII = new Regex("[ ]{2,}", options);
_metaCriticsPublisherInner = regex.Replace(_metaCriticsPublisherInner, " ");
}
catch
{
_metaCriticsPublisherInner = "NOT PROVIDED";
}
//Rating
var _metaCriticsRating = node.Descendants("li").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("stat maturity_rating"));
var _metaCriticsRatingInner = "";
try
{
_metaCriticsRatingInner = _metaCriticsRating.First().ChildNodes[3].InnerText;
_metaCriticsRatingInner = Regex.Replace(_metaCriticsRatingInner, @"\t|\n|\r|\s", "");
}
catch
{
_metaCriticsRatingInner = "NOT PROVIDED";
}
Console.WriteLine("MetaCritics Game Name: " + _metaCriticsGameName);
Console.WriteLine("MetaCritics Score: " + _metaCriticsScoreInner);
Console.WriteLine("MetaCritics User Score: " + _metaCriticsUserScoreInner);
Console.WriteLine("MetaCritics Genre: " + _metaCriticsGenreWordsInner);
Console.WriteLine("MetaCritics Rating: " + _metaCriticsRatingInner);
Console.WriteLine("MetaCritics Publisher: " + _metaCriticsPublisherInner);
Console.WriteLine("//");
double _scoreParseResult = 0;
double _userScoreParseResult = 0;
Double.TryParse(_metaCriticsScoreInner, out _scoreParseResult);
Double.TryParse(_metaCriticsUserScoreInner, out _userScoreParseResult);
MetaCriticsGame _game = new MetaCriticsGame
{
Name = _metaCriticsGameName,
Score = _scoreParseResult,
UserScore = _userScoreParseResult,
GenreKeyWords = _metaCriticsGenreWordsInner,
Rating = _metaCriticsRatingInner,
Publisher = _metaCriticsPublisherInner
};
SaveData.SafeDataSQLDatabaseMetaCritics(_game);
}
}
Console.WriteLine("FINISHED METACRITICS!!");
Console.Beep(3000, 1000);
Console.ReadLine();
}
