public static void SafeDataSQLDatabaseGameSpot(GameSpotGame _game)
{
try
{
var db = new UpdateModelII();
db.GameSpotGames.Add(_game);
db.SaveChanges();
}
catch (Exception e)
{
//foreach (var validationErrors in e.EntityValidationErrors)
//{
// foreach (var validationError in validationErrors.ValidationErrors)
// {
// System.Console.WriteLine("Property: {0} Error: {1}", validationError.PropertyName, validationError.ErrorMessage);
// }
//}
string h = e.ToString();
}
}
public static void ScrapGameSpotWebsite()
{
for (int pageNumber = 101; pageNumber <= 200; pageNumber++)
{
var html = @"https://www.gamespot.com/new-games/?sort=score&game_filter_type%5Bplatform%5D=&game_filter_type%5BminRating%5D=&game_filter_type%5BtimeFrame%5D=&game_filter_type%5BstartDate%5D=&game_filter_type%5BendDate%5D=&game_filter_type%5Btheme%5D=&game_filter_type%5Bregion%5D=1&game_filter_type%5Bletter%5D=&page=" + pageNumber;
HtmlWeb web = new HtmlWeb();
var htmlDoc = web.Load(html);
var htmlNodes = htmlDoc.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("media media-game media--small"));
foreach (var node in htmlNodes)
{
var _gameSpotGameName = node.Descendants("h3").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("media-title"));
var _gameSpotScore = node.Descendants("strong").Where(d => d.Attributes.Contains("title") && d.Attributes["title"].Value.Contains("GameSpot Score"));
var _gameSpotScoreInner = "";
try
{
_gameSpotScoreInner = _gameSpotScore.First().InnerHtml;
}
catch
{
_gameSpotScoreInner = "0";
}
var _gameSpotRecommendation = node.Descendants("span").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("score-word"));
var _gameSpotRecommendationInner = "";
try
{
_gameSpotRecommendationInner = _gameSpotRecommendation.First().InnerText;
}
catch
{
_gameSpotRecommendationInner = "NOT PROVIDED";
}
var _gameSpotComments = node.Descendants("p").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("media-deck"));
var _innerGameSpotComments = _gameSpotComments.First().InnerText;
_innerGameSpotComments = _innerGameSpotComments.Replace("&", "&");
_innerGameSpotComments = _innerGameSpotComments.Replace("'", "'");
var _userScoreGameSpot = node.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("media-well--review-user"));
var _userScoreGameSpotInner = "";
try
{
_userScoreGameSpotInner = _userScoreGameSpot.First().SelectSingleNode("strong").InnerText;
}
catch
{
_userScoreGameSpotInner = "NOT PROVIDED";
}
var _releaseDateGameSpot = node.Descendants("time").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("media-date"));
var _releaseDateGameSpotInner = _releaseDateGameSpot.First().ChildAttributes("datetime").First().Value;
var _platFormsAvailable = node.Descendants("ul").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("system-list"));
var _platFormsAvailableInner = _platFormsAvailable.First().ChildNodes.Descendants("span").Where(d => d.Attributes.Contains("itemprop") && d.Attributes["itemprop"].Value.Contains("device"));
var _platFormsAvailableString = "";
int count = 0;
foreach (var child in _platFormsAvailableInner)
{
if (count == 0)
{
_platFormsAvailableString = _platFormsAvailableString + child.InnerText;
}
else
{
_platFormsAvailableString = _platFormsAvailableString + " " + child.InnerText;
}
count++;
}
var _urlBaseGameSpot = @"https://www.gamespot.com";
var _gameSpotWiki = "";
try
{
var _gameSpotGameWikiAttribute = node.ChildNodes[0].Attributes[1].Value;
_gameSpotWiki = _urlBaseGameSpot + _gameSpotGameWikiAttribute;
}
catch
{
_gameSpotWiki = "NOT PROVIDED";
}
double _scoreParseResult = 0;
double _userScoreParseResult = 0;
Double.TryParse(_gameSpotScoreInner, out _scoreParseResult);
Double.TryParse(_userScoreGameSpotInner, out _userScoreParseResult);
var _innerGameSpotGameName = _gameSpotGameName.First().InnerHtml;
try
{
_innerGameSpotGameName = _innerGameSpotGameName.Replace("&", "&");
_innerGameSpotGameName = _innerGameSpotGameName.Replace("'", "'");
Regex rgx = new Regex("[^a-zA-Z0-9 &]");
_innerGameSpotGameName = rgx.Replace(_innerGameSpotGameName, "");
_innerGameSpotGameName = Regex.Replace(_innerGameSpotGameName, @"\t|\n|\r", "");
}
catch
{
_innerGameSpotGameName = "NOT PROVIDED";
}
GameSpotGame _game = new GameSpotGame
{
Name = _innerGameSpotGameName,
Score = _scoreParseResult,
Recommendation = _gameSpotRecommendationInner,
Comments = _innerGameSpotComments,
UserScore = _userScoreParseResult,
Wiki = _gameSpotWiki,
ReleaseDate = _releaseDateGameSpotInner,
Platforms = _platFormsAvailableString
};
SaveData.SafeDataSQLDatabaseGameSpot(_game);
Console.WriteLine("GameSpot Game Name: " + _gameSpotGameName.First().InnerHtml);
Console.WriteLine("GameSpot Score: " + _gameSpotScoreInner);
Console.WriteLine("GameSpot Reccommendation: " + _gameSpotRecommendationInner);
Console.WriteLine("GameSpot Comments: " + _innerGameSpotComments);
Console.WriteLine("GameSpot User Score: " + _userScoreGameSpotInner);
Console.WriteLine("GameSpot Wiki: " + _gameSpotWiki);
Console.WriteLine("Release Date: " + _releaseDateGameSpotInner);
Console.WriteLine("Platforms Available: " + _platFormsAvailableString);
Console.WriteLine("//");
}
}
Console.WriteLine("FINISHED GAMESPOT!");
Console.Beep(3000, 1000);
Console.ReadLine();
}
