public static void SafeDataSQLDatabaseIGN(IGNGame _game)
{
try
{
var db = new UpdateModelII();
db.IGNGames.Add(_game);
db.SaveChanges();
}
catch (DbUpdateException e)
{
//foreach (var validationErrors in e.GetBaseException)
//{
// foreach (var validationError in validationErrors.ValidationErrors)
// {
// System.Console.WriteLine("Property: {0} Error: {1}", validationError.PropertyName, validationError.ErrorMessage);
// }
//}
string h = e.GetBaseException().ToString();
h = h;
}
}
public static void ScrapIGNWebsite()
{
for (int pageNumber = 4600; pageNumber <= 4950; pageNumber += 50)
{
Thread.Sleep(1000);
var html = @"http://www.ign.com/games?startIndex=" + pageNumber + "&sortBy=popularity&sortOrder=desc";
HtmlWeb web = new HtmlWeb();
var htmlDoc = web.Load(html);
var htmlNodes = htmlDoc.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("id") && d.Attributes["id"].Value.Contains("item-list"));
var htmlNodesNextLevel = htmlNodes.First().ChildNodes[3].Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Contains("clear itemList-itemShort"));
int loop = 0;
foreach (var node in htmlNodesNextLevel)
{
Thread.Sleep(1000);
var _IGNGameName = node.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Equals("item-title"));
var _IGNGameNameSecondLevel = _IGNGameName.First().ChildNodes[1];
var _IGNGameNameInner = _IGNGameNameSecondLevel.InnerText;
var _IGNGamePage = _IGNGameNameSecondLevel.Attributes["href"].Value;
var _IGNScore = node.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Equals("grid_3"));
var _IGNScoreInner = "";
try
{
_IGNScoreInner = _IGNScore.First().InnerText;
}
catch
{
_IGNScoreInner = "NOT PROVIDED";
}
int retry = 0;
AGAIN:
Thread.Sleep(1000);
var rootHtml = @"http://www.ign.com";
rootHtml = @rootHtml + _IGNGamePage;
var _IGNWiki = "";
var htmlDocSpecificGame = new HtmlDocument();
htmlDocSpecificGame = web.Load(@rootHtml);
try
{
var children = htmlDocSpecificGame.DocumentNode.Descendants("a");
var _htmlNodeIGNWiki = htmlDocSpecificGame.DocumentNode.Descendants("a").Where(d => d.Attributes.Contains("title") && d.Attributes["title"].Value.Equals("wiki-guide"));
_IGNWiki = @_htmlNodeIGNWiki.First().Attributes["href"].Value;
}
catch
{
//var _htmlNodeIGNWiki = htmlDocSpecificGame.DocumentNode.Descendants("a").Where(d => d.Attributes.Contains("title") && d.Attributes["title"].Value.Equals("wiki-guide"));
//_IGNWiki = @_htmlNodeIGNWiki.First().Attributes["href"].Value;
if (retry == 1 || retry == 2)
{
_IGNWiki = "NOT PROVIDED";
}
else
{
retry++;
goto AGAIN;
}
}
var _IGNUserScoreInner = "";
try
{
var _IGNUserScore = htmlDocSpecificGame.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Equals("ratingValue"));
_IGNUserScoreInner = _IGNUserScore.ElementAt(1).InnerText;
}
catch (Exception E)
{
//var _IGNUserScore = htmlDocSpecificGame.DocumentNode.Descendants("div").Where(d => d.Attributes.Contains("class") && d.Attributes["class"].Value.Equals("ratingValue"));
//_IGNUserScoreInner = _IGNUserScore.First().ChildNodes.ElementAt(1).InnerText;
if (retry == 1 || retry == 2)
{
_IGNUserScoreInner = "0";
}
else
{
retry++;
goto AGAIN;
}
}
try
{
_IGNGameNameInner = _IGNGameNameInner.Replace("&", "&");
_IGNGameNameInner = _IGNGameNameInner.Replace("'", "'");
Regex rgx = new Regex("[^a-zA-Z0-9 &]");
_IGNGameNameInner = rgx.Replace(_IGNGameNameInner, "");
_IGNGameNameInner = Regex.Replace(_IGNGameNameInner, @"\t|\n|\r", "");
RegexOptions options = RegexOptions.None;
Regex regex = new Regex("[ ]{2,}", options);
_IGNGameNameInner = regex.Replace(_IGNGameNameInner, " ");
}
catch
{
_IGNGameNameInner = "NOT PROVIDED";
}
//Score
try
{
_IGNUserScoreInner = Regex.Replace(_IGNUserScoreInner, @"\t|\n|\r|,|\s", "");
}
catch
{
_IGNUserScoreInner = "0";
}
try
{
_IGNScoreInner = Regex.Replace(_IGNScoreInner, @"\t|\n|\r|,|\s", "");
RegexOptions options = RegexOptions.None;
Regex regex = new Regex("[ ]{2,}", options);
_IGNScoreInner = regex.Replace(_IGNScoreInner, " ");
}
catch
{
_IGNScoreInner = "0";
}
double _scoreParseResult = 0;
double _userScoreParseResult = 0;
Double.TryParse(_IGNScoreInner, out _scoreParseResult);
Double.TryParse(_IGNUserScoreInner, out _userScoreParseResult);
IGNGame _game = new IGNGame
{
Name = _IGNGameNameInner,
Score = _scoreParseResult,
UserScore = _userScoreParseResult,
Wiki = _IGNWiki
};
SaveData.SafeDataSQLDatabaseIGN(_game);
Console.WriteLine("IGN Game Name: " + _IGNGameNameInner);
Console.WriteLine("IGN Score: " + _IGNScoreInner);
Console.WriteLine("IGN User Score: " + _IGNUserScoreInner);
Console.WriteLine("IGN Wiki: " + _IGNWiki);
Console.WriteLine("//");
loop++;
}
}
Console.WriteLine("FINISHED IGN!");
Console.Beep(3000, 1000);
Console.ReadLine();
}