/// <summary>
/// add a single game entry
/// </summary>
/// <param name="game"></param>
public static void AddGame(MOBY_Game game)
{
using (var aG = new AsniScrapeAdminDbContext())
{
aG.MOBY_Game.Add(game);
aG.SaveChanges();
aG.Dispose();
}
}
/// <summary>
/// update a single game entry
/// </summary>
/// <param name="game"></param>
public static void UpdateGame(MOBY_Game game)
{
using (var uG = new AsniScrapeAdminDbContext())
{
uG.MOBY_Game.Update(game);
uG.SaveChanges();
uG.Dispose();
}
}
/// <summary>
/// save list of games to database (add or update logic included)
/// </summary>
/// <param name="games"></param>
public static void SaveToDatabase(List <MOBY_Game> games)
{
using (var db = new AsniScrapeAdminDbContext())
{
// get current database context
var current = db.MOBY_Game.AsNoTracking().ToList();
List <MOBY_Game> toAdd = new List <MOBY_Game>();
List <MOBY_Game> toUpdate = new List <MOBY_Game>();
// iterate through the games list and separete out games to be added and games to be updated
foreach (var g in games)
{
// games will generally be passed with mid = 0
if (g.mid > 0)
{
MOBY_Game t = (from a in current
where (a.mid == g.mid)
select a).SingleOrDefault();
if (t == null)
{
toAdd.Add(g);
}
else
{
toUpdate.Add(g);
}
}
else
{
MOBY_Game t = (from a in current
where (a.alias == g.alias && a.pid == g.pid)
select a).SingleOrDefault();
if (t == null)
{
toAdd.Add(g);
}
else
{
MOBY_Game m = t;
m.gameTitle = g.gameTitle;
m.alias = g.alias;
m.pid = g.pid;
m.releaseYear = g.releaseYear;
toUpdate.Add(m);
}
}
}
db.MOBY_Game.UpdateRange(toUpdate);
db.MOBY_Game.AddRange(toAdd);
db.SaveChanges();
}
}
/// <summary>
/// scrape all moby platform games
/// </summary>
public async void ScrapeMobyPlatformGames()
{
var mySettings = new MetroDialogSettings()
{
NegativeButtonText = "Cancel Scraping",
AnimateShow = false,
AnimateHide = false,
};
var controller = await mw.ShowProgressAsync("MedLaunch - Getting Basic Games List From mobygames.net", "", settings : mySettings);
controller.SetCancelable(true);
await Task.Delay(100);
await Task.Run(() =>
{
Task.Delay(1);
int count = 1;
int sysCount = mobyplatforms.Count();
controller.Minimum = 0;
controller.Maximum = sysCount;
foreach (var platform in mobyplatforms)
{
List <MOBY_Game> gs = new List <MOBY_Game>();
controller.SetProgress(Convert.ToDouble(count));
controller.SetMessage("Retrieving Game List for Platform: " + platform.name);
// get initial page
string url = platform.listURL;
string initialPage = ReturnWebpage(url, "", 10000);
bool isAttrib = false;
if (url.Contains("attribute/sheet"))
{
isAttrib = true;
}
int totalGames = 0;
if (!isAttrib)
{
/* Get the total number of games available for this system */
// split the html to list via line breaks
List <string> html = initialPage.Split('\n').ToList();
// get only the line that contains the number of games
string hLine = html.Where(a => a.Contains(" games)")).FirstOrDefault();
// get only the substring "xxx games"
string resultString = Regex.Match(hLine, @"(?<=\().+?(?=\))").Value;
// split by whitespace
string[] gArr = resultString.Split(' ');
// get int number of games
totalGames = Convert.ToInt32(gArr[0]);
}
else
{
List <string> html = initialPage.Split('\n').ToList();
string hLine = html.Where(a => a.Contains("(items")).FirstOrDefault();
string resultString = Regex.Match(hLine, @"(?<=\().+?(?=\))").Value;
string[] gArr = resultString.Split(' ');
totalGames = Convert.ToInt32(gArr.Last());
}
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(initialPage);
// build a list of page URLs
double numberofpages = Convert.ToDouble(totalGames) / 25;
int numberOfPages = Convert.ToInt32(Math.Ceiling(numberofpages));
// connect to every page and import all the game information
for (int i = 0; i < numberOfPages; i++)
{
int offset = i * 25;
string newUrl = url.Replace("offset,0", "offset," + offset);
HtmlDocument hDoc = new HtmlDocument();
if (i == 0)
{
hDoc = doc;
}
else
{
string htmlRes = ReturnWebpage(newUrl, "", 10000);
hDoc.LoadHtml(htmlRes);
}
// get just the data table we are interested in
HtmlNode objectTable = hDoc.GetElementbyId("mof_object_list");
// iterate through each row and scrape the game information
int cGame = 1;
foreach (HtmlNode row in objectTable.SelectNodes("tbody/tr"))
{
int currentGameNumber = offset + cGame;
if (controller != null)
{
controller.SetMessage("Scraping basic list of all " + platform.name + " games\nGame: (" + currentGameNumber + " of " + totalGames + ")\nPage: (" + (i + 1) + " of " + numberOfPages + ")");
controller.Minimum = 1;
controller.Maximum = totalGames;
controller.SetProgress(Convert.ToDouble(currentGameNumber));
}
HtmlNode[] cells = (from a in row.SelectNodes("td")
select a).ToArray();
string Title = cells[0].InnerText.Trim();
//var allLi = row.SelectSingleNode("//a[@href]");
string URLstring = cells[0].InnerHtml.Trim();
Regex regex = new Regex("href\\s*=\\s*(?:\"(?<1>[^\"]*)\"|(?<1>\\S+))", RegexOptions.IgnoreCase);
Match match;
string URL = "";
for (match = regex.Match(URLstring); match.Success; match = match.NextMatch())
{
URL = match.Groups[1].ToString();
}
MOBY_Game game = new MOBY_Game();
game.pid = platform.pid;
game.gameTitle = WebUtility.HtmlDecode(Title);
game.alias = WebUtility.HtmlDecode(URL.Split('/').LastOrDefault());
game.releaseYear = cells[1].InnerText.Trim();
// add game to main list
gs.Add(game);
}
}
MOBY_Game.SaveToDatabase(gs);
}
});
await controller.CloseAsync();
if (controller.IsCanceled)
{
await mw.ShowMessageAsync("MOBY Master Games List Download", "Operation Cancelled");
}
else
{
await mw.ShowMessageAsync("MOBY Master Games List Download", "Scanning and Import Completed");
}
}
public async void MobyManualMatch(bool AutoMatchOnSingle100Score)
{
var mySettings = new MetroDialogSettings()
{
NegativeButtonText = "Cancel Scraping",
AnimateShow = false,
AnimateHide = false,
};
var controller = await mw.ShowProgressAsync("Attempting manual match based on word count - mobygames to thegamesdb", "", settings : mySettings);
controller.SetCancelable(true);
await Task.Delay(100);
await Task.Run(() =>
{
Task.Delay(1);
int progress = 0;
List <Junction> jList = new List <Junction>();
List <MasterView> unmatched = (from a in MasterView.GetMasterView()
where a.mid == null
select a).ToList();
controller.Minimum = 0;
controller.Maximum = unmatched.Count();
int co = unmatched.Count();
int matched = 0;
int notmatched = 0;
controller.SetProgress(progress);
controller.SetMessage("TOTAL: " + co + "\n\nMatched: " + matched + "\nUnmatched: " + notmatched);
List <int[]> list = new List <int[]>();
List <MOBY_Game> mgames = MOBY_Game.GetGames().ToList();
bool cancel = false;
foreach (MasterView m in unmatched)
{
if (cancel == true)
{
break;
}
controller.SetProgress(progress);
controller.SetMessage("TOTAL: " + co + "\n\nMatched: " + matched + "\nUnmatched: " + notmatched);
progress++;
int gid = m.gid;
int pid = m.pid;
// build SearchObject
SearchObject so = new SearchObject();
so.searchId = gid;
so.searchString = m.GDBTitle.Trim();
so.listToSearch = (from a in mgames
where a.pid == pid
select new SearchList {
id = a.mid, name = a.gameTitle
}).ToList();
// get search results
so = SearchFunctions.WordCountMatch(so);
// get results list separately
List <SearchResult> results = so.searchResults.OrderByDescending(a => a.score).ToList();
// if automatch on unique 100 score is selected
if (AutoMatchOnSingle100Score)
{
var hundreds = results.Where(a => a.score == 100).ToList();
if (hundreds.Count == 1)
{
Junction j = new Junction();
j.gid = gid;
j.mid = hundreds.First().resultId;
jList.Add(j);
matched++;
//controller.SetMessage("Saving to Database...");
//Junction.SaveToDatabase(jList);
//jList = new List<Junction>();
continue;
}
else
{
notmatched++;
continue;
}
}
else
{
// iterate through each result and prompt user to accept / as for next result / or skip completely
foreach (var res in results)
{
string title = "Matching " + m.PlatformAlias + " - " + m.GDBTitle;
StringBuilder sb = new StringBuilder();
sb.Append("Potential match found for " + m.GDBTitle + " \n(" + m.PlatformAlias + "): ");
sb.Append(res.resultString);
sb.Append("\n\n");
sb.Append("Word score: " + res.score);
sb.Append("\n\n\n");
sb.Append("Press YES to accept this match and save to the database\n");
sb.Append("Press NO to show the next match for this entry\n");
sb.Append("Press CANCEL to skip this entirely and move to the next search object");
MessageBoxResult mbr = MessageBox.Show(sb.ToString(), title, MessageBoxButton.YesNoCancel, MessageBoxImage.Question, MessageBoxResult.None, MessageBoxOptions.DefaultDesktopOnly);
if (mbr == MessageBoxResult.Yes)
{
// match has been confirmed. create a junction record
Junction j = new Junction();
j.gid = gid;
j.mid = res.resultId;
jList.Add(j);
matched++;
controller.SetMessage("Saving to Database...");
Junction.SaveToDatabase(jList);
jList = new List <Junction>();
break;
}
if (mbr == MessageBoxResult.No)
{
continue;
}
if (mbr == MessageBoxResult.Cancel)
{
break;
}
if (mbr == MessageBoxResult.None)
{
cancel = true;
break;
}
}
}
controller.SetProgress(progress);
controller.SetMessage("TOTAL: " + co + "\n\nMatched: " + matched + "\nUnmatched: " + notmatched);
}
if (AutoMatchOnSingle100Score)
{
controller.SetMessage("Saving to Database...");
Junction.SaveToDatabase(jList);
}
});
await controller.CloseAsync();
if (controller.IsCanceled)
{
await mw.ShowMessageAsync("ExactMatch Matching", "Operation Cancelled");
}
else
{
await mw.ShowMessageAsync("ExactMatch Matching", "Scanning and Import Completed");
}
}
public async void MobyExactMatch()
{
var mySettings = new MetroDialogSettings()
{
NegativeButtonText = "Cancel Scraping",
AnimateShow = false,
AnimateHide = false,
};
var controller = await mw.ShowProgressAsync("Attempting exact match - mobygames to thegamesdb", "", settings : mySettings);
controller.SetCancelable(true);
await Task.Delay(100);
await Task.Run(() =>
{
Task.Delay(1);
int progress = 0;
List <MasterView> unmatched = (from a in MasterView.GetMasterView()
where a.mid == null
select a).ToList();
controller.Minimum = 0;
controller.Maximum = unmatched.Count();
int co = unmatched.Count();
int matched = 0;
int notmatched = 0;
controller.SetProgress(progress);
controller.SetMessage("TOTAL: " + co + "\n\nMatched: " + matched + "\nUnmatched: " + notmatched);
List <int[]> list = new List <int[]>();
List <MOBY_Game> mgames = MOBY_Game.GetGames().ToList();
foreach (MasterView m in unmatched)
{
controller.SetProgress(progress);
controller.SetMessage("TOTAL: " + co + "\n\nMatched: " + matched + "\nUnmatched: " + notmatched);
progress++;
int gid = m.gid;
string gdbTitle = m.GDBTitle.Trim();
// lookup in mobygames
MOBY_Game mg = (from a in mgames
where a.pid == m.pid &&
a.gameTitle.Trim().ToLower() == gdbTitle.ToLower()
select a).FirstOrDefault();
if (mg == null)
{
notmatched++;
continue;
}
matched++;
int[] i = new int[2];
i[0] = m.gid;
i[1] = mg.mid;
list.Add(i);
controller.SetProgress(progress);
controller.SetMessage("TOTAL: " + co + "\n\nMatched: " + matched + "\nUnmatched: " + notmatched);
}
controller.SetMessage("Saving to Database...");
List <Junction> jList = new List <Junction>();
foreach (var a in list)
{
Junction j = new Junction();
j.gid = a[0];
j.mid = a[1];
jList.Add(j);
}
Junction.SaveToDatabase(jList);
});
await controller.CloseAsync();
if (controller.IsCanceled)
{
await mw.ShowMessageAsync("ExactMatch Matching", "Operation Cancelled");
}
else
{
await mw.ShowMessageAsync("ExactMatch Matching", "Scanning and Import Completed");
}
}