/// <summary> /// add a single game entry /// </summary> /// <param name="game"></param> public static void AddGame(MOBY_Game game) { using (var aG = new AsniScrapeAdminDbContext()) { aG.MOBY_Game.Add(game); aG.SaveChanges(); aG.Dispose(); } }
/// <summary> /// update a single game entry /// </summary> /// <param name="game"></param> public static void UpdateGame(MOBY_Game game) { using (var uG = new AsniScrapeAdminDbContext()) { uG.MOBY_Game.Update(game); uG.SaveChanges(); uG.Dispose(); } }
/// <summary> /// save list of games to database (add or update logic included) /// </summary> /// <param name="games"></param> public static void SaveToDatabase(List <MOBY_Game> games) { using (var db = new AsniScrapeAdminDbContext()) { // get current database context var current = db.MOBY_Game.AsNoTracking().ToList(); List <MOBY_Game> toAdd = new List <MOBY_Game>(); List <MOBY_Game> toUpdate = new List <MOBY_Game>(); // iterate through the games list and separete out games to be added and games to be updated foreach (var g in games) { // games will generally be passed with mid = 0 if (g.mid > 0) { MOBY_Game t = (from a in current where (a.mid == g.mid) select a).SingleOrDefault(); if (t == null) { toAdd.Add(g); } else { toUpdate.Add(g); } } else { MOBY_Game t = (from a in current where (a.alias == g.alias && a.pid == g.pid) select a).SingleOrDefault(); if (t == null) { toAdd.Add(g); } else { MOBY_Game m = t; m.gameTitle = g.gameTitle; m.alias = g.alias; m.pid = g.pid; m.releaseYear = g.releaseYear; toUpdate.Add(m); } } } db.MOBY_Game.UpdateRange(toUpdate); db.MOBY_Game.AddRange(toAdd); db.SaveChanges(); } }
/// <summary> /// scrape all moby platform games /// </summary> public async void ScrapeMobyPlatformGames() { var mySettings = new MetroDialogSettings() { NegativeButtonText = "Cancel Scraping", AnimateShow = false, AnimateHide = false, }; var controller = await mw.ShowProgressAsync("MedLaunch - Getting Basic Games List From mobygames.net", "", settings : mySettings); controller.SetCancelable(true); await Task.Delay(100); await Task.Run(() => { Task.Delay(1); int count = 1; int sysCount = mobyplatforms.Count(); controller.Minimum = 0; controller.Maximum = sysCount; foreach (var platform in mobyplatforms) { List <MOBY_Game> gs = new List <MOBY_Game>(); controller.SetProgress(Convert.ToDouble(count)); controller.SetMessage("Retrieving Game List for Platform: " + platform.name); // get initial page string url = platform.listURL; string initialPage = ReturnWebpage(url, "", 10000); bool isAttrib = false; if (url.Contains("attribute/sheet")) { isAttrib = true; } int totalGames = 0; if (!isAttrib) { /* Get the total number of games available for this system */ // split the html to list via line breaks List <string> html = initialPage.Split('\n').ToList(); // get only the line that contains the number of games string hLine = html.Where(a => a.Contains(" games)")).FirstOrDefault(); // get only the substring "xxx games" string resultString = Regex.Match(hLine, @"(?<=\().+?(?=\))").Value; // split by whitespace string[] gArr = resultString.Split(' '); // get int number of games totalGames = Convert.ToInt32(gArr[0]); } else { List <string> html = initialPage.Split('\n').ToList(); string hLine = html.Where(a => a.Contains("(items")).FirstOrDefault(); string resultString = Regex.Match(hLine, @"(?<=\().+?(?=\))").Value; string[] gArr = resultString.Split(' '); totalGames = Convert.ToInt32(gArr.Last()); } HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(initialPage); // build a list of page URLs double numberofpages = Convert.ToDouble(totalGames) / 25; int numberOfPages = Convert.ToInt32(Math.Ceiling(numberofpages)); // connect to every page and import all the game information for (int i = 0; i < numberOfPages; i++) { int offset = i * 25; string newUrl = url.Replace("offset,0", "offset," + offset); HtmlDocument hDoc = new HtmlDocument(); if (i == 0) { hDoc = doc; } else { string htmlRes = ReturnWebpage(newUrl, "", 10000); hDoc.LoadHtml(htmlRes); } // get just the data table we are interested in HtmlNode objectTable = hDoc.GetElementbyId("mof_object_list"); // iterate through each row and scrape the game information int cGame = 1; foreach (HtmlNode row in objectTable.SelectNodes("tbody/tr")) { int currentGameNumber = offset + cGame; if (controller != null) { controller.SetMessage("Scraping basic list of all " + platform.name + " games\nGame: (" + currentGameNumber + " of " + totalGames + ")\nPage: (" + (i + 1) + " of " + numberOfPages + ")"); controller.Minimum = 1; controller.Maximum = totalGames; controller.SetProgress(Convert.ToDouble(currentGameNumber)); } HtmlNode[] cells = (from a in row.SelectNodes("td") select a).ToArray(); string Title = cells[0].InnerText.Trim(); //var allLi = row.SelectSingleNode("//a[@href]"); string URLstring = cells[0].InnerHtml.Trim(); Regex regex = new Regex("href\\s*=\\s*(?:\"(?<1>[^\"]*)\"|(?<1>\\S+))", RegexOptions.IgnoreCase); Match match; string URL = ""; for (match = regex.Match(URLstring); match.Success; match = match.NextMatch()) { URL = match.Groups[1].ToString(); } MOBY_Game game = new MOBY_Game(); game.pid = platform.pid; game.gameTitle = WebUtility.HtmlDecode(Title); game.alias = WebUtility.HtmlDecode(URL.Split('/').LastOrDefault()); game.releaseYear = cells[1].InnerText.Trim(); // add game to main list gs.Add(game); } } MOBY_Game.SaveToDatabase(gs); } }); await controller.CloseAsync(); if (controller.IsCanceled) { await mw.ShowMessageAsync("MOBY Master Games List Download", "Operation Cancelled"); } else { await mw.ShowMessageAsync("MOBY Master Games List Download", "Scanning and Import Completed"); } }
public async void MobyManualMatch(bool AutoMatchOnSingle100Score) { var mySettings = new MetroDialogSettings() { NegativeButtonText = "Cancel Scraping", AnimateShow = false, AnimateHide = false, }; var controller = await mw.ShowProgressAsync("Attempting manual match based on word count - mobygames to thegamesdb", "", settings : mySettings); controller.SetCancelable(true); await Task.Delay(100); await Task.Run(() => { Task.Delay(1); int progress = 0; List <Junction> jList = new List <Junction>(); List <MasterView> unmatched = (from a in MasterView.GetMasterView() where a.mid == null select a).ToList(); controller.Minimum = 0; controller.Maximum = unmatched.Count(); int co = unmatched.Count(); int matched = 0; int notmatched = 0; controller.SetProgress(progress); controller.SetMessage("TOTAL: " + co + "\n\nMatched: " + matched + "\nUnmatched: " + notmatched); List <int[]> list = new List <int[]>(); List <MOBY_Game> mgames = MOBY_Game.GetGames().ToList(); bool cancel = false; foreach (MasterView m in unmatched) { if (cancel == true) { break; } controller.SetProgress(progress); controller.SetMessage("TOTAL: " + co + "\n\nMatched: " + matched + "\nUnmatched: " + notmatched); progress++; int gid = m.gid; int pid = m.pid; // build SearchObject SearchObject so = new SearchObject(); so.searchId = gid; so.searchString = m.GDBTitle.Trim(); so.listToSearch = (from a in mgames where a.pid == pid select new SearchList { id = a.mid, name = a.gameTitle }).ToList(); // get search results so = SearchFunctions.WordCountMatch(so); // get results list separately List <SearchResult> results = so.searchResults.OrderByDescending(a => a.score).ToList(); // if automatch on unique 100 score is selected if (AutoMatchOnSingle100Score) { var hundreds = results.Where(a => a.score == 100).ToList(); if (hundreds.Count == 1) { Junction j = new Junction(); j.gid = gid; j.mid = hundreds.First().resultId; jList.Add(j); matched++; //controller.SetMessage("Saving to Database..."); //Junction.SaveToDatabase(jList); //jList = new List<Junction>(); continue; } else { notmatched++; continue; } } else { // iterate through each result and prompt user to accept / as for next result / or skip completely foreach (var res in results) { string title = "Matching " + m.PlatformAlias + " - " + m.GDBTitle; StringBuilder sb = new StringBuilder(); sb.Append("Potential match found for " + m.GDBTitle + " \n(" + m.PlatformAlias + "): "); sb.Append(res.resultString); sb.Append("\n\n"); sb.Append("Word score: " + res.score); sb.Append("\n\n\n"); sb.Append("Press YES to accept this match and save to the database\n"); sb.Append("Press NO to show the next match for this entry\n"); sb.Append("Press CANCEL to skip this entirely and move to the next search object"); MessageBoxResult mbr = MessageBox.Show(sb.ToString(), title, MessageBoxButton.YesNoCancel, MessageBoxImage.Question, MessageBoxResult.None, MessageBoxOptions.DefaultDesktopOnly); if (mbr == MessageBoxResult.Yes) { // match has been confirmed. create a junction record Junction j = new Junction(); j.gid = gid; j.mid = res.resultId; jList.Add(j); matched++; controller.SetMessage("Saving to Database..."); Junction.SaveToDatabase(jList); jList = new List <Junction>(); break; } if (mbr == MessageBoxResult.No) { continue; } if (mbr == MessageBoxResult.Cancel) { break; } if (mbr == MessageBoxResult.None) { cancel = true; break; } } } controller.SetProgress(progress); controller.SetMessage("TOTAL: " + co + "\n\nMatched: " + matched + "\nUnmatched: " + notmatched); } if (AutoMatchOnSingle100Score) { controller.SetMessage("Saving to Database..."); Junction.SaveToDatabase(jList); } }); await controller.CloseAsync(); if (controller.IsCanceled) { await mw.ShowMessageAsync("ExactMatch Matching", "Operation Cancelled"); } else { await mw.ShowMessageAsync("ExactMatch Matching", "Scanning and Import Completed"); } }
public async void MobyExactMatch() { var mySettings = new MetroDialogSettings() { NegativeButtonText = "Cancel Scraping", AnimateShow = false, AnimateHide = false, }; var controller = await mw.ShowProgressAsync("Attempting exact match - mobygames to thegamesdb", "", settings : mySettings); controller.SetCancelable(true); await Task.Delay(100); await Task.Run(() => { Task.Delay(1); int progress = 0; List <MasterView> unmatched = (from a in MasterView.GetMasterView() where a.mid == null select a).ToList(); controller.Minimum = 0; controller.Maximum = unmatched.Count(); int co = unmatched.Count(); int matched = 0; int notmatched = 0; controller.SetProgress(progress); controller.SetMessage("TOTAL: " + co + "\n\nMatched: " + matched + "\nUnmatched: " + notmatched); List <int[]> list = new List <int[]>(); List <MOBY_Game> mgames = MOBY_Game.GetGames().ToList(); foreach (MasterView m in unmatched) { controller.SetProgress(progress); controller.SetMessage("TOTAL: " + co + "\n\nMatched: " + matched + "\nUnmatched: " + notmatched); progress++; int gid = m.gid; string gdbTitle = m.GDBTitle.Trim(); // lookup in mobygames MOBY_Game mg = (from a in mgames where a.pid == m.pid && a.gameTitle.Trim().ToLower() == gdbTitle.ToLower() select a).FirstOrDefault(); if (mg == null) { notmatched++; continue; } matched++; int[] i = new int[2]; i[0] = m.gid; i[1] = mg.mid; list.Add(i); controller.SetProgress(progress); controller.SetMessage("TOTAL: " + co + "\n\nMatched: " + matched + "\nUnmatched: " + notmatched); } controller.SetMessage("Saving to Database..."); List <Junction> jList = new List <Junction>(); foreach (var a in list) { Junction j = new Junction(); j.gid = a[0]; j.mid = a[1]; jList.Add(j); } Junction.SaveToDatabase(jList); }); await controller.CloseAsync(); if (controller.IsCanceled) { await mw.ShowMessageAsync("ExactMatch Matching", "Operation Cancelled"); } else { await mw.ShowMessageAsync("ExactMatch Matching", "Scanning and Import Completed"); } }