public static List <ScraperMaster> GetMasterList() { List <ScraperMaster> mList = new List <ScraperMaster>(); // get from masterview List <MasterView> mv = ScrapeDB.AllScrapeData; //MasterView.GetMasterView(); // get gamedocs List <Game_Doc> docs = Game_Doc.GetDocs(); // iterate through foreach (var entry in mv) { ScraperMaster sm = new ScraperMaster(); sm.gid = entry.gid; sm.pid = entry.pid; sm.mid = entry.mid; sm.GDBTitle = entry.GDBTitle; sm.GDBPlatformName = entry.PlatformName; sm.GDBPlatformAlias = entry.PlatformAlias; sm.GDBYear = entry.GDBYear; sm.MOBYTitle = entry.MOBYTitle; sm.MOBYAlias = entry.MOBYAlias; sm.MOBYPlatformName = entry.MOBYPlatformName; sm.MOBYPlatformAlias = entry.MOBYPlatformAlias; sm.MOBYYear = entry.MOBYYear; List <string> ds = (from a in docs where a.gid == sm.gid select a.downloadUrl).ToList(); if (ds.Count > 0) { sm.Game_Docs.AddRange(ds); } mList.Add(sm); } return(mList); }
public static void ScrapeBasicDocsList(ProgressDialogController controller) { List <ReplacementDocs> rdlist = new List <ReplacementDocs>(); // iterate through mednafen systems var systems = GSystem.GetSystems(); foreach (var sys in systems) { controller.SetMessage("Getting manual links for: " + sys.systemName + "\n"); if (sys.systemId == 16 || sys.systemId == 17 || sys.systemId == 18) { continue; } List <int> rdsystems = ConvertSystemId2RDSystemId(sys.systemId); // iterate through replacementdocs systems foreach (int s in rdsystems) { // get the whole page for this system WebOps wo = new WebOps(); wo.BaseUrl = "http://www.replacementdocs.com/download.php?"; wo.Params = "1.list." + s.ToString() + ".1000.download_name.ASC"; wo.Timeout = 20000; string result = wo.ApiCall(); HtmlDocument doc = new HtmlDocument(); doc.LoadHtml(result); HtmlNode table = doc.DocumentNode.SelectSingleNode("//table[contains(@class, 'fborder')]"); // iterate through each table row foreach (HtmlNode row in table.ChildNodes) { if (row.ChildNodes.Count > 0) { HtmlNode[] cells = (from a in row.SelectNodes("td") select a).ToArray(); if (cells[0].InnerHtml.Contains("download.php?view.")) { // this is a data cell string title = cells[0].InnerText.Replace("\t", "").Trim(); string url = cells[0].InnerHtml.Replace("\t", "").Trim().Replace("<a href='download.php?view.", ""); string[] urlArr = url.Split('\''); string fileId = urlArr[0]; var recordcheck = (from a in rdlist where a.GameName == title && a.TGBSystemName == ConvertRDSystemId2TGBPlatformName(s) select a).ToList(); if (recordcheck.Count > 0) { ReplacementDocs r = recordcheck.FirstOrDefault(); r.Urls.Add("http://www.replacementdocs.com/request.php?" + fileId); r.Urls.Distinct(); rdlist.Add(r); } else { ReplacementDocs r = new ReplacementDocs(); r.GameName = title; r.TGBSystemName = ConvertRDSystemId2TGBPlatformName(s); r.Urls.Add("http://www.replacementdocs.com/request.php?" + fileId); r.Urls.Distinct(); rdlist.Add(r); } rdlist.Distinct(); } } } } } // Add to scrapeDB foreach (var m in rdlist) { foreach (string entry in m.Urls) { Game_Doc gd = new Game_Doc(); int pid = GDB_Platform.GetPlatforms().Where(a => a.name == m.TGBSystemName).FirstOrDefault().pid; gd.pid = pid; gd.gameName = m.GameName; gd.downloadUrl = entry; Game_Doc.AddDoc(gd); } } /* * // save rdlist to json * string json = JsonConvert.SerializeObject(rdlist, Formatting.Indented); * File.WriteAllText(AppDomain.CurrentDomain.BaseDirectory + @"..\..\Data\System\replacementdocs-manuals.json", json); */ }