public static List <ScraperMaster> GetMasterList()
{
List <ScraperMaster> mList = new List <ScraperMaster>();
// get from masterview
List <MasterView> mv = ScrapeDB.AllScrapeData; //MasterView.GetMasterView();
// get gamedocs
List <Game_Doc> docs = Game_Doc.GetDocs();
// iterate through
foreach (var entry in mv)
{
ScraperMaster sm = new ScraperMaster();
sm.gid = entry.gid;
sm.pid = entry.pid;
sm.mid = entry.mid;
sm.GDBTitle = entry.GDBTitle;
sm.GDBPlatformName = entry.PlatformName;
sm.GDBPlatformAlias = entry.PlatformAlias;
sm.GDBYear = entry.GDBYear;
sm.MOBYTitle = entry.MOBYTitle;
sm.MOBYAlias = entry.MOBYAlias;
sm.MOBYPlatformName = entry.MOBYPlatformName;
sm.MOBYPlatformAlias = entry.MOBYPlatformAlias;
sm.MOBYYear = entry.MOBYYear;
List <string> ds = (from a in docs
where a.gid == sm.gid
select a.downloadUrl).ToList();
if (ds.Count > 0)
{
sm.Game_Docs.AddRange(ds);
}
mList.Add(sm);
}
return(mList);
}
public static void ScrapeBasicDocsList(ProgressDialogController controller)
{
List <ReplacementDocs> rdlist = new List <ReplacementDocs>();
// iterate through mednafen systems
var systems = GSystem.GetSystems();
foreach (var sys in systems)
{
controller.SetMessage("Getting manual links for: " + sys.systemName + "\n");
if (sys.systemId == 16 || sys.systemId == 17 || sys.systemId == 18)
{
continue;
}
List <int> rdsystems = ConvertSystemId2RDSystemId(sys.systemId);
// iterate through replacementdocs systems
foreach (int s in rdsystems)
{
// get the whole page for this system
WebOps wo = new WebOps();
wo.BaseUrl = "http://www.replacementdocs.com/download.php?";
wo.Params = "1.list." + s.ToString() + ".1000.download_name.ASC";
wo.Timeout = 20000;
string result = wo.ApiCall();
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(result);
HtmlNode table = doc.DocumentNode.SelectSingleNode("//table[contains(@class, 'fborder')]");
// iterate through each table row
foreach (HtmlNode row in table.ChildNodes)
{
if (row.ChildNodes.Count > 0)
{
HtmlNode[] cells = (from a in row.SelectNodes("td")
select a).ToArray();
if (cells[0].InnerHtml.Contains("download.php?view."))
{
// this is a data cell
string title = cells[0].InnerText.Replace("\t", "").Trim();
string url = cells[0].InnerHtml.Replace("\t", "").Trim().Replace("<a href='download.php?view.", "");
string[] urlArr = url.Split('\'');
string fileId = urlArr[0];
var recordcheck = (from a in rdlist
where a.GameName == title && a.TGBSystemName == ConvertRDSystemId2TGBPlatformName(s)
select a).ToList();
if (recordcheck.Count > 0)
{
ReplacementDocs r = recordcheck.FirstOrDefault();
r.Urls.Add("http://www.replacementdocs.com/request.php?" + fileId);
r.Urls.Distinct();
rdlist.Add(r);
}
else
{
ReplacementDocs r = new ReplacementDocs();
r.GameName = title;
r.TGBSystemName = ConvertRDSystemId2TGBPlatformName(s);
r.Urls.Add("http://www.replacementdocs.com/request.php?" + fileId);
r.Urls.Distinct();
rdlist.Add(r);
}
rdlist.Distinct();
}
}
}
}
}
// Add to scrapeDB
foreach (var m in rdlist)
{
foreach (string entry in m.Urls)
{
Game_Doc gd = new Game_Doc();
int pid = GDB_Platform.GetPlatforms().Where(a => a.name == m.TGBSystemName).FirstOrDefault().pid;
gd.pid = pid;
gd.gameName = m.GameName;
gd.downloadUrl = entry;
Game_Doc.AddDoc(gd);
}
}
/*
* // save rdlist to json
* string json = JsonConvert.SerializeObject(rdlist, Formatting.Indented);
* File.WriteAllText(AppDomain.CurrentDomain.BaseDirectory + @"..\..\Data\System\replacementdocs-manuals.json", json);
*/
}
