Ejemplo n.º 1
0
        public static PSX_Games ScrapeIndividualInfo(PSX_Games record)
        {
            // load page to string
            string html = new WebClient().DownloadString(record.infoUrl);

            HtmlDocument doc = new HtmlDocument();

            doc.OptionFixNestedTags = true;
            doc.LoadHtml(html);

            // get all tables
            var findTables = from table in doc.DocumentNode.SelectNodes("//table").Cast <HtmlNode>()
                             from row in table.SelectNodes("tr").Cast <HtmlNode>()
                             select row;
            bool foundDev  = false;
            bool foundPub  = false;
            bool foundYear = false;

            foreach (HtmlNode row in findTables)
            {
                if (foundDev && foundPub && foundYear)
                {
                    break;
                }

                var cells = row.SelectNodes("th|td").ToArray();
                if (cells[0].InnerText.Contains("Developer"))
                {
                    record.developer = cells[1].InnerText.Replace("&nbsp;", "").Split('\t').Last().Trim().TrimEnd('.');
                    foundDev         = true;
                }
                if (cells[0].InnerText.Contains("Publisher"))
                {
                    record.publisher = cells[1].InnerText.Replace("&nbsp;", "").Split('\t').Last().Trim().TrimEnd('.');
                    foundPub         = true;
                }
                if (cells[0].InnerText.Contains("Date Released"))
                {
                    int      da   = 0;
                    string[] dArr = cells[1].InnerText.Split(' ');
                    // test whether numeric value
                    bool result = int.TryParse(dArr.Last().Split('\t').Last().Trim(), out da);
                    if (result == true)
                    {
                        record.year = dArr.Last().Split('\t').Last().Trim();
                    }

                    foundYear = true;
                }
            }

            return(record);
        }
Ejemplo n.º 2
0
        /// <summary>
        /// process psxdatacenter DB for every system
        /// </summary>
        /// <returns></returns>
        public static List <DAT_Rom> Go()
        {
            // create empty list
            List <DAT_Rom> l = new List <DAT_Rom>();

            // get all psx games from debug database
            List <PSX_Games> games = PSX_Games.GetGames();

            // iterate through each game and parse the information into a new object
            foreach (var g in games)
            {
                DAT_Rom dr = new DAT_Rom();
                dr.country       = g.region;
                dr.datProviderId = 5;
                dr.developer     = g.developer;
                dr.language      = g.languages;
                dr.pid           = 10;
                dr.publisher     = g.publisher;
                dr.year          = g.year;
                dr.otherFlags    = g.serial;

                // names
                dr.romName = g.name;
                dr.name    = g.name
                             .Replace("[Disc 1]", "")
                             .Replace("[Disc 2]", "")
                             .Replace("[Disc 3]", "")
                             .Replace("[Disc 4]", "")
                             .Replace("[Disc 5]", "")
                             .Replace("[Disc 6]", "")
                             .Replace("[Disc 7]", "")
                             .Replace("[Disc 8]", "")
                             .Replace("[Disc 9]", "")
                             .Replace("[Disc 10]", "")
                             .Trim();

                l.Add(dr);
            }

            l.Distinct();
            return(l);
        }
Ejemplo n.º 3
0
        public static int[] SaveToDatabase(List <PSX_Games> games)
        {
            // get current rom list
            List <PSX_Games> current = PSX_Games.GetGames();

            int added   = 0;
            int updated = 0;

            // create temp objects pre-database actions
            List <PSX_Games> toAdd    = new List <PSX_Games>();
            List <PSX_Games> toUpdate = new List <PSX_Games>();

            // iterate through each incoming rom
            foreach (var g in games)
            {
                // attempt rom lookup in current
                var lo = (from a in current
                          where
                          a.serial == g.serial
                          select a).ToList();

                PSX_Games l = new PSX_Games();

                if (lo.Count == 1)
                {
                    l = lo.Single();
                }

                else
                {
                    // more than one serial number result - some multi-disc games share the same serial number
                    // match on name
                    l = lo.Where(a => a.name == g.name).FirstOrDefault();

                    if (l == null)
                    {
                        // give up
                        continue;
                    }
                }

                if (l == null)
                {
                    // no entry found
                    toAdd.Add(g);
                }
                else
                {
                    PSX_Games game = new PSX_Games();
                    game           = l;
                    game.infoUrl   = g.infoUrl;
                    game.name      = g.name;
                    game.languages = g.languages;
                    game.region    = g.region;

                    if (game.publisher == null)
                    {
                        game.publisher = g.publisher;
                    }
                    if (game.developer == null)
                    {
                        game.developer = g.developer;
                    }
                    if (game.year == null)
                    {
                        game.year = g.year;
                    }

                    // entry found

                    if (game != l)
                    {
                        toUpdate.Add(game);
                    }
                }
            }

            using (var db = new PsxDataCenterAdminDbContext())
            {
                // add new entries
                db.PSX_Games.AddRange(toAdd);

                // update existing entries
                db.PSX_Games.UpdateRange(toUpdate);


                db.SaveChanges();

                added   = toAdd.Count();
                updated = toUpdate.Count();

                return(new int[] { added, updated });
            }
        }
Ejemplo n.º 4
0
        public static List <PSX_Games> ScrapeInitialList(bool downloadFiles)
        {
            List <PSX_Games> list = new List <PSX_Games>();
            HtmlDocument     doc  = new HtmlDocument();

            doc.OptionFixNestedTags = true;

            List <string> urls = new List <string>
            {
                "http://psxdatacenter.com/ulist.html",
                "http://psxdatacenter.com/plist.html",
                "http://psxdatacenter.com/jlist.html",
            };

            List <string> locals = new List <string>
            {
                AppDomain.CurrentDomain.BaseDirectory + @"..\..\_Debug\DATDB\DATFiles\PSXDATACENTER\ntscu.html",
                AppDomain.CurrentDomain.BaseDirectory + @"..\..\_Debug\DATDB\DATFiles\PSXDATACENTER\pal.html",
                AppDomain.CurrentDomain.BaseDirectory + @"..\..\_Debug\DATDB\DATFiles\PSXDATACENTER\ntscj.html",
            };

            if (downloadFiles == true)
            {
                // download the webpages
                for (int i = 0; i < urls.Count(); i++)
                {
                    // download the latest version of the webpage
                    string webResult = "";
                    using (var client = new WebClient())
                    {
                        webResult = client.DownloadString(urls[i]);
                    }
                    // save to disk
                    File.WriteAllText(locals[i], webResult);
                }
            }


            for (int i = 0; i < urls.Count(); i++)
            {
                // load html file to string
                string s = File.ReadAllText(locals[i]);
                doc.LoadHtml(s);

                // get all tables

                var tables = doc.DocumentNode
                             .Descendants("table")
                             .Where(d => d.Attributes.Contains("class") &&
                                    d.Attributes["class"].Value.Contains("sectiontable")).Cast <HtmlNode>();


                var findTables = from table in doc.DocumentNode.SelectNodes("//table")
                                 .Where(a => a.Attributes.Contains("class") && a.Attributes["class"].Value.Contains("sectiontable")).Cast <HtmlNode>()
                                 from row in table.SelectNodes("tr").Cast <HtmlNode>()
                                 select row;

                // iterate through rows
                foreach (HtmlNode row in findTables)
                {
                    if (row.InnerText == "\r\n" || row.InnerText.Trim() == "")
                    {
                        continue;
                    }

                    var cells = row.SelectNodes("th|td").ToArray();

                    // Get all the serial numbers first and iterate through them (if no serial number present - ignore)
                    string[] snss = cells[1].InnerHtml.Split(new string[] { "<br>" }, StringSplitOptions.None);
                    string[] sns  = snss.Where(a => a.Trim() != "").ToArray();

                    for (int sn = 0; sn < sns.Length; sn++)
                    {
                        PSX_Games record = new PSX_Games();
                        record.serial = sns[sn].Trim();

                        // get game name
                        string gText = cells[2].InnerText.Replace("&nbsp;", "").Replace("\r\n", "").Trim().Replace("  ", " ");
                        if (sns.Length > 1)
                        {
                            gText += " [Disc " + (sn + 1) + "]";
                        }
                        record.name = gText;

                        // Languages
                        string langs = cells[3].InnerText.Replace("\r\n", "").Replace("&nbsp;", "").Trim();
                        record.languages = langs;

                        // info url
                        if (cells[0].InnerHtml != "")
                        {
                            /*
                             * int pFrom = cells[0].InnerHtml.IndexOf("<a href=") + "<a href=".Length;
                             * int pTo = cells[0].InnerHtml.LastIndexOf(" target=");
                             * string result = cells[0].InnerHtml.Substring(pFrom, pTo - pFrom).Replace("\"", "");
                             */

                            string[] arr1   = cells[0].InnerHtml.Split(new string[] { "href=\"games/" }, StringSplitOptions.None);
                            string   result = "http://psxdatacenter.com/games/" + arr1[1].Replace("\">INFO</a>", "").Trim();


                            record.infoUrl = result;
                        }

                        // region
                        if (record.serial.Contains("SLPS") ||
                            record.serial.Contains("SCPS") ||
                            record.serial.Contains("SLPM") ||
                            record.serial.Contains("SIPS"))
                        {
                            record.region = "JAP";
                        }

                        if (record.serial.Contains("SLES") ||
                            record.serial.Contains("SCES"))
                        {
                            record.region = "EUR";
                        }

                        if (record.serial.Contains("SLUS") ||
                            record.serial.Contains("LSP") ||
                            record.serial.Contains("SCUS"))
                        {
                            record.region = "USA";
                        }


                        // add to master list
                        list.Add(record);
                    }
                }
            }

            // update in database
            int[] r = PSX_Games.SaveToDatabase(list);

            MessageBox.Show("Games Added: " + r[0] +
                            "\nGames Updated: " + r[1]);


            return(list);
        }
Ejemplo n.º 5
0
        public static async void GetExtraDetail()
        {
            PsxDc c = new PsxDc();

            var mySettings = new MetroDialogSettings()
            {
                NegativeButtonText = "Cancel",
                AnimateShow        = false,
                AnimateHide        = false
            };
            string output     = "Getting games that need additional info scraped...\n";
            var    controller = await c.mw.ShowProgressAsync("PSXDATACENTER Builder", output, true, settings : mySettings);

            controller.SetCancelable(true);
            controller.SetIndeterminate();
            await Task.Delay(1);

            await Task.Run(() =>
            {
                // load games
                List <PSX_Games> games = PSX_Games.GetGames()
                                         .Where(
                    a => a.infoUrl != null && a.infoUrl.Trim() != "" &&
                    (
                        a.publisher == null ||
                        a.developer == null ||
                        a.year == null
                    )
                    )
                                         .OrderBy(x => x.id)
                                         .ToList();

                int totalGamesToProcess = games.Count();

                //List<PSX_Games> gWorking = new List<PSX_Games>();

                output = "Performing lookups...\n";

                // iterate through each game and get extra details
                for (int i = 0; i < totalGamesToProcess; i++)
                {
                    controller.SetMessage(output + "\nGame " + i + " of " + totalGamesToProcess);
                    var re = ScrapeIndividualInfo(games[i]);
                    if (re == null)
                    {
                        continue;
                    }

                    // some arbitrary wait time to not spam the server
                    //System.Threading.Thread.Sleep(1000);

                    // add/update the database
                    PSX_Games.SaveToDatabase(re);
                }
            });

            await controller.CloseAsync();

            if (controller.IsCanceled)
            {
                await c.mw.ShowMessageAsync("DAT Builder", "Linking Cancelled");
            }
            else
            {
                await c.mw.ShowMessageAsync("DAT Builder", "Linking Completed");
            }
        }