public static void ParseSearchNibl(string[] searchQuery) { HtmlParser parser = new HtmlParser(); var niblSearchResults = new List<NiblSearchInformation>(); foreach (string searchParam in searchQuery) { parser.ParseUrl("http://nibl.co.uk/bots.php?search=" + searchParam); string[,,] test = parser.CutHtml("botlistitem"); int bound0 = test.GetUpperBound(0); int bound1 = test.GetUpperBound(1); for (int i = 0; i <= bound0; i++) { string bot = ""; string pack = ""; string filename = ""; string res = ""; for (int a = 0; a <= bound1; a++) { string tag = test[i, a, 0]; string textInsidetag = test[i, a, 1]; try { if (tag.Contains("botname") || textInsidetag.Contains("botname")) { bot = textInsidetag.Split('<')[0].Trim(); } if (tag.Contains("packnumber")) { pack = textInsidetag.Trim(); } if (tag.Contains("filename")) { filename = textInsidetag.Split('<')[0].Trim(); if (filename.Contains("480") || filename.Contains("360") || filename.Contains("576") || filename.Contains("SD")) { res = "480"; } else if (filename.Contains("720") || filename.Contains("768")) { res = "720"; } else if (filename.Contains("1080") || filename.Contains("1000")) { res = "1080"; } else { res = "unknown"; } } if (filename != "" && bot != "" && pack != "") { niblSearchResults.Add(new NiblSearchInformation() { botname = bot, filename = filename, packnumber = pack, resolution = res }); break; } } catch (Exception e) { } } } } var sd = new List<NiblSearchInformation>(); var hd = new List<NiblSearchInformation>(); var fhd = new List<NiblSearchInformation>(); var unknown = new List<NiblSearchInformation>(); var bots = new List<string>(); foreach (NiblSearchInformation info in niblSearchResults) { if (info.resolution.Contains("480")) { sd.Add(info); } else if (info.resolution.Contains("720")) { hd.Add(info); } else if (info.resolution.Contains("1080")) { fhd.Add(info); } else { unknown.Add(info); } } Dictionary<string, Dictionary<string, List<NiblSearchInformation>>> values = new Dictionary<string, Dictionary<string, List<NiblSearchInformation>>>(); Dictionary<string, List<NiblSearchInformation>> everyResolution = new Dictionary<string, List<NiblSearchInformation>>(); string currentBot = " "; string previousBot = ""; foreach (NiblSearchInformation info in niblSearchResults) { currentBot = info.botname; if (currentBot != previousBot) { if (!values.ContainsKey(currentBot)) { bots.Add(currentBot); } previousBot = currentBot; } } foreach (string currentBotToSearch in bots) { var search480p = sd.FindAll(x => x.botname.Contains(currentBotToSearch)); var search720p = hd.FindAll(x => x.botname.Contains(currentBotToSearch)); var search1080p = fhd.FindAll(x => x.botname.Contains(currentBotToSearch)); var searchUnkown = unknown.FindAll(x => x.botname.Contains(currentBotToSearch)); everyResolution.Add("480", search480p); everyResolution.Add("720", search720p); everyResolution.Add("1080", search1080p); everyResolution.Add("unknown", searchUnkown); values.Add(currentBotToSearch, everyResolution); everyResolution = new Dictionary<string, List<NiblSearchInformation>>(); } string actualJson = JsonConvert.SerializeObject(values, Formatting.Indented); niblSearchJson = "[{\"NIBL\" : " + actualJson + "}]"; }
/// <ParseAnimeSeason> /// <summary> /// Parses all the anime from the html source of Myanimelists.net 's seasons page /// </summary /// </ParseAnimeSeason> private static void ParseAnimeSeason(string url, string yearAndseason) { if (!Directory.Exists("DataStorage/Anime")) { Directory.CreateDirectory("DataStorage/Anime/"); } string path = "DataStorage/Anime/anime" + yearAndseason + ".json"; string localJson = ""; if (!File.Exists(path)) { var animeSeasonsJson = new List<AnimeInfo>(); string prevAnime = ""; HtmlParser parseCurrentSeason = new HtmlParser(); parseCurrentSeason.ParseUrl(url); string[,,] test = parseCurrentSeason.CutHtml("seasonal-anime js-seasonal-anime"); int bound0 = test.GetUpperBound(0); int bound1 = test.GetUpperBound(1); for (int i = 0; i <= bound0; i++) { string animeTitle = ""; string animeCover = ""; string animeSynopsis = ""; string animeGenres = ""; string animeScore = ""; string animeId = ""; for (int a = 0; a <= bound1; a++) { string tag = test[i, a, 0]; string textInsidetag = test[i, a, 1]; try { if (tag.Contains("link-image") || textInsidetag.Contains("link-image")) { animeTitle = textInsidetag.Trim(); } if (tag.Contains("lazyload") || textInsidetag.Contains("lazyload")) { animeCover = tag.Split('"')[3].Split('"')[0]; string[] partsOfUrl = animeCover.Split('/'); animeId = partsOfUrl[partsOfUrl.Length - 1].Split('.')[0]; } if (tag.Contains("synopsis js-synopsis") || textInsidetag.Contains("synopsis js-synopsis")) { animeSynopsis = textInsidetag.Replace("\"", "\\\"").Replace("'", "\'").Split(new string[] { "<p" }, StringSplitOptions.None)[0]; } if (tag.Contains("genre") || textInsidetag.Contains("genre")) { animeGenres = textInsidetag.Trim(); } if (tag.Contains("score") || textInsidetag.Contains("score")) { animeScore = textInsidetag.Trim(); } if (animeScore != "" && animeGenres != "" && animeSynopsis != "" && animeId != "" && animeCover != "" && animeTitle != "") { if (prevAnime != animeTitle) { animeSeasonsJson.Add(new AnimeInfo() { id = animeId, title = animeTitle, cover = animeCover, synopsis = animeSynopsis, genres = animeGenres, score = animeScore }); prevAnime = animeTitle; } break; } } catch (Exception e) { } } } using (StreamWriter sw = File.CreateText(path)) { string actualJson = JsonConvert.SerializeObject(animeSeasonsJson, Formatting.Indented); DebugCallbackMethod("WRITING ANIME JSON"); localJson = "[{\"Anime\" : " + actualJson + "}]"; sw.Write(localJson); } DebugCallbackMethod(" |LAUNCH| " + Path.GetFileName(path) + " created!"); } else { DebugCallbackMethod(" |LAUNCH| " + "No need to parse html, anime already parsed, reading file instead!"); localJson = File.ReadAllText(path); } animePerSeasonJson = localJson; DebugCallbackMethod("DONE"); }
/// <ParseSearchAnime> /// <summary> /// Parses html received from the search page of Myanimelist.net /// </summary /// </ParseSearchAnime> private static void ParseSearchAnime(string url) { searchedAnimeJson = ""; string prevAnime = ""; HtmlParser parser = new HtmlParser(); parser.ParseUrl(url); var searchedAnimeJsonList = new List<AnimeInfo>(); string[,,] test = parser.CutHtml("picSurround"); int bound0 = test.GetUpperBound(0); int bound1 = test.GetUpperBound(1); for (int i = 0; i <= bound0; i++) { string animeTitle = ""; string animeCover = ""; string animeSynopsis = ""; string animeGenres = "N/A"; string animeScore = ""; string animeId = ""; for (int a = 0; a <= bound1; a++) { string tag = test[i, a, 0]; string textInsidetag = test[i, a, 1]; try { if (tag.Contains("lazyload")) { animeTitle = tag.Split('"')[11].Split('"')[0].Trim(); animeCover = tag.Split('"')[3].Split('"')[0].Replace("r/50x70/", "").Trim(); string[] partsOfUrl = animeCover.Split('/'); animeId = partsOfUrl[partsOfUrl.Length - 1].Split('.')[0]; } if (tag.Contains("pt4")) { animeSynopsis = textInsidetag.Trim(); } if (tag.Contains("borderClass ac bgColor1\" width=\"50\"") || tag.Contains("borderClass ac bgColor0\" width=\"50\"")) { animeScore = textInsidetag.Trim(); } if (animeTitle != "" && animeSynopsis != "" && animeCover != "" && animeId != "" && animeScore != "") { searchedAnimeJsonList.Add(new AnimeInfo() { id = animeId, title = animeTitle, cover = animeCover, synopsis = animeSynopsis, genres = animeGenres, score = animeScore }); break; } } catch (Exception e) { } } } string actualJson = JsonConvert.SerializeObject(searchedAnimeJsonList, Formatting.Indented); DebugCallbackMethod("WRITING ANIME JSON"); searchedAnimeJson = "[{\"Anime\" : " + actualJson + "}]"; DebugCallbackMethod("DONE"); }
/// <ParseAllSeasons> /// <summary> /// Gets all the available seasons from Myanimelist.net, and writes them to a json file, or if the json file already exist, reads the json file(faster) instead of html parsing /// </summary /// </ParseAllSeasons> private static void ParseAllSeasons() { if (!Directory.Exists("DataStorage")) { Directory.CreateDirectory("DataStorage"); } string path = "DataStorage/allSeasonsJson.json"; if (!File.Exists(path)) { string prevSeason = ""; var seasons = new List<AnimeSeasons>(); HtmlParser parseSeasons = new HtmlParser(); parseSeasons.ParseUrl("http://myanimelist.net/anime/season/archive"); string[,,] test = parseSeasons.CutHtml("<td>"); int bound0 = test.GetUpperBound(0); int bound1 = test.GetUpperBound(1); for (int i = 0; i <= bound0; i++) { string seasonUrl = ""; string seasonYear = ""; for (int a = 0; a <= bound1; a++) { string tag = test[i, a, 0]; string textInsidetag = test[i, a, 1]; try { if (tag.Contains("http://myanimelist.net/anime/season") || textInsidetag.Contains("http://myanimelist.net/anime/season")) { seasonUrl = tag.Replace("a href=\"", "").Replace("\"", "").Trim(); seasonYear = textInsidetag.Trim(); if (prevSeason != seasonYear) { seasons.Add(new AnimeSeasons() { season = seasonYear, url = seasonUrl }); prevSeason = seasonYear; } break; } } catch (Exception e) { } } } using (StreamWriter sw = File.CreateText(path)) { string actualJson = JsonConvert.SerializeObject(seasons, Formatting.Indented); allSeasonsJson = "[{\"allSeasons\" : " + actualJson + "}]"; sw.Write(allSeasonsJson); } DebugCallbackMethod(" |LAUNCH| " + Path.GetFileName(path) + " created!"); } else { DebugCallbackMethod(" |LAUNCH| " + "No need to parse html, seasons already parsed, reading file instead!"); allSeasonsJson = File.ReadAllText(path); } }
public static void ParseSearchNibl(string[] searchQuery) { HtmlParser parser = new HtmlParser(); var niblSearchResults = new List <NiblSearchInformation>(); foreach (string searchParam in searchQuery) { parser.ParseUrl("http://nibl.co.uk/bots.php?search=" + searchParam); string[,,] test = parser.CutHtml("botlistitem"); int bound0 = test.GetUpperBound(0); int bound1 = test.GetUpperBound(1); for (int i = 0; i <= bound0; i++) { string bot = ""; string pack = ""; string filename = ""; string res = ""; for (int a = 0; a <= bound1; a++) { string tag = test[i, a, 0]; string textInsidetag = test[i, a, 1]; try { if (tag.Contains("botname") || textInsidetag.Contains("botname")) { bot = textInsidetag.Split('<')[0].Trim(); } if (tag.Contains("packnumber")) { pack = textInsidetag.Trim(); } if (tag.Contains("filename")) { filename = textInsidetag.Split('<')[0].Trim(); if (filename.Contains("480") || filename.Contains("360") || filename.Contains("576") || filename.Contains("SD")) { res = "480"; } else if (filename.Contains("720") || filename.Contains("768")) { res = "720"; } else if (filename.Contains("1080") || filename.Contains("1000")) { res = "1080"; } else { res = "unknown"; } } if (filename != "" && bot != "" && pack != "") { niblSearchResults.Add(new NiblSearchInformation() { botname = bot, filename = filename, packnumber = pack, resolution = res }); break; } } catch (Exception e) { } } } } var sd = new List <NiblSearchInformation>(); var hd = new List <NiblSearchInformation>(); var fhd = new List <NiblSearchInformation>(); var unknown = new List <NiblSearchInformation>(); var bots = new List <string>(); foreach (NiblSearchInformation info in niblSearchResults) { if (info.resolution.Contains("480")) { sd.Add(info); } else if (info.resolution.Contains("720")) { hd.Add(info); } else if (info.resolution.Contains("1080")) { fhd.Add(info); } else { unknown.Add(info); } } Dictionary <string, Dictionary <string, List <NiblSearchInformation> > > values = new Dictionary <string, Dictionary <string, List <NiblSearchInformation> > >(); Dictionary <string, List <NiblSearchInformation> > everyResolution = new Dictionary <string, List <NiblSearchInformation> >(); string currentBot = " "; string previousBot = ""; foreach (NiblSearchInformation info in niblSearchResults) { currentBot = info.botname; if (currentBot != previousBot) { if (!values.ContainsKey(currentBot)) { bots.Add(currentBot); } previousBot = currentBot; } } foreach (string currentBotToSearch in bots) { var search480p = sd.FindAll(x => x.botname.Contains(currentBotToSearch)); var search720p = hd.FindAll(x => x.botname.Contains(currentBotToSearch)); var search1080p = fhd.FindAll(x => x.botname.Contains(currentBotToSearch)); var searchUnkown = unknown.FindAll(x => x.botname.Contains(currentBotToSearch)); everyResolution.Add("480", search480p); everyResolution.Add("720", search720p); everyResolution.Add("1080", search1080p); everyResolution.Add("unknown", searchUnkown); values.Add(currentBotToSearch, everyResolution); everyResolution = new Dictionary <string, List <NiblSearchInformation> >(); } string actualJson = JsonConvert.SerializeObject(values, Formatting.Indented); niblSearchJson = "[{\"NIBL\" : " + actualJson + "}]"; }
/// <ParseAllSeasons> /// <summary> /// Gets all the available seasons from Myanimelist.net, and writes them to a json file, or if the json file already exist, reads the json file(faster) instead of html parsing /// </summary /// </ParseAllSeasons> private static void ParseAllSeasons() { if (!Directory.Exists("DataStorage")) { Directory.CreateDirectory("DataStorage"); } string path = "DataStorage/allSeasonsJson.json"; if (!File.Exists(path)) { string prevSeason = ""; var seasons = new List <AnimeSeasons>(); HtmlParser parseSeasons = new HtmlParser(); parseSeasons.ParseUrl("http://myanimelist.net/anime/season/archive"); string[,,] test = parseSeasons.CutHtml("<td>"); int bound0 = test.GetUpperBound(0); int bound1 = test.GetUpperBound(1); for (int i = 0; i <= bound0; i++) { string seasonUrl = ""; string seasonYear = ""; for (int a = 0; a <= bound1; a++) { string tag = test[i, a, 0]; string textInsidetag = test[i, a, 1]; try { if (tag.Contains("http://myanimelist.net/anime/season") || textInsidetag.Contains("http://myanimelist.net/anime/season")) { seasonUrl = tag.Replace("a href=\"", "").Replace("\"", "").Trim(); seasonYear = textInsidetag.Trim(); if (prevSeason != seasonYear) { seasons.Add(new AnimeSeasons() { season = seasonYear, url = seasonUrl }); prevSeason = seasonYear; } break; } } catch (Exception e) { } } } using (StreamWriter sw = File.CreateText(path)) { string actualJson = JsonConvert.SerializeObject(seasons, Formatting.Indented); allSeasonsJson = "[{\"allSeasons\" : " + actualJson + "}]"; sw.Write(allSeasonsJson); } DebugCallbackMethod(" |LAUNCH| " + Path.GetFileName(path) + " created!"); } else { DebugCallbackMethod(" |LAUNCH| " + "No need to parse html, seasons already parsed, reading file instead!"); allSeasonsJson = File.ReadAllText(path); } }
/// <ParseSearchAnime> /// <summary> /// Parses html received from the search page of Myanimelist.net /// </summary /// </ParseSearchAnime> private static void ParseSearchAnime(string url) { searchedAnimeJson = ""; string prevAnime = ""; HtmlParser parser = new HtmlParser(); parser.ParseUrl(url); var searchedAnimeJsonList = new List <AnimeInfo>(); string[,,] test = parser.CutHtml("picSurround"); int bound0 = test.GetUpperBound(0); int bound1 = test.GetUpperBound(1); for (int i = 0; i <= bound0; i++) { string animeTitle = ""; string animeCover = ""; string animeSynopsis = ""; string animeGenres = "N/A"; string animeScore = ""; string animeId = ""; for (int a = 0; a <= bound1; a++) { string tag = test[i, a, 0]; string textInsidetag = test[i, a, 1]; try { if (tag.Contains("lazyload")) { animeTitle = tag.Split('"')[11].Split('"')[0].Trim(); animeCover = tag.Split('"')[3].Split('"')[0].Replace("r/50x70/", "").Trim(); string[] partsOfUrl = animeCover.Split('/'); animeId = partsOfUrl[partsOfUrl.Length - 1].Split('.')[0]; } if (tag.Contains("pt4")) { animeSynopsis = textInsidetag.Trim(); } if (tag.Contains("borderClass ac bgColor1\" width=\"50\"") || tag.Contains("borderClass ac bgColor0\" width=\"50\"")) { animeScore = textInsidetag.Trim(); } if (animeTitle != "" && animeSynopsis != "" && animeCover != "" && animeId != "" && animeScore != "") { searchedAnimeJsonList.Add(new AnimeInfo() { id = animeId, title = animeTitle, cover = animeCover, synopsis = animeSynopsis, genres = animeGenres, score = animeScore }); break; } } catch (Exception e) { } } } string actualJson = JsonConvert.SerializeObject(searchedAnimeJsonList, Formatting.Indented); DebugCallbackMethod("WRITING ANIME JSON"); searchedAnimeJson = "[{\"Anime\" : " + actualJson + "}]"; DebugCallbackMethod("DONE"); }
/// <ParseAnimeSeason> /// <summary> /// Parses all the anime from the html source of Myanimelists.net 's seasons page /// </summary /// </ParseAnimeSeason> private static void ParseAnimeSeason(string url, string yearAndseason) { if (!Directory.Exists("DataStorage/Anime")) { Directory.CreateDirectory("DataStorage/Anime/"); } string path = "DataStorage/Anime/anime" + yearAndseason + ".json"; string localJson = ""; if (!File.Exists(path)) { var animeSeasonsJson = new List <AnimeInfo>(); string prevAnime = ""; HtmlParser parseCurrentSeason = new HtmlParser(); parseCurrentSeason.ParseUrl(url); string[,,] test = parseCurrentSeason.CutHtml("seasonal-anime js-seasonal-anime"); int bound0 = test.GetUpperBound(0); int bound1 = test.GetUpperBound(1); for (int i = 0; i <= bound0; i++) { string animeTitle = ""; string animeCover = ""; string animeSynopsis = ""; string animeGenres = ""; string animeScore = ""; string animeId = ""; for (int a = 0; a <= bound1; a++) { string tag = test[i, a, 0]; string textInsidetag = test[i, a, 1]; try { if (tag.Contains("link-image") || textInsidetag.Contains("link-image")) { animeTitle = textInsidetag.Trim(); } if (tag.Contains("lazyload") || textInsidetag.Contains("lazyload")) { animeCover = tag.Split('"')[3].Split('"')[0]; string[] partsOfUrl = animeCover.Split('/'); animeId = partsOfUrl[partsOfUrl.Length - 1].Split('.')[0]; } if (tag.Contains("synopsis js-synopsis") || textInsidetag.Contains("synopsis js-synopsis")) { animeSynopsis = textInsidetag.Replace("\"", "\\\"").Replace("'", "\'").Split(new string[] { "<p" }, StringSplitOptions.None)[0]; } if (tag.Contains("genre") || textInsidetag.Contains("genre")) { animeGenres = textInsidetag.Trim(); } if (tag.Contains("score") || textInsidetag.Contains("score")) { animeScore = textInsidetag.Trim(); } if (animeScore != "" && animeGenres != "" && animeSynopsis != "" && animeId != "" && animeCover != "" && animeTitle != "") { if (prevAnime != animeTitle) { animeSeasonsJson.Add(new AnimeInfo() { id = animeId, title = animeTitle, cover = animeCover, synopsis = animeSynopsis, genres = animeGenres, score = animeScore }); prevAnime = animeTitle; } break; } } catch (Exception e) { } } } using (StreamWriter sw = File.CreateText(path)) { string actualJson = JsonConvert.SerializeObject(animeSeasonsJson, Formatting.Indented); DebugCallbackMethod("WRITING ANIME JSON"); localJson = "[{\"Anime\" : " + actualJson + "}]"; sw.Write(localJson); } DebugCallbackMethod(" |LAUNCH| " + Path.GetFileName(path) + " created!"); } else { DebugCallbackMethod(" |LAUNCH| " + "No need to parse html, anime already parsed, reading file instead!"); localJson = File.ReadAllText(path); } animePerSeasonJson = localJson; DebugCallbackMethod("DONE"); }