public static void ParseSearchNibl(string[] searchQuery)
{
HtmlParser parser = new HtmlParser();
var niblSearchResults = new List<NiblSearchInformation>();
foreach (string searchParam in searchQuery)
{
parser.ParseUrl("http://nibl.co.uk/bots.php?search=" + searchParam);
string[,,] test = parser.CutHtml("botlistitem");
int bound0 = test.GetUpperBound(0);
int bound1 = test.GetUpperBound(1);
for (int i = 0; i <= bound0; i++)
{
string bot = "";
string pack = "";
string filename = "";
string res = "";
for (int a = 0; a <= bound1; a++)
{
string tag = test[i, a, 0];
string textInsidetag = test[i, a, 1];
try
{
if (tag.Contains("botname") || textInsidetag.Contains("botname"))
{
bot = textInsidetag.Split('<')[0].Trim();
}
if (tag.Contains("packnumber"))
{
pack = textInsidetag.Trim();
}
if (tag.Contains("filename"))
{
filename = textInsidetag.Split('<')[0].Trim();
if (filename.Contains("480") || filename.Contains("360") || filename.Contains("576") || filename.Contains("SD"))
{
res = "480";
}
else if (filename.Contains("720") || filename.Contains("768"))
{
res = "720";
}
else if (filename.Contains("1080") || filename.Contains("1000"))
{
res = "1080";
}
else
{
res = "unknown";
}
}
if (filename != "" && bot != "" && pack != "")
{
niblSearchResults.Add(new NiblSearchInformation() { botname = bot, filename = filename, packnumber = pack, resolution = res });
break;
}
}
catch (Exception e)
{
}
}
}
}
var sd = new List<NiblSearchInformation>();
var hd = new List<NiblSearchInformation>();
var fhd = new List<NiblSearchInformation>();
var unknown = new List<NiblSearchInformation>();
var bots = new List<string>();
foreach (NiblSearchInformation info in niblSearchResults)
{
if (info.resolution.Contains("480"))
{
sd.Add(info);
}
else if (info.resolution.Contains("720"))
{
hd.Add(info);
}
else if (info.resolution.Contains("1080"))
{
fhd.Add(info);
}
else
{
unknown.Add(info);
}
}
Dictionary<string, Dictionary<string, List<NiblSearchInformation>>> values = new Dictionary<string, Dictionary<string, List<NiblSearchInformation>>>();
Dictionary<string, List<NiblSearchInformation>> everyResolution = new Dictionary<string, List<NiblSearchInformation>>();
string currentBot = " ";
string previousBot = "";
foreach (NiblSearchInformation info in niblSearchResults)
{
currentBot = info.botname;
if (currentBot != previousBot)
{
if (!values.ContainsKey(currentBot))
{
bots.Add(currentBot);
}
previousBot = currentBot;
}
}
foreach (string currentBotToSearch in bots)
{
var search480p = sd.FindAll(x => x.botname.Contains(currentBotToSearch));
var search720p = hd.FindAll(x => x.botname.Contains(currentBotToSearch));
var search1080p = fhd.FindAll(x => x.botname.Contains(currentBotToSearch));
var searchUnkown = unknown.FindAll(x => x.botname.Contains(currentBotToSearch));
everyResolution.Add("480", search480p);
everyResolution.Add("720", search720p);
everyResolution.Add("1080", search1080p);
everyResolution.Add("unknown", searchUnkown);
values.Add(currentBotToSearch, everyResolution);
everyResolution = new Dictionary<string, List<NiblSearchInformation>>();
}
string actualJson = JsonConvert.SerializeObject(values, Formatting.Indented);
niblSearchJson = "[{\"NIBL\" : " + actualJson + "}]";
}
/// <ParseAnimeSeason>
/// <summary>
/// Parses all the anime from the html source of Myanimelists.net 's seasons page
/// </summary
/// </ParseAnimeSeason>
private static void ParseAnimeSeason(string url, string yearAndseason)
{
if (!Directory.Exists("DataStorage/Anime"))
{
Directory.CreateDirectory("DataStorage/Anime/");
}
string path = "DataStorage/Anime/anime" + yearAndseason + ".json";
string localJson = "";
if (!File.Exists(path))
{
var animeSeasonsJson = new List<AnimeInfo>();
string prevAnime = "";
HtmlParser parseCurrentSeason = new HtmlParser();
parseCurrentSeason.ParseUrl(url);
string[,,] test = parseCurrentSeason.CutHtml("seasonal-anime js-seasonal-anime");
int bound0 = test.GetUpperBound(0);
int bound1 = test.GetUpperBound(1);
for (int i = 0; i <= bound0; i++)
{
string animeTitle = "";
string animeCover = "";
string animeSynopsis = "";
string animeGenres = "";
string animeScore = "";
string animeId = "";
for (int a = 0; a <= bound1; a++)
{
string tag = test[i, a, 0];
string textInsidetag = test[i, a, 1];
try
{
if (tag.Contains("link-image") || textInsidetag.Contains("link-image"))
{
animeTitle = textInsidetag.Trim();
}
if (tag.Contains("lazyload") || textInsidetag.Contains("lazyload"))
{
animeCover = tag.Split('"')[3].Split('"')[0];
string[] partsOfUrl = animeCover.Split('/');
animeId = partsOfUrl[partsOfUrl.Length - 1].Split('.')[0];
}
if (tag.Contains("synopsis js-synopsis") || textInsidetag.Contains("synopsis js-synopsis"))
{
animeSynopsis = textInsidetag.Replace("\"", "\\\"").Replace("'", "\'").Split(new string[] { "<p" }, StringSplitOptions.None)[0];
}
if (tag.Contains("genre") || textInsidetag.Contains("genre"))
{
animeGenres = textInsidetag.Trim();
}
if (tag.Contains("score") || textInsidetag.Contains("score"))
{
animeScore = textInsidetag.Trim();
}
if (animeScore != "" && animeGenres != "" && animeSynopsis != "" && animeId != "" && animeCover != "" && animeTitle != "")
{
if (prevAnime != animeTitle)
{
animeSeasonsJson.Add(new AnimeInfo() { id = animeId, title = animeTitle, cover = animeCover, synopsis = animeSynopsis, genres = animeGenres, score = animeScore });
prevAnime = animeTitle;
}
break;
}
}
catch (Exception e)
{
}
}
}
using (StreamWriter sw = File.CreateText(path))
{
string actualJson = JsonConvert.SerializeObject(animeSeasonsJson, Formatting.Indented);
DebugCallbackMethod("WRITING ANIME JSON");
localJson = "[{\"Anime\" : " + actualJson + "}]";
sw.Write(localJson);
}
DebugCallbackMethod(" |LAUNCH| " + Path.GetFileName(path) + " created!");
}
else
{
DebugCallbackMethod(" |LAUNCH| " + "No need to parse html, anime already parsed, reading file instead!");
localJson = File.ReadAllText(path);
}
animePerSeasonJson = localJson;
DebugCallbackMethod("DONE");
}
/// <ParseSearchAnime>
/// <summary>
/// Parses html received from the search page of Myanimelist.net
/// </summary
/// </ParseSearchAnime>
private static void ParseSearchAnime(string url)
{
searchedAnimeJson = "";
string prevAnime = "";
HtmlParser parser = new HtmlParser();
parser.ParseUrl(url);
var searchedAnimeJsonList = new List<AnimeInfo>();
string[,,] test = parser.CutHtml("picSurround");
int bound0 = test.GetUpperBound(0);
int bound1 = test.GetUpperBound(1);
for (int i = 0; i <= bound0; i++)
{
string animeTitle = "";
string animeCover = "";
string animeSynopsis = "";
string animeGenres = "N/A";
string animeScore = "";
string animeId = "";
for (int a = 0; a <= bound1; a++)
{
string tag = test[i, a, 0];
string textInsidetag = test[i, a, 1];
try
{
if (tag.Contains("lazyload"))
{
animeTitle = tag.Split('"')[11].Split('"')[0].Trim();
animeCover = tag.Split('"')[3].Split('"')[0].Replace("r/50x70/", "").Trim();
string[] partsOfUrl = animeCover.Split('/');
animeId = partsOfUrl[partsOfUrl.Length - 1].Split('.')[0];
}
if (tag.Contains("pt4"))
{
animeSynopsis = textInsidetag.Trim();
}
if (tag.Contains("borderClass ac bgColor1\" width=\"50\"") || tag.Contains("borderClass ac bgColor0\" width=\"50\""))
{
animeScore = textInsidetag.Trim();
}
if (animeTitle != "" && animeSynopsis != "" && animeCover != "" && animeId != "" && animeScore != "")
{
searchedAnimeJsonList.Add(new AnimeInfo() { id = animeId, title = animeTitle, cover = animeCover, synopsis = animeSynopsis, genres = animeGenres, score = animeScore });
break;
}
}
catch (Exception e)
{
}
}
}
string actualJson = JsonConvert.SerializeObject(searchedAnimeJsonList, Formatting.Indented);
DebugCallbackMethod("WRITING ANIME JSON");
searchedAnimeJson = "[{\"Anime\" : " + actualJson + "}]";
DebugCallbackMethod("DONE");
}
/// <ParseAllSeasons>
/// <summary>
/// Gets all the available seasons from Myanimelist.net, and writes them to a json file, or if the json file already exist, reads the json file(faster) instead of html parsing
/// </summary
/// </ParseAllSeasons>
private static void ParseAllSeasons()
{
if (!Directory.Exists("DataStorage"))
{
Directory.CreateDirectory("DataStorage");
}
string path = "DataStorage/allSeasonsJson.json";
if (!File.Exists(path))
{
string prevSeason = "";
var seasons = new List<AnimeSeasons>();
HtmlParser parseSeasons = new HtmlParser();
parseSeasons.ParseUrl("http://myanimelist.net/anime/season/archive");
string[,,] test = parseSeasons.CutHtml("<td>");
int bound0 = test.GetUpperBound(0);
int bound1 = test.GetUpperBound(1);
for (int i = 0; i <= bound0; i++)
{
string seasonUrl = "";
string seasonYear = "";
for (int a = 0; a <= bound1; a++)
{
string tag = test[i, a, 0];
string textInsidetag = test[i, a, 1];
try
{
if (tag.Contains("http://myanimelist.net/anime/season") || textInsidetag.Contains("http://myanimelist.net/anime/season"))
{
seasonUrl = tag.Replace("a href=\"", "").Replace("\"", "").Trim();
seasonYear = textInsidetag.Trim();
if (prevSeason != seasonYear)
{
seasons.Add(new AnimeSeasons() { season = seasonYear, url = seasonUrl });
prevSeason = seasonYear;
}
break;
}
}
catch (Exception e)
{
}
}
}
using (StreamWriter sw = File.CreateText(path))
{
string actualJson = JsonConvert.SerializeObject(seasons, Formatting.Indented);
allSeasonsJson = "[{\"allSeasons\" : " + actualJson + "}]";
sw.Write(allSeasonsJson);
}
DebugCallbackMethod(" |LAUNCH| " + Path.GetFileName(path) + " created!");
}
else
{
DebugCallbackMethod(" |LAUNCH| " + "No need to parse html, seasons already parsed, reading file instead!");
allSeasonsJson = File.ReadAllText(path);
}
}
public static void ParseSearchNibl(string[] searchQuery)
{
HtmlParser parser = new HtmlParser();
var niblSearchResults = new List <NiblSearchInformation>();
foreach (string searchParam in searchQuery)
{
parser.ParseUrl("http://nibl.co.uk/bots.php?search=" + searchParam);
string[,,] test = parser.CutHtml("botlistitem");
int bound0 = test.GetUpperBound(0);
int bound1 = test.GetUpperBound(1);
for (int i = 0; i <= bound0; i++)
{
string bot = "";
string pack = "";
string filename = "";
string res = "";
for (int a = 0; a <= bound1; a++)
{
string tag = test[i, a, 0];
string textInsidetag = test[i, a, 1];
try
{
if (tag.Contains("botname") || textInsidetag.Contains("botname"))
{
bot = textInsidetag.Split('<')[0].Trim();
}
if (tag.Contains("packnumber"))
{
pack = textInsidetag.Trim();
}
if (tag.Contains("filename"))
{
filename = textInsidetag.Split('<')[0].Trim();
if (filename.Contains("480") || filename.Contains("360") || filename.Contains("576") || filename.Contains("SD"))
{
res = "480";
}
else if (filename.Contains("720") || filename.Contains("768"))
{
res = "720";
}
else if (filename.Contains("1080") || filename.Contains("1000"))
{
res = "1080";
}
else
{
res = "unknown";
}
}
if (filename != "" && bot != "" && pack != "")
{
niblSearchResults.Add(new NiblSearchInformation()
{
botname = bot, filename = filename, packnumber = pack, resolution = res
});
break;
}
}
catch (Exception e)
{
}
}
}
}
var sd = new List <NiblSearchInformation>();
var hd = new List <NiblSearchInformation>();
var fhd = new List <NiblSearchInformation>();
var unknown = new List <NiblSearchInformation>();
var bots = new List <string>();
foreach (NiblSearchInformation info in niblSearchResults)
{
if (info.resolution.Contains("480"))
{
sd.Add(info);
}
else if (info.resolution.Contains("720"))
{
hd.Add(info);
}
else if (info.resolution.Contains("1080"))
{
fhd.Add(info);
}
else
{
unknown.Add(info);
}
}
Dictionary <string, Dictionary <string, List <NiblSearchInformation> > > values = new Dictionary <string, Dictionary <string, List <NiblSearchInformation> > >();
Dictionary <string, List <NiblSearchInformation> > everyResolution = new Dictionary <string, List <NiblSearchInformation> >();
string currentBot = " ";
string previousBot = "";
foreach (NiblSearchInformation info in niblSearchResults)
{
currentBot = info.botname;
if (currentBot != previousBot)
{
if (!values.ContainsKey(currentBot))
{
bots.Add(currentBot);
}
previousBot = currentBot;
}
}
foreach (string currentBotToSearch in bots)
{
var search480p = sd.FindAll(x => x.botname.Contains(currentBotToSearch));
var search720p = hd.FindAll(x => x.botname.Contains(currentBotToSearch));
var search1080p = fhd.FindAll(x => x.botname.Contains(currentBotToSearch));
var searchUnkown = unknown.FindAll(x => x.botname.Contains(currentBotToSearch));
everyResolution.Add("480", search480p);
everyResolution.Add("720", search720p);
everyResolution.Add("1080", search1080p);
everyResolution.Add("unknown", searchUnkown);
values.Add(currentBotToSearch, everyResolution);
everyResolution = new Dictionary <string, List <NiblSearchInformation> >();
}
string actualJson = JsonConvert.SerializeObject(values, Formatting.Indented);
niblSearchJson = "[{\"NIBL\" : " + actualJson + "}]";
}
/// <ParseAllSeasons>
/// <summary>
/// Gets all the available seasons from Myanimelist.net, and writes them to a json file, or if the json file already exist, reads the json file(faster) instead of html parsing
/// </summary
/// </ParseAllSeasons>
private static void ParseAllSeasons()
{
if (!Directory.Exists("DataStorage"))
{
Directory.CreateDirectory("DataStorage");
}
string path = "DataStorage/allSeasonsJson.json";
if (!File.Exists(path))
{
string prevSeason = "";
var seasons = new List <AnimeSeasons>();
HtmlParser parseSeasons = new HtmlParser();
parseSeasons.ParseUrl("http://myanimelist.net/anime/season/archive");
string[,,] test = parseSeasons.CutHtml("<td>");
int bound0 = test.GetUpperBound(0);
int bound1 = test.GetUpperBound(1);
for (int i = 0; i <= bound0; i++)
{
string seasonUrl = "";
string seasonYear = "";
for (int a = 0; a <= bound1; a++)
{
string tag = test[i, a, 0];
string textInsidetag = test[i, a, 1];
try
{
if (tag.Contains("http://myanimelist.net/anime/season") || textInsidetag.Contains("http://myanimelist.net/anime/season"))
{
seasonUrl = tag.Replace("a href=\"", "").Replace("\"", "").Trim();
seasonYear = textInsidetag.Trim();
if (prevSeason != seasonYear)
{
seasons.Add(new AnimeSeasons()
{
season = seasonYear, url = seasonUrl
});
prevSeason = seasonYear;
}
break;
}
}
catch (Exception e)
{
}
}
}
using (StreamWriter sw = File.CreateText(path))
{
string actualJson = JsonConvert.SerializeObject(seasons, Formatting.Indented);
allSeasonsJson = "[{\"allSeasons\" : " + actualJson + "}]";
sw.Write(allSeasonsJson);
}
DebugCallbackMethod(" |LAUNCH| " + Path.GetFileName(path) + " created!");
}
else
{
DebugCallbackMethod(" |LAUNCH| " + "No need to parse html, seasons already parsed, reading file instead!");
allSeasonsJson = File.ReadAllText(path);
}
}
/// <ParseSearchAnime>
/// <summary>
/// Parses html received from the search page of Myanimelist.net
/// </summary
/// </ParseSearchAnime>
private static void ParseSearchAnime(string url)
{
searchedAnimeJson = "";
string prevAnime = "";
HtmlParser parser = new HtmlParser();
parser.ParseUrl(url);
var searchedAnimeJsonList = new List <AnimeInfo>();
string[,,] test = parser.CutHtml("picSurround");
int bound0 = test.GetUpperBound(0);
int bound1 = test.GetUpperBound(1);
for (int i = 0; i <= bound0; i++)
{
string animeTitle = "";
string animeCover = "";
string animeSynopsis = "";
string animeGenres = "N/A";
string animeScore = "";
string animeId = "";
for (int a = 0; a <= bound1; a++)
{
string tag = test[i, a, 0];
string textInsidetag = test[i, a, 1];
try
{
if (tag.Contains("lazyload"))
{
animeTitle = tag.Split('"')[11].Split('"')[0].Trim();
animeCover = tag.Split('"')[3].Split('"')[0].Replace("r/50x70/", "").Trim();
string[] partsOfUrl = animeCover.Split('/');
animeId = partsOfUrl[partsOfUrl.Length - 1].Split('.')[0];
}
if (tag.Contains("pt4"))
{
animeSynopsis = textInsidetag.Trim();
}
if (tag.Contains("borderClass ac bgColor1\" width=\"50\"") || tag.Contains("borderClass ac bgColor0\" width=\"50\""))
{
animeScore = textInsidetag.Trim();
}
if (animeTitle != "" && animeSynopsis != "" && animeCover != "" && animeId != "" && animeScore != "")
{
searchedAnimeJsonList.Add(new AnimeInfo()
{
id = animeId, title = animeTitle, cover = animeCover, synopsis = animeSynopsis, genres = animeGenres, score = animeScore
});
break;
}
}
catch (Exception e)
{
}
}
}
string actualJson = JsonConvert.SerializeObject(searchedAnimeJsonList, Formatting.Indented);
DebugCallbackMethod("WRITING ANIME JSON");
searchedAnimeJson = "[{\"Anime\" : " + actualJson + "}]";
DebugCallbackMethod("DONE");
}
/// <ParseAnimeSeason>
/// <summary>
/// Parses all the anime from the html source of Myanimelists.net 's seasons page
/// </summary
/// </ParseAnimeSeason>
private static void ParseAnimeSeason(string url, string yearAndseason)
{
if (!Directory.Exists("DataStorage/Anime"))
{
Directory.CreateDirectory("DataStorage/Anime/");
}
string path = "DataStorage/Anime/anime" + yearAndseason + ".json";
string localJson = "";
if (!File.Exists(path))
{
var animeSeasonsJson = new List <AnimeInfo>();
string prevAnime = "";
HtmlParser parseCurrentSeason = new HtmlParser();
parseCurrentSeason.ParseUrl(url);
string[,,] test = parseCurrentSeason.CutHtml("seasonal-anime js-seasonal-anime");
int bound0 = test.GetUpperBound(0);
int bound1 = test.GetUpperBound(1);
for (int i = 0; i <= bound0; i++)
{
string animeTitle = "";
string animeCover = "";
string animeSynopsis = "";
string animeGenres = "";
string animeScore = "";
string animeId = "";
for (int a = 0; a <= bound1; a++)
{
string tag = test[i, a, 0];
string textInsidetag = test[i, a, 1];
try
{
if (tag.Contains("link-image") || textInsidetag.Contains("link-image"))
{
animeTitle = textInsidetag.Trim();
}
if (tag.Contains("lazyload") || textInsidetag.Contains("lazyload"))
{
animeCover = tag.Split('"')[3].Split('"')[0];
string[] partsOfUrl = animeCover.Split('/');
animeId = partsOfUrl[partsOfUrl.Length - 1].Split('.')[0];
}
if (tag.Contains("synopsis js-synopsis") || textInsidetag.Contains("synopsis js-synopsis"))
{
animeSynopsis = textInsidetag.Replace("\"", "\\\"").Replace("'", "\'").Split(new string[] { "<p" }, StringSplitOptions.None)[0];
}
if (tag.Contains("genre") || textInsidetag.Contains("genre"))
{
animeGenres = textInsidetag.Trim();
}
if (tag.Contains("score") || textInsidetag.Contains("score"))
{
animeScore = textInsidetag.Trim();
}
if (animeScore != "" && animeGenres != "" && animeSynopsis != "" && animeId != "" && animeCover != "" && animeTitle != "")
{
if (prevAnime != animeTitle)
{
animeSeasonsJson.Add(new AnimeInfo()
{
id = animeId, title = animeTitle, cover = animeCover, synopsis = animeSynopsis, genres = animeGenres, score = animeScore
});
prevAnime = animeTitle;
}
break;
}
}
catch (Exception e)
{
}
}
}
using (StreamWriter sw = File.CreateText(path))
{
string actualJson = JsonConvert.SerializeObject(animeSeasonsJson, Formatting.Indented);
DebugCallbackMethod("WRITING ANIME JSON");
localJson = "[{\"Anime\" : " + actualJson + "}]";
sw.Write(localJson);
}
DebugCallbackMethod(" |LAUNCH| " + Path.GetFileName(path) + " created!");
}
else
{
DebugCallbackMethod(" |LAUNCH| " + "No need to parse html, anime already parsed, reading file instead!");
localJson = File.ReadAllText(path);
}
animePerSeasonJson = localJson;
DebugCallbackMethod("DONE");
}
