private static void getUniversityCatalog()
{
Console.WriteLine("Login to get links from database");
var conn = openDatabase();
Console.WriteLine("Starting collection");
MySqlCommand command = new MySqlCommand();
command.Connection = conn;
List <string> values = new List <string>();
string sql = "SELECT link FROM idp.course_names";
command.CommandText = sql;
MySqlDataReader reader = command.ExecuteReader();
while (reader.Read())
{
values.Add(digitsOnly.Replace(reader.GetString(0), "").Substring(1));
//Console.WriteLine(digitsOnly.Replace(reader.GetString(0),"").Substring(1));
}
conn.Close();
reader.Close();
MySqlConnection comm = openDatabase();
MySqlCommand cmd = new MySqlCommand();
cmd.Connection = comm;
foreach (string item in values)
{
// Console.WriteLine(item);
string url = $"http://catalog.stcloudstate.edu/Catalog/ViewCatalog.aspx?pageid=viewcatalog&catalogid=8&topicgroupid={item}";
// string url = $"http://catalog.stcloudstate.edu/Catalog/ViewCatalog.aspx?pageid=viewcatalog&catalogid=8&topicgroupid=107";
Console.WriteLine("Current url: " + url);
string name, description, credits;
string shortVersion, href;
int courseNumber;
HtmlWeb web = new HtmlWeb();
HtmlDocument document = web.Load(url);
Console.WriteLine("Webpage loaded");
HtmlNode node = document.DocumentNode.SelectSingleNode("//body//span[@id='ctl00_ctl00_mainLayoutContent_mainContent_pager']");
string maxPages;
if (node != null)
{
maxPages = node.LastChild.PreviousSibling.PreviousSibling.InnerText;
}
else
{
maxPages = "1";
}
// number of pages to concat onto thing
int tryer;
int.TryParse(maxPages, out tryer);
HtmlNodeCollection collection = document.DocumentNode.SelectNodes("//table[@class='DeAcFormTable']");
//Number of pages to grab
for (int j = 0; j < tryer; j++)
{
document = web.Load(url + $"&pg={j + 1 }");
//Amount of tables on the page
collection = document.DocumentNode.SelectNodes("//table[@class='DeAcFormTable']");
for (int i = 0; i < collection.LongCount(); i += 2)
{
//Section to be parsed
//------------------------------------------------------------------------------------------------------------------------
name = document.DocumentNode.SelectSingleNode($"//table[{i + 1}]//h3").InnerText;
description = document.DocumentNode.SelectSingleNode($"//table[{i + 2}]//td[2]").InnerText;
credits = document.DocumentNode.SelectSingleNode($"//table[{i + 1}]//td[2]").InnerText;
HtmlNodeCollection tempPrereqs = document.DocumentNode.SelectNodes($"//table[{i + 2}]//tr[2]//a[@class='topictooltip']");
HtmlNodeCollection tempOffered = document.DocumentNode.SelectNodes($"//table[{i + 2}]//tr[last()]//li");
//------------------------------------------------------------------------------------------------------------------------
shortVersion = name.Split(' ')[0];
int.TryParse(name.Split(' ')[1].Replace(".", String.Empty), out courseNumber);
name = name.Substring(name.LastIndexOf('.') + 1).TrimStart();
credits = credits.Substring(credits.LastIndexOf(':') + 2);
Console.WriteLine(name + " " + courseNumber);
//Database push
//----------------------------------------------------
Console.WriteLine("Trying to input this into the database:");
Console.WriteLine($"Course number: {courseNumber}, Short Version: {shortVersion}, Name: {name}\nDescription: {description}");
cmd.CommandText = string.Format("INSERT IGNORE INTO idp.course_collection(course_number,short,name,description) VALUES('{0}','{1}','{2}','{3}')", courseNumber, rgx.Replace(shortVersion, ""), rgx.Replace(name, ""), rgx.Replace(description, ""));
cmd.ExecuteNonQuery();
if (tempOffered != null)
{
foreach (HtmlNode item1 in tempOffered)
{
Console.WriteLine(item1.InnerText + ", ");
if (!item1.InnerText.Contains("GOAL"))
{
cmd.CommandText = string.Format("INSERT INTO idp.seasons(course_number,seasons) VALUES('{0}','{1}')", courseNumber, item1.InnerText);
cmd.ExecuteNonQuery();
}
else
{
Console.WriteLine($"********************{item1.InnerText}**************************");
}
}
}
else
{
cmd.CommandText = string.Format("INSERT INTO idp.seasons(course_number,seasons) VALUES('{0}','{1}')", courseNumber, "Demand");
cmd.ExecuteNonQuery();
}
if (tempPrereqs != null)
{
foreach (HtmlNode item1 in tempPrereqs)
{
Console.Write(item1.InnerText + ", ");
cmd.CommandText = string.Format("INSERT IGNORE INTO idp.prereqs(course_number,prereq,link) VALUES('{0}','{1}','{2}')", courseNumber, item1.InnerText, item1.Attributes["href"].Value);
cmd.ExecuteNonQuery();
}
}
Console.WriteLine();
}
}
}
comm.Close();
}