private List <string> GetProfessorsCourseDescription(string html)
{
List <string> course_desription = new List <string>();
List <string> course_hrefs = GetProfessorsCourseHrefs(html);
List <string> descriptive_pages = new List <string>();
List <CQ> cqList = new List <CQ>();
foreach (var href in course_hrefs)
{
cqList.Add(CQ.Create(RequestGetter.GetRequestByUrl(href)));
}
foreach (CQ cq in cqList)
{
foreach (IDomObject page in cq.Find("section"))
{
if (page.GetAttribute("class") == "description")
{
if (page.Cq().Text().Contains("Опис курсу"))
{
course_desription.Add(page.Cq().Text());
}
}
}
}
return(course_desription);
}
public void FillDepartmentsInfoDict(Dictionary <string, string> facultyCollegeUrlDict)
{
foreach (string facultyCollegeUrl in facultyCollegeUrlDict.Values)
{
string requestUrl = (facultyCollegeUrl + "/about/departments");
if (facultyCollegeUrl == "http://natcollege.lnu.edu.ua")
{
requestUrl = (facultyCollegeUrl + "/about/labs");
}
Dictionary <string, string> departmentUrlDict = GetDepartmnetUrlDict(RequestGetter.GetRequestByUrl(requestUrl));
Dictionary <string, Department> departmentDict = new Dictionary <string, Department>();
foreach (KeyValuePair <string, string> departmentUrl in departmentUrlDict)
{
Department newDepartment = new Department();
newDepartment.FillDepartmentInfo(RequestGetter.GetRequestByUrl(departmentUrl.Value));
departmentDict.Add(departmentUrl.Key, newDepartment);
}
}
}
private List <string> GetProfessors_CourseCurriculum(string html)
{
List <string> curiculum = new List <string>();
List <string> course_hrefs = GetProfessorsCourseHrefs(html);
List <CQ> cqList = new List <CQ>();
foreach (var href in course_hrefs)
{
cqList.Add(CQ.Create(RequestGetter.GetRequestByUrl(href)));
}
foreach (CQ cq in cqList)
{
foreach (IDomObject page in cq.Find("article"))
{
if (page.GetAttribute("class") == "content course")
{
curiculum.Add(page.Cq().Text());
}
}
}
return(curiculum);
}
private List <string> GetProfessorsCourseMaterials(string html)
{
List <string> course_materials = new List <string>();
List <string> course_hrefs = GetProfessorsCourseHrefs(html);
List <CQ> cqList = new List <CQ>();
foreach (var href in course_hrefs)
{
cqList.Add(CQ.Create(RequestGetter.GetRequestByUrl(href)));
}
foreach (CQ cq in cqList)
{
foreach (IDomObject page in cq.Find("section"))
{
if (page.GetAttribute("class") == "documents")
{
course_materials.Add(page.Cq().Text());
}
}
}
return(course_materials);
}
public void FillGapsInBD()
{
List <string> linksToFill = new List <string>();
using (var con = new NpgsqlConnection(connStr))
{
con.Open();
NpgsqlCommand comd = con.CreateCommand();
//comd.CommandText = "SELECT to_regclass('dbo.\"Professors\"');";
//bool exist = comd.ExecuteScalar() == null ? true : false;
//if (exist)
//{
NpgsqlCommand cmd = con.CreateCommand();
cmd.CommandText = "Select link from dbo.\"Professors\" where pib =''";
using (NpgsqlDataReader rdr = cmd.ExecuteReader())
{
while (rdr.Read())
{
linksToFill.Add(rdr.GetString(0));
}
}
}
using (var db = new IndividualContext())
{
foreach (string lecturerUrl in linksToFill)
{
Professor newProfessor = new Professor();
newProfessor.FillProfessorInfo(RequestGetter.GetRequestByUrl(lecturerUrl));
newProfessor.link = lecturerUrl;
db.Professors.Add(newProfessor);
db.Entry(newProfessor).State = EntityState.Modified;
db.SaveChanges();
}
}
}
public FillData(bool startNewDB = false)
{
this.startNewDB = startNewDB;
string facultyListUrl = @"http://www.lnu.edu.ua/about/faculties/";
string collegeListUrl = @"http://www.lnu.edu.ua/about/colleges/";
facultyCollegeUrlDict = GetFacultyCollegeUrlDict(RequestGetter.GetRequestByUrl(facultyListUrl));
foreach (var kvp in GetFacultyCollegeUrlDict(RequestGetter.GetRequestByUrl(collegeListUrl)))
{
facultyCollegeUrlDict.Add(kvp.Key, kvp.Value);
}
facultyCollegeUrlDict.Remove("економічний факультет");
facultyCollegeUrlDict.Remove("біологічний факультет");
facultyCollegeUrlDict.Remove("факультет електроніки та комп’ютерних технологій");
facultyCollegeUrlDict.Remove("механіко-математичний факультет");
facultyCollegeUrlDict.Remove("факультет управління фінансами та бізнесу");
// Dictionary<string, List<string>> facultyStaffUrlDict = GetFacultyStaffUrlDict(GetRequestByUrl(url));
//Dictionary<string, string> departmentUrlDict = GetDepartmnetUrlDict(GetRequestByUrl(url1));
//FillDepartmentsInfoDict(facultyCollegeUrlDict);
}
public void FillStaffBD()
{
string LastFaculty = "";
string LastDepartment = "";
int startFacultyIndex = 0;
if (!startNewDB)
{
using (var con = new NpgsqlConnection(connStr))
{
con.Open();
NpgsqlCommand comd = con.CreateCommand();
//comd.CommandText = "SELECT to_regclass('dbo.\"Professors\"');";
//bool exist = comd.ExecuteScalar() == null ? true : false;
//if (exist)
//{
NpgsqlCommand cmd = con.CreateCommand();
cmd.CommandText = "Select department,faculty from dbo.\"Professors\" order by id desc limit 1";
using (NpgsqlDataReader rdr = cmd.ExecuteReader())
{
while (rdr.Read())
{
LastDepartment = rdr.GetString(0).ToLower();
LastFaculty = rdr.GetString(1).ToLower();
}
}
//}
}
startFacultyIndex = facultyCollegeUrlDict.Keys.ToList().IndexOf(LastFaculty);
}
//else{
// using (var con = new NpgsqlConnection(connStr))
// {
// con.Open();
// NpgsqlCommand comd = con.CreateCommand();
// NpgsqlCommand cmd = con.CreateCommand();
// cmd.CommandText = "delete from dbo.\"Professors\"";
// cmd.ExecuteNonQuery();
// }
//}
bool restart;
do
{
restart = false;
try
{
foreach (string facultyCollegeUrl in facultyCollegeUrlDict.Values.Skip(startFacultyIndex))
{
string requestUrl = (facultyCollegeUrl + "/about/staff");
Dictionary <string, List <string> > facultyStaffUrl = GetFacultyStaffUrlDict(RequestGetter.GetRequestByUrl(requestUrl));
int startStaffIndex = -1;
if (!startNewDB)
{
startStaffIndex = facultyStaffUrl.Keys.ToList().IndexOf("кафедра " + LastDepartment);
if (startStaffIndex == -1)
{
startStaffIndex = facultyStaffUrl.Keys.ToList().IndexOf(LastDepartment.Replace("лабораторії ", ""));
}
if (startStaffIndex == -1)
{
startStaffIndex = facultyStaffUrl.Keys.ToList().IndexOf(LastDepartment.Replace("лабораторії", "лабораторія"));
}
}
using (var db = new IndividualContext())
{
foreach (KeyValuePair <string, List <string> > department in facultyStaffUrl.Skip(startStaffIndex + 1))
{
List <Professor> departmentStaff = new List <Professor>();
foreach (string lecturerUrl in department.Value)
{
Professor newProfessor = new Professor();
newProfessor.FillProfessorInfo(RequestGetter.GetRequestByUrl(lecturerUrl));
newProfessor.link = lecturerUrl;
//departmentStaff.Add(newProfessor);
db.Professors.Add(newProfessor);
}
//facultyStaffDict.Add(department.Key, departmentStaff);//title-get department title
db.SaveChanges();
}
}
}
}
catch (System.Net.WebException)
{
restart = true;
}
} while (restart);
}