public IEnumerable <UTCourse> FetchItems() { IItemFetcher <UTCourse> fallCourseFetcher = new UTEngCourseInfoFetcher(WebUrlConstants.EngTimetableFall); IItemFetcher <UTCourse> winterCourseFetcher = new UTEngCourseInfoFetcher(WebUrlConstants.EngTimetableWinter); IEnumerable <UTCourse> allCourses = fallCourseFetcher.FetchItems().Concat <UTCourse>(winterCourseFetcher.FetchItems()); IEnumerable <UTCourse> coursesDetail = new UTEngCourseDetailFetcher(WebUrlConstants.EngCalendar).FetchItems(); // Merge course info and course detail allCourses = allCourses.GroupJoin(coursesDetail, (x => x.Abbr), (x => x.Abbr), ((x, y) => this.CombineInfoDetail(x, y.FirstOrDefault())), new UTCourseAbbrComparer()); UTEngHssCsChecker checker = new UTEngHssCsChecker(); foreach (UTCourse course in allCourses) { course.Faculty = "Engineering"; if (checker.CheckEngCs(course.Code)) { course.AddCategory("cs"); } if (!CoursesCollection.ContainsKey(course.Abbr)) { CoursesCollection.Add(course.Abbr, course); } } // Match the prerequisites to postrequisites foreach (UTCourse course in CoursesCollection.Values) { if (!String.IsNullOrEmpty(course.Prerequisites)) { foreach (Match match in CodeRegex.Matches(course.Prerequisites)) { string abbr = match.Value; TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "F"); TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "S"); TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "Y"); } } } return(CoursesCollection.Values); }
public IEnumerable <UTCourse> FetchItems() { IItemFetcher <UTDepartment> depFetcher = new UTArtsciDepartmentFetcher(); List <UTCourse> coursesDetail = new List <UTCourse>(); UTEngHssCsChecker checker = new UTEngHssCsChecker(); UTArtsciCourseDetailPageNumberFetcher pagenumberFetcher = new UTArtsciCourseDetailPageNumberFetcher( String.Format(WebUrlConstants.ArtsciCourseDetailNew, 0)); int numPages = pagenumberFetcher.FetchItems().First(); object _sync = new object(); Parallel.For(0, numPages, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount * 4 }, delegate(int page) //for (int page = 0; page < numPages; page++) { UTArtsciCourseDetailFetcherNew fetcher = new UTArtsciCourseDetailFetcherNew( String.Format(WebUrlConstants.ArtsciCourseDetailNew, page)); IEnumerable <UTCourse> _courses = fetcher.FetchItems(); lock (_sync) { coursesDetail.AddRange(_courses); } } ); string artsciUrl = WebUrlConstants.ArtsciTimetableNew; List <UTCourse> courses = new List <UTCourse>(); //Parallel.For((int)'A', (int)'Z' + 1, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount }, delegate (int code) for (char code = 'A'; code <= 'Z'; code++) { // Memory issue from the server when searching for only one character. for (char code2 = 'A'; code2 <= 'Z'; code2++) { char codeChar = (char)code; IItemFetcher <UTCourse> courseInfoFetcherNew = new UTArtsciCourseInfoFetcherNew2(artsciUrl + codeChar + code2); IEnumerable <UTCourse> _courses = courseInfoFetcherNew.FetchItems(); lock (_sync) { courses.AddRange(_courses); } } } //); // Add hss/cs/department info //foreach (UTCourse course in courses) Parallel.ForEach <UTCourse>(courses, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount }, delegate(UTCourse course) { course.Faculty = "Artsci"; // Check cs/hss requirement for engineering students if (checker.CheckHss(course.Code + course.SemesterPrefix)) { course.AddCategory("hss"); } if (checker.CheckArtsciCs(course.Code + course.SemesterPrefix)) { course.AddCategory("cs"); } }); //} // Merge course info and course detail IEnumerable <UTCourse> coursesTotal = courses.GroupJoin(coursesDetail, (x => x.Abbr), (x => x.Abbr), ((x, y) => this.CombineInfoDetail(x, y.FirstOrDefault())), new UTCourseAbbrComparer()); // Add to unique dictionary foreach (UTCourse course in coursesTotal) { if (!CoursesCollection.ContainsKey(course.Abbr)) { CoursesCollection.Add(course.Abbr, course); } else { UTCourse existingCourse = CoursesCollection[course.Abbr]; foreach (CourseSection section in course.Sections) { bool found = false; foreach (CourseSection section2 in existingCourse.Sections) { if (section.Name == section2.Name) { found = true; break; } } if (!found) { existingCourse.Sections.Add(section); } } } } // Match the prerequisites to postrequisites foreach (UTCourse course in CoursesCollection.Values) { if (!String.IsNullOrEmpty(course.Prerequisites)) { foreach (Match match in CodeRegex.Matches(course.Prerequisites)) { string abbr = match.Groups["code"].Value; TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "H1F"); TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "H1S"); TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "Y1Y"); TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "H1Y"); } } } return(CoursesCollection.Values); }
public IEnumerable <UTCourse> FetchItems() { IItemFetcher <UTCourse> fallCourseFetcher = new UTEngCourseInfoFetcher(WebUrlConstants.EngTimetableFall); IItemFetcher <UTCourse> winterCourseFetcher = new UTEngCourseInfoFetcher(WebUrlConstants.EngTimetableWinter); IEnumerable <UTCourse> allCourses = fallCourseFetcher.FetchItems().Concat <UTCourse>(winterCourseFetcher.FetchItems()); //IEnumerable<UTCourse> courseDetail = new UTEngCourseDetailFetcher(WebUrlConstants.EngCalendar).FetchItems(); List <UTCourse> courseDetail = new List <UTCourse>(); Parallel.ForEach <UTCourse>(allCourses, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount }, delegate(UTCourse course) { string session; if (String.Equals(course.Semester, "S")) { session = WebUrlConstants.ArtsciSessionWinter; } else { session = WebUrlConstants.ArtsciSession; } IEnumerable <UTCourse> courseDetail_ = new UTCourseDetailFetcher(String.Format(WebUrlConstants.CourseFinderCourse, course.Abbr, session)).FetchItems(); lock (this) { if (courseDetail_.Count() > 0) { UTCourse course_ = courseDetail_.First(); courseDetail.Add(course_); Console.WriteLine("Engineering Course Detail: {0} | {1}", course_.Abbr, course.Abbr); } else { Console.WriteLine("Engineering Course Detail Not Found: {0}", course.Abbr); } } }); // Merge course info and course detail allCourses = allCourses.GroupJoin(courseDetail, (x => x.Abbr), (x => x.Abbr), ((x, y) => this.CombineInfoDetail(x, y.FirstOrDefault())), new UTCourseAbbrComparer()); UTEngHssCsChecker checker = new UTEngHssCsChecker(); foreach (UTCourse course in allCourses) { course.Faculty = "Engineering"; if (checker.CheckEngCs(course.Code)) { course.AddCategory("cs"); } if (!CoursesCollection.ContainsKey(course.Abbr)) { CoursesCollection.Add(course.Abbr, course); } } // Match the prerequisites to postrequisites foreach (UTCourse course in CoursesCollection.Values) { if (!String.IsNullOrEmpty(course.Prerequisites)) { foreach (Match match in CodeRegex.Matches(course.Prerequisites)) { string abbr = match.Value; TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "F"); TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "S"); TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "Y"); } } } return(CoursesCollection.Values); }
public IEnumerable <UTCourse> FetchItems() { IItemFetcher <UTDepartment> depFetcher = new UTArtsciDepartmentFetcher(); List <UTCourse> coursesDetail = new List <UTCourse>(); List <UTDepartment> deps = new List <UTDepartment>(depFetcher.FetchItems()); UTEngHssCsChecker checker = new UTEngHssCsChecker(); Dictionary <string, UTDepartment> depDict = new Dictionary <string, UTDepartment>(); // Parallel threads for fetching each department Parallel.ForEach <UTDepartment>(depFetcher.FetchItems(), new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount }, delegate(UTDepartment dep) //foreach (UTDepartment dep in depFetcher.FetchItems()) { IItemFetcher <UTCourse> courseDetailFetcher = new UTArtsciCourseDetailFetcher(dep.DetailUrl); coursesDetail.AddRange(courseDetailFetcher.FetchItems()); if (!depDict.ContainsKey(dep.Abbr)) { depDict.Add(dep.Abbr, dep); } }); //} string artsciUrl = WebUrlConstants.ArtsciTimetableNew; List <UTCourse> courses = new List <UTCourse>(); object _sync = new object(); Parallel.For((int)'A', (int)'Z' + 1, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount }, delegate(int code) //for (char c = 'A'; c < 'Z'; c++) { char codeChar = (char)code; IItemFetcher <UTCourse> courseInfoFetcherNew = new UTArtsciCourseInfoFetcherNew2(artsciUrl + codeChar); IEnumerable <UTCourse> _courses = courseInfoFetcherNew.FetchItems(); lock (_sync) { courses.AddRange(_courses); } } ); // Add hss/cs/department info //foreach (UTCourse course in courses) Parallel.ForEach <UTCourse>(courses, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount }, delegate(UTCourse course) { // Add department info if (depDict.ContainsKey(course.Code.Substring(0, 3))) { course.Department = depDict[course.Code.Substring(0, 3)].Name; } course.Faculty = "Artsci"; // Check cs/hss requirement for engineering students if (checker.CheckHss(course.Code + course.SemesterPrefix)) { course.AddCategory("hss"); } if (checker.CheckArtsciCs(course.Code + course.SemesterPrefix)) { course.AddCategory("cs"); } }); //} // Merge course info and course detail IEnumerable <UTCourse> coursesTotal = courses.GroupJoin(coursesDetail, (x => x.Abbr), (x => x.Abbr), ((x, y) => this.CombineInfoDetail(x, y.FirstOrDefault())), new UTCourseAbbrComparer()); // Add to unique dictionary foreach (UTCourse course in coursesTotal) { if (!CoursesCollection.ContainsKey(course.Abbr)) { CoursesCollection.Add(course.Abbr, course); } else { UTCourse existingCourse = CoursesCollection[course.Abbr]; foreach (CourseSection section in course.Sections) { bool found = false; foreach (CourseSection section2 in existingCourse.Sections) { if (section.Name == section2.Name) { found = true; break; } } if (!found) { existingCourse.Sections.Add(section); } } } } // Match the prerequisites to postrequisites foreach (UTCourse course in CoursesCollection.Values) { if (!String.IsNullOrEmpty(course.Prerequisites)) { foreach (Match match in CodeRegex.Matches(course.Prerequisites)) { string abbr = match.Groups["code"].Value; TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "H1F"); TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "H1S"); TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "Y1Y"); TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "H1Y"); } } } return(CoursesCollection.Values); }