public IEnumerable <UTCourse> FetchItems()
{
IItemFetcher <UTDepartment> depFetcher = new UTArtsciDepartmentFetcher();
List <UTCourse> coursesDetail = new List <UTCourse>();
UTEngHssCsChecker checker = new UTEngHssCsChecker();
UTArtsciCourseDetailPageNumberFetcher pagenumberFetcher = new UTArtsciCourseDetailPageNumberFetcher(
String.Format(WebUrlConstants.ArtsciCourseDetailNew, 0));
int numPages = pagenumberFetcher.FetchItems().First();
object _sync = new object();
Parallel.For(0, numPages, new ParallelOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount * 4
}, delegate(int page)
//for (int page = 0; page < numPages; page++)
{
UTArtsciCourseDetailFetcherNew fetcher = new UTArtsciCourseDetailFetcherNew(
String.Format(WebUrlConstants.ArtsciCourseDetailNew, page));
IEnumerable <UTCourse> _courses = fetcher.FetchItems();
lock (_sync)
{
coursesDetail.AddRange(_courses);
}
}
);
string artsciUrl = WebUrlConstants.ArtsciTimetableNew;
List <UTCourse> courses = new List <UTCourse>();
//Parallel.For((int)'A', (int)'Z' + 1, new ParallelOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount }, delegate (int code)
for (char code = 'A'; code <= 'Z'; code++)
{
// Memory issue from the server when searching for only one character.
for (char code2 = 'A'; code2 <= 'Z'; code2++)
{
char codeChar = (char)code;
IItemFetcher <UTCourse> courseInfoFetcherNew = new UTArtsciCourseInfoFetcherNew2(artsciUrl + codeChar + code2);
IEnumerable <UTCourse> _courses = courseInfoFetcherNew.FetchItems();
lock (_sync)
{
courses.AddRange(_courses);
}
}
}
//);
// Add hss/cs/department info
//foreach (UTCourse course in courses)
Parallel.ForEach <UTCourse>(courses, new ParallelOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount
}, delegate(UTCourse course)
{
course.Faculty = "Artsci";
// Check cs/hss requirement for engineering students
if (checker.CheckHss(course.Code + course.SemesterPrefix))
{
course.AddCategory("hss");
}
if (checker.CheckArtsciCs(course.Code + course.SemesterPrefix))
{
course.AddCategory("cs");
}
});
//}
// Merge course info and course detail
IEnumerable <UTCourse> coursesTotal = courses.GroupJoin(coursesDetail,
(x => x.Abbr),
(x => x.Abbr),
((x, y) => this.CombineInfoDetail(x, y.FirstOrDefault())),
new UTCourseAbbrComparer());
// Add to unique dictionary
foreach (UTCourse course in coursesTotal)
{
if (!CoursesCollection.ContainsKey(course.Abbr))
{
CoursesCollection.Add(course.Abbr, course);
}
else
{
UTCourse existingCourse = CoursesCollection[course.Abbr];
foreach (CourseSection section in course.Sections)
{
bool found = false;
foreach (CourseSection section2 in existingCourse.Sections)
{
if (section.Name == section2.Name)
{
found = true;
break;
}
}
if (!found)
{
existingCourse.Sections.Add(section);
}
}
}
}
// Match the prerequisites to postrequisites
foreach (UTCourse course in CoursesCollection.Values)
{
if (!String.IsNullOrEmpty(course.Prerequisites))
{
foreach (Match match in CodeRegex.Matches(course.Prerequisites))
{
string abbr = match.Groups["code"].Value;
TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "H1F");
TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "H1S");
TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "Y1Y");
TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "H1Y");
}
}
}
return(CoursesCollection.Values);
}
public IEnumerable <UTCourse> FetchItems()
{
IItemFetcher <UTDepartment> depFetcher = new UTArtsciDepartmentFetcher();
List <UTCourse> coursesDetail = new List <UTCourse>();
List <UTDepartment> deps = new List <UTDepartment>(depFetcher.FetchItems());
UTEngHssCsChecker checker = new UTEngHssCsChecker();
Dictionary <string, UTDepartment> depDict = new Dictionary <string, UTDepartment>();
// Parallel threads for fetching each department
Parallel.ForEach <UTDepartment>(depFetcher.FetchItems(), new ParallelOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount
}, delegate(UTDepartment dep)
//foreach (UTDepartment dep in depFetcher.FetchItems())
{
IItemFetcher <UTCourse> courseDetailFetcher = new UTArtsciCourseDetailFetcher(dep.DetailUrl);
coursesDetail.AddRange(courseDetailFetcher.FetchItems());
if (!depDict.ContainsKey(dep.Abbr))
{
depDict.Add(dep.Abbr, dep);
}
});
//}
string artsciUrl = WebUrlConstants.ArtsciTimetableNew;
List <UTCourse> courses = new List <UTCourse>();
object _sync = new object();
Parallel.For((int)'A', (int)'Z' + 1, new ParallelOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount
}, delegate(int code)
//for (char c = 'A'; c < 'Z'; c++)
{
char codeChar = (char)code;
IItemFetcher <UTCourse> courseInfoFetcherNew = new UTArtsciCourseInfoFetcherNew2(artsciUrl + codeChar);
IEnumerable <UTCourse> _courses = courseInfoFetcherNew.FetchItems();
lock (_sync)
{
courses.AddRange(_courses);
}
}
);
// Add hss/cs/department info
//foreach (UTCourse course in courses)
Parallel.ForEach <UTCourse>(courses, new ParallelOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount
}, delegate(UTCourse course)
{
// Add department info
if (depDict.ContainsKey(course.Code.Substring(0, 3)))
{
course.Department = depDict[course.Code.Substring(0, 3)].Name;
}
course.Faculty = "Artsci";
// Check cs/hss requirement for engineering students
if (checker.CheckHss(course.Code + course.SemesterPrefix))
{
course.AddCategory("hss");
}
if (checker.CheckArtsciCs(course.Code + course.SemesterPrefix))
{
course.AddCategory("cs");
}
});
//}
// Merge course info and course detail
IEnumerable <UTCourse> coursesTotal = courses.GroupJoin(coursesDetail,
(x => x.Abbr),
(x => x.Abbr),
((x, y) => this.CombineInfoDetail(x, y.FirstOrDefault())),
new UTCourseAbbrComparer());
// Add to unique dictionary
foreach (UTCourse course in coursesTotal)
{
if (!CoursesCollection.ContainsKey(course.Abbr))
{
CoursesCollection.Add(course.Abbr, course);
}
else
{
UTCourse existingCourse = CoursesCollection[course.Abbr];
foreach (CourseSection section in course.Sections)
{
bool found = false;
foreach (CourseSection section2 in existingCourse.Sections)
{
if (section.Name == section2.Name)
{
found = true;
break;
}
}
if (!found)
{
existingCourse.Sections.Add(section);
}
}
}
}
// Match the prerequisites to postrequisites
foreach (UTCourse course in CoursesCollection.Values)
{
if (!String.IsNullOrEmpty(course.Prerequisites))
{
foreach (Match match in CodeRegex.Matches(course.Prerequisites))
{
string abbr = match.Groups["code"].Value;
TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "H1F");
TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "H1S");
TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "Y1Y");
TryMatchPreq(this.CoursesCollection, course.Abbr, abbr, "H1Y");
}
}
}
return(CoursesCollection.Values);
}