public static async Task Main(string[] args)
{
var home = new AppHome("uva_super_report");
Console.WriteLine($"Using config path: {home.ConfigPath}");
if (!home.ConfigPresent())
{
Console.WriteLine("Need to generate a config file.");
home.CreateConfig(new DocumentSyntax {
Tables =
{
new TableSyntax("tokens")
{
Items =
{
{ "token", "PUT_TOKEN_HERE" }
}
},
new TableSyntax("limits")
{
Items =
{
{ "sample_skip", 0 },
{ "sample_take", 0 },
{ "courses_per_teacher", 0 },
{ "assignments_per_course", 0 },
{ "submissions_per_assignment", 0 },
{ "teachers_only", false }
}
}
}
});
Console.WriteLine("Created a new config file. Please go put in your token.");
return;
}
Console.WriteLine("Found config file.");
var config = home.GetConfig();
Debug.Assert(config != null, nameof(config) + " != null");
var token = config.GetTable("tokens").Get <string>("token");
var limits = config.GetTable("limits");
var sampleTake = (int)limits.GetOr <long>("sample_take");
var sampleSkip = (int)limits.GetOr <long>("sample_skip");
var coursesPerTeacher = (int)limits.GetOr <long>("courses_per_teacher");
var assignmentsPerCourse = (int)limits.GetOr <long>("assignments_per_course");
var submissionsPerAssignment = (int)limits.GetOr <long>("submissions_per_assignment");
var teachersOnly = limits.GetOr <bool>("teachers_only");
Console.WriteLine($"SKIPPING {sampleSkip} users.");
Console.WriteLine($"TAKING {(sampleTake == default ? "ALL" : sampleTake.ToString())} users.");
Console.WriteLine($"TAKING {(coursesPerTeacher == default ? "ALL" : coursesPerTeacher.ToString())} courses per teacher.");
Console.WriteLine($"TAKING {(assignmentsPerCourse == default ? "ALL" : assignmentsPerCourse.ToString())} assignments per course.");
Console.WriteLine($"TAKING {(submissionsPerAssignment == default ? "ALL" : submissionsPerAssignment.ToString())} submissions per assignment.");
// --------------------------------------------------------------------
var api = new Api(token, "https://uview.instructure.com/api/v1/");
var studentsObj = new JObject();
var teachersObj = new JObject();
var coursesObj = new JObject();
var submissionsObj = new JObject();
var assignmentsOverallObj = new JObject();
var assignmentsIndividualObj = new JObject();
var individualCoursePerformanceObj = new JObject();
var overallCoursePerformanceObj = new JObject();
var teacherPerformanceObj = new JObject();
var started = DateTime.Now;
var document = new JObject {
["teachers"] = teachersObj,
["students"] = studentsObj,
["courses"] = coursesObj,
["submissions"] = submissionsObj,
["assignmentsOverall"] = assignmentsOverallObj,
["assignmentsIndividual"] = assignmentsIndividualObj,
["individualCoursePerformance"] = individualCoursePerformanceObj,
["overallCoursePerformance"] = overallCoursePerformanceObj,
["teacherPerformance"] = teacherPerformanceObj,
["limits"] = new JObject {
["sampleTake"] = sampleTake,
["sampleSkip"] = sampleSkip,
["coursesPerTeacher"] = coursesPerTeacher,
["assignmentsPerCourse"] = assignmentsPerCourse,
["submissionsPerAssignment"] = submissionsPerAssignment,
["teachersOnly"] = teachersOnly
},
["dateStarted"] = started.ToIso8601Date()
};
var sample = api.StreamUsers()
.Where(u => !u.Name.ToLowerInvariant().Contains("test"))
.Where(u => !u.SisUserId?.StartsWith("pG") ?? false);
if (teachersOnly)
{
Console.WriteLine("TAKING TEACHERS ONLY.");
sample = sample.WhereAwait(async u => await u.IsTeacher());
}
sample = sample.Skip(sampleSkip);
if (sampleTake != default)
{
sample = sample.Take(sampleTake);
}
await foreach (var user in sample)
{
try {
if (await user.IsTeacher())
{
#region CurrentUserIsTeacher
if (!teachersObj.ContainsKey(user.Id.ToString()))
{
teachersObj[user.Id.ToString()] = new JObject {
["sisId"] = user.SisUserId,
["fullName"] = user.Name
};
}
var enrollmentsStream = api.StreamUserEnrollments(user.Id, TeacherEnrollment.Yield())
.SelectAwait(async e => (e, await api.GetCourse(e.CourseId)))
.Where(ec => !ec.Item2.Name.ToLowerInvariant().Contains("advisory"));
if (coursesPerTeacher != default)
{
enrollmentsStream = enrollmentsStream.Take(coursesPerTeacher);
}
await foreach (var(enrollment, course) in enrollmentsStream)
{
if (!coursesObj.ContainsKey(course.Id.ToString()))
{
coursesObj[course.Id.ToString()] = new JObject {
["sisId"] = course.SisCourseId,
["name"] = course.Name
};
}
if (!overallCoursePerformanceObj.ContainsKey(course.Id.ToString()))
{
var studentEnrollments =
api.StreamCourseEnrollments(course.Id, StudentEnrollment.Yield());
var courseScores = new List <decimal>();
await foreach (var studentEnrollment in studentEnrollments)
{
var grades = studentEnrollment.Grades;
if (!individualCoursePerformanceObj.ContainsKey(enrollment.Id.ToString()))
{
individualCoursePerformanceObj[enrollment.Id.ToString()] = new JObject {
["studentId"] = user.Id,
["courseId"] = enrollment.CourseId,
["currentLetterGrade"] = grades.CurrentGrade,
["finalLetterGrade"] = grades.FinalGrade,
["currentScore"] = grades.CurrentScore,
["finalScore"] = grades.FinalScore,
["activitySecondsSpent"] = studentEnrollment.TotalActivityTime,
["lastAttendedAt"] = studentEnrollment.LastAttendedAt?.ToIso8601Date(),
["lastActivityAt"] = studentEnrollment.LastActivityAt?.ToIso8601Date()
};
}
var currentScore = grades.CurrentScore;
courseScores.Add(string.IsNullOrEmpty(currentScore) ? 0
: Convert.ToDecimal(grades.CurrentScore));
}
if (!courseScores.Any())
{
continue;
}
var courseScoreStats = new Stats(courseScores);
var pass = courseScores.Count(s => s > 66.5m);
overallCoursePerformanceObj[course.Id.ToString()] = new JObject {
["gradesInSample"] = courseScores.Count,
["meanCourseScore"] = courseScoreStats.Mean,
["modeCourseScore"] = courseScoreStats.Mode,
["25thPercentileCourseScore"] = courseScoreStats.Q1,
["medianCourseScore"] = courseScoreStats.Median,
["75thPercentileCourseScore"] = courseScoreStats.Q3,
["courseScoreStandardDeviation"] = courseScoreStats.Sigma,
["coursePassCount"] = pass,
["courseFailCount"] = courseScores.Count - pass
};
}
var assignments = (await api.StreamCourseAssignments(course.Id)
.Where(a => a.Published)
.ToListAsync())
.DistinctBy(a => a.Id);
if (assignmentsPerCourse != default)
{
assignments = assignments.Take(assignmentsPerCourse);
}
foreach (var assignment in assignments)
{
var allSubmissionsStream = api.StreamSubmissionVersions(course.Id, assignment.Id);
if (submissionsPerAssignment != default)
{
allSubmissionsStream = allSubmissionsStream.Take(submissionsPerAssignment);
}
var allSubmissions = await allSubmissionsStream.ToListAsync();
// just listing every submission here
foreach (var submission in allSubmissions)
{
var submitter = await api.GetUser(submission.UserId);
submissionsObj[$"c_{course.Id}|a_{assignment.Id}|u_{submitter.Id}|n_{submission.Attempt??0}"] = new JObject {
["assignmentId"] = assignment.Id,
["courseId"] = course.Id,
["userId"] = submitter.Id,
["versionNumber"] = submission.Attempt,
["submissionDate"] = submission.SubmittedAt,
["pointsEarned"] = submission.Score,
["dateSubmitted"] = submission.SubmittedAt,
["dateGraded"] = submission.GradedAt,
["gradeMatchesCurrent"] = submission.GradeMatchesCurrentSubmission,
["late"] = submission.Late ?? false,
["missing"] = submission.Missing ?? false
};
}
// now narrow down for assignment performance
var submissions = allSubmissions.Where(s => s.Score != null)
.GroupBy(s => s.UserId)
.Select(sg => sg.First())
.ToList();
if (!submissions.Any())
{
continue;
}
var scores = submissions.Select(s => s.Score)
.Cast <decimal>()
.ToList();
var stats = new Stats(scores);
if (assignmentsOverallObj.ContainsKey($"c_{course.Id}|a_{assignment.Id}"))
{
Console.WriteLine($"WARN: Duplicated assignment?\nc={course.Id}\na={assignment.Id}\n\n{assignment.ToPrettyString()}\nSkipping!------\n");
continue;
}
assignmentsOverallObj[$"c_{course.Id}|a_{assignment.Id}"] = new JObject {
["assignmentName"] = assignment.Name,
["assignmentId"] = assignment.Id,
["courseId"] = course.Id,
["teacherId"] = user.Id,
["countedInFinalGrade"] = !(assignment.OmitFromFinalGrade ?? false),
["pointsPossible"] = assignment.PointsPossible,
["createdDate"] = assignment.CreatedAt.ToIso8601Date(),
["dueDate"] = assignment.DueAt?.ToIso8601Date(),
["gradesInSample"] = submissions.Count,
["meanScore"] = stats.Mean,
["modeScore"] = stats.Mode,
["25thPercentileScore"] = stats.Q1,
["medianScore"] = stats.Median,
["75thPercentileScore"] = stats.Q3,
["scoreStandardDeviation"] = stats.Sigma
};
// here we're listing the submissions per user that actually count for grades
foreach (var submission in submissions)
{
var submitter = await api.GetUser(submission.UserId);
Debug.Assert(!assignmentsIndividualObj.ContainsKey($"c_{course.Id}|a_{assignment.Id}|u_{submitter.Id}"));
var score = submission.Score.Value;
var z = Convert.ToDouble(score - stats.Mean) / stats.Sigma;
var iqr = stats.Q3 - stats.Q1;
TimeSpan?minutesSubmittedBeforeDueDate = null;
if (submission.SubmittedAt != null && assignment.DueAt != null)
{
minutesSubmittedBeforeDueDate = assignment.DueAt.Value - submission.SubmittedAt.Value;
}
assignmentsIndividualObj[$"c_{course.Id}|a_{assignment.Id}|u_{submitter.Id}"] = new JObject {
["assignmentId"] = assignment.Id,
["courseId"] = course.Id,
["userId"] = submitter.Id,
["submissionDate"] = submission.SubmittedAt,
["pointsEarned"] = score,
["z"] = z,
["isUnusual"] = Math.Abs(z) > 1.96,
["isMinorOutlier"] = score <stats.Q1 - iqr * 1.5m ||
score> stats.Q3 + iqr * 1.5m,
["isMajorOutlier"] = score <stats.Q1 - iqr * 3m ||
score> stats.Q3 + iqr * 3m,
["minutesSubmittedBeforeDueDate"] = minutesSubmittedBeforeDueDate?.TotalMinutes
};
}
}
}
teacherPerformanceObj[user.Id.ToString()] = new JObject {
};
#endregion
}
else
{
#region CurrentUserIsStudent
if (!studentsObj.ContainsKey(user.Id.ToString()))
{
var lastLogin = api.StreamUserAuthenticationEvents(user.Id)
.Where(e => e.Event == EventType.Login)
.Select(e => e.CreatedAt)
.DefaultIfEmpty()
.MaxAsync();
studentsObj[user.Id.ToString()] = new JObject {
["sisId"] = user.SisUserId,
["fullName"] = user.Name,
["lastLogin"] = (await lastLogin).ToIso8601Date()
};
}
await foreach (var enrollment in api.StreamUserEnrollments(user.Id))
{
if (!coursesObj.ContainsKey(enrollment.CourseId.ToString()))
{
var course = await api.GetCourse(enrollment.CourseId);
coursesObj[course.Id.ToString()] = new JObject {
["sisId"] = course.SisCourseId,
["name"] = course.Name
};
}
}
#endregion
}
} catch (Exception e) {
Console.WriteLine($"Caught an exception while processing user id {user.Id}\n{e}\n-------\n");
}
}
document["dateCompleted"] = DateTime.Now.ToIso8601Date();
document["usersInReport"] = studentsObj.Count + teachersObj.Count;
var outPath = Path.Combine(home.NsDir, $"SuperReport_{started.Ticks}.json");
File.WriteAllText(outPath, document.ToString(Indented) + "\n");
Console.WriteLine($"Wrote report to {outPath}");
}
public static async Task Main(string[] args)
{
var home = new AppHome("export_attendance_columns_csv");
Console.WriteLine($"Using config path: {home.ConfigPath}");
if (!home.ConfigPresent())
{
Console.WriteLine("Need to generate a config file.");
home.CreateConfig(new DocumentSyntax {
Tables =
{
new TableSyntax("tokens")
{
Items =
{
{ "token", "PUT_TOKEN_HERE" }
}
},
new TableSyntax("options")
{
Items =
{
{ "include_hidden", false }
}
},
new TableSyntax("debug")
{
Items =
{
{ "limit_to", -1 }
}
}
}
});
Console.WriteLine("Created a new config file. Please go put in your token and input path.");
return;
}
Console.WriteLine("Found config file.");
var config = home.GetConfig();
Debug.Assert(config != null, nameof(config) + " != null");
var token = config.GetTable("tokens")
.Get <string>("token");
var includeHidden = config.GetTable("options")
.Get <bool>("include_hidden");
var courseLimit = config.GetTable("debug")
.Get <long>("limit_to");
var api = new Api(token, "https://uview.instructure.com/api/v1/");
if (courseLimit > 0)
{
Console.WriteLine($"[DEBUG] Limited to course id {courseLimit}");
}
var table = new StringBuilder("course_id,column_name,student_id,enrollment_id,column_value\n");
try {
var courses = courseLimit <= 0 ? api.StreamCourses()
: (await api.GetCourse(Convert.ToUInt64(courseLimit))).YieldAsync();
await foreach (var course in courses)
{
try {
var enrollments = api.StreamCourseEnrollments(course.Id, StudentEnrollment.Yield())
.ToDictionaryAsync(e => e.UserId);
await foreach (var col in api.StreamCustomGradebookColumns(course.Id, includeHidden))
{
await foreach (var datum in api.StreamColumnEntries(col.Id, course.Id))
{
(await enrollments).TryGetValue(datum.UserId, out var thisEnrollment);
table.Append($"{course.Id}," +
$"\"{col.Title}\"," +
$"{datum.UserId}," +
$"{thisEnrollment?.Id.ToString() ?? "NULL"}" +
$",\"{datum.Content}\"\n");
}
}
} catch (Exception e) {
Console.WriteLine($"Threw up during course {course.Id}:\n{e}\nContinuing onwards.");
}
}
var now = DateTime.Now;
var outPath = Path.Combine(home.NsDir, $"AttendanceColumns_{now.Year}_{now.Month}_{now.Day}.csv");
File.WriteAllText(outPath, table.ToString());
Console.WriteLine($"Wrote report to {outPath}");
} catch (Exception e) {
Console.WriteLine($"Threw up:\n{e}");
}
}
internal static async Task Truancy(string token, string outPath, TomlTable truancyConfig, bool shortInterval = false)
{
if (!Database.UseSis)
{
Console.WriteLine("Please enable SIS to run the Truancy report.");
return;
}
if (truancyConfig == null)
{
Console.WriteLine("Truancy config table is null.");
return;
}
var sisIdYear = truancyConfig.Get <string>("sis_id_year");
var subaccountsFilter = truancyConfig.Get <TomlArray>("subaccounts").Cast <string>().ToArray();
Console.WriteLine("Running Truancy...");
var api = new Api(token, "https://uview.instructure.com/api/v1/");
var sb = new StringBuilder("user_id,sis_id,last_access,first_name,last_name,grade,phone,cell,address," +
"city,state,zip,mother_name,father_name,guardian_name,mother_email,father_email,guardian_email," +
"mother_cell,father_cell,guardian_cell,dob,entry_date,gender,school," +
"district_of_residence,age,ethnicity,language,guardian_relationship,has_sped,has_504,failing_courses");
await using var enumerationDb = await Database.Connect();
await using var dataDb = await Database.Connect();
await foreach (var sis in enumerationDb.GetTruancyCheckStudents())
{
try {
var user = await api.StreamUsers(sis)
.FirstOrDefaultAsync(u => u.SisUserId == sis);
if (user == null)
{
Console.WriteLine($"Warning: User with sis `{sis}` does not seem to exist in Canvas.");
sb.Append($"\n?,{sis},indeterminate,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?");
continue;
}
var mostRecent = await api.StreamUserPageViews(user.Id)
.Where(pv => pv.Links?.RealUser == null ||
pv.Links.RealUser.Value == user.Id) // ignore masqueraded views
.Where(pv => pv.Url.Length > "https://".Length) // ignore weird blank requests
.Where(pv => !string.IsNullOrWhiteSpace(pv.UserAgent)) // ignore weird blank user agents
.FirstOrDefaultAsync();
if (!shortInterval)
{
if (mostRecent != default && mostRecent.CreatedAt.AddDays(8) >= DateTime.Now)
{
continue;
}
}
else
{
if (mostRecent != default &&
mostRecent.CreatedAt.AddDays(2) >= DateTime.Now ||
mostRecent.CreatedAt.AddDays(8) < DateTime.Now)
{
continue;
}
}
var failingCourses = new LinkedList <Course>();
await foreach (var e in api.StreamUserEnrollments(user.Id, StudentEnrollment.Yield()))
{
var course = await api.GetCourse(e.CourseId, includes : Api.IndividualLevelCourseIncludes.Term);
if (course.Term?.EndAt != null && course.Term.EndAt < DateTime.Now)
{
continue;
}
// Concluded enrollments are sometimes reported by the api as active, so we have to try our best
// to disregard "active" enrollments in courses from past years.
if ("active" != e.EnrollmentState)
{
continue;
}
// Disregard if the course is unpublished...
if ("available" != course.WorkflowState)
{
continue;
}
// Disregard if no SIS
if (string.IsNullOrWhiteSpace(course.SisCourseId))
{
continue;
}
// Disregard if the SIS doesn't follow the standard course format with the current year
var m = CourseSisIdPattern.Match(course.SisCourseId);
if (!m.Success || m.Groups["year"].Value != sisIdYear)
{
continue;
}
var subaccountNames = await GetAccountSet(api, await api.GetAccount(course.AccountId))
.ThenApply(s => s.Select(a => a?.Name));
if (!subaccountsFilter.Intersect(subaccountNames).Any())
{
continue;
}
var grade = e.Grades?.CurrentGrade;
var score = e.Grades?.CurrentScore;
if (string.IsNullOrWhiteSpace(grade) && string.IsNullOrWhiteSpace(score))
{
failingCourses.AddLast(course);
}
else if ("F" == grade)
{
failingCourses.AddLast(course);
}
else if (double.TryParse(score, out var nScore) && nScore <= 66)
{
failingCourses.AddLast(course);
}
}
if (!failingCourses.Any())
{
continue;
}
var dtStr = mostRecent?.CreatedAt.ToString("yyyy-MM-dd'T'HH':'mm':'ssK") ?? "never";
var data = await dataDb.GetTruancyInfo(sis);
var failingCourseNames = "\"" + string.Join(";", failingCourses.Select(c => c.Name).Distinct()) + "\"";
sb.Append($"\n{user.Id},{sis},{dtStr},{data.FirstName},{data.LastName},{data.Grade},{data.Phone}," +
$"{data.Cell},{data.Address},{data.City},{data.State},{data.Zip},{data.MotherName}," +
$"{data.FatherName},{data.GuardianName},{data.MotherEmail},{data.FatherEmail}," +
$"{data.GuardianEmail},{data.MotherCell},{data.FatherCell},{data.GuardianCell}," +
$"{data.DateOfBirth},{data.EntryDate},{data.Gender},{data.School}," +
$"{data.ResidenceDistrictName},{data.Age},{data.Ethnicity},{data.Language}," +
$"\"{data.GuardianRelationship}\",{data.HasSped},{data.Has504},{failingCourseNames}");
} catch (Exception e) {
Console.WriteLine($"Warning: exception during user with sis `{sis}`\n{e}");
}
}
File.WriteAllText(outPath, sb.ToString());
Console.WriteLine($"Wrote report to {outPath} at {DateTime.Now:HH':'mm':'ss}");
}
