public IActionResult SelectCourses(int districtId, string orgSourceId) { var repo = new DistrictRepo(db, districtId); ViewBag.districtId = districtId; ViewBag.orgSourceId = orgSourceId; // TODO: Add config to determine how to pick up courses. // Direct Mapping: Get all courses that belong to this school //var model = repo.Lines<CsvCourse>().Where(c => // JsonConvert.DeserializeObject<CsvCourse>(c.RawData).orgSourcedId == orgSourceId).ToList(); // Indirect Mapping: Get classes that belong to this School, // then get the courses for them that match the sourceId. // then apply unique on courseSourcedId // TODO: Optimize the following var courseSourcedIds = repo.Lines <CsvClass>() .Where(c => JsonConvert.DeserializeObject <CsvClass>(c.RawData).schoolSourcedId == orgSourceId) .Select(c => JsonConvert.DeserializeObject <CsvClass>(c.RawData).courseSourcedId); var courses = repo.Lines <CsvCourse>() .Where(cr => courseSourcedIds.Contains(cr.SourcedId)) .OrderBy(cr => cr.SourcedId) .Distinct(); return(View(courses)); }
/// <summary> /// Identifies records that were missing from the feed and marks them as Deleted /// </summary> public async Task MarkDeleted(DateTime start) { foreach (var line in await Repo.Lines().Where(l => l.LastSeen < start).ToListAsync()) { line.LoadStatus = LoadStatus.Deleted; await Repo.Committer.InvokeIfChunk(); } await Repo.Committer.InvokeIfAny(); }
public async Task <IActionResult> SelectCourses(int districtId, string orgSourceId, IEnumerable <string> SelectedCourses) { var repo = new DistrictRepo(db, districtId); var model = await repo.Lines <CsvCourse>() //.Where(c => JsonConvert.DeserializeObject<CsvCourse>(c.RawData).orgSourcedId == orgSourceId.ToString()) .Where(c => SelectedCourses.Contains(c.SourcedId)) .ToListAsync(); ViewBag.districtId = districtId; foreach (var course in model) { bool include = SelectedCourses.Contains(course.SourcedId); if (course.IncludeInSync == include) { continue; } course.IncludeInSync = include; course.Touch(); repo.PushLineHistory(course, isNewData: false); } await repo.Committer.Invoke(); return(RedirectToAction(nameof(SelectCourses), new { districtId, orgSourceId }).WithSuccess("Courses saved successfully")); }
public async Task <IActionResult> SelectOrgs(int districtId) { var repo = new DistrictRepo(db, districtId); var model = await repo.Lines <CsvOrg>().ToListAsync(); ViewBag.districtId = districtId; return(View(model)); }
private async Task ApplyLineParallel <T>(DataSyncLine line) where T : CsvBaseObject { // we need a new DataContext to avoid concurrency issues using (var scope = Services.CreateScope()) using (var db = scope.ServiceProvider.GetRequiredService <ApplicationDbContext>()) { // re-create the Repo and Data pulled from it var repo = new DistrictRepo(db, DistrictId); var newLine = await repo.Lines <T>().SingleAsync(l => l.DataSyncLineId == line.DataSyncLineId); await ApplyLine <T>(repo, newLine); await repo.Committer.Invoke(); } }
public async Task <IActionResult> DistrictReport(int districtId) { var repo = new DistrictRepo(db, districtId); ViewBag.districtId = districtId; var org = await ReportLine <CsvOrg>(repo); org.SyncEnabled = repo.District.SyncOrgs; var course = await ReportLine <CsvCourse>(repo); course.SyncEnabled = repo.District.SyncCourses; var academicSession = await ReportLine <CsvAcademicSession>(repo); academicSession.SyncEnabled = repo.District.SyncAcademicSessions; var _class = await ReportLine <CsvClass>(repo); _class.SyncEnabled = repo.District.SyncClasses; var user = await ReportLine <CsvUser>(repo); user.SyncEnabled = repo.District.SyncUsers; var enrollment = await ReportLine <CsvEnrollment>(repo); enrollment.SyncEnabled = repo.District.SyncEnrollment; var total = await ReportLine(repo.Lines().AsNoTracking(), "Totals"); total.SyncEnabled = true; var model = new[] { org, course, academicSession, _class, user, enrollment, total }; return(View(model)); }
public async Task <IActionResult> DataSyncLineEdit(DataSyncLine postedLine) { var repo = new DistrictRepo(db, postedLine.DistrictId); var line = await repo.Lines().SingleOrDefaultAsync(l => l.DataSyncLineId == postedLine.DataSyncLineId); // not currently editable //bool isNewData = line.RawData != postedLine.RawData; line.TargetId = postedLine.TargetId; line.IncludeInSync = postedLine.IncludeInSync; line.LoadStatus = postedLine.LoadStatus; line.SyncStatus = postedLine.SyncStatus; line.Touch(); repo.PushLineHistory(line, isNewData: false); await repo.Committer.Invoke(); return(RedirectToAction(nameof(DataSyncLineEdit), line.DataSyncLineId).WithSuccess("Dataline updated successfully")); }
/// <summary> /// Apply all records of a given entity type to the LMS /// </summary> public async Task ApplyLines <T>() where T : CsvBaseObject { for (int last = 0; ;) { using (var scope = Services.CreateScope()) using (var db = scope.ServiceProvider.GetRequiredService <ApplicationDbContext>()) { var repo = new DistrictRepo(db, DistrictId); // filter on all lines that are included and ready to be applied or apply was failed var lines = repo.Lines <T>().Where( l => l.IncludeInSync && (l.SyncStatus == SyncStatus.ReadyToApply || l.SyncStatus == SyncStatus.ApplyFailed)); // how many records are remaining to process? int curr = await lines.CountAsync(); if (curr == 0) { break; } // after each process, the remaining record count should go down // this avoids and infinite loop in case there is an problem processing // basically, we bail if no progress is made at all if (last > 0 && last <= curr) { throw new ProcessingException(Logger, "Apply failed to update SyncStatus of applied record. This indicates that some apply calls are failing and hence the apply process was aborted."); } last = curr; // process chunks of lines in parallel IEnumerable <Task> tasks = await lines .AsNoTracking() .Take(ParallelChunkSize) .Select(line => ApplyLineParallel <T>(line)) .ToListAsync(); await Task.WhenAll(tasks); } } }
public async Task <IActionResult> SelectOrgs(int districtId, IEnumerable <string> SelectedOrgs) { var repo = new DistrictRepo(db, districtId); var model = await repo.Lines <CsvOrg>().ToListAsync(); foreach (var org in model) { bool include = SelectedOrgs.Contains(org.SourcedId); if (org.IncludeInSync == include) { continue; } org.IncludeInSync = include; org.Touch(); repo.PushLineHistory(org, isNewData: false); } await repo.Committer.Invoke(); return(RedirectToAction(nameof(SelectOrgs), new { districtId }).WithSuccess("Orgs saved successfully")); }
public async Task <IViewComponentResult> InvokeAsync(int districtId) { var repo = new DistrictRepo(db, districtId); var model = await OneRosterSync.Net.Controllers.DataSyncController.ReportLine(repo.Lines(), ""); return(View(viewName: "Stats", model: model)); }
private async Task ApplyLine <T>(DistrictRepo repo, DataSyncLine line) where T : CsvBaseObject { switch (line.LoadStatus) { case LoadStatus.None: Logger.Here().LogWarning($"None should not be flagged for Sync: {line.RawData}"); return; } ApiPostBase data; var apiManager = new ApiManager(repo.District.LmsApiBaseUrl) { ApiAuthenticator = ApiAuthenticatorFactory.GetApiAuthenticator(repo.District.LmsApiAuthenticatorType, repo.District.LmsApiAuthenticationJsonData) }; if (line.Table == nameof(CsvEnrollment)) { var enrollment = new ApiEnrollmentPost(line.RawData); CsvEnrollment csvEnrollment = JsonConvert.DeserializeObject <CsvEnrollment>(line.RawData); DataSyncLine cls = repo.Lines <CsvClass>().SingleOrDefault(l => l.SourcedId == csvEnrollment.classSourcedId); DataSyncLine usr = repo.Lines <CsvUser>().SingleOrDefault(l => l.SourcedId == csvEnrollment.userSourcedId); var map = new EnrollmentMap { classTargetId = cls?.TargetId, userTargetId = usr?.TargetId, }; // this provides a mapping of LMS TargetIds (rather than sourcedId's) enrollment.EnrollmentMap = map; enrollment.ClassTargetId = cls?.TargetId; enrollment.UserTargetId = usr?.TargetId; // cache map in the database (for display/troubleshooting only) line.EnrollmentMap = JsonConvert.SerializeObject(map); data = enrollment; } else if (line.Table == nameof(CsvClass)) { var classCsv = JsonConvert.DeserializeObject <CsvClass>(line.RawData); // Get course & school of this class var course = repo.Lines <CsvCourse>().SingleOrDefault(l => l.SourcedId == classCsv.courseSourcedId); var courseCsv = JsonConvert.DeserializeObject <CsvCourse>(course.RawData); // Get Term of this class // TODO: Handle multiple terms, termSourceIds can be a comma separated list of terms. var term = repo.Lines <CsvAcademicSession>().SingleOrDefault(s => s.SourcedId == classCsv.termSourcedIds); var org = repo.Lines <CsvOrg>().SingleOrDefault(o => o.SourcedId == classCsv.schoolSourcedId); var _class = new ApiClassPost(line.RawData) { CourseTargetId = course.TargetId, SchoolTargetId = org.TargetId, TermTargetId = string.IsNullOrWhiteSpace(term.TargetId) ? "2018" : term.TargetId, //TODO: Add a default term setting in District Entity Period = classCsv.periods }; data = _class; } else { data = new ApiPost <T>(line.RawData); } data.DistrictId = repo.District.TargetId; data.DistrictName = repo.District.Name; data.LastSeen = line.LastSeen; data.SourcedId = line.SourcedId; data.TargetId = line.TargetId; data.Status = line.LoadStatus.ToString(); var response = await apiManager.Post(GetEntityEndpoint(data.EntityType.ToLower(), repo), data); if (response.Success) { line.SyncStatus = SyncStatus.Applied; if (!string.IsNullOrEmpty(response.TargetId)) { line.TargetId = response.TargetId; } line.Error = null; } else { line.SyncStatus = SyncStatus.ApplyFailed; line.Error = response.ErrorMessage; // The Lms can send false success if the entity already exist. In such a case we read the targetId if (!string.IsNullOrEmpty(response.TargetId)) { line.TargetId = response.TargetId; } } line.Touch(); repo.PushLineHistory(line, isNewData: false); }
private async Task <bool> ProcessRecord <T>(T record, string table, DateTime now) where T : CsvBaseObject { if (string.IsNullOrEmpty(record.sourcedId)) { throw new ProcessingException(Logger.Here(), $"Record of type {typeof(T).Name} contains no SourcedId: {JsonConvert.SerializeObject(record)}"); } DataSyncLine line = await Repo.Lines <T>().SingleOrDefaultAsync(l => l.SourcedId == record.sourcedId); bool isNewRecord = line == null; Repo.CurrentHistory.NumRows++; string data = JsonConvert.SerializeObject(record); if (isNewRecord) { // already deleted if (record.isDeleted) { Repo.CurrentHistory.NumDeleted++; return(false); } Repo.CurrentHistory.NumAdded++; line = new DataSyncLine { SourcedId = record.sourcedId, DistrictId = Repo.DistrictId, LoadStatus = LoadStatus.Added, LastSeen = now, Table = table, }; Repo.AddLine(line); } else // existing record, check if it has changed { line.LastSeen = now; line.Touch(); // no change to the data, skip! if (line.RawData == data) { if (line.SyncStatus != SyncStatus.Loaded) { line.LoadStatus = LoadStatus.NoChange; } return(false); } // status should be deleted if (record.isDeleted) { Repo.CurrentHistory.NumDeleted++; line.LoadStatus = LoadStatus.Deleted; } else if (line.SyncStatus == SyncStatus.Loaded && line.LoadStatus == LoadStatus.Added) { Repo.CurrentHistory.NumAdded++; line.LoadStatus = LoadStatus.Added; // if added, leave added } else { Repo.CurrentHistory.NumModified++; line.LoadStatus = LoadStatus.Modified; } } line.RawData = data; line.SourcedId = record.sourcedId; line.SyncStatus = SyncStatus.Loaded; Repo.PushLineHistory(line, isNewData: true); return(isNewRecord); }
public async Task <IActionResult> UploadMappingFiles(IFormFile mappingFile, int districtId, string tableName) { var path = Path.GetTempFileName(); var repo = new DistrictRepo(db, districtId); var mapCount = 0; var lineCount = 0; if (string.IsNullOrWhiteSpace(tableName) || mappingFile == null) { return(View(nameof(DistrictEntityMapping), repo.District) .WithDanger($"Please select Table Name and select a file first.")); } using (var stream = new FileStream(path, FileMode.Create)) { await mappingFile.CopyToAsync(stream); } using (var file = System.IO.File.OpenText(path)) { using (var csv = new CsvHelper.CsvReader(file)) { csv.Configuration.MissingFieldFound = null; csv.Configuration.HasHeaderRecord = true; csv.Read(); csv.ReadHeader(); for (int i = 0; await csv.ReadAsync(); i++) { dynamic record = null; try { record = csv.GetRecord <dynamic>(); string sourcedId = record.sourcedId; string targetId = record.targetId; mapCount++; var line = repo.Lines() .Where(l => l.SourcedId == sourcedId && l.Table == tableName) .FirstOrDefault(); if (line != null) { line.TargetId = targetId; line.Touch(); lineCount++; } } catch (Exception ex) { Logger.Here().LogError(ex, ex.Message); return(View(nameof(DistrictEntityMapping), repo.District) .WithDanger($"Failed to apply Mappings. {ex.Message}")); } } await repo.Committer.Invoke(); } } return(View(nameof(DistrictEntityMapping), repo.District) .WithSuccess($"Successfully Processed Mapping for {tableName}. Mapping applied to {lineCount} records out of {mapCount} mapping records.")); }
private static async Task <DataSyncLineReportLine> ReportLine <T>(DistrictRepo repo) where T : CsvBaseObject { var lines = repo.Lines <T>().AsNoTracking(); return(await ReportLine(lines, typeof(T).Name)); }
public async Task <IActionResult> DataSyncLines(int districtId, int page = 1, string table = null, string filter = null, LoadStatus?loadStatus = null, SyncStatus?syncStatus = null) { var repo = new DistrictRepo(db, districtId); if (repo.District == null) { return(NotFound($"District {districtId} not found")); } ViewData["DistrictName"] = repo.District.Name; var query = repo.Lines().AsNoTracking(); if (!string.IsNullOrEmpty(table)) { query = query.Where(l => l.Table == table); } if (!string.IsNullOrEmpty(filter)) { query = query.Where(l => l.SourcedId.Contains(filter) || l.TargetId.Contains(filter)); } if (loadStatus.HasValue) { query = query.Where(l => l.LoadStatus == loadStatus.Value); } if (syncStatus.HasValue) { query = query.Where(l => l.SyncStatus == syncStatus.Value); } var orderedQuery = query.OrderByDescending(l => l.LastSeen); var model = await PagingList.CreateAsync(orderedQuery, 10, page); model.Action = nameof(DataSyncLines); model.RouteValue = new RouteValueDictionary { { "districtId", districtId }, { "table", table }, { "filter", filter }, }; if (loadStatus.HasValue) { model.RouteValue["loadStatus"] = (int)loadStatus.Value; } if (syncStatus.HasValue) { model.RouteValue["syncStatus"] = (int)syncStatus.Value; } // kludge to remove empty values foreach (var kvp in model.RouteValue.Where(kvp => kvp.Value == null).ToList()) { model.RouteValue.Remove(kvp.Key); } return(View(model)); }