public async Task <IActionResult> SelectCourses(int districtId, string orgSourceId, IEnumerable <string> SelectedCourses) { var repo = new DistrictRepo(db, districtId); var model = await repo.Lines <CsvCourse>() //.Where(c => JsonConvert.DeserializeObject<CsvCourse>(c.RawData).orgSourcedId == orgSourceId.ToString()) .Where(c => SelectedCourses.Contains(c.SourcedId)) .ToListAsync(); ViewBag.districtId = districtId; foreach (var course in model) { bool include = SelectedCourses.Contains(course.SourcedId); if (course.IncludeInSync == include) { continue; } course.IncludeInSync = include; course.Touch(); repo.PushLineHistory(course, isNewData: false); } await repo.Committer.Invoke(); return(RedirectToAction(nameof(SelectCourses), new { districtId, orgSourceId }).WithSuccess("Courses saved successfully")); }
private string GetEntityEndpoint(string entityType, DistrictRepo repo) { switch (entityType) { case "org": return(repo.District.LmsOrgEndPoint); case "course": return(repo.District.LmsCourseEndPoint); case "class": return(repo.District.LmsClassEndPoint); case "user": return(repo.District.LmsUserEndPoint); case "enrollment": return(repo.District.LmsEnrollmentEndPoint); case "academicsession": return(repo.District.LmsAcademicSessionEndPoint); default: throw new ArgumentOutOfRangeException(nameof(entityType), entityType, "An unknown entity was provided for which there is not endpoint."); } }
public IActionResult SelectCourses(int districtId, string orgSourceId) { var repo = new DistrictRepo(db, districtId); ViewBag.districtId = districtId; ViewBag.orgSourceId = orgSourceId; // TODO: Add config to determine how to pick up courses. // Direct Mapping: Get all courses that belong to this school //var model = repo.Lines<CsvCourse>().Where(c => // JsonConvert.DeserializeObject<CsvCourse>(c.RawData).orgSourcedId == orgSourceId).ToList(); // Indirect Mapping: Get classes that belong to this School, // then get the courses for them that match the sourceId. // then apply unique on courseSourcedId // TODO: Optimize the following var courseSourcedIds = repo.Lines <CsvClass>() .Where(c => JsonConvert.DeserializeObject <CsvClass>(c.RawData).schoolSourcedId == orgSourceId) .Select(c => JsonConvert.DeserializeObject <CsvClass>(c.RawData).courseSourcedId); var courses = repo.Lines <CsvCourse>() .Where(cr => courseSourcedIds.Contains(cr.SourcedId)) .OrderBy(cr => cr.SourcedId) .Distinct(); return(View(courses)); }
private async Task ScanForDistrictsToProcess(ApplicationDbContext db) { var now = DateTime.UtcNow; // First find districts scheduled and mark them "FullProcess" var districts = await db.Districts.Where(d => d.NextProcessingTime.HasValue && d.NextProcessingTime <= now).ToListAsync(); if (districts.Any()) { foreach (var d in districts) { d.ProcessingAction = ProcessingAction.FullProcess; } await db.SaveChangesAsync(); } // Now find any district that has any ProcessingAction districts = await db.Districts.Where(d => d.ProcessingAction != ProcessingAction.None).ToListAsync(); // walk the districts ready to be processed foreach (var district in districts) { var worker = new RosterProcessorWorker(district.DistrictId, Services, district.ProcessingAction); TaskQueue.QueueBackgroundWorkItem(async token => await worker.Invoke(token)); // clear the action out and reset the next processing time so it won't get picked up again district.ProcessingAction = ProcessingAction.None; DistrictRepo.UpdateNextProcessingTime(district); district.Touch(); await db.SaveChangesAsync(); } }
// GET: District public ActionResult Index() { DistrictRepo districtRepro = new DistrictRepo(); List <District> districtList = new List <District>(); districtList = districtRepro.GetAllDistrictInfo(); return(View(districtList)); }
public async Task <IViewComponentResult> InvokeAsync(int districtId) { var repo = new DistrictRepo(db, districtId); var model = await OneRosterSync.Net.Controllers.DataSyncController.ReportLine(repo.Lines(), ""); return(View(viewName: "Stats", model: model)); }
private void DestroyContext() { Repo = null; Db.Dispose(); Db = null; ServiceScope.Dispose(); ServiceScope = null; }
public async Task <IActionResult> SelectOrgs(int districtId) { var repo = new DistrictRepo(db, districtId); var model = await repo.Lines <CsvOrg>().ToListAsync(); ViewBag.districtId = districtId; return(View(model)); }
public async Task <IActionResult> DistrictDelete(int districtId) { var repo = new DistrictRepo(db, districtId); await repo.DeleteDistrict(); return(RedirectToAction(nameof(DistrictList)).WithSuccess("District deleted successfully")); }
public async Task <IActionResult> DistrictEdit(District postedDistrict) { if (!ModelState.IsValid) { return(View(postedDistrict)); } var district = await db.Districts.FindAsync(postedDistrict.DistrictId); if (postedDistrict.DailyProcessingTime.HasValue) { TimeSpan t = postedDistrict.DailyProcessingTime.Value; TimeSpan min = new TimeSpan(0); TimeSpan max = new TimeSpan(hours: 24, minutes: 0, seconds: 0); if (t < min || t > max) { ModelState.AddModelError( key: nameof(postedDistrict.DailyProcessingTime), errorMessage: "Invalid Daily Processing Time. Please enter a time between 0:0:0 and 23:59:59. Or clear to disable daily processing."); return(View(postedDistrict)); } } district.BasePath = postedDistrict.BasePath; district.DailyProcessingTime = postedDistrict.DailyProcessingTime; district.EmailsEachProcess = postedDistrict.EmailsEachProcess; district.EmailsOnChanges = postedDistrict.EmailsOnChanges; district.IsApprovalRequired = postedDistrict.IsApprovalRequired; district.LmsApiBaseUrl = postedDistrict.LmsApiBaseUrl; district.Name = postedDistrict.Name; district.TargetId = postedDistrict.TargetId; district.LmsApiAuthenticatorType = postedDistrict.LmsApiAuthenticatorType; district.LmsApiAuthenticationJsonData = postedDistrict.LmsApiAuthenticationJsonData; district.LmsOrgEndPoint = postedDistrict.LmsOrgEndPoint; district.LmsCourseEndPoint = postedDistrict.LmsCourseEndPoint; district.LmsClassEndPoint = postedDistrict.LmsClassEndPoint; district.LmsUserEndPoint = postedDistrict.LmsUserEndPoint; district.LmsEnrollmentEndPoint = postedDistrict.LmsEnrollmentEndPoint; district.LmsAcademicSessionEndPoint = postedDistrict.LmsAcademicSessionEndPoint; district.SyncEnrollment = postedDistrict.SyncEnrollment; district.SyncAcademicSessions = postedDistrict.SyncAcademicSessions; district.SyncClasses = postedDistrict.SyncClasses; district.SyncCourses = postedDistrict.SyncCourses; district.SyncOrgs = postedDistrict.SyncOrgs; district.SyncUsers = postedDistrict.SyncUsers; DistrictRepo.UpdateNextProcessingTime(district); district.Touch(); await db.SaveChangesAsync(); return(RedirectToDistrict(district.DistrictId)); }
public async Task <IViewComponentResult> InvokeAsync(int districtId, bool current) { var repo = new DistrictRepo(db, districtId); var model = await repo.DataSyncHistories .AsNoTracking() .OrderByDescending(h => h.Modified) .Take(20) .ToListAsync(); string view = current ? "CurrentHistory" : "ListOfHistories"; return(View(viewName: view, model: model)); }
private async Task ApplyLineParallel <T>(DataSyncLine line) where T : CsvBaseObject { // we need a new DataContext to avoid concurrency issues using (var scope = Services.CreateScope()) using (var db = scope.ServiceProvider.GetRequiredService <ApplicationDbContext>()) { // re-create the Repo and Data pulled from it var repo = new DistrictRepo(db, DistrictId); var newLine = await repo.Lines <T>().SingleAsync(l => l.DataSyncLineId == line.DataSyncLineId); await ApplyLine <T>(repo, newLine); await repo.Committer.Invoke(); } }
public ActionResult Delete(int id, District district) { try { DistrictRepo districtRepo = new DistrictRepo(); string msg = districtRepo.deleteDistrict(id); // TODO: Add delete logic here return(RedirectToAction("Index")); } catch { return(View()); } }
public async Task SeedAsync(DataSeedContext context) { var provinceE = new ProvinceEntity("Western"); var districtE = new DistrictEntity("Colombo", provinceE.Id); var cityE = new CityEntity(districtE.Id) { CityName = "Delkanda", Geolocation = "6.784568:79.546545" }; await ProvinceRepo.InsertAsync(provinceE); await DistrictRepo.InsertAsync(districtE); await CityRepo.InsertAsync(cityE); }
public async Task <IActionResult> DistrictReport(int districtId) { var repo = new DistrictRepo(db, districtId); ViewBag.districtId = districtId; var org = await ReportLine <CsvOrg>(repo); org.SyncEnabled = repo.District.SyncOrgs; var course = await ReportLine <CsvCourse>(repo); course.SyncEnabled = repo.District.SyncCourses; var academicSession = await ReportLine <CsvAcademicSession>(repo); academicSession.SyncEnabled = repo.District.SyncAcademicSessions; var _class = await ReportLine <CsvClass>(repo); _class.SyncEnabled = repo.District.SyncClasses; var user = await ReportLine <CsvUser>(repo); user.SyncEnabled = repo.District.SyncUsers; var enrollment = await ReportLine <CsvEnrollment>(repo); enrollment.SyncEnabled = repo.District.SyncEnrollment; var total = await ReportLine(repo.Lines().AsNoTracking(), "Totals"); total.SyncEnabled = true; var model = new[] { org, course, academicSession, _class, user, enrollment, total }; return(View(model)); }
public ActionResult Edit(int id, District district) { try { // TODO: Add update logic here DistrictRepo districtRepro = new DistrictRepo(); districtRepro.updateDistrictinfo(district); return(RedirectToAction("Index")); } catch { return(View()); } }
public ActionResult Create(District district) { try { DistrictRepo districtrepo = new DistrictRepo(); string msg = districtrepo.insertDistrict(district); // TODO: Add insert logic here return(RedirectToAction("Index")); } catch { return(View()); } }
public async Task <IActionResult> DataSyncLineEdit(DataSyncLine postedLine) { var repo = new DistrictRepo(db, postedLine.DistrictId); var line = await repo.Lines().SingleOrDefaultAsync(l => l.DataSyncLineId == postedLine.DataSyncLineId); // not currently editable //bool isNewData = line.RawData != postedLine.RawData; line.TargetId = postedLine.TargetId; line.IncludeInSync = postedLine.IncludeInSync; line.LoadStatus = postedLine.LoadStatus; line.SyncStatus = postedLine.SyncStatus; line.Touch(); repo.PushLineHistory(line, isNewData: false); await repo.Committer.Invoke(); return(RedirectToAction(nameof(DataSyncLineEdit), line.DataSyncLineId).WithSuccess("Dataline updated successfully")); }
/// <summary> /// Apply all records of a given entity type to the LMS /// </summary> public async Task ApplyLines <T>() where T : CsvBaseObject { for (int last = 0; ;) { using (var scope = Services.CreateScope()) using (var db = scope.ServiceProvider.GetRequiredService <ApplicationDbContext>()) { var repo = new DistrictRepo(db, DistrictId); // filter on all lines that are included and ready to be applied or apply was failed var lines = repo.Lines <T>().Where( l => l.IncludeInSync && (l.SyncStatus == SyncStatus.ReadyToApply || l.SyncStatus == SyncStatus.ApplyFailed)); // how many records are remaining to process? int curr = await lines.CountAsync(); if (curr == 0) { break; } // after each process, the remaining record count should go down // this avoids and infinite loop in case there is an problem processing // basically, we bail if no progress is made at all if (last > 0 && last <= curr) { throw new ProcessingException(Logger, "Apply failed to update SyncStatus of applied record. This indicates that some apply calls are failing and hence the apply process was aborted."); } last = curr; // process chunks of lines in parallel IEnumerable <Task> tasks = await lines .AsNoTracking() .Take(ParallelChunkSize) .Select(line => ApplyLineParallel <T>(line)) .ToListAsync(); await Task.WhenAll(tasks); } } }
public async Task <IActionResult> SelectOrgs(int districtId, IEnumerable <string> SelectedOrgs) { var repo = new DistrictRepo(db, districtId); var model = await repo.Lines <CsvOrg>().ToListAsync(); foreach (var org in model) { bool include = SelectedOrgs.Contains(org.SourcedId); if (org.IncludeInSync == include) { continue; } org.IncludeInSync = include; org.Touch(); repo.PushLineHistory(org, isNewData: false); } await repo.Committer.Invoke(); return(RedirectToAction(nameof(SelectOrgs), new { districtId }).WithSuccess("Orgs saved successfully")); }
public IActionResult DistrictClone(int districtId) { var repo = new DistrictRepo(db, districtId); var clonedDistrict = repo.District.ShallowCopy(); clonedDistrict.DistrictId = 0; clonedDistrict.Name = $"Clone of {clonedDistrict.Name}"; clonedDistrict.ProcessingStatus = ProcessingStatus.None; clonedDistrict.ProcessingAction = ProcessingAction.None; clonedDistrict.Created = DateTime.Now; clonedDistrict.Modified = DateTime.Now; db.Add(clonedDistrict); db.SaveChanges(); clonedDistrict.BasePath = $"CSVFiles/{clonedDistrict.DistrictId}"; db.SaveChanges(); return(RedirectToAction(nameof(DistrictList)).WithSuccess($"District cloned as {clonedDistrict.Name}")); }
public Loader(DistrictRepo repo, string basePath) { Repo = repo; BasePath = basePath; }
public async Task <IActionResult> DistrictEntityMapping(int districtId) { var repo = new DistrictRepo(db, districtId); return(View(repo.District)); }
private static async Task <DataSyncLineReportLine> ReportLine <T>(DistrictRepo repo) where T : CsvBaseObject { var lines = repo.Lines <T>().AsNoTracking(); return(await ReportLine(lines, typeof(T).Name)); }
public async Task <IActionResult> UploadMappingFiles(IFormFile mappingFile, int districtId, string tableName) { var path = Path.GetTempFileName(); var repo = new DistrictRepo(db, districtId); var mapCount = 0; var lineCount = 0; if (string.IsNullOrWhiteSpace(tableName) || mappingFile == null) { return(View(nameof(DistrictEntityMapping), repo.District) .WithDanger($"Please select Table Name and select a file first.")); } using (var stream = new FileStream(path, FileMode.Create)) { await mappingFile.CopyToAsync(stream); } using (var file = System.IO.File.OpenText(path)) { using (var csv = new CsvHelper.CsvReader(file)) { csv.Configuration.MissingFieldFound = null; csv.Configuration.HasHeaderRecord = true; csv.Read(); csv.ReadHeader(); for (int i = 0; await csv.ReadAsync(); i++) { dynamic record = null; try { record = csv.GetRecord <dynamic>(); string sourcedId = record.sourcedId; string targetId = record.targetId; mapCount++; var line = repo.Lines() .Where(l => l.SourcedId == sourcedId && l.Table == tableName) .FirstOrDefault(); if (line != null) { line.TargetId = targetId; line.Touch(); lineCount++; } } catch (Exception ex) { Logger.Here().LogError(ex, ex.Message); return(View(nameof(DistrictEntityMapping), repo.District) .WithDanger($"Failed to apply Mappings. {ex.Message}")); } } await repo.Committer.Invoke(); } } return(View(nameof(DistrictEntityMapping), repo.District) .WithSuccess($"Successfully Processed Mapping for {tableName}. Mapping applied to {lineCount} records out of {mapCount} mapping records.")); }
public async Task <IActionResult> DataSyncLines(int districtId, int page = 1, string table = null, string filter = null, LoadStatus?loadStatus = null, SyncStatus?syncStatus = null) { var repo = new DistrictRepo(db, districtId); if (repo.District == null) { return(NotFound($"District {districtId} not found")); } ViewData["DistrictName"] = repo.District.Name; var query = repo.Lines().AsNoTracking(); if (!string.IsNullOrEmpty(table)) { query = query.Where(l => l.Table == table); } if (!string.IsNullOrEmpty(filter)) { query = query.Where(l => l.SourcedId.Contains(filter) || l.TargetId.Contains(filter)); } if (loadStatus.HasValue) { query = query.Where(l => l.LoadStatus == loadStatus.Value); } if (syncStatus.HasValue) { query = query.Where(l => l.SyncStatus == syncStatus.Value); } var orderedQuery = query.OrderByDescending(l => l.LastSeen); var model = await PagingList.CreateAsync(orderedQuery, 10, page); model.Action = nameof(DataSyncLines); model.RouteValue = new RouteValueDictionary { { "districtId", districtId }, { "table", table }, { "filter", filter }, }; if (loadStatus.HasValue) { model.RouteValue["loadStatus"] = (int)loadStatus.Value; } if (syncStatus.HasValue) { model.RouteValue["syncStatus"] = (int)syncStatus.Value; } // kludge to remove empty values foreach (var kvp in model.RouteValue.Where(kvp => kvp.Value == null).ToList()) { model.RouteValue.Remove(kvp.Key); } return(View(model)); }
private async Task ApplyLine <T>(DistrictRepo repo, DataSyncLine line) where T : CsvBaseObject { switch (line.LoadStatus) { case LoadStatus.None: Logger.Here().LogWarning($"None should not be flagged for Sync: {line.RawData}"); return; } ApiPostBase data; var apiManager = new ApiManager(repo.District.LmsApiBaseUrl) { ApiAuthenticator = ApiAuthenticatorFactory.GetApiAuthenticator(repo.District.LmsApiAuthenticatorType, repo.District.LmsApiAuthenticationJsonData) }; if (line.Table == nameof(CsvEnrollment)) { var enrollment = new ApiEnrollmentPost(line.RawData); CsvEnrollment csvEnrollment = JsonConvert.DeserializeObject <CsvEnrollment>(line.RawData); DataSyncLine cls = repo.Lines <CsvClass>().SingleOrDefault(l => l.SourcedId == csvEnrollment.classSourcedId); DataSyncLine usr = repo.Lines <CsvUser>().SingleOrDefault(l => l.SourcedId == csvEnrollment.userSourcedId); var map = new EnrollmentMap { classTargetId = cls?.TargetId, userTargetId = usr?.TargetId, }; // this provides a mapping of LMS TargetIds (rather than sourcedId's) enrollment.EnrollmentMap = map; enrollment.ClassTargetId = cls?.TargetId; enrollment.UserTargetId = usr?.TargetId; // cache map in the database (for display/troubleshooting only) line.EnrollmentMap = JsonConvert.SerializeObject(map); data = enrollment; } else if (line.Table == nameof(CsvClass)) { var classCsv = JsonConvert.DeserializeObject <CsvClass>(line.RawData); // Get course & school of this class var course = repo.Lines <CsvCourse>().SingleOrDefault(l => l.SourcedId == classCsv.courseSourcedId); var courseCsv = JsonConvert.DeserializeObject <CsvCourse>(course.RawData); // Get Term of this class // TODO: Handle multiple terms, termSourceIds can be a comma separated list of terms. var term = repo.Lines <CsvAcademicSession>().SingleOrDefault(s => s.SourcedId == classCsv.termSourcedIds); var org = repo.Lines <CsvOrg>().SingleOrDefault(o => o.SourcedId == classCsv.schoolSourcedId); var _class = new ApiClassPost(line.RawData) { CourseTargetId = course.TargetId, SchoolTargetId = org.TargetId, TermTargetId = string.IsNullOrWhiteSpace(term.TargetId) ? "2018" : term.TargetId, //TODO: Add a default term setting in District Entity Period = classCsv.periods }; data = _class; } else { data = new ApiPost <T>(line.RawData); } data.DistrictId = repo.District.TargetId; data.DistrictName = repo.District.Name; data.LastSeen = line.LastSeen; data.SourcedId = line.SourcedId; data.TargetId = line.TargetId; data.Status = line.LoadStatus.ToString(); var response = await apiManager.Post(GetEntityEndpoint(data.EntityType.ToLower(), repo), data); if (response.Success) { line.SyncStatus = SyncStatus.Applied; if (!string.IsNullOrEmpty(response.TargetId)) { line.TargetId = response.TargetId; } line.Error = null; } else { line.SyncStatus = SyncStatus.ApplyFailed; line.Error = response.ErrorMessage; // The Lms can send false success if the entity already exist. In such a case we read the targetId if (!string.IsNullOrEmpty(response.TargetId)) { line.TargetId = response.TargetId; } } line.Touch(); repo.PushLineHistory(line, isNewData: false); }
public Analyzer(ILogger logger, DistrictRepo repo) { Repo = repo; }
private void CreateContext() { ServiceScope = Services.CreateScope(); Db = ServiceScope.ServiceProvider.GetRequiredService <ApplicationDbContext>(); Repo = new DistrictRepo(Db, DistrictId); }