public async Task <IActionResult> SelectCourses(int districtId, string orgSourceId, IEnumerable <string> SelectedCourses)
{
var repo = new DistrictRepo(db, districtId);
var model = await repo.Lines <CsvCourse>()
//.Where(c => JsonConvert.DeserializeObject<CsvCourse>(c.RawData).orgSourcedId == orgSourceId.ToString())
.Where(c => SelectedCourses.Contains(c.SourcedId))
.ToListAsync();
ViewBag.districtId = districtId;
foreach (var course in model)
{
bool include = SelectedCourses.Contains(course.SourcedId);
if (course.IncludeInSync == include)
{
continue;
}
course.IncludeInSync = include;
course.Touch();
repo.PushLineHistory(course, isNewData: false);
}
await repo.Committer.Invoke();
return(RedirectToAction(nameof(SelectCourses), new { districtId, orgSourceId }).WithSuccess("Courses saved successfully"));
}
private string GetEntityEndpoint(string entityType, DistrictRepo repo)
{
switch (entityType)
{
case "org":
return(repo.District.LmsOrgEndPoint);
case "course":
return(repo.District.LmsCourseEndPoint);
case "class":
return(repo.District.LmsClassEndPoint);
case "user":
return(repo.District.LmsUserEndPoint);
case "enrollment":
return(repo.District.LmsEnrollmentEndPoint);
case "academicsession":
return(repo.District.LmsAcademicSessionEndPoint);
default:
throw new ArgumentOutOfRangeException(nameof(entityType), entityType, "An unknown entity was provided for which there is not endpoint.");
}
}
public IActionResult SelectCourses(int districtId, string orgSourceId)
{
var repo = new DistrictRepo(db, districtId);
ViewBag.districtId = districtId;
ViewBag.orgSourceId = orgSourceId;
// TODO: Add config to determine how to pick up courses.
// Direct Mapping: Get all courses that belong to this school
//var model = repo.Lines<CsvCourse>().Where(c =>
// JsonConvert.DeserializeObject<CsvCourse>(c.RawData).orgSourcedId == orgSourceId).ToList();
// Indirect Mapping: Get classes that belong to this School,
// then get the courses for them that match the sourceId.
// then apply unique on courseSourcedId
// TODO: Optimize the following
var courseSourcedIds = repo.Lines <CsvClass>()
.Where(c => JsonConvert.DeserializeObject <CsvClass>(c.RawData).schoolSourcedId == orgSourceId)
.Select(c => JsonConvert.DeserializeObject <CsvClass>(c.RawData).courseSourcedId);
var courses = repo.Lines <CsvCourse>()
.Where(cr => courseSourcedIds.Contains(cr.SourcedId))
.OrderBy(cr => cr.SourcedId)
.Distinct();
return(View(courses));
}
private async Task ScanForDistrictsToProcess(ApplicationDbContext db)
{
var now = DateTime.UtcNow;
// First find districts scheduled and mark them "FullProcess"
var districts = await db.Districts.Where(d => d.NextProcessingTime.HasValue && d.NextProcessingTime <= now).ToListAsync();
if (districts.Any())
{
foreach (var d in districts)
{
d.ProcessingAction = ProcessingAction.FullProcess;
}
await db.SaveChangesAsync();
}
// Now find any district that has any ProcessingAction
districts = await db.Districts.Where(d => d.ProcessingAction != ProcessingAction.None).ToListAsync();
// walk the districts ready to be processed
foreach (var district in districts)
{
var worker = new RosterProcessorWorker(district.DistrictId, Services, district.ProcessingAction);
TaskQueue.QueueBackgroundWorkItem(async token => await worker.Invoke(token));
// clear the action out and reset the next processing time so it won't get picked up again
district.ProcessingAction = ProcessingAction.None;
DistrictRepo.UpdateNextProcessingTime(district);
district.Touch();
await db.SaveChangesAsync();
}
}
// GET: District
public ActionResult Index()
{
DistrictRepo districtRepro = new DistrictRepo();
List <District> districtList = new List <District>();
districtList = districtRepro.GetAllDistrictInfo();
return(View(districtList));
}
public async Task <IViewComponentResult> InvokeAsync(int districtId)
{
var repo = new DistrictRepo(db, districtId);
var model = await OneRosterSync.Net.Controllers.DataSyncController.ReportLine(repo.Lines(), "");
return(View(viewName: "Stats", model: model));
}
private void DestroyContext()
{
Repo = null;
Db.Dispose();
Db = null;
ServiceScope.Dispose();
ServiceScope = null;
}
public async Task <IActionResult> SelectOrgs(int districtId)
{
var repo = new DistrictRepo(db, districtId);
var model = await repo.Lines <CsvOrg>().ToListAsync();
ViewBag.districtId = districtId;
return(View(model));
}
public async Task <IActionResult> DistrictDelete(int districtId)
{
var repo = new DistrictRepo(db, districtId);
await repo.DeleteDistrict();
return(RedirectToAction(nameof(DistrictList)).WithSuccess("District deleted successfully"));
}
public async Task <IActionResult> DistrictEdit(District postedDistrict)
{
if (!ModelState.IsValid)
{
return(View(postedDistrict));
}
var district = await db.Districts.FindAsync(postedDistrict.DistrictId);
if (postedDistrict.DailyProcessingTime.HasValue)
{
TimeSpan t = postedDistrict.DailyProcessingTime.Value;
TimeSpan min = new TimeSpan(0);
TimeSpan max = new TimeSpan(hours: 24, minutes: 0, seconds: 0);
if (t < min || t > max)
{
ModelState.AddModelError(
key: nameof(postedDistrict.DailyProcessingTime),
errorMessage: "Invalid Daily Processing Time. Please enter a time between 0:0:0 and 23:59:59. Or clear to disable daily processing.");
return(View(postedDistrict));
}
}
district.BasePath = postedDistrict.BasePath;
district.DailyProcessingTime = postedDistrict.DailyProcessingTime;
district.EmailsEachProcess = postedDistrict.EmailsEachProcess;
district.EmailsOnChanges = postedDistrict.EmailsOnChanges;
district.IsApprovalRequired = postedDistrict.IsApprovalRequired;
district.LmsApiBaseUrl = postedDistrict.LmsApiBaseUrl;
district.Name = postedDistrict.Name;
district.TargetId = postedDistrict.TargetId;
district.LmsApiAuthenticatorType = postedDistrict.LmsApiAuthenticatorType;
district.LmsApiAuthenticationJsonData = postedDistrict.LmsApiAuthenticationJsonData;
district.LmsOrgEndPoint = postedDistrict.LmsOrgEndPoint;
district.LmsCourseEndPoint = postedDistrict.LmsCourseEndPoint;
district.LmsClassEndPoint = postedDistrict.LmsClassEndPoint;
district.LmsUserEndPoint = postedDistrict.LmsUserEndPoint;
district.LmsEnrollmentEndPoint = postedDistrict.LmsEnrollmentEndPoint;
district.LmsAcademicSessionEndPoint = postedDistrict.LmsAcademicSessionEndPoint;
district.SyncEnrollment = postedDistrict.SyncEnrollment;
district.SyncAcademicSessions = postedDistrict.SyncAcademicSessions;
district.SyncClasses = postedDistrict.SyncClasses;
district.SyncCourses = postedDistrict.SyncCourses;
district.SyncOrgs = postedDistrict.SyncOrgs;
district.SyncUsers = postedDistrict.SyncUsers;
DistrictRepo.UpdateNextProcessingTime(district);
district.Touch();
await db.SaveChangesAsync();
return(RedirectToDistrict(district.DistrictId));
}
public async Task <IViewComponentResult> InvokeAsync(int districtId, bool current)
{
var repo = new DistrictRepo(db, districtId);
var model = await repo.DataSyncHistories
.AsNoTracking()
.OrderByDescending(h => h.Modified)
.Take(20)
.ToListAsync();
string view = current ? "CurrentHistory" : "ListOfHistories";
return(View(viewName: view, model: model));
}
private async Task ApplyLineParallel <T>(DataSyncLine line) where T : CsvBaseObject
{
// we need a new DataContext to avoid concurrency issues
using (var scope = Services.CreateScope())
using (var db = scope.ServiceProvider.GetRequiredService <ApplicationDbContext>())
{
// re-create the Repo and Data pulled from it
var repo = new DistrictRepo(db, DistrictId);
var newLine = await repo.Lines <T>().SingleAsync(l => l.DataSyncLineId == line.DataSyncLineId);
await ApplyLine <T>(repo, newLine);
await repo.Committer.Invoke();
}
}
public ActionResult Delete(int id, District district)
{
try
{
DistrictRepo districtRepo = new DistrictRepo();
string msg = districtRepo.deleteDistrict(id);
// TODO: Add delete logic here
return(RedirectToAction("Index"));
}
catch
{
return(View());
}
}
public async Task SeedAsync(DataSeedContext context)
{
var provinceE = new ProvinceEntity("Western");
var districtE = new DistrictEntity("Colombo", provinceE.Id);
var cityE = new CityEntity(districtE.Id)
{
CityName = "Delkanda",
Geolocation = "6.784568:79.546545"
};
await ProvinceRepo.InsertAsync(provinceE);
await DistrictRepo.InsertAsync(districtE);
await CityRepo.InsertAsync(cityE);
}
public async Task <IActionResult> DistrictReport(int districtId)
{
var repo = new DistrictRepo(db, districtId);
ViewBag.districtId = districtId;
var org = await ReportLine <CsvOrg>(repo);
org.SyncEnabled = repo.District.SyncOrgs;
var course = await ReportLine <CsvCourse>(repo);
course.SyncEnabled = repo.District.SyncCourses;
var academicSession = await ReportLine <CsvAcademicSession>(repo);
academicSession.SyncEnabled = repo.District.SyncAcademicSessions;
var _class = await ReportLine <CsvClass>(repo);
_class.SyncEnabled = repo.District.SyncClasses;
var user = await ReportLine <CsvUser>(repo);
user.SyncEnabled = repo.District.SyncUsers;
var enrollment = await ReportLine <CsvEnrollment>(repo);
enrollment.SyncEnabled = repo.District.SyncEnrollment;
var total = await ReportLine(repo.Lines().AsNoTracking(), "Totals");
total.SyncEnabled = true;
var model = new[]
{
org,
course,
academicSession,
_class,
user,
enrollment,
total
};
return(View(model));
}
public ActionResult Edit(int id, District district)
{
try
{
// TODO: Add update logic here
DistrictRepo districtRepro = new DistrictRepo();
districtRepro.updateDistrictinfo(district);
return(RedirectToAction("Index"));
}
catch
{
return(View());
}
}
public ActionResult Create(District district)
{
try
{
DistrictRepo districtrepo = new DistrictRepo();
string msg = districtrepo.insertDistrict(district);
// TODO: Add insert logic here
return(RedirectToAction("Index"));
}
catch
{
return(View());
}
}
public async Task <IActionResult> DataSyncLineEdit(DataSyncLine postedLine)
{
var repo = new DistrictRepo(db, postedLine.DistrictId);
var line = await repo.Lines().SingleOrDefaultAsync(l => l.DataSyncLineId == postedLine.DataSyncLineId);
// not currently editable
//bool isNewData = line.RawData != postedLine.RawData;
line.TargetId = postedLine.TargetId;
line.IncludeInSync = postedLine.IncludeInSync;
line.LoadStatus = postedLine.LoadStatus;
line.SyncStatus = postedLine.SyncStatus;
line.Touch();
repo.PushLineHistory(line, isNewData: false);
await repo.Committer.Invoke();
return(RedirectToAction(nameof(DataSyncLineEdit), line.DataSyncLineId).WithSuccess("Dataline updated successfully"));
}
/// <summary>
/// Apply all records of a given entity type to the LMS
/// </summary>
public async Task ApplyLines <T>() where T : CsvBaseObject
{
for (int last = 0; ;)
{
using (var scope = Services.CreateScope())
using (var db = scope.ServiceProvider.GetRequiredService <ApplicationDbContext>())
{
var repo = new DistrictRepo(db, DistrictId);
// filter on all lines that are included and ready to be applied or apply was failed
var lines = repo.Lines <T>().Where(
l => l.IncludeInSync && (l.SyncStatus == SyncStatus.ReadyToApply || l.SyncStatus == SyncStatus.ApplyFailed));
// how many records are remaining to process?
int curr = await lines.CountAsync();
if (curr == 0)
{
break;
}
// after each process, the remaining record count should go down
// this avoids and infinite loop in case there is an problem processing
// basically, we bail if no progress is made at all
if (last > 0 && last <= curr)
{
throw new ProcessingException(Logger, "Apply failed to update SyncStatus of applied record. This indicates that some apply calls are failing and hence the apply process was aborted.");
}
last = curr;
// process chunks of lines in parallel
IEnumerable <Task> tasks = await lines
.AsNoTracking()
.Take(ParallelChunkSize)
.Select(line => ApplyLineParallel <T>(line))
.ToListAsync();
await Task.WhenAll(tasks);
}
}
}
public async Task <IActionResult> SelectOrgs(int districtId, IEnumerable <string> SelectedOrgs)
{
var repo = new DistrictRepo(db, districtId);
var model = await repo.Lines <CsvOrg>().ToListAsync();
foreach (var org in model)
{
bool include = SelectedOrgs.Contains(org.SourcedId);
if (org.IncludeInSync == include)
{
continue;
}
org.IncludeInSync = include;
org.Touch();
repo.PushLineHistory(org, isNewData: false);
}
await repo.Committer.Invoke();
return(RedirectToAction(nameof(SelectOrgs), new { districtId }).WithSuccess("Orgs saved successfully"));
}
public IActionResult DistrictClone(int districtId)
{
var repo = new DistrictRepo(db, districtId);
var clonedDistrict = repo.District.ShallowCopy();
clonedDistrict.DistrictId = 0;
clonedDistrict.Name = $"Clone of {clonedDistrict.Name}";
clonedDistrict.ProcessingStatus = ProcessingStatus.None;
clonedDistrict.ProcessingAction = ProcessingAction.None;
clonedDistrict.Created = DateTime.Now;
clonedDistrict.Modified = DateTime.Now;
db.Add(clonedDistrict);
db.SaveChanges();
clonedDistrict.BasePath = $"CSVFiles/{clonedDistrict.DistrictId}";
db.SaveChanges();
return(RedirectToAction(nameof(DistrictList)).WithSuccess($"District cloned as {clonedDistrict.Name}"));
}
public Loader(DistrictRepo repo, string basePath)
{
Repo = repo;
BasePath = basePath;
}
public async Task <IActionResult> DistrictEntityMapping(int districtId)
{
var repo = new DistrictRepo(db, districtId);
return(View(repo.District));
}
private static async Task <DataSyncLineReportLine> ReportLine <T>(DistrictRepo repo) where T : CsvBaseObject
{
var lines = repo.Lines <T>().AsNoTracking();
return(await ReportLine(lines, typeof(T).Name));
}
public async Task <IActionResult> UploadMappingFiles(IFormFile mappingFile, int districtId, string tableName)
{
var path = Path.GetTempFileName();
var repo = new DistrictRepo(db, districtId);
var mapCount = 0;
var lineCount = 0;
if (string.IsNullOrWhiteSpace(tableName) || mappingFile == null)
{
return(View(nameof(DistrictEntityMapping), repo.District)
.WithDanger($"Please select Table Name and select a file first."));
}
using (var stream = new FileStream(path, FileMode.Create))
{
await mappingFile.CopyToAsync(stream);
}
using (var file = System.IO.File.OpenText(path))
{
using (var csv = new CsvHelper.CsvReader(file))
{
csv.Configuration.MissingFieldFound = null;
csv.Configuration.HasHeaderRecord = true;
csv.Read();
csv.ReadHeader();
for (int i = 0; await csv.ReadAsync(); i++)
{
dynamic record = null;
try
{
record = csv.GetRecord <dynamic>();
string sourcedId = record.sourcedId;
string targetId = record.targetId;
mapCount++;
var line = repo.Lines()
.Where(l => l.SourcedId == sourcedId && l.Table == tableName)
.FirstOrDefault();
if (line != null)
{
line.TargetId = targetId;
line.Touch();
lineCount++;
}
}
catch (Exception ex)
{
Logger.Here().LogError(ex, ex.Message);
return(View(nameof(DistrictEntityMapping), repo.District)
.WithDanger($"Failed to apply Mappings. {ex.Message}"));
}
}
await repo.Committer.Invoke();
}
}
return(View(nameof(DistrictEntityMapping), repo.District)
.WithSuccess($"Successfully Processed Mapping for {tableName}. Mapping applied to {lineCount} records out of {mapCount} mapping records."));
}
public async Task <IActionResult> DataSyncLines(int districtId,
int page = 1, string table = null, string filter = null,
LoadStatus?loadStatus = null, SyncStatus?syncStatus = null)
{
var repo = new DistrictRepo(db, districtId);
if (repo.District == null)
{
return(NotFound($"District {districtId} not found"));
}
ViewData["DistrictName"] = repo.District.Name;
var query = repo.Lines().AsNoTracking();
if (!string.IsNullOrEmpty(table))
{
query = query.Where(l => l.Table == table);
}
if (!string.IsNullOrEmpty(filter))
{
query = query.Where(l => l.SourcedId.Contains(filter) || l.TargetId.Contains(filter));
}
if (loadStatus.HasValue)
{
query = query.Where(l => l.LoadStatus == loadStatus.Value);
}
if (syncStatus.HasValue)
{
query = query.Where(l => l.SyncStatus == syncStatus.Value);
}
var orderedQuery = query.OrderByDescending(l => l.LastSeen);
var model = await PagingList.CreateAsync(orderedQuery, 10, page);
model.Action = nameof(DataSyncLines);
model.RouteValue = new RouteValueDictionary
{
{ "districtId", districtId },
{ "table", table },
{ "filter", filter },
};
if (loadStatus.HasValue)
{
model.RouteValue["loadStatus"] = (int)loadStatus.Value;
}
if (syncStatus.HasValue)
{
model.RouteValue["syncStatus"] = (int)syncStatus.Value;
}
// kludge to remove empty values
foreach (var kvp in model.RouteValue.Where(kvp => kvp.Value == null).ToList())
{
model.RouteValue.Remove(kvp.Key);
}
return(View(model));
}
private async Task ApplyLine <T>(DistrictRepo repo, DataSyncLine line) where T : CsvBaseObject
{
switch (line.LoadStatus)
{
case LoadStatus.None:
Logger.Here().LogWarning($"None should not be flagged for Sync: {line.RawData}");
return;
}
ApiPostBase data;
var apiManager = new ApiManager(repo.District.LmsApiBaseUrl)
{
ApiAuthenticator = ApiAuthenticatorFactory.GetApiAuthenticator(repo.District.LmsApiAuthenticatorType,
repo.District.LmsApiAuthenticationJsonData)
};
if (line.Table == nameof(CsvEnrollment))
{
var enrollment = new ApiEnrollmentPost(line.RawData);
CsvEnrollment csvEnrollment = JsonConvert.DeserializeObject <CsvEnrollment>(line.RawData);
DataSyncLine cls = repo.Lines <CsvClass>().SingleOrDefault(l => l.SourcedId == csvEnrollment.classSourcedId);
DataSyncLine usr = repo.Lines <CsvUser>().SingleOrDefault(l => l.SourcedId == csvEnrollment.userSourcedId);
var map = new EnrollmentMap
{
classTargetId = cls?.TargetId,
userTargetId = usr?.TargetId,
};
// this provides a mapping of LMS TargetIds (rather than sourcedId's)
enrollment.EnrollmentMap = map;
enrollment.ClassTargetId = cls?.TargetId;
enrollment.UserTargetId = usr?.TargetId;
// cache map in the database (for display/troubleshooting only)
line.EnrollmentMap = JsonConvert.SerializeObject(map);
data = enrollment;
}
else if (line.Table == nameof(CsvClass))
{
var classCsv = JsonConvert.DeserializeObject <CsvClass>(line.RawData);
// Get course & school of this class
var course = repo.Lines <CsvCourse>().SingleOrDefault(l => l.SourcedId == classCsv.courseSourcedId);
var courseCsv = JsonConvert.DeserializeObject <CsvCourse>(course.RawData);
// Get Term of this class
// TODO: Handle multiple terms, termSourceIds can be a comma separated list of terms.
var term = repo.Lines <CsvAcademicSession>().SingleOrDefault(s => s.SourcedId == classCsv.termSourcedIds);
var org = repo.Lines <CsvOrg>().SingleOrDefault(o => o.SourcedId == classCsv.schoolSourcedId);
var _class = new ApiClassPost(line.RawData)
{
CourseTargetId = course.TargetId,
SchoolTargetId = org.TargetId,
TermTargetId = string.IsNullOrWhiteSpace(term.TargetId) ? "2018" : term.TargetId, //TODO: Add a default term setting in District Entity
Period = classCsv.periods
};
data = _class;
}
else
{
data = new ApiPost <T>(line.RawData);
}
data.DistrictId = repo.District.TargetId;
data.DistrictName = repo.District.Name;
data.LastSeen = line.LastSeen;
data.SourcedId = line.SourcedId;
data.TargetId = line.TargetId;
data.Status = line.LoadStatus.ToString();
var response = await apiManager.Post(GetEntityEndpoint(data.EntityType.ToLower(), repo), data);
if (response.Success)
{
line.SyncStatus = SyncStatus.Applied;
if (!string.IsNullOrEmpty(response.TargetId))
{
line.TargetId = response.TargetId;
}
line.Error = null;
}
else
{
line.SyncStatus = SyncStatus.ApplyFailed;
line.Error = response.ErrorMessage;
// The Lms can send false success if the entity already exist. In such a case we read the targetId
if (!string.IsNullOrEmpty(response.TargetId))
{
line.TargetId = response.TargetId;
}
}
line.Touch();
repo.PushLineHistory(line, isNewData: false);
}
public Analyzer(ILogger logger, DistrictRepo repo)
{
Repo = repo;
}
private void CreateContext()
{
ServiceScope = Services.CreateScope();
Db = ServiceScope.ServiceProvider.GetRequiredService <ApplicationDbContext>();
Repo = new DistrictRepo(Db, DistrictId);
}
