public IActionResult SelectCourses(int districtId, string orgSourceId)
{
var repo = new DistrictRepo(db, districtId);
ViewBag.districtId = districtId;
ViewBag.orgSourceId = orgSourceId;
// TODO: Add config to determine how to pick up courses.
// Direct Mapping: Get all courses that belong to this school
//var model = repo.Lines<CsvCourse>().Where(c =>
// JsonConvert.DeserializeObject<CsvCourse>(c.RawData).orgSourcedId == orgSourceId).ToList();
// Indirect Mapping: Get classes that belong to this School,
// then get the courses for them that match the sourceId.
// then apply unique on courseSourcedId
// TODO: Optimize the following
var courseSourcedIds = repo.Lines <CsvClass>()
.Where(c => JsonConvert.DeserializeObject <CsvClass>(c.RawData).schoolSourcedId == orgSourceId)
.Select(c => JsonConvert.DeserializeObject <CsvClass>(c.RawData).courseSourcedId);
var courses = repo.Lines <CsvCourse>()
.Where(cr => courseSourcedIds.Contains(cr.SourcedId))
.OrderBy(cr => cr.SourcedId)
.Distinct();
return(View(courses));
}
/// <summary>
/// Identifies records that were missing from the feed and marks them as Deleted
/// </summary>
public async Task MarkDeleted(DateTime start)
{
foreach (var line in await Repo.Lines().Where(l => l.LastSeen < start).ToListAsync())
{
line.LoadStatus = LoadStatus.Deleted;
await Repo.Committer.InvokeIfChunk();
}
await Repo.Committer.InvokeIfAny();
}
public async Task <IActionResult> SelectCourses(int districtId, string orgSourceId, IEnumerable <string> SelectedCourses)
{
var repo = new DistrictRepo(db, districtId);
var model = await repo.Lines <CsvCourse>()
//.Where(c => JsonConvert.DeserializeObject<CsvCourse>(c.RawData).orgSourcedId == orgSourceId.ToString())
.Where(c => SelectedCourses.Contains(c.SourcedId))
.ToListAsync();
ViewBag.districtId = districtId;
foreach (var course in model)
{
bool include = SelectedCourses.Contains(course.SourcedId);
if (course.IncludeInSync == include)
{
continue;
}
course.IncludeInSync = include;
course.Touch();
repo.PushLineHistory(course, isNewData: false);
}
await repo.Committer.Invoke();
return(RedirectToAction(nameof(SelectCourses), new { districtId, orgSourceId }).WithSuccess("Courses saved successfully"));
}
public async Task <IActionResult> SelectOrgs(int districtId)
{
var repo = new DistrictRepo(db, districtId);
var model = await repo.Lines <CsvOrg>().ToListAsync();
ViewBag.districtId = districtId;
return(View(model));
}
private async Task ApplyLineParallel <T>(DataSyncLine line) where T : CsvBaseObject
{
// we need a new DataContext to avoid concurrency issues
using (var scope = Services.CreateScope())
using (var db = scope.ServiceProvider.GetRequiredService <ApplicationDbContext>())
{
// re-create the Repo and Data pulled from it
var repo = new DistrictRepo(db, DistrictId);
var newLine = await repo.Lines <T>().SingleAsync(l => l.DataSyncLineId == line.DataSyncLineId);
await ApplyLine <T>(repo, newLine);
await repo.Committer.Invoke();
}
}
public async Task <IActionResult> DistrictReport(int districtId)
{
var repo = new DistrictRepo(db, districtId);
ViewBag.districtId = districtId;
var org = await ReportLine <CsvOrg>(repo);
org.SyncEnabled = repo.District.SyncOrgs;
var course = await ReportLine <CsvCourse>(repo);
course.SyncEnabled = repo.District.SyncCourses;
var academicSession = await ReportLine <CsvAcademicSession>(repo);
academicSession.SyncEnabled = repo.District.SyncAcademicSessions;
var _class = await ReportLine <CsvClass>(repo);
_class.SyncEnabled = repo.District.SyncClasses;
var user = await ReportLine <CsvUser>(repo);
user.SyncEnabled = repo.District.SyncUsers;
var enrollment = await ReportLine <CsvEnrollment>(repo);
enrollment.SyncEnabled = repo.District.SyncEnrollment;
var total = await ReportLine(repo.Lines().AsNoTracking(), "Totals");
total.SyncEnabled = true;
var model = new[]
{
org,
course,
academicSession,
_class,
user,
enrollment,
total
};
return(View(model));
}
public async Task <IActionResult> DataSyncLineEdit(DataSyncLine postedLine)
{
var repo = new DistrictRepo(db, postedLine.DistrictId);
var line = await repo.Lines().SingleOrDefaultAsync(l => l.DataSyncLineId == postedLine.DataSyncLineId);
// not currently editable
//bool isNewData = line.RawData != postedLine.RawData;
line.TargetId = postedLine.TargetId;
line.IncludeInSync = postedLine.IncludeInSync;
line.LoadStatus = postedLine.LoadStatus;
line.SyncStatus = postedLine.SyncStatus;
line.Touch();
repo.PushLineHistory(line, isNewData: false);
await repo.Committer.Invoke();
return(RedirectToAction(nameof(DataSyncLineEdit), line.DataSyncLineId).WithSuccess("Dataline updated successfully"));
}
/// <summary>
/// Apply all records of a given entity type to the LMS
/// </summary>
public async Task ApplyLines <T>() where T : CsvBaseObject
{
for (int last = 0; ;)
{
using (var scope = Services.CreateScope())
using (var db = scope.ServiceProvider.GetRequiredService <ApplicationDbContext>())
{
var repo = new DistrictRepo(db, DistrictId);
// filter on all lines that are included and ready to be applied or apply was failed
var lines = repo.Lines <T>().Where(
l => l.IncludeInSync && (l.SyncStatus == SyncStatus.ReadyToApply || l.SyncStatus == SyncStatus.ApplyFailed));
// how many records are remaining to process?
int curr = await lines.CountAsync();
if (curr == 0)
{
break;
}
// after each process, the remaining record count should go down
// this avoids and infinite loop in case there is an problem processing
// basically, we bail if no progress is made at all
if (last > 0 && last <= curr)
{
throw new ProcessingException(Logger, "Apply failed to update SyncStatus of applied record. This indicates that some apply calls are failing and hence the apply process was aborted.");
}
last = curr;
// process chunks of lines in parallel
IEnumerable <Task> tasks = await lines
.AsNoTracking()
.Take(ParallelChunkSize)
.Select(line => ApplyLineParallel <T>(line))
.ToListAsync();
await Task.WhenAll(tasks);
}
}
}
public async Task <IActionResult> SelectOrgs(int districtId, IEnumerable <string> SelectedOrgs)
{
var repo = new DistrictRepo(db, districtId);
var model = await repo.Lines <CsvOrg>().ToListAsync();
foreach (var org in model)
{
bool include = SelectedOrgs.Contains(org.SourcedId);
if (org.IncludeInSync == include)
{
continue;
}
org.IncludeInSync = include;
org.Touch();
repo.PushLineHistory(org, isNewData: false);
}
await repo.Committer.Invoke();
return(RedirectToAction(nameof(SelectOrgs), new { districtId }).WithSuccess("Orgs saved successfully"));
}
public async Task <IViewComponentResult> InvokeAsync(int districtId)
{
var repo = new DistrictRepo(db, districtId);
var model = await OneRosterSync.Net.Controllers.DataSyncController.ReportLine(repo.Lines(), "");
return(View(viewName: "Stats", model: model));
}
private async Task ApplyLine <T>(DistrictRepo repo, DataSyncLine line) where T : CsvBaseObject
{
switch (line.LoadStatus)
{
case LoadStatus.None:
Logger.Here().LogWarning($"None should not be flagged for Sync: {line.RawData}");
return;
}
ApiPostBase data;
var apiManager = new ApiManager(repo.District.LmsApiBaseUrl)
{
ApiAuthenticator = ApiAuthenticatorFactory.GetApiAuthenticator(repo.District.LmsApiAuthenticatorType,
repo.District.LmsApiAuthenticationJsonData)
};
if (line.Table == nameof(CsvEnrollment))
{
var enrollment = new ApiEnrollmentPost(line.RawData);
CsvEnrollment csvEnrollment = JsonConvert.DeserializeObject <CsvEnrollment>(line.RawData);
DataSyncLine cls = repo.Lines <CsvClass>().SingleOrDefault(l => l.SourcedId == csvEnrollment.classSourcedId);
DataSyncLine usr = repo.Lines <CsvUser>().SingleOrDefault(l => l.SourcedId == csvEnrollment.userSourcedId);
var map = new EnrollmentMap
{
classTargetId = cls?.TargetId,
userTargetId = usr?.TargetId,
};
// this provides a mapping of LMS TargetIds (rather than sourcedId's)
enrollment.EnrollmentMap = map;
enrollment.ClassTargetId = cls?.TargetId;
enrollment.UserTargetId = usr?.TargetId;
// cache map in the database (for display/troubleshooting only)
line.EnrollmentMap = JsonConvert.SerializeObject(map);
data = enrollment;
}
else if (line.Table == nameof(CsvClass))
{
var classCsv = JsonConvert.DeserializeObject <CsvClass>(line.RawData);
// Get course & school of this class
var course = repo.Lines <CsvCourse>().SingleOrDefault(l => l.SourcedId == classCsv.courseSourcedId);
var courseCsv = JsonConvert.DeserializeObject <CsvCourse>(course.RawData);
// Get Term of this class
// TODO: Handle multiple terms, termSourceIds can be a comma separated list of terms.
var term = repo.Lines <CsvAcademicSession>().SingleOrDefault(s => s.SourcedId == classCsv.termSourcedIds);
var org = repo.Lines <CsvOrg>().SingleOrDefault(o => o.SourcedId == classCsv.schoolSourcedId);
var _class = new ApiClassPost(line.RawData)
{
CourseTargetId = course.TargetId,
SchoolTargetId = org.TargetId,
TermTargetId = string.IsNullOrWhiteSpace(term.TargetId) ? "2018" : term.TargetId, //TODO: Add a default term setting in District Entity
Period = classCsv.periods
};
data = _class;
}
else
{
data = new ApiPost <T>(line.RawData);
}
data.DistrictId = repo.District.TargetId;
data.DistrictName = repo.District.Name;
data.LastSeen = line.LastSeen;
data.SourcedId = line.SourcedId;
data.TargetId = line.TargetId;
data.Status = line.LoadStatus.ToString();
var response = await apiManager.Post(GetEntityEndpoint(data.EntityType.ToLower(), repo), data);
if (response.Success)
{
line.SyncStatus = SyncStatus.Applied;
if (!string.IsNullOrEmpty(response.TargetId))
{
line.TargetId = response.TargetId;
}
line.Error = null;
}
else
{
line.SyncStatus = SyncStatus.ApplyFailed;
line.Error = response.ErrorMessage;
// The Lms can send false success if the entity already exist. In such a case we read the targetId
if (!string.IsNullOrEmpty(response.TargetId))
{
line.TargetId = response.TargetId;
}
}
line.Touch();
repo.PushLineHistory(line, isNewData: false);
}
private async Task <bool> ProcessRecord <T>(T record, string table, DateTime now) where T : CsvBaseObject
{
if (string.IsNullOrEmpty(record.sourcedId))
{
throw new ProcessingException(Logger.Here(),
$"Record of type {typeof(T).Name} contains no SourcedId: {JsonConvert.SerializeObject(record)}");
}
DataSyncLine line = await Repo.Lines <T>().SingleOrDefaultAsync(l => l.SourcedId == record.sourcedId);
bool isNewRecord = line == null;
Repo.CurrentHistory.NumRows++;
string data = JsonConvert.SerializeObject(record);
if (isNewRecord)
{
// already deleted
if (record.isDeleted)
{
Repo.CurrentHistory.NumDeleted++;
return(false);
}
Repo.CurrentHistory.NumAdded++;
line = new DataSyncLine
{
SourcedId = record.sourcedId,
DistrictId = Repo.DistrictId,
LoadStatus = LoadStatus.Added,
LastSeen = now,
Table = table,
};
Repo.AddLine(line);
}
else // existing record, check if it has changed
{
line.LastSeen = now;
line.Touch();
// no change to the data, skip!
if (line.RawData == data)
{
if (line.SyncStatus != SyncStatus.Loaded)
{
line.LoadStatus = LoadStatus.NoChange;
}
return(false);
}
// status should be deleted
if (record.isDeleted)
{
Repo.CurrentHistory.NumDeleted++;
line.LoadStatus = LoadStatus.Deleted;
}
else if (line.SyncStatus == SyncStatus.Loaded && line.LoadStatus == LoadStatus.Added)
{
Repo.CurrentHistory.NumAdded++;
line.LoadStatus = LoadStatus.Added; // if added, leave added
}
else
{
Repo.CurrentHistory.NumModified++;
line.LoadStatus = LoadStatus.Modified;
}
}
line.RawData = data;
line.SourcedId = record.sourcedId;
line.SyncStatus = SyncStatus.Loaded;
Repo.PushLineHistory(line, isNewData: true);
return(isNewRecord);
}
public async Task <IActionResult> UploadMappingFiles(IFormFile mappingFile, int districtId, string tableName)
{
var path = Path.GetTempFileName();
var repo = new DistrictRepo(db, districtId);
var mapCount = 0;
var lineCount = 0;
if (string.IsNullOrWhiteSpace(tableName) || mappingFile == null)
{
return(View(nameof(DistrictEntityMapping), repo.District)
.WithDanger($"Please select Table Name and select a file first."));
}
using (var stream = new FileStream(path, FileMode.Create))
{
await mappingFile.CopyToAsync(stream);
}
using (var file = System.IO.File.OpenText(path))
{
using (var csv = new CsvHelper.CsvReader(file))
{
csv.Configuration.MissingFieldFound = null;
csv.Configuration.HasHeaderRecord = true;
csv.Read();
csv.ReadHeader();
for (int i = 0; await csv.ReadAsync(); i++)
{
dynamic record = null;
try
{
record = csv.GetRecord <dynamic>();
string sourcedId = record.sourcedId;
string targetId = record.targetId;
mapCount++;
var line = repo.Lines()
.Where(l => l.SourcedId == sourcedId && l.Table == tableName)
.FirstOrDefault();
if (line != null)
{
line.TargetId = targetId;
line.Touch();
lineCount++;
}
}
catch (Exception ex)
{
Logger.Here().LogError(ex, ex.Message);
return(View(nameof(DistrictEntityMapping), repo.District)
.WithDanger($"Failed to apply Mappings. {ex.Message}"));
}
}
await repo.Committer.Invoke();
}
}
return(View(nameof(DistrictEntityMapping), repo.District)
.WithSuccess($"Successfully Processed Mapping for {tableName}. Mapping applied to {lineCount} records out of {mapCount} mapping records."));
}
private static async Task <DataSyncLineReportLine> ReportLine <T>(DistrictRepo repo) where T : CsvBaseObject
{
var lines = repo.Lines <T>().AsNoTracking();
return(await ReportLine(lines, typeof(T).Name));
}
public async Task <IActionResult> DataSyncLines(int districtId,
int page = 1, string table = null, string filter = null,
LoadStatus?loadStatus = null, SyncStatus?syncStatus = null)
{
var repo = new DistrictRepo(db, districtId);
if (repo.District == null)
{
return(NotFound($"District {districtId} not found"));
}
ViewData["DistrictName"] = repo.District.Name;
var query = repo.Lines().AsNoTracking();
if (!string.IsNullOrEmpty(table))
{
query = query.Where(l => l.Table == table);
}
if (!string.IsNullOrEmpty(filter))
{
query = query.Where(l => l.SourcedId.Contains(filter) || l.TargetId.Contains(filter));
}
if (loadStatus.HasValue)
{
query = query.Where(l => l.LoadStatus == loadStatus.Value);
}
if (syncStatus.HasValue)
{
query = query.Where(l => l.SyncStatus == syncStatus.Value);
}
var orderedQuery = query.OrderByDescending(l => l.LastSeen);
var model = await PagingList.CreateAsync(orderedQuery, 10, page);
model.Action = nameof(DataSyncLines);
model.RouteValue = new RouteValueDictionary
{
{ "districtId", districtId },
{ "table", table },
{ "filter", filter },
};
if (loadStatus.HasValue)
{
model.RouteValue["loadStatus"] = (int)loadStatus.Value;
}
if (syncStatus.HasValue)
{
model.RouteValue["syncStatus"] = (int)syncStatus.Value;
}
// kludge to remove empty values
foreach (var kvp in model.RouteValue.Where(kvp => kvp.Value == null).ToList())
{
model.RouteValue.Remove(kvp.Key);
}
return(View(model));
}
