/// <summary>
/// Helper to mark a record to be included in the next push to LMS
/// </summary>
private void IncludeReadyTouch(DataSyncLine line)
{
line.IncludeInSync = true;
line.SyncStatus = SyncStatus.ReadyToApply;
line.Touch();
Repo.PushLineHistory(line, isNewData: false);
}
public void PushLineHistory(DataSyncLine line, bool isNewData)
{
Db.DataSyncHistoryDetails.Add(new DataSyncHistoryDetail
{
DataSyncLineId = line.DataSyncLineId,
DataSyncHistoryId = CurrentHistory.DataSyncHistoryId,
DataNew = isNewData ? line.RawData : (string)null,
IncludeInSync = line.IncludeInSync,
LoadStatus = line.LoadStatus,
SyncStatus = line.SyncStatus,
Table = line.Table,
});
}
private async Task ApplyLineParallel <T>(DataSyncLine line) where T : CsvBaseObject
{
// we need a new DataContext to avoid concurrency issues
using (var scope = Services.CreateScope())
using (var db = scope.ServiceProvider.GetRequiredService <ApplicationDbContext>())
{
// re-create the Repo and Data pulled from it
var repo = new DistrictRepo(db, DistrictId);
var newLine = await repo.Lines <T>().SingleAsync(l => l.DataSyncLineId == line.DataSyncLineId);
await ApplyLine <T>(repo, newLine);
await repo.Committer.Invoke();
}
}
public async Task <IActionResult> DataSyncLineEdit(DataSyncLine postedLine)
{
var repo = new DistrictRepo(db, postedLine.DistrictId);
var line = await repo.Lines().SingleOrDefaultAsync(l => l.DataSyncLineId == postedLine.DataSyncLineId);
// not currently editable
//bool isNewData = line.RawData != postedLine.RawData;
line.TargetId = postedLine.TargetId;
line.IncludeInSync = postedLine.IncludeInSync;
line.LoadStatus = postedLine.LoadStatus;
line.SyncStatus = postedLine.SyncStatus;
line.Touch();
repo.PushLineHistory(line, isNewData: false);
await repo.Committer.Invoke();
return(RedirectToAction(nameof(DataSyncLineEdit), line.DataSyncLineId).WithSuccess("Dataline updated successfully"));
}
private async Task ApplyLine <T>(DistrictRepo repo, DataSyncLine line) where T : CsvBaseObject
{
switch (line.LoadStatus)
{
case LoadStatus.None:
Logger.Here().LogWarning($"None should not be flagged for Sync: {line.RawData}");
return;
}
ApiPostBase data;
var apiManager = new ApiManager(repo.District.LmsApiBaseUrl)
{
ApiAuthenticator = ApiAuthenticatorFactory.GetApiAuthenticator(repo.District.LmsApiAuthenticatorType,
repo.District.LmsApiAuthenticationJsonData)
};
if (line.Table == nameof(CsvEnrollment))
{
var enrollment = new ApiEnrollmentPost(line.RawData);
CsvEnrollment csvEnrollment = JsonConvert.DeserializeObject <CsvEnrollment>(line.RawData);
DataSyncLine cls = repo.Lines <CsvClass>().SingleOrDefault(l => l.SourcedId == csvEnrollment.classSourcedId);
DataSyncLine usr = repo.Lines <CsvUser>().SingleOrDefault(l => l.SourcedId == csvEnrollment.userSourcedId);
var map = new EnrollmentMap
{
classTargetId = cls?.TargetId,
userTargetId = usr?.TargetId,
};
// this provides a mapping of LMS TargetIds (rather than sourcedId's)
enrollment.EnrollmentMap = map;
enrollment.ClassTargetId = cls?.TargetId;
enrollment.UserTargetId = usr?.TargetId;
// cache map in the database (for display/troubleshooting only)
line.EnrollmentMap = JsonConvert.SerializeObject(map);
data = enrollment;
}
else if (line.Table == nameof(CsvClass))
{
var classCsv = JsonConvert.DeserializeObject <CsvClass>(line.RawData);
// Get course & school of this class
var course = repo.Lines <CsvCourse>().SingleOrDefault(l => l.SourcedId == classCsv.courseSourcedId);
var courseCsv = JsonConvert.DeserializeObject <CsvCourse>(course.RawData);
// Get Term of this class
// TODO: Handle multiple terms, termSourceIds can be a comma separated list of terms.
var term = repo.Lines <CsvAcademicSession>().SingleOrDefault(s => s.SourcedId == classCsv.termSourcedIds);
var org = repo.Lines <CsvOrg>().SingleOrDefault(o => o.SourcedId == classCsv.schoolSourcedId);
var _class = new ApiClassPost(line.RawData)
{
CourseTargetId = course.TargetId,
SchoolTargetId = org.TargetId,
TermTargetId = string.IsNullOrWhiteSpace(term.TargetId) ? "2018" : term.TargetId, //TODO: Add a default term setting in District Entity
Period = classCsv.periods
};
data = _class;
}
else
{
data = new ApiPost <T>(line.RawData);
}
data.DistrictId = repo.District.TargetId;
data.DistrictName = repo.District.Name;
data.LastSeen = line.LastSeen;
data.SourcedId = line.SourcedId;
data.TargetId = line.TargetId;
data.Status = line.LoadStatus.ToString();
var response = await apiManager.Post(GetEntityEndpoint(data.EntityType.ToLower(), repo), data);
if (response.Success)
{
line.SyncStatus = SyncStatus.Applied;
if (!string.IsNullOrEmpty(response.TargetId))
{
line.TargetId = response.TargetId;
}
line.Error = null;
}
else
{
line.SyncStatus = SyncStatus.ApplyFailed;
line.Error = response.ErrorMessage;
// The Lms can send false success if the entity already exist. In such a case we read the targetId
if (!string.IsNullOrEmpty(response.TargetId))
{
line.TargetId = response.TargetId;
}
}
line.Touch();
repo.PushLineHistory(line, isNewData: false);
}
private static bool IsUnappliedChangeWithoutIncludedInSync(DataSyncLine line) => (line.LoadStatus != LoadStatus.NoChange || line.SyncStatus != SyncStatus.Applied);
/// <summary> /// This is used to determine if any change needs to be pushed to the LMS and is included in sync. /// Basically if a record has changed OR has never been Applied /// which can happen if a record is loaded and later caused to be included in the Sync /// </summary> private static bool IsUnappliedChange(DataSyncLine line) => line.IncludeInSync && (line.LoadStatus != LoadStatus.NoChange || line.SyncStatus != SyncStatus.Applied);
private async Task <bool> ProcessRecord <T>(T record, string table, DateTime now) where T : CsvBaseObject
{
if (string.IsNullOrEmpty(record.sourcedId))
{
throw new ProcessingException(Logger.Here(),
$"Record of type {typeof(T).Name} contains no SourcedId: {JsonConvert.SerializeObject(record)}");
}
DataSyncLine line = await Repo.Lines <T>().SingleOrDefaultAsync(l => l.SourcedId == record.sourcedId);
bool isNewRecord = line == null;
Repo.CurrentHistory.NumRows++;
string data = JsonConvert.SerializeObject(record);
if (isNewRecord)
{
// already deleted
if (record.isDeleted)
{
Repo.CurrentHistory.NumDeleted++;
return(false);
}
Repo.CurrentHistory.NumAdded++;
line = new DataSyncLine
{
SourcedId = record.sourcedId,
DistrictId = Repo.DistrictId,
LoadStatus = LoadStatus.Added,
LastSeen = now,
Table = table,
};
Repo.AddLine(line);
}
else // existing record, check if it has changed
{
line.LastSeen = now;
line.Touch();
// no change to the data, skip!
if (line.RawData == data)
{
if (line.SyncStatus != SyncStatus.Loaded)
{
line.LoadStatus = LoadStatus.NoChange;
}
return(false);
}
// status should be deleted
if (record.isDeleted)
{
Repo.CurrentHistory.NumDeleted++;
line.LoadStatus = LoadStatus.Deleted;
}
else if (line.SyncStatus == SyncStatus.Loaded && line.LoadStatus == LoadStatus.Added)
{
Repo.CurrentHistory.NumAdded++;
line.LoadStatus = LoadStatus.Added; // if added, leave added
}
else
{
Repo.CurrentHistory.NumModified++;
line.LoadStatus = LoadStatus.Modified;
}
}
line.RawData = data;
line.SourcedId = record.sourcedId;
line.SyncStatus = SyncStatus.Loaded;
Repo.PushLineHistory(line, isNewData: true);
return(isNewRecord);
}
/// <summary>
/// Add a DataSyncLine to the db and associated with this District
/// </summary>
public void AddLine(DataSyncLine line)
{
line.DistrictId = DistrictId;
Db.DataSyncLines.Add(line);
}
