private async Task ApplyLine <T>(DistrictRepo repo, DataSyncLine line) where T : CsvBaseObject
{
switch (line.LoadStatus)
{
case LoadStatus.None:
Logger.Here().LogWarning($"None should not be flagged for Sync: {line.RawData}");
return;
}
ApiPostBase data;
var apiManager = new ApiManager(repo.District.LmsApiBaseUrl)
{
ApiAuthenticator = ApiAuthenticatorFactory.GetApiAuthenticator(repo.District.LmsApiAuthenticatorType,
repo.District.LmsApiAuthenticationJsonData)
};
if (line.Table == nameof(CsvEnrollment))
{
var enrollment = new ApiEnrollmentPost(line.RawData);
CsvEnrollment csvEnrollment = JsonConvert.DeserializeObject <CsvEnrollment>(line.RawData);
DataSyncLine cls = repo.Lines <CsvClass>().SingleOrDefault(l => l.SourcedId == csvEnrollment.classSourcedId);
DataSyncLine usr = repo.Lines <CsvUser>().SingleOrDefault(l => l.SourcedId == csvEnrollment.userSourcedId);
var map = new EnrollmentMap
{
classTargetId = cls?.TargetId,
userTargetId = usr?.TargetId,
};
// this provides a mapping of LMS TargetIds (rather than sourcedId's)
enrollment.EnrollmentMap = map;
enrollment.ClassTargetId = cls?.TargetId;
enrollment.UserTargetId = usr?.TargetId;
// cache map in the database (for display/troubleshooting only)
line.EnrollmentMap = JsonConvert.SerializeObject(map);
data = enrollment;
}
else if (line.Table == nameof(CsvClass))
{
var classCsv = JsonConvert.DeserializeObject <CsvClass>(line.RawData);
// Get course & school of this class
var course = repo.Lines <CsvCourse>().SingleOrDefault(l => l.SourcedId == classCsv.courseSourcedId);
var courseCsv = JsonConvert.DeserializeObject <CsvCourse>(course.RawData);
// Get Term of this class
// TODO: Handle multiple terms, termSourceIds can be a comma separated list of terms.
var term = repo.Lines <CsvAcademicSession>().SingleOrDefault(s => s.SourcedId == classCsv.termSourcedIds);
var org = repo.Lines <CsvOrg>().SingleOrDefault(o => o.SourcedId == classCsv.schoolSourcedId);
var _class = new ApiClassPost(line.RawData)
{
CourseTargetId = course.TargetId,
SchoolTargetId = org.TargetId,
TermTargetId = string.IsNullOrWhiteSpace(term.TargetId) ? "2018" : term.TargetId, //TODO: Add a default term setting in District Entity
Period = classCsv.periods
};
data = _class;
}
else
{
data = new ApiPost <T>(line.RawData);
}
data.DistrictId = repo.District.TargetId;
data.DistrictName = repo.District.Name;
data.LastSeen = line.LastSeen;
data.SourcedId = line.SourcedId;
data.TargetId = line.TargetId;
data.Status = line.LoadStatus.ToString();
var response = await apiManager.Post(GetEntityEndpoint(data.EntityType.ToLower(), repo), data);
if (response.Success)
{
line.SyncStatus = SyncStatus.Applied;
if (!string.IsNullOrEmpty(response.TargetId))
{
line.TargetId = response.TargetId;
}
line.Error = null;
}
else
{
line.SyncStatus = SyncStatus.ApplyFailed;
line.Error = response.ErrorMessage;
// The Lms can send false success if the entity already exist. In such a case we read the targetId
if (!string.IsNullOrEmpty(response.TargetId))
{
line.TargetId = response.TargetId;
}
}
line.Touch();
repo.PushLineHistory(line, isNewData: false);
}
/// <summary>
/// Analyze the records to determine which should be included in the feed
/// based on dependencies.
/// </summary>
public async Task Analyze()
{
// load some small tables into memory for performance
var cache = new DataLineCache();
await cache.Load(Repo.Lines(), new[] { nameof(CsvOrg), nameof(CsvCourse), nameof(CsvClass) });
var orgIds = new List<string>();
// include Orgs that have been selected for sync
foreach (var org in cache.GetMap<CsvOrg>().Values.Where(IsUnappliedChange))
{
orgIds.Add(org.SourcedId);
IncludeReadyTouch(org);
}
await Repo.Committer.Invoke();
// courses are manually marked for sync, so choose only those
foreach (var course in cache.GetMap<CsvCourse>().Values.Where(l => l.IncludeInSync).Where(IsUnappliedChange))
IncludeReadyTouch(course);
await Repo.Committer.Invoke();
// now walk the classes and include those which map to an included course and are part of the selected orgs.
var classMap = cache.GetMap<CsvClass>();
var courseIds = cache.GetMap<CsvCourse>()
.Values
.Where(l => l.IncludeInSync)
.Select(l => l.SourcedId)
.ToHashSet();
foreach (var _class in classMap.Values.Where(IsUnappliedChangeWithoutIncludedInSync))
{
CsvClass csvClass = JsonConvert.DeserializeObject<CsvClass>(_class.RawData);
if (courseIds.Contains(csvClass.courseSourcedId) && orgIds.Contains(csvClass.schoolSourcedId))
IncludeReadyTouch(_class);
await Repo.Committer.InvokeIfChunk();
}
await Repo.Committer.InvokeIfAny();
// process enrollments in the database associated with the District based on the conditions below (in chunks of 200)
await Repo.Lines<CsvEnrollment>().ForEachInChunksAsync(chunkSize: 200,
action: async (enrollment) =>
{
CsvEnrollment csvEnrollment = JsonConvert.DeserializeObject<CsvEnrollment>(enrollment.RawData);
// figure out if we need to process this enrollment
if (!classMap.ContainsKey(csvEnrollment.classSourcedId) || // look up class associated with enrollment
!classMap[csvEnrollment.classSourcedId].IncludeInSync || // only include enrollment if the class is included
!IsUnappliedChangeWithoutIncludedInSync(enrollment)) // only include if unapplied change in enrollment
return;
var user = await Repo.Lines<CsvUser>().SingleOrDefaultAsync(l => l.SourcedId == csvEnrollment.userSourcedId);
if (user == null) // should never happen
{
enrollment.Error = $"Missing user for {csvEnrollment.userSourcedId}";
Logger.Here().LogError($"Missing user for enrollment for line {enrollment.DataSyncLineId}");
return;
}
// mark enrollment for sync
IncludeReadyTouch(enrollment);
// mark user for sync
//DataSyncLine user = userMap[csvEnrollment.userSourcedId];
if (IsUnappliedChangeWithoutIncludedInSync(user))
IncludeReadyTouch(user);
},
onChunkComplete: async () => await Repo.Committer.Invoke());
// now process any user changes we may have missed
await Repo.Lines<CsvUser>().Where(u => u.IncludeInSync
&& u.LoadStatus != LoadStatus.NoChange
&& u.SyncStatus != SyncStatus.ReadyToApply)
.ForEachInChunksForShrinkingList(chunkSize: 200,
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
action: async (user) => IncludeReadyTouch(user),
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
onChunkComplete: async () => await Repo.Committer.Invoke());
await Repo.Committer.Invoke();
}
