public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
Offset = o;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
Complete = true;
return(result.Value);
}
var observationPeriods =
BuildObservationPeriods(9999, ObservationPeriodsRaw.ToArray())
.ToArray();
if (Excluded(person, observationPeriods))
{
Complete = true;
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var visitDetails = BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
if (visitOccurrence.IdUndefined)
{
visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
visitOccurrence.IdUndefined = false;
}
if (!visitOccurrences.ContainsKey(visitOccurrence.Id))
{
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
var ecntrId = visitOccurrence.AdditionalFields["ecntr_id"];
if (long.TryParse(ecntrId, out var visitId))
{
if (!_visitIdMap.ContainsKey(visitId))
{
_visitIdMap.Add(visitId, new List <VisitOccurrence>());
}
_visitIdMap[visitId].Add(visitOccurrence);
}
}
foreach (var visitDetail in visitDetails)
{
var vo = GetVisitOccurrence(visitDetail);
visitDetail.VisitOccurrenceId = vo?.Id ?? 0;
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
foreach (var e in ConditionOccurrencesRaw)
{
SetVisitId(e);
}
foreach (var e in ProcedureOccurrencesRaw)
{
SetVisitId(e);
}
foreach (var e in ObservationsRaw)
{
SetVisitId(e);
}
foreach (var e in DrugExposuresRaw)
{
SetVisitId(e);
}
foreach (var e in MeasurementsRaw)
{
SetVisitId(e);
}
foreach (var e in DeviceExposureRaw)
{
SetVisitId(e);
}
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var co in conditionOccurrences)
{
co.Id = Offset.GetKeyOffset(co.PersonId).ConditionOccurrenceId;
}
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var procedureOccurrence in procedureOccurrences)
{
procedureOccurrence.Id =
Offset.GetKeyOffset(procedureOccurrence.PersonId).ProcedureOccurrenceId;
}
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var ob in observations)
{
ob.Id = Offset.GetKeyOffset(ob.PersonId).ObservationId;
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(visitOccurrences.Values.ToArray());
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
SetProviderIds(visitDetails);
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, null,
observationPeriods,
new PayerPlanPeriod[0],
drugExposures,
conditionOccurrences,
procedureOccurrences,
observations,
measurements,
visitOccurrences.Values.ToArray(), visitDetails, new Cohort[0], deviceExposure, new Note[0]);
Complete = true;
return(Attrition.None);
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
Offset = o;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
Complete = true;
return(result.Value);
}
var observationPeriods = new[]
{
new ObservationPeriod
{
PersonId = person.PersonId,
StartDate = DateTime.MinValue,
EndDate = DateTime.MaxValue
}
};
var visitDetails = BuildVisitDetails(VisitDetailsRaw.ToArray(), VisitOccurrencesRaw.ToArray(), observationPeriods).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
if (!visitOccurrence.EndDate.HasValue)
{
visitOccurrence.EndDate = visitOccurrence.StartDate;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var co in conditionOccurrences)
{
co.Id = Offset.GetKeyOffset(co.PersonId).ConditionOccurrenceId;
}
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var procedureOccurrence in procedureOccurrences)
{
procedureOccurrence.Id =
Offset.GetKeyOffset(procedureOccurrence.PersonId).ProcedureOccurrenceId;
}
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var ob in observations)
{
ob.Id = Offset.GetKeyOffset(ob.PersonId).ObservationId;
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
SetVisitDetailId(drugExposures);
SetVisitDetailId(conditionOccurrences);
SetVisitDetailId(procedureOccurrences);
SetVisitDetailId(measurements);
SetVisitDetailId(observations);
SetVisitDetailId(deviceExposure);
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(visitOccurrences.Values.ToArray());
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
SetProviderIds(visitDetails);
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, null,
new ObservationPeriod[0],
new PayerPlanPeriod[0],
drugExposures,
conditionOccurrences,
procedureOccurrences,
observations,
measurements,
visitOccurrences.Values.ToArray(), visitDetails, new Cohort[0], deviceExposure, new Note[0]);
Complete = true;
return(Attrition.None);
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
Offset = o;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray();
// Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH
if (Excluded(person, observationPeriods))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
if (visitOccurrence.IdUndefined)
{
visitOccurrence.Id =
Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
}
// HIX-1491 Cerner Year_of_birth should use earliest admission
if (visitOccurrences.Count > 0)
{
var minDate = visitOccurrences.Values.Min(vo => vo.StartDate);
if (person.YearOfBirth > minDate.Year)
{
person.YearOfBirth = minDate.Year;
}
}
if (person.YearOfBirth < 1900)
{
return(Attrition.ImplausibleYOBPast);
}
if (person.YearOfBirth > DateTime.Now.Year)
{
return(Attrition.ImplausibleYOBFuture);
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure =
BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
// set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities
SetPayerPlanPeriodId(payerPlanPeriods, drugExposures, procedureOccurrences,
visitOccurrences.Values.ToArray(),
deviceExposure);
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
var drugCosts = BuildDrugCosts(drugExposures).ToArray();
var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray();
var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray();
var devicCosts = BuildDeviceCosts(deviceExposure).ToArray();
var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray();
var notes = BuildNote(NoteRecords.ToArray(), visitOccurrences, observationPeriods).ToArray();
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, death, observationPeriods, payerPlanPeriods, drugExposures,
conditionOccurrences, procedureOccurrences, observations, measurements,
visitOccurrences.Values.ToArray(), null, cohort, deviceExposure, notes);
var pg = new PregnancyAlgorithm();
foreach (var pe in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
pe.Id = Offset.GetKeyOffset(pe.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(pe);
}
return(Attrition.None);
}
//private VisitOccurrence GetVisitOccurrence(IEntity ent)
//{
// if (_rawVisits.ContainsKey(ent.SourceRecordGuid))
// {
// var vo = _rawVisits[ent.SourceRecordGuid];
// if (vo.Id == 0 && _rawVisits.ContainsKey(vo.SourceRecordGuid) &&
// _rawVisits[vo.SourceRecordGuid].SourceRecordGuid != ent.SourceRecordGuid)
// {
// vo = _rawVisits[vo.SourceRecordGuid];
// }
// return vo;
// }
// return null;
//}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager om)
{
this.Offset = om;
this.ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray();
// Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH
if (Excluded(person, observationPeriods))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray();
var vDetails = BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(
VisitOccurrencesRaw
.Where(v => v.AdditionalFields["sort_index"] == "1" && v.AdditionalFields["ordinal"] == "1")
.ToArray(), observationPeriods))
{
if (!_rawVisits.ContainsKey(visitOccurrence.SourceRecordGuid))
{
_rawVisits.Add(visitOccurrence.SourceRecordGuid, visitOccurrence);
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
var visitDetails = new Dictionary <long, VisitDetail>();
foreach (var group in vDetails.GroupBy(d => d.Id))
{
var vd = group.First();
if (visitOccurrences.ContainsKey(vd.Id))
{
vd.VisitOccurrenceId = vd.Id;
vd.ProviderId = visitOccurrences[vd.Id].ProviderId;
}
else
{
var visits = visitOccurrences.Values.Where(vo =>
vo.TypeConceptId == vd.TypeConceptId && vo.StartDate >= vd.StartDate &&
vo.EndDate <= vo.EndDate).ToArray();
vd.VisitOccurrenceId = visits.Any() ? visits.First().Id : 0;
}
vd.SourceValue = $"visitid:{vd.VisitOccurrenceId};encid:{vd.AdditionalFields["encid"]}";
visitDetails.Add(vd.Id, vd);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
UpdateVisitOccurrenceIds(visitOccurrences, visitDetails);
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure =
BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
// set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities
SetPayerPlanPeriodId(payerPlanPeriods, drugExposures, procedureOccurrences,
visitOccurrences.Values.ToArray(),
deviceExposure);
var notes = BuildNote(NoteRecords.ToArray(), visitOccurrences, observationPeriods).ToArray();
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
//SetProviderIds(visitDetails);
SetProviderIds(drugExposures, visitOccurrences);
SetProviderIds(conditionOccurrences, visitOccurrences);
SetProviderIds(measurements, visitOccurrences);
SetProviderIds(procedureOccurrences, visitOccurrences);
SetProviderIds(deviceExposure, visitOccurrences);
SetProviderIds(observations, visitOccurrences);
SetProviderIds(notes, visitOccurrences);
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
var drugCosts = BuildDrugCosts(drugExposures).ToArray();
var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray();
var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray();
var devicCosts = BuildDeviceCosts(deviceExposure).ToArray();
var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray();
if (death != null)
{
death = CleanUpDeath(visitOccurrences.Values, death);
death = CleanUpDeath(drugExposures, death);
death = CleanUpDeath(conditionOccurrences, death);
death = CleanUpDeath(procedureOccurrences, death);
death = CleanUpDeath(measurements, death);
death = CleanUpDeath(observations, death);
death = CleanUpDeath(deviceExposure, death);
}
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, death, observationPeriods, payerPlanPeriods, drugExposures,
conditionOccurrences, procedureOccurrences, observations, measurements,
visitOccurrences.Values.ToArray(), visitDetails.Values.ToArray(), cohort, deviceExposure, notes);
var pg = new PregnancyAlgorithm.PregnancyAlgorithm();
foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
r.Id = KeyMasterOffsetManager.GetKeyOffset(r.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(r);
}
return(Attrition.None);
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
Offset = o;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
Complete = true;
return(result.Value);
}
var obs = ObservationPeriodsRaw.Where(op => op.TypeConceptId > 0).ToArray();
ObservationPeriod[] observationPeriods;
if (obs.Length == 0)
{
observationPeriods = new ObservationPeriod[0];
}
else
{
observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.Where(op => op.TypeConceptId > 0).ToArray())
.ToArray();
}
if (Excluded(person, observationPeriods))
{
Complete = true;
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var visitDetails = BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
if (!visitOccurrences.ContainsKey(visitOccurrence.Id))
{
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
}
foreach (var visitDetail in visitDetails)
{
var vo = GetVisitOccurrence(visitDetail);
visitDetail.VisitOccurrenceId = vo?.Id ?? 0;
if (visitDetail.VisitOccurrenceId.HasValue && !_visitDetails.ContainsKey(visitDetail.VisitOccurrenceId.Value))
{
_visitDetails.Add(visitDetail.VisitOccurrenceId.Value, new List <VisitDetail>());
}
_visitDetails[visitDetail.VisitOccurrenceId.Value].Add(visitDetail);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var co in conditionOccurrences)
{
co.Id = Offset.GetKeyOffset(co.PersonId).ConditionOccurrenceId;
}
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var procedureOccurrence in procedureOccurrences)
{
procedureOccurrence.Id =
Offset.GetKeyOffset(procedureOccurrence.PersonId).ProcedureOccurrenceId;
}
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var ob in observations)
{
ob.Id = Offset.GetKeyOffset(ob.PersonId).ObservationId;
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
Console.WriteLine($"PeronsId={person.PersonSourceValue};" +
$"cond={conditionOccurrences.Length};" +
$"drg={drugExposures.Length};" +
$"proc={procedureOccurrences.Length};" +
$"mes={measurements.Length};" +
$"obser={observations.Length};" +
$"dev={deviceExposure.Length};" +
$"vo={visitOccurrences.Values.Count};" +
$"vd={visitDetails.Length}");
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(visitOccurrences.Values.ToArray());
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
SetProviderIds(visitDetails);
SetVisitDetailId(drugExposures, visitOccurrences.Values.ToArray());
SetVisitDetailId(conditionOccurrences, visitOccurrences.Values.ToArray());
SetVisitDetailId(procedureOccurrences, visitOccurrences.Values.ToArray());
SetVisitDetailId(measurements, visitOccurrences.Values.ToArray());
SetVisitDetailId(observations, visitOccurrences.Values.ToArray());
SetVisitDetailId(deviceExposure, visitOccurrences.Values.ToArray());
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, null,
observationPeriods,
new PayerPlanPeriod[0],
drugExposures,
conditionOccurrences,
procedureOccurrences,
observations,
measurements,
visitOccurrences.Values.ToArray(), visitDetails, new Cohort[0], deviceExposure, new Note[0]);
Complete = true;
return(Attrition.None);
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
Offset = o;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var buildPerson = result.Key;
if (buildPerson == null)
{
return(result.Value);
}
_currentPerson = buildPerson;
var observationPeriods =
BuildObservationPeriods(buildPerson.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray();
int age;
int.TryParse(PersonRecords.ToArray()[0].AdditionalFields["age"], out age);
_personAge = age;
int ageday;
int.TryParse(PersonRecords.ToArray()[0].AdditionalFields["ageday"], out ageday);
if (age > 0)
{
buildPerson.YearOfBirth = observationPeriods[0].StartDate.Year - age;
}
else if (string.IsNullOrEmpty(PersonRecords.ToArray()[0].AdditionalFields["ageday"]) &&
PersonRecords.ToArray()[0].AdditionalFields["age_neonate"] == "1")
{
buildPerson.YearOfBirth = observationPeriods[0].StartDate.Year;
}
else if (PersonRecords.ToArray()[0].AdditionalFields["ageday"] == null &&
PersonRecords.ToArray()[0].AdditionalFields["age_neonate"] != "1" || ageday < 0)
{
var dateOfBirth = observationPeriods[0].StartDate.AddDays(-180);
buildPerson.YearOfBirth = dateOfBirth.Year;
}
else if (ageday >= 0)
{
var dateOfBirth = observationPeriods[0].StartDate.AddDays(ageday * -1);
buildPerson.YearOfBirth = dateOfBirth.Year;
buildPerson.DayOfBirth = dateOfBirth.Day;
buildPerson.MonthOfBirth = dateOfBirth.Month;
}
if (!buildPerson.YearOfBirth.HasValue || buildPerson.YearOfBirth < 1907)
{
return(Attrition.ImplausibleYOBPast); // HIX-1417
}
// Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH
if (Excluded(buildPerson, observationPeriods))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
if (visitOccurrence.IdUndefined)
{
visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure =
BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
var drugCosts = BuildDrugCosts(drugExposures).ToArray();
var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray();
var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray();
var devicCosts = BuildDeviceCosts(deviceExposure).ToArray();
foreach (var cost in visitCosts)
{
if (cost.TotalPaid == 0)
{
continue;
}
var cost52 = new Cost(cost.PersonId)
{
CostId = Offset.GetKeyOffset(cost.PersonId).VisitCostId,
CurrencyConceptId = 44818668,
TypeId = 5032,
Domain = "Visit",
EventId = cost.Id,
TotalCharge = cost.TotalPaid
};
ChunkData.AddCostData(cost52);
}
var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray();
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(buildPerson, death, observationPeriods, payerPlanPeriods, drugExposures,
conditionOccurrences, procedureOccurrences, observations, measurements,
visitOccurrences.Values.ToArray(), null, cohort, deviceExposure, new Note[0]);
var pg = new PregnancyAlgorithm();
foreach (var pe in pg.GetPregnancyEpisodes(Vocabulary, buildPerson, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == buildPerson.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == buildPerson.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == buildPerson.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == buildPerson.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == buildPerson.PersonId).ToArray()))
{
pe.Id = Offset.GetKeyOffset(pe.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(pe);
}
return(Attrition.None);
}
/// <summary>
/// Build person entity and all person related entities like: DrugExposures, ConditionOccurrences, ProcedureOccurrences... from raw data sets
/// </summary>
public override Attrition Build(ChunkData data, KeyMasterOffsetManager om)
{
Offset = om;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
if (!ObservationPeriodsRaw.Any(op => op.StartDate < op.EndDate))
{
return(Attrition.InvalidObservationTime);
}
var op = ObservationPeriodsRaw.Where(op =>
op.StartDate < op.EndDate &&
op.StartDate.Year >= person.YearOfBirth &&
op.EndDate.Value.Year >= person.YearOfBirth &&
op.StartDate.Year <= DateTime.Now.Year).ToArray();
if (op.Length == 0)
{
return(Attrition.InvalidObservationTime);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, op).ToArray();
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.Where(pp =>
pp.StartDate.Year >= person.YearOfBirth &&
pp.EndDate.Value.Year >= person.YearOfBirth &&
pp.StartDate.Year <= DateTime.Now.Year).ToArray(), null).ToArray();
var visitDetails = new Dictionary <long, VisitDetail>();
var visitDetIds = new List <long>();
foreach (var vd in BuildVisitDetails(null, VisitOccurrencesRaw.Where(vo =>
vo.StartDate.Year >= person.YearOfBirth &&
vo.EndDate.Value.Year >= person.YearOfBirth &&
vo.StartDate.Year <= DateTime.Now.Year &&
vo.EndDate.Value.Year <= DateTime.Now.Year).ToArray(), observationPeriods).ToArray())
{
if (person.MonthOfBirth.HasValue && vd.StartDate.Year < person.YearOfBirth.Value &&
vd.StartDate.Month < person.MonthOfBirth ||
vd.StartDate.Year < person.YearOfBirth.Value)
{
if (vd.StartDate.Year < person.YearOfBirth.Value)
{
if (DateTime.TryParse(person.AdditionalFields["frd"], out var frd))
{
vd.StartDate = frd;
vd.EndDate = frd;
}
else
{
continue;
}
}
}
if (visitDetails.ContainsKey(vd.Id))
{
continue;
}
visitDetails.Add(vd.Id, vd);
visitDetIds.Add(vd.Id);
}
long?prevVisitDetId = null;
foreach (var visitId in visitDetIds.OrderBy(v => v))
{
if (prevVisitDetId.HasValue)
{
visitDetails[visitId].PrecedingVisitDetailId = prevVisitDetId;
}
prevVisitDetId = visitId;
}
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var byStartDate in visitDetails.Values.GroupBy(v => v.StartDate))
{
var vd = byStartDate.First();
var providerId = byStartDate.Min(v => v.ProviderId);
var careSiteId = byStartDate.Min(v => v.CareSiteId);
var sourceValue = byStartDate.Min(v => v.SourceValue);
var visitOccurrenceId = byStartDate.Min(v => v.Id);
var visitOccurrence = new VisitOccurrence(vd)
{
//Id = Offset.GetKeyOffset(vd.PersonId).VisitOccurrenceId,
Id = visitOccurrenceId,
ProviderId = providerId,
CareSiteId = careSiteId,
SourceValue = sourceValue
};
foreach (var visitDetail in byStartDate)
{
visitDetail.VisitOccurrenceId = visitOccurrence.Id;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
SetVisitOccurrenceId(ConditionOccurrencesRaw, visitDetails);
SetVisitOccurrenceId(ProcedureOccurrencesRaw, visitDetails);
SetVisitOccurrenceId(DrugExposuresRaw, visitDetails);
SetVisitOccurrenceId(DeviceExposureRaw, visitDetails);
SetVisitOccurrenceId(ObservationsRaw, visitDetails);
SetVisitOccurrenceId(MeasurementsRaw, visitDetails);
var drugExposures = BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences = BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var procedureOccurrences = BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
if (death != null)
{
person.TimeOfDeath = death.StartDate;
if (death.StartDate < observationPeriods.Min(op => op.StartDate))
{
return(Attrition.UnacceptablePatientQuality);
}
if (death.StartDate.Year < person.YearOfBirth || death.StartDate.Year > DateTime.Now.Year)
{
death = null;
}
}
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person,
death,
observationPeriods,
payerPlanPeriods,
Clean(drugExposures, person).ToArray(),
Clean(conditionOccurrences, person).ToArray(),
Clean(procedureOccurrences, person).ToArray(),
Clean(observations, person).ToArray(),
Clean(measurements, person).ToArray(),
visitOccurrences.Values.ToArray(),
visitDetails.Values.ToArray(), null,
Clean(deviceExposure, person).ToArray(), null);
var pg = new PregnancyAlgorithm();
foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
r.Id = Offset.GetKeyOffset(r.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(r);
}
return(Attrition.None);
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager om)
{
this.Offset = om;
this.ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray();
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray();
var dedupVisits = VisitOccurrencesRaw.Where(v => v.AdditionalFields["sort_index"] == "1").ToArray();
var vDetails = BuildVisitDetails(null, dedupVisits, observationPeriods).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(dedupVisits, observationPeriods))
{
visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
visitOccurrence.TypeConceptId = 44818517;
if (!visitOccurrence.EndDate.HasValue)
{
visitOccurrence.EndDate = visitOccurrence.StartDate;
}
if (!_rawVisits.ContainsKey(visitOccurrence.SourceRecordGuid))
{
_rawVisits.Add(visitOccurrence.SourceRecordGuid, visitOccurrence);
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
var visitDetails = new Dictionary <long, VisitDetail>();
foreach (var group in vDetails.GroupBy(d => d.Id))
{
var vd = group.First();
var visitOccurrenceId = GetVisitOccurrenceId(vd);
if (visitOccurrenceId.HasValue && visitOccurrences.ContainsKey(visitOccurrenceId.Value))
{
vd.VisitOccurrenceId = visitOccurrenceId;
}
else
{
var visits = visitOccurrences.Values
.Where(vo => vd.StartDate.Date >= vo.StartDate.Date &&
vd.EndDate.Value.Date <= vo.EndDate.Value.Date).ToArray();
if (visits.Any())
{
vd.VisitOccurrenceId = visits.First().Id;
}
}
visitDetails.Add(vd.Id, vd);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure =
BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
// set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities
SetPayerPlanPeriodId(payerPlanPeriods, drugExposures, procedureOccurrences,
visitOccurrences.Values.ToArray(),
deviceExposure);
var notes = BuildNote(NoteRecords.ToArray(), visitOccurrences, observationPeriods).ToArray();
List <DateTime?> mins = new List <DateTime?>();
List <DateTime?> maxs = new List <DateTime?>();
mins.Add(GetMinDate(drugExposures));
mins.Add(GetMinDate(conditionOccurrences));
mins.Add(GetMinDate(procedureOccurrences));
mins.Add(GetMinDate(observations));
mins.Add(GetMinDate(deviceExposure));
mins.Add(GetMinDate(measurements));
mins.Add(GetMinDate(visitOccurrences.Values));
mins.Add(GetMinDate(visitDetails.Values));
maxs.Add(GetMaxDate(drugExposures));
maxs.Add(GetMaxDate(conditionOccurrences));
maxs.Add(GetMaxDate(procedureOccurrences));
maxs.Add(GetMaxDate(observations));
maxs.Add(GetMaxDate(deviceExposure));
maxs.Add(GetMaxDate(measurements));
maxs.Add(GetMaxDate(visitOccurrences.Values));
maxs.Add(GetMaxDate(visitDetails.Values));
var min = mins.Min();
var max = maxs.Max();
var observationPeriodsFinal = new List <ObservationPeriod>(1)
{
observationPeriods[0]
};
if (min.HasValue)
{
observationPeriodsFinal[0].StartDate = min.Value;
if (max.HasValue)
{
observationPeriodsFinal[0].EndDate = max;
}
else
{
observationPeriodsFinal[0].EndDate = min.Value;
}
}
// Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH
if (Excluded(person, observationPeriodsFinal))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
SetProviderIds(deviceExposure);
SetProviderIds(measurements);
SetProviderIds(visitOccurrences.Values);
SetProviderIds(visitDetails.Values);
SetVisitOccurrenceId(conditionOccurrences, visitDetails.Values.ToArray());
SetVisitOccurrenceId(procedureOccurrences, visitDetails.Values.ToArray());
SetVisitOccurrenceId(drugExposures, visitDetails.Values.ToArray());
SetVisitOccurrenceId(deviceExposure, visitDetails.Values.ToArray());
SetVisitOccurrenceId(observations, visitDetails.Values.ToArray());
SetVisitOccurrenceId(measurements, visitDetails.Values.ToArray());
//SetVisitOccurrenceId(visitDetails.Values, visitOccurrences.Values.ToArray());
SetProviderIds(drugExposures, visitOccurrences);
SetProviderIds(conditionOccurrences, visitOccurrences);
SetProviderIds(measurements, visitOccurrences);
SetProviderIds(procedureOccurrences, visitOccurrences);
SetProviderIds(deviceExposure, visitOccurrences);
SetProviderIds(observations, visitOccurrences);
//SetProviderIds(notes, visitOccurrences); TMP: NOTE
if (notes.Any())
{
if (visitOccurrences.Count > 0)
{
foreach (var e in notes.Where(e => !e.ProviderId.HasValue))
{
if (e.AdditionalFields != null && e.AdditionalFields.ContainsKey("encid") && !string.IsNullOrEmpty(e.AdditionalFields["encid"]))
{
var encid = e.AdditionalFields["encid"];
var vd = visitDetails.Values.FirstOrDefault(v => v.AdditionalFields["encid"] == encid);
if (vd == null)
{
continue;
}
e.VisitDetailId = vd.Id;
e.VisitOccurrenceId = vd.VisitOccurrenceId;
}
if (!e.VisitOccurrenceId.HasValue)
{
continue;
}
if (visitOccurrences.ContainsKey(e.VisitOccurrenceId.Value))
{
e.ProviderId = visitOccurrences[e.VisitOccurrenceId.Value].ProviderId;
}
}
}
}
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriodsFinal.ToArray());
var drugCosts = BuildDrugCosts(drugExposures).ToArray();
var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray();
var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray();
var devicCosts = BuildDeviceCosts(deviceExposure).ToArray();
var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriodsFinal.ToArray()).ToArray();
if (death != null)
{
death = CleanUpDeath(visitDetails.Values, death);
death = CleanUpDeath(visitOccurrences.Values, death);
death = CleanUpDeath(drugExposures, death);
death = CleanUpDeath(conditionOccurrences, death);
death = CleanUpDeath(procedureOccurrences, death);
death = CleanUpDeath(measurements, death);
death = CleanUpDeath(observations, death);
death = CleanUpDeath(deviceExposure, death);
}
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, death, observationPeriodsFinal.ToArray(), payerPlanPeriods, drugExposures,
conditionOccurrences, procedureOccurrences, observations, measurements,
visitOccurrences.Values.ToArray(), visitDetails.Values.ToArray(), cohort, deviceExposure, notes);
var pg = new PregnancyAlgorithm();
foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriodsFinal.ToArray(),
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
r.Id = Offset.GetKeyOffset(r.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(r);
}
return(Attrition.None);
}
