public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
Offset = o;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
Complete = true;
return(result.Value);
}
var observationPeriods =
BuildObservationPeriods(9999, ObservationPeriodsRaw.ToArray())
.ToArray();
if (Excluded(person, observationPeriods))
{
Complete = true;
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var visitDetails = BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
if (visitOccurrence.IdUndefined)
{
visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
visitOccurrence.IdUndefined = false;
}
if (!visitOccurrences.ContainsKey(visitOccurrence.Id))
{
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
var ecntrId = visitOccurrence.AdditionalFields["ecntr_id"];
if (long.TryParse(ecntrId, out var visitId))
{
if (!_visitIdMap.ContainsKey(visitId))
{
_visitIdMap.Add(visitId, new List <VisitOccurrence>());
}
_visitIdMap[visitId].Add(visitOccurrence);
}
}
foreach (var visitDetail in visitDetails)
{
var vo = GetVisitOccurrence(visitDetail);
visitDetail.VisitOccurrenceId = vo?.Id ?? 0;
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
foreach (var e in ConditionOccurrencesRaw)
{
SetVisitId(e);
}
foreach (var e in ProcedureOccurrencesRaw)
{
SetVisitId(e);
}
foreach (var e in ObservationsRaw)
{
SetVisitId(e);
}
foreach (var e in DrugExposuresRaw)
{
SetVisitId(e);
}
foreach (var e in MeasurementsRaw)
{
SetVisitId(e);
}
foreach (var e in DeviceExposureRaw)
{
SetVisitId(e);
}
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var co in conditionOccurrences)
{
co.Id = Offset.GetKeyOffset(co.PersonId).ConditionOccurrenceId;
}
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var procedureOccurrence in procedureOccurrences)
{
procedureOccurrence.Id =
Offset.GetKeyOffset(procedureOccurrence.PersonId).ProcedureOccurrenceId;
}
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var ob in observations)
{
ob.Id = Offset.GetKeyOffset(ob.PersonId).ObservationId;
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(visitOccurrences.Values.ToArray());
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
SetProviderIds(visitDetails);
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, null,
observationPeriods,
new PayerPlanPeriod[0],
drugExposures,
conditionOccurrences,
procedureOccurrences,
observations,
measurements,
visitOccurrences.Values.ToArray(), visitDetails, new Cohort[0], deviceExposure, new Note[0]);
Complete = true;
return(Attrition.None);
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
Offset = o;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
Complete = true;
return(result.Value);
}
var observationPeriods = new[]
{
new ObservationPeriod
{
PersonId = person.PersonId,
StartDate = DateTime.MinValue,
EndDate = DateTime.MaxValue
}
};
var visitDetails = BuildVisitDetails(VisitDetailsRaw.ToArray(), VisitOccurrencesRaw.ToArray(), observationPeriods).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
if (!visitOccurrence.EndDate.HasValue)
{
visitOccurrence.EndDate = visitOccurrence.StartDate;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var co in conditionOccurrences)
{
co.Id = Offset.GetKeyOffset(co.PersonId).ConditionOccurrenceId;
}
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var procedureOccurrence in procedureOccurrences)
{
procedureOccurrence.Id =
Offset.GetKeyOffset(procedureOccurrence.PersonId).ProcedureOccurrenceId;
}
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var ob in observations)
{
ob.Id = Offset.GetKeyOffset(ob.PersonId).ObservationId;
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
SetVisitDetailId(drugExposures);
SetVisitDetailId(conditionOccurrences);
SetVisitDetailId(procedureOccurrences);
SetVisitDetailId(measurements);
SetVisitDetailId(observations);
SetVisitDetailId(deviceExposure);
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(visitOccurrences.Values.ToArray());
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
SetProviderIds(visitDetails);
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, null,
new ObservationPeriod[0],
new PayerPlanPeriod[0],
drugExposures,
conditionOccurrences,
procedureOccurrences,
observations,
measurements,
visitOccurrences.Values.ToArray(), visitDetails, new Cohort[0], deviceExposure, new Note[0]);
Complete = true;
return(Attrition.None);
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
Offset = o;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray();
// Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH
if (Excluded(person, observationPeriods))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
if (visitOccurrence.IdUndefined)
{
visitOccurrence.Id =
Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
}
// HIX-1491 Cerner Year_of_birth should use earliest admission
if (visitOccurrences.Count > 0)
{
var minDate = visitOccurrences.Values.Min(vo => vo.StartDate);
if (person.YearOfBirth > minDate.Year)
{
person.YearOfBirth = minDate.Year;
}
}
if (person.YearOfBirth < 1900)
{
return(Attrition.ImplausibleYOBPast);
}
if (person.YearOfBirth > DateTime.Now.Year)
{
return(Attrition.ImplausibleYOBFuture);
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure =
BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
// set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities
SetPayerPlanPeriodId(payerPlanPeriods, drugExposures, procedureOccurrences,
visitOccurrences.Values.ToArray(),
deviceExposure);
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
var drugCosts = BuildDrugCosts(drugExposures).ToArray();
var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray();
var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray();
var devicCosts = BuildDeviceCosts(deviceExposure).ToArray();
var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray();
var notes = BuildNote(NoteRecords.ToArray(), visitOccurrences, observationPeriods).ToArray();
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, death, observationPeriods, payerPlanPeriods, drugExposures,
conditionOccurrences, procedureOccurrences, observations, measurements,
visitOccurrences.Values.ToArray(), null, cohort, deviceExposure, notes);
var pg = new PregnancyAlgorithm();
foreach (var pe in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
pe.Id = Offset.GetKeyOffset(pe.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(pe);
}
return(Attrition.None);
}
/// <summary>
/// Build person entity and all person related entities like: DrugExposures, ConditionOccurrences, ProcedureOccurrences... from raw data sets
/// </summary>
public override Attrition Build(ChunkData data, KeyMasterOffsetManager om)
{
this.Offset = om;
this.ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
if (!ObservationPeriodsRaw.Any(op => op.StartDate < op.EndDate))
{
return(Attrition.InvalidObservationTime);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap,
ObservationPeriodsRaw.Where(op => op.StartDate < op.EndDate).ToArray()).ToArray();
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray();
var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
// Build and clenaup visit occurrences entities
foreach (var visitOccurrence in CleanupVisits(
BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods), cohort,
observationPeriods))
{
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
var drugExposures = BuildDrugExposures(DrugExposuresRaw.Where(de => de.StartDate < DateTime.Now).ToArray(),
visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var conditionOccurrences =
Cleanup(
BuildConditionOccurrences(
ConditionOccurrencesRaw.Where(co => co.StartDate < DateTime.Now).ToArray(), visitOccurrences,
observationPeriods),
cohort).ToArray();
var procedureOccurrences =
Cleanup(
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods),
cohort).ToArray();
var observations =
Cleanup(BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods), cohort)
.ToArray();
var measurements =
Cleanup(BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods), cohort)
.ToArray();
// set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities
SetPayerPlanPeriodId(payerPlanPeriods, drugExposures, procedureOccurrences,
visitOccurrences.Values.ToArray(), new DeviceExposure[] { });
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
var drugCosts = BuildDrugCosts(drugExposures).ToArray();
var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray();
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, death,
observationPeriods,
payerPlanPeriods,
drugExposures,
CleanupCondition(conditionOccurrences).ToArray(),
procedureOccurrences,
CleanupObservations(observations, measurements, conditionOccurrences, procedureOccurrences).ToArray(),
measurements,
visitOccurrences.Values.ToArray(), null, cohort, deviceExposure, new Note[0]);
var pg = new PregnancyAlgorithm.PregnancyAlgorithm();
foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
r.Id = KeyMasterOffsetManager.GetKeyOffset(r.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(r);
}
return(Attrition.None);
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
this.Offset = o;
this.ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
if (person.YearOfBirth > DateTime.Now.Year)
{
return(Attrition.ImplausibleYOBFuture);
}
var op = ObservationPeriodsRaw.Where(op =>
op.StartDate.Year >= person.YearOfBirth &&
op.EndDate.Value.Year >= person.YearOfBirth &&
op.StartDate.Year <= DateTime.Now.Year).ToArray();
if (op.Length == 0)
{
return(Attrition.InvalidObservationTime);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, op).ToArray();
if (Excluded(person, observationPeriods))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var fisrtOP = observationPeriods.Min(op => op.StartDate);
if (fisrtOP.Year == person.YearOfBirth)
{
person.MonthOfBirth = fisrtOP.Month;
person.DayOfBirth = fisrtOP.Day;
}
else
{
person.MonthOfBirth = null;
person.DayOfBirth = null;
}
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.Where(pp =>
pp.StartDate.Year >= person.YearOfBirth &&
pp.EndDate.Value.Year >= person.YearOfBirth &&
pp.StartDate.Year <= DateTime.Now.Year).ToArray(), null).ToArray();
List <VisitDetail> visitDetails = new List <VisitDetail>();
foreach (var vd in BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).Where(vd =>
vd.StartDate.Year >= person.YearOfBirth &&
vd.EndDate.Value.Year >= person.YearOfBirth &&
vd.StartDate.Year <= DateTime.Now.Year))
{
if (vd.StartDate.Year < person.YearOfBirth || vd.StartDate.Year > DateTime.Now.Year)
{
continue;
}
if (vd.EndDate.HasValue && (
vd.EndDate.Value.Year < person.YearOfBirth ||
vd.EndDate.Value.Year > DateTime.Now.Year))
{
continue;
}
visitDetails.Add(vd);
}
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
if (visitOccurrence.StartDate.Year < person.YearOfBirth || visitOccurrence.StartDate.Year > DateTime.Now.Year)
{
continue;
}
if (visitOccurrence.EndDate.HasValue && (
visitOccurrence.EndDate.Value.Year < person.YearOfBirth ||
visitOccurrence.EndDate.Value.Year > DateTime.Now.Year))
{
continue;
}
//if (visitOccurrence.IdUndefined)
{
visitOccurrence.Id =
Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
visitOccurrence.IdUndefined = false;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
foreach (var visitDetail in visitDetails)
{
var patPlanid = visitDetail.AdditionalFields["pat_planid"];
var clmid = string.Empty;
var locCd = string.Empty;
if (visitDetail.AdditionalFields.ContainsKey("clmid"))
{
clmid = visitDetail.AdditionalFields["clmid"];
}
if (visitDetail.AdditionalFields.ContainsKey("loc_cd"))
{
locCd = visitDetail.AdditionalFields["loc_cd"];
}
if (!_rawVisitDetails.ContainsKey(patPlanid))
{
_rawVisitDetails.Add(patPlanid, new Dictionary <string, Dictionary <string, List <VisitDetail> > >());
}
if (!_rawVisitDetails[patPlanid].ContainsKey(clmid))
{
_rawVisitDetails[patPlanid].Add(clmid, new Dictionary <string, List <VisitDetail> >());
}
if (!_rawVisitDetails[patPlanid][clmid].ContainsKey(locCd))
{
_rawVisitDetails[patPlanid][clmid].Add(locCd, new List <VisitDetail>());
}
_rawVisitDetails[patPlanid][clmid][locCd].Add(visitDetail);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var co in conditionOccurrences)
{
co.Id = Offset.GetKeyOffset(co.PersonId).ConditionOccurrenceId;
}
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var procedureOccurrence in procedureOccurrences)
{
procedureOccurrence.Id = Offset.GetKeyOffset(procedureOccurrence.PersonId)
.ProcedureOccurrenceId;
}
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var drugExposures = BuildDrugExposures(FilteroutDrugClaims(DrugExposuresRaw).ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
if (death != null && person.YearOfBirth.HasValue && person.YearOfBirth.Value > 0 &&
person.YearOfBirth > death.StartDate.Year)
{
death = null;
}
if (death != null)
{
// DOD
if (death.TypeConceptId == 32519)
{
foreach (var visitOccurrence in visitOccurrences.Values.Where(v => (v.ConceptId == 9202 || v.ConceptId == 581458) && v.StartDate > death.StartDate.AddDays(30)))
{
visitOccurrence.ConceptId = -1;
}
if (visitOccurrences.Values.Any(v => (v.ConceptId == 9201 || v.ConceptId == 9203) && v.StartDate > death.StartDate.AddDays(30)))
{
death = null;
}
}
if (death != null)
{
if (death.StartDate.Year < person.YearOfBirth || death.StartDate.Year > DateTime.Now.Year)
{
death = null;
}
foreach (var payerPlanPeriod in payerPlanPeriods)
{
if (payerPlanPeriod.EndDate.Value == death.StartDate)
{
payerPlanPeriod.StopReasonConceptId = 352;
}
}
}
}
foreach (var visitDetail in visitDetails)
{
var vo = GetVisitOccurrence(visitDetail);
if (vo != null && visitOccurrences.ContainsKey(vo.Id))
{
if (visitOccurrences[vo.Id].ConceptId == -1)
{
visitDetail.ConceptId = -1;
}
else
{
visitDetail.VisitOccurrenceId = vo.Id;
}
}
else
{
visitDetail.ConceptId = -1;
}
}
SetVisitDetailId(drugExposures);
SetVisitDetailId(conditionOccurrences);
SetVisitDetailId(procedureOccurrences);
SetVisitDetailId(measurements);
SetVisitDetailId(observations);
SetVisitDetailId(deviceExposure);
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person,
death,
observationPeriods,
payerPlanPeriods,
Clean(drugExposures, person).ToArray(),
Clean(conditionOccurrences, person).ToArray(),
Clean(procedureOccurrences, person).ToArray(),
Clean(observations, person).ToArray(),
Clean(measurements, person).ToArray(),
visitOccurrences.Values.Where(v => v.ConceptId >= 0).ToArray(),
visitDetails.Where(v => v.ConceptId >= 0).ToArray(), new Cohort[0],
Clean(deviceExposure, person).ToArray(),
new Note[0]);
var pg = new PregnancyAlgorithm();
foreach (var episode in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
episode.Id = Offset.GetKeyOffset(episode.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(episode);
if (episode.ConceptId == 433260 && _potentialChilds.Count > 0)
{
foreach (var child in _potentialChilds)
{
var childId = child.Key;
foreach (var birthdate in child.Value)
{
if (episode.EndDate.Value.Between(birthdate.AddDays(-60), birthdate.AddDays(60)))
{
//40485452 Child of subject
//40478925 Mother of subject
ChunkData.FactRelationships.Add(new FactRelationship
{
DomainConceptId1 = 56,
DomainConceptId2 = 56,
FactId1 = episode.PersonId,
FactId2 = childId,
RelationshipConceptId = 40478925
});
break;
}
}
}
}
}
return(Attrition.None);
}
//private VisitOccurrence GetVisitOccurrence(IEntity ent)
//{
// if (_rawVisits.ContainsKey(ent.SourceRecordGuid))
// {
// var vo = _rawVisits[ent.SourceRecordGuid];
// if (vo.Id == 0 && _rawVisits.ContainsKey(vo.SourceRecordGuid) &&
// _rawVisits[vo.SourceRecordGuid].SourceRecordGuid != ent.SourceRecordGuid)
// {
// vo = _rawVisits[vo.SourceRecordGuid];
// }
// return vo;
// }
// return null;
//}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager om)
{
this.Offset = om;
this.ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray();
// Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH
if (Excluded(person, observationPeriods))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray();
var vDetails = BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(
VisitOccurrencesRaw
.Where(v => v.AdditionalFields["sort_index"] == "1" && v.AdditionalFields["ordinal"] == "1")
.ToArray(), observationPeriods))
{
if (!_rawVisits.ContainsKey(visitOccurrence.SourceRecordGuid))
{
_rawVisits.Add(visitOccurrence.SourceRecordGuid, visitOccurrence);
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
var visitDetails = new Dictionary <long, VisitDetail>();
foreach (var group in vDetails.GroupBy(d => d.Id))
{
var vd = group.First();
if (visitOccurrences.ContainsKey(vd.Id))
{
vd.VisitOccurrenceId = vd.Id;
vd.ProviderId = visitOccurrences[vd.Id].ProviderId;
}
else
{
var visits = visitOccurrences.Values.Where(vo =>
vo.TypeConceptId == vd.TypeConceptId && vo.StartDate >= vd.StartDate &&
vo.EndDate <= vo.EndDate).ToArray();
vd.VisitOccurrenceId = visits.Any() ? visits.First().Id : 0;
}
vd.SourceValue = $"visitid:{vd.VisitOccurrenceId};encid:{vd.AdditionalFields["encid"]}";
visitDetails.Add(vd.Id, vd);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
UpdateVisitOccurrenceIds(visitOccurrences, visitDetails);
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure =
BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
// set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities
SetPayerPlanPeriodId(payerPlanPeriods, drugExposures, procedureOccurrences,
visitOccurrences.Values.ToArray(),
deviceExposure);
var notes = BuildNote(NoteRecords.ToArray(), visitOccurrences, observationPeriods).ToArray();
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
//SetProviderIds(visitDetails);
SetProviderIds(drugExposures, visitOccurrences);
SetProviderIds(conditionOccurrences, visitOccurrences);
SetProviderIds(measurements, visitOccurrences);
SetProviderIds(procedureOccurrences, visitOccurrences);
SetProviderIds(deviceExposure, visitOccurrences);
SetProviderIds(observations, visitOccurrences);
SetProviderIds(notes, visitOccurrences);
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
var drugCosts = BuildDrugCosts(drugExposures).ToArray();
var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray();
var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray();
var devicCosts = BuildDeviceCosts(deviceExposure).ToArray();
var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray();
if (death != null)
{
death = CleanUpDeath(visitOccurrences.Values, death);
death = CleanUpDeath(drugExposures, death);
death = CleanUpDeath(conditionOccurrences, death);
death = CleanUpDeath(procedureOccurrences, death);
death = CleanUpDeath(measurements, death);
death = CleanUpDeath(observations, death);
death = CleanUpDeath(deviceExposure, death);
}
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, death, observationPeriods, payerPlanPeriods, drugExposures,
conditionOccurrences, procedureOccurrences, observations, measurements,
visitOccurrences.Values.ToArray(), visitDetails.Values.ToArray(), cohort, deviceExposure, notes);
var pg = new PregnancyAlgorithm.PregnancyAlgorithm();
foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
r.Id = KeyMasterOffsetManager.GetKeyOffset(r.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(r);
}
return(Attrition.None);
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
Offset = o;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
Complete = true;
return(result.Value);
}
var obs = ObservationPeriodsRaw.Where(op => op.TypeConceptId > 0).ToArray();
ObservationPeriod[] observationPeriods;
if (obs.Length == 0)
{
observationPeriods = new ObservationPeriod[0];
}
else
{
observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.Where(op => op.TypeConceptId > 0).ToArray())
.ToArray();
}
if (Excluded(person, observationPeriods))
{
Complete = true;
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var visitDetails = BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
if (!visitOccurrences.ContainsKey(visitOccurrence.Id))
{
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
}
foreach (var visitDetail in visitDetails)
{
var vo = GetVisitOccurrence(visitDetail);
visitDetail.VisitOccurrenceId = vo?.Id ?? 0;
if (visitDetail.VisitOccurrenceId.HasValue && !_visitDetails.ContainsKey(visitDetail.VisitOccurrenceId.Value))
{
_visitDetails.Add(visitDetail.VisitOccurrenceId.Value, new List <VisitDetail>());
}
_visitDetails[visitDetail.VisitOccurrenceId.Value].Add(visitDetail);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var co in conditionOccurrences)
{
co.Id = Offset.GetKeyOffset(co.PersonId).ConditionOccurrenceId;
}
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var procedureOccurrence in procedureOccurrences)
{
procedureOccurrence.Id =
Offset.GetKeyOffset(procedureOccurrence.PersonId).ProcedureOccurrenceId;
}
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var ob in observations)
{
ob.Id = Offset.GetKeyOffset(ob.PersonId).ObservationId;
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
Console.WriteLine($"PeronsId={person.PersonSourceValue};" +
$"cond={conditionOccurrences.Length};" +
$"drg={drugExposures.Length};" +
$"proc={procedureOccurrences.Length};" +
$"mes={measurements.Length};" +
$"obser={observations.Length};" +
$"dev={deviceExposure.Length};" +
$"vo={visitOccurrences.Values.Count};" +
$"vd={visitDetails.Length}");
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(visitOccurrences.Values.ToArray());
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
SetProviderIds(visitDetails);
SetVisitDetailId(drugExposures, visitOccurrences.Values.ToArray());
SetVisitDetailId(conditionOccurrences, visitOccurrences.Values.ToArray());
SetVisitDetailId(procedureOccurrences, visitOccurrences.Values.ToArray());
SetVisitDetailId(measurements, visitOccurrences.Values.ToArray());
SetVisitDetailId(observations, visitOccurrences.Values.ToArray());
SetVisitDetailId(deviceExposure, visitOccurrences.Values.ToArray());
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, null,
observationPeriods,
new PayerPlanPeriod[0],
drugExposures,
conditionOccurrences,
procedureOccurrences,
observations,
measurements,
visitOccurrences.Values.ToArray(), visitDetails, new Cohort[0], deviceExposure, new Note[0]);
Complete = true;
return(Attrition.None);
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
Offset = o;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var buildPerson = result.Key;
if (buildPerson == null)
{
return(result.Value);
}
_currentPerson = buildPerson;
var observationPeriods =
BuildObservationPeriods(buildPerson.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray();
int age;
int.TryParse(PersonRecords.ToArray()[0].AdditionalFields["age"], out age);
_personAge = age;
int ageday;
int.TryParse(PersonRecords.ToArray()[0].AdditionalFields["ageday"], out ageday);
if (age > 0)
{
buildPerson.YearOfBirth = observationPeriods[0].StartDate.Year - age;
}
else if (string.IsNullOrEmpty(PersonRecords.ToArray()[0].AdditionalFields["ageday"]) &&
PersonRecords.ToArray()[0].AdditionalFields["age_neonate"] == "1")
{
buildPerson.YearOfBirth = observationPeriods[0].StartDate.Year;
}
else if (PersonRecords.ToArray()[0].AdditionalFields["ageday"] == null &&
PersonRecords.ToArray()[0].AdditionalFields["age_neonate"] != "1" || ageday < 0)
{
var dateOfBirth = observationPeriods[0].StartDate.AddDays(-180);
buildPerson.YearOfBirth = dateOfBirth.Year;
}
else if (ageday >= 0)
{
var dateOfBirth = observationPeriods[0].StartDate.AddDays(ageday * -1);
buildPerson.YearOfBirth = dateOfBirth.Year;
buildPerson.DayOfBirth = dateOfBirth.Day;
buildPerson.MonthOfBirth = dateOfBirth.Month;
}
if (!buildPerson.YearOfBirth.HasValue || buildPerson.YearOfBirth < 1907)
{
return(Attrition.ImplausibleYOBPast); // HIX-1417
}
// Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH
if (Excluded(buildPerson, observationPeriods))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
if (visitOccurrence.IdUndefined)
{
visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure =
BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
var drugCosts = BuildDrugCosts(drugExposures).ToArray();
var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray();
var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray();
var devicCosts = BuildDeviceCosts(deviceExposure).ToArray();
foreach (var cost in visitCosts)
{
if (cost.TotalPaid == 0)
{
continue;
}
var cost52 = new Cost(cost.PersonId)
{
CostId = Offset.GetKeyOffset(cost.PersonId).VisitCostId,
CurrencyConceptId = 44818668,
TypeId = 5032,
Domain = "Visit",
EventId = cost.Id,
TotalCharge = cost.TotalPaid
};
ChunkData.AddCostData(cost52);
}
var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray();
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(buildPerson, death, observationPeriods, payerPlanPeriods, drugExposures,
conditionOccurrences, procedureOccurrences, observations, measurements,
visitOccurrences.Values.ToArray(), null, cohort, deviceExposure, new Note[0]);
var pg = new PregnancyAlgorithm();
foreach (var pe in pg.GetPregnancyEpisodes(Vocabulary, buildPerson, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == buildPerson.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == buildPerson.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == buildPerson.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == buildPerson.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == buildPerson.PersonId).ToArray()))
{
pe.Id = Offset.GetKeyOffset(pe.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(pe);
}
return(Attrition.None);
}