public override Attrition Build(ChunkData data, KeyMasterOffsetManager o) { Offset = o; ChunkData = data; var result = BuildPerson(PersonRecords.ToList()); var person = result.Key; if (person == null) { Complete = true; return(result.Value); } var observationPeriods = BuildObservationPeriods(9999, ObservationPeriodsRaw.ToArray()) .ToArray(); if (Excluded(person, observationPeriods)) { Complete = true; return(Attrition.ImplausibleYOBPostEarliestOP); } var visitDetails = BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).ToArray(); var visitOccurrences = new Dictionary <long, VisitOccurrence>(); var visitIds = new List <long>(); foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods)) { if (visitOccurrence.IdUndefined) { visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId; visitOccurrence.IdUndefined = false; } if (!visitOccurrences.ContainsKey(visitOccurrence.Id)) { visitOccurrences.Add(visitOccurrence.Id, visitOccurrence); visitIds.Add(visitOccurrence.Id); } var ecntrId = visitOccurrence.AdditionalFields["ecntr_id"]; if (long.TryParse(ecntrId, out var visitId)) { if (!_visitIdMap.ContainsKey(visitId)) { _visitIdMap.Add(visitId, new List <VisitOccurrence>()); } _visitIdMap[visitId].Add(visitOccurrence); } } foreach (var visitDetail in visitDetails) { var vo = GetVisitOccurrence(visitDetail); visitDetail.VisitOccurrenceId = vo?.Id ?? 0; } long?prevVisitId = null; foreach (var visitId in visitIds.OrderBy(v => v)) { if (prevVisitId.HasValue) { visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId; } prevVisitId = visitId; } foreach (var e in ConditionOccurrencesRaw) { SetVisitId(e); } foreach (var e in ProcedureOccurrencesRaw) { SetVisitId(e); } foreach (var e in ObservationsRaw) { SetVisitId(e); } foreach (var e in DrugExposuresRaw) { SetVisitId(e); } foreach (var e in MeasurementsRaw) { SetVisitId(e); } foreach (var e in DeviceExposureRaw) { SetVisitId(e); } var conditionOccurrences = BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); foreach (var co in conditionOccurrences) { co.Id = Offset.GetKeyOffset(co.PersonId).ConditionOccurrenceId; } var procedureOccurrences = BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); foreach (var procedureOccurrence in procedureOccurrences) { procedureOccurrence.Id = Offset.GetKeyOffset(procedureOccurrence.PersonId).ProcedureOccurrenceId; } var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); foreach (var ob in observations) { ob.Id = Offset.GetKeyOffset(ob.PersonId).ObservationId; } var drugExposures = BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); // set corresponding ProviderIds SetProviderIds(drugExposures); SetProviderIds(conditionOccurrences); SetProviderIds(visitOccurrences.Values.ToArray()); SetProviderIds(procedureOccurrences); SetProviderIds(observations); SetProviderIds(visitDetails); // push built entities to ChunkBuilder for further save to CDM database AddToChunk(person, null, observationPeriods, new PayerPlanPeriod[0], drugExposures, conditionOccurrences, procedureOccurrences, observations, measurements, visitOccurrences.Values.ToArray(), visitDetails, new Cohort[0], deviceExposure, new Note[0]); Complete = true; return(Attrition.None); }
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o) { Offset = o; ChunkData = data; var result = BuildPerson(PersonRecords.ToList()); var person = result.Key; if (person == null) { Complete = true; return(result.Value); } var observationPeriods = new[] { new ObservationPeriod { PersonId = person.PersonId, StartDate = DateTime.MinValue, EndDate = DateTime.MaxValue } }; var visitDetails = BuildVisitDetails(VisitDetailsRaw.ToArray(), VisitOccurrencesRaw.ToArray(), observationPeriods).ToArray(); var visitOccurrences = new Dictionary <long, VisitOccurrence>(); var visitIds = new List <long>(); foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods)) { if (!visitOccurrence.EndDate.HasValue) { visitOccurrence.EndDate = visitOccurrence.StartDate; } visitOccurrences.Add(visitOccurrence.Id, visitOccurrence); visitIds.Add(visitOccurrence.Id); } long?prevVisitId = null; foreach (var visitId in visitIds.OrderBy(v => v)) { if (prevVisitId.HasValue) { visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId; } prevVisitId = visitId; } var conditionOccurrences = BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); foreach (var co in conditionOccurrences) { co.Id = Offset.GetKeyOffset(co.PersonId).ConditionOccurrenceId; } var procedureOccurrences = BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); foreach (var procedureOccurrence in procedureOccurrences) { procedureOccurrence.Id = Offset.GetKeyOffset(procedureOccurrence.PersonId).ProcedureOccurrenceId; } var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); foreach (var ob in observations) { ob.Id = Offset.GetKeyOffset(ob.PersonId).ObservationId; } var drugExposures = BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); SetVisitDetailId(drugExposures); SetVisitDetailId(conditionOccurrences); SetVisitDetailId(procedureOccurrences); SetVisitDetailId(measurements); SetVisitDetailId(observations); SetVisitDetailId(deviceExposure); // set corresponding ProviderIds SetProviderIds(drugExposures); SetProviderIds(conditionOccurrences); SetProviderIds(visitOccurrences.Values.ToArray()); SetProviderIds(procedureOccurrences); SetProviderIds(observations); SetProviderIds(visitDetails); // push built entities to ChunkBuilder for further save to CDM database AddToChunk(person, null, new ObservationPeriod[0], new PayerPlanPeriod[0], drugExposures, conditionOccurrences, procedureOccurrences, observations, measurements, visitOccurrences.Values.ToArray(), visitDetails, new Cohort[0], deviceExposure, new Note[0]); Complete = true; return(Attrition.None); }
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o) { Offset = o; ChunkData = data; var result = BuildPerson(PersonRecords.ToList()); var person = result.Key; if (person == null) { return(result.Value); } var observationPeriods = BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray(); // Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH if (Excluded(person, observationPeriods)) { return(Attrition.ImplausibleYOBPostEarliestOP); } var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray(); var visitOccurrences = new Dictionary <long, VisitOccurrence>(); foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods)) { if (visitOccurrence.IdUndefined) { visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId; } visitOccurrences.Add(visitOccurrence.Id, visitOccurrence); } // HIX-1491 Cerner Year_of_birth should use earliest admission if (visitOccurrences.Count > 0) { var minDate = visitOccurrences.Values.Min(vo => vo.StartDate); if (person.YearOfBirth > minDate.Year) { person.YearOfBirth = minDate.Year; } } if (person.YearOfBirth < 1900) { return(Attrition.ImplausibleYOBPast); } if (person.YearOfBirth > DateTime.Now.Year) { return(Attrition.ImplausibleYOBFuture); } var drugExposures = BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray(); var conditionOccurrences = BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var procedureOccurrences = BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray(); // set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities SetPayerPlanPeriodId(payerPlanPeriods, drugExposures, procedureOccurrences, visitOccurrences.Values.ToArray(), deviceExposure); // set corresponding ProviderIds SetProviderIds(drugExposures); SetProviderIds(conditionOccurrences); SetProviderIds(procedureOccurrences); SetProviderIds(observations); var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods); var drugCosts = BuildDrugCosts(drugExposures).ToArray(); var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray(); var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray(); var devicCosts = BuildDeviceCosts(deviceExposure).ToArray(); var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray(); var notes = BuildNote(NoteRecords.ToArray(), visitOccurrences, observationPeriods).ToArray(); // push built entities to ChunkBuilder for further save to CDM database AddToChunk(person, death, observationPeriods, payerPlanPeriods, drugExposures, conditionOccurrences, procedureOccurrences, observations, measurements, visitOccurrences.Values.ToArray(), null, cohort, deviceExposure, notes); var pg = new PregnancyAlgorithm(); foreach (var pe in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods, ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray())) { pe.Id = Offset.GetKeyOffset(pe.PersonId).ConditionEraId; ChunkData.ConditionEra.Add(pe); } return(Attrition.None); }
/// <summary> /// Build person entity and all person related entities like: DrugExposures, ConditionOccurrences, ProcedureOccurrences... from raw data sets /// </summary> public override Attrition Build(ChunkData data, KeyMasterOffsetManager om) { this.Offset = om; this.ChunkData = data; var result = BuildPerson(PersonRecords.ToList()); var person = result.Key; if (person == null) { return(result.Value); } if (!ObservationPeriodsRaw.Any(op => op.StartDate < op.EndDate)) { return(Attrition.InvalidObservationTime); } var observationPeriods = BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.Where(op => op.StartDate < op.EndDate).ToArray()).ToArray(); var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray(); var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray(); var visitOccurrences = new Dictionary <long, VisitOccurrence>(); var visitIds = new List <long>(); // Build and clenaup visit occurrences entities foreach (var visitOccurrence in CleanupVisits( BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods), cohort, observationPeriods)) { visitOccurrences.Add(visitOccurrence.Id, visitOccurrence); visitIds.Add(visitOccurrence.Id); } long?prevVisitId = null; foreach (var visitId in visitIds.OrderBy(v => v)) { if (prevVisitId.HasValue) { visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId; } prevVisitId = visitId; } var drugExposures = BuildDrugExposures(DrugExposuresRaw.Where(de => de.StartDate < DateTime.Now).ToArray(), visitOccurrences, observationPeriods) .ToArray(); var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var conditionOccurrences = Cleanup( BuildConditionOccurrences( ConditionOccurrencesRaw.Where(co => co.StartDate < DateTime.Now).ToArray(), visitOccurrences, observationPeriods), cohort).ToArray(); var procedureOccurrences = Cleanup( BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods), cohort).ToArray(); var observations = Cleanup(BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods), cohort) .ToArray(); var measurements = Cleanup(BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods), cohort) .ToArray(); // set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities SetPayerPlanPeriodId(payerPlanPeriods, drugExposures, procedureOccurrences, visitOccurrences.Values.ToArray(), new DeviceExposure[] { }); // set corresponding ProviderIds SetProviderIds(drugExposures); SetProviderIds(conditionOccurrences); SetProviderIds(procedureOccurrences); SetProviderIds(observations); var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods); var drugCosts = BuildDrugCosts(drugExposures).ToArray(); var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray(); // push built entities to ChunkBuilder for further save to CDM database AddToChunk(person, death, observationPeriods, payerPlanPeriods, drugExposures, CleanupCondition(conditionOccurrences).ToArray(), procedureOccurrences, CleanupObservations(observations, measurements, conditionOccurrences, procedureOccurrences).ToArray(), measurements, visitOccurrences.Values.ToArray(), null, cohort, deviceExposure, new Note[0]); var pg = new PregnancyAlgorithm.PregnancyAlgorithm(); foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods, ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray())) { r.Id = KeyMasterOffsetManager.GetKeyOffset(r.PersonId).ConditionEraId; ChunkData.ConditionEra.Add(r); } return(Attrition.None); }
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o) { this.Offset = o; this.ChunkData = data; var result = BuildPerson(PersonRecords.ToList()); var person = result.Key; if (person == null) { return(result.Value); } if (person.YearOfBirth > DateTime.Now.Year) { return(Attrition.ImplausibleYOBFuture); } var op = ObservationPeriodsRaw.Where(op => op.StartDate.Year >= person.YearOfBirth && op.EndDate.Value.Year >= person.YearOfBirth && op.StartDate.Year <= DateTime.Now.Year).ToArray(); if (op.Length == 0) { return(Attrition.InvalidObservationTime); } var observationPeriods = BuildObservationPeriods(person.ObservationPeriodGap, op).ToArray(); if (Excluded(person, observationPeriods)) { return(Attrition.ImplausibleYOBPostEarliestOP); } var fisrtOP = observationPeriods.Min(op => op.StartDate); if (fisrtOP.Year == person.YearOfBirth) { person.MonthOfBirth = fisrtOP.Month; person.DayOfBirth = fisrtOP.Day; } else { person.MonthOfBirth = null; person.DayOfBirth = null; } var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.Where(pp => pp.StartDate.Year >= person.YearOfBirth && pp.EndDate.Value.Year >= person.YearOfBirth && pp.StartDate.Year <= DateTime.Now.Year).ToArray(), null).ToArray(); List <VisitDetail> visitDetails = new List <VisitDetail>(); foreach (var vd in BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).Where(vd => vd.StartDate.Year >= person.YearOfBirth && vd.EndDate.Value.Year >= person.YearOfBirth && vd.StartDate.Year <= DateTime.Now.Year)) { if (vd.StartDate.Year < person.YearOfBirth || vd.StartDate.Year > DateTime.Now.Year) { continue; } if (vd.EndDate.HasValue && ( vd.EndDate.Value.Year < person.YearOfBirth || vd.EndDate.Value.Year > DateTime.Now.Year)) { continue; } visitDetails.Add(vd); } var visitOccurrences = new Dictionary <long, VisitOccurrence>(); var visitIds = new List <long>(); foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods)) { if (visitOccurrence.StartDate.Year < person.YearOfBirth || visitOccurrence.StartDate.Year > DateTime.Now.Year) { continue; } if (visitOccurrence.EndDate.HasValue && ( visitOccurrence.EndDate.Value.Year < person.YearOfBirth || visitOccurrence.EndDate.Value.Year > DateTime.Now.Year)) { continue; } //if (visitOccurrence.IdUndefined) { visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId; visitOccurrence.IdUndefined = false; } visitOccurrences.Add(visitOccurrence.Id, visitOccurrence); visitIds.Add(visitOccurrence.Id); } foreach (var visitDetail in visitDetails) { var patPlanid = visitDetail.AdditionalFields["pat_planid"]; var clmid = string.Empty; var locCd = string.Empty; if (visitDetail.AdditionalFields.ContainsKey("clmid")) { clmid = visitDetail.AdditionalFields["clmid"]; } if (visitDetail.AdditionalFields.ContainsKey("loc_cd")) { locCd = visitDetail.AdditionalFields["loc_cd"]; } if (!_rawVisitDetails.ContainsKey(patPlanid)) { _rawVisitDetails.Add(patPlanid, new Dictionary <string, Dictionary <string, List <VisitDetail> > >()); } if (!_rawVisitDetails[patPlanid].ContainsKey(clmid)) { _rawVisitDetails[patPlanid].Add(clmid, new Dictionary <string, List <VisitDetail> >()); } if (!_rawVisitDetails[patPlanid][clmid].ContainsKey(locCd)) { _rawVisitDetails[patPlanid][clmid].Add(locCd, new List <VisitDetail>()); } _rawVisitDetails[patPlanid][clmid][locCd].Add(visitDetail); } long?prevVisitId = null; foreach (var visitId in visitIds.OrderBy(v => v)) { if (prevVisitId.HasValue) { visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId; } prevVisitId = visitId; } var conditionOccurrences = BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); foreach (var co in conditionOccurrences) { co.Id = Offset.GetKeyOffset(co.PersonId).ConditionOccurrenceId; } var procedureOccurrences = BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); foreach (var procedureOccurrence in procedureOccurrences) { procedureOccurrence.Id = Offset.GetKeyOffset(procedureOccurrence.PersonId) .ProcedureOccurrenceId; } var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var drugExposures = BuildDrugExposures(FilteroutDrugClaims(DrugExposuresRaw).ToArray(), visitOccurrences, observationPeriods) .ToArray(); var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods); if (death != null && person.YearOfBirth.HasValue && person.YearOfBirth.Value > 0 && person.YearOfBirth > death.StartDate.Year) { death = null; } if (death != null) { // DOD if (death.TypeConceptId == 32519) { foreach (var visitOccurrence in visitOccurrences.Values.Where(v => (v.ConceptId == 9202 || v.ConceptId == 581458) && v.StartDate > death.StartDate.AddDays(30))) { visitOccurrence.ConceptId = -1; } if (visitOccurrences.Values.Any(v => (v.ConceptId == 9201 || v.ConceptId == 9203) && v.StartDate > death.StartDate.AddDays(30))) { death = null; } } if (death != null) { if (death.StartDate.Year < person.YearOfBirth || death.StartDate.Year > DateTime.Now.Year) { death = null; } foreach (var payerPlanPeriod in payerPlanPeriods) { if (payerPlanPeriod.EndDate.Value == death.StartDate) { payerPlanPeriod.StopReasonConceptId = 352; } } } } foreach (var visitDetail in visitDetails) { var vo = GetVisitOccurrence(visitDetail); if (vo != null && visitOccurrences.ContainsKey(vo.Id)) { if (visitOccurrences[vo.Id].ConceptId == -1) { visitDetail.ConceptId = -1; } else { visitDetail.VisitOccurrenceId = vo.Id; } } else { visitDetail.ConceptId = -1; } } SetVisitDetailId(drugExposures); SetVisitDetailId(conditionOccurrences); SetVisitDetailId(procedureOccurrences); SetVisitDetailId(measurements); SetVisitDetailId(observations); SetVisitDetailId(deviceExposure); // push built entities to ChunkBuilder for further save to CDM database AddToChunk(person, death, observationPeriods, payerPlanPeriods, Clean(drugExposures, person).ToArray(), Clean(conditionOccurrences, person).ToArray(), Clean(procedureOccurrences, person).ToArray(), Clean(observations, person).ToArray(), Clean(measurements, person).ToArray(), visitOccurrences.Values.Where(v => v.ConceptId >= 0).ToArray(), visitDetails.Where(v => v.ConceptId >= 0).ToArray(), new Cohort[0], Clean(deviceExposure, person).ToArray(), new Note[0]); var pg = new PregnancyAlgorithm(); foreach (var episode in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods, ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray())) { episode.Id = Offset.GetKeyOffset(episode.PersonId).ConditionEraId; ChunkData.ConditionEra.Add(episode); if (episode.ConceptId == 433260 && _potentialChilds.Count > 0) { foreach (var child in _potentialChilds) { var childId = child.Key; foreach (var birthdate in child.Value) { if (episode.EndDate.Value.Between(birthdate.AddDays(-60), birthdate.AddDays(60))) { //40485452 Child of subject //40478925 Mother of subject ChunkData.FactRelationships.Add(new FactRelationship { DomainConceptId1 = 56, DomainConceptId2 = 56, FactId1 = episode.PersonId, FactId2 = childId, RelationshipConceptId = 40478925 }); break; } } } } } return(Attrition.None); }
//private VisitOccurrence GetVisitOccurrence(IEntity ent) //{ // if (_rawVisits.ContainsKey(ent.SourceRecordGuid)) // { // var vo = _rawVisits[ent.SourceRecordGuid]; // if (vo.Id == 0 && _rawVisits.ContainsKey(vo.SourceRecordGuid) && // _rawVisits[vo.SourceRecordGuid].SourceRecordGuid != ent.SourceRecordGuid) // { // vo = _rawVisits[vo.SourceRecordGuid]; // } // return vo; // } // return null; //} public override Attrition Build(ChunkData data, KeyMasterOffsetManager om) { this.Offset = om; this.ChunkData = data; var result = BuildPerson(PersonRecords.ToList()); var person = result.Key; if (person == null) { return(result.Value); } var observationPeriods = BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray(); // Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH if (Excluded(person, observationPeriods)) { return(Attrition.ImplausibleYOBPostEarliestOP); } var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray(); var vDetails = BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).ToArray(); var visitOccurrences = new Dictionary <long, VisitOccurrence>(); var visitIds = new List <long>(); foreach (var visitOccurrence in BuildVisitOccurrences( VisitOccurrencesRaw .Where(v => v.AdditionalFields["sort_index"] == "1" && v.AdditionalFields["ordinal"] == "1") .ToArray(), observationPeriods)) { if (!_rawVisits.ContainsKey(visitOccurrence.SourceRecordGuid)) { _rawVisits.Add(visitOccurrence.SourceRecordGuid, visitOccurrence); } visitOccurrences.Add(visitOccurrence.Id, visitOccurrence); visitIds.Add(visitOccurrence.Id); } var visitDetails = new Dictionary <long, VisitDetail>(); foreach (var group in vDetails.GroupBy(d => d.Id)) { var vd = group.First(); if (visitOccurrences.ContainsKey(vd.Id)) { vd.VisitOccurrenceId = vd.Id; vd.ProviderId = visitOccurrences[vd.Id].ProviderId; } else { var visits = visitOccurrences.Values.Where(vo => vo.TypeConceptId == vd.TypeConceptId && vo.StartDate >= vd.StartDate && vo.EndDate <= vo.EndDate).ToArray(); vd.VisitOccurrenceId = visits.Any() ? visits.First().Id : 0; } vd.SourceValue = $"visitid:{vd.VisitOccurrenceId};encid:{vd.AdditionalFields["encid"]}"; visitDetails.Add(vd.Id, vd); } long?prevVisitId = null; foreach (var visitId in visitIds.OrderBy(v => v)) { if (prevVisitId.HasValue) { visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId; } prevVisitId = visitId; } UpdateVisitOccurrenceIds(visitOccurrences, visitDetails); var drugExposures = BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray(); var conditionOccurrences = BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var procedureOccurrences = BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray(); // set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities SetPayerPlanPeriodId(payerPlanPeriods, drugExposures, procedureOccurrences, visitOccurrences.Values.ToArray(), deviceExposure); var notes = BuildNote(NoteRecords.ToArray(), visitOccurrences, observationPeriods).ToArray(); // set corresponding ProviderIds SetProviderIds(drugExposures); SetProviderIds(conditionOccurrences); SetProviderIds(procedureOccurrences); SetProviderIds(observations); //SetProviderIds(visitDetails); SetProviderIds(drugExposures, visitOccurrences); SetProviderIds(conditionOccurrences, visitOccurrences); SetProviderIds(measurements, visitOccurrences); SetProviderIds(procedureOccurrences, visitOccurrences); SetProviderIds(deviceExposure, visitOccurrences); SetProviderIds(observations, visitOccurrences); SetProviderIds(notes, visitOccurrences); var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods); var drugCosts = BuildDrugCosts(drugExposures).ToArray(); var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray(); var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray(); var devicCosts = BuildDeviceCosts(deviceExposure).ToArray(); var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray(); if (death != null) { death = CleanUpDeath(visitOccurrences.Values, death); death = CleanUpDeath(drugExposures, death); death = CleanUpDeath(conditionOccurrences, death); death = CleanUpDeath(procedureOccurrences, death); death = CleanUpDeath(measurements, death); death = CleanUpDeath(observations, death); death = CleanUpDeath(deviceExposure, death); } // push built entities to ChunkBuilder for further save to CDM database AddToChunk(person, death, observationPeriods, payerPlanPeriods, drugExposures, conditionOccurrences, procedureOccurrences, observations, measurements, visitOccurrences.Values.ToArray(), visitDetails.Values.ToArray(), cohort, deviceExposure, notes); var pg = new PregnancyAlgorithm.PregnancyAlgorithm(); foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods, ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(), ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray())) { r.Id = KeyMasterOffsetManager.GetKeyOffset(r.PersonId).ConditionEraId; ChunkData.ConditionEra.Add(r); } return(Attrition.None); }
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o) { Offset = o; ChunkData = data; var result = BuildPerson(PersonRecords.ToList()); var person = result.Key; if (person == null) { Complete = true; return(result.Value); } var obs = ObservationPeriodsRaw.Where(op => op.TypeConceptId > 0).ToArray(); ObservationPeriod[] observationPeriods; if (obs.Length == 0) { observationPeriods = new ObservationPeriod[0]; } else { observationPeriods = BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.Where(op => op.TypeConceptId > 0).ToArray()) .ToArray(); } if (Excluded(person, observationPeriods)) { Complete = true; return(Attrition.ImplausibleYOBPostEarliestOP); } var visitDetails = BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).ToArray(); var visitOccurrences = new Dictionary <long, VisitOccurrence>(); var visitIds = new List <long>(); foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods)) { visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId; if (!visitOccurrences.ContainsKey(visitOccurrence.Id)) { visitOccurrences.Add(visitOccurrence.Id, visitOccurrence); visitIds.Add(visitOccurrence.Id); } } foreach (var visitDetail in visitDetails) { var vo = GetVisitOccurrence(visitDetail); visitDetail.VisitOccurrenceId = vo?.Id ?? 0; if (visitDetail.VisitOccurrenceId.HasValue && !_visitDetails.ContainsKey(visitDetail.VisitOccurrenceId.Value)) { _visitDetails.Add(visitDetail.VisitOccurrenceId.Value, new List <VisitDetail>()); } _visitDetails[visitDetail.VisitOccurrenceId.Value].Add(visitDetail); } long?prevVisitId = null; foreach (var visitId in visitIds.OrderBy(v => v)) { if (prevVisitId.HasValue) { visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId; } prevVisitId = visitId; } var conditionOccurrences = BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); foreach (var co in conditionOccurrences) { co.Id = Offset.GetKeyOffset(co.PersonId).ConditionOccurrenceId; } var procedureOccurrences = BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); foreach (var procedureOccurrence in procedureOccurrences) { procedureOccurrence.Id = Offset.GetKeyOffset(procedureOccurrence.PersonId).ProcedureOccurrenceId; } var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); foreach (var ob in observations) { ob.Id = Offset.GetKeyOffset(ob.PersonId).ObservationId; } var drugExposures = BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); Console.WriteLine($"PeronsId={person.PersonSourceValue};" + $"cond={conditionOccurrences.Length};" + $"drg={drugExposures.Length};" + $"proc={procedureOccurrences.Length};" + $"mes={measurements.Length};" + $"obser={observations.Length};" + $"dev={deviceExposure.Length};" + $"vo={visitOccurrences.Values.Count};" + $"vd={visitDetails.Length}"); // set corresponding ProviderIds SetProviderIds(drugExposures); SetProviderIds(conditionOccurrences); SetProviderIds(visitOccurrences.Values.ToArray()); SetProviderIds(procedureOccurrences); SetProviderIds(observations); SetProviderIds(visitDetails); SetVisitDetailId(drugExposures, visitOccurrences.Values.ToArray()); SetVisitDetailId(conditionOccurrences, visitOccurrences.Values.ToArray()); SetVisitDetailId(procedureOccurrences, visitOccurrences.Values.ToArray()); SetVisitDetailId(measurements, visitOccurrences.Values.ToArray()); SetVisitDetailId(observations, visitOccurrences.Values.ToArray()); SetVisitDetailId(deviceExposure, visitOccurrences.Values.ToArray()); // push built entities to ChunkBuilder for further save to CDM database AddToChunk(person, null, observationPeriods, new PayerPlanPeriod[0], drugExposures, conditionOccurrences, procedureOccurrences, observations, measurements, visitOccurrences.Values.ToArray(), visitDetails, new Cohort[0], deviceExposure, new Note[0]); Complete = true; return(Attrition.None); }
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o) { Offset = o; ChunkData = data; var result = BuildPerson(PersonRecords.ToList()); var buildPerson = result.Key; if (buildPerson == null) { return(result.Value); } _currentPerson = buildPerson; var observationPeriods = BuildObservationPeriods(buildPerson.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray(); int age; int.TryParse(PersonRecords.ToArray()[0].AdditionalFields["age"], out age); _personAge = age; int ageday; int.TryParse(PersonRecords.ToArray()[0].AdditionalFields["ageday"], out ageday); if (age > 0) { buildPerson.YearOfBirth = observationPeriods[0].StartDate.Year - age; } else if (string.IsNullOrEmpty(PersonRecords.ToArray()[0].AdditionalFields["ageday"]) && PersonRecords.ToArray()[0].AdditionalFields["age_neonate"] == "1") { buildPerson.YearOfBirth = observationPeriods[0].StartDate.Year; } else if (PersonRecords.ToArray()[0].AdditionalFields["ageday"] == null && PersonRecords.ToArray()[0].AdditionalFields["age_neonate"] != "1" || ageday < 0) { var dateOfBirth = observationPeriods[0].StartDate.AddDays(-180); buildPerson.YearOfBirth = dateOfBirth.Year; } else if (ageday >= 0) { var dateOfBirth = observationPeriods[0].StartDate.AddDays(ageday * -1); buildPerson.YearOfBirth = dateOfBirth.Year; buildPerson.DayOfBirth = dateOfBirth.Day; buildPerson.MonthOfBirth = dateOfBirth.Month; } if (!buildPerson.YearOfBirth.HasValue || buildPerson.YearOfBirth < 1907) { return(Attrition.ImplausibleYOBPast); // HIX-1417 } // Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH if (Excluded(buildPerson, observationPeriods)) { return(Attrition.ImplausibleYOBPostEarliestOP); } var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray(); var visitOccurrences = new Dictionary <long, VisitOccurrence>(); foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods)) { if (visitOccurrence.IdUndefined) { visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId; } visitOccurrences.Add(visitOccurrence.Id, visitOccurrence); } var drugExposures = BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray(); var conditionOccurrences = BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var procedureOccurrences = BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods) .ToArray(); var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray(); var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods); var drugCosts = BuildDrugCosts(drugExposures).ToArray(); var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray(); var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray(); var devicCosts = BuildDeviceCosts(deviceExposure).ToArray(); foreach (var cost in visitCosts) { if (cost.TotalPaid == 0) { continue; } var cost52 = new Cost(cost.PersonId) { CostId = Offset.GetKeyOffset(cost.PersonId).VisitCostId, CurrencyConceptId = 44818668, TypeId = 5032, Domain = "Visit", EventId = cost.Id, TotalCharge = cost.TotalPaid }; ChunkData.AddCostData(cost52); } var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray(); // push built entities to ChunkBuilder for further save to CDM database AddToChunk(buildPerson, death, observationPeriods, payerPlanPeriods, drugExposures, conditionOccurrences, procedureOccurrences, observations, measurements, visitOccurrences.Values.ToArray(), null, cohort, deviceExposure, new Note[0]); var pg = new PregnancyAlgorithm(); foreach (var pe in pg.GetPregnancyEpisodes(Vocabulary, buildPerson, observationPeriods, ChunkData.ConditionOccurrences.Where(e => e.PersonId == buildPerson.PersonId).ToArray(), ChunkData.ProcedureOccurrences.Where(e => e.PersonId == buildPerson.PersonId).ToArray(), ChunkData.Observations.Where(e => e.PersonId == buildPerson.PersonId).ToArray(), ChunkData.Measurements.Where(e => e.PersonId == buildPerson.PersonId).ToArray(), ChunkData.DrugExposures.Where(e => e.PersonId == buildPerson.PersonId).ToArray())) { pe.Id = Offset.GetKeyOffset(pe.PersonId).ConditionEraId; ChunkData.ConditionEra.Add(pe); } return(Attrition.None); }