/// <summary>
/// Build person entity and all person related entities like: DrugExposures, ConditionOccurrences, ProcedureOccurrences... from raw data sets
/// </summary>
public override Attrition Build(ChunkData data, KeyMasterOffsetManager om)
{
Offset = om;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
if (!ObservationPeriodsRaw.Any(op => op.StartDate < op.EndDate))
{
return(Attrition.InvalidObservationTime);
}
var op = ObservationPeriodsRaw.Where(op =>
op.StartDate < op.EndDate &&
op.StartDate.Year >= person.YearOfBirth &&
op.EndDate.Value.Year >= person.YearOfBirth &&
op.StartDate.Year <= DateTime.Now.Year).ToArray();
if (op.Length == 0)
{
return(Attrition.InvalidObservationTime);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, op).ToArray();
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.Where(pp =>
pp.StartDate.Year >= person.YearOfBirth &&
pp.EndDate.Value.Year >= person.YearOfBirth &&
pp.StartDate.Year <= DateTime.Now.Year).ToArray(), null).ToArray();
var visitDetails = new Dictionary <long, VisitDetail>();
var visitDetIds = new List <long>();
foreach (var vd in BuildVisitDetails(null, VisitOccurrencesRaw.Where(vo =>
vo.StartDate.Year >= person.YearOfBirth &&
vo.EndDate.Value.Year >= person.YearOfBirth &&
vo.StartDate.Year <= DateTime.Now.Year &&
vo.EndDate.Value.Year <= DateTime.Now.Year).ToArray(), observationPeriods).ToArray())
{
if (person.MonthOfBirth.HasValue && vd.StartDate.Year < person.YearOfBirth.Value &&
vd.StartDate.Month < person.MonthOfBirth ||
vd.StartDate.Year < person.YearOfBirth.Value)
{
if (vd.StartDate.Year < person.YearOfBirth.Value)
{
if (DateTime.TryParse(person.AdditionalFields["frd"], out var frd))
{
vd.StartDate = frd;
vd.EndDate = frd;
}
else
{
continue;
}
}
}
if (visitDetails.ContainsKey(vd.Id))
{
continue;
}
visitDetails.Add(vd.Id, vd);
visitDetIds.Add(vd.Id);
}
long?prevVisitDetId = null;
foreach (var visitId in visitDetIds.OrderBy(v => v))
{
if (prevVisitDetId.HasValue)
{
visitDetails[visitId].PrecedingVisitDetailId = prevVisitDetId;
}
prevVisitDetId = visitId;
}
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var byStartDate in visitDetails.Values.GroupBy(v => v.StartDate))
{
var vd = byStartDate.First();
var providerId = byStartDate.Min(v => v.ProviderId);
var careSiteId = byStartDate.Min(v => v.CareSiteId);
var sourceValue = byStartDate.Min(v => v.SourceValue);
var visitOccurrenceId = byStartDate.Min(v => v.Id);
var visitOccurrence = new VisitOccurrence(vd)
{
//Id = Offset.GetKeyOffset(vd.PersonId).VisitOccurrenceId,
Id = visitOccurrenceId,
ProviderId = providerId,
CareSiteId = careSiteId,
SourceValue = sourceValue
};
foreach (var visitDetail in byStartDate)
{
visitDetail.VisitOccurrenceId = visitOccurrence.Id;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
SetVisitOccurrenceId(ConditionOccurrencesRaw, visitDetails);
SetVisitOccurrenceId(ProcedureOccurrencesRaw, visitDetails);
SetVisitOccurrenceId(DrugExposuresRaw, visitDetails);
SetVisitOccurrenceId(DeviceExposureRaw, visitDetails);
SetVisitOccurrenceId(ObservationsRaw, visitDetails);
SetVisitOccurrenceId(MeasurementsRaw, visitDetails);
var drugExposures = BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences = BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var procedureOccurrences = BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
if (death != null)
{
person.TimeOfDeath = death.StartDate;
if (death.StartDate < observationPeriods.Min(op => op.StartDate))
{
return(Attrition.UnacceptablePatientQuality);
}
if (death.StartDate.Year < person.YearOfBirth || death.StartDate.Year > DateTime.Now.Year)
{
death = null;
}
}
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person,
death,
observationPeriods,
payerPlanPeriods,
Clean(drugExposures, person).ToArray(),
Clean(conditionOccurrences, person).ToArray(),
Clean(procedureOccurrences, person).ToArray(),
Clean(observations, person).ToArray(),
Clean(measurements, person).ToArray(),
visitOccurrences.Values.ToArray(),
visitDetails.Values.ToArray(), null,
Clean(deviceExposure, person).ToArray(), null);
var pg = new PregnancyAlgorithm();
foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
r.Id = Offset.GetKeyOffset(r.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(r);
}
return(Attrition.None);
}
//private VisitOccurrence GetVisitOccurrence(IEntity ent)
//{
// if (_rawVisits.ContainsKey(ent.SourceRecordGuid))
// {
// var vo = _rawVisits[ent.SourceRecordGuid];
// if (vo.Id == 0 && _rawVisits.ContainsKey(vo.SourceRecordGuid) &&
// _rawVisits[vo.SourceRecordGuid].SourceRecordGuid != ent.SourceRecordGuid)
// {
// vo = _rawVisits[vo.SourceRecordGuid];
// }
// return vo;
// }
// return null;
//}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager om)
{
this.Offset = om;
this.ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray();
// Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH
if (Excluded(person, observationPeriods))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray();
var vDetails = BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(
VisitOccurrencesRaw
.Where(v => v.AdditionalFields["sort_index"] == "1" && v.AdditionalFields["ordinal"] == "1")
.ToArray(), observationPeriods))
{
if (!_rawVisits.ContainsKey(visitOccurrence.SourceRecordGuid))
{
_rawVisits.Add(visitOccurrence.SourceRecordGuid, visitOccurrence);
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
var visitDetails = new Dictionary <long, VisitDetail>();
foreach (var group in vDetails.GroupBy(d => d.Id))
{
var vd = group.First();
if (visitOccurrences.ContainsKey(vd.Id))
{
vd.VisitOccurrenceId = vd.Id;
vd.ProviderId = visitOccurrences[vd.Id].ProviderId;
}
else
{
var visits = visitOccurrences.Values.Where(vo =>
vo.TypeConceptId == vd.TypeConceptId && vo.StartDate >= vd.StartDate &&
vo.EndDate <= vo.EndDate).ToArray();
vd.VisitOccurrenceId = visits.Any() ? visits.First().Id : 0;
}
vd.SourceValue = $"visitid:{vd.VisitOccurrenceId};encid:{vd.AdditionalFields["encid"]}";
visitDetails.Add(vd.Id, vd);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
UpdateVisitOccurrenceIds(visitOccurrences, visitDetails);
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure =
BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
// set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities
SetPayerPlanPeriodId(payerPlanPeriods, drugExposures, procedureOccurrences,
visitOccurrences.Values.ToArray(),
deviceExposure);
var notes = BuildNote(NoteRecords.ToArray(), visitOccurrences, observationPeriods).ToArray();
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
//SetProviderIds(visitDetails);
SetProviderIds(drugExposures, visitOccurrences);
SetProviderIds(conditionOccurrences, visitOccurrences);
SetProviderIds(measurements, visitOccurrences);
SetProviderIds(procedureOccurrences, visitOccurrences);
SetProviderIds(deviceExposure, visitOccurrences);
SetProviderIds(observations, visitOccurrences);
SetProviderIds(notes, visitOccurrences);
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
var drugCosts = BuildDrugCosts(drugExposures).ToArray();
var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray();
var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray();
var devicCosts = BuildDeviceCosts(deviceExposure).ToArray();
var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray();
if (death != null)
{
death = CleanUpDeath(visitOccurrences.Values, death);
death = CleanUpDeath(drugExposures, death);
death = CleanUpDeath(conditionOccurrences, death);
death = CleanUpDeath(procedureOccurrences, death);
death = CleanUpDeath(measurements, death);
death = CleanUpDeath(observations, death);
death = CleanUpDeath(deviceExposure, death);
}
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, death, observationPeriods, payerPlanPeriods, drugExposures,
conditionOccurrences, procedureOccurrences, observations, measurements,
visitOccurrences.Values.ToArray(), visitDetails.Values.ToArray(), cohort, deviceExposure, notes);
var pg = new PregnancyAlgorithm.PregnancyAlgorithm();
foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
r.Id = KeyMasterOffsetManager.GetKeyOffset(r.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(r);
}
return(Attrition.None);
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager om)
{
this.Offset = om;
this.ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray();
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray();
var dedupVisits = VisitOccurrencesRaw.Where(v => v.AdditionalFields["sort_index"] == "1").ToArray();
var vDetails = BuildVisitDetails(null, dedupVisits, observationPeriods).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(dedupVisits, observationPeriods))
{
visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
visitOccurrence.TypeConceptId = 44818517;
if (!visitOccurrence.EndDate.HasValue)
{
visitOccurrence.EndDate = visitOccurrence.StartDate;
}
if (!_rawVisits.ContainsKey(visitOccurrence.SourceRecordGuid))
{
_rawVisits.Add(visitOccurrence.SourceRecordGuid, visitOccurrence);
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
var visitDetails = new Dictionary <long, VisitDetail>();
foreach (var group in vDetails.GroupBy(d => d.Id))
{
var vd = group.First();
var visitOccurrenceId = GetVisitOccurrenceId(vd);
if (visitOccurrenceId.HasValue && visitOccurrences.ContainsKey(visitOccurrenceId.Value))
{
vd.VisitOccurrenceId = visitOccurrenceId;
}
else
{
var visits = visitOccurrences.Values
.Where(vo => vd.StartDate.Date >= vo.StartDate.Date &&
vd.EndDate.Value.Date <= vo.EndDate.Value.Date).ToArray();
if (visits.Any())
{
vd.VisitOccurrenceId = visits.First().Id;
}
}
visitDetails.Add(vd.Id, vd);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure =
BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
// set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities
SetPayerPlanPeriodId(payerPlanPeriods, drugExposures, procedureOccurrences,
visitOccurrences.Values.ToArray(),
deviceExposure);
var notes = BuildNote(NoteRecords.ToArray(), visitOccurrences, observationPeriods).ToArray();
List <DateTime?> mins = new List <DateTime?>();
List <DateTime?> maxs = new List <DateTime?>();
mins.Add(GetMinDate(drugExposures));
mins.Add(GetMinDate(conditionOccurrences));
mins.Add(GetMinDate(procedureOccurrences));
mins.Add(GetMinDate(observations));
mins.Add(GetMinDate(deviceExposure));
mins.Add(GetMinDate(measurements));
mins.Add(GetMinDate(visitOccurrences.Values));
mins.Add(GetMinDate(visitDetails.Values));
maxs.Add(GetMaxDate(drugExposures));
maxs.Add(GetMaxDate(conditionOccurrences));
maxs.Add(GetMaxDate(procedureOccurrences));
maxs.Add(GetMaxDate(observations));
maxs.Add(GetMaxDate(deviceExposure));
maxs.Add(GetMaxDate(measurements));
maxs.Add(GetMaxDate(visitOccurrences.Values));
maxs.Add(GetMaxDate(visitDetails.Values));
var min = mins.Min();
var max = maxs.Max();
var observationPeriodsFinal = new List <ObservationPeriod>(1)
{
observationPeriods[0]
};
if (min.HasValue)
{
observationPeriodsFinal[0].StartDate = min.Value;
if (max.HasValue)
{
observationPeriodsFinal[0].EndDate = max;
}
else
{
observationPeriodsFinal[0].EndDate = min.Value;
}
}
// Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH
if (Excluded(person, observationPeriodsFinal))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
SetProviderIds(deviceExposure);
SetProviderIds(measurements);
SetProviderIds(visitOccurrences.Values);
SetProviderIds(visitDetails.Values);
SetVisitOccurrenceId(conditionOccurrences, visitDetails.Values.ToArray());
SetVisitOccurrenceId(procedureOccurrences, visitDetails.Values.ToArray());
SetVisitOccurrenceId(drugExposures, visitDetails.Values.ToArray());
SetVisitOccurrenceId(deviceExposure, visitDetails.Values.ToArray());
SetVisitOccurrenceId(observations, visitDetails.Values.ToArray());
SetVisitOccurrenceId(measurements, visitDetails.Values.ToArray());
//SetVisitOccurrenceId(visitDetails.Values, visitOccurrences.Values.ToArray());
SetProviderIds(drugExposures, visitOccurrences);
SetProviderIds(conditionOccurrences, visitOccurrences);
SetProviderIds(measurements, visitOccurrences);
SetProviderIds(procedureOccurrences, visitOccurrences);
SetProviderIds(deviceExposure, visitOccurrences);
SetProviderIds(observations, visitOccurrences);
//SetProviderIds(notes, visitOccurrences); TMP: NOTE
if (notes.Any())
{
if (visitOccurrences.Count > 0)
{
foreach (var e in notes.Where(e => !e.ProviderId.HasValue))
{
if (e.AdditionalFields != null && e.AdditionalFields.ContainsKey("encid") && !string.IsNullOrEmpty(e.AdditionalFields["encid"]))
{
var encid = e.AdditionalFields["encid"];
var vd = visitDetails.Values.FirstOrDefault(v => v.AdditionalFields["encid"] == encid);
if (vd == null)
{
continue;
}
e.VisitDetailId = vd.Id;
e.VisitOccurrenceId = vd.VisitOccurrenceId;
}
if (!e.VisitOccurrenceId.HasValue)
{
continue;
}
if (visitOccurrences.ContainsKey(e.VisitOccurrenceId.Value))
{
e.ProviderId = visitOccurrences[e.VisitOccurrenceId.Value].ProviderId;
}
}
}
}
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriodsFinal.ToArray());
var drugCosts = BuildDrugCosts(drugExposures).ToArray();
var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray();
var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray();
var devicCosts = BuildDeviceCosts(deviceExposure).ToArray();
var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriodsFinal.ToArray()).ToArray();
if (death != null)
{
death = CleanUpDeath(visitDetails.Values, death);
death = CleanUpDeath(visitOccurrences.Values, death);
death = CleanUpDeath(drugExposures, death);
death = CleanUpDeath(conditionOccurrences, death);
death = CleanUpDeath(procedureOccurrences, death);
death = CleanUpDeath(measurements, death);
death = CleanUpDeath(observations, death);
death = CleanUpDeath(deviceExposure, death);
}
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, death, observationPeriodsFinal.ToArray(), payerPlanPeriods, drugExposures,
conditionOccurrences, procedureOccurrences, observations, measurements,
visitOccurrences.Values.ToArray(), visitDetails.Values.ToArray(), cohort, deviceExposure, notes);
var pg = new PregnancyAlgorithm();
foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriodsFinal.ToArray(),
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
r.Id = Offset.GetKeyOffset(r.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(r);
}
return(Attrition.None);
}