public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
this.Offset = o;
this.ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
if (person.YearOfBirth > DateTime.Now.Year)
{
return(Attrition.ImplausibleYOBFuture);
}
var op = ObservationPeriodsRaw.Where(op =>
op.StartDate.Year >= person.YearOfBirth &&
op.EndDate.Value.Year >= person.YearOfBirth &&
op.StartDate.Year <= DateTime.Now.Year).ToArray();
if (op.Length == 0)
{
return(Attrition.InvalidObservationTime);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, op).ToArray();
if (Excluded(person, observationPeriods))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var fisrtOP = observationPeriods.Min(op => op.StartDate);
if (fisrtOP.Year == person.YearOfBirth)
{
person.MonthOfBirth = fisrtOP.Month;
person.DayOfBirth = fisrtOP.Day;
}
else
{
person.MonthOfBirth = null;
person.DayOfBirth = null;
}
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.Where(pp =>
pp.StartDate.Year >= person.YearOfBirth &&
pp.EndDate.Value.Year >= person.YearOfBirth &&
pp.StartDate.Year <= DateTime.Now.Year).ToArray(), null).ToArray();
List <VisitDetail> visitDetails = new List <VisitDetail>();
foreach (var vd in BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).Where(vd =>
vd.StartDate.Year >= person.YearOfBirth &&
vd.EndDate.Value.Year >= person.YearOfBirth &&
vd.StartDate.Year <= DateTime.Now.Year))
{
if (vd.StartDate.Year < person.YearOfBirth || vd.StartDate.Year > DateTime.Now.Year)
{
continue;
}
if (vd.EndDate.HasValue && (
vd.EndDate.Value.Year < person.YearOfBirth ||
vd.EndDate.Value.Year > DateTime.Now.Year))
{
continue;
}
visitDetails.Add(vd);
}
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
if (visitOccurrence.StartDate.Year < person.YearOfBirth || visitOccurrence.StartDate.Year > DateTime.Now.Year)
{
continue;
}
if (visitOccurrence.EndDate.HasValue && (
visitOccurrence.EndDate.Value.Year < person.YearOfBirth ||
visitOccurrence.EndDate.Value.Year > DateTime.Now.Year))
{
continue;
}
//if (visitOccurrence.IdUndefined)
{
visitOccurrence.Id =
Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
visitOccurrence.IdUndefined = false;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
foreach (var visitDetail in visitDetails)
{
var patPlanid = visitDetail.AdditionalFields["pat_planid"];
var clmid = string.Empty;
var locCd = string.Empty;
if (visitDetail.AdditionalFields.ContainsKey("clmid"))
{
clmid = visitDetail.AdditionalFields["clmid"];
}
if (visitDetail.AdditionalFields.ContainsKey("loc_cd"))
{
locCd = visitDetail.AdditionalFields["loc_cd"];
}
if (!_rawVisitDetails.ContainsKey(patPlanid))
{
_rawVisitDetails.Add(patPlanid, new Dictionary <string, Dictionary <string, List <VisitDetail> > >());
}
if (!_rawVisitDetails[patPlanid].ContainsKey(clmid))
{
_rawVisitDetails[patPlanid].Add(clmid, new Dictionary <string, List <VisitDetail> >());
}
if (!_rawVisitDetails[patPlanid][clmid].ContainsKey(locCd))
{
_rawVisitDetails[patPlanid][clmid].Add(locCd, new List <VisitDetail>());
}
_rawVisitDetails[patPlanid][clmid][locCd].Add(visitDetail);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var co in conditionOccurrences)
{
co.Id = Offset.GetKeyOffset(co.PersonId).ConditionOccurrenceId;
}
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var procedureOccurrence in procedureOccurrences)
{
procedureOccurrence.Id = Offset.GetKeyOffset(procedureOccurrence.PersonId)
.ProcedureOccurrenceId;
}
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var drugExposures = BuildDrugExposures(FilteroutDrugClaims(DrugExposuresRaw).ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
if (death != null && person.YearOfBirth.HasValue && person.YearOfBirth.Value > 0 &&
person.YearOfBirth > death.StartDate.Year)
{
death = null;
}
if (death != null)
{
// DOD
if (death.TypeConceptId == 32519)
{
foreach (var visitOccurrence in visitOccurrences.Values.Where(v => (v.ConceptId == 9202 || v.ConceptId == 581458) && v.StartDate > death.StartDate.AddDays(30)))
{
visitOccurrence.ConceptId = -1;
}
if (visitOccurrences.Values.Any(v => (v.ConceptId == 9201 || v.ConceptId == 9203) && v.StartDate > death.StartDate.AddDays(30)))
{
death = null;
}
}
if (death != null)
{
if (death.StartDate.Year < person.YearOfBirth || death.StartDate.Year > DateTime.Now.Year)
{
death = null;
}
foreach (var payerPlanPeriod in payerPlanPeriods)
{
if (payerPlanPeriod.EndDate.Value == death.StartDate)
{
payerPlanPeriod.StopReasonConceptId = 352;
}
}
}
}
foreach (var visitDetail in visitDetails)
{
var vo = GetVisitOccurrence(visitDetail);
if (vo != null && visitOccurrences.ContainsKey(vo.Id))
{
if (visitOccurrences[vo.Id].ConceptId == -1)
{
visitDetail.ConceptId = -1;
}
else
{
visitDetail.VisitOccurrenceId = vo.Id;
}
}
else
{
visitDetail.ConceptId = -1;
}
}
SetVisitDetailId(drugExposures);
SetVisitDetailId(conditionOccurrences);
SetVisitDetailId(procedureOccurrences);
SetVisitDetailId(measurements);
SetVisitDetailId(observations);
SetVisitDetailId(deviceExposure);
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person,
death,
observationPeriods,
payerPlanPeriods,
Clean(drugExposures, person).ToArray(),
Clean(conditionOccurrences, person).ToArray(),
Clean(procedureOccurrences, person).ToArray(),
Clean(observations, person).ToArray(),
Clean(measurements, person).ToArray(),
visitOccurrences.Values.Where(v => v.ConceptId >= 0).ToArray(),
visitDetails.Where(v => v.ConceptId >= 0).ToArray(), new Cohort[0],
Clean(deviceExposure, person).ToArray(),
new Note[0]);
var pg = new PregnancyAlgorithm();
foreach (var episode in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
episode.Id = Offset.GetKeyOffset(episode.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(episode);
if (episode.ConceptId == 433260 && _potentialChilds.Count > 0)
{
foreach (var child in _potentialChilds)
{
var childId = child.Key;
foreach (var birthdate in child.Value)
{
if (episode.EndDate.Value.Between(birthdate.AddDays(-60), birthdate.AddDays(60)))
{
//40485452 Child of subject
//40478925 Mother of subject
ChunkData.FactRelationships.Add(new FactRelationship
{
DomainConceptId1 = 56,
DomainConceptId2 = 56,
FactId1 = episode.PersonId,
FactId2 = childId,
RelationshipConceptId = 40478925
});
break;
}
}
}
}
}
return(Attrition.None);
}
/// <summary>
/// Build person entity and all person related entities like: DrugExposures, ConditionOccurrences, ProcedureOccurrences... from raw data sets
/// </summary>
public override Attrition Build(ChunkData data, KeyMasterOffsetManager om)
{
Offset = om;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
if (!ObservationPeriodsRaw.Any(op => op.StartDate < op.EndDate))
{
return(Attrition.InvalidObservationTime);
}
var op = ObservationPeriodsRaw.Where(op =>
op.StartDate < op.EndDate &&
op.StartDate.Year >= person.YearOfBirth &&
op.EndDate.Value.Year >= person.YearOfBirth &&
op.StartDate.Year <= DateTime.Now.Year).ToArray();
if (op.Length == 0)
{
return(Attrition.InvalidObservationTime);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, op).ToArray();
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.Where(pp =>
pp.StartDate.Year >= person.YearOfBirth &&
pp.EndDate.Value.Year >= person.YearOfBirth &&
pp.StartDate.Year <= DateTime.Now.Year).ToArray(), null).ToArray();
var visitDetails = new Dictionary <long, VisitDetail>();
var visitDetIds = new List <long>();
foreach (var vd in BuildVisitDetails(null, VisitOccurrencesRaw.Where(vo =>
vo.StartDate.Year >= person.YearOfBirth &&
vo.EndDate.Value.Year >= person.YearOfBirth &&
vo.StartDate.Year <= DateTime.Now.Year &&
vo.EndDate.Value.Year <= DateTime.Now.Year).ToArray(), observationPeriods).ToArray())
{
if (person.MonthOfBirth.HasValue && vd.StartDate.Year < person.YearOfBirth.Value &&
vd.StartDate.Month < person.MonthOfBirth ||
vd.StartDate.Year < person.YearOfBirth.Value)
{
if (vd.StartDate.Year < person.YearOfBirth.Value)
{
if (DateTime.TryParse(person.AdditionalFields["frd"], out var frd))
{
vd.StartDate = frd;
vd.EndDate = frd;
}
else
{
continue;
}
}
}
if (visitDetails.ContainsKey(vd.Id))
{
continue;
}
visitDetails.Add(vd.Id, vd);
visitDetIds.Add(vd.Id);
}
long?prevVisitDetId = null;
foreach (var visitId in visitDetIds.OrderBy(v => v))
{
if (prevVisitDetId.HasValue)
{
visitDetails[visitId].PrecedingVisitDetailId = prevVisitDetId;
}
prevVisitDetId = visitId;
}
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var byStartDate in visitDetails.Values.GroupBy(v => v.StartDate))
{
var vd = byStartDate.First();
var providerId = byStartDate.Min(v => v.ProviderId);
var careSiteId = byStartDate.Min(v => v.CareSiteId);
var sourceValue = byStartDate.Min(v => v.SourceValue);
var visitOccurrenceId = byStartDate.Min(v => v.Id);
var visitOccurrence = new VisitOccurrence(vd)
{
//Id = Offset.GetKeyOffset(vd.PersonId).VisitOccurrenceId,
Id = visitOccurrenceId,
ProviderId = providerId,
CareSiteId = careSiteId,
SourceValue = sourceValue
};
foreach (var visitDetail in byStartDate)
{
visitDetail.VisitOccurrenceId = visitOccurrence.Id;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
SetVisitOccurrenceId(ConditionOccurrencesRaw, visitDetails);
SetVisitOccurrenceId(ProcedureOccurrencesRaw, visitDetails);
SetVisitOccurrenceId(DrugExposuresRaw, visitDetails);
SetVisitOccurrenceId(DeviceExposureRaw, visitDetails);
SetVisitOccurrenceId(ObservationsRaw, visitDetails);
SetVisitOccurrenceId(MeasurementsRaw, visitDetails);
var drugExposures = BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences = BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var procedureOccurrences = BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
if (death != null)
{
person.TimeOfDeath = death.StartDate;
if (death.StartDate < observationPeriods.Min(op => op.StartDate))
{
return(Attrition.UnacceptablePatientQuality);
}
if (death.StartDate.Year < person.YearOfBirth || death.StartDate.Year > DateTime.Now.Year)
{
death = null;
}
}
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person,
death,
observationPeriods,
payerPlanPeriods,
Clean(drugExposures, person).ToArray(),
Clean(conditionOccurrences, person).ToArray(),
Clean(procedureOccurrences, person).ToArray(),
Clean(observations, person).ToArray(),
Clean(measurements, person).ToArray(),
visitOccurrences.Values.ToArray(),
visitDetails.Values.ToArray(), null,
Clean(deviceExposure, person).ToArray(), null);
var pg = new PregnancyAlgorithm();
foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
r.Id = Offset.GetKeyOffset(r.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(r);
}
return(Attrition.None);
}
