public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
Offset = o;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray();
// Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH
if (Excluded(person, observationPeriods))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
if (visitOccurrence.IdUndefined)
{
visitOccurrence.Id =
Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
}
// HIX-1491 Cerner Year_of_birth should use earliest admission
if (visitOccurrences.Count > 0)
{
var minDate = visitOccurrences.Values.Min(vo => vo.StartDate);
if (person.YearOfBirth > minDate.Year)
{
person.YearOfBirth = minDate.Year;
}
}
if (person.YearOfBirth < 1900)
{
return(Attrition.ImplausibleYOBPast);
}
if (person.YearOfBirth > DateTime.Now.Year)
{
return(Attrition.ImplausibleYOBFuture);
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure =
BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
// set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities
SetPayerPlanPeriodId(payerPlanPeriods, drugExposures, procedureOccurrences,
visitOccurrences.Values.ToArray(),
deviceExposure);
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
var drugCosts = BuildDrugCosts(drugExposures).ToArray();
var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray();
var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray();
var devicCosts = BuildDeviceCosts(deviceExposure).ToArray();
var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray();
var notes = BuildNote(NoteRecords.ToArray(), visitOccurrences, observationPeriods).ToArray();
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, death, observationPeriods, payerPlanPeriods, drugExposures,
conditionOccurrences, procedureOccurrences, observations, measurements,
visitOccurrences.Values.ToArray(), null, cohort, deviceExposure, notes);
var pg = new PregnancyAlgorithm();
foreach (var pe in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
pe.Id = Offset.GetKeyOffset(pe.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(pe);
}
return(Attrition.None);
}
/// <summary>
/// This method is called for every Lambda invocation. This method takes in an S3 event object and can be used
/// to respond to S3 notifications.
/// </summary>
/// <param name="evnt"></param>
/// <param name="context"></param>
/// <returns></returns>
public async Task <string> FunctionHandle(S3Event evnt, ILambdaContext context)
{
Console.WriteLine("FunctionHandler");
var s3Event = evnt.Records?[0].S3;
if (s3Event == null)
{
return(null);
}
var pr = string.Empty;
_bucket = Environment.GetEnvironmentVariable("Bucket");
_awsAccessKeyId = Environment.GetEnvironmentVariable("S3AwsAccessKeyId");
_awsSecretAccessKey = Environment.GetEnvironmentVariable("S3AwsSecretAccessKey");
_cdmFolder = Environment.GetEnvironmentVariable("CDMFolder");
_resultFolder = Environment.GetEnvironmentVariable("ResultFolder");
_attemptFileRemoved = false;
try
{
var result = new List <EraEntity>();
var vendor = Enum.Parse <Vendors>(s3Event.Object.Key.Split('.')[0]);
var buildingId = int.Parse(s3Event.Object.Key.Split('.')[1]);
var chunkId = int.Parse(s3Event.Object.Key.Split('.')[2]);
pr = s3Event.Object.Key.Split('.')[3].Trim();
Console.WriteLine($"vendor={vendor};buildingId={buildingId};chunkId={chunkId};");
int personCount = 0;
var prefix = $"{vendor}/{buildingId}/{_cdmFolder}/";
using (var client = new AmazonS3Client(_awsAccessKeyId, _awsSecretAccessKey,
Amazon.RegionEndpoint.USEast1))
{
var vocab = new Vocabulary2();
vocab.Fill(client, _bucket, vendor, buildingId);
Console.WriteLine("vocab done");
foreach (var key in GetKeys(prefix + "PERSON/PERSON." + chunkId + "." + pr + "."))
{
var persons = GetPersons(key).ToArray();
var observationPeriods = GetObservationPeriods(key).ToArray();
var conditionOccurrences = GetConditionOccurrence(key).ToArray();
var procedureOccurrences = GetProcedureOccurrences(key).ToArray();
var observations = GetObservations(key).ToArray();
var measurements = GetMeasurements(key).ToArray();
var drugExposures = GetDrugExposures(key).ToArray();
foreach (var person in persons)
{
var pg = new PregnancyAlgorithm();
foreach (var r in pg.GetPregnancyEpisodes(vocab, person,
observationPeriods.Where(e => e.PersonId == person.PersonId).ToArray(),
conditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
procedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
observations.Where(e => e.PersonId == person.PersonId).ToArray(),
measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
drugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
result.Add(r);
}
personCount++;
}
Console.WriteLine(chunkId + "." + pr + " | Total person:" + personCount + " | pregnancy episodes:" + result.Count);
}
}
Console.WriteLine("DONE :" + chunkId + "." + pr + " | Total person:" + personCount + " | pregnancy episodes:" + result.Count);
var fileName = $"CONDITION_ERA/CONDITION_ERA.{chunkId}.{pr}.txt.gz";
var bucket = $"{_bucket}/{vendor}/{buildingId}/{_resultFolder}";
var reader = new ConditionEraDataReader2(result.ToList());
using (var client = new AmazonS3Client(_awsAccessKeyId, _awsSecretAccessKey,
Amazon.RegionEndpoint.USEast1))
using (var stream = reader.GetStream(S3StorageType.CSV))
{
var putObject = client.PutObjectAsync(new PutObjectRequest
{
BucketName = bucket,
Key = fileName,
ServerSideEncryptionMethod = ServerSideEncryptionMethod.AES256,
StorageClass = S3StorageClass.Standard,
InputStream = stream
});
putObject.Wait();
}
GC.Collect();
Console.WriteLine("SAVED: " + fileName);
RemoveAttemptFile(s3Event);
return("Done");
}
catch (Exception e)
{
context.Logger.LogLine($"Error getting object {s3Event.Object.Key} from bucket {s3Event.Bucket.Name}. Make sure they exist and your bucket is in the same region as this function.");
context.Logger.LogLine(e.Message);
context.Logger.LogLine(e.StackTrace);
throw;
}
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
this.Offset = o;
this.ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
if (person.YearOfBirth > DateTime.Now.Year)
{
return(Attrition.ImplausibleYOBFuture);
}
var op = ObservationPeriodsRaw.Where(op =>
op.StartDate.Year >= person.YearOfBirth &&
op.EndDate.Value.Year >= person.YearOfBirth &&
op.StartDate.Year <= DateTime.Now.Year).ToArray();
if (op.Length == 0)
{
return(Attrition.InvalidObservationTime);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, op).ToArray();
if (Excluded(person, observationPeriods))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var fisrtOP = observationPeriods.Min(op => op.StartDate);
if (fisrtOP.Year == person.YearOfBirth)
{
person.MonthOfBirth = fisrtOP.Month;
person.DayOfBirth = fisrtOP.Day;
}
else
{
person.MonthOfBirth = null;
person.DayOfBirth = null;
}
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.Where(pp =>
pp.StartDate.Year >= person.YearOfBirth &&
pp.EndDate.Value.Year >= person.YearOfBirth &&
pp.StartDate.Year <= DateTime.Now.Year).ToArray(), null).ToArray();
List <VisitDetail> visitDetails = new List <VisitDetail>();
foreach (var vd in BuildVisitDetails(null, VisitOccurrencesRaw.ToArray(), observationPeriods).Where(vd =>
vd.StartDate.Year >= person.YearOfBirth &&
vd.EndDate.Value.Year >= person.YearOfBirth &&
vd.StartDate.Year <= DateTime.Now.Year))
{
if (vd.StartDate.Year < person.YearOfBirth || vd.StartDate.Year > DateTime.Now.Year)
{
continue;
}
if (vd.EndDate.HasValue && (
vd.EndDate.Value.Year < person.YearOfBirth ||
vd.EndDate.Value.Year > DateTime.Now.Year))
{
continue;
}
visitDetails.Add(vd);
}
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
if (visitOccurrence.StartDate.Year < person.YearOfBirth || visitOccurrence.StartDate.Year > DateTime.Now.Year)
{
continue;
}
if (visitOccurrence.EndDate.HasValue && (
visitOccurrence.EndDate.Value.Year < person.YearOfBirth ||
visitOccurrence.EndDate.Value.Year > DateTime.Now.Year))
{
continue;
}
//if (visitOccurrence.IdUndefined)
{
visitOccurrence.Id =
Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
visitOccurrence.IdUndefined = false;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
foreach (var visitDetail in visitDetails)
{
var patPlanid = visitDetail.AdditionalFields["pat_planid"];
var clmid = string.Empty;
var locCd = string.Empty;
if (visitDetail.AdditionalFields.ContainsKey("clmid"))
{
clmid = visitDetail.AdditionalFields["clmid"];
}
if (visitDetail.AdditionalFields.ContainsKey("loc_cd"))
{
locCd = visitDetail.AdditionalFields["loc_cd"];
}
if (!_rawVisitDetails.ContainsKey(patPlanid))
{
_rawVisitDetails.Add(patPlanid, new Dictionary <string, Dictionary <string, List <VisitDetail> > >());
}
if (!_rawVisitDetails[patPlanid].ContainsKey(clmid))
{
_rawVisitDetails[patPlanid].Add(clmid, new Dictionary <string, List <VisitDetail> >());
}
if (!_rawVisitDetails[patPlanid][clmid].ContainsKey(locCd))
{
_rawVisitDetails[patPlanid][clmid].Add(locCd, new List <VisitDetail>());
}
_rawVisitDetails[patPlanid][clmid][locCd].Add(visitDetail);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var co in conditionOccurrences)
{
co.Id = Offset.GetKeyOffset(co.PersonId).ConditionOccurrenceId;
}
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
foreach (var procedureOccurrence in procedureOccurrences)
{
procedureOccurrence.Id = Offset.GetKeyOffset(procedureOccurrence.PersonId)
.ProcedureOccurrenceId;
}
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var drugExposures = BuildDrugExposures(FilteroutDrugClaims(DrugExposuresRaw).ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
if (death != null && person.YearOfBirth.HasValue && person.YearOfBirth.Value > 0 &&
person.YearOfBirth > death.StartDate.Year)
{
death = null;
}
if (death != null)
{
// DOD
if (death.TypeConceptId == 32519)
{
foreach (var visitOccurrence in visitOccurrences.Values.Where(v => (v.ConceptId == 9202 || v.ConceptId == 581458) && v.StartDate > death.StartDate.AddDays(30)))
{
visitOccurrence.ConceptId = -1;
}
if (visitOccurrences.Values.Any(v => (v.ConceptId == 9201 || v.ConceptId == 9203) && v.StartDate > death.StartDate.AddDays(30)))
{
death = null;
}
}
if (death != null)
{
if (death.StartDate.Year < person.YearOfBirth || death.StartDate.Year > DateTime.Now.Year)
{
death = null;
}
foreach (var payerPlanPeriod in payerPlanPeriods)
{
if (payerPlanPeriod.EndDate.Value == death.StartDate)
{
payerPlanPeriod.StopReasonConceptId = 352;
}
}
}
}
foreach (var visitDetail in visitDetails)
{
var vo = GetVisitOccurrence(visitDetail);
if (vo != null && visitOccurrences.ContainsKey(vo.Id))
{
if (visitOccurrences[vo.Id].ConceptId == -1)
{
visitDetail.ConceptId = -1;
}
else
{
visitDetail.VisitOccurrenceId = vo.Id;
}
}
else
{
visitDetail.ConceptId = -1;
}
}
SetVisitDetailId(drugExposures);
SetVisitDetailId(conditionOccurrences);
SetVisitDetailId(procedureOccurrences);
SetVisitDetailId(measurements);
SetVisitDetailId(observations);
SetVisitDetailId(deviceExposure);
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person,
death,
observationPeriods,
payerPlanPeriods,
Clean(drugExposures, person).ToArray(),
Clean(conditionOccurrences, person).ToArray(),
Clean(procedureOccurrences, person).ToArray(),
Clean(observations, person).ToArray(),
Clean(measurements, person).ToArray(),
visitOccurrences.Values.Where(v => v.ConceptId >= 0).ToArray(),
visitDetails.Where(v => v.ConceptId >= 0).ToArray(), new Cohort[0],
Clean(deviceExposure, person).ToArray(),
new Note[0]);
var pg = new PregnancyAlgorithm();
foreach (var episode in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
episode.Id = Offset.GetKeyOffset(episode.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(episode);
if (episode.ConceptId == 433260 && _potentialChilds.Count > 0)
{
foreach (var child in _potentialChilds)
{
var childId = child.Key;
foreach (var birthdate in child.Value)
{
if (episode.EndDate.Value.Between(birthdate.AddDays(-60), birthdate.AddDays(60)))
{
//40485452 Child of subject
//40478925 Mother of subject
ChunkData.FactRelationships.Add(new FactRelationship
{
DomainConceptId1 = 56,
DomainConceptId2 = 56,
FactId1 = episode.PersonId,
FactId2 = childId,
RelationshipConceptId = 40478925
});
break;
}
}
}
}
}
return(Attrition.None);
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
{
Offset = o;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var buildPerson = result.Key;
if (buildPerson == null)
{
return(result.Value);
}
_currentPerson = buildPerson;
var observationPeriods =
BuildObservationPeriods(buildPerson.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray();
int age;
int.TryParse(PersonRecords.ToArray()[0].AdditionalFields["age"], out age);
_personAge = age;
int ageday;
int.TryParse(PersonRecords.ToArray()[0].AdditionalFields["ageday"], out ageday);
if (age > 0)
{
buildPerson.YearOfBirth = observationPeriods[0].StartDate.Year - age;
}
else if (string.IsNullOrEmpty(PersonRecords.ToArray()[0].AdditionalFields["ageday"]) &&
PersonRecords.ToArray()[0].AdditionalFields["age_neonate"] == "1")
{
buildPerson.YearOfBirth = observationPeriods[0].StartDate.Year;
}
else if (PersonRecords.ToArray()[0].AdditionalFields["ageday"] == null &&
PersonRecords.ToArray()[0].AdditionalFields["age_neonate"] != "1" || ageday < 0)
{
var dateOfBirth = observationPeriods[0].StartDate.AddDays(-180);
buildPerson.YearOfBirth = dateOfBirth.Year;
}
else if (ageday >= 0)
{
var dateOfBirth = observationPeriods[0].StartDate.AddDays(ageday * -1);
buildPerson.YearOfBirth = dateOfBirth.Year;
buildPerson.DayOfBirth = dateOfBirth.Day;
buildPerson.MonthOfBirth = dateOfBirth.Month;
}
if (!buildPerson.YearOfBirth.HasValue || buildPerson.YearOfBirth < 1907)
{
return(Attrition.ImplausibleYOBPast); // HIX-1417
}
// Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH
if (Excluded(buildPerson, observationPeriods))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
foreach (var visitOccurrence in BuildVisitOccurrences(VisitOccurrencesRaw.ToArray(), observationPeriods))
{
if (visitOccurrence.IdUndefined)
{
visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure =
BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
var drugCosts = BuildDrugCosts(drugExposures).ToArray();
var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray();
var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray();
var devicCosts = BuildDeviceCosts(deviceExposure).ToArray();
foreach (var cost in visitCosts)
{
if (cost.TotalPaid == 0)
{
continue;
}
var cost52 = new Cost(cost.PersonId)
{
CostId = Offset.GetKeyOffset(cost.PersonId).VisitCostId,
CurrencyConceptId = 44818668,
TypeId = 5032,
Domain = "Visit",
EventId = cost.Id,
TotalCharge = cost.TotalPaid
};
ChunkData.AddCostData(cost52);
}
var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriods).ToArray();
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(buildPerson, death, observationPeriods, payerPlanPeriods, drugExposures,
conditionOccurrences, procedureOccurrences, observations, measurements,
visitOccurrences.Values.ToArray(), null, cohort, deviceExposure, new Note[0]);
var pg = new PregnancyAlgorithm();
foreach (var pe in pg.GetPregnancyEpisodes(Vocabulary, buildPerson, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == buildPerson.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == buildPerson.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == buildPerson.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == buildPerson.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == buildPerson.PersonId).ToArray()))
{
pe.Id = Offset.GetKeyOffset(pe.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(pe);
}
return(Attrition.None);
}
/// <summary>
/// Build person entity and all person related entities like: DrugExposures, ConditionOccurrences, ProcedureOccurrences... from raw data sets
/// </summary>
public override Attrition Build(ChunkData data, KeyMasterOffsetManager om)
{
Offset = om;
ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
if (!ObservationPeriodsRaw.Any(op => op.StartDate < op.EndDate))
{
return(Attrition.InvalidObservationTime);
}
var op = ObservationPeriodsRaw.Where(op =>
op.StartDate < op.EndDate &&
op.StartDate.Year >= person.YearOfBirth &&
op.EndDate.Value.Year >= person.YearOfBirth &&
op.StartDate.Year <= DateTime.Now.Year).ToArray();
if (op.Length == 0)
{
return(Attrition.InvalidObservationTime);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, op).ToArray();
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.Where(pp =>
pp.StartDate.Year >= person.YearOfBirth &&
pp.EndDate.Value.Year >= person.YearOfBirth &&
pp.StartDate.Year <= DateTime.Now.Year).ToArray(), null).ToArray();
var visitDetails = new Dictionary <long, VisitDetail>();
var visitDetIds = new List <long>();
foreach (var vd in BuildVisitDetails(null, VisitOccurrencesRaw.Where(vo =>
vo.StartDate.Year >= person.YearOfBirth &&
vo.EndDate.Value.Year >= person.YearOfBirth &&
vo.StartDate.Year <= DateTime.Now.Year &&
vo.EndDate.Value.Year <= DateTime.Now.Year).ToArray(), observationPeriods).ToArray())
{
if (person.MonthOfBirth.HasValue && vd.StartDate.Year < person.YearOfBirth.Value &&
vd.StartDate.Month < person.MonthOfBirth ||
vd.StartDate.Year < person.YearOfBirth.Value)
{
if (vd.StartDate.Year < person.YearOfBirth.Value)
{
if (DateTime.TryParse(person.AdditionalFields["frd"], out var frd))
{
vd.StartDate = frd;
vd.EndDate = frd;
}
else
{
continue;
}
}
}
if (visitDetails.ContainsKey(vd.Id))
{
continue;
}
visitDetails.Add(vd.Id, vd);
visitDetIds.Add(vd.Id);
}
long?prevVisitDetId = null;
foreach (var visitId in visitDetIds.OrderBy(v => v))
{
if (prevVisitDetId.HasValue)
{
visitDetails[visitId].PrecedingVisitDetailId = prevVisitDetId;
}
prevVisitDetId = visitId;
}
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var byStartDate in visitDetails.Values.GroupBy(v => v.StartDate))
{
var vd = byStartDate.First();
var providerId = byStartDate.Min(v => v.ProviderId);
var careSiteId = byStartDate.Min(v => v.CareSiteId);
var sourceValue = byStartDate.Min(v => v.SourceValue);
var visitOccurrenceId = byStartDate.Min(v => v.Id);
var visitOccurrence = new VisitOccurrence(vd)
{
//Id = Offset.GetKeyOffset(vd.PersonId).VisitOccurrenceId,
Id = visitOccurrenceId,
ProviderId = providerId,
CareSiteId = careSiteId,
SourceValue = sourceValue
};
foreach (var visitDetail in byStartDate)
{
visitDetail.VisitOccurrenceId = visitOccurrence.Id;
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
SetVisitOccurrenceId(ConditionOccurrencesRaw, visitDetails);
SetVisitOccurrenceId(ProcedureOccurrencesRaw, visitDetails);
SetVisitOccurrenceId(DrugExposuresRaw, visitDetails);
SetVisitOccurrenceId(DeviceExposureRaw, visitDetails);
SetVisitOccurrenceId(ObservationsRaw, visitDetails);
SetVisitOccurrenceId(MeasurementsRaw, visitDetails);
var drugExposures = BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var deviceExposure = BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences = BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var procedureOccurrences = BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriods);
if (death != null)
{
person.TimeOfDeath = death.StartDate;
if (death.StartDate < observationPeriods.Min(op => op.StartDate))
{
return(Attrition.UnacceptablePatientQuality);
}
if (death.StartDate.Year < person.YearOfBirth || death.StartDate.Year > DateTime.Now.Year)
{
death = null;
}
}
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person,
death,
observationPeriods,
payerPlanPeriods,
Clean(drugExposures, person).ToArray(),
Clean(conditionOccurrences, person).ToArray(),
Clean(procedureOccurrences, person).ToArray(),
Clean(observations, person).ToArray(),
Clean(measurements, person).ToArray(),
visitOccurrences.Values.ToArray(),
visitDetails.Values.ToArray(), null,
Clean(deviceExposure, person).ToArray(), null);
var pg = new PregnancyAlgorithm();
foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriods,
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
r.Id = Offset.GetKeyOffset(r.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(r);
}
return(Attrition.None);
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager offset)
{
Offset = offset;
ChunkData = data;
var p = new List <Person>();
var op = new List <ObservationPeriod>();
var vo = new List <VisitOccurrence>();
var co = new List <ConditionOccurrence>();
var de = new List <DrugExposure>();
var o = new List <Observation>();
foreach (var record in PersonRecords)
{
var table = record.AdditionalFields["table_name"]?.ToLower();
var variable = record.AdditionalFields["variable"]?.ToLower();
var value = record.AdditionalFields["value"]?.ToLower();
var description = string.Empty;
if (record.AdditionalFields.ContainsKey("value_description"))
{
description = record.AdditionalFields["value_description"]?.ToLower();
}
if (string.IsNullOrEmpty(table) || string.IsNullOrEmpty(variable))
{
continue;
}
decimal?decimalValue = null;
if (decimal.TryParse(value, out var dValue))
{
decimalValue = dValue;
}
var valueAsString = value;
if (!string.IsNullOrEmpty(description))
{
valueAsString = description;
}
var year = 2000;
if (table.StartsWith("demo"))
{
p.Add(record);
// GetYear => empty -> 2000 'B' -> 2002, 'C' -> 2004, 'D' -> 2006 ....
year = GetYear(table.Replace("demo", "").Replace("_", ""));
if (op.Count == 0)
{
op.Add(new ObservationPeriod
{
Id = Offset.GetKeyOffset(record.PersonId).ObservationId,
PersonId = record.PersonId,
TypeConceptId = 0,
StartDate = new DateTime(year - 1, 1, 1),
EndDate = new DateTime(year, 12, 31)
});
vo.Add(new VisitOccurrence(record)
{
Id = Offset.GetKeyOffset(record.PersonId).VisitOccurrenceId,
ConceptId = 0,
StartDate = new DateTime(year - 1, 1, 1),
EndDate = new DateTime(year - 1, 1, 1),
TypeConceptId = 44818519
});
}
}
else if (table.StartsWith("rxq_rx") && (variable.StartsWith("rxddrgid") || variable.StartsWith("rxddrug")))
{
year = GetYear(table.Replace("rxq_rx", "").Replace("_", ""));
string sourceValue = null;
if (!string.IsNullOrEmpty(value) && value.Contains('_'))
{
var values = value.Split('_');
// use the characters after the last underscore(_)
// 60000_1_7__d00910
if (table == "rxq_rx" || table == "rxq_rx_b")
{
sourceValue = values.Last();
}
// use the characters before the first underscore (_)
// d00848_365_1_DICLOFENAC
else
{
sourceValue = values[0];
}
}
var result = Vocabulary.Lookup(sourceValue, "Drug", DateTime.MinValue);
if (result.Any())
{
foreach (var lookupValue in result)
{
var conceptId = lookupValue.ConceptId;
if (conceptId.HasValue)
{
de.Add(new DrugExposure(record)
{
Id = Offset.GetKeyOffset(record.PersonId).DrugExposureId,
ConceptId = conceptId.Value,
StartDate = new DateTime(year - 1, 1, 1),
EndDate = new DateTime(year - 1, 1, 1),
SourceValue = sourceValue,
TypeConceptId = 44787730
});
}
}
}
}
else if (!string.IsNullOrEmpty(value))
{
var yearField = table.Split('_').Last();
year = GetYear(yearField == table ? null : yearField);
var sourceValue = variable + "_" + value;
if (variable == "bmxbmi")
{
int?conceptId = null;
if (double.TryParse(value, out var bmxbmi))
{
if (bmxbmi < 18.5)
{
conceptId = 435372;
}
else if (bmxbmi > 25.0 && bmxbmi < 29.9)
{
conceptId = 437525;
}
else
{
conceptId = 4215968;
}
}
co.Add(new ConditionOccurrence(record)
{
Id = Offset.GetKeyOffset(record.PersonId).ConditionOccurrenceId,
ConceptId = conceptId.Value,
StartDate = new DateTime(year, 1, 1),
EndDate = null,
TypeConceptId = 45905770,
SourceValue = sourceValue
});
}
else
{
var result = Vocabulary.Lookup(sourceValue, "Condition", DateTime.MinValue);
if (result.Any())
{
foreach (var lookupValue in result)
{
var conceptId = lookupValue.ConceptId;
if (conceptId.HasValue)
{
co.Add(new ConditionOccurrence(record)
{
Id = Offset.GetKeyOffset(record.PersonId).ConditionOccurrenceId,
ConceptId = conceptId.Value,
StartDate = new DateTime(year, 1, 1),
EndDate = null,
TypeConceptId = 45905770,
SourceValue = sourceValue
});
}
}
}
}
}
o.Add(new Observation(record)
{
Id = Offset.GetKeyOffset(record.PersonId).ObservationId,
ValueAsNumber = decimalValue,
ValueAsString = decimalValue.HasValue ? valueAsString : value,
ConceptId = 0,
TypeConceptId = 44814721,
SourceValue = variable,
StartDate = new DateTime(year, 1, 1)
});
}
var r = BuildPerson(p);
var person = r.Key;
if (person == null)
{
Complete = true;
return(r.Value);
}
if (vo.Count > 0)
{
foreach (var entity in o)
{
entity.VisitOccurrenceId = vo[0].Id;
}
foreach (var entity in de)
{
entity.VisitOccurrenceId = vo[0].Id;
}
foreach (var entity in co)
{
entity.VisitOccurrenceId = vo[0].Id;
}
}
AddToChunk(person, null, op.ToArray(), new PayerPlanPeriod[0], de.ToArray(),
co.ToArray(), new ProcedureOccurrence[0], o.ToArray(), new Measurement[0],
vo.ToArray(), new VisitDetail[0], new Cohort[0], new DeviceExposure[0], new Note[0]);
Complete = true;
var pg = new PregnancyAlgorithm();
foreach (var pe in pg.GetPregnancyEpisodes(Vocabulary, person, op.ToArray(),
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
pe.Id = Offset.GetKeyOffset(pe.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(pe);
}
return(Attrition.None);
}
public override Attrition Build(ChunkData data, KeyMasterOffsetManager om)
{
this.Offset = om;
this.ChunkData = data;
var result = BuildPerson(PersonRecords.ToList());
var person = result.Key;
if (person == null)
{
return(result.Value);
}
var observationPeriods =
BuildObservationPeriods(person.ObservationPeriodGap, ObservationPeriodsRaw.ToArray()).ToArray();
var payerPlanPeriods = BuildPayerPlanPeriods(PayerPlanPeriodsRaw.ToArray(), null).ToArray();
var dedupVisits = VisitOccurrencesRaw.Where(v => v.AdditionalFields["sort_index"] == "1").ToArray();
var vDetails = BuildVisitDetails(null, dedupVisits, observationPeriods).ToArray();
var visitOccurrences = new Dictionary <long, VisitOccurrence>();
var visitIds = new List <long>();
foreach (var visitOccurrence in BuildVisitOccurrences(dedupVisits, observationPeriods))
{
visitOccurrence.Id = Offset.GetKeyOffset(visitOccurrence.PersonId).VisitOccurrenceId;
visitOccurrence.TypeConceptId = 44818517;
if (!visitOccurrence.EndDate.HasValue)
{
visitOccurrence.EndDate = visitOccurrence.StartDate;
}
if (!_rawVisits.ContainsKey(visitOccurrence.SourceRecordGuid))
{
_rawVisits.Add(visitOccurrence.SourceRecordGuid, visitOccurrence);
}
visitOccurrences.Add(visitOccurrence.Id, visitOccurrence);
visitIds.Add(visitOccurrence.Id);
}
var visitDetails = new Dictionary <long, VisitDetail>();
foreach (var group in vDetails.GroupBy(d => d.Id))
{
var vd = group.First();
var visitOccurrenceId = GetVisitOccurrenceId(vd);
if (visitOccurrenceId.HasValue && visitOccurrences.ContainsKey(visitOccurrenceId.Value))
{
vd.VisitOccurrenceId = visitOccurrenceId;
}
else
{
var visits = visitOccurrences.Values
.Where(vo => vd.StartDate.Date >= vo.StartDate.Date &&
vd.EndDate.Value.Date <= vo.EndDate.Value.Date).ToArray();
if (visits.Any())
{
vd.VisitOccurrenceId = visits.First().Id;
}
}
visitDetails.Add(vd.Id, vd);
}
long?prevVisitId = null;
foreach (var visitId in visitIds.OrderBy(v => v))
{
if (prevVisitId.HasValue)
{
visitOccurrences[visitId].PrecedingVisitOccurrenceId = prevVisitId;
}
prevVisitId = visitId;
}
var drugExposures =
BuildDrugExposures(DrugExposuresRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
var conditionOccurrences =
BuildConditionOccurrences(ConditionOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var procedureOccurrences =
BuildProcedureOccurrences(ProcedureOccurrencesRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var observations = BuildObservations(ObservationsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var measurements = BuildMeasurement(MeasurementsRaw.ToArray(), visitOccurrences, observationPeriods)
.ToArray();
var deviceExposure =
BuildDeviceExposure(DeviceExposureRaw.ToArray(), visitOccurrences, observationPeriods).ToArray();
// set corresponding PlanPeriodIds to drug exposure entities and procedure occurrence entities
SetPayerPlanPeriodId(payerPlanPeriods, drugExposures, procedureOccurrences,
visitOccurrences.Values.ToArray(),
deviceExposure);
var notes = BuildNote(NoteRecords.ToArray(), visitOccurrences, observationPeriods).ToArray();
List <DateTime?> mins = new List <DateTime?>();
List <DateTime?> maxs = new List <DateTime?>();
mins.Add(GetMinDate(drugExposures));
mins.Add(GetMinDate(conditionOccurrences));
mins.Add(GetMinDate(procedureOccurrences));
mins.Add(GetMinDate(observations));
mins.Add(GetMinDate(deviceExposure));
mins.Add(GetMinDate(measurements));
mins.Add(GetMinDate(visitOccurrences.Values));
mins.Add(GetMinDate(visitDetails.Values));
maxs.Add(GetMaxDate(drugExposures));
maxs.Add(GetMaxDate(conditionOccurrences));
maxs.Add(GetMaxDate(procedureOccurrences));
maxs.Add(GetMaxDate(observations));
maxs.Add(GetMaxDate(deviceExposure));
maxs.Add(GetMaxDate(measurements));
maxs.Add(GetMaxDate(visitOccurrences.Values));
maxs.Add(GetMaxDate(visitDetails.Values));
var min = mins.Min();
var max = maxs.Max();
var observationPeriodsFinal = new List <ObservationPeriod>(1)
{
observationPeriods[0]
};
if (min.HasValue)
{
observationPeriodsFinal[0].StartDate = min.Value;
if (max.HasValue)
{
observationPeriodsFinal[0].EndDate = max;
}
else
{
observationPeriodsFinal[0].EndDate = min.Value;
}
}
// Delete any individual that has an OBSERVATION_PERIOD that is >= 2 years prior to the YEAR_OF_BIRTH
if (Excluded(person, observationPeriodsFinal))
{
return(Attrition.ImplausibleYOBPostEarliestOP);
}
// set corresponding ProviderIds
SetProviderIds(drugExposures);
SetProviderIds(conditionOccurrences);
SetProviderIds(procedureOccurrences);
SetProviderIds(observations);
SetProviderIds(deviceExposure);
SetProviderIds(measurements);
SetProviderIds(visitOccurrences.Values);
SetProviderIds(visitDetails.Values);
SetVisitOccurrenceId(conditionOccurrences, visitDetails.Values.ToArray());
SetVisitOccurrenceId(procedureOccurrences, visitDetails.Values.ToArray());
SetVisitOccurrenceId(drugExposures, visitDetails.Values.ToArray());
SetVisitOccurrenceId(deviceExposure, visitDetails.Values.ToArray());
SetVisitOccurrenceId(observations, visitDetails.Values.ToArray());
SetVisitOccurrenceId(measurements, visitDetails.Values.ToArray());
//SetVisitOccurrenceId(visitDetails.Values, visitOccurrences.Values.ToArray());
SetProviderIds(drugExposures, visitOccurrences);
SetProviderIds(conditionOccurrences, visitOccurrences);
SetProviderIds(measurements, visitOccurrences);
SetProviderIds(procedureOccurrences, visitOccurrences);
SetProviderIds(deviceExposure, visitOccurrences);
SetProviderIds(observations, visitOccurrences);
//SetProviderIds(notes, visitOccurrences); TMP: NOTE
if (notes.Any())
{
if (visitOccurrences.Count > 0)
{
foreach (var e in notes.Where(e => !e.ProviderId.HasValue))
{
if (e.AdditionalFields != null && e.AdditionalFields.ContainsKey("encid") && !string.IsNullOrEmpty(e.AdditionalFields["encid"]))
{
var encid = e.AdditionalFields["encid"];
var vd = visitDetails.Values.FirstOrDefault(v => v.AdditionalFields["encid"] == encid);
if (vd == null)
{
continue;
}
e.VisitDetailId = vd.Id;
e.VisitOccurrenceId = vd.VisitOccurrenceId;
}
if (!e.VisitOccurrenceId.HasValue)
{
continue;
}
if (visitOccurrences.ContainsKey(e.VisitOccurrenceId.Value))
{
e.ProviderId = visitOccurrences[e.VisitOccurrenceId.Value].ProviderId;
}
}
}
}
var death = BuildDeath(DeathRecords.ToArray(), visitOccurrences, observationPeriodsFinal.ToArray());
var drugCosts = BuildDrugCosts(drugExposures).ToArray();
var procedureCosts = BuildProcedureCosts(procedureOccurrences).ToArray();
var visitCosts = BuildVisitCosts(visitOccurrences.Values.ToArray()).ToArray();
var devicCosts = BuildDeviceCosts(deviceExposure).ToArray();
var cohort = BuildCohort(CohortRecords.ToArray(), observationPeriodsFinal.ToArray()).ToArray();
if (death != null)
{
death = CleanUpDeath(visitDetails.Values, death);
death = CleanUpDeath(visitOccurrences.Values, death);
death = CleanUpDeath(drugExposures, death);
death = CleanUpDeath(conditionOccurrences, death);
death = CleanUpDeath(procedureOccurrences, death);
death = CleanUpDeath(measurements, death);
death = CleanUpDeath(observations, death);
death = CleanUpDeath(deviceExposure, death);
}
// push built entities to ChunkBuilder for further save to CDM database
AddToChunk(person, death, observationPeriodsFinal.ToArray(), payerPlanPeriods, drugExposures,
conditionOccurrences, procedureOccurrences, observations, measurements,
visitOccurrences.Values.ToArray(), visitDetails.Values.ToArray(), cohort, deviceExposure, notes);
var pg = new PregnancyAlgorithm();
foreach (var r in pg.GetPregnancyEpisodes(Vocabulary, person, observationPeriodsFinal.ToArray(),
ChunkData.ConditionOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.ProcedureOccurrences.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Observations.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.Measurements.Where(e => e.PersonId == person.PersonId).ToArray(),
ChunkData.DrugExposures.Where(e => e.PersonId == person.PersonId).ToArray()))
{
r.Id = Offset.GetKeyOffset(r.PersonId).ConditionEraId;
ChunkData.ConditionEra.Add(r);
}
return(Attrition.None);
}