public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataReader reader, KeyMasterOffset keyMaster)
{
var concepts = base.GetConcepts(concept, reader, keyMaster).ToList();
if (concepts.Count > 0)
{
var baseConcept = concepts[0];
var died = reader.GetInt(Died);
if (died.HasValue && died.Value == 1)
{
yield return new Death((Entity) baseConcept) {Primary = true};
}
else //Medical claim diagnostic code indicating death
{
if (Concepts.Count() > 1)
{
for (int i = 1; i < Concepts.Count(); i++)
{
foreach (var entity in PopulateDeath(reader, Concepts[i], baseConcept)) yield return entity;
}
}
else
{
foreach (var entity in PopulateDeath(reader, Concepts[0], baseConcept)) yield return entity;
}
}
}
}
private static void UpdateOffset(KeyMasterOffset offset)
{
var visitOffset = 0;
if (offset.VisitOccurrenceIdChanged)
visitOffset = offset.VisitOccurrenceId;
var periodOffset = 0;
if (offset.PayerPlanPeriodIdChanged)
periodOffset = offset.PayerPlanPeriodId;
var drugOffset = 0;
if (offset.DrugExposureIdChanged)
drugOffset = offset.DrugExposureId;
var procedureOffset = 0;
if (offset.ProcedureOccurrenceIdChanged)
procedureOffset = offset.ProcedureOccurrenceId;
var dbKeyOffset = new DbKeyOffset(Settings.Current.Building.BuilderConnectionString);
foreach (var reader in dbKeyOffset.Update(Settings.Current.Building.Id.Value, visitOffset, periodOffset, drugOffset, procedureOffset))
{
offset.VisitOccurrenceOffset = reader.GetLong("VisitOccurrence").Value - visitOffset;
offset.PayerPlanPeriodOffset = reader.GetLong("PayerPlanPeriod").Value - periodOffset;
offset.DrugExposureOffset = reader.GetLong("DrugExposure").Value - drugOffset;
offset.ProcedureOccurrenceOffset = reader.GetLong("ProcedureOccurrence").Value - procedureOffset;
}
}
public ChunkData(int chunkId, int subChunkId)
{
ChunkId = chunkId;
SubChunkId = subChunkId;
KeyMasterOffset = new KeyMasterOffset();
Persons = new ConcurrentQueue <Person>();
Deaths = new ConcurrentQueue <Death>();
ObservationPeriods = new ConcurrentQueue <ObservationPeriod>();
PayerPlanPeriods = new ConcurrentQueue <PayerPlanPeriod>();
ConditionOccurrences = new ConcurrentQueue <ConditionOccurrence>();
DrugExposures = new ConcurrentQueue <DrugExposure>();
ProcedureOccurrences = new ConcurrentQueue <ProcedureOccurrence>();
Observations = new ConcurrentQueue <Observation>();
Measurements = new ConcurrentQueue <Measurement>();
VisitOccurrences = new ConcurrentQueue <VisitOccurrence>();
VisitCost = new ConcurrentQueue <VisitCost>();
DrugCost = new ConcurrentQueue <DrugCost>();
ProcedureCost = new ConcurrentQueue <ProcedureCost>();
ConditionEra = new ConcurrentQueue <EraEntity>();
DrugEra = new ConcurrentQueue <EraEntity>();
Cohort = new ConcurrentQueue <Cohort>();
DeviceExposure = new ConcurrentQueue <DeviceExposure>();
DeviceCost = new ConcurrentQueue <DeviceCost>();
Cost = new ConcurrentQueue <Cost>();
Note = new ConcurrentQueue <Note>();
}
public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataReader reader, KeyMasterOffset keyMaster)
{
var id = string.IsNullOrEmpty(Id) ? KeyMaster.GetLocationId() : int.Parse(reader[Id].ToString());
yield return new Location
{
Id = id,
State = reader.GetString(State),
SourceValue = reader.GetString(SourceValue)
};
}
public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataReader reader, KeyMasterOffset keyMaster)
{
long? relevantConditionConceptId = 0;
DateTime? endDate = null;
if(Concepts.Length == 2)
{
var relevantConcepts = Concepts[1].GetValues(Vocabulary, Concepts[1].Fields[0], reader);
if (relevantConcepts.Count > 0)
relevantConditionConceptId = relevantConcepts.Min(c => c.ConceptId);
}
foreach (var c in base.GetConcepts(concept, reader, keyMaster))
{
var e = (Entity) c;
var calculatedDaysSupply = GetDaysSupply(reader, e);
if (!string.IsNullOrEmpty(EndDate))
{
if (calculatedDaysSupply > 0 && calculatedDaysSupply <= 365)
endDate = e.StartDate.AddDays(calculatedDaysSupply - 1);
else
endDate = reader.GetDateTime(EndDate);
}
var refillString = reader.GetString(Refill);
int? refill = null;
if (!string.IsNullOrEmpty(refillString))
{
int refillValue;
// in Optum Refill looks like: 01, 02...
int.TryParse(refillString, out refillValue);
refill = refillValue;
}
yield return new DrugExposure(e)
{
Id = keyMaster.DrugExposureId,
Refills = refill,
DaysSupply = reader.GetInt(DaysSupply),
CalculatedDaysSupply = calculatedDaysSupply,
Quantity = GetQuantity(reader),
RelevantConditionConceptId = relevantConditionConceptId,
EndDate = endDate == DateTime.MinValue ? null : endDate,
GetEraConceptIdsCall = Vocabulary.LookupIngredientLevel
};
}
}
public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataReader reader, KeyMasterOffset keyMaster)
{
var id = string.IsNullOrEmpty(Id) ? KeyMaster.GetCareSiteId() : int.Parse(reader[Id].ToString());
yield return new CareSite
{
Id = id,
LocationId = reader.GetLong(LocationId).Value,
PlaceOfSvcSourceValue = reader.GetString(PlaceOfSvcSourceValue),
SourceValue = id.ToString()
};
}
public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataReader reader, KeyMasterOffset keyMaster)
{
var conceptField = Concepts[0].Fields[0];
var id = string.IsNullOrEmpty(Id) ? KeyMaster.GetOrganizationId() : int.Parse(reader[Id].ToString());
yield return new Organization
{
Id = id,
ConceptId = conceptField.DefaultConceptId.HasValue ? conceptField.DefaultConceptId.Value : 0,
LocationId = 0,
SourceValue = reader[conceptField.Key].ToString().Trim(),
PlaceOfSvcSourceValue = string.IsNullOrEmpty(PlaceOfService) ? "" : reader[PlaceOfService].ToString()
};
}
public ChunkData()
{
KeyMasterOffset = new KeyMasterOffset();
Persons = new ConcurrentStack <Person>();
Deaths = new ConcurrentStack <Death>();
ObservationPeriods = new ConcurrentStack <ObservationPeriod>();
PayerPlanPeriods = new ConcurrentStack <PayerPlanPeriod>();
ConditionOccurrences = new ConcurrentStack <ConditionOccurrence>();
DrugExposures = new ConcurrentStack <DrugExposure>();
ProcedureOccurrences = new ConcurrentStack <ProcedureOccurrence>();
Observations = new ConcurrentStack <Observation>();
VisitOccurrences = new ConcurrentStack <VisitOccurrence>();
DrugCost = new ConcurrentStack <DrugCost>();
ProcedureCost = new ConcurrentStack <ProcedureCost>();
ConditionEra = new ConcurrentStack <EraEntity>();
DrugEra = new ConcurrentStack <EraEntity>();
Cohort = new ConcurrentStack <Cohort>();
}
public ChunkData()
{
KeyMasterOffset = new KeyMasterOffset();
Persons = new ConcurrentStack<Person>();
Deaths = new ConcurrentStack<Death>();
ObservationPeriods = new ConcurrentStack<ObservationPeriod>();
PayerPlanPeriods = new ConcurrentStack<PayerPlanPeriod>();
ConditionOccurrences = new ConcurrentStack<ConditionOccurrence>();
DrugExposures = new ConcurrentStack<DrugExposure>();
ProcedureOccurrences = new ConcurrentStack<ProcedureOccurrence>();
Observations = new ConcurrentStack<Observation>();
VisitOccurrences = new ConcurrentStack<VisitOccurrence>();
DrugCost = new ConcurrentStack<DrugCost>();
ProcedureCost = new ConcurrentStack<ProcedureCost>();
ConditionEra = new ConcurrentStack<EraEntity>();
DrugEra = new ConcurrentStack<EraEntity>();
Cohort = new ConcurrentStack<Cohort>();
}
public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataReader reader, KeyMasterOffset keyMaster)
{
long? relevantConditionConceptId = null;
if (Concepts.Length == 2)
{
var relevantConcepts = Concepts[1].GetValues(Vocabulary, Concepts[1].Fields[0], reader);
if (relevantConcepts.Count > 0)
relevantConditionConceptId = relevantConcepts.Min(c => c.ConceptId);
}
foreach (var e in base.GetConcepts(concept, reader, keyMaster))
{
yield return
new ProcedureOccurrence((Entity) e)
{
Id = keyMaster.ProcedureOccurrenceId,
ReleventConditionConceptId = relevantConditionConceptId
};
}
}
public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataReader reader, KeyMasterOffset keyMaster)
{
var visitOccurrences = base.GetConcepts(concept, reader, keyMaster).ToList();
if(visitOccurrences.Count > 0)
{
var id = reader.GetLong(Id);
var visitOccurrence = new VisitOccurrence((Entity) visitOccurrences[0])
{
CareSiteId = reader.GetInt(CareSiteId) ?? 0
};
if(id.HasValue)
{
visitOccurrence.Id = id.Value;
}
else
{
visitOccurrence.IdUndefined = true;
}
yield return visitOccurrence;
}
}
public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataReader reader, KeyMasterOffset keyMaster)
{
var personId = reader.GetLong(PersonId);
var startDate = reader.GetDateTime(StartDate);
var endDate = reader.GetDateTime(EndDate);
if (personId.HasValue)
{
Dictionary<string, string> additionalFields = null;
if (AdditionalFields != null)
{
additionalFields = new Dictionary<string, string>(AdditionalFields.Length);
foreach (var additionalField in AdditionalFields)
{
additionalFields.Add(additionalField.ToLower(), reader.GetString(additionalField));
}
}
var payerSourceValue = reader.GetString(PayerSource);
if (string.IsNullOrWhiteSpace(payerSourceValue))
payerSourceValue = null;
yield return new PayerPlanPeriod
{
PersonId = personId.Value,
StartDate = startDate,
EndDate = endDate,
PayerSourceValue = payerSourceValue,
PlanSourceValue = reader.GetString(PlanSource),
FamilySourceValue = reader.GetString(FamilySource),
VisitOccurrenceId = reader.GetLong(VisitOccurrenceId),
AdditionalFields = additionalFields
};
}
}
// A custom DataReader is implemented to prevent the need for the HashSet to be transformed to a DataTable for loading by SqlBulkCopy
public PayerPlanPeriodDataReader(List<PayerPlanPeriod> batch, KeyMasterOffset offset)
{
enumerator = batch.GetEnumerator();
this.offset = offset;
}
public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataReader reader, KeyMasterOffset keyMaster)
{
long id;
var idUndefined = false;
if (string.IsNullOrEmpty(Id))
{
id = KeyMaster.GetProviderId();
idUndefined = true;
}
else
{
id = reader.GetLong(Id).Value;
}
if (concept == null)
{
yield return new Provider
{
Id = id,
CareSiteId = reader.GetInt(CareSiteId) ?? 0,
ProviderSourceValue = reader.GetString(ProviderSourceValue),
SourceValue = reader.GetString(SpecialtySourceValue)
};
}
else
{
var conceptField = concept.Fields[0];
var source = reader.GetString(conceptField.Key) ?? reader.GetString(conceptField.SourceKey);
if (source != null && source.Length == 0)
source = null;
var specialtyConceptIds = concept.GetValues(Vocabulary, conceptField, reader).ToList();
long? specialtyConcept = null;
//(Unknown Physician Specialty)
long defaultConceptId = 38004514;
if (conceptField.DefaultConceptId.HasValue)
{
defaultConceptId = conceptField.DefaultConceptId.Value;
}
if (specialtyConceptIds.Count > 0)
{
specialtyConcept = specialtyConceptIds[0].ConceptId;
}
yield return new Provider
{
Id = id,
IdUndefined = idUndefined,
CareSiteId = reader.GetInt(CareSiteId) ?? 0,
ConceptId = specialtyConcept.HasValue ? specialtyConcept.Value : defaultConceptId,
ProviderSourceValue = reader.GetString(ProviderSourceValue),
SourceValue =
string.IsNullOrEmpty(conceptField.SourceKey) ? source : reader.GetString(conceptField.SourceKey)
};
}
}
// A custom DataReader is implemented to prevent the need for the HashSet to be transformed to a DataTable for loading by SqlBulkCopy
public ConditionOccurrenceDataReader(List<ConditionOccurrence> batch, KeyMasterOffset offset)
{
conditionEnumerator = batch.GetEnumerator();
this.offset = offset;
}
public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataReader reader, KeyMasterOffset keyMaster)
{
var locationSourceValue = reader.GetString(Location);
int? locationId = null;
if (!string.IsNullOrEmpty(Location))
{
locationId = Vocabulary.LookupLocation(locationSourceValue);
}
var observationPeriodGap = 32;
if(!string.IsNullOrEmpty(ObservationPeriodGap))
{
observationPeriodGap = int.Parse(ObservationPeriodGap);
}
var genderSource = reader.GetString(Gender);
var genderConceptId = Vocabulary.LookupGender(genderSource);
var personId = reader.GetLong(PersonId);
if(personId.HasValue)
{
yield return new Person
{
ObservationPeriodGap = observationPeriodGap,
PersonId = personId.Value,
StartDate = reader.GetDateTime(StartDate),
EndDate = reader.GetDateTime(EndDate),
PersonSourceValue = reader.GetString(PersonSourceValue),
GenderSourceValue = genderSource,
GenderConceptId = genderConceptId,
LocationId = locationId,
YearOfBirth = reader.GetInt(YearOfBirth),
MonthOfBirth = reader.GetInt(MonthOfBirth),
DayOfBirth = reader.GetInt(DayOfBirth),
LocationSourceValue = locationSourceValue,
CareSiteId = reader.GetInt(CareSiteId),
EthnicitySourceValue = reader.GetString(Ethnicity),
EthnicityConceptId = reader.GetInt(EthnicityConceptId) ?? 0,
RaceSourceValue = reader.GetString(Race),
RaceConceptId = reader.GetInt(RaceConceptId) ?? 0,
ProviderId = reader.GetInt(ProviderId)
};
}
}
public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataReader reader, KeyMasterOffset keyMaster)
{
var obsConcepts = base.GetConcepts(concept, reader, keyMaster).ToList();
if (obsConcepts.Count > 0)
{
var unitConcept = GetUnitConcept(reader);
long? relevantConditionConceptId = null;
if (Concepts.Length == 3)
{
var relevantConditionConcepts = Concepts[2].GetValues(Vocabulary, Concepts[2].Fields[0], reader);
if(relevantConditionConcepts.Count > 0)
relevantConditionConceptId = relevantConditionConcepts.Min(c => c.ConceptId);
}
var createObservation = new Func<Observation>(() => new Observation((Entity)obsConcepts[0])
{
SourceValue = string.IsNullOrWhiteSpace(obsConcepts[0].SourceValue) ? null : obsConcepts[0].SourceValue,
RangeLow = reader.GetDecimal(RangeLow),
RangeHigh = reader.GetDecimal(RangeHigh),
UnitsConceptId = unitConcept.Key ?? 0,
UnitsSourceValue = string.IsNullOrWhiteSpace(unitConcept.Value) ? null : unitConcept.Value,
ValueAsConceptId = reader.GetInt(ValueAsConceptId) ?? 0,
RelevantConditionConceptId = relevantConditionConceptId ?? 0
});
if (ValuesAsNumber != null && ValuesAsNumber.Length > 1)
{
foreach (var valueAsNumber in ValuesAsNumber)
{
var value = reader.GetDecimal(valueAsNumber);
if (value != null)
{
var observation = createObservation();
observation.ValueAsNumber = Math.Round(value.Value, 3);
observation.SourceValue = observation.ValueAsNumber.ToString();
observation.TypeConceptId = 38000277;
yield return observation;
}
}
}
else if (ValuesAsString != null && ValuesAsString.Length > 1)
{
foreach (var valueAsString in ValuesAsString)
{
var value = reader.GetString(valueAsString);
if (!string.IsNullOrEmpty(value) && value != "-" && value != "9") //TMP
{
var observation = createObservation();
observation.ValueAsString = reader.GetString(valueAsString);
observation.SourceValue = valueAsString;
observation.TypeConceptId = 38000278;
yield return observation;
}
}
}
else if (!string.IsNullOrEmpty(obsConcepts[0].SourceValue))
{
var observation = createObservation();
if (ValuesAsNumber != null && ValuesAsNumber.Length > 0)
{
observation.ValueAsNumber = reader.GetDecimal(ValuesAsNumber[0]);
if (observation.ValueAsNumber.HasValue)
{
observation.ValueAsNumber = Math.Round(observation.ValueAsNumber.Value, 3);
}
}
if (ValuesAsString != null && ValuesAsString.Length > 0)
{
observation.ValueAsString = reader.GetString(ValuesAsString[0]);
// Workaround, map ValueAsString (usind SourceLookup) and reset SourceValue to original value
if (observation.AdditionalFields != null && observation.AdditionalFields.ContainsKey("original_source"))
{
observation.ValueAsString = observation.SourceValue;
observation.SourceValue = observation.AdditionalFields["original_source"];
}
}
yield return observation;
}
}
}
// A custom DataReader is implemented to prevent the need for the HashSet to be transformed to a DataTable for loading by SqlBulkCopy
public ProcedureCostDataReader(List<ProcedureCost> batch, KeyMasterOffset offset)
{
enumerator = batch.GetEnumerator();
this.offset = offset;
}
public static List<PayerPlanPeriod> GetPayerPlanPeriods(PayerPlanPeriod[] entities, Func<PayerPlanPeriod, PayerPlanPeriod, bool> canBeCombined, KeyMasterOffset keyMaster)
{
return GetPayerPlanPeriods(entities, 32, canBeCombined, keyMaster);
}
// A custom DataReader is implemented to prevent the need for the HashSet to be transformed to a DataTable for loading by SqlBulkCopy
public VisitOccurrenceDataReader(List<VisitOccurrence> batch, KeyMasterOffset offset)
{
visitEnumerator = batch.GetEnumerator();
this.offset = offset;
}
// A custom DataReader is implemented to prevent the need for the HashSet to be transformed to a DataTable for loading by SqlBulkCopy
public DrugExposureDataReader(List<DrugExposure> batch, KeyMasterOffset offset)
{
exposureEnumerator = batch.GetEnumerator();
this.offset = offset;
}
// A custom DataReader is implemented to prevent the need for the HashSet to be transformed to a DataTable for loading by SqlBulkCopy
public ProcedureOccurrenceDataReader(List<ProcedureOccurrence> batch, KeyMasterOffset offset)
{
procedureEnumerator = batch.GetEnumerator();
this.offset = offset;
}
public static List<PayerPlanPeriod> GetPayerPlanPeriods(PayerPlanPeriod[] entities, int gap, Func<PayerPlanPeriod, PayerPlanPeriod, bool> canBeCombined, KeyMasterOffset keyMaster)
{
var result = new List<PayerPlanPeriod>();
if (entities.Length == 0) return result;
var ordered =
entities.OrderBy(e => e.StartDate).ToList();
var payerPlanPeriod = ordered[0];
var list = new List<PayerPlanPeriod>();
foreach (var entity in ordered)
{
if (canBeCombined(entity, payerPlanPeriod))
{
list.Add(entity);
continue;
}
result.AddRange(GetEras(list, gap, -1).Select(i => new PayerPlanPeriod
{
Id = keyMaster.PayerPlanPeriodId,
PersonId = i.PersonId,
StartDate = i.StartDate,
EndDate = i.EndDate,
PayerSourceValue =
payerPlanPeriod.PayerSourceValue,
FamilySourceValue =
payerPlanPeriod.FamilySourceValue,
PlanSourceValue =
payerPlanPeriod.PlanSourceValue
}));
list.Clear();
payerPlanPeriod = entity;
list.Add(entity);
}
result.AddRange(GetEras(list, gap, -1).Select(i => new PayerPlanPeriod
{
Id = keyMaster.PayerPlanPeriodId,
PersonId = i.PersonId,
StartDate = i.StartDate,
EndDate = i.EndDate,
PayerSourceValue =
payerPlanPeriod.PayerSourceValue,
FamilySourceValue =
payerPlanPeriod.FamilySourceValue,
PlanSourceValue =
payerPlanPeriod.PlanSourceValue
}));
return result;
}
// A custom DataReader is implemented to prevent the need for the HashSet to be transformed to a DataTable for loading by SqlBulkCopy
public ObservationDataReader(List<Observation> batch, KeyMasterOffset offset)
{
observationEnumerator = batch.GetEnumerator();
this.offset = offset;
}
public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataReader reader, KeyMasterOffset keyMaster)
{
return base.GetConcepts(concept, reader, keyMaster).Select(e => new ConditionOccurrence((Entity)e));
}
public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataReader reader, KeyMasterOffset keyMaster)
{
throw new NotImplementedException();
}
public virtual IEnumerable<IEntity> GetConcepts(Concept concept, IDataReader reader, KeyMasterOffset keyMaster)
{
var personId = reader.GetLong(PersonId);
if (personId.HasValue)
{
var startDate = reader.GetDateTime(StartDate);
var endDate = startDate;
if (!string.IsNullOrEmpty(EndDate))
endDate = reader.GetDateTime(EndDate);
Dictionary<string, string> additionalFields = null;
if (AdditionalFields != null)
{
additionalFields = new Dictionary<string, string>(AdditionalFields.Length);
foreach (var additionalField in AdditionalFields)
{
additionalFields.Add(additionalField.ToLower(), reader.GetString(additionalField));
}
}
foreach (var field in concept.Fields)
{
var sourceValue = field.DefaultSource;
if (string.IsNullOrEmpty(sourceValue))
sourceValue = reader.GetString(field.Key);
if (!field.IsNullable && string.IsNullOrEmpty(sourceValue) && field.DefaultConceptId == null && field.ConceptId == null)
continue;
// Used when: field.Key used for conceptId mapping and
// field.SourceKey used for SourceValue (by default field.Key and field.SourceKey are identical)
if (!string.IsNullOrEmpty(field.SourceKey))
sourceValue = reader.GetString(field.SourceKey);
if(!string.IsNullOrEmpty(concept.SourceLookup))
{
var source = Vocabulary.LookupSource(sourceValue, concept.SourceLookup);
if (!string.IsNullOrEmpty(source))
sourceValue = source;
}
foreach (var lookupValue in concept.GetValues(Vocabulary, field, reader))
{
var cId = lookupValue.ConceptId;
if (!cId.HasValue && field.DefaultConceptId.HasValue)
cId = field.DefaultConceptId;
if (!concept.IdRequired || cId.HasValue)
{
yield return new Entity
{
IsUnique = IsUnique,
PersonId = personId.Value,
SourceValue = sourceValue,
ConceptId = cId.HasValue ? cId.Value : 0,
TypeConceptId = concept.GetTypeId(field, reader),
StartDate = startDate,
EndDate = endDate == DateTime.MinValue ? (DateTime?) null : endDate,
ProviderId = reader.GetInt(ProviderId),
ProviderKey = reader.GetString(ProviderIdKey),
VisitOccurrenceId = reader.GetLong(VisitOccurrenceId),
AdditionalFields = additionalFields,
ValidStartDate = lookupValue.ValidStartDate,
ValidEndDate = lookupValue.ValidEndDate
};
}
}
}
}
}