static IServiceCollection ConfigureDeidentificationOptions(this IServiceCollection services, IConfiguration config)
{
var deident = new DeidentificationOptions();
deident.Patient.Enabled = config.GetValue <bool>(Config.Deidentification.Patient.Enabled);
if (deident.Patient.Enabled)
{
deident.Patient.DateShifting.WithIncrement(config.GetValue <string>(Config.Deidentification.Patient.DateShifting.Increment));
deident.Patient.DateShifting.LowerBound = config.GetValue <int>(Config.Deidentification.Patient.DateShifting.LowerBound);
deident.Patient.DateShifting.UpperBound = config.GetValue <int>(Config.Deidentification.Patient.DateShifting.UpperBound);
if (deident.Patient.DateShifting.LowerBound == 0 && deident.Patient.DateShifting.UpperBound == 0)
{
throw new LeafConfigurationException("Patient De-identification is enabled but Date Shifting Lower Bound and Upper Bound are both set to zero");
}
if (deident.Patient.DateShifting.LowerBound >= deident.Patient.DateShifting.UpperBound)
{
throw new LeafConfigurationException($"Patient De-identification Date Shifting Lower Bound must be less than Upper Bound, but is set to {deident.Patient.DateShifting.LowerBound} vs {deident.Patient.DateShifting.UpperBound}");
}
}
deident.Cohort.Enabled = config.GetValue <bool>(Config.Deidentification.Cohort.Enabled);
if (deident.Cohort.Enabled)
{
deident.Cohort.Noise.Enabled = config.GetValue <bool>(Config.Deidentification.Cohort.Noise.Enabled);
if (deident.Cohort.Noise.Enabled)
{
deident.Cohort.Noise.LowerBound = config.GetValue <int>(Config.Deidentification.Cohort.Noise.LowerBound);
deident.Cohort.Noise.UpperBound = config.GetValue <int>(Config.Deidentification.Cohort.Noise.UpperBound);
if (deident.Cohort.Noise.LowerBound == 0 && deident.Cohort.Noise.UpperBound == 0)
{
throw new LeafConfigurationException("Cohort De-identification Noise is enabled but Lower Bound and Upper Bound are both set to zero");
}
if (deident.Cohort.Noise.LowerBound >= deident.Cohort.Noise.UpperBound)
{
throw new LeafConfigurationException($"Cohort De-identification Noise Lower Bound must be less than Upper Bound, but is set to {deident.Cohort.Noise.LowerBound} vs {deident.Cohort.Noise.UpperBound}");
}
}
deident.Cohort.LowCellSizeMasking.Enabled = config.GetValue <bool>(Config.Deidentification.Cohort.LowCellSizeMasking.Enabled);
if (deident.Cohort.LowCellSizeMasking.Enabled)
{
deident.Cohort.LowCellSizeMasking.Threshold = config.GetValue <int>(Config.Deidentification.Cohort.LowCellSizeMasking.Threshold);
if (deident.Cohort.LowCellSizeMasking.Threshold <= 0)
{
throw new LeafConfigurationException($"Cohort De-identification Low Cell Size Masking must be greater than or equal to one, but is set to {deident.Cohort.LowCellSizeMasking.Threshold}");
}
}
}
services.Configure <DeidentificationOptions>(opts =>
{
opts.Patient = deident.Patient;
opts.Cohort = deident.Cohort;
});
return(services);
}
public DemographicsExecutor(
IUserContext userContext,
ILogger <DemographicsExecutor> log,
IOptions <ClinDbOptions> dbOpts,
IOptions <DeidentificationOptions> deidentOpts)
{
this.log = log;
this.dbOpts = dbOpts.Value;
this.deidentOpts = deidentOpts.Value;
user = userContext;
}
Func <EncounterDatasetRecord, Encounter> GetConverter(bool anonymize, DeidentificationOptions opts)
{
if (anonymize)
{
var shift = opts.Patient.DateShifting;
var anon = new Anonymizer <EncounterDatasetRecord>(Pepper, shift.Increment.ToString(), shift.LowerBound, shift.UpperBound);
return((rec) =>
{
anon.Anonymize(rec);
return rec.ToEncounter();
});
}
return((rec) => rec.ToEncounter());
}
public ConfigController(
IOptions <AuthenticationOptions> authenticationOptions,
IOptions <LeafVersionOptions> versionOptions,
IOptions <CohortOptions> cohortOptions,
IOptions <ClientOptions> clientOptions,
IOptions <AttestationOptions> attestationOptions,
IOptions <DeidentificationOptions> deidentOptions)
{
this.authenticationOptions = authenticationOptions.Value;
this.versionOptions = versionOptions.Value;
this.cohortOptions = cohortOptions.Value;
this.clientOptions = clientOptions.Value;
this.attestationOptions = attestationOptions.Value;
this.deidentOptions = deidentOptions.Value;
}
public DatasetProvider(
DatasetCompilerValidationContextProvider contextProvider,
IDatasetSqlCompiler compiler,
IDatasetExecutor datasetService,
IOptions <ClientOptions> clientOpts,
IOptions <DeidentificationOptions> deidentOpts,
ILogger <DatasetProvider> log)
{
this.contextProvider = contextProvider;
this.compiler = compiler;
this.executor = datasetService;
this.clientOpts = clientOpts.Value;
this.deidentOpts = deidentOpts.Value;
this.log = log;
}
public QueryManager(
IQueryService service,
IOptions <DeidentificationOptions> obfuscationOptions,
ILogger <QueryManager> log,
IUserContext user,
PanelConverter converter,
PanelValidator validator)
{
this.service = service;
this.deidentOpts = obfuscationOptions.Value;
this.log = log;
this.user = user;
this.converter = converter;
this.validator = validator;
}
public DemographicProvider(
IUserContext user,
DemographicCompilerValidationContextProvider contextProvider,
IOptions <ClientOptions> clientOpts,
IOptions <DeidentificationOptions> deidentOpts,
IDemographicSqlCompiler compiler,
IDemographicsExecutor executor,
ILogger <DemographicProvider> log)
{
this.user = user;
this.contextProvider = contextProvider;
this.compiler = compiler;
this.executor = executor;
this.clientOpts = clientOpts.Value;
this.deidentOpts = deidentOpts.Value;
this.log = log;
}
public void Obfuscate(ref PatientCount count, PanelValidationContext ctx, DeidentificationOptions opts)
{
if (!opts.Cohort.Enabled)
{
return;
}
// If low cell sizes should be masked and count less than or equal to threshold, set to threshold.
if (opts.Cohort.LowCellSizeMasking.Enabled && count.Value <= opts.Cohort.LowCellSizeMasking.Threshold)
{
count.Value = opts.Cohort.LowCellSizeMasking.Threshold;
count.PlusMinus = opts.Cohort.LowCellSizeMasking.Threshold;
count.WithinLowCellThreshold = true;
return;
}
// Bail if noise obfuscation not enabled
if (!opts.Cohort.Noise.Enabled)
{
return;
}
// Ensure that variations of the same query (with concepts and panels moved around but the query logic identical)
// always returns the same string of Guid Ids for concepts.
var orderedIds = GetDeterministicConceptIdsAsString(ctx.Allowed);
// Hash into a byte array.
var hashed = md5.ComputeHash(Encoding.UTF8.GetBytes(orderedIds));
// Seed a random number generator from the hash.
var generator = new Random(BitConverter.ToInt32(hashed, 0));
// Compute a random shifted value between the lower and upper bounds
var shift = 0;
while (shift == 0)
{
shift = generator.Next(opts.Cohort.Noise.LowerBound, opts.Cohort.Noise.UpperBound);
}
count.Value += shift;
count.PlusMinus = Math.Max(Math.Abs(opts.Cohort.Noise.LowerBound), Math.Abs(opts.Cohort.Noise.UpperBound));
}
public CohortCounter(
IOptions <RuntimeOptions> opts,
IOptions <DeidentificationOptions> deidentOpts,
PanelConverter converter,
PanelValidator validator,
IPatientCohortService counter,
ICohortCacheService cohortCache,
IObfuscationService obfuscator,
IUserContext user,
ILogger <CohortCounter> log)
{
this.runtime = opts.Value.Runtime;
this.deidentOpts = deidentOpts.Value;
this.converter = converter;
this.validator = validator;
this.counter = counter;
this.obfuscator = obfuscator;
this.cohortCache = cohortCache;
this.user = user;
this.log = log;
}
public void Original_Value_Changed()
{
var orig = 50;
var obfuscator = new ObfuscationService();
var opts = new DeidentificationOptions {
Cohort = new DeidentificationOptions.CohortObfuscationOptions {
Enabled = true, Noise = new DeidentificationOptions.CohortObfuscationOptions.NoiseOptions {
Enabled = true, LowerBound = -10, UpperBound = 10
}
}
};
var count = new PatientCount {
Value = orig
};
var ctx = MockPanel.Context();
obfuscator.Obfuscate(ref count, ctx, opts);
Assert.NotEqual(orig, count.Value);
Assert.Equal(count.PlusMinus, Math.Max(opts.Cohort.Noise.LowerBound, opts.Cohort.Noise.UpperBound));
Assert.False(count.WithinLowCellThreshold);
}
public void Under_Threshold_Not_Shifted()
{
var orig = 5;
var obfuscator = new ObfuscationService();
var opts = new DeidentificationOptions {
Cohort = new DeidentificationOptions.CohortObfuscationOptions {
Enabled = true, LowCellSizeMasking = new DeidentificationOptions.CohortObfuscationOptions.LowCellSizeMaskingOptions {
Enabled = true, Threshold = 10
}
}
};
var count = new PatientCount {
Value = orig
};
var ctx = MockPanel.Context();
obfuscator.Obfuscate(ref count, ctx, opts);
Assert.NotEqual(orig, count.Value);
Assert.Equal(count.Value, opts.Cohort.LowCellSizeMasking.Threshold);
Assert.True(count.WithinLowCellThreshold);
}
public override IEnumerable <ShapedDataset> Marshal(SqlDataReader reader, bool anonymize, DeidentificationOptions opts)
{
var fields = (_context.DatasetQuery as DynamicDatasetQuery).Schema.Fields
.Where(f => _schema.Fields.Any(sf => sf.Name == f.Name) && (!anonymize || !f.Phi || (f.Phi && f.Mask)))
.Select(f => f.ToSchemaField());
var records = new List <ShapedDataset>();
var converter = GetConverter(anonymize, fields, opts);
while (reader.Read())
{
var record = GetRecord(reader, fields);
var dyn = converter(record);
records.Add(dyn);
}
return(records);
}
public void Same_Logic_Different_Structure_Produces_Same_Shift()
{
var g1 = Guid.NewGuid();
var g2 = Guid.NewGuid();
var g3 = Guid.NewGuid();
var orig = 50;
var obfuscator = new ObfuscationService();
var opts = new DeidentificationOptions {
Cohort = new DeidentificationOptions.CohortObfuscationOptions {
Enabled = true, Noise = new DeidentificationOptions.CohortObfuscationOptions.NoiseOptions {
Enabled = true, LowerBound = -10, UpperBound = 10
}
}
};
var count1 = new PatientCount {
Value = orig
};
var count2 = new PatientCount {
Value = orig
};
var count3 = new PatientCount {
Value = orig
};
var count4 = new PatientCount {
Value = orig
};
var ctx1 = MockPanel.Context();
var ctx2 = MockPanel.Context();
var ctx3 = MockPanel.Context();
var ctx4 = MockPanel.Context();
// Set context one with two panels, the first with two concepts, second with one.
ctx1.Allowed.ElementAt(0).SubPanels.ElementAt(0).PanelItems = new List <PanelItem> {
new PanelItem {
Concept = new Concept {
Id = g1
}
}, new PanelItem {
Concept = new Concept {
Id = g2
}
}
};
ctx1.Allowed.Append(new Panel {
SubPanels = new List <SubPanel> {
new SubPanel {
PanelItems = new List <PanelItem> {
new PanelItem {
Concept = new Concept {
Id = g3
}
}
}
}
}
});
// Set context two to same as one, but with panel one concept order flipped.
ctx2.Allowed.ElementAt(0).SubPanels.ElementAt(0).PanelItems = new List <PanelItem> {
new PanelItem {
Concept = new Concept {
Id = g2
}
}, new PanelItem {
Concept = new Concept {
Id = g1
}
}
};
ctx2.Allowed.Append(new Panel {
SubPanels = new List <SubPanel> {
new SubPanel {
PanelItems = new List <PanelItem> {
new PanelItem {
Concept = new Concept {
Id = g3
}
}
}
}
}
});
// Set context three to same as one, but with panel order flipped.
ctx3.Allowed.ElementAt(0).SubPanels.ElementAt(0).PanelItems = new List <PanelItem> {
new PanelItem {
Concept = new Concept {
Id = g1
}
}, new PanelItem {
Concept = new Concept {
Id = g2
}
}
};
ctx3.Allowed.Prepend(new Panel {
SubPanels = new List <SubPanel> {
new SubPanel {
PanelItems = new List <PanelItem> {
new PanelItem {
Concept = new Concept {
Id = g3
}
}
}
}
}
});
// Set context four to a combination of two and three, with both concept order and panel order flipped.
ctx4.Allowed.ElementAt(0).SubPanels.ElementAt(0).PanelItems = new List <PanelItem> {
new PanelItem {
Concept = new Concept {
Id = g2
}
}, new PanelItem {
Concept = new Concept {
Id = g1
}
}
};
ctx4.Allowed.Prepend(new Panel {
SubPanels = new List <SubPanel> {
new SubPanel {
PanelItems = new List <PanelItem> {
new PanelItem {
Concept = new Concept {
Id = g3
}
}
}
}
}
});
obfuscator.Obfuscate(ref count1, ctx1, opts);
obfuscator.Obfuscate(ref count2, ctx2, opts);
obfuscator.Obfuscate(ref count3, ctx3, opts);
obfuscator.Obfuscate(ref count4, ctx4, opts);
Assert.NotEqual(orig, count1.Value);
Assert.NotEqual(orig, count2.Value);
Assert.NotEqual(orig, count3.Value);
Assert.NotEqual(orig, count4.Value);
Assert.False(count1.WithinLowCellThreshold);
Assert.False(count2.WithinLowCellThreshold);
Assert.False(count3.WithinLowCellThreshold);
Assert.False(count4.WithinLowCellThreshold);
Assert.Equal(count1.Value, count2.Value);
Assert.Equal(count1.Value, count3.Value);
Assert.Equal(count1.Value, count4.Value);
}
Func <DynamicDatasetRecord, DynamicShapedDatumSet> GetConverter(bool anonymize, IEnumerable <SchemaFieldSelector> fields, DeidentificationOptions opts)
{
if (anonymize)
{
var shift = opts.Patient.DateShifting;
var anon = new DynamicAnonymizer(Pepper, shift.Increment.ToString(), shift.LowerBound, shift.UpperBound);
return((rec) =>
{
anon.Anonymize(rec, fields);
return rec.ToDatumSet();
});
}
return((rec) => rec.ToDatumSet());
}
public override IEnumerable <ShapedDataset> Marshal(SqlDataReader reader, bool anonymize, DeidentificationOptions opts)
{
var records = new List <ShapedDataset>();
var converter = GetConverter(anonymize, opts);
while (reader.Read())
{
var record = GetRecord(reader);
var enc = converter(record);
records.Add(enc);
}
return(records);
}
public PatientDemographicContext Marshal(SqlDataReader reader, bool anonymize, DeidentificationOptions opts)
{
var exported = new List <PatientDemographic>();
var cohort = new List <PatientDemographic>();
var exportConverter = GetExportConverter(anonymize, opts);
while (reader.Read())
{
var cohortRecord = GetCohortRecord(reader);
cohort.Add(cohortRecord.ToIdentifiedPatientDemographic());
if (cohortRecord.Exported)
{
var export = exportConverter(cohortRecord);
exported.Add(export);
}
}
return(new PatientDemographicContext
{
Exported = exported,
Cohort = cohort
});
}
public abstract IEnumerable<ShapedDataset> Marshal(SqlDataReader reader, bool anonymize, DeidentificationOptions opts);
Func <PatientDemographicRecord, PatientDemographic> GetExportConverter(bool anonymize, DeidentificationOptions opts)
{
if (anonymize)
{
var shift = opts.Patient.DateShifting;
var anon = new Anonymizer <PatientDemographicRecord>(Pepper, shift.Increment.ToString(), shift.LowerBound, shift.UpperBound);
return((rec) =>
{
anon.Anonymize(rec);
rec.Age = rec.CalculateAge();
return rec.ToAnonymousPatientDemographic();
});
}
return((rec) =>
{
rec.Age = rec.CalculateAge();
return rec.ToIdentifiedPatientDemographic();
});
}
