static IServiceCollection ConfigureDeidentificationOptions(this IServiceCollection services, IConfiguration config) { var deident = new DeidentificationOptions(); deident.Patient.Enabled = config.GetValue <bool>(Config.Deidentification.Patient.Enabled); if (deident.Patient.Enabled) { deident.Patient.DateShifting.WithIncrement(config.GetValue <string>(Config.Deidentification.Patient.DateShifting.Increment)); deident.Patient.DateShifting.LowerBound = config.GetValue <int>(Config.Deidentification.Patient.DateShifting.LowerBound); deident.Patient.DateShifting.UpperBound = config.GetValue <int>(Config.Deidentification.Patient.DateShifting.UpperBound); if (deident.Patient.DateShifting.LowerBound == 0 && deident.Patient.DateShifting.UpperBound == 0) { throw new LeafConfigurationException("Patient De-identification is enabled but Date Shifting Lower Bound and Upper Bound are both set to zero"); } if (deident.Patient.DateShifting.LowerBound >= deident.Patient.DateShifting.UpperBound) { throw new LeafConfigurationException($"Patient De-identification Date Shifting Lower Bound must be less than Upper Bound, but is set to {deident.Patient.DateShifting.LowerBound} vs {deident.Patient.DateShifting.UpperBound}"); } } deident.Cohort.Enabled = config.GetValue <bool>(Config.Deidentification.Cohort.Enabled); if (deident.Cohort.Enabled) { deident.Cohort.Noise.Enabled = config.GetValue <bool>(Config.Deidentification.Cohort.Noise.Enabled); if (deident.Cohort.Noise.Enabled) { deident.Cohort.Noise.LowerBound = config.GetValue <int>(Config.Deidentification.Cohort.Noise.LowerBound); deident.Cohort.Noise.UpperBound = config.GetValue <int>(Config.Deidentification.Cohort.Noise.UpperBound); if (deident.Cohort.Noise.LowerBound == 0 && deident.Cohort.Noise.UpperBound == 0) { throw new LeafConfigurationException("Cohort De-identification Noise is enabled but Lower Bound and Upper Bound are both set to zero"); } if (deident.Cohort.Noise.LowerBound >= deident.Cohort.Noise.UpperBound) { throw new LeafConfigurationException($"Cohort De-identification Noise Lower Bound must be less than Upper Bound, but is set to {deident.Cohort.Noise.LowerBound} vs {deident.Cohort.Noise.UpperBound}"); } } deident.Cohort.LowCellSizeMasking.Enabled = config.GetValue <bool>(Config.Deidentification.Cohort.LowCellSizeMasking.Enabled); if (deident.Cohort.LowCellSizeMasking.Enabled) { deident.Cohort.LowCellSizeMasking.Threshold = config.GetValue <int>(Config.Deidentification.Cohort.LowCellSizeMasking.Threshold); if (deident.Cohort.LowCellSizeMasking.Threshold <= 0) { throw new LeafConfigurationException($"Cohort De-identification Low Cell Size Masking must be greater than or equal to one, but is set to {deident.Cohort.LowCellSizeMasking.Threshold}"); } } } services.Configure <DeidentificationOptions>(opts => { opts.Patient = deident.Patient; opts.Cohort = deident.Cohort; }); return(services); }
public DemographicsExecutor( IUserContext userContext, ILogger <DemographicsExecutor> log, IOptions <ClinDbOptions> dbOpts, IOptions <DeidentificationOptions> deidentOpts) { this.log = log; this.dbOpts = dbOpts.Value; this.deidentOpts = deidentOpts.Value; user = userContext; }
Func <EncounterDatasetRecord, Encounter> GetConverter(bool anonymize, DeidentificationOptions opts) { if (anonymize) { var shift = opts.Patient.DateShifting; var anon = new Anonymizer <EncounterDatasetRecord>(Pepper, shift.Increment.ToString(), shift.LowerBound, shift.UpperBound); return((rec) => { anon.Anonymize(rec); return rec.ToEncounter(); }); } return((rec) => rec.ToEncounter()); }
public ConfigController( IOptions <AuthenticationOptions> authenticationOptions, IOptions <LeafVersionOptions> versionOptions, IOptions <CohortOptions> cohortOptions, IOptions <ClientOptions> clientOptions, IOptions <AttestationOptions> attestationOptions, IOptions <DeidentificationOptions> deidentOptions) { this.authenticationOptions = authenticationOptions.Value; this.versionOptions = versionOptions.Value; this.cohortOptions = cohortOptions.Value; this.clientOptions = clientOptions.Value; this.attestationOptions = attestationOptions.Value; this.deidentOptions = deidentOptions.Value; }
public DatasetProvider( DatasetCompilerValidationContextProvider contextProvider, IDatasetSqlCompiler compiler, IDatasetExecutor datasetService, IOptions <ClientOptions> clientOpts, IOptions <DeidentificationOptions> deidentOpts, ILogger <DatasetProvider> log) { this.contextProvider = contextProvider; this.compiler = compiler; this.executor = datasetService; this.clientOpts = clientOpts.Value; this.deidentOpts = deidentOpts.Value; this.log = log; }
public QueryManager( IQueryService service, IOptions <DeidentificationOptions> obfuscationOptions, ILogger <QueryManager> log, IUserContext user, PanelConverter converter, PanelValidator validator) { this.service = service; this.deidentOpts = obfuscationOptions.Value; this.log = log; this.user = user; this.converter = converter; this.validator = validator; }
public DemographicProvider( IUserContext user, DemographicCompilerValidationContextProvider contextProvider, IOptions <ClientOptions> clientOpts, IOptions <DeidentificationOptions> deidentOpts, IDemographicSqlCompiler compiler, IDemographicsExecutor executor, ILogger <DemographicProvider> log) { this.user = user; this.contextProvider = contextProvider; this.compiler = compiler; this.executor = executor; this.clientOpts = clientOpts.Value; this.deidentOpts = deidentOpts.Value; this.log = log; }
public void Obfuscate(ref PatientCount count, PanelValidationContext ctx, DeidentificationOptions opts) { if (!opts.Cohort.Enabled) { return; } // If low cell sizes should be masked and count less than or equal to threshold, set to threshold. if (opts.Cohort.LowCellSizeMasking.Enabled && count.Value <= opts.Cohort.LowCellSizeMasking.Threshold) { count.Value = opts.Cohort.LowCellSizeMasking.Threshold; count.PlusMinus = opts.Cohort.LowCellSizeMasking.Threshold; count.WithinLowCellThreshold = true; return; } // Bail if noise obfuscation not enabled if (!opts.Cohort.Noise.Enabled) { return; } // Ensure that variations of the same query (with concepts and panels moved around but the query logic identical) // always returns the same string of Guid Ids for concepts. var orderedIds = GetDeterministicConceptIdsAsString(ctx.Allowed); // Hash into a byte array. var hashed = md5.ComputeHash(Encoding.UTF8.GetBytes(orderedIds)); // Seed a random number generator from the hash. var generator = new Random(BitConverter.ToInt32(hashed, 0)); // Compute a random shifted value between the lower and upper bounds var shift = 0; while (shift == 0) { shift = generator.Next(opts.Cohort.Noise.LowerBound, opts.Cohort.Noise.UpperBound); } count.Value += shift; count.PlusMinus = Math.Max(Math.Abs(opts.Cohort.Noise.LowerBound), Math.Abs(opts.Cohort.Noise.UpperBound)); }
public CohortCounter( IOptions <RuntimeOptions> opts, IOptions <DeidentificationOptions> deidentOpts, PanelConverter converter, PanelValidator validator, IPatientCohortService counter, ICohortCacheService cohortCache, IObfuscationService obfuscator, IUserContext user, ILogger <CohortCounter> log) { this.runtime = opts.Value.Runtime; this.deidentOpts = deidentOpts.Value; this.converter = converter; this.validator = validator; this.counter = counter; this.obfuscator = obfuscator; this.cohortCache = cohortCache; this.user = user; this.log = log; }
public void Original_Value_Changed() { var orig = 50; var obfuscator = new ObfuscationService(); var opts = new DeidentificationOptions { Cohort = new DeidentificationOptions.CohortObfuscationOptions { Enabled = true, Noise = new DeidentificationOptions.CohortObfuscationOptions.NoiseOptions { Enabled = true, LowerBound = -10, UpperBound = 10 } } }; var count = new PatientCount { Value = orig }; var ctx = MockPanel.Context(); obfuscator.Obfuscate(ref count, ctx, opts); Assert.NotEqual(orig, count.Value); Assert.Equal(count.PlusMinus, Math.Max(opts.Cohort.Noise.LowerBound, opts.Cohort.Noise.UpperBound)); Assert.False(count.WithinLowCellThreshold); }
public void Under_Threshold_Not_Shifted() { var orig = 5; var obfuscator = new ObfuscationService(); var opts = new DeidentificationOptions { Cohort = new DeidentificationOptions.CohortObfuscationOptions { Enabled = true, LowCellSizeMasking = new DeidentificationOptions.CohortObfuscationOptions.LowCellSizeMaskingOptions { Enabled = true, Threshold = 10 } } }; var count = new PatientCount { Value = orig }; var ctx = MockPanel.Context(); obfuscator.Obfuscate(ref count, ctx, opts); Assert.NotEqual(orig, count.Value); Assert.Equal(count.Value, opts.Cohort.LowCellSizeMasking.Threshold); Assert.True(count.WithinLowCellThreshold); }
public override IEnumerable <ShapedDataset> Marshal(SqlDataReader reader, bool anonymize, DeidentificationOptions opts) { var fields = (_context.DatasetQuery as DynamicDatasetQuery).Schema.Fields .Where(f => _schema.Fields.Any(sf => sf.Name == f.Name) && (!anonymize || !f.Phi || (f.Phi && f.Mask))) .Select(f => f.ToSchemaField()); var records = new List <ShapedDataset>(); var converter = GetConverter(anonymize, fields, opts); while (reader.Read()) { var record = GetRecord(reader, fields); var dyn = converter(record); records.Add(dyn); } return(records); }
public void Same_Logic_Different_Structure_Produces_Same_Shift() { var g1 = Guid.NewGuid(); var g2 = Guid.NewGuid(); var g3 = Guid.NewGuid(); var orig = 50; var obfuscator = new ObfuscationService(); var opts = new DeidentificationOptions { Cohort = new DeidentificationOptions.CohortObfuscationOptions { Enabled = true, Noise = new DeidentificationOptions.CohortObfuscationOptions.NoiseOptions { Enabled = true, LowerBound = -10, UpperBound = 10 } } }; var count1 = new PatientCount { Value = orig }; var count2 = new PatientCount { Value = orig }; var count3 = new PatientCount { Value = orig }; var count4 = new PatientCount { Value = orig }; var ctx1 = MockPanel.Context(); var ctx2 = MockPanel.Context(); var ctx3 = MockPanel.Context(); var ctx4 = MockPanel.Context(); // Set context one with two panels, the first with two concepts, second with one. ctx1.Allowed.ElementAt(0).SubPanels.ElementAt(0).PanelItems = new List <PanelItem> { new PanelItem { Concept = new Concept { Id = g1 } }, new PanelItem { Concept = new Concept { Id = g2 } } }; ctx1.Allowed.Append(new Panel { SubPanels = new List <SubPanel> { new SubPanel { PanelItems = new List <PanelItem> { new PanelItem { Concept = new Concept { Id = g3 } } } } } }); // Set context two to same as one, but with panel one concept order flipped. ctx2.Allowed.ElementAt(0).SubPanels.ElementAt(0).PanelItems = new List <PanelItem> { new PanelItem { Concept = new Concept { Id = g2 } }, new PanelItem { Concept = new Concept { Id = g1 } } }; ctx2.Allowed.Append(new Panel { SubPanels = new List <SubPanel> { new SubPanel { PanelItems = new List <PanelItem> { new PanelItem { Concept = new Concept { Id = g3 } } } } } }); // Set context three to same as one, but with panel order flipped. ctx3.Allowed.ElementAt(0).SubPanels.ElementAt(0).PanelItems = new List <PanelItem> { new PanelItem { Concept = new Concept { Id = g1 } }, new PanelItem { Concept = new Concept { Id = g2 } } }; ctx3.Allowed.Prepend(new Panel { SubPanels = new List <SubPanel> { new SubPanel { PanelItems = new List <PanelItem> { new PanelItem { Concept = new Concept { Id = g3 } } } } } }); // Set context four to a combination of two and three, with both concept order and panel order flipped. ctx4.Allowed.ElementAt(0).SubPanels.ElementAt(0).PanelItems = new List <PanelItem> { new PanelItem { Concept = new Concept { Id = g2 } }, new PanelItem { Concept = new Concept { Id = g1 } } }; ctx4.Allowed.Prepend(new Panel { SubPanels = new List <SubPanel> { new SubPanel { PanelItems = new List <PanelItem> { new PanelItem { Concept = new Concept { Id = g3 } } } } } }); obfuscator.Obfuscate(ref count1, ctx1, opts); obfuscator.Obfuscate(ref count2, ctx2, opts); obfuscator.Obfuscate(ref count3, ctx3, opts); obfuscator.Obfuscate(ref count4, ctx4, opts); Assert.NotEqual(orig, count1.Value); Assert.NotEqual(orig, count2.Value); Assert.NotEqual(orig, count3.Value); Assert.NotEqual(orig, count4.Value); Assert.False(count1.WithinLowCellThreshold); Assert.False(count2.WithinLowCellThreshold); Assert.False(count3.WithinLowCellThreshold); Assert.False(count4.WithinLowCellThreshold); Assert.Equal(count1.Value, count2.Value); Assert.Equal(count1.Value, count3.Value); Assert.Equal(count1.Value, count4.Value); }
Func <DynamicDatasetRecord, DynamicShapedDatumSet> GetConverter(bool anonymize, IEnumerable <SchemaFieldSelector> fields, DeidentificationOptions opts) { if (anonymize) { var shift = opts.Patient.DateShifting; var anon = new DynamicAnonymizer(Pepper, shift.Increment.ToString(), shift.LowerBound, shift.UpperBound); return((rec) => { anon.Anonymize(rec, fields); return rec.ToDatumSet(); }); } return((rec) => rec.ToDatumSet()); }
public override IEnumerable <ShapedDataset> Marshal(SqlDataReader reader, bool anonymize, DeidentificationOptions opts) { var records = new List <ShapedDataset>(); var converter = GetConverter(anonymize, opts); while (reader.Read()) { var record = GetRecord(reader); var enc = converter(record); records.Add(enc); } return(records); }
public PatientDemographicContext Marshal(SqlDataReader reader, bool anonymize, DeidentificationOptions opts) { var exported = new List <PatientDemographic>(); var cohort = new List <PatientDemographic>(); var exportConverter = GetExportConverter(anonymize, opts); while (reader.Read()) { var cohortRecord = GetCohortRecord(reader); cohort.Add(cohortRecord.ToIdentifiedPatientDemographic()); if (cohortRecord.Exported) { var export = exportConverter(cohortRecord); exported.Add(export); } } return(new PatientDemographicContext { Exported = exported, Cohort = cohort }); }
public abstract IEnumerable<ShapedDataset> Marshal(SqlDataReader reader, bool anonymize, DeidentificationOptions opts);
Func <PatientDemographicRecord, PatientDemographic> GetExportConverter(bool anonymize, DeidentificationOptions opts) { if (anonymize) { var shift = opts.Patient.DateShifting; var anon = new Anonymizer <PatientDemographicRecord>(Pepper, shift.Increment.ToString(), shift.LowerBound, shift.UpperBound); return((rec) => { anon.Anonymize(rec); rec.Age = rec.CalculateAge(); return rec.ToAnonymousPatientDemographic(); }); } return((rec) => { rec.Age = rec.CalculateAge(); return rec.ToIdentifiedPatientDemographic(); }); }