public Resource DeIdentify([FromBody] Resource resource) { if (resource == null) { logger.LogWarning("Bad Request: received request body is empty."); Response.StatusCode = StatusCodes.Status400BadRequest; return(BadRequestOutcome); } logger.LogDebug("De-Identifying resource {resourceType}/{resourceId}", resource.TypeName, resource.Id); var settings = new AnonymizerSettings(); if (resource is Parameters param) { // parse dynamic rule settings var dynamicSettings = param.GetSingle("settings")?.Part; if (dynamicSettings?.Any() == true) { settings.DynamicRuleSettings = dynamicSettings.ToDictionary(p => p.Name, p => p.Value as object); } return(Anonymize(param.GetSingle("resource").Resource, settings)); } return(Anonymize(resource)); }
private void ValidateInput(AnonymizerSettings settings, Resource resource) { if (settings != null && settings.ValidateInput) { _validator.ValidateInput(resource); } }
private void ValidateOutput(AnonymizerSettings settings, Resource anonymizedNode) { if (settings != null && settings.ValidateOutput) { _validator.ValidateOutput(anonymizedNode); } }
public AnonymizationVisitor(AnonymizationFhirPathRule[] rules, Dictionary <string, IAnonymizerProcessor> processors, AnonymizerSettings settings = null) { _rules = rules; _processors = processors; _settings = settings; }
public ITypedElement AnonymizeElement(ITypedElement element, AnonymizerSettings settings = null) { EnsureArg.IsNotNull(element, nameof(element)); ElementNode resourceNode = ElementNode.FromElement(element); return(resourceNode.Anonymize(_rules, _processors)); }
public void GivenIsPrettyOutputSetTrue_WhenAnonymizeJson_PrettyJsonOutputShouldBeReturned() { var settings = new AnonymizerSettings() { IsPrettyOutput = true }; var result = _engine.AnonymizeJson(TestPatientSample, settings); Assert.Equal(PrettyOutputTarget, result); }
public void GivenIsPrettyOutputSetTrue_WhenAnonymizeJson_PrettyJsonOutputShouldBeReturned() { AnonymizerEngine engine = new AnonymizerEngine(Path.Combine("TestConfigurations", "configuration-test-sample.json")); var settings = new AnonymizerSettings() { IsPrettyOutput = true }; var result = engine.AnonymizeJson(TestPatientSample, settings); Assert.Equal(PrettyOutputTarget, result); }
public Resource AnonymizeResource(Resource resource, AnonymizerSettings settings = null) { EnsureArg.IsNotNull(resource, nameof(resource)); ValidateInput(settings, resource); var anonymizedResource = AnonymizeElement(resource.ToTypedElement()).ToPoco <Resource>(); ValidateOutput(settings, anonymizedResource); return(anonymizedResource); }
public Resource AnonymizeResource(Resource resource, AnonymizerSettings settings = null) { EnsureArg.IsNotNull(resource, nameof(resource)); ValidateInput(settings, resource); var resourceNode = ElementNode.FromElement(resource.ToTypedElement()); var anonymizedResource = resourceNode.Anonymize(_rules, _processors).ToPoco <Resource>(); ValidateOutput(settings, anonymizedResource); return(anonymizedResource); }
private async Task AnonymizeSingleBlobInJsonFormatAsync(BlobClient inputBlobClient, BlockBlobClient outputBlobClient, string blobName, string inputFolderPrefix) { try { using Stream contentStream = await OperationExecutionHelper.InvokeWithTimeoutRetryAsync <Stream>(async () => { Stream contentStream = new MemoryStream(); await inputBlobClient.DownloadToAsync(contentStream).ConfigureAwait(false); contentStream.Position = 0; return(contentStream); }, TimeSpan.FromSeconds(FhirAzureConstants.DefaultBlockDownloadTimeoutInSeconds), FhirAzureConstants.DefaultBlockDownloadTimeoutRetryCount, isRetrableException : OperationExecutionHelper.IsRetrableException).ConfigureAwait(false); using (var reader = new StreamReader(contentStream)) { string input = await reader.ReadToEndAsync(); var engine = AnonymizerEngine.CreateWithFileContext(_configFile, blobName, inputFolderPrefix); var settings = new AnonymizerSettings() { IsPrettyOutput = true }; string output = engine.AnonymizeJson(input, settings); using (MemoryStream outputStream = new MemoryStream(reader.CurrentEncoding.GetBytes(output))) { await OperationExecutionHelper.InvokeWithTimeoutRetryAsync(async() => { outputStream.Position = 0; using MemoryStream stream = new MemoryStream(); await outputStream.CopyToAsync(stream).ConfigureAwait(false); stream.Position = 0; return(await outputBlobClient.UploadAsync(stream).ConfigureAwait(false)); }, TimeSpan.FromSeconds(FhirAzureConstants.DefaultBlockUploadTimeoutInSeconds), FhirAzureConstants.DefaultBlockUploadTimeoutRetryCount, isRetrableException : OperationExecutionHelper.IsRetrableException).ConfigureAwait(false); } Console.WriteLine($"[{blobName}]: Anonymize completed."); } } catch (Exception ex) { Console.WriteLine($"[{blobName}]: Anonymize failed, you can find detail error message in stderr.txt."); Console.Error.WriteLine($"[{blobName}]: Failed to anonymize blob. \nErrorMessage: {ex.Message}\n Details: {ex.ToString()} \nStackTrace: {ex.StackTrace}"); } }
public string AnonymizeJson(string json, AnonymizerSettings settings = null) { EnsureArg.IsNotNullOrEmpty(json, nameof(json)); var resource = _parser.Parse <Resource>(json); Resource anonymizedResource = AnonymizeResource(resource, settings); FhirJsonSerializationSettings serializationSettings = new FhirJsonSerializationSettings { Pretty = settings != null && settings.IsPrettyOutput }; return(anonymizedResource.ToJson(serializationSettings)); }
public string AnonymizeJson(string json, AnonymizerSettings settings = null) { EnsureArg.IsNotNullOrEmpty(json, nameof(json)); var element = ParseJsonToTypedElement(json); var anonymizedElement = AnonymizeElement(element); var serializationSettings = new FhirJsonSerializationSettings { Pretty = settings != null && settings.IsPrettyOutput }; return(anonymizedElement.ToJson(serializationSettings)); }
public ITypedElement AnonymizeElement(ITypedElement element, AnonymizerSettings settings = null) { EnsureArg.IsNotNull(element, nameof(element)); try { ElementNode resourceNode = ParseTypedElementToElementNode(element); return(resourceNode.Anonymize(_rules, _processors)); } catch (AnonymizerProcessingException) { if (_configurationManager.Configuration.processingErrors == ProcessingErrorsOption.Skip) { // Return empty resource. return(new EmptyElement(element.InstanceType)); } throw; } }
public async Task <string> FileAnonymize(string fileName) { var resourceOutputFileName = GetResourceOutputFileName(fileName, _inputFolder, _outputFolder); if (_options.IsRecursive) { var resourceOutputFolder = Path.GetDirectoryName(resourceOutputFileName); Directory.CreateDirectory(resourceOutputFolder); } if (_options.SkipExistedFile && File.Exists(resourceOutputFileName)) { Console.WriteLine($"Skip processing on file {fileName}."); return(string.Empty); } string resourceJson = await File.ReadAllTextAsync(fileName).ConfigureAwait(false); try { var engine = AnonymizerEngine.CreateWithFileContext(_configFilePath, fileName, _inputFolder); var settings = new AnonymizerSettings() { IsPrettyOutput = true, ValidateInput = _options.ValidateInput, ValidateOutput = _options.ValidateOutput }; var resourceResult = engine.AnonymizeJson(resourceJson, settings); await File.WriteAllTextAsync(resourceOutputFileName, resourceResult).ConfigureAwait(false); } catch (Exception innerException) { Console.Error.WriteLine($"[{fileName}] Error:\nResource: {resourceJson}\nErrorMessage: {innerException.ToString()}"); throw; } return(string.Empty); }
public async Task <string> FileAnonymize(string fileName) { var resourceOutputFileName = GetResourceOutputFileName(fileName, _inputFolder, _outputFolder); if (_isRecursive) { var resourceOutputFolder = Path.GetDirectoryName(resourceOutputFileName); Directory.CreateDirectory(resourceOutputFolder); } string resourceJson = await File.ReadAllTextAsync(fileName).ConfigureAwait(false); using (FileStream outputStream = new FileStream(resourceOutputFileName, FileMode.Create)) { using StreamWriter writer = new StreamWriter(outputStream); try { var engine = AnonymizerEngine.CreateWithFileContext(_configFilePath, fileName, _inputFolder); var settings = new AnonymizerSettings() { IsPrettyOutput = true, ValidateInput = _validateInput, ValidateOutput = _validateOutput }; var resourceResult = engine.AnonymizeJson(resourceJson, settings); await writer.WriteAsync(resourceResult).ConfigureAwait(false); await writer.FlushAsync().ConfigureAwait(false); } catch (Exception innerException) { Console.Error.WriteLine($"[{fileName}] Error:\nResource: {resourceJson}\nErrorMessage: {innerException.ToString()}"); throw; } } return(string.Empty); }
private Resource Anonymize(Resource resource, AnonymizerSettings anonymizerSettings = null) { using var activity = Program.ActivitySource.StartActivity(nameof(Anonymize)); activity?.AddTag("resource.type", resource.TypeName); activity?.AddTag("resource.id", resource.Id); if (resource is Bundle bundle) { activity?.AddTag("bundle.size", bundle.Entry.Count); BundleSizeHistogram.WithLabels(nameof(DeIdentify)).Observe(bundle.Entry.Count); } try { return(anonymizer.AnonymizeResource(resource, anonymizerSettings)); } catch (Exception exc) { logger.LogError(exc, "Anonymize failed"); Response.StatusCode = StatusCodes.Status500InternalServerError; return(GetInternalErrorOutcome(exc)); } }
public string AnonymizeJson(string json, AnonymizerSettings settings = null) { EnsureArg.IsNotNullOrEmpty(json, nameof(json)); ElementNode root; Resource resource; try { resource = _parser.Parse <Resource>(json); root = ElementNode.FromElement(resource.ToTypedElement()); } catch (Exception innerException) { throw new Exception("Failed to parse json resource, please check the json content.", innerException); } if (settings != null && settings.ValidateInput) { _validator.ValidateInput(resource); } var anonymizedNode = AnonymizeResourceNode(root); if (settings != null && settings.ValidateOutput) { anonymizedNode.RemoveNullChildren(); _validator.ValidateOutput(anonymizedNode.ToPoco <Resource>()); } FhirJsonSerializationSettings serializationSettings = new FhirJsonSerializationSettings { Pretty = settings != null && settings.IsPrettyOutput }; return(anonymizedNode.ToJson(serializationSettings)); }
public DecryptSegmentVisitor(AnonymizerSettings decryptSettings) { _decryptSettings = decryptSettings; }
public DecryptSegmentVisitor(IDocumentItemFactory documentItemFactory, IPropertiesFactory propertiesFactory, AnonymizerSettings decryptSettings) { _documentItemFactory = documentItemFactory; _propertiesFactory = propertiesFactory; _decryptSettings = decryptSettings; }
public static ElementNode Anonymize(this ElementNode node, AnonymizationFhirPathRule[] rules, Dictionary <string, IAnonymizerProcessor> processors, AnonymizerSettings settings = null) { var visitor = new AnonymizationVisitor(rules, processors, settings); node.Accept(visitor); node.RemoveNullChildren(); return(node); }
public Resource DePseudonymizeResource(Resource resource, AnonymizerSettings settings = null) { return(AnonymizeResource(resource, settings)); }
public async Task AnonymizeAsync() { var directorySearchOption = _isRecursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; var bulkResourceFileList = Directory.EnumerateFiles(_inputFolder, "*.ndjson", directorySearchOption).ToList(); Console.WriteLine($"Find {bulkResourceFileList.Count()} bulk data resource files in '{_inputFolder}'."); foreach (var bulkResourceFileName in bulkResourceFileList) { Console.WriteLine($"Processing {bulkResourceFileName}"); var bulkResourceOutputFileName = GetResourceOutputFileName(bulkResourceFileName, _inputFolder, _outputFolder); if (_isRecursive) { var resourceOutputFolder = Path.GetDirectoryName(bulkResourceOutputFileName); Directory.CreateDirectory(resourceOutputFolder); } int completedCount = 0; int failedCount = 0; int consumeCompletedCount = 0; using (FileStream inputStream = new FileStream(bulkResourceFileName, FileMode.Open)) using (FileStream outputStream = new FileStream(bulkResourceOutputFileName, FileMode.Create)) { using FhirStreamReader reader = new FhirStreamReader(inputStream); using FhirStreamConsumer consumer = new FhirStreamConsumer(outputStream); Func <string, string> anonymizeFunction = (content) => { try { var engine = AnonymizerEngine.CreateWithFileContext(_configFilePath, bulkResourceFileName, _inputFolder); var settings = new AnonymizerSettings() { IsPrettyOutput = false, ValidateInput = _validateInput, ValidateOutput = _validateOutput }; return(engine.AnonymizeJson(content, settings)); } catch (Exception ex) { Console.Error.WriteLine($"Error:\nResource: {content}\nErrorMessage: {ex.ToString()}"); throw; } }; Stopwatch stopWatch = new Stopwatch(); stopWatch.Start(); FhirPartitionedExecutor <string, string> executor = new FhirPartitionedExecutor <string, string>(reader, consumer, anonymizeFunction); executor.PartitionCount = Environment.ProcessorCount * 2; Progress <BatchAnonymizeProgressDetail> progress = new Progress <BatchAnonymizeProgressDetail>(); progress.ProgressChanged += (obj, args) => { Interlocked.Add(ref completedCount, args.ProcessCompleted); Interlocked.Add(ref failedCount, args.ProcessFailed); Interlocked.Add(ref consumeCompletedCount, args.ConsumeCompleted); Console.WriteLine($"[{stopWatch.Elapsed.ToString()}][tid:{args.CurrentThreadId}]: {completedCount} Process completed. {failedCount} Process failed. {consumeCompletedCount} Consume completed."); }; await executor.ExecuteAsync(CancellationToken.None, false, progress).ConfigureAwait(false); } Console.WriteLine($"Finished processing '{bulkResourceFileName}'!"); } }
public DecryptDataProcessor(AnonymizerSettings decryptSettings) { _decryptSettings = decryptSettings; }