public Resource DeIdentify([FromBody] Resource resource)
{
if (resource == null)
{
logger.LogWarning("Bad Request: received request body is empty.");
Response.StatusCode = StatusCodes.Status400BadRequest;
return(BadRequestOutcome);
}
logger.LogDebug("De-Identifying resource {resourceType}/{resourceId}",
resource.TypeName, resource.Id);
var settings = new AnonymizerSettings();
if (resource is Parameters param)
{
// parse dynamic rule settings
var dynamicSettings = param.GetSingle("settings")?.Part;
if (dynamicSettings?.Any() == true)
{
settings.DynamicRuleSettings = dynamicSettings.ToDictionary(p => p.Name, p => p.Value as object);
}
return(Anonymize(param.GetSingle("resource").Resource, settings));
}
return(Anonymize(resource));
}
private void ValidateInput(AnonymizerSettings settings, Resource resource)
{
if (settings != null && settings.ValidateInput)
{
_validator.ValidateInput(resource);
}
}
private void ValidateOutput(AnonymizerSettings settings, Resource anonymizedNode)
{
if (settings != null && settings.ValidateOutput)
{
_validator.ValidateOutput(anonymizedNode);
}
}
public AnonymizationVisitor(AnonymizationFhirPathRule[] rules,
Dictionary <string, IAnonymizerProcessor> processors,
AnonymizerSettings settings = null)
{
_rules = rules;
_processors = processors;
_settings = settings;
}
public ITypedElement AnonymizeElement(ITypedElement element, AnonymizerSettings settings = null)
{
EnsureArg.IsNotNull(element, nameof(element));
ElementNode resourceNode = ElementNode.FromElement(element);
return(resourceNode.Anonymize(_rules, _processors));
}
public void GivenIsPrettyOutputSetTrue_WhenAnonymizeJson_PrettyJsonOutputShouldBeReturned()
{
var settings = new AnonymizerSettings()
{
IsPrettyOutput = true
};
var result = _engine.AnonymizeJson(TestPatientSample, settings);
Assert.Equal(PrettyOutputTarget, result);
}
public void GivenIsPrettyOutputSetTrue_WhenAnonymizeJson_PrettyJsonOutputShouldBeReturned()
{
AnonymizerEngine engine = new AnonymizerEngine(Path.Combine("TestConfigurations", "configuration-test-sample.json"));
var settings = new AnonymizerSettings()
{
IsPrettyOutput = true
};
var result = engine.AnonymizeJson(TestPatientSample, settings);
Assert.Equal(PrettyOutputTarget, result);
}
public Resource AnonymizeResource(Resource resource, AnonymizerSettings settings = null)
{
EnsureArg.IsNotNull(resource, nameof(resource));
ValidateInput(settings, resource);
var anonymizedResource = AnonymizeElement(resource.ToTypedElement()).ToPoco <Resource>();
ValidateOutput(settings, anonymizedResource);
return(anonymizedResource);
}
public Resource AnonymizeResource(Resource resource, AnonymizerSettings settings = null)
{
EnsureArg.IsNotNull(resource, nameof(resource));
ValidateInput(settings, resource);
var resourceNode = ElementNode.FromElement(resource.ToTypedElement());
var anonymizedResource = resourceNode.Anonymize(_rules, _processors).ToPoco <Resource>();
ValidateOutput(settings, anonymizedResource);
return(anonymizedResource);
}
private async Task AnonymizeSingleBlobInJsonFormatAsync(BlobClient inputBlobClient, BlockBlobClient outputBlobClient, string blobName, string inputFolderPrefix)
{
try
{
using Stream contentStream = await OperationExecutionHelper.InvokeWithTimeoutRetryAsync <Stream>(async () =>
{
Stream contentStream = new MemoryStream();
await inputBlobClient.DownloadToAsync(contentStream).ConfigureAwait(false);
contentStream.Position = 0;
return(contentStream);
},
TimeSpan.FromSeconds(FhirAzureConstants.DefaultBlockDownloadTimeoutInSeconds),
FhirAzureConstants.DefaultBlockDownloadTimeoutRetryCount,
isRetrableException : OperationExecutionHelper.IsRetrableException).ConfigureAwait(false);
using (var reader = new StreamReader(contentStream))
{
string input = await reader.ReadToEndAsync();
var engine = AnonymizerEngine.CreateWithFileContext(_configFile, blobName, inputFolderPrefix);
var settings = new AnonymizerSettings()
{
IsPrettyOutput = true
};
string output = engine.AnonymizeJson(input, settings);
using (MemoryStream outputStream = new MemoryStream(reader.CurrentEncoding.GetBytes(output)))
{
await OperationExecutionHelper.InvokeWithTimeoutRetryAsync(async() =>
{
outputStream.Position = 0;
using MemoryStream stream = new MemoryStream();
await outputStream.CopyToAsync(stream).ConfigureAwait(false);
stream.Position = 0;
return(await outputBlobClient.UploadAsync(stream).ConfigureAwait(false));
},
TimeSpan.FromSeconds(FhirAzureConstants.DefaultBlockUploadTimeoutInSeconds),
FhirAzureConstants.DefaultBlockUploadTimeoutRetryCount,
isRetrableException : OperationExecutionHelper.IsRetrableException).ConfigureAwait(false);
}
Console.WriteLine($"[{blobName}]: Anonymize completed.");
}
}
catch (Exception ex)
{
Console.WriteLine($"[{blobName}]: Anonymize failed, you can find detail error message in stderr.txt.");
Console.Error.WriteLine($"[{blobName}]: Failed to anonymize blob. \nErrorMessage: {ex.Message}\n Details: {ex.ToString()} \nStackTrace: {ex.StackTrace}");
}
}
public string AnonymizeJson(string json, AnonymizerSettings settings = null)
{
EnsureArg.IsNotNullOrEmpty(json, nameof(json));
var resource = _parser.Parse <Resource>(json);
Resource anonymizedResource = AnonymizeResource(resource, settings);
FhirJsonSerializationSettings serializationSettings = new FhirJsonSerializationSettings
{
Pretty = settings != null && settings.IsPrettyOutput
};
return(anonymizedResource.ToJson(serializationSettings));
}
public string AnonymizeJson(string json, AnonymizerSettings settings = null)
{
EnsureArg.IsNotNullOrEmpty(json, nameof(json));
var element = ParseJsonToTypedElement(json);
var anonymizedElement = AnonymizeElement(element);
var serializationSettings = new FhirJsonSerializationSettings
{
Pretty = settings != null && settings.IsPrettyOutput
};
return(anonymizedElement.ToJson(serializationSettings));
}
public ITypedElement AnonymizeElement(ITypedElement element, AnonymizerSettings settings = null)
{
EnsureArg.IsNotNull(element, nameof(element));
try
{
ElementNode resourceNode = ParseTypedElementToElementNode(element);
return(resourceNode.Anonymize(_rules, _processors));
}
catch (AnonymizerProcessingException)
{
if (_configurationManager.Configuration.processingErrors == ProcessingErrorsOption.Skip)
{
// Return empty resource.
return(new EmptyElement(element.InstanceType));
}
throw;
}
}
public async Task <string> FileAnonymize(string fileName)
{
var resourceOutputFileName = GetResourceOutputFileName(fileName, _inputFolder, _outputFolder);
if (_options.IsRecursive)
{
var resourceOutputFolder = Path.GetDirectoryName(resourceOutputFileName);
Directory.CreateDirectory(resourceOutputFolder);
}
if (_options.SkipExistedFile && File.Exists(resourceOutputFileName))
{
Console.WriteLine($"Skip processing on file {fileName}.");
return(string.Empty);
}
string resourceJson = await File.ReadAllTextAsync(fileName).ConfigureAwait(false);
try
{
var engine = AnonymizerEngine.CreateWithFileContext(_configFilePath, fileName, _inputFolder);
var settings = new AnonymizerSettings()
{
IsPrettyOutput = true,
ValidateInput = _options.ValidateInput,
ValidateOutput = _options.ValidateOutput
};
var resourceResult = engine.AnonymizeJson(resourceJson, settings);
await File.WriteAllTextAsync(resourceOutputFileName, resourceResult).ConfigureAwait(false);
}
catch (Exception innerException)
{
Console.Error.WriteLine($"[{fileName}] Error:\nResource: {resourceJson}\nErrorMessage: {innerException.ToString()}");
throw;
}
return(string.Empty);
}
public async Task <string> FileAnonymize(string fileName)
{
var resourceOutputFileName = GetResourceOutputFileName(fileName, _inputFolder, _outputFolder);
if (_isRecursive)
{
var resourceOutputFolder = Path.GetDirectoryName(resourceOutputFileName);
Directory.CreateDirectory(resourceOutputFolder);
}
string resourceJson = await File.ReadAllTextAsync(fileName).ConfigureAwait(false);
using (FileStream outputStream = new FileStream(resourceOutputFileName, FileMode.Create))
{
using StreamWriter writer = new StreamWriter(outputStream);
try
{
var engine = AnonymizerEngine.CreateWithFileContext(_configFilePath, fileName, _inputFolder);
var settings = new AnonymizerSettings()
{
IsPrettyOutput = true,
ValidateInput = _validateInput,
ValidateOutput = _validateOutput
};
var resourceResult = engine.AnonymizeJson(resourceJson, settings);
await writer.WriteAsync(resourceResult).ConfigureAwait(false);
await writer.FlushAsync().ConfigureAwait(false);
}
catch (Exception innerException)
{
Console.Error.WriteLine($"[{fileName}] Error:\nResource: {resourceJson}\nErrorMessage: {innerException.ToString()}");
throw;
}
}
return(string.Empty);
}
private Resource Anonymize(Resource resource, AnonymizerSettings anonymizerSettings = null)
{
using var activity = Program.ActivitySource.StartActivity(nameof(Anonymize));
activity?.AddTag("resource.type", resource.TypeName);
activity?.AddTag("resource.id", resource.Id);
if (resource is Bundle bundle)
{
activity?.AddTag("bundle.size", bundle.Entry.Count);
BundleSizeHistogram.WithLabels(nameof(DeIdentify)).Observe(bundle.Entry.Count);
}
try
{
return(anonymizer.AnonymizeResource(resource, anonymizerSettings));
}
catch (Exception exc)
{
logger.LogError(exc, "Anonymize failed");
Response.StatusCode = StatusCodes.Status500InternalServerError;
return(GetInternalErrorOutcome(exc));
}
}
public string AnonymizeJson(string json, AnonymizerSettings settings = null)
{
EnsureArg.IsNotNullOrEmpty(json, nameof(json));
ElementNode root;
Resource resource;
try
{
resource = _parser.Parse <Resource>(json);
root = ElementNode.FromElement(resource.ToTypedElement());
}
catch (Exception innerException)
{
throw new Exception("Failed to parse json resource, please check the json content.", innerException);
}
if (settings != null && settings.ValidateInput)
{
_validator.ValidateInput(resource);
}
var anonymizedNode = AnonymizeResourceNode(root);
if (settings != null && settings.ValidateOutput)
{
anonymizedNode.RemoveNullChildren();
_validator.ValidateOutput(anonymizedNode.ToPoco <Resource>());
}
FhirJsonSerializationSettings serializationSettings = new FhirJsonSerializationSettings
{
Pretty = settings != null && settings.IsPrettyOutput
};
return(anonymizedNode.ToJson(serializationSettings));
}
public DecryptSegmentVisitor(AnonymizerSettings decryptSettings)
{
_decryptSettings = decryptSettings;
}
public DecryptSegmentVisitor(IDocumentItemFactory documentItemFactory, IPropertiesFactory propertiesFactory, AnonymizerSettings decryptSettings)
{
_documentItemFactory = documentItemFactory;
_propertiesFactory = propertiesFactory;
_decryptSettings = decryptSettings;
}
public static ElementNode Anonymize(this ElementNode node, AnonymizationFhirPathRule[] rules,
Dictionary <string, IAnonymizerProcessor> processors, AnonymizerSettings settings = null)
{
var visitor = new AnonymizationVisitor(rules, processors, settings);
node.Accept(visitor);
node.RemoveNullChildren();
return(node);
}
public Resource DePseudonymizeResource(Resource resource, AnonymizerSettings settings = null)
{
return(AnonymizeResource(resource, settings));
}
public async Task AnonymizeAsync()
{
var directorySearchOption = _isRecursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
var bulkResourceFileList = Directory.EnumerateFiles(_inputFolder, "*.ndjson", directorySearchOption).ToList();
Console.WriteLine($"Find {bulkResourceFileList.Count()} bulk data resource files in '{_inputFolder}'.");
foreach (var bulkResourceFileName in bulkResourceFileList)
{
Console.WriteLine($"Processing {bulkResourceFileName}");
var bulkResourceOutputFileName = GetResourceOutputFileName(bulkResourceFileName, _inputFolder, _outputFolder);
if (_isRecursive)
{
var resourceOutputFolder = Path.GetDirectoryName(bulkResourceOutputFileName);
Directory.CreateDirectory(resourceOutputFolder);
}
int completedCount = 0;
int failedCount = 0;
int consumeCompletedCount = 0;
using (FileStream inputStream = new FileStream(bulkResourceFileName, FileMode.Open))
using (FileStream outputStream = new FileStream(bulkResourceOutputFileName, FileMode.Create))
{
using FhirStreamReader reader = new FhirStreamReader(inputStream);
using FhirStreamConsumer consumer = new FhirStreamConsumer(outputStream);
Func <string, string> anonymizeFunction = (content) =>
{
try
{
var engine = AnonymizerEngine.CreateWithFileContext(_configFilePath, bulkResourceFileName, _inputFolder);
var settings = new AnonymizerSettings()
{
IsPrettyOutput = false,
ValidateInput = _validateInput,
ValidateOutput = _validateOutput
};
return(engine.AnonymizeJson(content, settings));
}
catch (Exception ex)
{
Console.Error.WriteLine($"Error:\nResource: {content}\nErrorMessage: {ex.ToString()}");
throw;
}
};
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
FhirPartitionedExecutor <string, string> executor = new FhirPartitionedExecutor <string, string>(reader, consumer, anonymizeFunction);
executor.PartitionCount = Environment.ProcessorCount * 2;
Progress <BatchAnonymizeProgressDetail> progress = new Progress <BatchAnonymizeProgressDetail>();
progress.ProgressChanged += (obj, args) =>
{
Interlocked.Add(ref completedCount, args.ProcessCompleted);
Interlocked.Add(ref failedCount, args.ProcessFailed);
Interlocked.Add(ref consumeCompletedCount, args.ConsumeCompleted);
Console.WriteLine($"[{stopWatch.Elapsed.ToString()}][tid:{args.CurrentThreadId}]: {completedCount} Process completed. {failedCount} Process failed. {consumeCompletedCount} Consume completed.");
};
await executor.ExecuteAsync(CancellationToken.None, false, progress).ConfigureAwait(false);
}
Console.WriteLine($"Finished processing '{bulkResourceFileName}'!");
}
}
public DecryptDataProcessor(AnonymizerSettings decryptSettings)
{
_decryptSettings = decryptSettings;
}
