public void GivenCommonResourceWithStu3OnlyElement_WhenAnonymizing_ExceptionShouldBeThrown(string testFile)
{
AnonymizerEngine engine = new AnonymizerEngine("r4-configuration-sample.json");
string testContent = File.ReadAllText(ResourceTestsFile(testFile));
Assert.Throws <StructuralTypeException>(() => engine.AnonymizeJson(testContent));
}
public void GivenCommonResourceWithR4OnlyField_WhenAnonymizing_ExceptionShouldBeThrown(string testFile)
{
AnonymizerEngine engine = new AnonymizerEngine("stu3-configuration-sample.json");
string testContent = File.ReadAllText(ResourceTestsFile(testFile));
Assert.Throws <FormatException>(() => engine.AnonymizeJson(testContent));
}
public void GivenAPatientResource_WhenAnonymizingWithProcessingError_IfParameterNotGiven_ExceptionWillBeThrown()
{
AnonymizerEngine engine = new AnonymizerEngine(Path.Combine("Configurations", "configuration-without-processing-error.json"));
string testContent = File.ReadAllText(ResourceTestsFile("patient-basic.json"));
Assert.Throws <AnonymizerProcessingException>(() => engine.AnonymizeJson(testContent));
}
public void GivenABundleResource_WhenAnonymizingWithProcessingError_IfRaise_ExceptionWillBeThrown()
{
AnonymizerEngine engine = new AnonymizerEngine(Path.Combine("Configurations", "configuration-raise-processing-error.json"));
string testContent = File.ReadAllText(CollectionResourceTestsFile("bundle-basic.json"));
Assert.Throws <AnonymizerProcessingException>(() => engine.AnonymizeJson(testContent));
}
internal static async Task AnonymizeAsync(AnonymizerOptions options)
{
var engine = new AnonymizerEngine(
options.ConfigurationFilePath,
new AnonymizerEngineOptions(options.ValidateInput, options.ValidateOutput));
if (options.InputFile != null && options.OutputFile != null)
{
if (IsSamePath(options.InputFile, options.OutputFile))
{
throw new ArgumentException("Input and output file path are the same! Please check file names.");
}
await AnonymizeOneFileAsync(options.InputFile, options.OutputFile, engine);
}
else if (options.InputFolder != null && options.OutputFolder != null)
{
if (IsSamePath(options.InputFolder, options.OutputFolder))
{
throw new ArgumentException("Input and output folders are the same! Please choose another folder.");
}
foreach (string file in Directory.EnumerateFiles(options.InputFolder, "*.dcm", SearchOption.AllDirectories))
{
await AnonymizeOneFileAsync(file, Path.Join(options.OutputFolder, Path.GetRelativePath(options.InputFolder, file)), engine);
}
Console.WriteLine("Anonymization finished!");
}
else
{
throw new ArgumentException("Invalid parameters. Please specify inputFile (or inputFolder) and outputFile (or outputFolder) at the same time.\r\nSamples:\r\n [-i inputFile -o outputFile]\r\nor\r\n [-I inputFolder -O outputFolder]");
}
}
public void Run(bool force = false)
{
_engine = new AnonymizerEngine("./configuration-sample.json");
var input = LoadActivityInput();
AnonymizeDataset(input, force).Wait();
}
public void GivenIsPrettyOutputSetFalse_WhenAnonymizeJson_OneLineJsonOutputShouldBeReturned()
{
AnonymizerEngine engine = new AnonymizerEngine(Path.Combine("TestConfigurations", "configuration-test-sample.json"));
var result = engine.AnonymizeJson(TestPatientSample);
Assert.Equal(OneLineOutputTarget, result);
}
public void GivenAR4OnlyResource_WhenAnonymizing_ExceptionShouldBeThrown(string testFile, string ResourceName)
{
AnonymizerEngine engine = new AnonymizerEngine("stu3-configuration-sample.json");
string testContent = File.ReadAllText(ResourceTestsFile(testFile));
var ex = Assert.Throws <FormatException>(() => engine.AnonymizeJson(testContent));
var expectedError = "type (at Cannot locate type information for type '" + ResourceName + "')";
Assert.Equal(expectedError, ex.Message.ToString());
}
public static void VerifySingleJsonResourceFromFile(AnonymizerEngine engine, string testFile, string targetFile)
{
Console.WriteLine($"VerifySingleJsonResourceFromFile. TestFile: {testFile}, TargetFile: {targetFile}");
string testContent = File.ReadAllText(testFile);
string targetContent = File.ReadAllText(targetFile);
string resultAfterAnonymize = engine.AnonymizeJson(testContent);
Assert.Equal(Standardize(targetContent), Standardize(resultAfterAnonymize));
}
public void Load(IServiceCollection services)
{
AnonymizerEngine.InitializeFhirPathExtensionSymbols();
services.AddFactory <IScoped <IAnonymizer> >();
services.Add <ExportAnonymizerFactory>()
.Transient()
.AsService <IAnonymizerFactory>();
}
public void GivenAPatientResource_WhenAnonymizingWithMultipleSubstituteConfig_ThenAnonymizedJsonShouldBeReturned()
{
// Child node is substituted first
AnonymizerEngine engine = new AnonymizerEngine(Path.Combine("Configurations", "substitute-multiple.json"));
FunctionalTestUtility.VerifySingleJsonResourceFromFile(engine, ResourceTestsFile("patient-substitute-multiple.json"), ResourceTestsFile("patient-substitute-multiple-target.json"));
// Parent node is substituted first
engine = new AnonymizerEngine(Path.Combine("Configurations", "substitute-multiple-2.json"));
FunctionalTestUtility.VerifySingleJsonResourceFromFile(engine, ResourceTestsFile("patient-substitute-multiple-2.json"), ResourceTestsFile("patient-substitute-multiple-2-target.json"));
}
public async Task Run(bool force = false)
{
// Increase connection limit of single endpoint: 2 => 128
System.Net.ServicePointManager.DefaultConnectionLimit = 128;
AnonymizerEngine.InitializeFhirPathExtensionSymbols();
var input = LoadActivityInput();
await AnonymizeDataset(input, force).ConfigureAwait(false);
}
public void GivenCommonResourceWithR4OnlyValue_WhenAnonymizing_ExceptionShouldBeThrown(string testFile)
{
AnonymizerEngine engine = new AnonymizerEngine("stu3-configuration-sample.json");
string testContent = File.ReadAllText(ResourceTestsFile(testFile));
var exception = Assert.Throws <InvalidInputException>(() => engine.AnonymizeJson(testContent));
Assert.Equal("The input FHIR resource is invalid", exception.Message);
Assert.True(exception.InnerException is StructuralTypeException);
Assert.StartsWith("Type checking the data: Encountered unknown element", exception.InnerException.Message);
}
public void GivenAR4OnlyResource_WhenAnonymizing_ExceptionShouldBeThrown(string testFile, string ResourceName)
{
AnonymizerEngine engine = new AnonymizerEngine("stu3-configuration-sample.json");
string testContent = File.ReadAllText(ResourceTestsFile(testFile));
var exception = Assert.Throws <InvalidInputException>(() => engine.AnonymizeJson(testContent));
var expectedError = "type (at Cannot locate type information for type '" + ResourceName + "')";
Assert.StartsWith("The input FHIR resource JSON is invalid.", exception.Message);
Assert.True(exception.InnerException is FormatException);
Assert.Equal(expectedError, exception.InnerException.Message);
}
public void GivenIsPrettyOutputSetTrue_WhenAnonymizeJson_PrettyJsonOutputShouldBeReturned()
{
AnonymizerEngine engine = new AnonymizerEngine(Path.Combine("TestConfigurations", "configuration-test-sample.json"));
var settings = new AnonymizerSettings()
{
IsPrettyOutput = true
};
var result = engine.AnonymizeJson(TestPatientSample, settings);
Assert.Equal(PrettyOutputTarget, result);
}
public void GivenAnonymizerEngine_AddingCustomProcessor_WhenAnonymize_CorrectResultWillBeReturned()
{
var factory = new CustomProcessorFactory();
factory.RegisterProcessors(typeof(MaskProcessor));
AnonymizerEngine engine = new AnonymizerEngine(Path.Combine("TestConfigurations", "configuration-custom-Processor.json"), factory);
var result = engine.AnonymizeJson(TestPatientSample);
Assert.Equal(CustomTarget, result);
}
private async Task AnonymizeSingleBlobInJsonFormatAsync(BlobClient inputBlobClient, BlockBlobClient outputBlobClient, string blobName, string inputFolderPrefix)
{
try
{
using Stream contentStream = await OperationExecutionHelper.InvokeWithTimeoutRetryAsync <Stream>(async () =>
{
Stream contentStream = new MemoryStream();
await inputBlobClient.DownloadToAsync(contentStream).ConfigureAwait(false);
contentStream.Position = 0;
return(contentStream);
},
TimeSpan.FromSeconds(FhirAzureConstants.DefaultBlockDownloadTimeoutInSeconds),
FhirAzureConstants.DefaultBlockDownloadTimeoutRetryCount,
isRetrableException : OperationExecutionHelper.IsRetrableException).ConfigureAwait(false);
using (var reader = new StreamReader(contentStream))
{
string input = await reader.ReadToEndAsync();
var engine = AnonymizerEngine.CreateWithFileContext(_configFile, blobName, inputFolderPrefix);
var settings = new AnonymizerSettings()
{
IsPrettyOutput = true
};
string output = engine.AnonymizeJson(input, settings);
using (MemoryStream outputStream = new MemoryStream(reader.CurrentEncoding.GetBytes(output)))
{
await OperationExecutionHelper.InvokeWithTimeoutRetryAsync(async() =>
{
outputStream.Position = 0;
using MemoryStream stream = new MemoryStream();
await outputStream.CopyToAsync(stream).ConfigureAwait(false);
stream.Position = 0;
return(await outputBlobClient.UploadAsync(stream).ConfigureAwait(false));
},
TimeSpan.FromSeconds(FhirAzureConstants.DefaultBlockUploadTimeoutInSeconds),
FhirAzureConstants.DefaultBlockUploadTimeoutRetryCount,
isRetrableException : OperationExecutionHelper.IsRetrableException).ConfigureAwait(false);
}
Console.WriteLine($"[{blobName}]: Anonymize completed.");
}
}
catch (Exception ex)
{
Console.WriteLine($"[{blobName}]: Anonymize failed, you can find detail error message in stderr.txt.");
Console.Error.WriteLine($"[{blobName}]: Failed to anonymize blob. \nErrorMessage: {ex.Message}\n Details: {ex.ToString()} \nStackTrace: {ex.StackTrace}");
}
}
public FilesAnonymizerForJsonFormatResource(
string configFilePath,
string inputFolder,
string outputFolder,
AnonymizationToolOptions options)
{
_inputFolder = inputFolder;
_outputFolder = outputFolder;
_configFilePath = configFilePath;
_options = options;
AnonymizerEngine.InitializeFhirPathExtensionSymbols();
}
internal static async Task AnonymizeAsync(Options options)
{
try
{
DicomFile dicomFile = await DicomFile.OpenAsync(options.InputFile).ConfigureAwait(false);
//FilterByVR(dicomFile, "PN");
var engine = new AnonymizerEngine(options.ConfigurationFilePath);
engine.Anonymize(dicomFile.Dataset);
dicomFile.Save(options.OutputFile);
Console.WriteLine($"Finished processing '{options.InputFile}'!");
}
catch (Exception ex)
{
Console.WriteLine($"Process failed! {ex}");
}
}
private async Task AnonymizeSingleBlobInNdJsonFormatAsync(BlobClient inputBlobClient, BlockBlobClient outputBlobClient, string blobName, string inputFolderPrefix)
{
var processedCount = 0;
int skippedCount = 0;
var consumedCount = 0;
using FhirBlobDataStream inputStream = new FhirBlobDataStream(inputBlobClient);
FhirStreamReader reader = new FhirStreamReader(inputStream);
FhirBlobConsumer consumer = new FhirBlobConsumer(outputBlobClient);
var engine = AnonymizerEngine.CreateWithFileContext(_configFile, blobName, inputFolderPrefix);
Func <string, string> anonymizerFunction = (item) =>
{
try
{
return(engine.AnonymizeJson(item));
}
catch (Exception ex)
{
Console.WriteLine($"[{blobName}]: Anonymize partial failed, you can find detail error message in stderr.txt.");
Console.Error.WriteLine($"[{blobName}]: Resource: {item}\nErrorMessage: {ex.Message}\n Details: {ex.ToString()}\nStackTrace: {ex.StackTrace}");
throw;
}
};
Stopwatch stopWatch = Stopwatch.StartNew();
FhirPartitionedExecutor <string, string> executor = new FhirPartitionedExecutor <string, string>(reader, consumer, anonymizerFunction);
executor.PartitionCount = Environment.ProcessorCount * 2;
Progress <BatchAnonymizeProgressDetail> progress = new Progress <BatchAnonymizeProgressDetail>();
progress.ProgressChanged += (obj, args) =>
{
Interlocked.Add(ref processedCount, args.ProcessCompleted);
Interlocked.Add(ref skippedCount, args.ProcessSkipped);
Interlocked.Add(ref consumedCount, args.ConsumeCompleted);
Console.WriteLine($"[{stopWatch.Elapsed.ToString()}][tid:{args.CurrentThreadId}]: {processedCount} Completed. {skippedCount} Skipped. {consumedCount} consume completed.");
};
await executor.ExecuteAsync(CancellationToken.None, progress).ConfigureAwait(false);
}
public async Task <string> FileAnonymize(string fileName)
{
var resourceOutputFileName = GetResourceOutputFileName(fileName, _inputFolder, _outputFolder);
if (_options.IsRecursive)
{
var resourceOutputFolder = Path.GetDirectoryName(resourceOutputFileName);
Directory.CreateDirectory(resourceOutputFolder);
}
if (_options.SkipExistedFile && File.Exists(resourceOutputFileName))
{
Console.WriteLine($"Skip processing on file {fileName}.");
return(string.Empty);
}
string resourceJson = await File.ReadAllTextAsync(fileName).ConfigureAwait(false);
try
{
var engine = AnonymizerEngine.CreateWithFileContext(_configFilePath, fileName, _inputFolder);
var settings = new AnonymizerSettings()
{
IsPrettyOutput = true,
ValidateInput = _options.ValidateInput,
ValidateOutput = _options.ValidateOutput
};
var resourceResult = engine.AnonymizeJson(resourceJson, settings);
await File.WriteAllTextAsync(resourceOutputFileName, resourceResult).ConfigureAwait(false);
}
catch (Exception innerException)
{
Console.Error.WriteLine($"[{fileName}] Error:\nResource: {resourceJson}\nErrorMessage: {innerException.ToString()}");
throw;
}
return(string.Empty);
}
public async Task <IAnonymizer> CreateAnonymizerAsync(string configurationLocation, CancellationToken cancellationToken)
{
EnsureArg.IsNotNullOrEmpty(configurationLocation, nameof(configurationLocation));
using (Stream stream = new MemoryStream())
{
try
{
await _artifactProvider.FetchAsync(configurationLocation, stream, cancellationToken);
stream.Position = 0;
}
catch (FileNotFoundException ex)
{
throw new AnonymizationConfigurationNotFoundException(ex.Message, ex);
}
catch (Exception ex)
{
_logger.LogError($"Failed to fetch Anonymization configuration file: {configurationLocation}");
throw new AnonymizationConfigurationFetchException(ex.Message, ex);
}
using (StreamReader reader = new StreamReader(stream))
{
string configurationContent = await reader.ReadToEndAsync();
try
{
var engine = new AnonymizerEngine(AnonymizerConfigurationManager.CreateFromSettingsInJson(configurationContent));
return(new ExportAnonymizer(engine));
}
catch (Exception ex)
{
_logger.LogError($"Failed to parse configuration file: {ex.Message}");
throw new FailedToParseAnonymizationConfigurationException(ex.Message, ex);
}
}
}
}
public async Task <string> FileAnonymize(string fileName)
{
var resourceOutputFileName = GetResourceOutputFileName(fileName, _inputFolder, _outputFolder);
if (_isRecursive)
{
var resourceOutputFolder = Path.GetDirectoryName(resourceOutputFileName);
Directory.CreateDirectory(resourceOutputFolder);
}
string resourceJson = await File.ReadAllTextAsync(fileName).ConfigureAwait(false);
using (FileStream outputStream = new FileStream(resourceOutputFileName, FileMode.Create))
{
using StreamWriter writer = new StreamWriter(outputStream);
try
{
var engine = AnonymizerEngine.CreateWithFileContext(_configFilePath, fileName, _inputFolder);
var settings = new AnonymizerSettings()
{
IsPrettyOutput = true,
ValidateInput = _validateInput,
ValidateOutput = _validateOutput
};
var resourceResult = engine.AnonymizeJson(resourceJson, settings);
await writer.WriteAsync(resourceResult).ConfigureAwait(false);
await writer.FlushAsync().ConfigureAwait(false);
}
catch (Exception innerException)
{
Console.Error.WriteLine($"[{fileName}] Error:\nResource: {resourceJson}\nErrorMessage: {innerException.ToString()}");
throw;
}
}
return(string.Empty);
}
public ExportAnonymizer(AnonymizerEngine engine)
{
EnsureArg.IsNotNull(engine, nameof(engine));
_engine = engine;
}
public async Task AnonymizeAsync()
{
var directorySearchOption = _isRecursive ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly;
var bulkResourceFileList = Directory.EnumerateFiles(_inputFolder, "*.ndjson", directorySearchOption).ToList();
Console.WriteLine($"Find {bulkResourceFileList.Count()} bulk data resource files in '{_inputFolder}'.");
foreach (var bulkResourceFileName in bulkResourceFileList)
{
Console.WriteLine($"Processing {bulkResourceFileName}");
var bulkResourceOutputFileName = GetResourceOutputFileName(bulkResourceFileName, _inputFolder, _outputFolder);
if (_isRecursive)
{
var resourceOutputFolder = Path.GetDirectoryName(bulkResourceOutputFileName);
Directory.CreateDirectory(resourceOutputFolder);
}
int completedCount = 0;
int failedCount = 0;
int consumeCompletedCount = 0;
using (FileStream inputStream = new FileStream(bulkResourceFileName, FileMode.Open))
using (FileStream outputStream = new FileStream(bulkResourceOutputFileName, FileMode.Create))
{
using FhirStreamReader reader = new FhirStreamReader(inputStream);
using FhirStreamConsumer consumer = new FhirStreamConsumer(outputStream);
Func <string, string> anonymizeFunction = (content) =>
{
try
{
var engine = AnonymizerEngine.CreateWithFileContext(_configFilePath, bulkResourceFileName, _inputFolder);
var settings = new AnonymizerSettings()
{
IsPrettyOutput = false,
ValidateInput = _validateInput,
ValidateOutput = _validateOutput
};
return(engine.AnonymizeJson(content, settings));
}
catch (Exception ex)
{
Console.Error.WriteLine($"Error:\nResource: {content}\nErrorMessage: {ex.ToString()}");
throw;
}
};
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
FhirPartitionedExecutor <string, string> executor = new FhirPartitionedExecutor <string, string>(reader, consumer, anonymizeFunction);
executor.PartitionCount = Environment.ProcessorCount * 2;
Progress <BatchAnonymizeProgressDetail> progress = new Progress <BatchAnonymizeProgressDetail>();
progress.ProgressChanged += (obj, args) =>
{
Interlocked.Add(ref completedCount, args.ProcessCompleted);
Interlocked.Add(ref failedCount, args.ProcessFailed);
Interlocked.Add(ref consumeCompletedCount, args.ConsumeCompleted);
Console.WriteLine($"[{stopWatch.Elapsed.ToString()}][tid:{args.CurrentThreadId}]: {completedCount} Process completed. {failedCount} Process failed. {consumeCompletedCount} Consume completed.");
};
await executor.ExecuteAsync(CancellationToken.None, false, progress).ConfigureAwait(false);
}
Console.WriteLine($"Finished processing '{bulkResourceFileName}'!");
}
}
public void GivenCommonResourceWithStu3OnlyField_WhenAnonymizing_AnonymizedJsonShouldBeReturned(string testFile, string targetFile)
{
AnonymizerEngine engine = new AnonymizerEngine("stu3-configuration-sample.json");
FunctionalTestUtility.VerifySingleJsonResourceFromFile(engine, ResourceTestsFile(testFile), ResourceTestsFile(targetFile));
}
public FhirResourceDataProcessor(string configFilePath)
{
_engine = new AnonymizerEngine(configFilePath);
}
public void GivenAPatientResource_WhenRedactAll_ThenRedactedJsonShouldBeReturned()
{
AnonymizerEngine engine = new AnonymizerEngine(Path.Combine("Configurations", "redact-all-config.json"));
FunctionalTestUtility.VerifySingleJsonResourceFromFile(engine, ResourceTestsFile("patient-basic.json"), ResourceTestsFile("patient-redact-all-target.json"));
}
public void GivenAPatientResource_WhenAnonymizingWithNoPartialRedactConfig_ThenAnonymizedJsonShouldBeReturned()
{
AnonymizerEngine engine = new AnonymizerEngine(Path.Combine("Configurations", "common-no-partial-config.json"));
FunctionalTestUtility.VerifySingleJsonResourceFromFile(engine, ResourceTestsFile("patient-no-partial.json"), ResourceTestsFile("patient-no-partial-target.json"));
}
public void GivenAPatientResourceWithNullDatetime_WhenAnonymizing_ThenAnonymizedJsonShouldBeReturned()
{
AnonymizerEngine engine = new AnonymizerEngine(Path.Combine("Configurations", "common-config.json"));
FunctionalTestUtility.VerifySingleJsonResourceFromFile(engine, ResourceTestsFile("patient-null-date.json"), ResourceTestsFile("patient-null-date-target.json"));
}
