public async Task GivenAPartitionedExecutor_WhenExecute_ResultShouldBeReturnedInOrder()
{
int itemCount = 9873;
var testConsumer = new TestFhirDataConsumer(itemCount);
FhirPartitionedExecutor executor = new FhirPartitionedExecutor(new TestFhirDataReader(itemCount), testConsumer, (content) => content)
{
BatchSize = 100,
PartitionCount = 10
};
int totalCount = 0;
int consumeCount = 0;
Progress <BatchAnonymizeProgressDetail> progress = new Progress <BatchAnonymizeProgressDetail>();
progress.ProgressChanged += (obj, args) =>
{
Interlocked.Add(ref totalCount, args.ProcessCompleted);
Interlocked.Add(ref consumeCount, args.ConsumeCompleted);
};
await executor.ExecuteAsync(CancellationToken.None, progress : progress);
Assert.Equal(itemCount, totalCount);
Assert.Equal(itemCount, testConsumer.CurrentOffset);
Assert.Equal(99, testConsumer.BatchCount);
Assert.Equal(9873, consumeCount);
}
public async Task GivenAPartitionedExecutor_WhenExecute_ResultShouldBeReturnedInOrder()
{
int itemCount = 9873;
var testConsumer = new TestFhirDataConsumer(itemCount);
FhirPartitionedExecutor <string, string> executor = new FhirPartitionedExecutor <string, string>(new TestFhirDataReader(itemCount), testConsumer, (content) => content)
{
BatchSize = 100,
PartitionCount = 10
};
int totalCount = 0;
int consumeCount = 0;
Progress <BatchAnonymizeProgressDetail> progress = new Progress <BatchAnonymizeProgressDetail>();
progress.ProgressChanged += (obj, args) =>
{
Interlocked.Add(ref totalCount, args.ProcessCompleted);
Interlocked.Add(ref consumeCount, args.ConsumeCompleted);
};
await executor.ExecuteAsync(CancellationToken.None, progress : progress);
Assert.Equal(itemCount, testConsumer.CurrentOffset);
Assert.Equal(99, testConsumer.BatchCount);
// Progress report is triggered by event, wait 1 second here in case progress not report.
await Task.Delay(TimeSpan.FromSeconds(1));
Assert.Equal(itemCount, totalCount);
Assert.Equal(itemCount, consumeCount);
}
public async Task GivenAPartitionedExecutor_WhenIOExceptionThrowFromConsumer_ExecutionShouldStop()
{
int itemCount = 91873;
var testConsumer = new TestFhirDataConsumer(itemCount)
{
BreakOnOffset = 2342
};
var reader = new TestFhirDataReader(itemCount);
FhirPartitionedExecutor <string, string> executor = new FhirPartitionedExecutor <string, string>(reader, testConsumer, (content) => content);
await Assert.ThrowsAsync <IOException>(async() => await executor.ExecuteAsync(CancellationToken.None));
}
public async Task GivenAPartitionedExecutorBreakOnExceptionEnabled_WhenExceptionThrow_ExecutionShouldStop()
{
int itemCount = 9873;
var testConsumer = new TestFhirDataConsumer(itemCount);
FhirPartitionedExecutor executor = new FhirPartitionedExecutor(new TestFhirDataReader(itemCount), testConsumer, (content) => content);
executor.AnonymizerFunction = (content) =>
{
throw new InvalidOperationException();
};
await Assert.ThrowsAsync <InvalidOperationException>(async() => await executor.ExecuteAsync(CancellationToken.None, true));
}
public async Task GivenAPartitionedExecutor_WhenCancelled_OperationCancelledExceptionShouldBeThrown()
{
int itemCount = 9873;
var testConsumer = new TestFhirDataConsumer(itemCount);
FhirPartitionedExecutor executor = new FhirPartitionedExecutor(new TestFhirDataReader(itemCount), testConsumer, (content) => content);
executor.AnonymizerFunction = (content) =>
{
Thread.Sleep(10);
return(content);
};
CancellationTokenSource source = new CancellationTokenSource();
source.CancelAfter(1000);
await Assert.ThrowsAsync <OperationCanceledException>(async() => await executor.ExecuteAsync(source.Token));
}
public async Task GivenAPartitionedExecutorNotKeepOrder_WhenExecute_AllResultShouldBeReturned()
{
int itemCount = 29873;
var testConsumer = new TestFhirDataConsumer(itemCount)
{
CheckOrder = false
};
Random random = new Random();
Func <string, Task <string> > anonymizeFunc = async content =>
{
if (random.Next() % 100 == 0)
{
await Task.Delay(TimeSpan.FromMilliseconds(200));
}
return(await Task.FromResult(content));
};
FhirPartitionedExecutor <string, string> executor = new FhirPartitionedExecutor <string, string>(new TestFhirDataReader(itemCount), testConsumer, anonymizeFunc)
{
BatchSize = 100,
PartitionCount = 12,
KeepOrder = false
};
int totalCount = 0;
int consumeCount = 0;
Progress <BatchAnonymizeProgressDetail> progress = new Progress <BatchAnonymizeProgressDetail>();
progress.ProgressChanged += (obj, args) =>
{
Interlocked.Add(ref totalCount, args.ProcessCompleted);
Interlocked.Add(ref consumeCount, args.ConsumeCompleted);
};
await executor.ExecuteAsync(CancellationToken.None, progress : progress);
Assert.Equal(itemCount, testConsumer.CurrentOffset);
Assert.Equal(299, testConsumer.BatchCount);
// Progress report is triggered by event, wait 1 second here in case progress not report.
await Task.Delay(TimeSpan.FromSeconds(1));
Assert.Equal(itemCount, totalCount);
Assert.Equal(itemCount, consumeCount);
}