public async Task GivenAPartitionedExecutor_WhenExecute_ResultShouldBeReturnedInOrder() { int itemCount = 9873; var testConsumer = new TestFhirDataConsumer(itemCount); FhirPartitionedExecutor <string, string> executor = new FhirPartitionedExecutor <string, string>(new TestFhirDataReader(itemCount), testConsumer, (content) => content) { BatchSize = 100, PartitionCount = 10 }; int totalCount = 0; int consumeCount = 0; Progress <BatchAnonymizeProgressDetail> progress = new Progress <BatchAnonymizeProgressDetail>(); progress.ProgressChanged += (obj, args) => { Interlocked.Add(ref totalCount, args.ProcessCompleted); Interlocked.Add(ref consumeCount, args.ConsumeCompleted); }; await executor.ExecuteAsync(CancellationToken.None, progress : progress); Assert.Equal(itemCount, testConsumer.CurrentOffset); Assert.Equal(99, testConsumer.BatchCount); // Progress report is triggered by event, wait 1 second here in case progress not report. await Task.Delay(TimeSpan.FromSeconds(1)); Assert.Equal(itemCount, totalCount); Assert.Equal(itemCount, consumeCount); }
public async Task GivenAPartitionedExecutor_WhenIOExceptionThrowFromConsumer_ExecutionShouldStop() { int itemCount = 91873; var testConsumer = new TestFhirDataConsumer(itemCount) { BreakOnOffset = 2342 }; var reader = new TestFhirDataReader(itemCount); FhirPartitionedExecutor <string, string> executor = new FhirPartitionedExecutor <string, string>(reader, testConsumer, (content) => content); await Assert.ThrowsAsync <IOException>(async() => await executor.ExecuteAsync(CancellationToken.None)); }
public async Task GivenAPartitionedExecutorBreakOnExceptionEnabled_WhenExceptionThrow_ExecutionShouldStop() { int itemCount = 9873; var testConsumer = new TestFhirDataConsumer(itemCount); Func <string, string> invalidOperationFunc = (content) => { throw new InvalidOperationException(); }; var executor = new FhirPartitionedExecutor <string, string>( new TestFhirDataReader(itemCount), testConsumer, invalidOperationFunc); await Assert.ThrowsAsync <InvalidOperationException>(async() => await executor.ExecuteAsync(CancellationToken.None, true)); }
public async Task GivenAPartitionedExecutorNotKeepOrder_WhenExecute_AllResultShouldBeReturned() { int itemCount = 29873; var testConsumer = new TestFhirDataConsumer(itemCount) { CheckOrder = false }; Random random = new Random(); Func <string, Task <string> > anonymizeFunc = async content => { if (random.Next() % 100 == 0) { await Task.Delay(TimeSpan.FromMilliseconds(200)); } return(await Task.FromResult(content)); }; FhirPartitionedExecutor <string, string> executor = new FhirPartitionedExecutor <string, string>(new TestFhirDataReader(itemCount), testConsumer, anonymizeFunc) { BatchSize = 100, PartitionCount = 12, KeepOrder = false }; int totalCount = 0; int consumeCount = 0; Progress <BatchAnonymizeProgressDetail> progress = new Progress <BatchAnonymizeProgressDetail>(); progress.ProgressChanged += (obj, args) => { Interlocked.Add(ref totalCount, args.ProcessCompleted); Interlocked.Add(ref consumeCount, args.ConsumeCompleted); }; await executor.ExecuteAsync(CancellationToken.None, progress : progress); Assert.Equal(itemCount, testConsumer.CurrentOffset); Assert.Equal(299, testConsumer.BatchCount); // Progress report is triggered by event, wait 1 second here in case progress not report. await Task.Delay(TimeSpan.FromSeconds(1)); Assert.Equal(itemCount, totalCount); Assert.Equal(itemCount, consumeCount); }
public async Task GivenAPartitionedExecutor_WhenCancelled_OperationCancelledExceptionShouldBeThrown() { int itemCount = 9873; var testConsumer = new TestFhirDataConsumer(itemCount); var executor = new FhirPartitionedExecutor <string, string>( new TestFhirDataReader(itemCount), testConsumer, (content) => { Thread.Sleep(10); return(content); }); CancellationTokenSource source = new CancellationTokenSource(); source.CancelAfter(1000); await Assert.ThrowsAsync <OperationCanceledException>(async() => await executor.ExecuteAsync(source.Token)); }