public void HappyFlow() { // 500m on i9-9900 takes about 5-8secs.. bottle-necked by consumer thread. (Release build) long expectPayloadCount = 500000000; TestContext.WriteLine($"Pipe allocation {_stopwatch.Elapsed}"); IPipe <long> sut = new BatchingPipe <long>(100000000); TestContext.WriteLine($"Pipe allocation end {_stopwatch.Elapsed}"); // Limited buffer prevents this test eating more than ~1GB Task producingTask = RunProducingTask(sut, expectPayloadCount, 100000000); TestContext.WriteLine($"Consumer stating {_stopwatch.Elapsed}"); long expectedNextValue = 0; foreach (long actualValue in sut.ToEnumerable()) { // Tried using NUnit assert first - Assert.AreEqual(expectedNextValue++, actualValue) // But it introduced 99%+ of cpu overhead making this test run for minutes instead of seconds. if (expectedNextValue++ != actualValue) { Assert.Fail($"Order was not preserved e:{expectedNextValue-1} a:{actualValue}"); } } Assert.IsTrue(producingTask.IsCompletedSuccessfully); Assert.AreEqual(expectPayloadCount, expectedNextValue); TestContext.WriteLine($"Consumer completed {_stopwatch.Elapsed}"); }
public IPipe <T> Create() { BatchingPipe <T> pipe = new BatchingPipe <T>(); PipeCreatedEvent?.Invoke(this, pipe); return(pipe); }
public void HappyFlow() { BatchingPipe <Packet <byte> > packetsPipe = new BatchingPipe <Packet <byte> >(); _ = packetsPipe.ProduceCompleteAsync(FixedTestData.CreatePackets()); Stream sut = packetsPipe.ToReadOnlyStream(); FixedTestData.AssertStream(sut); }
public async Task HappyFlowHeavyPooledBatches() { // 5B (5000m) on i9-9900 takes about 25secs.. bottle-necked by producer thread. (Release build) long expectPayloadCount = 5000000000; TestContext.WriteLine($"Pipe allocation {_stopwatch.Elapsed}"); IPipe <long> sut = new BatchingPipe <long>(100000000); TestContext.WriteLine($"Pipe allocation complete {_stopwatch.Elapsed}"); // with unlimited buffer, test run was sitting between 3-16GB of ram usage. Task producingTask = RunProducingTask(sut, expectPayloadCount, Int32.MaxValue); TestContext.WriteLine($"Consumer stating {_stopwatch.Elapsed}"); ConcurrentBag <BatchSummary> results = new ConcurrentBag <BatchSummary>(); await sut .ToPacketsAsyncEnumerable() .ParallelForEachAsync((packet, index) => { try { results.Add(BatchSummary.Validate(packet, index)); return(Task.CompletedTask); } finally { packet.Dispose(); } } ); TestContext.WriteLine($"Consumer completed {_stopwatch.Elapsed}"); Assert.IsTrue(producingTask.IsCompletedSuccessfully); BatchSummary[] resultsArray = results .Where(p => !p.Empty) .OrderBy(r => r.Index) .ToArray(); Assert.Positive(resultsArray.Length); Assert.IsTrue(resultsArray.All(r => r.Valid)); for (int i = 1; i < resultsArray.Length; i++) { if (resultsArray[i].First - 1 != resultsArray[i - 1].Last) { Assert.Fail("Bad ordering detected"); } } Assert.AreEqual(expectPayloadCount - 1, resultsArray[^ 1].Last);
public void End() { if (_queue != null) { _queue.ProducingCompleted(); _queue = null; _writerTask.Wait(); AssertTask(); } }
public async Task HappyFlow() { Stream inputStream = FixedTestData.CreateStream(); IPipe <Packet <byte> > duplicateBuffer = new BatchingPipe <Packet <byte> >(); DuplicateStream sut = new DuplicateStream(inputStream, duplicateBuffer.ToWriteOnlyStream()); // As sut is read for assertion, in the process it should start to write data to duplication destination Task assertSutStream = Task.Run(() => FixedTestData.AssertStream(sut)); // Thus with next task we should be able to assert both streams at the same time. Task assertDuplicatedStream = Task.Run(() => FixedTestData.AssertStream(duplicateBuffer.ToReadOnlyStream())); await assertSutStream; await assertDuplicatedStream; }
public void HappyFlow() { Stream testData = new CombinedStream(FixedTestData.CreateStreams()); BatchingPipe <Packet <byte> > packetsPipe = new BatchingPipe <Packet <byte> >(); Stream sut = packetsPipe.ToWriteOnlyStream(); Task.Run(async() => { await testData.CopyToAsync(sut); await sut.DisposeAsync(); // Closing/Disposing adapter triggers completion of producer by default. }); FixedTestData.AssertStream(packetsPipe.ToReadOnlyStream()); //FixedTestData.DebugStream(packetsPipe.ToReadOnlyStream()); }
public static IEnumerable <IEnumerable <T> > ToBatchesAsync <T>(this IEnumerable <T> source, int batchSize) { using IEnumerator <T> enumerator = source.GetEnumerator(); int index = 0; while (enumerator.MoveNext()) { BatchingPipe <T> batch = new BatchingPipe <T>(); Task task = Task.Run(() => ProduceBatch(enumerator, batch, batchSize)); yield return(batch.ToEnumerable()); Task.WaitAll(task); if (!task.IsCompletedSuccessfully) { throw new Exception("Failure while reading source enumerable", task.Exception); } } }
public void ActivateOptions() { _pipe = new BatchingPipe <LoggingEvent>(); _cancellationTokenSource = new CancellationTokenSource(); _consumerTask = StartConsumer(_cancellationTokenSource.Token); }
public void Begin() { _queue = new BatchingPipe <IResult>(); _writerTask = _queue.ToEnumerableAsync(Process); }
private void OnPipeCreatedEvent(IPipeFactory <IResult> sender, BatchingPipe <IResult> pipe) { _muxer.Add(pipe); }