public async Task TestOrdering_Sync_OrderedDisabled(bool trustedEnumeration) { // If ordering were enabled, this test would hang. var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, EnsureOrdered = false }; var mres = new ManualResetEventSlim(); var tb = new TransformManyBlock <int, int>(i => { if (i == 0) { mres.Wait(); } return(trustedEnumeration ? new[] { i } : Enumerable.Repeat(i, 1)); }, options); tb.Post(0); tb.Post(1); Assert.Equal(1, await tb.ReceiveAsync()); mres.Set(); Assert.Equal(0, await tb.ReceiveAsync()); tb.Complete(); await tb.Completion; }
static void ShowTransformManyBlock() { // <snippet6> // Create a TransformManyBlock<string, char> object that splits // a string into its individual characters. var transformManyBlock = new TransformManyBlock <string, char>( s => s.ToCharArray()); // Post two messages to the first block. transformManyBlock.Post("Hello"); transformManyBlock.Post("World"); // Receive all output values from the block. for (int i = 0; i < ("Hello" + "World").Length; i++) { Console.WriteLine(transformManyBlock.Receive()); } /* Output: * H * e * l * l * o * W * o * r * l * d */ // </snippet6> }
public async Task TestOrdering_Sync_BlockingEnumeration_NoDeadlock(bool ensureOrdered) { // If iteration of the yielded enumerables happened while holding a lock, this would deadlock. var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, EnsureOrdered = ensureOrdered }; ManualResetEventSlim mres1 = new ManualResetEventSlim(), mres2 = new ManualResetEventSlim(); var tb = new TransformManyBlock <int, int>(i => i == 0 ? BlockableIterator(mres1, mres2) : BlockableIterator(mres2, mres1), options); tb.Post(0); tb.Post(1); Assert.Equal(42, await tb.ReceiveAsync()); Assert.Equal(42, await tb.ReceiveAsync()); tb.Complete(); await tb.Completion; IEnumerable <int> BlockableIterator(ManualResetEventSlim wait, ManualResetEventSlim release) { release.Set(); wait.Wait(); yield return(42); } }
public void TransformManyOverfullTest2() { var scheduler = new TestScheduler(); n = 0; var transform = new TransformManyBlock <int, int> ( i => ComputeResults(), new ExecutionDataflowBlockOptions { BoundedCapacity = 100, TaskScheduler = scheduler }); for (int i = 0; i < 100; i++) { Assert.IsTrue(transform.Post(i)); } Assert.IsFalse(transform.Post(101)); Assert.GreaterOrEqual(scheduler.ExecuteAll(), 1); Assert.IsFalse(transform.Post(102)); Assert.AreEqual(10000, Volatile.Read(ref n)); }
public void TransformManyBlockTest() { var block = new TransformManyBlock <int, int> ( i => new[] { -i, i }, new ExecutionDataflowBlockOptions { BoundedCapacity = 1 }); Assert.IsTrue(block.Post(1)); Assert.IsFalse(block.Post(2)); Assert.IsFalse(block.Completion.Wait(100)); Assert.IsFalse(block.Post(3)); Assert.AreEqual(-1, block.Receive()); Assert.IsFalse(block.Post(4)); Assert.AreEqual(1, block.Receive()); Assert.IsTrue(block.Post(5)); Assert.AreEqual(-5, block.Receive()); Assert.AreEqual(5, block.Receive()); }
public override void Post(Job job) { //Console.WriteLine($"{job.Id} posting"); for (var i = 0; i < 100; ++i) { job.AddData(job.Id); } _start.Post(job); }
public void TestPostAsyncEnumerable() { foreach (bool bounded in DataflowTestHelpers.BooleanValues) { var tb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToAsyncEnumerable, new ExecutionDataflowBlockOptions { BoundedCapacity = bounded ? 1 : -1 }); Assert.True(tb.Post(0)); tb.Complete(); Assert.False(tb.Post(0)); } }
public override void Post(Job job) { //Console.WriteLine($"{job.Id} posting"); for (var i = 0; i < 10; ++i) { job.AddData("data " + i, new DataProperty[] { new DataProperty { Name = (i % 2 == 0) ? "Even" : "Odd", Value = "" } }); } _start.Post(job); }
public ISourceBlock <DownloadedFile> DownloadFiles(string[] fileIds, string securityCookieString, string securityCookieDomain) { var urls = CreateUrls(fileIds); // we have to use TransformManyBlock here, because we want to be able to return 0 or 1 items var block = new TransformManyBlock <string, DownloadedFile>( async url => { var httpClientHandler = new HttpClientHandler(); if (!string.IsNullOrEmpty(securityCookieString)) { var securityCookie = new Cookie(FormsAuthentication.FormsCookieName, securityCookieString); securityCookie.Domain = securityCookieDomain; httpClientHandler.CookieContainer.Add(securityCookie); } return(await DownloadFile(url, httpClientHandler)); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Properties.Settings.Default.maxConcurrentDownloads }); foreach (var url in urls) { block.Post(url); } block.Complete(); return(block); }
public async Task FindProjectJsonAsync() { TransformManyBlock <GitHubRepo, SearchResult> repoSearchBlock = new TransformManyBlock <GitHubRepo, SearchResult>(repo => SearchRepoAsync(repo), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount * 4 //MaxDegreeOfParallelism = 1 }); ActionBlock <SearchResult> downloadFileBlock = new ActionBlock <SearchResult>(DownloadFileAsync, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount * 4 //MaxDegreeOfParallelism = 1 }); repoSearchBlock.LinkTo(downloadFileBlock, new DataflowLinkOptions() { PropagateCompletion = true }); foreach (var repo in _storage.GetAllRepos()) { if (_cancelToken.IsCancellationRequested) { break; } repoSearchBlock.Post(repo); } repoSearchBlock.Complete(); await downloadFileBlock.Completion; }
public async Task TestMultiplePublishersWithPropagateCompletion() { var block1 = new TransformManyBlock <string, char>(x => x.ToCharArray()); var block2 = new TransformManyBlock <string, char>(x => x.ToCharArray()); var target = new BufferBlock <char>(); var propagate = new DataflowLinkOptions { PropagateCompletion = true }; block1.LinkTo(target, propagate); block2.LinkTo(target, propagate); block1.Post("a"); // This propagates completion through to target block1.Complete(); await AssertCompletes(block1.Completion); Assert.IsTrue(await target.OutputAvailableAsync(), "target should have message waiting"); Assert.IsTrue(!target.Completion.IsCompleted, "target won't be complete until buffer empty"); // The target won't receive this, because it's already started completing (via propagation) block2.Post("b"); block2.Complete(); // Pulling this out of the buffer allows the target to complete Assert.AreEqual('a', target.Receive()); // This sholud happen pretty quickly await AssertCompletes(target.Completion); }
public void Start(CoreValues cv) { CoreValues = cv; MyCamera = new Camera(cv.Origin, cv.LookAt, cv.VecUp, cv.FovY, cv.AspectRatio, cv.Aperture, cv.DistToFocus); Image = new Vec3[cv.Width * cv.Height]; World = HitableList.DefaultWorld; Count = cv.Width * cv.Height; PixelFloats = new TransformManyBlock <int, InfoStruct>(GetSampleRays); MergeColors = new ActionBlock <InfoStruct>(DoMergeColors, new ExecutionDataflowBlockOptions { EnsureOrdered = false, MaxDegreeOfParallelism = -1 }); //writeImage = new ActionBlock<InfoStruct>(DoWriteImage, new ExecutionDataflowBlockOptions { EnsureOrdered = false, MaxDegreeOfParallelism = -1 }); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; PixelFloats.LinkTo(MergeColors, linkOptions); //MergeColors.LinkTo(writeImage, linkOptions); PixelFloats.Post(0); PixelFloats.Complete(); MergeColors.Completion.Wait(); DoSaveImage(); }
private async Task Start(string path) { var uiScheduler = TaskScheduler.FromCurrentSynchronizationContext(); dirToFilesBlock = new TransformManyBlock <string, string>((Func <string, IEnumerable <string> >)(GetFileSystemItems), new ExecutionDataflowBlockOptions() { CancellationToken = ct }); fileActionBlock = new ActionBlock <string>((Action <string>)ProcessFile, new ExecutionDataflowBlockOptions() { CancellationToken = ct, TaskScheduler = uiScheduler }); // Order of LinkTo's important here! dirToFilesBlock.LinkTo(dirToFilesBlock, new DataflowLinkOptions() { PropagateCompletion = true }, IsDirectory); dirToFilesBlock.LinkTo(fileActionBlock, new DataflowLinkOptions() { PropagateCompletion = true }, IsRequiredDocType); // Kick off the recursion. dirToFilesBlock.Post(path); await ProcessingIsComplete(); dirToFilesBlock.Complete(); await Task.WhenAll(dirToFilesBlock.Completion, fileActionBlock.Completion); }
public IEnumerable <IMetaData> TransformManyBlockUsage(string stringToSplit) { Console.WriteLine($"Inside {nameof(TplDataflow1ExecutionBlocksController)} - {nameof(TransformManyBlockUsage)}"); // Create the members of the pipeline. var transformManyBlockSplitAnInputStringIntoArray = new TransformManyBlock <string, string>(input => Functions.SplitAnInputStringIntoArray(input, SplitterSeparator) ); var transformBlockCreateASingleMedatadataFromAString = new TransformBlock <string, IMetaData>(stringInput => Functions.CreateASingleMedatadataFromAString(stringInput) ); // Connect the dataflow blocks to form a pipeline. transformManyBlockSplitAnInputStringIntoArray.LinkTo(transformBlockCreateASingleMedatadataFromAString, DataflowOptions.LinkOptions); // Start TransformManyBlockUsage pipeline with the input values. transformManyBlockSplitAnInputStringIntoArray.Post(stringToSplit); // Mark the head of the pipeline as complete. transformManyBlockSplitAnInputStringIntoArray.Complete(); // Equivalent of transformManyBlockSplitAnInputStringIntoArray.OutputCount var ouputCount = stringToSplit?.Split(SplitterSeparator, StringSplitOptions.RemoveEmptyEntries).Length ?? 0; for (var i = 0; i < ouputCount; i++) { yield return(transformBlockCreateASingleMedatadataFromAString.Receive()); } }
public async Task TestExceptions() { var tb1 = new TransformManyBlock <int, int>((Func <int, IEnumerable <int> >)(i => { throw new InvalidCastException(); })); var tb2 = new TransformManyBlock <int, int>((Func <int, Task <IEnumerable <int> > >)(i => { throw new InvalidProgramException(); })); var tb3 = new TransformManyBlock <int, int>((Func <int, Task <IEnumerable <int> > >)(i => Task.Run((Func <IEnumerable <int> >)(() => { throw new InvalidTimeZoneException(); })))); var tb4 = new TransformManyBlock <int, int>(i => ExceptionAfter(3)); var tb5 = new TransformManyBlock <int, int>(i => Task.Run(() => ExceptionAfter(3))); for (int i = 0; i < 3; i++) { tb1.Post(i); tb2.Post(i); tb3.Post(i); tb4.Post(i); tb5.Post(i); } await Assert.ThrowsAsync <InvalidCastException>(() => tb1.Completion); await Assert.ThrowsAsync <InvalidProgramException>(() => tb2.Completion); await Assert.ThrowsAsync <InvalidTimeZoneException>(() => tb3.Completion); await Assert.ThrowsAsync <FormatException>(() => tb4.Completion); await Assert.ThrowsAsync <FormatException>(() => tb5.Completion); Assert.All(new[] { tb1, tb2, tb3 }, tb => Assert.True(tb.InputCount == 0 && tb.OutputCount == 0)); }
public static async Task RunSampleOne(Func <int, Task <User[]> > getUsers, Func <User, Task> writeUser, int pageCount) { var getDataBlock = new TransformManyBlock <int, User>( transform: async(pageNumber) => await getUsers(pageNumber), dataflowBlockOptions: new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 3 }); var writeDataBlock = new ActionBlock <User>( action: async(user) => await writeUser(user), dataflowBlockOptions: new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 }); getDataBlock.LinkTo(writeDataBlock, new DataflowLinkOptions { PropagateCompletion = true }); for (var i = 1; i <= pageCount; i++) { getDataBlock.Post(i); } getDataBlock.Complete(); await writeDataBlock.Completion; }
public async Task RunAsync() { if (!IsBuilt) { Logger.LogError("Error run not arranged conveyor. Use {arrangeMethod}() before {runMethod}().", nameof(Arrange), nameof(RunAsync)); throw new InvalidOperationException($"Error run not arranged conveyor. Use {nameof(Arrange)}() before {nameof(RunAsync)}()."); } //Run conveyor Stopwatch stopWatch = Stopwatch.StartNew(); try { dataNamesEnumeratorBlock.Post(filePath); dataNamesEnumeratorBlock.Complete(); //Wait result await writeFilesBlock.Completion; } catch (AggregateException exs) { foreach (Exception ex in exs.InnerExceptions) { Logger.LogError("Errors while processing: {err}", ex.Message); } } stopWatch.Stop(); elapsedTime = stopWatch.ElapsedMilliseconds; }
public async Task TestOrdering() { const int iters = 9999; foreach (int mmpt in new[] { DataflowBlockOptions.Unbounded, 1 }) { foreach (int dop in new[] { 1, 2, DataflowBlockOptions.Unbounded }) { var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, MaxMessagesPerTask = mmpt }; var tb = new TransformManyBlock <int, int>(i => new[] { i, i + 1, i + 2 }, options); for (int i = 0; i < iters; i += 3) { Assert.True(tb.Post(i)); } for (int i = 0; i < iters; i++) { Assert.Equal(expected: i, actual: await tb.ReceiveAsync()); } tb.Complete(); await tb.Completion; } } }
public async Task TestCircularLinking() { const int Iters = 200; foreach (bool sync in DataflowTestHelpers.BooleanValues) { var tcs = new TaskCompletionSource <bool>(); Func <int, IEnumerable <int> > body = i => { if (i >= Iters) { tcs.SetResult(true); } return(Enumerable.Repeat(i + 1, 1)); }; TransformManyBlock <int, int> tb = sync ? new TransformManyBlock <int, int>(body) : new TransformManyBlock <int, int>(i => Task.Run(() => body(i))); using (tb.LinkTo(tb)) { tb.Post(0); await tcs.Task; tb.Complete(); } } }
public async Task TestPrecanceled() { var bb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable, new ExecutionDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true) }); int ignoredValue; IList <int> ignoredValues; IDisposable link = bb.LinkTo(DataflowBlock.NullTarget <int>()); Assert.NotNull(link); link.Dispose(); Assert.False(bb.Post(42)); var t = bb.SendAsync(42); Assert.True(t.IsCompleted); Assert.False(t.Result); Assert.False(bb.TryReceiveAll(out ignoredValues)); Assert.False(bb.TryReceive(out ignoredValue)); Assert.NotNull(bb.Completion); await Assert.ThrowsAnyAsync <OperationCanceledException>(() => bb.Completion); bb.Complete(); // just make sure it doesn't throw }
internal void start() { var consumerBlock = Consumer("consumer 1"); var consumer2Block = Consumer("\t\tconsumer 2"); var producerBlock = new TransformManyBlock <int, int>(x => Enumerable.Range(0, x)); producerBlock.LinkTo(consumerBlock, new DataflowLinkOptions { PropagateCompletion = true, }); producerBlock.LinkTo(consumer2Block, new DataflowLinkOptions { PropagateCompletion = true }); if (!producerBlock.Post(10)) { Console.WriteLine("Failed Post"); } producerBlock.Complete(); producerBlock.Completion.ContinueWith(p => print_status(p, "producer")); consumerBlock.Completion.ContinueWith(p => print_status(p, "consumer 1")); consumer2Block.Completion.ContinueWith(p => print_status(p, "\t\tconsumer 2")); }
public void DeferredUsageTest () { int insIndex = -1; int[] array = new int[5 + 3]; var evt = new CountdownEvent (array.Length); var block = new ActionBlock<int> (i => { array[Interlocked.Increment (ref insIndex)] = i; evt.Signal (); }); var trsm = new TransformManyBlock<int, int> (i => Enumerable.Range (0, i)); trsm.Post (5); trsm.Post (3); trsm.LinkTo (block); evt.Wait (); CollectionAssert.AreEquivalent (new int[] { 0, 1, 2, 3, 4, 0, 1, 2 }, array); }
public Task WalkAsync(string path) { directoriesRemaining = 1; directoryBrowseBlock.Post(path); return(fileActionBlock.Completion); }
public void ProcessDirectory(string dir) { imageCollectionBlock.Post(dir); //DirectoryInfo directory = new DirectoryInfo(dir); //foreach (FileInfo file in directory.GetFiles("*.jpg", SearchOption.AllDirectories)) //{ // loadAndToGreyBlock.Post(new Uri(file.FullName)); //} }
public void DeferredUsageTest() { int insIndex = -1; int[] array = new int[5 + 3]; var evt = new CountdownEvent(array.Length); var block = new ActionBlock <int> (i => { array[Interlocked.Increment(ref insIndex)] = i; evt.Signal(); }); var trsm = new TransformManyBlock <int, int> (i => Enumerable.Range(0, i)); trsm.Post(5); trsm.Post(3); trsm.LinkTo(block); evt.Wait(); CollectionAssert.AreEquivalent(new int[] { 0, 1, 2, 3, 4, 0, 1, 2 }, array); }
public static async Task PurgeExpiredAsync(CloudBlobContainer inboxContainer) { Requires.NotNull(inboxContainer, "inboxContainer"); var deleteBlobsExpiringBefore = DateTime.UtcNow; int purgedBlobCount = 0; var searchExpiredBlobs = new TransformManyBlock <CloudBlobContainer, ICloudBlob>( async c => { try { var results = await c.ListBlobsSegmentedAsync( string.Empty, useFlatBlobListing: true, pageSize: 50, details: BlobListingDetails.Metadata, options: new BlobRequestOptions(), operationContext: null); return(from blob in results.OfType <ICloudBlob>() let expires = DateTime.Parse(blob.Metadata[ExpirationDateMetadataKey]) where expires < deleteBlobsExpiringBefore select blob); } catch (StorageException ex) { var webException = ex.InnerException as WebException; if (webException != null) { var httpResponse = (HttpWebResponse)webException.Response; if (httpResponse.StatusCode == HttpStatusCode.NotFound) { // it's legit that some tests never created the container to begin with. return(Enumerable.Empty <ICloudBlob>()); } } throw; } }, new ExecutionDataflowBlockOptions { BoundedCapacity = 4, }); var deleteBlobBlock = new ActionBlock <ICloudBlob>( blob => { Interlocked.Increment(ref purgedBlobCount); return(blob.DeleteAsync()); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 4, BoundedCapacity = 100, }); searchExpiredBlobs.LinkTo(deleteBlobBlock, new DataflowLinkOptions { PropagateCompletion = true }); searchExpiredBlobs.Post(inboxContainer); searchExpiredBlobs.Complete(); await deleteBlobBlock.Completion; }
private static void Main() { var consumeBlock = new ActionBlock <int>(new Action <int>(Consumer)); var transformManyBlock = new TransformManyBlock <int, int>(new Func <int, IEnumerable <int> >(ProduceMany)); transformManyBlock.LinkTo(consumeBlock); transformManyBlock.Post(10); Console.ReadLine(); }
private static void DataStructure_TransformManyBlock() { // Create a TransformManyBlock<string, char> object that splits // a string into its individual characters. var transformManyBlock = new TransformManyBlock <string, char>( //返回的为IEnumrable<char> 委托类型Func<TInput, IEnumerable<TOutput>> //var transformManyBlock = new TransformBlock<string, char>( //返回的为char 委托类型Func<TInput, TOutput> s => s.ToCharArray()); // Post two messages to the first block. transformManyBlock.Post("Hello"); transformManyBlock.Post("World"); // Receive all output values from the block. for (int i = 0; i < ("Hello" + "World").Length; i++) { Console.WriteLine(transformManyBlock.Receive()); } }
public static void TestSync() { //这个地方就是将TransformManyBlock中的多个参数依次转换成了一个参数给ActionBlock tmb.LinkTo(ab); for (int i = 0; i < 4; i++) { tmb.Post(i); } Console.WriteLine("Finished post"); }
public async Task TestBlockBufferCount() { var block1 = new TransformBlock <int, int>(i => 2 * i); var block2 = new TransformManyBlock <int, int>(i => new [] { i }); var block3 = new ActionBlock <int>(i => { Thread.Sleep(1000); }); block1.Post(0); block2.Post(0); block2.Post(0); block3.Post(0); block3.Post(0); block3.Post(0); await Task.Delay(200); Assert.AreEqual(1, block1.GetBufferCount()); Assert.AreEqual(2, block2.GetBufferCount()); Assert.AreEqual(2, block3.GetBufferCount()); }
public async Task TestBlockBufferCount() { var block1 = new TransformBlock<int, int>(i => 2 * i); var block2 = new TransformManyBlock<int, int>(i => new [] { i }); var block3 = new ActionBlock<int>(i => { Thread.Sleep(1000); }); block1.Post(0); block2.Post(0); block2.Post(0); block3.Post(0); block3.Post(0); block3.Post(0); await Task.Delay(200); Assert.AreEqual(1, block1.GetBufferCount().Total()); Assert.AreEqual(2, block2.GetBufferCount().Total()); Assert.AreEqual(2, block3.GetBufferCount().Total()); }
public async Task TestReserveReleaseConsume() { var tb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable); tb.Post(1); await DataflowTestHelpers.TestReserveAndRelease(tb); tb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable); tb.Post(2); await DataflowTestHelpers.TestReserveAndConsume(tb); }
private static TransformManyBlock<int, int> ConstructTransformManyWithNMessages(int messagesCount) { var block = new TransformManyBlock<int, int>(i => new int[] { i }); for (int i = 0; i < messagesCount; i++) { block.Post(i); } // Spin until the messages have been properly buffered up. // Otherwise TryReceive fails. SpinWait.SpinUntil(() => block.OutputCount == messagesCount); return block; }
private static void Main(string[] args) { var generatorBlock = new TransformManyBlock<int, string>(num => GenerateStrings(num)); var writerBlock = new TransformBlock<string, string>(str => WriteString(str), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5 }); var finishingBlock = new ActionBlock<string>(str => { writerBlock.Completion.Wait(); Console.WriteLine(Thread.CurrentThread.ManagedThreadId + ": finished - " + str); }); generatorBlock.LinkTo(writerBlock, new DataflowLinkOptions{PropagateCompletion = true}); writerBlock.LinkTo(finishingBlock, new DataflowLinkOptions { PropagateCompletion = true }); for (var i = 1; i <= 3; i++) { Console.WriteLine("Posted " + i*10); generatorBlock.Post(i*10); } generatorBlock.Complete(); finishingBlock.Completion.Wait(); Console.WriteLine("Pipeline is finished"); Console.ReadKey(); }
public async Task TestCountZeroAtCompletion() { var cts = new CancellationTokenSource(); var tb = new TransformManyBlock<int, int>(DataflowTestHelpers.ToEnumerable, new ExecutionDataflowBlockOptions() { CancellationToken = cts.Token }); tb.Post(1); cts.Cancel(); await Assert.ThrowsAnyAsync<OperationCanceledException>(() => tb.Completion); Assert.Equal(expected: 0, actual: tb.InputCount); Assert.Equal(expected: 0, actual: tb.OutputCount); cts = new CancellationTokenSource(); tb = new TransformManyBlock<int, int>(DataflowTestHelpers.ToEnumerable); tb.Post(1); ((IDataflowBlock)tb).Fault(new InvalidOperationException()); await Assert.ThrowsAnyAsync<InvalidOperationException>(() => tb.Completion); Assert.Equal(expected: 0, actual: tb.InputCount); Assert.Equal(expected: 0, actual: tb.OutputCount); }
public async Task TestPrecanceled() { var bb = new TransformManyBlock<int, int>(DataflowTestHelpers.ToEnumerable, new ExecutionDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true) }); int ignoredValue; IList<int> ignoredValues; IDisposable link = bb.LinkTo(DataflowBlock.NullTarget<int>()); Assert.NotNull(link); link.Dispose(); Assert.False(bb.Post(42)); var t = bb.SendAsync(42); Assert.True(t.IsCompleted); Assert.False(t.Result); Assert.False(bb.TryReceiveAll(out ignoredValues)); Assert.False(bb.TryReceive(out ignoredValue)); Assert.NotNull(bb.Completion); await Assert.ThrowsAnyAsync<OperationCanceledException>(() => bb.Completion); bb.Complete(); // just make sure it doesn't throw }
public async Task TestExceptions() { var tb1 = new TransformManyBlock<int, int>((Func<int, IEnumerable<int>>)(i => { throw new InvalidCastException(); })); var tb2 = new TransformManyBlock<int, int>((Func<int, Task<IEnumerable<int>>>)(i => { throw new InvalidProgramException(); })); var tb3 = new TransformManyBlock<int, int>((Func<int, Task<IEnumerable<int>>>)(i => Task.Run((Func<IEnumerable<int>>)(() => { throw new InvalidTimeZoneException(); })))); var tb4 = new TransformManyBlock<int, int>(i => ExceptionAfter(3)); var tb5 = new TransformManyBlock<int, int>(i => Task.Run(() => ExceptionAfter(3))); for (int i = 0; i < 3; i++) { tb1.Post(i); tb2.Post(i); tb3.Post(i); tb4.Post(i); tb5.Post(i); } await Assert.ThrowsAsync<InvalidCastException>(() => tb1.Completion); await Assert.ThrowsAsync<InvalidProgramException>(() => tb2.Completion); await Assert.ThrowsAsync<InvalidTimeZoneException>(() => tb3.Completion); await Assert.ThrowsAsync<FormatException>(() => tb4.Completion); await Assert.ThrowsAsync<FormatException>(() => tb5.Completion); Assert.All(new[] { tb1, tb2, tb3 }, tb => Assert.True(tb.InputCount == 0 && tb.OutputCount == 0)); }
public async Task TestOrdering_Sync_OrderedDisabled(bool trustedEnumeration) { // If ordering were enabled, this test would hang. var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, EnsureOrdered = false }; var mres = new ManualResetEventSlim(); var tb = new TransformManyBlock<int, int>(i => { if (i == 0) mres.Wait(); return trustedEnumeration ? new[] { i } : Enumerable.Repeat(i, 1); }, options); tb.Post(0); tb.Post(1); Assert.Equal(1, await tb.ReceiveAsync()); mres.Set(); Assert.Equal(0, await tb.ReceiveAsync()); tb.Complete(); await tb.Completion; }
public void AsyncNullTest () { var scheduler = new TestScheduler (); var block = new TransformManyBlock<int, int> ( i => (Task<IEnumerable<int>>)null, new ExecutionDataflowBlockOptions { TaskScheduler = scheduler }); Assert.IsTrue (block.Post (1)); scheduler.ExecuteAll (); Assert.IsFalse (block.Completion.Wait (100)); block.Complete (); Assert.IsTrue (block.Completion.Wait (100)); }
public async void Crawl() { var scraper = new TransformManyBlock<LinkInfo, LinkInfo>( async linkInfo => { var linkInfos = new List<LinkInfo>(); try { var response = await linkInfo.Uri.GetResponse(); if (!IsSuccessStatus(response, linkInfo)) return linkInfos; var html = await response.Content.ReadAsStringAsync(); var doc = new HtmlDocument(); doc.LoadHtml(html); linkInfos = doc.ToLinkInfos(linkInfo.Uri); Log(Context.GoodHTML, linkInfo.Uri.AbsoluteUri); } catch (Exception error) { Log(Context.BadHTML, "Error: {0} (URL: {1})", error.Message, linkInfo.Uri); } return linkInfos; }, new ExecutionDataflowBlockOptions() { CancellationToken = cts.Token }); var fetcher = new ActionBlock<LinkInfo>( async linkInfo => { try { var fileName = linkInfo.GetFileName(linkInfo.Uri, "Downloads"); if (File.Exists(fileName)) { Log(Context.DupMedia, linkInfo.Uri.AbsoluteUri); return; } var response = await linkInfo.Uri.GetResponse(); if (!IsSuccessStatus(response, linkInfo)) return; var webStream = await response.Content.ReadAsStreamAsync(); fileName.EnsurePathExists(); using (var fileStream = File.OpenWrite(fileName)) await webStream.CopyToAsync(fileStream); Log(Context.GoodMedia, linkInfo.Uri.AbsoluteUri); } catch (Exception error) { Log(Context.BadMedia, "Error: {0} (URL: {1})", error.Message, linkInfo.Uri); } }, new ExecutionDataflowBlockOptions() { CancellationToken = cts.Token, MaxDegreeOfParallelism = Environment.ProcessorCount * 12 }); scraper.Completion.SetOnlyOnFaultedCompletion(error => HandleErrors(error)); fetcher.Completion.SetOnlyOnFaultedCompletion(error => HandleErrors(error)); scraper.LinkTo(scraper, new Predicate<LinkInfo>(li => li.Kind == LinkKind.HTML)); scraper.LinkTo(fetcher, new Predicate<LinkInfo>(li => li.Kind == LinkKind.Media)); scraper.Post(new LinkInfo(new Uri("http://www.bbc.com/news/"))); try { await Task.WhenAll(scraper.Completion, fetcher.Completion); } catch (OperationCanceledException) { } catch (Exception error) { Log(Context.Failure, "Error: " + error.Message); } if (OnFinished != null) OnFinished(this, EventArgs.Empty); }
public void RunTransformManyBlockConformanceTests() { bool passed = true; #region Sync { // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++) { Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory = options => { TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => new[] { i }, (ExecutionDataflowBlockOptions)options); ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); transformManyBlock.LinkTo(actionBlock); return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false }; }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token }; passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true); } // Test chained Post/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained Post all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true; for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16; Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i); Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive(); localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16)); Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test multiple yielded results { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { return Enumerable.Range(0, 10); }); t.Post(42); t.Complete(); for (int i = 0; i < 10; i++) { localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that OperationCanceledExceptions are ignored { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { if ((i % 2) == 0) throw new OperationCanceledException(); return new[] { i }; }); for (int i = 0; i < 10; i++) t.Post(i); t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test using a precanceled token { bool localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token }; var t = new TransformManyBlock<int, int>(i => new[] { i }, dbo); int ignoredValue; IList<int> ignoredValues; localPassed &= t.LinkTo(new ActionBlock<int>(delegate { })) != null; localPassed &= t.SendAsync(42).Result == false; localPassed &= t.TryReceiveAll(out ignoredValues) == false; localPassed &= t.Post(42) == false; localPassed &= t.OutputCount == 0; localPassed &= t.TryReceive(out ignoredValue) == false; localPassed &= t.Completion != null; t.Complete(); } catch (Exception) { localPassed = false; } Console.WriteLine(" > {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, IEnumerable<int>>(i => { throw new InvalidOperationException(); })); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test reuse of a list and array { bool localPassed = true; foreach (bool bounded in new[] { false, true }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (IList<int> list in new IList<int>[] { new int[1], new List<int>() { 0 }, new Collection<int>() { 0 } }) { int nextExpectedValue = 1; TransformManyBlock<int, int> tmb1 = null; tmb1 = new TransformManyBlock<int, int>(i => { if (i == 1000) { tmb1.Complete(); return (IEnumerable<int>)null; } else if (dop == 1) { list[0] = i + 1; return (IEnumerable<int>)list; } else if (list is int[]) { return new int[1] { i + 1 }; } else if (list is List<int>) { return new List<int>() { i + 1 }; } else return new Collection<int>() { i + 1 }; }, dbo); TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i => { if (i != nextExpectedValue) { localPassed = false; tmb1.Complete(); } nextExpectedValue++; return i; }); tmb1.LinkTo(tmb2); tmb2.LinkTo(tmb1); tmb1.SendAsync(0).Wait(); tmb1.Completion.Wait(); } } } Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test throwing an OCE { bool localPassed = true; foreach (bool bounded in new[] { true, false }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (int mode in new[] { 0, 1, 2 }) { const int ITERS = 50; var mres = new ManualResetEventSlim(); var tmb = new TransformManyBlock<int, int>(i => { if (i < ITERS - 1) throw new OperationCanceledException(); if (mode == 0) return new int[] { i }; else if (mode == 1) return new List<int>() { i }; else return Enumerable.Repeat(i, 1); }, dbo); var ab = new ActionBlock<int>(i => { if (i != ITERS - 1) localPassed = false; mres.Set(); }); tmb.LinkTo(ab); for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait(); mres.Wait(); } } } Console.WriteLine("{0}: Canceled invocation", localPassed ? "Success" : "Failure"); passed &= localPassed; } } #endregion #region Async { // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++) { Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory = options => { TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }), (ExecutionDataflowBlockOptions)options); ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); transformManyBlock.LinkTo(actionBlock); return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false }; }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token }; passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true); } // Test chained Post/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained Post all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true; for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16; Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i); Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive(); localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16)); Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test multiple yielded results { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)Enumerable.Range(0, 10).ToArray())); t.Post(42); t.Complete(); for (int i = 0; i < 10; i++) { localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that OperationCanceledExceptions are ignored { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { if ((i % 2) == 0) throw new OperationCanceledException(); return new[] { i }; }); for (int i = 0; i < 10; i++) t.Post(i); t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that null tasks are ignored { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { if ((i % 2) == 0) return null; return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }); }); for (int i = 0; i < 10; i++) t.Post(i); t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that null tasks are ignored when a reordering buffer is in place { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => { if (i == 0) { Task.Delay(1000).Wait(); return null; } return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }); }), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 }); t.Post(0); t.Post(1); try { localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1; } catch { localPassed = false; } Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting from the delegate { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => { throw new InvalidOperationException(); })); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting from the task { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => Task<IEnumerable<int>>.Factory.StartNew(() => { throw new InvalidOperationException(); }))); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test reuse of a list and array { bool localPassed = true; foreach (bool bounded in new[] { false, true }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (IList<int> list in new IList<int>[] { new int[1], new List<int>() { 0 }, new Collection<int>() { 0 } }) { int nextExpectedValue = 1; TransformManyBlock<int, int> tmb1 = null; tmb1 = new TransformManyBlock<int, int>(i => { return Task.Factory.StartNew(() => { if (i == 1000) { tmb1.Complete(); return (IEnumerable<int>)null; } else if (dop == 1) { list[0] = i + 1; return (IEnumerable<int>)list; } else if (list is int[]) { return new int[1] { i + 1 }; } else if (list is List<int>) { return new List<int>() { i + 1 }; } else return new Collection<int>() { i + 1 }; }); }, dbo); TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i => { if (i != nextExpectedValue) { localPassed = false; tmb1.Complete(); } nextExpectedValue++; return i; }); tmb1.LinkTo(tmb2); tmb2.LinkTo(tmb1); tmb1.SendAsync(0).Wait(); tmb1.Completion.Wait(); } } } Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test throwing an OCE { bool localPassed = true; foreach (bool bounded in new[] { true, false }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (int mode in new[] { 0, 1, 2 }) { const int ITERS = 50; var mres = new ManualResetEventSlim(); var tmb = new TransformManyBlock<int, int>(i => { var cts = new CancellationTokenSource(); return Task.Factory.StartNew(() => { if (i < ITERS - 1) { cts.Cancel(); cts.Token.ThrowIfCancellationRequested(); } if (mode == 0) return new int[] { i }; else if (mode == 1) return new List<int>() { i }; else return Enumerable.Repeat(i, 1); }, cts.Token); }, dbo); var ab = new ActionBlock<int>(i => { if (i != ITERS - 1) localPassed = false; mres.Set(); }); tmb.LinkTo(ab); for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait(); mres.Wait(); } } } Console.WriteLine(" > {0}: Canceled invocation", localPassed ? "Success" : "Failure"); passed &= localPassed; } } #endregion Assert.True(passed, "Test failed."); }
internal static bool TransformManyEnumerableToAction() { const int ITERS = 2; var data = new[] { 1 }; var tm = new TransformManyBlock<int, int>(i => data); int completedCount = 0; var c = new ActionBlock<int>(i => completedCount++); tm.LinkWithCompletion(c); for (int i = 0; i < ITERS; i++) tm.Post(i); tm.Complete(); c.Completion.Wait(); return completedCount == ITERS; }
public void AsyncCancelledTest () { var scheduler = new TestScheduler (); var block = new TransformManyBlock<int, int> ( i => { var tcs = new TaskCompletionSource<IEnumerable<int>> (); tcs.SetCanceled (); return tcs.Task; }, new ExecutionDataflowBlockOptions { TaskScheduler = scheduler }); Assert.IsTrue (block.Post (1)); scheduler.ExecuteAll (); Assert.IsFalse (block.Completion.Wait (100)); }
public void NullResultTest () { bool received = false; var transformMany = new TransformManyBlock<int, int> (i => (IEnumerable<int>)null); var action = new ActionBlock<int> (i => received = true); transformMany.LinkTo (action); Assert.IsTrue (transformMany.Post (1)); transformMany.Complete (); Assert.IsTrue (transformMany.Completion.Wait (100)); Assert.IsFalse (received); }
public async Task TestReserveReleaseConsume() { var tb = new TransformManyBlock<int, int>(DataflowTestHelpers.ToEnumerable); tb.Post(1); await DataflowTestHelpers.TestReserveAndRelease(tb); tb = new TransformManyBlock<int, int>(DataflowTestHelpers.ToEnumerable); tb.Post(2); await DataflowTestHelpers.TestReserveAndConsume(tb); }
public async Task TestMultipleYields() { const int Messages = 10; var t = new TransformManyBlock<int, int>(i => Enumerable.Range(0, Messages)); t.Post(42); t.Complete(); for (int i = 0; i < Messages; i++) { Assert.False(t.Completion.IsCompleted); Assert.Equal(expected: i, actual: await t.ReceiveAsync()); } await t.Completion; }
public async static Task<Animat> SeedAsync(ScenarioSpecies species, GeoRect geoRect, Bathymetry bathymetry) { var yxzFileName = Path.Combine(Path.GetTempPath(), Path.GetFileNameWithoutExtension(Path.GetRandomFileName()) + ".txt"); bathymetry.ToYXZ(yxzFileName, -1); var mbs = new C3mbs(); mbsRESULT mbsResult; if (mbsRESULT.OK != (mbsResult = mbs.SetOutputDirectory(Path.GetTempPath()))) throw new AnimatInterfaceMMBSException("SetOutputDirectory Error:" + mbs.ResultToTc(mbsResult)); var config = mbs.GetConfiguration(); config.enabled = false; // binary output enabled/disabled config.durationLess = true; // make sure we're in durationless mode. mbs.SetConfiguration(config); mbsResult = mbs.LoadBathymetryFromTextFile(yxzFileName); if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException("Bathymetry failed to load: " + mbs.ResultToTc(mbsResult)); mbsResult = mbs.AddSpecies(species.SpeciesDefinitionFilePath); if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException(string.Format("C3mbs::AddSpecies FATAL error {0} for species {1}", mbs.ResultToTc(mbsResult), species.SpeciesDefinitionFilePath)); var bounds = new GeoArray(geoRect.NorthWest, geoRect.NorthEast, geoRect.SouthEast, geoRect.SouthWest, geoRect.NorthWest); var result = new Animat { ScenarioSpecies = species }; var area = bounds.Area; //Debug.WriteLine("Area: {0}",area); var transformManyBlock = new TransformManyBlock<int, Geo<float>>(count => { var geos = new List<Geo<float>>(); for (var i = 0; i < count; i++) { var location = bounds.RandomLocationWithinPerimeter(); var depth = bathymetry.Samples.GetNearestPointAsync(location).Result.Data; mbsRESULT retval; lock (mbs) retval = mbs.AddIndividualAnimat(0, new mbsPosition { latitude = location.Latitude, longitude = location.Longitude, depth = 0 }); if (mbsRESULT.OK == retval) geos.Add(new Geo<float>(location.Latitude, location.Longitude, (float)(depth * Random.NextDouble()))); } return geos; }, new ExecutionDataflowBlockOptions { TaskScheduler = TaskScheduler.Default, BoundedCapacity = -1, MaxDegreeOfParallelism = -1, }); var bufferBlock = new BufferBlock<Geo<float>>(); transformManyBlock.LinkTo(bufferBlock); var population = (int)Math.Round(area * species.PopulationDensity); result.TotalAnimats = population; const int blockSize = 100; while (population > 0) { transformManyBlock.Post(population > blockSize ? blockSize : population); population -= blockSize; } transformManyBlock.Complete(); await transformManyBlock.Completion; //mbsResult = mbs.InitializeRun(); //if (mbsRESULT.OK == mbsResult) while (mbsRUNSTATE.INITIALIZING == mbs.GetRunState()) Thread.Sleep(1); //else throw new AnimatInterfaceMMBSException("C3mbs::Initialize FATAL error " + mbs.ResultToTc(mbsResult)); mbsResult = mbs.FinishRun(); if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException("C3mbs::FinishRun FATAL error " + mbs.ResultToTc(mbsResult)); IList<Geo<float>> animatGeos; if (bufferBlock.TryReceiveAll(out animatGeos)) result.Locations.AddRange(animatGeos); return result; }
public async Task TestOrdering() { const int iters = 9999; foreach (int mmpt in new[] { DataflowBlockOptions.Unbounded, 1 }) foreach (int dop in new[] { 1, 2, DataflowBlockOptions.Unbounded }) { var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, MaxMessagesPerTask = mmpt }; var tb = new TransformManyBlock<int, int>(i => new[] { i, i + 1, i + 2 }, options); for (int i = 0; i < iters; i += 3) { Assert.True(tb.Post(i)); } for (int i = 0; i < iters; i++) { Assert.Equal(expected: i, actual: await tb.ReceiveAsync()); } tb.Complete(); await tb.Completion; } }
public async static Task<Animat> SeedAsyncWithout3MB(ScenarioSpecies species, GeoRect geoRect, Bathymetry bathymetry) { var bounds = new GeoArray(geoRect.NorthWest, geoRect.NorthEast, geoRect.SouthEast, geoRect.SouthWest, geoRect.NorthWest); var result = new Animat { ScenarioSpecies = species }; var area = bounds.Area; //Debug.WriteLine("Area: {0}",area); var transformManyBlock = new TransformManyBlock<int, Geo<float>>(count => { var geos = new List<Geo<float>>(); for (var i = 0; i < count; i++) { var location = bounds.RandomLocationWithinPerimeter(); var depth = bathymetry.Samples.GetNearestPointAsync(location).Result.Data; if (depth < -50) geos.Add(new Geo<float>(location.Latitude, location.Longitude, (float)(depth * Random.NextDouble()))); } return geos; }, new ExecutionDataflowBlockOptions { TaskScheduler = TaskScheduler.Default, BoundedCapacity = -1, MaxDegreeOfParallelism = -1, }); var bufferBlock = new BufferBlock<Geo<float>>(); transformManyBlock.LinkTo(bufferBlock); var population = (int)Math.Round(area * species.PopulationDensity); result.TotalAnimats = population; const int blockSize = 100; while (population > 0) { transformManyBlock.Post(population > blockSize ? blockSize : population); population -= blockSize; } transformManyBlock.Complete(); await transformManyBlock.Completion; IList<Geo<float>> animatGeos; if (bufferBlock.TryReceiveAll(out animatGeos)) result.Locations.AddRange(animatGeos); return result; }
/// <summary> /// Purges all blobs set to expire prior to the specified date. /// </summary> /// <param name="deleteBlobsExpiringBefore"> /// All blobs scheduled to expire prior to this date will be purged. The default value /// is interpreted as <see cref="DateTime.UtcNow"/>. /// </param> /// <returns>The task representing the asynchronous operation.</returns> public async Task PurgeBlobsExpiringBeforeAsync(DateTime deleteBlobsExpiringBefore = default(DateTime)) { if (deleteBlobsExpiringBefore == default(DateTime)) { deleteBlobsExpiringBefore = DateTime.UtcNow; } Requires.Argument(deleteBlobsExpiringBefore.Kind == DateTimeKind.Utc, "expirationUtc", "UTC required."); var searchExpiredDirectoriesBlock = new TransformManyBlock<CloudBlobContainer, CloudBlobDirectory>( async c => { var results = await c.ListBlobsSegmentedAsync(); return from directory in results.OfType<CloudBlobDirectory>() let expires = DateTime.Parse(directory.Uri.Segments[directory.Uri.Segments.Length - 1].TrimEnd('/')) where expires < deleteBlobsExpiringBefore select directory; }, new ExecutionDataflowBlockOptions { BoundedCapacity = 4, }); var deleteDirectoryBlock = new TransformManyBlock<CloudBlobDirectory, CloudBlockBlob>( async directory => { var results = await directory.ListBlobsSegmentedAsync(); return results.OfType<CloudBlockBlob>(); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, BoundedCapacity = 4, }); var deleteBlobBlock = new ActionBlock<CloudBlockBlob>( blob => blob.DeleteAsync(), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 4, BoundedCapacity = 100, }); searchExpiredDirectoriesBlock.LinkTo(deleteDirectoryBlock, new DataflowLinkOptions { PropagateCompletion = true }); deleteDirectoryBlock.LinkTo(deleteBlobBlock, new DataflowLinkOptions { PropagateCompletion = true }); searchExpiredDirectoriesBlock.Post(this.container); searchExpiredDirectoriesBlock.Complete(); await deleteBlobBlock.Completion; }
public static async Task PurgeExpiredAsync(CloudBlobContainer inboxContainer) { Requires.NotNull(inboxContainer, "inboxContainer"); var deleteBlobsExpiringBefore = DateTime.UtcNow; int purgedBlobCount = 0; var searchExpiredBlobs = new TransformManyBlock<CloudBlobContainer, ICloudBlob>( async c => { try { var results = await c.ListBlobsSegmentedAsync( string.Empty, useFlatBlobListing: true, pageSize: 50, details: BlobListingDetails.Metadata, options: new BlobRequestOptions(), operationContext: null); return from blob in results.OfType<ICloudBlob>() let expires = DateTime.Parse(blob.Metadata[ExpirationDateMetadataKey]) where expires < deleteBlobsExpiringBefore select blob; } catch (StorageException ex) { var webException = ex.InnerException as WebException; if (webException != null) { var httpResponse = (HttpWebResponse)webException.Response; if (httpResponse.StatusCode == HttpStatusCode.NotFound) { // it's legit that some tests never created the container to begin with. return Enumerable.Empty<ICloudBlob>(); } } throw; } }, new ExecutionDataflowBlockOptions { BoundedCapacity = 4, }); var deleteBlobBlock = new ActionBlock<ICloudBlob>( blob => { Interlocked.Increment(ref purgedBlobCount); return blob.DeleteAsync(); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 4, BoundedCapacity = 100, }); searchExpiredBlobs.LinkTo(deleteBlobBlock, new DataflowLinkOptions { PropagateCompletion = true }); searchExpiredBlobs.Post(inboxContainer); searchExpiredBlobs.Complete(); await deleteBlobBlock.Completion; }