internal static void Run() { const string w1 = "Hello"; const string w2 = "World"; // Create a TransformManyBlock<string, char> object that splits // a string into its individual characters. var transformManyBlock = new TransformManyBlock <string, char>(s => s.ToCharArray()); // Chained buffer block var buffer = new BufferBlock <char>(); transformManyBlock.LinkTo(buffer); transformManyBlock.Completion.ContinueWith(_ => buffer.Complete()); // Post two messages to the first block. transformManyBlock.SendAsync(w1); transformManyBlock.SendAsync(w2); transformManyBlock.Complete(); transformManyBlock.Completion.Wait(); // Receive all output values from the last block. IList <char> chars; while (!buffer.TryReceiveAll(out chars)) { // Needed if we don't wait for transformManyBlock.Completion Console.Write("."); Thread.SpinWait(10000); } Console.WriteLine(chars == null ? "null" : string.Join(", ", chars)); }
protected override async Task ExecuteNodeTreeAsync(ExecutionContext context, ObjectExecutionNode rootNode) { //Options var blockOptions = new ExecutionDataflowBlockOptions { CancellationToken = context.CancellationToken, MaxDegreeOfParallelism = 1024 }; //Execute Block var block = new TransformManyBlock <Job, Job>( async job => { var node = await ExecuteNodeAsync(job.Context, job.Node).ConfigureAwait(false); var childJobs = (job.Node as IParentExecutionNode)? .GetChildNodes() .Select(child => job.CreateChildJob(job.Context, child)) .ToArray(); job.Complete(); return(childJobs); }, blockOptions); //Link to self block.LinkTo(block); //Start var rootJob = new Job(context, rootNode); await block.SendAsync(rootJob); //Wait until done await rootJob.Completion; }
public override async Task ProcessAsync() { var settings = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 10, }; var listFilesBlock = new TransformManyBlock <string, string>(_fileSystem.GetFileNames, settings); var getVehiclesBlock = new TransformManyBlock <string, IVehicle>(GetVehicles, settings); var transformBlock = new TransformBlock <IVehicle, Truck>(TransformAsync, settings); var doubleBlock = new TransformBlock <Truck, Truck>(DoubleDoorsAsync, settings); var batchBlock = new BatchBlock <Truck>(10); var saveBlock = new ActionBlock <IEnumerable <Truck> >(SaveTrucksAsync, settings); DataflowLinkOptions linkOptions = new DataflowLinkOptions() { PropagateCompletion = true }; listFilesBlock.LinkTo(getVehiclesBlock, linkOptions); getVehiclesBlock.LinkTo(transformBlock, linkOptions); transformBlock.LinkTo(doubleBlock, linkOptions); doubleBlock.LinkTo(batchBlock, linkOptions); batchBlock.LinkTo(saveBlock, linkOptions); await listFilesBlock.SendAsync(_directory); listFilesBlock.Complete(); await saveBlock.Completion; }
public async Task TestPrecanceled() { var bb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable, new ExecutionDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true) }); int ignoredValue; IList <int> ignoredValues; IDisposable link = bb.LinkTo(DataflowBlock.NullTarget <int>()); Assert.NotNull(link); link.Dispose(); Assert.False(bb.Post(42)); var t = bb.SendAsync(42); Assert.True(t.IsCompleted); Assert.False(t.Result); Assert.False(bb.TryReceiveAll(out ignoredValues)); Assert.False(bb.TryReceive(out ignoredValue)); Assert.NotNull(bb.Completion); await Assert.ThrowsAnyAsync <OperationCanceledException>(() => bb.Completion); bb.Complete(); // just make sure it doesn't throw }
public async Task TestProducerConsumer() { foreach (TaskScheduler scheduler in new[] { TaskScheduler.Default, new ConcurrentExclusiveSchedulerPair().ConcurrentScheduler }) { foreach (int maxMessagesPerTask in new[] { DataflowBlockOptions.Unbounded, 1, 2 }) { foreach (int boundedCapacity in new[] { DataflowBlockOptions.Unbounded, 1, 2 }) { foreach (int dop in new[] { 1, 2 }) { foreach (int elementsPerItem in new[] { 1, 3, 5 }) { foreach (bool sync in DataflowTestHelpers.BooleanValues) { const int Messages = 50; var options = new ExecutionDataflowBlockOptions { BoundedCapacity = boundedCapacity, MaxDegreeOfParallelism = dop, MaxMessagesPerTask = maxMessagesPerTask, TaskScheduler = scheduler }; TransformManyBlock <int, int> tb = sync ? new TransformManyBlock <int, int>(i => Enumerable.Repeat(i, elementsPerItem), options) : new TransformManyBlock <int, int>(i => TaskShim.Run(() => Enumerable.Repeat(i, elementsPerItem)), options); await TaskShim.WhenAll( TaskShim.Run(async delegate { // consumer int i = 0; int processed = 0; while (await tb.OutputAvailableAsync()) { Assert.Equal(expected: i, actual: await tb.ReceiveAsync()); processed++; if (processed % elementsPerItem == 0) { i++; } } }), TaskShim.Run(async delegate { // producer for (int i = 0; i < Messages; i++) { await tb.SendAsync(i); } tb.Complete(); })); } } } } } } }
/// <summary> /// Reflects over a set of assemblies and produces MEF parts for every applicable type. /// </summary> /// <param name="assemblyPaths">The paths to assemblies to search for MEF parts.</param> /// <param name="progress">An optional way to receive progress updates on how discovery is progressing.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns>A set of generated parts.</returns> public async Task <DiscoveredParts> CreatePartsAsync(IEnumerable <string> assemblyPaths, IProgress <DiscoveryProgress> progress = null, CancellationToken cancellationToken = default(CancellationToken)) { Requires.NotNull(assemblyPaths, nameof(assemblyPaths)); var exceptions = new List <PartDiscoveryException>(); var tuple = this.CreateAssemblyDiscoveryBlockChain(progress, cancellationToken); var assemblyLoader = new TransformManyBlock <string, Assembly>( path => { try { #if NET45 return(new Assembly[] { Assembly.Load(AssemblyName.GetAssemblyName(path)) }); #elif NETCOREAPP1_0 return(new Assembly[] { System.Runtime.Loader.AssemblyLoadContext.Default.LoadFromAssemblyPath(path) }); #else throw new NotSupportedException(); #endif } catch (Exception ex) { lock (exceptions) { exceptions.Add(new PartDiscoveryException(string.Format(CultureInfo.CurrentCulture, Strings.UnableToLoadAssembly, path), ex) { AssemblyPath = path }); } return(Enumerable.Empty <Assembly>()); } }, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = Environment.ProcessorCount, }); assemblyLoader.LinkTo(tuple.Item1, new DataflowLinkOptions { PropagateCompletion = true }); foreach (var assemblyPath in assemblyPaths) { await assemblyLoader.SendAsync(assemblyPath); } assemblyLoader.Complete(); var result = await tuple.Item2; return(result.Merge(new DiscoveredParts(Enumerable.Empty <ComposablePartDefinition>(), exceptions))); }
/// <summary> /// Reflects over a set of assemblies and produces MEF parts for every applicable type. /// </summary> /// <param name="assemblyPaths">The paths to assemblies to search for MEF parts.</param> /// <param name="progress">An optional way to receive progress updates on how discovery is progressing.</param> /// <param name="cancellationToken">The cancellation token.</param> /// <returns>A set of generated parts.</returns> public async Task <DiscoveredParts> CreatePartsAsync(IEnumerable <string> assemblyPaths, IProgress <DiscoveryProgress>?progress = null, CancellationToken cancellationToken = default(CancellationToken)) { Requires.NotNull(assemblyPaths, nameof(assemblyPaths)); var exceptions = new List <PartDiscoveryException>(); var tuple = this.CreateAssemblyDiscoveryBlockChain(progress, cancellationToken); var assemblyLoader = new TransformManyBlock <string, Assembly>( path => { try { return(new Assembly[] { this.Resolver.AssemblyLoader.LoadAssembly(Utilities.GetAssemblyNameWithCodebasePath(path)) }); } catch (Exception ex) { lock (exceptions) { exceptions.Add(new PartDiscoveryException(string.Format(CultureInfo.CurrentCulture, Strings.UnableToLoadAssembly, path), ex) { AssemblyPath = path }); } return(Enumerable.Empty <Assembly>()); } }, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = Environment.ProcessorCount, }); assemblyLoader.LinkTo(tuple.Item1, new DataflowLinkOptions { PropagateCompletion = true }); foreach (var assemblyPath in assemblyPaths) { await assemblyLoader.SendAsync(assemblyPath).ConfigureAwait(false); } assemblyLoader.Complete(); var result = await tuple.Item2.ConfigureAwait(false); return(result.Merge(new DiscoveredParts(Enumerable.Empty <ComposablePartDefinition>(), exceptions))); }
private async Task <Task> ProduceParticipantActionsAsync( Func <Task <IEnumerable <Task <int[]> > > > getParticipants, Func <int[], Task <Action[]> > getParticipantActions, ITargetBlock <Action[]> actionQueue, int maxDegreeOfParallelism ) { var participantsQueryBlock = new TransformManyBlock <Task <int[]>, int>(async task => await task, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = -1 }); var participantsBuffer = new BatchBlock <int>(_batchOptions.Value.BatchSize); var getParticipantActionsBlock = new ActionBlock <int[]>(async participantIds => { var actions = await getParticipantActions(participantIds); bool sendStatus = await actionQueue.SendAsync(actions); if (!sendStatus) { throw new InvalidOperationException("Revisit bottleneck"); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }); participantsQueryBlock.LinkTo(participantsBuffer, PropagateCompletion); participantsBuffer.LinkTo(getParticipantActionsBlock, PropagateCompletion); foreach (var participantTask in await getParticipants()) { bool sendStatus = await participantsQueryBlock.SendAsync(participantTask); if (!sendStatus) { throw new InvalidOperationException("Revisit bottleneck"); } } participantsQueryBlock.Complete(); return(getParticipantActionsBlock.Completion); }
public async Task <bool> SendDiscoveryJobAsync(IPeer peer) { return(await _discoverNodesDataflow.SendAsync(peer)); }
public async Task Run() { var current = 0; var pending = 0; var repositoryCount = 0; var fetchBlock = new TransformManyBlock <IGitService, Repository>( async service => { var friendlyServiceName = service.Provider.ToString(); logger.LogInformation($"Fetching repositories from {friendlyServiceName} ..."); var providerSettings = settings.Git.Where(x => x.Type == service.Provider); var repositories = await service.GetRepositoryUrisAsync(providerSettings); if (providerSettings.Any() && repositories.Count == 0) { throw new InvalidOperationException($"{friendlyServiceName} API returned 0 repositories. Please check your credentials."); } Interlocked.Add(ref repositoryCount, repositories.Count); logger.LogInformation($"Fetched {repositories.Count} repositories from {friendlyServiceName}. ({repositoryCount} total repositories to backup) "); return(repositories); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = 1000, MaxDegreeOfParallelism = settings.Backup.MaxDegreeOfParallelism }); var mirrorBlock = new ActionBlock <Repository>( async repository => { Interlocked.Increment(ref current); Interlocked.Increment(ref pending); logger.LogInformation($"Cloning {current} of {repositoryCount} ({pending} running) ({repository.HttpsUrl})"); await backupRunner.Backup(repository); logger.LogInformation($"Finished {repository.HttpsUrl} ({pending} pending)"); Interlocked.Decrement(ref pending); }, new ExecutionDataflowBlockOptions() { BoundedCapacity = 1000, MaxDegreeOfParallelism = settings.Backup.MaxDegreeOfParallelism, }); fetchBlock.LinkTo(mirrorBlock, new DataflowLinkOptions() { PropagateCompletion = true }); foreach (var service in gitServices) { await fetchBlock.SendAsync(service); } fetchBlock.Complete(); await mirrorBlock.Completion; await backupRunner.Cleanup(); await heartbeat.Notify(); logger.LogInformation("Done."); }
public void RunTransformManyBlockConformanceTests() { bool passed = true; #region Sync { // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; passed& feedMethod < FeedMethod._Count; feedMethod++) { Func <DataflowBlockOptions, TargetProperties <int> > transformManyBlockFactory = options => { TransformManyBlock <int, int> transformManyBlock = new TransformManyBlock <int, int>(i => new[] { i }, (ExecutionDataflowBlockOptions)options); ActionBlock <int> actionBlock = new ActionBlock <int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); transformManyBlock.LinkTo(actionBlock); return(new TargetProperties <int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false }); }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token }; passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true); } // Test chained Post/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained Post all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) { localPassed &= network.Post(i) == true; } for (int i = 0; i < ITERS; i++) { localPassed &= ((IReceivableSourceBlock <int>)network).Receive() == i * 16; } Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 })); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) { tasks[i - 1] = network.SendAsync(i); } Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) { total += ((IReceivableSourceBlock <int>)network).Receive(); } localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16)); Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test multiple yielded results { bool localPassed = true; var t = new TransformManyBlock <int, int>(i => { return(Enumerable.Range(0, 10)); }); t.Post(42); t.Complete(); for (int i = 0; i < 10; i++) { localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that OperationCanceledExceptions are ignored { bool localPassed = true; var t = new TransformManyBlock <int, int>(i => { if ((i % 2) == 0) { throw new OperationCanceledException(); } return(new[] { i }); }); for (int i = 0; i < 10; i++) { t.Post(i); } t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) { localPassed &= t.Receive() == i; } } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test using a precanceled token { bool localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token }; var t = new TransformManyBlock <int, int>(i => new[] { i }, dbo); int ignoredValue; IList <int> ignoredValues; localPassed &= t.LinkTo(new ActionBlock <int>(delegate { })) != null; localPassed &= t.SendAsync(42).Result == false; localPassed &= t.TryReceiveAll(out ignoredValues) == false; localPassed &= t.Post(42) == false; localPassed &= t.OutputCount == 0; localPassed &= t.TryReceive(out ignoredValue) == false; localPassed &= t.Completion != null; t.Complete(); } catch (Exception) { localPassed = false; } Console.WriteLine(" > {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting { bool localPassed = true; var t = new TransformManyBlock <int, int>(new Func <int, IEnumerable <int> >(i => { throw new InvalidOperationException(); })); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test reuse of a list and array { bool localPassed = true; foreach (bool bounded in new[] { false, true }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (IList <int> list in new IList <int>[] { new int[1], new List <int>() { 0 }, new Collection <int>() { 0 } }) { int nextExpectedValue = 1; TransformManyBlock <int, int> tmb1 = null; tmb1 = new TransformManyBlock <int, int>(i => { if (i == 1000) { tmb1.Complete(); return((IEnumerable <int>)null); } else if (dop == 1) { list[0] = i + 1; return((IEnumerable <int>)list); } else if (list is int[]) { return(new int[1] { i + 1 }); } else if (list is List <int> ) { return(new List <int>() { i + 1 }); } else { return new Collection <int>() { i + 1 } }; }, dbo); TransformBlock <int, int> tmb2 = new TransformBlock <int, int>(i => { if (i != nextExpectedValue) { localPassed = false; tmb1.Complete(); } nextExpectedValue++; return(i); }); tmb1.LinkTo(tmb2); tmb2.LinkTo(tmb1); tmb1.SendAsync(0).Wait(); tmb1.Completion.Wait(); } } } Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test throwing an OCE { bool localPassed = true; foreach (bool bounded in new[] { true, false }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (int mode in new[] { 0, 1, 2 }) { const int ITERS = 50; var mres = new ManualResetEventSlim(); var tmb = new TransformManyBlock <int, int>(i => { if (i < ITERS - 1) { throw new OperationCanceledException(); } if (mode == 0) { return new int[] { i } } ; else if (mode == 1) { return new List <int>() { i } } ; else { return(Enumerable.Repeat(i, 1)); } }, dbo); var ab = new ActionBlock <int>(i => { if (i != ITERS - 1) { localPassed = false; } mres.Set(); }); tmb.LinkTo(ab); for (int i = 0; i < ITERS; i++) { tmb.SendAsync(i).Wait(); } mres.Wait(); } } } Console.WriteLine("{0}: Canceled invocation", localPassed ? "Success" : "Failure"); passed &= localPassed; } } #endregion #region Async { // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; passed& feedMethod < FeedMethod._Count; feedMethod++) { Func <DataflowBlockOptions, TargetProperties <int> > transformManyBlockFactory = options => { TransformManyBlock <int, int> transformManyBlock = new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i }), (ExecutionDataflowBlockOptions)options); ActionBlock <int> actionBlock = new ActionBlock <int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); transformManyBlock.LinkTo(actionBlock); return(new TargetProperties <int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false }); }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token }; passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true); } // Test chained Post/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained Post all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) { localPassed &= network.Post(i) == true; } for (int i = 0; i < ITERS; i++) { localPassed &= ((IReceivableSourceBlock <int>)network).Receive() == i * 16; } Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 }))); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) { tasks[i - 1] = network.SendAsync(i); } Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) { total += ((IReceivableSourceBlock <int>)network).Receive(); } localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16)); Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test multiple yielded results { bool localPassed = true; var t = new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>)Enumerable.Range(0, 10).ToArray())); t.Post(42); t.Complete(); for (int i = 0; i < 10; i++) { localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that OperationCanceledExceptions are ignored { bool localPassed = true; var t = new TransformManyBlock <int, int>(i => { if ((i % 2) == 0) { throw new OperationCanceledException(); } return(new[] { i }); }); for (int i = 0; i < 10; i++) { t.Post(i); } t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) { localPassed &= t.Receive() == i; } } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that null tasks are ignored { bool localPassed = true; var t = new TransformManyBlock <int, int>(i => { if ((i % 2) == 0) { return(null); } return(Task.Run(() => (IEnumerable <int>) new[] { i })); }); for (int i = 0; i < 10; i++) { t.Post(i); } t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) { localPassed &= t.Receive() == i; } } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that null tasks are ignored when a reordering buffer is in place { bool localPassed = true; var t = new TransformManyBlock <int, int>(new Func <int, Task <IEnumerable <int> > >(i => { if (i == 0) { Task.Delay(1000).Wait(); return(null); } return(Task.Run(() => (IEnumerable <int>) new[] { i })); }), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 }); t.Post(0); t.Post(1); try { localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1; } catch { localPassed = false; } Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting from the delegate { bool localPassed = true; var t = new TransformManyBlock <int, int>(new Func <int, Task <IEnumerable <int> > >(i => { throw new InvalidOperationException(); })); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting from the task { bool localPassed = true; var t = new TransformManyBlock <int, int>(new Func <int, Task <IEnumerable <int> > >(i => Task.Run <IEnumerable <int> >(new Func <IEnumerable <int> >(() => { throw new InvalidOperationException(); })))); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test reuse of a list and array { bool localPassed = true; foreach (bool bounded in new[] { false, true }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (IList <int> list in new IList <int>[] { new int[1], new List <int>() { 0 }, new Collection <int>() { 0 } }) { int nextExpectedValue = 1; TransformManyBlock <int, int> tmb1 = null; tmb1 = new TransformManyBlock <int, int>(i => { return(Task.Run(() => { if (i == 1000) { tmb1.Complete(); return (IEnumerable <int>)null; } else if (dop == 1) { list[0] = i + 1; return (IEnumerable <int>)list; } else if (list is int[]) { return new int[1] { i + 1 }; } else if (list is List <int> ) { return new List <int>() { i + 1 }; } else { return new Collection <int>() { i + 1 } }; })); }, dbo); TransformBlock <int, int> tmb2 = new TransformBlock <int, int>(i => { if (i != nextExpectedValue) { localPassed = false; tmb1.Complete(); } nextExpectedValue++; return(i); }); tmb1.LinkTo(tmb2); tmb2.LinkTo(tmb1); tmb1.SendAsync(0).Wait(); tmb1.Completion.Wait(); } } } Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test throwing an OCE { bool localPassed = true; foreach (bool bounded in new[] { true, false }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (int mode in new[] { 0, 1, 2 }) { const int ITERS = 50; var mres = new ManualResetEventSlim(); var tmb = new TransformManyBlock <int, int>(i => { var cts = new CancellationTokenSource(); return(Task.Run(() => { if (i < ITERS - 1) { cts.Cancel(); cts.Token.ThrowIfCancellationRequested(); } if (mode == 0) { return new int[] { i } } ; else if (mode == 1) { return new List <int>() { i } } ; else { return Enumerable.Repeat(i, 1); } }, cts.Token)); }, dbo); var ab = new ActionBlock <int>(i => { if (i != ITERS - 1) { localPassed = false; } mres.Set(); }); tmb.LinkTo(ab); for (int i = 0; i < ITERS; i++) { tmb.SendAsync(i).Wait(); } mres.Wait(); } } } Console.WriteLine(" > {0}: Canceled invocation", localPassed ? "Success" : "Failure"); passed &= localPassed; } } #endregion Assert.True(passed, "Test failed."); }
public void RunTransformManyBlockConformanceTests() { bool passed = true; #region Sync { // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++) { Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory = options => { TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => new[] { i }, (ExecutionDataflowBlockOptions)options); ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); transformManyBlock.LinkTo(actionBlock); return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false }; }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token }; passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true); } // Test chained Post/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained Post all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true; for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16; Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i); Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive(); localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16)); Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test multiple yielded results { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { return Enumerable.Range(0, 10); }); t.Post(42); t.Complete(); for (int i = 0; i < 10; i++) { localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that OperationCanceledExceptions are ignored { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { if ((i % 2) == 0) throw new OperationCanceledException(); return new[] { i }; }); for (int i = 0; i < 10; i++) t.Post(i); t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test using a precanceled token { bool localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token }; var t = new TransformManyBlock<int, int>(i => new[] { i }, dbo); int ignoredValue; IList<int> ignoredValues; localPassed &= t.LinkTo(new ActionBlock<int>(delegate { })) != null; localPassed &= t.SendAsync(42).Result == false; localPassed &= t.TryReceiveAll(out ignoredValues) == false; localPassed &= t.Post(42) == false; localPassed &= t.OutputCount == 0; localPassed &= t.TryReceive(out ignoredValue) == false; localPassed &= t.Completion != null; t.Complete(); } catch (Exception) { localPassed = false; } Console.WriteLine(" > {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, IEnumerable<int>>(i => { throw new InvalidOperationException(); })); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test reuse of a list and array { bool localPassed = true; foreach (bool bounded in new[] { false, true }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (IList<int> list in new IList<int>[] { new int[1], new List<int>() { 0 }, new Collection<int>() { 0 } }) { int nextExpectedValue = 1; TransformManyBlock<int, int> tmb1 = null; tmb1 = new TransformManyBlock<int, int>(i => { if (i == 1000) { tmb1.Complete(); return (IEnumerable<int>)null; } else if (dop == 1) { list[0] = i + 1; return (IEnumerable<int>)list; } else if (list is int[]) { return new int[1] { i + 1 }; } else if (list is List<int>) { return new List<int>() { i + 1 }; } else return new Collection<int>() { i + 1 }; }, dbo); TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i => { if (i != nextExpectedValue) { localPassed = false; tmb1.Complete(); } nextExpectedValue++; return i; }); tmb1.LinkTo(tmb2); tmb2.LinkTo(tmb1); tmb1.SendAsync(0).Wait(); tmb1.Completion.Wait(); } } } Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test throwing an OCE { bool localPassed = true; foreach (bool bounded in new[] { true, false }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (int mode in new[] { 0, 1, 2 }) { const int ITERS = 50; var mres = new ManualResetEventSlim(); var tmb = new TransformManyBlock<int, int>(i => { if (i < ITERS - 1) throw new OperationCanceledException(); if (mode == 0) return new int[] { i }; else if (mode == 1) return new List<int>() { i }; else return Enumerable.Repeat(i, 1); }, dbo); var ab = new ActionBlock<int>(i => { if (i != ITERS - 1) localPassed = false; mres.Set(); }); tmb.LinkTo(ab); for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait(); mres.Wait(); } } } Console.WriteLine("{0}: Canceled invocation", localPassed ? "Success" : "Failure"); passed &= localPassed; } } #endregion #region Async { // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++) { Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory = options => { TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }), (ExecutionDataflowBlockOptions)options); ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); transformManyBlock.LinkTo(actionBlock); return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false }; }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token }; passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true); } // Test chained Post/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained Post all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true; for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16; Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i); Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive(); localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16)); Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test multiple yielded results { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)Enumerable.Range(0, 10).ToArray())); t.Post(42); t.Complete(); for (int i = 0; i < 10; i++) { localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that OperationCanceledExceptions are ignored { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { if ((i % 2) == 0) throw new OperationCanceledException(); return new[] { i }; }); for (int i = 0; i < 10; i++) t.Post(i); t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that null tasks are ignored { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { if ((i % 2) == 0) return null; return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }); }); for (int i = 0; i < 10; i++) t.Post(i); t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that null tasks are ignored when a reordering buffer is in place { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => { if (i == 0) { Task.Delay(1000).Wait(); return null; } return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }); }), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 }); t.Post(0); t.Post(1); try { localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1; } catch { localPassed = false; } Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting from the delegate { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => { throw new InvalidOperationException(); })); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting from the task { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => Task<IEnumerable<int>>.Factory.StartNew(() => { throw new InvalidOperationException(); }))); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test reuse of a list and array { bool localPassed = true; foreach (bool bounded in new[] { false, true }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (IList<int> list in new IList<int>[] { new int[1], new List<int>() { 0 }, new Collection<int>() { 0 } }) { int nextExpectedValue = 1; TransformManyBlock<int, int> tmb1 = null; tmb1 = new TransformManyBlock<int, int>(i => { return Task.Factory.StartNew(() => { if (i == 1000) { tmb1.Complete(); return (IEnumerable<int>)null; } else if (dop == 1) { list[0] = i + 1; return (IEnumerable<int>)list; } else if (list is int[]) { return new int[1] { i + 1 }; } else if (list is List<int>) { return new List<int>() { i + 1 }; } else return new Collection<int>() { i + 1 }; }); }, dbo); TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i => { if (i != nextExpectedValue) { localPassed = false; tmb1.Complete(); } nextExpectedValue++; return i; }); tmb1.LinkTo(tmb2); tmb2.LinkTo(tmb1); tmb1.SendAsync(0).Wait(); tmb1.Completion.Wait(); } } } Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test throwing an OCE { bool localPassed = true; foreach (bool bounded in new[] { true, false }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (int mode in new[] { 0, 1, 2 }) { const int ITERS = 50; var mres = new ManualResetEventSlim(); var tmb = new TransformManyBlock<int, int>(i => { var cts = new CancellationTokenSource(); return Task.Factory.StartNew(() => { if (i < ITERS - 1) { cts.Cancel(); cts.Token.ThrowIfCancellationRequested(); } if (mode == 0) return new int[] { i }; else if (mode == 1) return new List<int>() { i }; else return Enumerable.Repeat(i, 1); }, cts.Token); }, dbo); var ab = new ActionBlock<int>(i => { if (i != ITERS - 1) localPassed = false; mres.Set(); }); tmb.LinkTo(ab); for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait(); mres.Wait(); } } } Console.WriteLine(" > {0}: Canceled invocation", localPassed ? "Success" : "Failure"); passed &= localPassed; } } #endregion Assert.True(passed, "Test failed."); }
public void Submit(SampleCollection samples) { _filterSampleCountBlock.SendAsync(samples).Wait(); }
public async Task TestPrecanceled() { var bb = new TransformManyBlock<int, int>(DataflowTestHelpers.ToEnumerable, new ExecutionDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true) }); int ignoredValue; IList<int> ignoredValues; IDisposable link = bb.LinkTo(DataflowBlock.NullTarget<int>()); Assert.NotNull(link); link.Dispose(); Assert.False(bb.Post(42)); var t = bb.SendAsync(42); Assert.True(t.IsCompleted); Assert.False(t.Result); Assert.False(bb.TryReceiveAll(out ignoredValues)); Assert.False(bb.TryReceive(out ignoredValue)); Assert.NotNull(bb.Completion); await Assert.ThrowsAnyAsync<OperationCanceledException>(() => bb.Completion); bb.Complete(); // just make sure it doesn't throw }
public async Task CalculateAsync(DateTimeRange period, string salesAreaName) { var hasErrors = false; _logger.LogInformation( $"Break availability calculation for '{salesAreaName}' sales area " + $"on {LogAsString.Log(period.Start.Date)} has been started."); try { List <IProgrammeForBreakAvailCalculation> programmes = await GetProgrammesAsync(period, salesAreaName) .ConfigureAwait(false); if (programmes.Count == 0) { _logger.LogWarning( $"No programmes found for sales area {salesAreaName} " + $"on {LogAsString.Log(period.Start.Date)}"); return; } DateTimeRange periodCoveringWholeProgrammes = PeriodIncludingAnyProgrammeSpanningMidnight(period, programmes); var spotsQueryTask = GetSpots(periodCoveringWholeProgrammes, salesAreaName); var breaksQueryTask = GetBreaks(periodCoveringWholeProgrammes, salesAreaName); await Task .WhenAll(spotsQueryTask, breaksQueryTask) .ConfigureAwait(false); List <ISpotForBreakAvailCalculation> spots = spotsQueryTask.Result; if (spots.Count > 0) { _logger.LogInformation( $"Found {LogAsString.Log(spots.Count)} spots " + $"for sales area {salesAreaName} " + $"on {LogAsString.Log(period.Start.Date)}" ); } else { _logger.LogWarning( $"No spots found for sales area {salesAreaName} " + $"on {LogAsString.Log(period.Start.Date)}"); } _logger.LogInformation( $"Found {LogAsString.Log(programmes.Count)} programmes " + $"for sales area {salesAreaName} on {LogAsString.Log(period.Start.Date)}. " + $"(Programme Ids: {programmes.ReducePropertyToCsv(p => p.ProgrammeId)})" ); List <BreakAvailability> breaks = breaksQueryTask.Result; if (breaks.Count > 0) { string breakExternalRefs = breaks.ReducePropertyToCsv(x => x.ExternalBreakRef); _logger.LogInformation( $"Found {LogAsString.Log(breaks.Count)} break(s) for sales area {salesAreaName} " + $"on {LogAsString.Log(period.Start.Date)}. " + $"[Break Ext. Refs: {breakExternalRefs}]" ); } else { _logger.LogWarning( $"No breaks found for sales area {salesAreaName} " + $"on {LogAsString.Log(period.Start.Date)}"); } var updateBreakHandler = new BreakAvailabilityUpdateHandler(); var calculator = new BreakAndOptimiserAvailabilityCalculator <BreakAvailability>(_logger, updateBreakHandler); calculator.Calculate(salesAreaName, programmes, breaks, spots); if (updateBreakHandler.UpdatedBreaks.Count > 0) { _ = await _internalBlock .SendAsync(updateBreakHandler.UpdatedBreaks.Values.ToArray(), _cancellationToken) .ConfigureAwait(false); } } catch (OperationCanceledException) { _logger.LogInformation( $"Break availability calculation for '{salesAreaName}' sales area " + $"on {LogAsString.Log(period.Start.Date)} has been cancelled."); hasErrors = true; } catch (Exception ex) { _logger.LogError($"Break availability calculation for '{salesAreaName}' sales area " + $"on {LogAsString.Log(period.Start.Date)} has been finished with errors."); hasErrors = true; throw new BreakAvailabilityCalculatorException(period, salesAreaName, ex); } finally { if (!hasErrors) { _logger.LogInformation( $"Break availability calculation for '{salesAreaName}' sales area " + $"on {LogAsString.Log(period.Start.Date)} has been finished successfully."); } }
public async Task TestArrayListReusePossibleForDop1() { foreach (int boundedCapacity in new[] { DataflowBlockOptions.Unbounded, 2 }) { foreach (bool sync in DataflowTestHelpers.BooleanValues) { foreach (int dop in new[] { 1, Environment.ProcessorCount }) { var dbo = new ExecutionDataflowBlockOptions { BoundedCapacity = boundedCapacity, MaxDegreeOfParallelism = dop }; foreach (IList <int> list in new IList <int>[] { new int[1], new List <int> { 0 }, new Collection <int> { 0 } }) { int nextExpectedValue = 1; TransformManyBlock <int, int> transform = null; Func <int, IEnumerable <int> > body = i => { if (i == 100) // we're done iterating { transform.Complete(); return((IEnumerable <int>)null); } else if (dop == 1) { list[0] = i + 1; // reuse the list over and over, but only at dop == 1 return((IEnumerable <int>)list); } else if (list is int[]) { return(new int[1] { i + 1 }); } else if (list is List <int> ) { return(new List <int>() { i + 1 }); } else { return(new Collection <int>() { i + 1 }); } }; transform = sync ? new TransformManyBlock <int, int>(body, dbo) : new TransformManyBlock <int, int>(i => Task.Run(() => body(i)), dbo); TransformBlock <int, int> verifier = new TransformBlock <int, int>(i => { Assert.Equal(expected: nextExpectedValue, actual: i); nextExpectedValue++; return(i); }); transform.LinkTo(verifier); verifier.LinkTo(transform); await transform.SendAsync(0); await transform.Completion; } } } } }
public async Task HandleAsync(CommandContext context, NextDelegate next) { if (context.Command is BulkUpdateContents bulkUpdates) { if (bulkUpdates.Jobs?.Length > 0) { var executionOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount / 2) }; var createCommandsBlock = new TransformManyBlock <BulkTask, BulkTaskCommand>(async task => { return(await CreateCommandsAsync(task)); }, executionOptions); var executeCommandBlock = new ActionBlock <BulkTaskCommand>(async command => { await ExecuteCommandAsync(command); }, executionOptions); createCommandsBlock.LinkTo(executeCommandBlock, new DataflowLinkOptions { PropagateCompletion = true }); var requestContext = contextProvider.Context.WithoutContentEnrichment().WithUnpublished(true); var requestedSchema = bulkUpdates.SchemaId.Name; var results = new ConcurrentBag <BulkUpdateResultItem>(); for (var i = 0; i < bulkUpdates.Jobs.Length; i++) { var task = new BulkTask( context.CommandBus, requestContext, requestedSchema, i, bulkUpdates.Jobs[i], bulkUpdates, results); await createCommandsBlock.SendAsync(task); } createCommandsBlock.Complete(); await executeCommandBlock.Completion; context.Complete(new BulkUpdateResult(results)); } else { context.Complete(new BulkUpdateResult()); } } else { await next(context); } }
public async Task HandleAsync(CommandContext context, NextDelegate next) { if (context.Command is BulkUpdateContents bulkUpdates) { if (bulkUpdates.Jobs?.Length > 0) { var executionOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount / 2) }; var createCommandsBlock = new TransformManyBlock <BulkTask, BulkTaskCommand>(async task => { try { return(await CreateCommandsAsync(task)); } catch (OperationCanceledException ex) { // Dataflow swallows operation cancelled exception. throw new AggregateException(ex); } }, executionOptions); var executeCommandBlock = new ActionBlock <BulkTaskCommand>(async command => { try { await ExecuteCommandAsync(command); } catch (OperationCanceledException ex) { // Dataflow swallows operation cancelled exception. throw new AggregateException(ex); } }, executionOptions); createCommandsBlock.BidirectionalLinkTo(executeCommandBlock); contextProvider.Context.Change(b => b .WithoutContentEnrichment() .WithoutCleanup() .WithUnpublished(true) .WithoutTotal()); var requestedSchema = bulkUpdates.SchemaId.Name; var results = new ConcurrentBag <BulkUpdateResultItem>(); for (var i = 0; i < bulkUpdates.Jobs.Length; i++) { var task = new BulkTask( context.CommandBus, requestedSchema, i, bulkUpdates.Jobs[i], bulkUpdates, results); if (!await createCommandsBlock.SendAsync(task)) { break; } } createCommandsBlock.Complete(); await executeCommandBlock.Completion; context.Complete(new BulkUpdateResult(results)); } else { context.Complete(new BulkUpdateResult()); } } else { await next(context); } }
public Task StartAsync() { return(m_Flattener.SendAsync(m_Source)); }