internal static void Run()
{
const string w1 = "Hello";
const string w2 = "World";
// Create a TransformManyBlock<string, char> object that splits
// a string into its individual characters.
var transformManyBlock = new TransformManyBlock <string, char>(s => s.ToCharArray());
// Chained buffer block
var buffer = new BufferBlock <char>();
transformManyBlock.LinkTo(buffer);
transformManyBlock.Completion.ContinueWith(_ => buffer.Complete());
// Post two messages to the first block.
transformManyBlock.SendAsync(w1);
transformManyBlock.SendAsync(w2);
transformManyBlock.Complete();
transformManyBlock.Completion.Wait();
// Receive all output values from the last block.
IList <char> chars;
while (!buffer.TryReceiveAll(out chars))
{
// Needed if we don't wait for transformManyBlock.Completion
Console.Write(".");
Thread.SpinWait(10000);
}
Console.WriteLine(chars == null ? "null" : string.Join(", ", chars));
}
protected override async Task ExecuteNodeTreeAsync(ExecutionContext context, ObjectExecutionNode rootNode)
{
//Options
var blockOptions = new ExecutionDataflowBlockOptions
{
CancellationToken = context.CancellationToken,
MaxDegreeOfParallelism = 1024
};
//Execute Block
var block = new TransformManyBlock <Job, Job>(
async job =>
{
var node = await ExecuteNodeAsync(job.Context, job.Node).ConfigureAwait(false);
var childJobs = (job.Node as IParentExecutionNode)?
.GetChildNodes()
.Select(child => job.CreateChildJob(job.Context, child))
.ToArray();
job.Complete();
return(childJobs);
},
blockOptions);
//Link to self
block.LinkTo(block);
//Start
var rootJob = new Job(context, rootNode);
await block.SendAsync(rootJob);
//Wait until done
await rootJob.Completion;
}
public override async Task ProcessAsync()
{
var settings = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 10,
};
var listFilesBlock = new TransformManyBlock <string, string>(_fileSystem.GetFileNames, settings);
var getVehiclesBlock = new TransformManyBlock <string, IVehicle>(GetVehicles, settings);
var transformBlock = new TransformBlock <IVehicle, Truck>(TransformAsync, settings);
var doubleBlock = new TransformBlock <Truck, Truck>(DoubleDoorsAsync, settings);
var batchBlock = new BatchBlock <Truck>(10);
var saveBlock = new ActionBlock <IEnumerable <Truck> >(SaveTrucksAsync, settings);
DataflowLinkOptions linkOptions = new DataflowLinkOptions()
{
PropagateCompletion = true
};
listFilesBlock.LinkTo(getVehiclesBlock, linkOptions);
getVehiclesBlock.LinkTo(transformBlock, linkOptions);
transformBlock.LinkTo(doubleBlock, linkOptions);
doubleBlock.LinkTo(batchBlock, linkOptions);
batchBlock.LinkTo(saveBlock, linkOptions);
await listFilesBlock.SendAsync(_directory);
listFilesBlock.Complete();
await saveBlock.Completion;
}
public async Task TestPrecanceled()
{
var bb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable,
new ExecutionDataflowBlockOptions {
CancellationToken = new CancellationToken(canceled: true)
});
int ignoredValue;
IList <int> ignoredValues;
IDisposable link = bb.LinkTo(DataflowBlock.NullTarget <int>());
Assert.NotNull(link);
link.Dispose();
Assert.False(bb.Post(42));
var t = bb.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
Assert.False(bb.TryReceiveAll(out ignoredValues));
Assert.False(bb.TryReceive(out ignoredValue));
Assert.NotNull(bb.Completion);
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => bb.Completion);
bb.Complete(); // just make sure it doesn't throw
}
public async Task TestProducerConsumer()
{
foreach (TaskScheduler scheduler in new[] { TaskScheduler.Default, new ConcurrentExclusiveSchedulerPair().ConcurrentScheduler })
{
foreach (int maxMessagesPerTask in new[] { DataflowBlockOptions.Unbounded, 1, 2 })
{
foreach (int boundedCapacity in new[] { DataflowBlockOptions.Unbounded, 1, 2 })
{
foreach (int dop in new[] { 1, 2 })
{
foreach (int elementsPerItem in new[] { 1, 3, 5 })
{
foreach (bool sync in DataflowTestHelpers.BooleanValues)
{
const int Messages = 50;
var options = new ExecutionDataflowBlockOptions
{
BoundedCapacity = boundedCapacity,
MaxDegreeOfParallelism = dop,
MaxMessagesPerTask = maxMessagesPerTask,
TaskScheduler = scheduler
};
TransformManyBlock <int, int> tb = sync ?
new TransformManyBlock <int, int>(i => Enumerable.Repeat(i, elementsPerItem), options) :
new TransformManyBlock <int, int>(i => TaskShim.Run(() => Enumerable.Repeat(i, elementsPerItem)), options);
await TaskShim.WhenAll(
TaskShim.Run(async delegate
{ // consumer
int i = 0;
int processed = 0;
while (await tb.OutputAvailableAsync())
{
Assert.Equal(expected: i, actual: await tb.ReceiveAsync());
processed++;
if (processed % elementsPerItem == 0)
{
i++;
}
}
}),
TaskShim.Run(async delegate
{ // producer
for (int i = 0; i < Messages; i++)
{
await tb.SendAsync(i);
}
tb.Complete();
}));
}
}
}
}
}
}
}
/// <summary>
/// Reflects over a set of assemblies and produces MEF parts for every applicable type.
/// </summary>
/// <param name="assemblyPaths">The paths to assemblies to search for MEF parts.</param>
/// <param name="progress">An optional way to receive progress updates on how discovery is progressing.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>A set of generated parts.</returns>
public async Task <DiscoveredParts> CreatePartsAsync(IEnumerable <string> assemblyPaths, IProgress <DiscoveryProgress> progress = null, CancellationToken cancellationToken = default(CancellationToken))
{
Requires.NotNull(assemblyPaths, nameof(assemblyPaths));
var exceptions = new List <PartDiscoveryException>();
var tuple = this.CreateAssemblyDiscoveryBlockChain(progress, cancellationToken);
var assemblyLoader = new TransformManyBlock <string, Assembly>(
path =>
{
try
{
#if NET45
return(new Assembly[] { Assembly.Load(AssemblyName.GetAssemblyName(path)) });
#elif NETCOREAPP1_0
return(new Assembly[] { System.Runtime.Loader.AssemblyLoadContext.Default.LoadFromAssemblyPath(path) });
#else
throw new NotSupportedException();
#endif
}
catch (Exception ex)
{
lock (exceptions)
{
exceptions.Add(new PartDiscoveryException(string.Format(CultureInfo.CurrentCulture, Strings.UnableToLoadAssembly, path), ex)
{
AssemblyPath = path
});
}
return(Enumerable.Empty <Assembly>());
}
},
new ExecutionDataflowBlockOptions
{
CancellationToken = cancellationToken,
MaxDegreeOfParallelism = Environment.ProcessorCount,
});
assemblyLoader.LinkTo(tuple.Item1, new DataflowLinkOptions {
PropagateCompletion = true
});
foreach (var assemblyPath in assemblyPaths)
{
await assemblyLoader.SendAsync(assemblyPath);
}
assemblyLoader.Complete();
var result = await tuple.Item2;
return(result.Merge(new DiscoveredParts(Enumerable.Empty <ComposablePartDefinition>(), exceptions)));
}
/// <summary>
/// Reflects over a set of assemblies and produces MEF parts for every applicable type.
/// </summary>
/// <param name="assemblyPaths">The paths to assemblies to search for MEF parts.</param>
/// <param name="progress">An optional way to receive progress updates on how discovery is progressing.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>A set of generated parts.</returns>
public async Task <DiscoveredParts> CreatePartsAsync(IEnumerable <string> assemblyPaths, IProgress <DiscoveryProgress>?progress = null, CancellationToken cancellationToken = default(CancellationToken))
{
Requires.NotNull(assemblyPaths, nameof(assemblyPaths));
var exceptions = new List <PartDiscoveryException>();
var tuple = this.CreateAssemblyDiscoveryBlockChain(progress, cancellationToken);
var assemblyLoader = new TransformManyBlock <string, Assembly>(
path =>
{
try
{
return(new Assembly[] { this.Resolver.AssemblyLoader.LoadAssembly(Utilities.GetAssemblyNameWithCodebasePath(path)) });
}
catch (Exception ex)
{
lock (exceptions)
{
exceptions.Add(new PartDiscoveryException(string.Format(CultureInfo.CurrentCulture, Strings.UnableToLoadAssembly, path), ex)
{
AssemblyPath = path
});
}
return(Enumerable.Empty <Assembly>());
}
},
new ExecutionDataflowBlockOptions
{
CancellationToken = cancellationToken,
MaxDegreeOfParallelism = Environment.ProcessorCount,
});
assemblyLoader.LinkTo(tuple.Item1, new DataflowLinkOptions {
PropagateCompletion = true
});
foreach (var assemblyPath in assemblyPaths)
{
await assemblyLoader.SendAsync(assemblyPath).ConfigureAwait(false);
}
assemblyLoader.Complete();
var result = await tuple.Item2.ConfigureAwait(false);
return(result.Merge(new DiscoveredParts(Enumerable.Empty <ComposablePartDefinition>(), exceptions)));
}
private async Task <Task> ProduceParticipantActionsAsync(
Func <Task <IEnumerable <Task <int[]> > > > getParticipants,
Func <int[], Task <Action[]> > getParticipantActions,
ITargetBlock <Action[]> actionQueue, int maxDegreeOfParallelism
)
{
var participantsQueryBlock = new TransformManyBlock <Task <int[]>, int>(async task => await task, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = -1
});
var participantsBuffer = new BatchBlock <int>(_batchOptions.Value.BatchSize);
var getParticipantActionsBlock = new ActionBlock <int[]>(async participantIds =>
{
var actions = await getParticipantActions(participantIds);
bool sendStatus = await actionQueue.SendAsync(actions);
if (!sendStatus)
{
throw new InvalidOperationException("Revisit bottleneck");
}
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxDegreeOfParallelism
});
participantsQueryBlock.LinkTo(participantsBuffer, PropagateCompletion);
participantsBuffer.LinkTo(getParticipantActionsBlock, PropagateCompletion);
foreach (var participantTask in await getParticipants())
{
bool sendStatus = await participantsQueryBlock.SendAsync(participantTask);
if (!sendStatus)
{
throw new InvalidOperationException("Revisit bottleneck");
}
}
participantsQueryBlock.Complete();
return(getParticipantActionsBlock.Completion);
}
public async Task <bool> SendDiscoveryJobAsync(IPeer peer)
{
return(await _discoverNodesDataflow.SendAsync(peer));
}
public async Task Run()
{
var current = 0;
var pending = 0;
var repositoryCount = 0;
var fetchBlock = new TransformManyBlock <IGitService, Repository>(
async service =>
{
var friendlyServiceName = service.Provider.ToString();
logger.LogInformation($"Fetching repositories from {friendlyServiceName} ...");
var providerSettings = settings.Git.Where(x => x.Type == service.Provider);
var repositories = await service.GetRepositoryUrisAsync(providerSettings);
if (providerSettings.Any() && repositories.Count == 0)
{
throw new InvalidOperationException($"{friendlyServiceName} API returned 0 repositories. Please check your credentials.");
}
Interlocked.Add(ref repositoryCount, repositories.Count);
logger.LogInformation($"Fetched {repositories.Count} repositories from {friendlyServiceName}. ({repositoryCount} total repositories to backup) ");
return(repositories);
}, new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 1000, MaxDegreeOfParallelism = settings.Backup.MaxDegreeOfParallelism
});
var mirrorBlock = new ActionBlock <Repository>(
async repository =>
{
Interlocked.Increment(ref current);
Interlocked.Increment(ref pending);
logger.LogInformation($"Cloning {current} of {repositoryCount} ({pending} running) ({repository.HttpsUrl})");
await backupRunner.Backup(repository);
logger.LogInformation($"Finished {repository.HttpsUrl} ({pending} pending)");
Interlocked.Decrement(ref pending);
}, new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 1000, MaxDegreeOfParallelism = settings.Backup.MaxDegreeOfParallelism,
});
fetchBlock.LinkTo(mirrorBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
foreach (var service in gitServices)
{
await fetchBlock.SendAsync(service);
}
fetchBlock.Complete();
await mirrorBlock.Completion;
await backupRunner.Cleanup();
await heartbeat.Notify();
logger.LogInformation("Done.");
}
public void RunTransformManyBlockConformanceTests()
{
bool passed = true;
#region Sync
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed& feedMethod < FeedMethod._Count; feedMethod++)
{
Func <DataflowBlockOptions, TargetProperties <int> > transformManyBlockFactory =
options =>
{
TransformManyBlock <int, int> transformManyBlock = new TransformManyBlock <int, int>(i => new[] { i }, (ExecutionDataflowBlockOptions)options);
ActionBlock <int> actionBlock = new ActionBlock <int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformManyBlock.LinkTo(actionBlock);
return(new TargetProperties <int> {
Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false
});
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
};
var mptOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10
};
var cancellationOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token
};
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++)
{
localPassed &= network.Post(i) == true;
}
for (int i = 0; i < ITERS; i++)
{
localPassed &= ((IReceivableSourceBlock <int>)network).Receive() == i * 16;
}
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => new[] { i * 2 }));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++)
{
tasks[i - 1] = network.SendAsync(i);
}
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++)
{
total += ((IReceivableSourceBlock <int>)network).Receive();
}
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test multiple yielded results
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(i =>
{
return(Enumerable.Range(0, 10));
});
t.Post(42);
t.Complete();
for (int i = 0; i < 10; i++)
{
localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(i =>
{
if ((i % 2) == 0)
{
throw new OperationCanceledException();
}
return(new[] { i });
});
for (int i = 0; i < 10; i++)
{
t.Post(i);
}
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0)
{
localPassed &= t.Receive() == i;
}
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test using a precanceled token
{
bool localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new ExecutionDataflowBlockOptions {
CancellationToken = cts.Token
};
var t = new TransformManyBlock <int, int>(i => new[] { i }, dbo);
int ignoredValue;
IList <int> ignoredValues;
localPassed &= t.LinkTo(new ActionBlock <int>(delegate { })) != null;
localPassed &= t.SendAsync(42).Result == false;
localPassed &= t.TryReceiveAll(out ignoredValues) == false;
localPassed &= t.Post(42) == false;
localPassed &= t.OutputCount == 0;
localPassed &= t.TryReceive(out ignoredValue) == false;
localPassed &= t.Completion != null;
t.Complete();
}
catch (Exception)
{
localPassed = false;
}
Console.WriteLine(" > {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(new Func <int, IEnumerable <int> >(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test reuse of a list and array
{
bool localPassed = true;
foreach (bool bounded in new[] { false, true })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop, BoundedCapacity = 2
} :
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop
};
foreach (IList <int> list in new IList <int>[]
{
new int[1],
new List <int>()
{
0
},
new Collection <int>()
{
0
}
})
{
int nextExpectedValue = 1;
TransformManyBlock <int, int> tmb1 = null;
tmb1 = new TransformManyBlock <int, int>(i =>
{
if (i == 1000)
{
tmb1.Complete();
return((IEnumerable <int>)null);
}
else if (dop == 1)
{
list[0] = i + 1;
return((IEnumerable <int>)list);
}
else if (list is int[])
{
return(new int[1] {
i + 1
});
}
else if (list is List <int> )
{
return(new List <int>()
{
i + 1
});
}
else
{
return new Collection <int>()
{
i + 1
}
};
}, dbo);
TransformBlock <int, int> tmb2 = new TransformBlock <int, int>(i =>
{
if (i != nextExpectedValue)
{
localPassed = false;
tmb1.Complete();
}
nextExpectedValue++;
return(i);
});
tmb1.LinkTo(tmb2);
tmb2.LinkTo(tmb1);
tmb1.SendAsync(0).Wait();
tmb1.Completion.Wait();
}
}
}
Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test throwing an OCE
{
bool localPassed = true;
foreach (bool bounded in new[] { true, false })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop, BoundedCapacity = 2
} :
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop
};
foreach (int mode in new[] { 0, 1, 2 })
{
const int ITERS = 50;
var mres = new ManualResetEventSlim();
var tmb = new TransformManyBlock <int, int>(i =>
{
if (i < ITERS - 1)
{
throw new OperationCanceledException();
}
if (mode == 0)
{
return new int[] { i }
}
;
else if (mode == 1)
{
return new List <int>()
{
i
}
}
;
else
{
return(Enumerable.Repeat(i, 1));
}
}, dbo);
var ab = new ActionBlock <int>(i =>
{
if (i != ITERS - 1)
{
localPassed = false;
}
mres.Set();
});
tmb.LinkTo(ab);
for (int i = 0; i < ITERS; i++)
{
tmb.SendAsync(i).Wait();
}
mres.Wait();
}
}
}
Console.WriteLine("{0}: Canceled invocation", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
#region Async
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed& feedMethod < FeedMethod._Count; feedMethod++)
{
Func <DataflowBlockOptions, TargetProperties <int> > transformManyBlockFactory =
options =>
{
TransformManyBlock <int, int> transformManyBlock = new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i }), (ExecutionDataflowBlockOptions)options);
ActionBlock <int> actionBlock = new ActionBlock <int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformManyBlock.LinkTo(actionBlock);
return(new TargetProperties <int> {
Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false
});
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount
};
var mptOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10
};
var cancellationOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token
};
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 })));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 })));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock <int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 })));
for (int i = 0; i < ITERS; i++)
{
localPassed &= network.Post(i) == true;
}
for (int i = 0; i < ITERS; i++)
{
localPassed &= ((IReceivableSourceBlock <int>)network).Receive() == i * 16;
}
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain <TransformManyBlock <int, int>, int>(4, () => new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>) new[] { i * 2 })));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++)
{
tasks[i - 1] = network.SendAsync(i);
}
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++)
{
total += ((IReceivableSourceBlock <int>)network).Receive();
}
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test multiple yielded results
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(i => Task.Run(() => (IEnumerable <int>)Enumerable.Range(0, 10).ToArray()));
t.Post(42);
t.Complete();
for (int i = 0; i < 10; i++)
{
localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(i =>
{
if ((i % 2) == 0)
{
throw new OperationCanceledException();
}
return(new[] { i });
});
for (int i = 0; i < 10; i++)
{
t.Post(i);
}
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0)
{
localPassed &= t.Receive() == i;
}
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(i =>
{
if ((i % 2) == 0)
{
return(null);
}
return(Task.Run(() => (IEnumerable <int>) new[] { i }));
});
for (int i = 0; i < 10; i++)
{
t.Post(i);
}
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0)
{
localPassed &= t.Receive() == i;
}
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored when a reordering buffer is in place
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(new Func <int, Task <IEnumerable <int> > >(i =>
{
if (i == 0)
{
Task.Delay(1000).Wait();
return(null);
}
return(Task.Run(() => (IEnumerable <int>) new[] { i }));
}), new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 2
});
t.Post(0);
t.Post(1);
try
{
localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1;
}
catch
{
localPassed = false;
}
Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the delegate
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(new Func <int, Task <IEnumerable <int> > >(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the task
{
bool localPassed = true;
var t = new TransformManyBlock <int, int>(new Func <int, Task <IEnumerable <int> > >(i => Task.Run <IEnumerable <int> >(new Func <IEnumerable <int> >(() => { throw new InvalidOperationException(); }))));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test reuse of a list and array
{
bool localPassed = true;
foreach (bool bounded in new[] { false, true })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop, BoundedCapacity = 2
} :
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop
};
foreach (IList <int> list in new IList <int>[]
{
new int[1],
new List <int>()
{
0
},
new Collection <int>()
{
0
}
})
{
int nextExpectedValue = 1;
TransformManyBlock <int, int> tmb1 = null;
tmb1 = new TransformManyBlock <int, int>(i =>
{
return(Task.Run(() =>
{
if (i == 1000)
{
tmb1.Complete();
return (IEnumerable <int>)null;
}
else if (dop == 1)
{
list[0] = i + 1;
return (IEnumerable <int>)list;
}
else if (list is int[])
{
return new int[1] {
i + 1
};
}
else if (list is List <int> )
{
return new List <int>()
{
i + 1
};
}
else
{
return new Collection <int>()
{
i + 1
}
};
}));
}, dbo);
TransformBlock <int, int> tmb2 = new TransformBlock <int, int>(i =>
{
if (i != nextExpectedValue)
{
localPassed = false;
tmb1.Complete();
}
nextExpectedValue++;
return(i);
});
tmb1.LinkTo(tmb2);
tmb2.LinkTo(tmb1);
tmb1.SendAsync(0).Wait();
tmb1.Completion.Wait();
}
}
}
Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test throwing an OCE
{
bool localPassed = true;
foreach (bool bounded in new[] { true, false })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop, BoundedCapacity = 2
} :
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop
};
foreach (int mode in new[] { 0, 1, 2 })
{
const int ITERS = 50;
var mres = new ManualResetEventSlim();
var tmb = new TransformManyBlock <int, int>(i =>
{
var cts = new CancellationTokenSource();
return(Task.Run(() =>
{
if (i < ITERS - 1)
{
cts.Cancel();
cts.Token.ThrowIfCancellationRequested();
}
if (mode == 0)
{
return new int[] { i }
}
;
else if (mode == 1)
{
return new List <int>()
{
i
}
}
;
else
{
return Enumerable.Repeat(i, 1);
}
}, cts.Token));
}, dbo);
var ab = new ActionBlock <int>(i =>
{
if (i != ITERS - 1)
{
localPassed = false;
}
mres.Set();
});
tmb.LinkTo(ab);
for (int i = 0; i < ITERS; i++)
{
tmb.SendAsync(i).Wait();
}
mres.Wait();
}
}
}
Console.WriteLine(" > {0}: Canceled invocation", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
Assert.True(passed, "Test failed.");
}
public void RunTransformManyBlockConformanceTests()
{
bool passed = true;
#region Sync
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory =
options =>
{
TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => new[] { i }, (ExecutionDataflowBlockOptions)options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformManyBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token };
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16;
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test multiple yielded results
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
return Enumerable.Range(0, 10);
});
t.Post(42);
t.Complete();
for (int i = 0; i < 10; i++)
{
localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return new[] { i };
});
for (int i = 0; i < 10; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test using a precanceled token
{
bool localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token };
var t = new TransformManyBlock<int, int>(i => new[] { i }, dbo);
int ignoredValue;
IList<int> ignoredValues;
localPassed &= t.LinkTo(new ActionBlock<int>(delegate { })) != null;
localPassed &= t.SendAsync(42).Result == false;
localPassed &= t.TryReceiveAll(out ignoredValues) == false;
localPassed &= t.Post(42) == false;
localPassed &= t.OutputCount == 0;
localPassed &= t.TryReceive(out ignoredValue) == false;
localPassed &= t.Completion != null;
t.Complete();
}
catch (Exception)
{
localPassed = false;
}
Console.WriteLine(" > {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, IEnumerable<int>>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test reuse of a list and array
{
bool localPassed = true;
foreach (bool bounded in new[] { false, true })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (IList<int> list in new IList<int>[]
{
new int[1],
new List<int>() { 0 },
new Collection<int>() { 0 }
})
{
int nextExpectedValue = 1;
TransformManyBlock<int, int> tmb1 = null;
tmb1 = new TransformManyBlock<int, int>(i =>
{
if (i == 1000)
{
tmb1.Complete();
return (IEnumerable<int>)null;
}
else if (dop == 1)
{
list[0] = i + 1;
return (IEnumerable<int>)list;
}
else if (list is int[])
{
return new int[1] { i + 1 };
}
else if (list is List<int>)
{
return new List<int>() { i + 1 };
}
else return new Collection<int>() { i + 1 };
}, dbo);
TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i =>
{
if (i != nextExpectedValue)
{
localPassed = false;
tmb1.Complete();
}
nextExpectedValue++;
return i;
});
tmb1.LinkTo(tmb2);
tmb2.LinkTo(tmb1);
tmb1.SendAsync(0).Wait();
tmb1.Completion.Wait();
}
}
}
Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test throwing an OCE
{
bool localPassed = true;
foreach (bool bounded in new[] { true, false })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (int mode in new[] { 0, 1, 2 })
{
const int ITERS = 50;
var mres = new ManualResetEventSlim();
var tmb = new TransformManyBlock<int, int>(i =>
{
if (i < ITERS - 1) throw new OperationCanceledException();
if (mode == 0) return new int[] { i };
else if (mode == 1) return new List<int>() { i };
else return Enumerable.Repeat(i, 1);
}, dbo);
var ab = new ActionBlock<int>(i =>
{
if (i != ITERS - 1) localPassed = false;
mres.Set();
});
tmb.LinkTo(ab);
for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait();
mres.Wait();
}
}
}
Console.WriteLine("{0}: Canceled invocation", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
#region Async
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory =
options =>
{
TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }), (ExecutionDataflowBlockOptions)options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformManyBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token };
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16;
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test multiple yielded results
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)Enumerable.Range(0, 10).ToArray()));
t.Post(42);
t.Complete();
for (int i = 0; i < 10; i++)
{
localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return new[] { i };
});
for (int i = 0; i < 10; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
if ((i % 2) == 0) return null;
return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i });
});
for (int i = 0; i < 10; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored when a reordering buffer is in place
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i =>
{
if (i == 0)
{
Task.Delay(1000).Wait();
return null;
}
return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i });
}), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 });
t.Post(0);
t.Post(1);
try
{
localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1;
}
catch
{
localPassed = false;
}
Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the delegate
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the task
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => Task<IEnumerable<int>>.Factory.StartNew(() => { throw new InvalidOperationException(); })));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test reuse of a list and array
{
bool localPassed = true;
foreach (bool bounded in new[] { false, true })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (IList<int> list in new IList<int>[]
{
new int[1],
new List<int>() { 0 },
new Collection<int>() { 0 }
})
{
int nextExpectedValue = 1;
TransformManyBlock<int, int> tmb1 = null;
tmb1 = new TransformManyBlock<int, int>(i =>
{
return Task.Factory.StartNew(() =>
{
if (i == 1000)
{
tmb1.Complete();
return (IEnumerable<int>)null;
}
else if (dop == 1)
{
list[0] = i + 1;
return (IEnumerable<int>)list;
}
else if (list is int[])
{
return new int[1] { i + 1 };
}
else if (list is List<int>)
{
return new List<int>() { i + 1 };
}
else return new Collection<int>() { i + 1 };
});
}, dbo);
TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i =>
{
if (i != nextExpectedValue)
{
localPassed = false;
tmb1.Complete();
}
nextExpectedValue++;
return i;
});
tmb1.LinkTo(tmb2);
tmb2.LinkTo(tmb1);
tmb1.SendAsync(0).Wait();
tmb1.Completion.Wait();
}
}
}
Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test throwing an OCE
{
bool localPassed = true;
foreach (bool bounded in new[] { true, false })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (int mode in new[] { 0, 1, 2 })
{
const int ITERS = 50;
var mres = new ManualResetEventSlim();
var tmb = new TransformManyBlock<int, int>(i =>
{
var cts = new CancellationTokenSource();
return Task.Factory.StartNew(() =>
{
if (i < ITERS - 1)
{
cts.Cancel();
cts.Token.ThrowIfCancellationRequested();
}
if (mode == 0) return new int[] { i };
else if (mode == 1) return new List<int>() { i };
else return Enumerable.Repeat(i, 1);
}, cts.Token);
}, dbo);
var ab = new ActionBlock<int>(i =>
{
if (i != ITERS - 1) localPassed = false;
mres.Set();
});
tmb.LinkTo(ab);
for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait();
mres.Wait();
}
}
}
Console.WriteLine(" > {0}: Canceled invocation", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
Assert.True(passed, "Test failed.");
}
public void Submit(SampleCollection samples)
{
_filterSampleCountBlock.SendAsync(samples).Wait();
}
public async Task TestPrecanceled()
{
var bb = new TransformManyBlock<int, int>(DataflowTestHelpers.ToEnumerable,
new ExecutionDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true) });
int ignoredValue;
IList<int> ignoredValues;
IDisposable link = bb.LinkTo(DataflowBlock.NullTarget<int>());
Assert.NotNull(link);
link.Dispose();
Assert.False(bb.Post(42));
var t = bb.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
Assert.False(bb.TryReceiveAll(out ignoredValues));
Assert.False(bb.TryReceive(out ignoredValue));
Assert.NotNull(bb.Completion);
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => bb.Completion);
bb.Complete(); // just make sure it doesn't throw
}
public async Task CalculateAsync(DateTimeRange period, string salesAreaName)
{
var hasErrors = false;
_logger.LogInformation(
$"Break availability calculation for '{salesAreaName}' sales area " +
$"on {LogAsString.Log(period.Start.Date)} has been started.");
try
{
List <IProgrammeForBreakAvailCalculation> programmes = await
GetProgrammesAsync(period, salesAreaName)
.ConfigureAwait(false);
if (programmes.Count == 0)
{
_logger.LogWarning(
$"No programmes found for sales area {salesAreaName} " +
$"on {LogAsString.Log(period.Start.Date)}");
return;
}
DateTimeRange periodCoveringWholeProgrammes =
PeriodIncludingAnyProgrammeSpanningMidnight(period, programmes);
var spotsQueryTask = GetSpots(periodCoveringWholeProgrammes, salesAreaName);
var breaksQueryTask = GetBreaks(periodCoveringWholeProgrammes, salesAreaName);
await Task
.WhenAll(spotsQueryTask, breaksQueryTask)
.ConfigureAwait(false);
List <ISpotForBreakAvailCalculation> spots = spotsQueryTask.Result;
if (spots.Count > 0)
{
_logger.LogInformation(
$"Found {LogAsString.Log(spots.Count)} spots "
+ $"for sales area {salesAreaName} " +
$"on {LogAsString.Log(period.Start.Date)}"
);
}
else
{
_logger.LogWarning(
$"No spots found for sales area {salesAreaName} " +
$"on {LogAsString.Log(period.Start.Date)}");
}
_logger.LogInformation(
$"Found {LogAsString.Log(programmes.Count)} programmes "
+ $"for sales area {salesAreaName} on {LogAsString.Log(period.Start.Date)}. "
+ $"(Programme Ids: {programmes.ReducePropertyToCsv(p => p.ProgrammeId)})"
);
List <BreakAvailability> breaks = breaksQueryTask.Result;
if (breaks.Count > 0)
{
string breakExternalRefs = breaks.ReducePropertyToCsv(x => x.ExternalBreakRef);
_logger.LogInformation(
$"Found {LogAsString.Log(breaks.Count)} break(s) for sales area {salesAreaName} "
+ $"on {LogAsString.Log(period.Start.Date)}. "
+ $"[Break Ext. Refs: {breakExternalRefs}]"
);
}
else
{
_logger.LogWarning(
$"No breaks found for sales area {salesAreaName} " +
$"on {LogAsString.Log(period.Start.Date)}");
}
var updateBreakHandler = new BreakAvailabilityUpdateHandler();
var calculator = new BreakAndOptimiserAvailabilityCalculator <BreakAvailability>(_logger, updateBreakHandler);
calculator.Calculate(salesAreaName, programmes, breaks, spots);
if (updateBreakHandler.UpdatedBreaks.Count > 0)
{
_ = await _internalBlock
.SendAsync(updateBreakHandler.UpdatedBreaks.Values.ToArray(), _cancellationToken)
.ConfigureAwait(false);
}
}
catch (OperationCanceledException)
{
_logger.LogInformation(
$"Break availability calculation for '{salesAreaName}' sales area " +
$"on {LogAsString.Log(period.Start.Date)} has been cancelled.");
hasErrors = true;
}
catch (Exception ex)
{
_logger.LogError($"Break availability calculation for '{salesAreaName}' sales area " +
$"on {LogAsString.Log(period.Start.Date)} has been finished with errors.");
hasErrors = true;
throw new BreakAvailabilityCalculatorException(period, salesAreaName, ex);
}
finally
{
if (!hasErrors)
{
_logger.LogInformation(
$"Break availability calculation for '{salesAreaName}' sales area " +
$"on {LogAsString.Log(period.Start.Date)} has been finished successfully.");
}
}
public async Task TestArrayListReusePossibleForDop1()
{
foreach (int boundedCapacity in new[] { DataflowBlockOptions.Unbounded, 2 })
{
foreach (bool sync in DataflowTestHelpers.BooleanValues)
{
foreach (int dop in new[] { 1, Environment.ProcessorCount })
{
var dbo = new ExecutionDataflowBlockOptions {
BoundedCapacity = boundedCapacity, MaxDegreeOfParallelism = dop
};
foreach (IList <int> list in new IList <int>[] { new int[1], new List <int> {
0
}, new Collection <int> {
0
} })
{
int nextExpectedValue = 1;
TransformManyBlock <int, int> transform = null;
Func <int, IEnumerable <int> > body = i => {
if (i == 100) // we're done iterating
{
transform.Complete();
return((IEnumerable <int>)null);
}
else if (dop == 1)
{
list[0] = i + 1; // reuse the list over and over, but only at dop == 1
return((IEnumerable <int>)list);
}
else if (list is int[])
{
return(new int[1] {
i + 1
});
}
else if (list is List <int> )
{
return(new List <int>()
{
i + 1
});
}
else
{
return(new Collection <int>()
{
i + 1
});
}
};
transform = sync ?
new TransformManyBlock <int, int>(body, dbo) :
new TransformManyBlock <int, int>(i => Task.Run(() => body(i)), dbo);
TransformBlock <int, int> verifier = new TransformBlock <int, int>(i => {
Assert.Equal(expected: nextExpectedValue, actual: i);
nextExpectedValue++;
return(i);
});
transform.LinkTo(verifier);
verifier.LinkTo(transform);
await transform.SendAsync(0);
await transform.Completion;
}
}
}
}
}
public async Task HandleAsync(CommandContext context, NextDelegate next)
{
if (context.Command is BulkUpdateContents bulkUpdates)
{
if (bulkUpdates.Jobs?.Length > 0)
{
var executionOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount / 2)
};
var createCommandsBlock = new TransformManyBlock <BulkTask, BulkTaskCommand>(async task =>
{
return(await CreateCommandsAsync(task));
}, executionOptions);
var executeCommandBlock = new ActionBlock <BulkTaskCommand>(async command =>
{
await ExecuteCommandAsync(command);
}, executionOptions);
createCommandsBlock.LinkTo(executeCommandBlock, new DataflowLinkOptions
{
PropagateCompletion = true
});
var requestContext = contextProvider.Context.WithoutContentEnrichment().WithUnpublished(true);
var requestedSchema = bulkUpdates.SchemaId.Name;
var results = new ConcurrentBag <BulkUpdateResultItem>();
for (var i = 0; i < bulkUpdates.Jobs.Length; i++)
{
var task = new BulkTask(
context.CommandBus,
requestContext,
requestedSchema,
i,
bulkUpdates.Jobs[i],
bulkUpdates,
results);
await createCommandsBlock.SendAsync(task);
}
createCommandsBlock.Complete();
await executeCommandBlock.Completion;
context.Complete(new BulkUpdateResult(results));
}
else
{
context.Complete(new BulkUpdateResult());
}
}
else
{
await next(context);
}
}
public async Task HandleAsync(CommandContext context, NextDelegate next)
{
if (context.Command is BulkUpdateContents bulkUpdates)
{
if (bulkUpdates.Jobs?.Length > 0)
{
var executionOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount / 2)
};
var createCommandsBlock = new TransformManyBlock <BulkTask, BulkTaskCommand>(async task =>
{
try
{
return(await CreateCommandsAsync(task));
}
catch (OperationCanceledException ex)
{
// Dataflow swallows operation cancelled exception.
throw new AggregateException(ex);
}
}, executionOptions);
var executeCommandBlock = new ActionBlock <BulkTaskCommand>(async command =>
{
try
{
await ExecuteCommandAsync(command);
}
catch (OperationCanceledException ex)
{
// Dataflow swallows operation cancelled exception.
throw new AggregateException(ex);
}
}, executionOptions);
createCommandsBlock.BidirectionalLinkTo(executeCommandBlock);
contextProvider.Context.Change(b => b
.WithoutContentEnrichment()
.WithoutCleanup()
.WithUnpublished(true)
.WithoutTotal());
var requestedSchema = bulkUpdates.SchemaId.Name;
var results = new ConcurrentBag <BulkUpdateResultItem>();
for (var i = 0; i < bulkUpdates.Jobs.Length; i++)
{
var task = new BulkTask(
context.CommandBus,
requestedSchema,
i,
bulkUpdates.Jobs[i],
bulkUpdates,
results);
if (!await createCommandsBlock.SendAsync(task))
{
break;
}
}
createCommandsBlock.Complete();
await executeCommandBlock.Completion;
context.Complete(new BulkUpdateResult(results));
}
else
{
context.Complete(new BulkUpdateResult());
}
}
else
{
await next(context);
}
}
public Task StartAsync()
{
return(m_Flattener.SendAsync(m_Source));
}
