static void Main(string[] args)
{
string s =
"http://cn.bing.com/search?q=MD5CryptoServiceProvider+slow&qs=n&pq=md5cryptoserviceprovider+slow&sc=0-25&sp=-1&sk=&cvid=67d40cbd8c424d55a3db83e6e9868267&first=51&FORM=PERE4";
using (MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider())
{
byte[] inBytes = Encoding.UTF8.GetBytes(s);
var bytes = md5.ComputeHash(inBytes);
Console.WriteLine(bytes.Length);
}
var splitter = new TransformBlock<string, KeyValuePair<string, int>>(
input =>
{
var splitted = input.Split('=');
return new KeyValuePair<string, int>(splitted[0], int.Parse(splitted[1]));
});
var dict = new Dictionary<string, int>();
var aggregater = new ActionBlock<KeyValuePair<string, int>>(
pair =>
{
int oldValue;
dict[pair.Key] = dict.TryGetValue(pair.Key, out oldValue) ? oldValue + pair.Value : pair.Value;
});
splitter.LinkTo(aggregater, new DataflowLinkOptions() { PropagateCompletion = true});
splitter.Post("a=1");
splitter.Post("b=2");
splitter.Post("a=5");
splitter.Complete();
aggregater.Completion.Wait();
Console.WriteLine("sum(a) = {0}", dict["a"]); //prints sum(a) = 6
//CalcAsync().Wait();
//SlowFlowAsync().Wait();
//FailDemoAsync().Wait();
//TransformAndLinkDemo().Wait();
//LinkLeftToDemo().Wait();
//CircularFlowAutoComplete().Wait();
//RecorderDemo().Wait();
BulkInserterDemo().Wait();
//BulkInserterDemo2().Wait();
//BroadcasterDemo().Wait();
//MyLoggerDemo().Wait();
//ETLLookupDemo().Wait();
}
static void Main1(string[] args)
{
// Create a shared CancellationTokenSource object to enable the
// TrySolution method to be cancelled.
var cts = new CancellationTokenSource();
// Create three TransformBlock<int, int> objects.
// Each TransformBlock<int, int> object calls the TrySolution method.
Func <int, int> action = n => TrySolution(n, cts.Token);
var trySolution1 = new TransformBlock <int, int>(action);
var trySolution2 = new TransformBlock <int, int>(action);
var trySolution3 = new TransformBlock <int, int>(action);
// Post data to each TransformBlock<int, int> object.
trySolution1.Post(11);
trySolution2.Post(21);
trySolution3.Post(31);
// Call the ReceiveFromAny<T> method to receive the result from the
// first TransformBlock<int, int> object to finish.
int result = ReceiveFromAny(trySolution1, trySolution2, trySolution3);
// Cancel all calls to TrySolution that are still active.
cts.Cancel();
// Print the result to the console.
Console.WriteLine("The solution is {0}.", result);
cts.Dispose();
}
public Task <Package> CompilePackageAsync(
string name,
IEnumerable <ICodeFileSource> fileSources,
FixedDictionary <string, Task <Package> > references,
TaskScheduler taskScheduler)
{
var lexer = new Lexer();
var parser = new CompilationUnitParser();
var parseBlock = new TransformBlock <ICodeFileSource, CompilationUnitSyntax>(
async(fileSource) =>
{
var file = await fileSource.LoadAsync();
var context = new ParseContext(file, new Diagnostics());
var tokens = lexer.Lex(context).WhereNotTrivia();
return(parser.Parse(tokens));
}, new ExecutionDataflowBlockOptions()
{
TaskScheduler = taskScheduler,
EnsureOrdered = false,
});
foreach (var fileSource in fileSources)
{
parseBlock.Post(fileSource);
}
parseBlock.Complete();
throw new NotImplementedException();
}
public static async Task Linking()
{
var multiplyBlock = new TransformBlock <int, int>(item => item * 2);
var subtractBlock = new TransformBlock <int, int>(item =>
{
return(item - 2);
});
multiplyBlock.LinkTo(subtractBlock, new DataflowLinkOptions
{
PropagateCompletion = true
});
foreach (var item in Enumerable.Repeat(1, 10).ToArray())
{
multiplyBlock.Post(item);
}
multiplyBlock.Complete();
/*
* Microsoft documentation is kinda misleading about waiting for the final block to complete
* whether a block is completed or not depends on the type of the block
* TransformBlock needs 3 conditions
* 1/ TransformBlock.Complete() has been called
* 2/ InputCount == 0 the block has applied its transformation to every incoming element
* 3/ Outputcount == 0 all transformed elements have left the output buffer
*/
await subtractBlock.Completion;
System.Console.WriteLine("Done"); // this part is unreachable
}
public override void Run()
{
/* Unbounded 表示块使用支持的最大并行数
* 块并行处理时，处理的数据顺序不一定与接收的顺序一致，但是输出时与接收的顺序一致
*/
TransformBlock <RunModel, string> transformBlock = new TransformBlock <RunModel, string>(
(model) =>
{
Helper.PrintLine($"开始处理 {model.Name}");
return($"I'm {model.Name}");
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
var models = this.CreateCollection();
// TransformBlock 可以同时作为输入和输出
Array.ForEach(models, model => transformBlock.Post(model));
for (int index = 0; index < models.Length; index++)
{
Helper.PrintLine(transformBlock.Receive());
}
}
public Task GenerateTests(List <string> files, DegreeOfParallelism degree)
{
var readFile = new TransformBlock <string, string>(
new Func <string, Task <string> >(reader.ReadCodeFromFile), new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = degree.AmountOfReadParallelProcess
});
var generateTests = new TransformBlock <string, List <TestClassDetails> >(
new Func <string, List <TestClassDetails> >(GetDetailsFromSourceCode), new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = degree.AmountOfGenerateParallelProcess
});
var writeTestsToFile = new ActionBlock <List <TestClassDetails> >(
tests => writer.WriteTestsToFiles(tests), new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = degree.AmountOfWriteParallelProcess
});
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
readFile.LinkTo(generateTests, linkOptions);
generateTests.LinkTo(writeTestsToFile, linkOptions);
foreach (var file in files)
{
readFile.Post(file);
}
readFile.Complete();
return(writeTestsToFile.Completion);
}
static async Task Main(string[] args)
{
var producer1 = new TransformBlock <string, string>(async n =>
{
await Task.Delay(150);
return(n);
});
var producer2 = new TransformBlock <string, string>(n =>
{
Task.Delay(500).Wait();
return(n);
});
var printBlock = new ActionBlock <string>(n => Console.WriteLine(n));
producer1.LinkTo(printBlock);
producer2.LinkTo(printBlock);
for (int i = 0; i < 10; i++)
{
producer1.Post($"Producer 1 message: {i}");
producer2.Post($"Producer 2 message: {i}");
}
producer1.Complete();
producer2.Complete();
await Task.WhenAll(new[] { producer1.Completion, producer2.Completion })
.ContinueWith(_ => printBlock.Complete())
.ContinueWith(_ => printBlock.Completion);
Console.WriteLine("done");
Console.ReadLine();
}
public void PropagateCompletionAfterWaitTest()
{
var evt = new ManualResetEventSlim();
var source = new TransformBlock <int, int> (
i =>
{
evt.Wait();
return(i);
});
var target = new BufferBlock <int> ();
Assert.IsNotNull(source.LinkTo(target,
new DataflowLinkOptions {
PropagateCompletion = true
}));
Assert.IsTrue(source.Post(42));
Assert.IsFalse(target.Completion.Wait(100));
source.Complete();
Assert.IsFalse(target.Completion.Wait(100));
evt.Set();
Assert.IsFalse(target.Completion.Wait(100));
Assert.AreEqual(42, target.Receive());
Assert.IsTrue(target.Completion.Wait(1000));
}
private static void Main()
{
//var greedy = new ExecutionDataflowBlockOptions();
var nonGreedy = new ExecutionDataflowBlockOptions {
BoundedCapacity = 1
};
var options = nonGreedy;
var firstBlock = new ActionBlock <int>(i => Do(i, 1, 2), options);
var secondBlock = new ActionBlock <int>(i => Do(i, 2, 1), options);
var thirdBlock = new ActionBlock <int>(i => Do(i, 3, 2), options);
var transform = new TransformBlock <int, int>(i => i * 2);
transform.LinkTo(firstBlock);
transform.LinkTo(secondBlock);
transform.LinkTo(thirdBlock);
for (var i = 0; i <= 10; i++)
{
transform.Post(i);
}
Console.ReadLine();
}
private static void Main()
{
Tuple <string, bool> IsHappy(string url)
{
Console.WriteLine(Thread.CurrentThread.ManagedThreadId);
using (var client = new WebClient())
{
var content = client.DownloadString(url);
return(Tuple.Create(url, content.ToLower().Contains("happy")));
}
}
var happySites = new List <string>();
var isHappySiteBlock = new TransformBlock <string, Tuple <string, bool> >(
(Func <string, Tuple <string, bool> >)IsHappy,
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1
});
var addToHappySiteBlock = new ActionBlock <Tuple <string, bool> >(tuple => happySites.Add(tuple.Item1));
isHappySiteBlock.LinkTo(
addToHappySiteBlock, new DataflowLinkOptions {
PropagateCompletion = true
}, tuple => tuple.Item2);
isHappySiteBlock.LinkTo(
DataflowBlock.NullTarget <Tuple <string, bool> >());
Array.ForEach(_HappySites, url => isHappySiteBlock.Post(url));
isHappySiteBlock.Complete();
addToHappySiteBlock.Completion.Wait();
happySites.ForEach(Console.WriteLine);
}
public void TestBoundedCapacity()
{
var t1 = new TransformBlock <int, int>(x => x + 2);
var t2 = new TransformBlock <int, string>(x => (x * 2).ToString());
var b = new BufferBlock <string>(new DataflowBlockOptions {
BoundedCapacity = 1
});
var log = new TransformBlock <string, string>(x => { Console.WriteLine($"TransformBlock has {x}"); return(x); });
var test = new ActionBlock <string>(x => Assert.Equal("8", x));
var options = new DataflowLinkOptions {
PropagateCompletion = true
};
t1.LinkTo(t2, options);
t2.LinkTo(b, options);
b.LinkTo(log, options);
log.LinkTo(test, options);
t1.Post(2);
t1.Complete();
test.Completion.Wait();
}
internal static void Run()
{
var tempFile = CreateRandomFile();
// Create an ActionBlock<int> object that prints to the console
// the number of bytes read.
var printResult = new ActionBlock <int>(zeroBytesRead =>
{
Console.WriteLine("{0} contains {1} zero bytes.", Path.GetFileName(tempFile), zeroBytesRead);
});
// Create a TransformBlock<string, int> object that calls the
// CountBytes function and returns its result.
var countBytes = new TransformBlock <string, int>(new Func <string, Task <int> >(CountBytes));
// Link the TransformBlock<string, int> object to the ActionBlock<int> object.
countBytes.LinkTo(printResult);
countBytes.Completion.ContinueWith(delegate { printResult.Complete(); });
// Post the path to the temporary file to the
// TransformBlock<string, int> object.
countBytes.Post(tempFile);
// Requests completion of the TransformBlock<string, int> object.
countBytes.Complete();
// Wait for the ActionBlock<int> object to print the message.
printResult.Completion.Wait();
// Delete the temporary file.
File.Delete(tempFile);
}
public void Run()
{
ConcurrentBag <int> values = new ConcurrentBag <int>();
var transformBlock = new TransformBlock <int, string>(n =>
{
Task.Delay(500).Wait();
values.Add(n);
return(n.ToString());
}, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 4
});
for (int i = 0; i < 10; i++)
{
transformBlock.Post(i);
Console.WriteLine("Input count:" + transformBlock.InputCount);
}
for (int i = 0; i < 10; i++)
{
Console.WriteLine("output count:" + transformBlock.OutputCount);
var result = transformBlock.Receive();
var listResult = 0;
values.TryTake(out listResult);
Console.WriteLine($"Result:{result} Output count:{transformBlock.OutputCount} input count:{transformBlock.InputCount} list item: {listResult}");
}
}
static async void Example4()
{
try
{
var multiplyBlock = new TransformBlock <int, int>(item =>
{
if (item == 1)
{
throw new InvalidOperationException("Blech.");
}
return(item * 2);
});
var subtracktBlock = new TransformBlock <int, int>(item => item - 2);
multiplyBlock.LinkTo(subtracktBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
multiplyBlock.Post(1);
await subtracktBlock.Completion;
}
catch (AggregateException ae)
{
//这里捕获异常。
ae.Flatten();
}
}
public static async Task DataFlowIntro()
{
try
{
var multiplyBlock = new TransformBlock <int, int>(item =>
{
if (item == 1)
{
throw new InvalidOperationException("Blech.");
}
return(item * 2);
});
var subtractBlock = new TransformBlock <int, int>(item => item - 2);
multiplyBlock.LinkTo(subtractBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
multiplyBlock.Post(10);
await subtractBlock.Completion;
}
catch (Exception e)
{
Console.WriteLine(e);
throw;
}
}
public async Task TestPrecanceled()
{
var bb = new TransformBlock <int, int>(i => i,
new ExecutionDataflowBlockOptions {
CancellationToken = new CancellationToken(canceled: true)
});
int ignoredValue;
IList <int> ignoredValues;
IDisposable link = bb.LinkTo(DataflowBlock.NullTarget <int>());
Assert.NotNull(link);
link.Dispose();
Assert.False(bb.Post(42));
var t = bb.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
Assert.False(bb.TryReceiveAll(out ignoredValues));
Assert.False(bb.TryReceive(out ignoredValue));
Assert.NotNull(bb.Completion);
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => bb.Completion);
bb.Complete(); // just make sure it doesn't throw
}
/// <summary>
/// Generate tests classes.
/// </summary>
/// <param name="reader">File reader.</param>
/// <param name="writer">File writer.</param>
/// <param name="source">Paths to files.</param>
/// <returns></returns>
public async Task Generate(CodeReader reader, CodeWriter writer, List <string> source)
{
DataflowLinkOptions linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
ExecutionDataflowBlockOptions processingTaskRestriction = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = _testsGeneratorConfig.MaxProcessingTasksCount
};
ExecutionDataflowBlockOptions outputTaskRestriction = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = _testsGeneratorConfig.MaxWritingTasksCount
};
TransformBlock <string, string> readingBlock =
new TransformBlock <string, string>(new Func <string, Task <string> >(reader.ReadAsync), processingTaskRestriction);
TransformBlock <string, GeneratedTestClass> producingBlock =
new TransformBlock <string, GeneratedTestClass>(new Func <string, GeneratedTestClass>(Produce), processingTaskRestriction);
ActionBlock <GeneratedTestClass> writingBlock = new ActionBlock <GeneratedTestClass>(
((generatedClass) => writer.WriteAsync(generatedClass).Wait()), outputTaskRestriction);
readingBlock.LinkTo(producingBlock, linkOptions);
producingBlock.LinkTo(writingBlock, linkOptions);
foreach (string path in source)
{
readingBlock.Post(path);
}
readingBlock.Complete();
await writingBlock.Completion;
}
static void Main(string[] args)
{
var options = new GroupingDataflowBlockOptions {
Greedy = false
};
var sync = new BatchBlock <string>(3, options);
// the delegate returns value tuple (.MET 4.7 / standard 2.0)
var chA = new TransformBlock <int, string>(
(Func <int, string>)ChannelA);
var chB = new TransformBlock <int, string>(
(Func <int, string>)ChannelB);
var chC = new TransformBlock <int, string>(
(Func <int, Task <string> >)ChannelC);
var presenter = new ActionBlock <string[]>((Action <string[]>)Present);
chA.LinkTo(sync);
chB.LinkTo(sync);
chC.LinkTo(sync);
sync.LinkTo(presenter);
for (int i = 1; i <= 20; i++)
{
chA.Post(i);
chB.Post(i);
chC.Post(i);
}
Console.ReadKey();
}
public async Task Can_TransformBlock_Be_Observable_And_Linked()
{
var totalValuesRecievedByActionBlock = new List <int>();
var totalValuesRecievedInStream = new List <int>();
var streamCompletionSource = new TaskCompletionSource <bool>();
var transformBlock = new TransformBlock <int, int>(_ => _);
var actionBlock = new ActionBlock <int>(x => Task.Delay(1000).ContinueWith(_ => totalValuesRecievedByActionBlock.Add(x)), new ExecutionDataflowBlockOptions()
{
BoundedCapacity = 1
});
var subscription = transformBlock.AsObservable().Subscribe(x => totalValuesRecievedInStream.Add(x),
() => streamCompletionSource.SetResult(true));
var actionLink = transformBlock.LinkTo(actionBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
foreach (var item in Enumerable.Range(0, 100))
{
transformBlock.Post(item);
}
await Task.Delay(2000);
transformBlock.Complete();
await actionBlock.Completion;
await streamCompletionSource.Task;
Assert.AreEqual(totalValuesRecievedByActionBlock, totalValuesRecievedInStream);
}
/// <summary>
/// Running async and return result
/// @see: https://codereview.stackexchange.com/a/212326
/// </summary>
/// <param name="items">!IEnumerableName! (dot) ForEachAsync</param>
/// <param name="action">Input type</param>
/// <param name="maxDegreesOfParallelism">Default of 6 task running at the same time </param>
/// <typeparam name="TSource">output type</typeparam>
/// <typeparam name="TResult">and output it self</typeparam>
/// <returns>Task with output</returns>
public static async Task <IEnumerable <TResult> > ForEachAsync <TSource, TResult>(
this IEnumerable <TSource> items,
Func <TSource, Task <TResult> > action,
int maxDegreesOfParallelism = 6)
{
var transformBlock = new TransformBlock <TSource, TResult>(action, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = maxDegreesOfParallelism
});
var bufferBlock = new BufferBlock <TResult>();
using (transformBlock.LinkTo(bufferBlock, new DataflowLinkOptions {
PropagateCompletion = true
}))
{
foreach (var item in items)
{
transformBlock.Post(item);
}
transformBlock.Complete();
await transformBlock.Completion;
}
bufferBlock.TryReceiveAll(out var result);
return(result);
}
private static async Task SequentialForkAsync()
{
// the output of transform block is sequential
// (dictate by the order of the original input)
IPropagatorBlock <int, string> block =
new TransformBlock <int, string>(i => UnitOfWorkAsync(i),
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = ITERATIONS
});
// the action block will get the transformation in the original order
var ab = new ActionBlock <string>(m => Console.WriteLine($"# Action: {m}"));
block.LinkTo(ab, new DataflowLinkOptions {
PropagateCompletion = true
});
for (int i = 0; i < ITERATIONS; i++)
{
block.Post(i);
}
block.Complete();
await ab.Completion.ConfigureAwait(false);
Console.WriteLine("Done");
}
public static void Run()
{
Func <int, int> fn = n =>
{
Thread.Sleep(1000);
return(n * n);
};
var tfBlock = new TransformBlock <int, int>(fn);
for (int i = 0; i < 10; i++)
{
tfBlock.Post(i);
}
// Receive Async returns a Task
for (int i = 0; i < 10; i++)
{
Task <int> resultTask = tfBlock.ReceiveAsync();
int result = resultTask.Result;
// Calling Result will wait until it has a value ready
Console.WriteLine(result);
}
Console.WriteLine("Done");
//Instead the ReceiveAsync() method returns a Task<T>
//that represents the receive operation.Calling the Result() method on the returned Task forces the
//program to wait until data becomes available essentially making it a synchronous operation like the
//previous example with the same console output
//RESULT IS THE SAME
}
/// <summary>
/// Process the data.
/// </summary>
public static void ProcessData <T1, T2>
(
[NotNull] this IEnumerable <T1> data,
[NotNull] TransformBlock <T1, T2> transformBlock,
[NotNull] ActionBlock <T2> actionBlock
)
{
Code.NotNull(data, "data");
Code.NotNull(transformBlock, "transformBlock");
Code.NotNull(actionBlock, "actionBlock");
DataflowLinkOptions options = new DataflowLinkOptions
{
PropagateCompletion = true
};
transformBlock.LinkTo(actionBlock, options);
foreach (T1 item in data)
{
transformBlock.Post(item);
}
transformBlock.Complete();
actionBlock.Completion.Wait();
}
public override Task GenerateTests(IEnumerable <string> classFiles, string pathToSave, int maxFileToRead, int maxThreads, int maxFileToWrite)
{
var maxFilesToLoadTasks = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxFileToRead
};
var maxTasksExecutedTasks = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxThreads
};
var maxFilesToWriteTasks = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxFileToWrite
};
var getText = new TransformBlock <string, string>(GetTextFromFileAsync, maxFilesToLoadTasks);
var getTests = new TransformBlock <string, string[]>(GetTestsFromText, maxTasksExecutedTasks);
_pathToSave = pathToSave;
var saveTests = new ActionBlock <string[]>(SaveTests, maxFilesToWriteTasks);
//to propagate message after IDataflowBlock.Complete()
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
getText.LinkTo(getTests, linkOptions);
getTests.LinkTo(saveTests, linkOptions);
foreach (var file in classFiles)
{
getText.Post(file);
}
getText.Complete();
return(saveTests.Completion);
}
public async Task TestCircularLinking()
{
const int Iters = 200;
foreach (bool sync in DataflowTestHelpers.BooleanValues)
{
var tcs = new TaskCompletionSource <bool>();
Func <int, int> body = i => {
if (i >= Iters)
{
tcs.SetResult(true);
}
return(i + 1);
};
TransformBlock <int, int> tb = sync ?
new TransformBlock <int, int>(body) :
new TransformBlock <int, int>(i => Task.Run(() => body(i)));
using (tb.LinkTo(tb))
{
tb.Post(0);
await tcs.Task;
tb.Complete();
}
}
}
static void Main(string[] args)
{
var b1 = new TransformBlock <int, string>(i => new string((char)('a' + i), i));
var b2 = new TransformManyBlock <string, char>(i => i);
var b3 = new ActionBlock <char>(async c =>
{
await Task.Delay(1000).ConfigureAwait(false);
Console.Write($"{c}, ");
});
var b4 = new ActionBlock <char>(async c =>
{
await Task.Delay(500).ConfigureAwait(false);
Trace.Write($"{c}, ");
});
var b5 = new BroadcastBlock <char>(i => i);
b1.LinkTo(b2);
b2.LinkTo(b5);
b5.LinkTo(b3);
b5.LinkTo(b4);
for (int i = 1; i <= 10; i++)
{
b1.Post(i);
}
Console.WriteLine($@"
b1: in {b1.InputCount} out {b1.OutputCount}
b2: in {b2.InputCount} out {b2.OutputCount}
b3: in {b3.InputCount}
b4: in {b4.InputCount}
");
Console.ReadKey();
}
public void TestDegreeOfParallelism()
{
var t1 = new TransformBlock <int, int>(x => x * 4);
var t2 = new TransformBlock <int, string>(
x => $"Hello number {x}",
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
}
);
var log = new TransformBlock <string, string>(x => { Console.WriteLine($"TransformBlock has {x}"); return(x); });
var test = new ActionBlock <string>(x => Assert.Equal("Hello number 8", x));
var options = new DataflowLinkOptions {
PropagateCompletion = true
};
t1.LinkTo(t2, options);
t2.LinkTo(log, options);
log.LinkTo(test, options);
t1.Post(2);
t1.Complete();
test.Completion.Wait();
}
public async Task Generate(List <string> inputFiles, string outputPath)
{
var linkOptions = new DataflowLinkOptions();
linkOptions.PropagateCompletion = true;
var readOptions = new ExecutionDataflowBlockOptions();
readOptions.MaxDegreeOfParallelism = config.CountOfReadThreads;
var processOptions = new ExecutionDataflowBlockOptions();
processOptions.MaxDegreeOfParallelism = config.CountOfProcessThreads;
var writeOptions = new ExecutionDataflowBlockOptions();
writeOptions.MaxDegreeOfParallelism = config.CountOfWriteThreads;
var readBlock = new TransformBlock <string, string>(fileName => AsyncReader.Read(fileName), readOptions);
var processBlock = new TransformBlock <string, List <TestInfo> >(sourceCode => GenerateTests(sourceCode), processOptions);
var writeBlock = new ActionBlock <List <TestInfo> >(output => AsyncWriter.Write(outputPath, output).Wait(), writeOptions);
readBlock.LinkTo(processBlock, linkOptions);
processBlock.LinkTo(writeBlock, linkOptions);
foreach (string file in inputFiles)
{
readBlock.Post(file);
}
readBlock.Complete();
await writeBlock.Completion;
}
static void Main(string[] args)
{
var transformBlock = new TransformBlock <int, string>(n =>
{
Task.Delay(500).Wait();
return(n.ToString());
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 3
}
);
for (int i = 0; i < 10; i++)
{
transformBlock.Post(i);
Console.WriteLine($"Number of messages in the input queue {transformBlock.InputCount}");
}
for (int i = 0; i < 10; i++)
{
var result = transformBlock.Receive();
Console.WriteLine($"Received:{result}");
Console.WriteLine($"Number of messages in the output queue {transformBlock.OutputCount}");
}
Console.WriteLine("Finished!");
Console.Read();
}
private static async Task SimpleDemoWithDelayAsync()
{
Console.WriteLine("TransformBlockDemo has started!");
var block = new TransformBlock <int, string>( // by default singlethreaded
async(input) =>
{
await Task.Delay(500).ConfigureAwait(false);
return(input.ToString());
});
for (int i = 0; i < 10; i++)
{
block.Post(i);
Console.WriteLine($"TransformBlock input queue count: {block.InputCount}");
}
block.Complete(); // No mo data.
while (await block.OutputAvailableAsync().ConfigureAwait(false))
{
Console.WriteLine($"TransformBlock OutputCount: {block.InputCount}");
var output = await block.ReceiveAsync().ConfigureAwait(false);
Console.WriteLine($"TransformBlock TransformOutput: {output}");
Console.WriteLine($"TransformBlock OutputCount: {block.OutputCount}"); // will always be 0, since receive data is a blocking action and this transformblock is single threaded
}
// wait for completion.
await block.Completion.ConfigureAwait(false);
Console.WriteLine("Finished!");
Console.ReadKey();
}
public static void Main(string[] args)
{
var block = new TransformBlock <int, string>(n =>
{
if (n == 5)
{
throw new Exception("Something went wrong");
}
return($"Message {n} processed");
}
);
var printBlock = new ActionBlock <string>(x => Console.WriteLine(x));
block.LinkTo(printBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
for (var i = 0; i < 10; i++)
{
if (!block.Post(i))
{
throw new ArgumentException();
}
}
block.Completion.Wait();
Console.WriteLine("Finished!");
Console.ReadKey();
}
static public void ProcessingByTPL_StraightForwardImplementation()
{
const string pathToFiles = @"..\..\..\..\DataFiles";
string[] files = Directory.GetFiles(pathToFiles, "*.txt", SearchOption.AllDirectories);
var loadDataFromFileBlock = new TransformBlock<string[], List<CustomerTextData>>(fileItems =>
{
var factory = new CustomerTextDataFactory();
return new List<CustomerTextData>(Array.ConvertAll(fileItems, factory.LoadFromFile));
});
var filterBlock = new TransformBlock<List<CustomerTextData>, List<CustomerTextData>>(textDataList =>
{
var filter = new FilterTextData(5);
return textDataList.Where(filter.Run).ToList();
});
var toListBlock = new TransformManyBlock<List<CustomerTextData>, CustomerTextData>(textDataList =>
{
var queue = new ConcurrentQueue<CustomerTextData>();
textDataList.ForEach(queue.Enqueue);
return queue;
});
var action = new ActionBlock<CustomerTextData>(textData =>
{
var weight = new WeightTextData();
int result = weight.Run(textData);
Trace.WriteLine(result);
Console.WriteLine(result);
});
loadDataFromFileBlock.LinkTo(filterBlock);
filterBlock.LinkTo(toListBlock);
toListBlock.LinkTo(action);
loadDataFromFileBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)filterBlock).Fault(t.Exception);
else filterBlock.Complete();
});
filterBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)toListBlock).Fault(t.Exception);
else toListBlock.Complete();
});
toListBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)action).Fault(t.Exception);
else action.Complete();
});
loadDataFromFileBlock.Post(files);
loadDataFromFileBlock.Complete();
action.Completion.Wait();
}
private static TransformBlock<int, string> ConstructTransformWithNMessages(int messagesCount)
{
var block = new TransformBlock<int, string>(i => i.ToString());
for (int i = 0; i < messagesCount; i++)
{
block.Post(i);
}
// Spin until the messages have been properly buffered up.
// Otherwise TryReceive fails.
SpinWait.SpinUntil(() => block.OutputCount == messagesCount);
return block;
}
public void BasicUsageTest ()
{
int[] array = new int[10];
var evt = new ManualResetEventSlim (false);
ActionBlock<int> action = new ActionBlock<int> ((i) => { array[Math.Abs (i)] = i; evt.Set (); });
TransformBlock<int, int> block = new TransformBlock<int, int> (i => -i);
block.LinkTo (action);
for (int i = 0; i < array.Length; ++i)
Assert.IsTrue (block.Post (i), "Not accepted");
evt.Wait ();
CollectionAssert.AreEqual (new int[] { 0, -1, -2, -3, -4, -5, -6, -7, -8, -9 }, array);
}
public void DeferredUsageTest ()
{
int[] array = new int[10];
var action = new ActionBlock<int> (i => array[Math.Abs (i)] = i);
var block = new TransformBlock<int, int> (i => -i);
for (int i = 0; i < array.Length; ++i)
Assert.IsTrue (block.Post (i), "Not accepted");
Thread.Sleep (300);
block.LinkTo (action);
Thread.Sleep (100);
CollectionAssert.AreEqual (new[] { 0, -1, -2, -3, -4, -5, -6, -7, -8, -9 }, array);
}
internal static bool TransformThroughFilterToAction()
{
const int ITERS = 2;
int completedCount = 0;
var t = new TransformBlock<int, int>(i => i);
var c = new ActionBlock<int>(i => completedCount++);
t.LinkTo(c, i => true);
t.Completion.ContinueWith(_ => c.Complete());
for (int i = 0; i < ITERS; i++) t.Post(i);
t.Complete();
c.Completion.Wait();
return completedCount == ITERS;
}
public void AsyncNullTest ()
{
var scheduler = new TestScheduler ();
var block = new TransformBlock<int, int> (
i => null,
new ExecutionDataflowBlockOptions { TaskScheduler = scheduler });
Assert.IsTrue (block.Post (1));
scheduler.ExecuteAll ();
Assert.IsFalse (block.Completion.Wait (100));
block.Complete ();
Assert.IsTrue (block.Completion.Wait (100));
}
public void Run()
{
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 8 };
var tb = new TransformBlock<int, int>(i => i * 2);
var ab = new ActionBlock<int>(i => this.Compute(i), options);
tb.LinkTo(ab);
for (var i = 0; i < 10; i++)
{
tb.Post(i);
}
tb.Complete();
tb.Completion.Wait();
Thread.Sleep(500);
}
public static void Example1()
{
var conf = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 4 };
ActionBlock<int> a = new ActionBlock<int>(i =>
{
Thread.Sleep(500);
Console.WriteLine(i);
}, conf);
TransformBlock<int, int> t = new TransformBlock<int, int>(i => i * 3);
t.LinkTo(a);
for (int i = 0; i < 12; i++)
{
t.Post(i);
}
}
public async Task TestBlockBufferCount()
{
var block1 = new TransformBlock<int, int>(i => 2 * i);
var block2 = new TransformManyBlock<int, int>(i => new [] { i });
var block3 = new ActionBlock<int>(i => { Thread.Sleep(1000); });
block1.Post(0);
block2.Post(0);
block2.Post(0);
block3.Post(0);
block3.Post(0);
block3.Post(0);
await Task.Delay(200);
Assert.AreEqual(1, block1.GetBufferCount().Total());
Assert.AreEqual(2, block2.GetBufferCount().Total());
Assert.AreEqual(2, block3.GetBufferCount().Total());
}
public void BasicUsageTest ()
{
int[] array = new int[10];
var evt = new CountdownEvent (10);
var action = new ActionBlock<int> (i =>
{
array [Math.Abs (i)] = i;
evt.Signal ();
});
var block = new TransformBlock<int, int> (i => -i);
block.LinkTo (action);
for (int i = 0; i < array.Length; ++i)
Assert.IsTrue (block.Post (i), "Not accepted");
evt.Wait ();
CollectionAssert.AreEqual (
new[] { 0, -1, -2, -3, -4, -5, -6, -7, -8, -9 }, array);
}
internal static bool TransformToAction()
{
bool passed = true;
const int ITERS = 2;
var t = new TransformBlock<int, int>(i => i * 2);
int completedCount = 0;
int prev = -2;
var c = new ActionBlock<int>(i =>
{
completedCount++;
if (i != prev + 2) passed &= false;
prev = i;
});
t.LinkWithCompletion(c);
for (int i = 0; i < ITERS; i++) t.Post(i);
t.Complete();
c.Completion.Wait();
Assert.True(completedCount == ITERS);
return passed;
}
public async static void TData()
{
var multiplyBlock = new TransformBlock<int, int>(item =>
{
var res = item * 2;
Console.WriteLine("{0} * 2 = {1}", item, res);
return res;
});
var divideBlock = new TransformBlock<int, int>(item =>
{
var res = item / 2;
Console.WriteLine("{0} / 2 = {1}", item, res);
return res;
});
multiplyBlock.LinkTo(divideBlock);
multiplyBlock.Post(2);
multiplyBlock.Complete();
await divideBlock.Completion;
}
static void Main(string[] args)
{
var multiplyBlock = new TransformBlock<int, int>(value => value * 2);
var subtractBlock = new TransformBlock<int, int>(value => value - 2);
var displayBlock = new ActionBlock<int>(value => Console.WriteLine(value));
// multiplyBlock ==> subtractBlock ==> displayBlock
var linkOptions = new DataflowLinkOptions { PropagateCompletion = true };
multiplyBlock.LinkTo(subtractBlock, linkOptions);
subtractBlock.LinkTo(displayBlock, linkOptions);
// Put data in the first block (multiplyBlock)
foreach (var i in Enumerable.Range(0, 10))
multiplyBlock.Post(i);
// Mark it as complete. Completion will propagate because of the link options.
multiplyBlock.Complete();
// Wait for the last block (displayBlock) to complete.
displayBlock.Completion.Wait();
Console.ReadKey();
}
public async Task TestFaultyLinkedTarget()
{
var tb = new TransformBlock<int, int>(i => i);
tb.LinkTo(new DelegatePropagator<int, int>
{
OfferMessageDelegate = delegate { throw new InvalidCastException(); }
});
tb.Post(42);
await Assert.ThrowsAsync<InvalidCastException>(() => tb.Completion);
}
public async Task TestPrecanceled()
{
var bb = new TransformBlock<int, int>(i => i,
new ExecutionDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true) });
int ignoredValue;
IList<int> ignoredValues;
IDisposable link = bb.LinkTo(DataflowBlock.NullTarget<int>());
Assert.NotNull(link);
link.Dispose();
Assert.False(bb.Post(42));
var t = bb.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
Assert.False(bb.TryReceiveAll(out ignoredValues));
Assert.False(bb.TryReceive(out ignoredValue));
Assert.NotNull(bb.Completion);
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => bb.Completion);
bb.Complete(); // just make sure it doesn't throw
}
public async Task TestExceptions()
{
var tb1 = new TransformBlock<int, int>((Func<int, int>)(i => { throw new InvalidCastException(); }));
var tb2 = new TransformBlock<int, int>((Func<int, Task<int>>)(i => { throw new InvalidProgramException(); }));
var tb3 = new TransformBlock<int, int>((Func<int, Task<int>>)(i => Task.Run((Func<int>)(() => { throw new InvalidTimeZoneException(); }))));
for (int i = 0; i < 3; i++)
{
tb1.Post(i);
tb2.Post(i);
tb3.Post(i);
}
await Assert.ThrowsAsync<InvalidCastException>(() => tb1.Completion);
await Assert.ThrowsAsync<InvalidProgramException>(() => tb2.Completion);
await Assert.ThrowsAsync<InvalidTimeZoneException>(() => tb3.Completion);
Assert.All(new[] { tb1, tb2, tb3 }, tb => Assert.True(tb.InputCount == 0 && tb.OutputCount == 0));
}
public async Task TestReserveReleaseConsume()
{
var tb = new TransformBlock<int, int>(i => i * 2);
tb.Post(1);
await DataflowTestHelpers.TestReserveAndRelease(tb);
tb = new TransformBlock<int, int>(i => i * 2);
tb.Post(2);
await DataflowTestHelpers.TestReserveAndConsume(tb);
}
public async Task TestCountZeroAtCompletion()
{
var cts = new CancellationTokenSource();
var tb = new TransformBlock<int, int>(i => i, new ExecutionDataflowBlockOptions() { CancellationToken = cts.Token });
tb.Post(1);
cts.Cancel();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => tb.Completion);
Assert.Equal(expected: 0, actual: tb.InputCount);
Assert.Equal(expected: 0, actual: tb.OutputCount);
cts = new CancellationTokenSource();
tb = new TransformBlock<int, int>(i => i);
tb.Post(1);
((IDataflowBlock)tb).Fault(new InvalidOperationException());
await Assert.ThrowsAnyAsync<InvalidOperationException>(() => tb.Completion);
Assert.Equal(expected: 0, actual: tb.InputCount);
Assert.Equal(expected: 0, actual: tb.OutputCount);
}
private static bool TestQuickStop(bool testThrow)
{
bool passed = true;
CancellationTokenSource cts = new CancellationTokenSource();
var options = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token };
var propagator = new TransformBlock<ThrowOn, ThrowOn>(x => { Task.Delay(200).Wait(); return x; }, options);
var thrower = new ThrowerBlock();
// Post enough messages to require long processing
for (int i = 0; i < 2; i++)
propagator.Post(testThrow && i == 1 ? ThrowOn.OfferMessage : ThrowOn.TryReceive); // Throw on the second message
// Link the thrower
propagator.LinkTo(thrower);
// Once a message has been processed, cancel the propagator (if we are testing cancellation)
SpinWait.SpinUntil(() => thrower.LastOperation == ThrowOn.OfferMessage);
if (!testThrow) cts.Cancel();
// Wait for the propagator to complete
try
{
var ranToCompletion = propagator.Completion.Wait(10000);
passed = false;
Console.WriteLine("Task is faulted or canceled (finished: {0}) - FAILED", ranToCompletion ? "ran to copmpletion" : "still running");
}
catch (AggregateException ae)
{
passed = testThrow ? ae.InnerException is InvalidOperationException : ae.InnerException is TaskCanceledException;
ae.Handle(e => true);
Console.WriteLine("Task is faulted or canceled (exception) - {0}", passed ? "Passed" : "FAILED");
}
return passed;
}
public Task RunAsync()
{
TransformBlock<Uri, PropertyBag> ingestBlock = new TransformBlock<Uri, PropertyBag>(input =>
{
PropertyBag result = new PropertyBag
{
OriginalUrl = input.ToString(),
UserAgent = _userAgent,
Step = new CrawlStep(input, 0)
};
return result;
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxDegreeOfParallelism
});
TransformBlock<PropertyBag, PropertyBag> ingestBlockForAggregation =
new TransformBlock<PropertyBag, PropertyBag>(input => input, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = MaxDegreeOfParallelism
});
CrawlIngestionHelper crawlIngestionHelper = new CrawlIngestionHelper(ingestBlockForAggregation, _userAgent);
TransformBlock<PropertyBag, PropertyBag>[] pipeline = Pipeline
.Select(pipelineStep =>
{
return new TransformBlock<PropertyBag, PropertyBag>(async propertyBag =>
{
if (propertyBag.StopPipelining)
{
return propertyBag;
}
try
{
propertyBag.StopPipelining = !await pipelineStep.Process(crawlIngestionHelper, propertyBag);
}
catch (Exception exception)
{
propertyBag.Exceptions.Add(exception);
}
return propertyBag;
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = pipelineStep.MaxDegreeOfParallelism
});
})
.ToArray();
ActionBlock<PropertyBag> terminationCheckerBlock = new ActionBlock<PropertyBag>(propertyBag =>
{
if (ingestBlock.InputCount == 0
&& ingestBlock.OutputCount == 0
&& !ingestBlock.Completion.IsCompleted
&& !ingestBlock.Completion.IsCanceled
&& !ingestBlock.Completion.IsFaulted
&& ingestBlockForAggregation.InputCount == 0
&& ingestBlockForAggregation.OutputCount == 0)
{
if (pipeline.Any(transformBlock => transformBlock.InputCount != 0 || transformBlock.OutputCount != 0))
{
return;
}
ingestBlock.Complete();
}
}, new ExecutionDataflowBlockOptions {MaxDegreeOfParallelism = 1});
ingestBlock.LinkTo(ingestBlockForAggregation, new DataflowLinkOptions {PropagateCompletion = true});
TransformBlock<PropertyBag, PropertyBag> previous = ingestBlockForAggregation;
foreach (TransformBlock<PropertyBag, PropertyBag> transformBlock in pipeline)
{
previous.LinkTo(transformBlock, new DataflowLinkOptions {PropagateCompletion = true});
previous = transformBlock;
}
previous.LinkTo(terminationCheckerBlock, new DataflowLinkOptions {PropagateCompletion = true});
foreach (Uri startUri in StartUris)
{
ingestBlock.Post(startUri);
}
return terminationCheckerBlock.Completion;
}
//[Fact(Skip = "Outerloop")]
public void RunTransformBlockConformanceTests()
{
bool passed = true;
// SYNC
#region Sync
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> transformBlockFactory =
options =>
{
TransformBlock<int, int> transformBlock = new TransformBlock<int, int>(i => i, (ExecutionDataflowBlockOptions)options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = transformBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2, CancellationToken = cancellationSource.Token };
passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, dopOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 1, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, cancellationOptions, 1, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2));
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16;
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => i * 2));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformBlock<int, int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return i;
});
for (int i = 0; i < 2; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 2; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test using a precanceled token
{
bool localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token };
var t = new TransformBlock<int, int>(i => i, dbo);
int ignoredValue;
IList<int> ignoredValues;
localPassed &= t.LinkTo(new ActionBlock<int>(delegate { })) != null;
localPassed &= t.SendAsync(42).Result == false;
localPassed &= t.TryReceiveAll(out ignoredValues) == false;
localPassed &= t.Post(42) == false;
localPassed &= t.OutputCount == 0;
localPassed &= t.TryReceive(out ignoredValue) == false;
localPassed &= t.Completion != null;
t.Complete();
}
catch (Exception)
{
localPassed = false;
}
Console.WriteLine(" {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting
{
bool localPassed = true;
var t = new TransformBlock<int, int>(new Func<int, int>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" {0}: Faulted handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
#region Async
// ASYNC (a copy of the sync but with constructors returning Task<T> instead of T
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> transformBlockFactory =
options =>
{
TransformBlock<int, int> transformBlock = new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i), (ExecutionDataflowBlockOptions)options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = transformBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 2, CancellationToken = cancellationSource.Token };
passed &= FeedTarget(transformBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, dopOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2)));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2)));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2)));
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16;
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformBlock<int, int>, int>(4, () => new TransformBlock<int, int>(i => Task.Factory.StartNew(() => i * 2)));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformBlock<int, int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return Task.Factory.StartNew(() => i);
});
for (int i = 0; i < 2; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 2; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored
{
bool localPassed = true;
var t = new TransformBlock<int, int>(i =>
{
if ((i % 2) == 0) return null;
return Task.Factory.StartNew(() => i);
});
for (int i = 0; i < 2; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 2; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: null tasks are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored when a reordering buffer is in place
{
bool localPassed = true;
var t = new TransformBlock<int, int>(i =>
{
if (i == 0)
{
Task.Delay(10).Wait();
return null;
}
return Task.Factory.StartNew(() => i);
}, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 });
t.Post(0);
t.Post(1);
try
{
localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1;
}
catch
{
localPassed = false;
}
Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the delegate
{
bool localPassed = true;
var t = new TransformBlock<int, int>(new Func<int, Task<int>>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the task
{
bool localPassed = true;
var t = new TransformBlock<int, int>(new Func<int, Task<int>>(i => Task<int>.Factory.StartNew(() => { throw new InvalidOperationException(); })));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
Assert.True(passed, "Test failed.");
}
public async Task TransformPingPong()
{
TransformBlock<int, int> t1 = null, t2 = null;
t1 = new TransformBlock<int, int>(i =>
{
if (i >= Iterations) t2.Complete();
return i + 1;
});
t2 = new TransformBlock<int, int>(i => i + 1);
t1.LinkTo(t2);
t2.LinkTo(t1);
t1.Post(0);
await t2.Completion;
}
//
// The test that verifies that internally, TransformBlock does not get confused
// between Func<T, Task<object>> and Func<T, object>.
//[Fact(Skip = "Outerloop")]
public void RunTransformCtorOverloadTest()
{
bool passed = true;
{
Func<object, Task<object>> f = x => Task<object>.FromResult(x);
TransformBlock<object, object> tf = new TransformBlock<object, object>(f);
var mre = new ManualResetEvent(false);
ActionBlock<object> a = new ActionBlock<object>(x =>
{
bool localPassed = !(x is Task<object>);
passed &= localPassed;
mre.Set();
});
tf.LinkTo(a);
tf.Post(new object());
mre.WaitOne();
}
{
Func<object, Task<object>> f = x => Task<object>.FromResult(x);
TransformBlock<object, object> tf = new TransformBlock<object, object>((Func<object, object>)f);
var mre = new ManualResetEvent(false);
ActionBlock<object> a = new ActionBlock<object>(x =>
{
bool localPassed = x is Task<object>;
passed &= localPassed;
mre.Set();
});
tf.LinkTo(a);
tf.Post(new object());
mre.WaitOne();
}
Assert.True(passed, "Test failed.");
}
internal static bool TransformThroughDiscardingFilterToAction()
{
const int ITERS = 2;
int completedCount = 0;
var t = new TransformBlock<int, int>(i => i);
var c = new ActionBlock<int>(i => completedCount++);
t.LinkTo(c, i => i % 2 == 0);
t.LinkTo(DataflowBlock.NullTarget<int>());
t.Completion.ContinueWith(_ => c.Complete());
for (int i = 0; i < ITERS; i++) t.Post(i);
t.Complete();
c.Completion.Wait();
return completedCount == ITERS / 2;
}
public async Task TestOrdering_Sync_OrderedDisabled()
{
// If ordering were enabled, this test would hang.
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, EnsureOrdered = false };
var mres = new ManualResetEventSlim();
var tb = new TransformBlock<int, int>(i =>
{
if (i == 0) mres.Wait();
return i;
}, options);
tb.Post(0);
tb.Post(1);
Assert.Equal(1, await tb.ReceiveAsync());
mres.Set();
Assert.Equal(0, await tb.ReceiveAsync());
tb.Complete();
await tb.Completion;
}
public async Task PostList(ITargetBlock<Core.Entities.BalanceDate> target)
{
Diag.ThreadPrint("PostList - start");
var transform = new TransformBlock<BalanceDate, Core.Entities.BalanceDate>(ef =>
mapper.Map<Core.Entities.BalanceDate>(ef), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 4 });
transform.LinkTo(target,new DataflowLinkOptions() { PropagateCompletion = true });
await Task.Run(() =>
{
Diag.ThreadPrint("PostList - task start");
using (FinanceEntities context = factory.CreateContext())
{
(from b in context.BalanceDates
.Include(a => a.BalanceDateBankAccounts)
.Include("BalanceDateBankAccounts.BankAccount")
.Include("BalanceDateBankAccounts.BankAccount.Bank")
select b).AsParallel().ForAll(ef => transform.Post(ef));
//await transform.Completion;
//transform.Completion.ContinueWith(t =>
//{
// if (t.IsFaulted) target.Fault(t.Exception);
// else
// {
// Diag.ThreadPrint("PostList - task set target complete");
// target.Complete();
// }
//});
transform.Complete();
}
Diag.ThreadPrint("PostList - task end");
}).ConfigureAwait(false);
Diag.ThreadPrint("PostList - end");
}
static void Main(string[] args)
{
//
// Create the members of the pipeline.
//
// Downloads the requested resource as a string.
var downloadString = new TransformBlock<string, string>(uri =>
{
Console.WriteLine("Downloading '{0}'...", uri);
return new WebClient().DownloadString(uri);
});
// Separates the specified text into an array of words.
var createWordList = new TransformBlock<string, string[]>(text =>
{
Console.WriteLine("Creating word list...");
// Remove common punctuation by replacing all non-letter characters
// with a space character to.
char[] tokens = text.ToArray();
for (int i = 0; i < tokens.Length; i++)
{
if (!char.IsLetter(tokens[i]))
tokens[i] = ' ';
}
text = new string(tokens);
// Separate the text into an array of words.
return text.Split(new char[] { ' ' },
StringSplitOptions.RemoveEmptyEntries);
});
// Removes short words, orders the resulting words alphabetically,
// and then remove duplicates.
var filterWordList = new TransformBlock<string[], string[]>(words =>
{
Console.WriteLine("Filtering word list...");
return words.Where(word => word.Length > 3).OrderBy(word => word)
.Distinct().ToArray();
});
// Finds all words in the specified collection whose reverse also
// exists in the collection.
var findReversedWords = new TransformManyBlock<string[], string>(words =>
{
Console.WriteLine("Finding reversed words...");
// Holds reversed words.
var reversedWords = new ConcurrentQueue<string>();
// Add each word in the original collection to the result whose
// reversed word also exists in the collection.
Parallel.ForEach(words, word =>
{
// Reverse the work.
string reverse = new string(word.Reverse().ToArray());
// Enqueue the word if the reversed version also exists
// in the collection.
if (Array.BinarySearch<string>(words, reverse) >= 0 &&
word != reverse)
{
reversedWords.Enqueue(word);
}
});
return reversedWords;
});
// Prints the provided reversed words to the console.
var printReversedWords = new ActionBlock<string>(reversedWord =>
{
Console.WriteLine("Found reversed words {0}/{1}",
reversedWord, new string(reversedWord.Reverse().ToArray()));
});
//
// Connect the dataflow blocks to form a pipeline.
//
downloadString.LinkTo(createWordList);
createWordList.LinkTo(filterWordList);
filterWordList.LinkTo(findReversedWords);
findReversedWords.LinkTo(printReversedWords);
//
// For each completion task in the pipeline, create a continuation task
// that marks the next block in the pipeline as completed.
// A completed dataflow block processes any buffered elements, but does
// not accept new elements.
//
downloadString.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)createWordList).Fault(t.Exception);
else createWordList.Complete();
});
createWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)filterWordList).Fault(t.Exception);
else filterWordList.Complete();
});
filterWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)findReversedWords).Fault(t.Exception);
else findReversedWords.Complete();
});
findReversedWords.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)printReversedWords).Fault(t.Exception);
else printReversedWords.Complete();
});
// Process "The Iliad of Homer" by Homer.
downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt");
// Mark the head of the pipeline as complete. The continuation tasks
// propagate completion through the pipeline as each part of the
// pipeline finishes.
downloadString.Complete();
// Wait for the last block in the pipeline to process all messages.
printReversedWords.Completion.Wait();
Console.ReadLine();
}
public async Task TestAsObservableAndAsObserver_ErrorPropagation()
{
// Test that exceptional data flows when exception occurs before and after subscription
foreach (bool beforeSubscription in DataflowTestHelpers.BooleanValues)
{
var tb = new TransformBlock<int, int>(i => {
if (i == 42) throw new InvalidOperationException("uh oh");
return i;
});
if (beforeSubscription)
{
tb.Post(42);
await Assert.ThrowsAsync<InvalidOperationException>(async () => await tb.Completion);
}
ITargetBlock<int>[] targets = Enumerable.Range(0, 3).Select(_ => new WriteOnceBlock<int>(i => i)).ToArray();
foreach (var target in targets)
{
tb.AsObservable().Subscribe(target.AsObserver());
}
if (!beforeSubscription)
{
tb.Post(42);
await Assert.ThrowsAsync<InvalidOperationException>(() => tb.Completion);
}
foreach (var target in targets)
{
await Assert.ThrowsAsync<AggregateException>(() => target.Completion);
}
}
}
public void AsyncCancelledTest ()
{
var scheduler = new TestScheduler ();
var block = new TransformBlock<int, int> (
i =>
{
var tcs = new TaskCompletionSource<int> ();
tcs.SetCanceled ();
return tcs.Task;
}, new ExecutionDataflowBlockOptions { TaskScheduler = scheduler });
Assert.IsTrue (block.Post (1));
scheduler.ExecuteAll ();
Assert.IsFalse (block.Completion.Wait (100));
}
