public async Task TestOrdering_Sync_OrderedDisabled(bool trustedEnumeration)
{
// If ordering were enabled, this test would hang.
var options = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 2, EnsureOrdered = false
};
var mres = new ManualResetEventSlim();
var tb = new TransformManyBlock <int, int>(i =>
{
if (i == 0)
{
mres.Wait();
}
return(trustedEnumeration ?
new[] { i } :
Enumerable.Repeat(i, 1));
}, options);
tb.Post(0);
tb.Post(1);
Assert.Equal(1, await tb.ReceiveAsync());
mres.Set();
Assert.Equal(0, await tb.ReceiveAsync());
tb.Complete();
await tb.Completion;
}
static void ShowTransformManyBlock()
{
// <snippet6>
// Create a TransformManyBlock<string, char> object that splits
// a string into its individual characters.
var transformManyBlock = new TransformManyBlock <string, char>(
s => s.ToCharArray());
// Post two messages to the first block.
transformManyBlock.Post("Hello");
transformManyBlock.Post("World");
// Receive all output values from the block.
for (int i = 0; i < ("Hello" + "World").Length; i++)
{
Console.WriteLine(transformManyBlock.Receive());
}
/* Output:
* H
* e
* l
* l
* o
* W
* o
* r
* l
* d
*/
// </snippet6>
}
public async Task TestOrdering_Sync_BlockingEnumeration_NoDeadlock(bool ensureOrdered)
{
// If iteration of the yielded enumerables happened while holding a lock, this would deadlock.
var options = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 2, EnsureOrdered = ensureOrdered
};
ManualResetEventSlim mres1 = new ManualResetEventSlim(), mres2 = new ManualResetEventSlim();
var tb = new TransformManyBlock <int, int>(i => i == 0 ? BlockableIterator(mres1, mres2) : BlockableIterator(mres2, mres1), options);
tb.Post(0);
tb.Post(1);
Assert.Equal(42, await tb.ReceiveAsync());
Assert.Equal(42, await tb.ReceiveAsync());
tb.Complete();
await tb.Completion;
IEnumerable <int> BlockableIterator(ManualResetEventSlim wait, ManualResetEventSlim release)
{
release.Set();
wait.Wait();
yield return(42);
}
}
public void TransformManyOverfullTest2()
{
var scheduler = new TestScheduler();
n = 0;
var transform = new TransformManyBlock <int, int> (
i => ComputeResults(),
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 100, TaskScheduler = scheduler
});
for (int i = 0; i < 100; i++)
{
Assert.IsTrue(transform.Post(i));
}
Assert.IsFalse(transform.Post(101));
Assert.GreaterOrEqual(scheduler.ExecuteAll(), 1);
Assert.IsFalse(transform.Post(102));
Assert.AreEqual(10000, Volatile.Read(ref n));
}
public void TransformManyBlockTest()
{
var block = new TransformManyBlock <int, int> (
i => new[] { -i, i },
new ExecutionDataflowBlockOptions {
BoundedCapacity = 1
});
Assert.IsTrue(block.Post(1));
Assert.IsFalse(block.Post(2));
Assert.IsFalse(block.Completion.Wait(100));
Assert.IsFalse(block.Post(3));
Assert.AreEqual(-1, block.Receive());
Assert.IsFalse(block.Post(4));
Assert.AreEqual(1, block.Receive());
Assert.IsTrue(block.Post(5));
Assert.AreEqual(-5, block.Receive());
Assert.AreEqual(5, block.Receive());
}
public override void Post(Job job)
{
//Console.WriteLine($"{job.Id} posting");
for (var i = 0; i < 100; ++i)
{
job.AddData(job.Id);
}
_start.Post(job);
}
public void TestPostAsyncEnumerable()
{
foreach (bool bounded in DataflowTestHelpers.BooleanValues)
{
var tb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToAsyncEnumerable, new ExecutionDataflowBlockOptions {
BoundedCapacity = bounded ? 1 : -1
});
Assert.True(tb.Post(0));
tb.Complete();
Assert.False(tb.Post(0));
}
}
public override void Post(Job job)
{
//Console.WriteLine($"{job.Id} posting");
for (var i = 0; i < 10; ++i)
{
job.AddData("data " + i, new DataProperty[] { new DataProperty {
Name = (i % 2 == 0) ? "Even" : "Odd", Value = ""
} });
}
_start.Post(job);
}
public ISourceBlock <DownloadedFile> DownloadFiles(string[] fileIds, string securityCookieString, string securityCookieDomain)
{
var urls = CreateUrls(fileIds);
// we have to use TransformManyBlock here, because we want to be able to return 0 or 1 items
var block = new TransformManyBlock <string, DownloadedFile>(
async url =>
{
var httpClientHandler = new HttpClientHandler();
if (!string.IsNullOrEmpty(securityCookieString))
{
var securityCookie = new Cookie(FormsAuthentication.FormsCookieName, securityCookieString);
securityCookie.Domain = securityCookieDomain;
httpClientHandler.CookieContainer.Add(securityCookie);
}
return(await DownloadFile(url, httpClientHandler));
}, new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = Properties.Settings.Default.maxConcurrentDownloads
});
foreach (var url in urls)
{
block.Post(url);
}
block.Complete();
return(block);
}
public async Task FindProjectJsonAsync()
{
TransformManyBlock <GitHubRepo, SearchResult> repoSearchBlock = new TransformManyBlock <GitHubRepo, SearchResult>(repo => SearchRepoAsync(repo),
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount * 4
//MaxDegreeOfParallelism = 1
});
ActionBlock <SearchResult> downloadFileBlock = new ActionBlock <SearchResult>(DownloadFileAsync, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount * 4
//MaxDegreeOfParallelism = 1
});
repoSearchBlock.LinkTo(downloadFileBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
foreach (var repo in _storage.GetAllRepos())
{
if (_cancelToken.IsCancellationRequested)
{
break;
}
repoSearchBlock.Post(repo);
}
repoSearchBlock.Complete();
await downloadFileBlock.Completion;
}
public async Task TestMultiplePublishersWithPropagateCompletion()
{
var block1 = new TransformManyBlock <string, char>(x => x.ToCharArray());
var block2 = new TransformManyBlock <string, char>(x => x.ToCharArray());
var target = new BufferBlock <char>();
var propagate = new DataflowLinkOptions {
PropagateCompletion = true
};
block1.LinkTo(target, propagate);
block2.LinkTo(target, propagate);
block1.Post("a");
// This propagates completion through to target
block1.Complete();
await AssertCompletes(block1.Completion);
Assert.IsTrue(await target.OutputAvailableAsync(), "target should have message waiting");
Assert.IsTrue(!target.Completion.IsCompleted, "target won't be complete until buffer empty");
// The target won't receive this, because it's already started completing (via propagation)
block2.Post("b");
block2.Complete();
// Pulling this out of the buffer allows the target to complete
Assert.AreEqual('a', target.Receive());
// This sholud happen pretty quickly
await AssertCompletes(target.Completion);
}
public void Start(CoreValues cv)
{
CoreValues = cv;
MyCamera = new Camera(cv.Origin, cv.LookAt, cv.VecUp, cv.FovY, cv.AspectRatio, cv.Aperture, cv.DistToFocus);
Image = new Vec3[cv.Width * cv.Height];
World = HitableList.DefaultWorld;
Count = cv.Width * cv.Height;
PixelFloats = new TransformManyBlock <int, InfoStruct>(GetSampleRays);
MergeColors = new ActionBlock <InfoStruct>(DoMergeColors, new ExecutionDataflowBlockOptions {
EnsureOrdered = false, MaxDegreeOfParallelism = -1
});
//writeImage = new ActionBlock<InfoStruct>(DoWriteImage, new ExecutionDataflowBlockOptions { EnsureOrdered = false, MaxDegreeOfParallelism = -1 });
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
PixelFloats.LinkTo(MergeColors, linkOptions);
//MergeColors.LinkTo(writeImage, linkOptions);
PixelFloats.Post(0);
PixelFloats.Complete();
MergeColors.Completion.Wait();
DoSaveImage();
}
private async Task Start(string path)
{
var uiScheduler = TaskScheduler.FromCurrentSynchronizationContext();
dirToFilesBlock = new TransformManyBlock <string, string>((Func <string, IEnumerable <string> >)(GetFileSystemItems), new ExecutionDataflowBlockOptions()
{
CancellationToken = ct
});
fileActionBlock = new ActionBlock <string>((Action <string>)ProcessFile, new ExecutionDataflowBlockOptions()
{
CancellationToken = ct, TaskScheduler = uiScheduler
});
// Order of LinkTo's important here!
dirToFilesBlock.LinkTo(dirToFilesBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
}, IsDirectory);
dirToFilesBlock.LinkTo(fileActionBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
}, IsRequiredDocType);
// Kick off the recursion.
dirToFilesBlock.Post(path);
await ProcessingIsComplete();
dirToFilesBlock.Complete();
await Task.WhenAll(dirToFilesBlock.Completion, fileActionBlock.Completion);
}
public IEnumerable <IMetaData> TransformManyBlockUsage(string stringToSplit)
{
Console.WriteLine($"Inside {nameof(TplDataflow1ExecutionBlocksController)} - {nameof(TransformManyBlockUsage)}");
// Create the members of the pipeline.
var transformManyBlockSplitAnInputStringIntoArray = new TransformManyBlock <string, string>(input =>
Functions.SplitAnInputStringIntoArray(input, SplitterSeparator)
);
var transformBlockCreateASingleMedatadataFromAString = new TransformBlock <string, IMetaData>(stringInput =>
Functions.CreateASingleMedatadataFromAString(stringInput)
);
// Connect the dataflow blocks to form a pipeline.
transformManyBlockSplitAnInputStringIntoArray.LinkTo(transformBlockCreateASingleMedatadataFromAString, DataflowOptions.LinkOptions);
// Start TransformManyBlockUsage pipeline with the input values.
transformManyBlockSplitAnInputStringIntoArray.Post(stringToSplit);
// Mark the head of the pipeline as complete.
transformManyBlockSplitAnInputStringIntoArray.Complete();
// Equivalent of transformManyBlockSplitAnInputStringIntoArray.OutputCount
var ouputCount = stringToSplit?.Split(SplitterSeparator, StringSplitOptions.RemoveEmptyEntries).Length ?? 0;
for (var i = 0; i < ouputCount; i++)
{
yield return(transformBlockCreateASingleMedatadataFromAString.Receive());
}
}
public async Task TestExceptions()
{
var tb1 = new TransformManyBlock <int, int>((Func <int, IEnumerable <int> >)(i => { throw new InvalidCastException(); }));
var tb2 = new TransformManyBlock <int, int>((Func <int, Task <IEnumerable <int> > >)(i => { throw new InvalidProgramException(); }));
var tb3 = new TransformManyBlock <int, int>((Func <int, Task <IEnumerable <int> > >)(i => Task.Run((Func <IEnumerable <int> >)(() => { throw new InvalidTimeZoneException(); }))));
var tb4 = new TransformManyBlock <int, int>(i => ExceptionAfter(3));
var tb5 = new TransformManyBlock <int, int>(i => Task.Run(() => ExceptionAfter(3)));
for (int i = 0; i < 3; i++)
{
tb1.Post(i);
tb2.Post(i);
tb3.Post(i);
tb4.Post(i);
tb5.Post(i);
}
await Assert.ThrowsAsync <InvalidCastException>(() => tb1.Completion);
await Assert.ThrowsAsync <InvalidProgramException>(() => tb2.Completion);
await Assert.ThrowsAsync <InvalidTimeZoneException>(() => tb3.Completion);
await Assert.ThrowsAsync <FormatException>(() => tb4.Completion);
await Assert.ThrowsAsync <FormatException>(() => tb5.Completion);
Assert.All(new[] { tb1, tb2, tb3 }, tb => Assert.True(tb.InputCount == 0 && tb.OutputCount == 0));
}
public static async Task RunSampleOne(Func <int, Task <User[]> > getUsers, Func <User, Task> writeUser, int pageCount)
{
var getDataBlock = new TransformManyBlock <int, User>(
transform: async(pageNumber) => await getUsers(pageNumber),
dataflowBlockOptions: new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 3
});
var writeDataBlock = new ActionBlock <User>(
action: async(user) => await writeUser(user),
dataflowBlockOptions: new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1
});
getDataBlock.LinkTo(writeDataBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
for (var i = 1; i <= pageCount; i++)
{
getDataBlock.Post(i);
}
getDataBlock.Complete();
await writeDataBlock.Completion;
}
public async Task RunAsync()
{
if (!IsBuilt)
{
Logger.LogError("Error run not arranged conveyor. Use {arrangeMethod}() before {runMethod}().", nameof(Arrange), nameof(RunAsync));
throw new InvalidOperationException($"Error run not arranged conveyor. Use {nameof(Arrange)}() before {nameof(RunAsync)}().");
}
//Run conveyor
Stopwatch stopWatch = Stopwatch.StartNew();
try
{
dataNamesEnumeratorBlock.Post(filePath);
dataNamesEnumeratorBlock.Complete();
//Wait result
await writeFilesBlock.Completion;
}
catch (AggregateException exs)
{
foreach (Exception ex in exs.InnerExceptions)
{
Logger.LogError("Errors while processing: {err}", ex.Message);
}
}
stopWatch.Stop();
elapsedTime = stopWatch.ElapsedMilliseconds;
}
public async Task TestOrdering()
{
const int iters = 9999;
foreach (int mmpt in new[] { DataflowBlockOptions.Unbounded, 1 })
{
foreach (int dop in new[] { 1, 2, DataflowBlockOptions.Unbounded })
{
var options = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = dop, MaxMessagesPerTask = mmpt
};
var tb = new TransformManyBlock <int, int>(i => new[] { i, i + 1, i + 2 }, options);
for (int i = 0; i < iters; i += 3)
{
Assert.True(tb.Post(i));
}
for (int i = 0; i < iters; i++)
{
Assert.Equal(expected: i, actual: await tb.ReceiveAsync());
}
tb.Complete();
await tb.Completion;
}
}
}
public async Task TestCircularLinking()
{
const int Iters = 200;
foreach (bool sync in DataflowTestHelpers.BooleanValues)
{
var tcs = new TaskCompletionSource <bool>();
Func <int, IEnumerable <int> > body = i => {
if (i >= Iters)
{
tcs.SetResult(true);
}
return(Enumerable.Repeat(i + 1, 1));
};
TransformManyBlock <int, int> tb = sync ?
new TransformManyBlock <int, int>(body) :
new TransformManyBlock <int, int>(i => Task.Run(() => body(i)));
using (tb.LinkTo(tb))
{
tb.Post(0);
await tcs.Task;
tb.Complete();
}
}
}
public async Task TestPrecanceled()
{
var bb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable,
new ExecutionDataflowBlockOptions {
CancellationToken = new CancellationToken(canceled: true)
});
int ignoredValue;
IList <int> ignoredValues;
IDisposable link = bb.LinkTo(DataflowBlock.NullTarget <int>());
Assert.NotNull(link);
link.Dispose();
Assert.False(bb.Post(42));
var t = bb.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
Assert.False(bb.TryReceiveAll(out ignoredValues));
Assert.False(bb.TryReceive(out ignoredValue));
Assert.NotNull(bb.Completion);
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => bb.Completion);
bb.Complete(); // just make sure it doesn't throw
}
internal void start()
{
var consumerBlock = Consumer("consumer 1");
var consumer2Block = Consumer("\t\tconsumer 2");
var producerBlock = new TransformManyBlock <int, int>(x => Enumerable.Range(0, x));
producerBlock.LinkTo(consumerBlock, new DataflowLinkOptions
{
PropagateCompletion = true,
});
producerBlock.LinkTo(consumer2Block, new DataflowLinkOptions
{
PropagateCompletion = true
});
if (!producerBlock.Post(10))
{
Console.WriteLine("Failed Post");
}
producerBlock.Complete();
producerBlock.Completion.ContinueWith(p => print_status(p, "producer"));
consumerBlock.Completion.ContinueWith(p => print_status(p, "consumer 1"));
consumer2Block.Completion.ContinueWith(p => print_status(p, "\t\tconsumer 2"));
}
public void DeferredUsageTest ()
{
int insIndex = -1;
int[] array = new int[5 + 3];
var evt = new CountdownEvent (array.Length);
var block = new ActionBlock<int> (i => { array[Interlocked.Increment (ref insIndex)] = i; evt.Signal (); });
var trsm = new TransformManyBlock<int, int> (i => Enumerable.Range (0, i));
trsm.Post (5);
trsm.Post (3);
trsm.LinkTo (block);
evt.Wait ();
CollectionAssert.AreEquivalent (new int[] { 0, 1, 2, 3, 4, 0, 1, 2 }, array);
}
public Task WalkAsync(string path)
{
directoriesRemaining = 1;
directoryBrowseBlock.Post(path);
return(fileActionBlock.Completion);
}
public void ProcessDirectory(string dir)
{
imageCollectionBlock.Post(dir);
//DirectoryInfo directory = new DirectoryInfo(dir);
//foreach (FileInfo file in directory.GetFiles("*.jpg", SearchOption.AllDirectories))
//{
// loadAndToGreyBlock.Post(new Uri(file.FullName));
//}
}
public void DeferredUsageTest()
{
int insIndex = -1;
int[] array = new int[5 + 3];
var evt = new CountdownEvent(array.Length);
var block = new ActionBlock <int> (i => { array[Interlocked.Increment(ref insIndex)] = i; evt.Signal(); });
var trsm = new TransformManyBlock <int, int> (i => Enumerable.Range(0, i));
trsm.Post(5);
trsm.Post(3);
trsm.LinkTo(block);
evt.Wait();
CollectionAssert.AreEquivalent(new int[] { 0, 1, 2, 3, 4, 0, 1, 2 }, array);
}
public static async Task PurgeExpiredAsync(CloudBlobContainer inboxContainer)
{
Requires.NotNull(inboxContainer, "inboxContainer");
var deleteBlobsExpiringBefore = DateTime.UtcNow;
int purgedBlobCount = 0;
var searchExpiredBlobs = new TransformManyBlock <CloudBlobContainer, ICloudBlob>(
async c => {
try {
var results = await c.ListBlobsSegmentedAsync(
string.Empty,
useFlatBlobListing: true,
pageSize: 50,
details: BlobListingDetails.Metadata,
options: new BlobRequestOptions(),
operationContext: null);
return(from blob in results.OfType <ICloudBlob>()
let expires = DateTime.Parse(blob.Metadata[ExpirationDateMetadataKey])
where expires < deleteBlobsExpiringBefore
select blob);
} catch (StorageException ex) {
var webException = ex.InnerException as WebException;
if (webException != null)
{
var httpResponse = (HttpWebResponse)webException.Response;
if (httpResponse.StatusCode == HttpStatusCode.NotFound)
{
// it's legit that some tests never created the container to begin with.
return(Enumerable.Empty <ICloudBlob>());
}
}
throw;
}
},
new ExecutionDataflowBlockOptions {
BoundedCapacity = 4,
});
var deleteBlobBlock = new ActionBlock <ICloudBlob>(
blob => {
Interlocked.Increment(ref purgedBlobCount);
return(blob.DeleteAsync());
},
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 4,
BoundedCapacity = 100,
});
searchExpiredBlobs.LinkTo(deleteBlobBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
searchExpiredBlobs.Post(inboxContainer);
searchExpiredBlobs.Complete();
await deleteBlobBlock.Completion;
}
private static void Main()
{
var consumeBlock = new ActionBlock <int>(new Action <int>(Consumer));
var transformManyBlock = new TransformManyBlock <int, int>(new Func <int, IEnumerable <int> >(ProduceMany));
transformManyBlock.LinkTo(consumeBlock);
transformManyBlock.Post(10);
Console.ReadLine();
}
private static void DataStructure_TransformManyBlock()
{
// Create a TransformManyBlock<string, char> object that splits
// a string into its individual characters.
var transformManyBlock = new TransformManyBlock <string, char>( //返回的为IEnumrable<char> 委托类型Func<TInput, IEnumerable<TOutput>>
//var transformManyBlock = new TransformBlock<string, char>( //返回的为char 委托类型Func<TInput, TOutput>
s => s.ToCharArray());
// Post two messages to the first block.
transformManyBlock.Post("Hello");
transformManyBlock.Post("World");
// Receive all output values from the block.
for (int i = 0; i < ("Hello" + "World").Length; i++)
{
Console.WriteLine(transformManyBlock.Receive());
}
}
public static void TestSync()
{
//这个地方就是将TransformManyBlock中的多个参数依次转换成了一个参数给ActionBlock
tmb.LinkTo(ab);
for (int i = 0; i < 4; i++)
{
tmb.Post(i);
}
Console.WriteLine("Finished post");
}
public async Task TestBlockBufferCount()
{
var block1 = new TransformBlock <int, int>(i => 2 * i);
var block2 = new TransformManyBlock <int, int>(i => new [] { i });
var block3 = new ActionBlock <int>(i => { Thread.Sleep(1000); });
block1.Post(0);
block2.Post(0);
block2.Post(0);
block3.Post(0);
block3.Post(0);
block3.Post(0);
await Task.Delay(200);
Assert.AreEqual(1, block1.GetBufferCount());
Assert.AreEqual(2, block2.GetBufferCount());
Assert.AreEqual(2, block3.GetBufferCount());
}
public async Task TestBlockBufferCount()
{
var block1 = new TransformBlock<int, int>(i => 2 * i);
var block2 = new TransformManyBlock<int, int>(i => new [] { i });
var block3 = new ActionBlock<int>(i => { Thread.Sleep(1000); });
block1.Post(0);
block2.Post(0);
block2.Post(0);
block3.Post(0);
block3.Post(0);
block3.Post(0);
await Task.Delay(200);
Assert.AreEqual(1, block1.GetBufferCount().Total());
Assert.AreEqual(2, block2.GetBufferCount().Total());
Assert.AreEqual(2, block3.GetBufferCount().Total());
}
public async Task TestReserveReleaseConsume()
{
var tb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable);
tb.Post(1);
await DataflowTestHelpers.TestReserveAndRelease(tb);
tb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable);
tb.Post(2);
await DataflowTestHelpers.TestReserveAndConsume(tb);
}
private static TransformManyBlock<int, int> ConstructTransformManyWithNMessages(int messagesCount)
{
var block = new TransformManyBlock<int, int>(i => new int[] { i });
for (int i = 0; i < messagesCount; i++)
{
block.Post(i);
}
// Spin until the messages have been properly buffered up.
// Otherwise TryReceive fails.
SpinWait.SpinUntil(() => block.OutputCount == messagesCount);
return block;
}
private static void Main(string[] args)
{
var generatorBlock = new TransformManyBlock<int, string>(num => GenerateStrings(num));
var writerBlock = new TransformBlock<string, string>(str => WriteString(str), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5 });
var finishingBlock = new ActionBlock<string>(str =>
{
writerBlock.Completion.Wait();
Console.WriteLine(Thread.CurrentThread.ManagedThreadId + ": finished - " + str);
});
generatorBlock.LinkTo(writerBlock, new DataflowLinkOptions{PropagateCompletion = true});
writerBlock.LinkTo(finishingBlock, new DataflowLinkOptions { PropagateCompletion = true });
for (var i = 1; i <= 3; i++)
{
Console.WriteLine("Posted " + i*10);
generatorBlock.Post(i*10);
}
generatorBlock.Complete();
finishingBlock.Completion.Wait();
Console.WriteLine("Pipeline is finished");
Console.ReadKey();
}
public async Task TestCountZeroAtCompletion()
{
var cts = new CancellationTokenSource();
var tb = new TransformManyBlock<int, int>(DataflowTestHelpers.ToEnumerable, new ExecutionDataflowBlockOptions() { CancellationToken = cts.Token });
tb.Post(1);
cts.Cancel();
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => tb.Completion);
Assert.Equal(expected: 0, actual: tb.InputCount);
Assert.Equal(expected: 0, actual: tb.OutputCount);
cts = new CancellationTokenSource();
tb = new TransformManyBlock<int, int>(DataflowTestHelpers.ToEnumerable);
tb.Post(1);
((IDataflowBlock)tb).Fault(new InvalidOperationException());
await Assert.ThrowsAnyAsync<InvalidOperationException>(() => tb.Completion);
Assert.Equal(expected: 0, actual: tb.InputCount);
Assert.Equal(expected: 0, actual: tb.OutputCount);
}
public async Task TestPrecanceled()
{
var bb = new TransformManyBlock<int, int>(DataflowTestHelpers.ToEnumerable,
new ExecutionDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true) });
int ignoredValue;
IList<int> ignoredValues;
IDisposable link = bb.LinkTo(DataflowBlock.NullTarget<int>());
Assert.NotNull(link);
link.Dispose();
Assert.False(bb.Post(42));
var t = bb.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
Assert.False(bb.TryReceiveAll(out ignoredValues));
Assert.False(bb.TryReceive(out ignoredValue));
Assert.NotNull(bb.Completion);
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => bb.Completion);
bb.Complete(); // just make sure it doesn't throw
}
public async Task TestExceptions()
{
var tb1 = new TransformManyBlock<int, int>((Func<int, IEnumerable<int>>)(i => { throw new InvalidCastException(); }));
var tb2 = new TransformManyBlock<int, int>((Func<int, Task<IEnumerable<int>>>)(i => { throw new InvalidProgramException(); }));
var tb3 = new TransformManyBlock<int, int>((Func<int, Task<IEnumerable<int>>>)(i => Task.Run((Func<IEnumerable<int>>)(() => { throw new InvalidTimeZoneException(); }))));
var tb4 = new TransformManyBlock<int, int>(i => ExceptionAfter(3));
var tb5 = new TransformManyBlock<int, int>(i => Task.Run(() => ExceptionAfter(3)));
for (int i = 0; i < 3; i++)
{
tb1.Post(i);
tb2.Post(i);
tb3.Post(i);
tb4.Post(i);
tb5.Post(i);
}
await Assert.ThrowsAsync<InvalidCastException>(() => tb1.Completion);
await Assert.ThrowsAsync<InvalidProgramException>(() => tb2.Completion);
await Assert.ThrowsAsync<InvalidTimeZoneException>(() => tb3.Completion);
await Assert.ThrowsAsync<FormatException>(() => tb4.Completion);
await Assert.ThrowsAsync<FormatException>(() => tb5.Completion);
Assert.All(new[] { tb1, tb2, tb3 }, tb => Assert.True(tb.InputCount == 0 && tb.OutputCount == 0));
}
public async Task TestOrdering_Sync_OrderedDisabled(bool trustedEnumeration)
{
// If ordering were enabled, this test would hang.
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, EnsureOrdered = false };
var mres = new ManualResetEventSlim();
var tb = new TransformManyBlock<int, int>(i =>
{
if (i == 0) mres.Wait();
return trustedEnumeration ?
new[] { i } :
Enumerable.Repeat(i, 1);
}, options);
tb.Post(0);
tb.Post(1);
Assert.Equal(1, await tb.ReceiveAsync());
mres.Set();
Assert.Equal(0, await tb.ReceiveAsync());
tb.Complete();
await tb.Completion;
}
public void AsyncNullTest ()
{
var scheduler = new TestScheduler ();
var block = new TransformManyBlock<int, int> (
i => (Task<IEnumerable<int>>)null,
new ExecutionDataflowBlockOptions { TaskScheduler = scheduler });
Assert.IsTrue (block.Post (1));
scheduler.ExecuteAll ();
Assert.IsFalse (block.Completion.Wait (100));
block.Complete ();
Assert.IsTrue (block.Completion.Wait (100));
}
public async void Crawl()
{
var scraper = new TransformManyBlock<LinkInfo, LinkInfo>(
async linkInfo =>
{
var linkInfos = new List<LinkInfo>();
try
{
var response = await linkInfo.Uri.GetResponse();
if (!IsSuccessStatus(response, linkInfo))
return linkInfos;
var html = await response.Content.ReadAsStringAsync();
var doc = new HtmlDocument();
doc.LoadHtml(html);
linkInfos = doc.ToLinkInfos(linkInfo.Uri);
Log(Context.GoodHTML, linkInfo.Uri.AbsoluteUri);
}
catch (Exception error)
{
Log(Context.BadHTML, "Error: {0} (URL: {1})", error.Message, linkInfo.Uri);
}
return linkInfos;
},
new ExecutionDataflowBlockOptions()
{
CancellationToken = cts.Token
});
var fetcher = new ActionBlock<LinkInfo>(
async linkInfo =>
{
try
{
var fileName = linkInfo.GetFileName(linkInfo.Uri, "Downloads");
if (File.Exists(fileName))
{
Log(Context.DupMedia, linkInfo.Uri.AbsoluteUri);
return;
}
var response = await linkInfo.Uri.GetResponse();
if (!IsSuccessStatus(response, linkInfo))
return;
var webStream = await response.Content.ReadAsStreamAsync();
fileName.EnsurePathExists();
using (var fileStream = File.OpenWrite(fileName))
await webStream.CopyToAsync(fileStream);
Log(Context.GoodMedia, linkInfo.Uri.AbsoluteUri);
}
catch (Exception error)
{
Log(Context.BadMedia, "Error: {0} (URL: {1})", error.Message, linkInfo.Uri);
}
},
new ExecutionDataflowBlockOptions()
{
CancellationToken = cts.Token,
MaxDegreeOfParallelism = Environment.ProcessorCount * 12
});
scraper.Completion.SetOnlyOnFaultedCompletion(error => HandleErrors(error));
fetcher.Completion.SetOnlyOnFaultedCompletion(error => HandleErrors(error));
scraper.LinkTo(scraper, new Predicate<LinkInfo>(li => li.Kind == LinkKind.HTML));
scraper.LinkTo(fetcher, new Predicate<LinkInfo>(li => li.Kind == LinkKind.Media));
scraper.Post(new LinkInfo(new Uri("http://www.bbc.com/news/")));
try
{
await Task.WhenAll(scraper.Completion, fetcher.Completion);
}
catch (OperationCanceledException)
{
}
catch (Exception error)
{
Log(Context.Failure, "Error: " + error.Message);
}
if (OnFinished != null)
OnFinished(this, EventArgs.Empty);
}
public void RunTransformManyBlockConformanceTests()
{
bool passed = true;
#region Sync
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory =
options =>
{
TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => new[] { i }, (ExecutionDataflowBlockOptions)options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformManyBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token };
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16;
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test multiple yielded results
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
return Enumerable.Range(0, 10);
});
t.Post(42);
t.Complete();
for (int i = 0; i < 10; i++)
{
localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return new[] { i };
});
for (int i = 0; i < 10; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test using a precanceled token
{
bool localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token };
var t = new TransformManyBlock<int, int>(i => new[] { i }, dbo);
int ignoredValue;
IList<int> ignoredValues;
localPassed &= t.LinkTo(new ActionBlock<int>(delegate { })) != null;
localPassed &= t.SendAsync(42).Result == false;
localPassed &= t.TryReceiveAll(out ignoredValues) == false;
localPassed &= t.Post(42) == false;
localPassed &= t.OutputCount == 0;
localPassed &= t.TryReceive(out ignoredValue) == false;
localPassed &= t.Completion != null;
t.Complete();
}
catch (Exception)
{
localPassed = false;
}
Console.WriteLine(" > {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, IEnumerable<int>>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test reuse of a list and array
{
bool localPassed = true;
foreach (bool bounded in new[] { false, true })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (IList<int> list in new IList<int>[]
{
new int[1],
new List<int>() { 0 },
new Collection<int>() { 0 }
})
{
int nextExpectedValue = 1;
TransformManyBlock<int, int> tmb1 = null;
tmb1 = new TransformManyBlock<int, int>(i =>
{
if (i == 1000)
{
tmb1.Complete();
return (IEnumerable<int>)null;
}
else if (dop == 1)
{
list[0] = i + 1;
return (IEnumerable<int>)list;
}
else if (list is int[])
{
return new int[1] { i + 1 };
}
else if (list is List<int>)
{
return new List<int>() { i + 1 };
}
else return new Collection<int>() { i + 1 };
}, dbo);
TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i =>
{
if (i != nextExpectedValue)
{
localPassed = false;
tmb1.Complete();
}
nextExpectedValue++;
return i;
});
tmb1.LinkTo(tmb2);
tmb2.LinkTo(tmb1);
tmb1.SendAsync(0).Wait();
tmb1.Completion.Wait();
}
}
}
Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test throwing an OCE
{
bool localPassed = true;
foreach (bool bounded in new[] { true, false })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (int mode in new[] { 0, 1, 2 })
{
const int ITERS = 50;
var mres = new ManualResetEventSlim();
var tmb = new TransformManyBlock<int, int>(i =>
{
if (i < ITERS - 1) throw new OperationCanceledException();
if (mode == 0) return new int[] { i };
else if (mode == 1) return new List<int>() { i };
else return Enumerable.Repeat(i, 1);
}, dbo);
var ab = new ActionBlock<int>(i =>
{
if (i != ITERS - 1) localPassed = false;
mres.Set();
});
tmb.LinkTo(ab);
for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait();
mres.Wait();
}
}
}
Console.WriteLine("{0}: Canceled invocation", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
#region Async
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory =
options =>
{
TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }), (ExecutionDataflowBlockOptions)options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformManyBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token };
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16;
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test multiple yielded results
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)Enumerable.Range(0, 10).ToArray()));
t.Post(42);
t.Complete();
for (int i = 0; i < 10; i++)
{
localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return new[] { i };
});
for (int i = 0; i < 10; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
if ((i % 2) == 0) return null;
return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i });
});
for (int i = 0; i < 10; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored when a reordering buffer is in place
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i =>
{
if (i == 0)
{
Task.Delay(1000).Wait();
return null;
}
return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i });
}), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 });
t.Post(0);
t.Post(1);
try
{
localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1;
}
catch
{
localPassed = false;
}
Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the delegate
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the task
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => Task<IEnumerable<int>>.Factory.StartNew(() => { throw new InvalidOperationException(); })));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test reuse of a list and array
{
bool localPassed = true;
foreach (bool bounded in new[] { false, true })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (IList<int> list in new IList<int>[]
{
new int[1],
new List<int>() { 0 },
new Collection<int>() { 0 }
})
{
int nextExpectedValue = 1;
TransformManyBlock<int, int> tmb1 = null;
tmb1 = new TransformManyBlock<int, int>(i =>
{
return Task.Factory.StartNew(() =>
{
if (i == 1000)
{
tmb1.Complete();
return (IEnumerable<int>)null;
}
else if (dop == 1)
{
list[0] = i + 1;
return (IEnumerable<int>)list;
}
else if (list is int[])
{
return new int[1] { i + 1 };
}
else if (list is List<int>)
{
return new List<int>() { i + 1 };
}
else return new Collection<int>() { i + 1 };
});
}, dbo);
TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i =>
{
if (i != nextExpectedValue)
{
localPassed = false;
tmb1.Complete();
}
nextExpectedValue++;
return i;
});
tmb1.LinkTo(tmb2);
tmb2.LinkTo(tmb1);
tmb1.SendAsync(0).Wait();
tmb1.Completion.Wait();
}
}
}
Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test throwing an OCE
{
bool localPassed = true;
foreach (bool bounded in new[] { true, false })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (int mode in new[] { 0, 1, 2 })
{
const int ITERS = 50;
var mres = new ManualResetEventSlim();
var tmb = new TransformManyBlock<int, int>(i =>
{
var cts = new CancellationTokenSource();
return Task.Factory.StartNew(() =>
{
if (i < ITERS - 1)
{
cts.Cancel();
cts.Token.ThrowIfCancellationRequested();
}
if (mode == 0) return new int[] { i };
else if (mode == 1) return new List<int>() { i };
else return Enumerable.Repeat(i, 1);
}, cts.Token);
}, dbo);
var ab = new ActionBlock<int>(i =>
{
if (i != ITERS - 1) localPassed = false;
mres.Set();
});
tmb.LinkTo(ab);
for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait();
mres.Wait();
}
}
}
Console.WriteLine(" > {0}: Canceled invocation", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
Assert.True(passed, "Test failed.");
}
internal static bool TransformManyEnumerableToAction()
{
const int ITERS = 2;
var data = new[] { 1 };
var tm = new TransformManyBlock<int, int>(i => data);
int completedCount = 0;
var c = new ActionBlock<int>(i => completedCount++);
tm.LinkWithCompletion(c);
for (int i = 0; i < ITERS; i++) tm.Post(i);
tm.Complete();
c.Completion.Wait();
return completedCount == ITERS;
}
public void AsyncCancelledTest ()
{
var scheduler = new TestScheduler ();
var block = new TransformManyBlock<int, int> (
i =>
{
var tcs = new TaskCompletionSource<IEnumerable<int>> ();
tcs.SetCanceled ();
return tcs.Task;
}, new ExecutionDataflowBlockOptions { TaskScheduler = scheduler });
Assert.IsTrue (block.Post (1));
scheduler.ExecuteAll ();
Assert.IsFalse (block.Completion.Wait (100));
}
public void NullResultTest ()
{
bool received = false;
var transformMany =
new TransformManyBlock<int, int> (i => (IEnumerable<int>)null);
var action = new ActionBlock<int> (i => received = true);
transformMany.LinkTo (action);
Assert.IsTrue (transformMany.Post (1));
transformMany.Complete ();
Assert.IsTrue (transformMany.Completion.Wait (100));
Assert.IsFalse (received);
}
public async Task TestReserveReleaseConsume()
{
var tb = new TransformManyBlock<int, int>(DataflowTestHelpers.ToEnumerable);
tb.Post(1);
await DataflowTestHelpers.TestReserveAndRelease(tb);
tb = new TransformManyBlock<int, int>(DataflowTestHelpers.ToEnumerable);
tb.Post(2);
await DataflowTestHelpers.TestReserveAndConsume(tb);
}
public async Task TestMultipleYields()
{
const int Messages = 10;
var t = new TransformManyBlock<int, int>(i => Enumerable.Range(0, Messages));
t.Post(42);
t.Complete();
for (int i = 0; i < Messages; i++)
{
Assert.False(t.Completion.IsCompleted);
Assert.Equal(expected: i, actual: await t.ReceiveAsync());
}
await t.Completion;
}
public async static Task<Animat> SeedAsync(ScenarioSpecies species, GeoRect geoRect, Bathymetry bathymetry)
{
var yxzFileName = Path.Combine(Path.GetTempPath(), Path.GetFileNameWithoutExtension(Path.GetRandomFileName()) + ".txt");
bathymetry.ToYXZ(yxzFileName, -1);
var mbs = new C3mbs();
mbsRESULT mbsResult;
if (mbsRESULT.OK != (mbsResult = mbs.SetOutputDirectory(Path.GetTempPath())))
throw new AnimatInterfaceMMBSException("SetOutputDirectory Error:" + mbs.ResultToTc(mbsResult));
var config = mbs.GetConfiguration();
config.enabled = false; // binary output enabled/disabled
config.durationLess = true; // make sure we're in durationless mode.
mbs.SetConfiguration(config);
mbsResult = mbs.LoadBathymetryFromTextFile(yxzFileName);
if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException("Bathymetry failed to load: " + mbs.ResultToTc(mbsResult));
mbsResult = mbs.AddSpecies(species.SpeciesDefinitionFilePath);
if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException(string.Format("C3mbs::AddSpecies FATAL error {0} for species {1}", mbs.ResultToTc(mbsResult), species.SpeciesDefinitionFilePath));
var bounds = new GeoArray(geoRect.NorthWest, geoRect.NorthEast, geoRect.SouthEast, geoRect.SouthWest, geoRect.NorthWest);
var result = new Animat { ScenarioSpecies = species };
var area = bounds.Area;
//Debug.WriteLine("Area: {0}",area);
var transformManyBlock = new TransformManyBlock<int, Geo<float>>(count =>
{
var geos = new List<Geo<float>>();
for (var i = 0; i < count; i++)
{
var location = bounds.RandomLocationWithinPerimeter();
var depth = bathymetry.Samples.GetNearestPointAsync(location).Result.Data;
mbsRESULT retval;
lock (mbs) retval = mbs.AddIndividualAnimat(0, new mbsPosition { latitude = location.Latitude, longitude = location.Longitude, depth = 0 });
if (mbsRESULT.OK == retval) geos.Add(new Geo<float>(location.Latitude, location.Longitude, (float)(depth * Random.NextDouble())));
}
return geos;
}, new ExecutionDataflowBlockOptions
{
TaskScheduler = TaskScheduler.Default,
BoundedCapacity = -1,
MaxDegreeOfParallelism = -1,
});
var bufferBlock = new BufferBlock<Geo<float>>();
transformManyBlock.LinkTo(bufferBlock);
var population = (int)Math.Round(area * species.PopulationDensity);
result.TotalAnimats = population;
const int blockSize = 100;
while (population > 0)
{
transformManyBlock.Post(population > blockSize ? blockSize : population);
population -= blockSize;
}
transformManyBlock.Complete();
await transformManyBlock.Completion;
//mbsResult = mbs.InitializeRun();
//if (mbsRESULT.OK == mbsResult) while (mbsRUNSTATE.INITIALIZING == mbs.GetRunState()) Thread.Sleep(1);
//else throw new AnimatInterfaceMMBSException("C3mbs::Initialize FATAL error " + mbs.ResultToTc(mbsResult));
mbsResult = mbs.FinishRun();
if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException("C3mbs::FinishRun FATAL error " + mbs.ResultToTc(mbsResult));
IList<Geo<float>> animatGeos;
if (bufferBlock.TryReceiveAll(out animatGeos))
result.Locations.AddRange(animatGeos);
return result;
}
public async Task TestOrdering()
{
const int iters = 9999;
foreach (int mmpt in new[] { DataflowBlockOptions.Unbounded, 1 })
foreach (int dop in new[] { 1, 2, DataflowBlockOptions.Unbounded })
{
var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, MaxMessagesPerTask = mmpt };
var tb = new TransformManyBlock<int, int>(i => new[] { i, i + 1, i + 2 }, options);
for (int i = 0; i < iters; i += 3)
{
Assert.True(tb.Post(i));
}
for (int i = 0; i < iters; i++)
{
Assert.Equal(expected: i, actual: await tb.ReceiveAsync());
}
tb.Complete();
await tb.Completion;
}
}
public async static Task<Animat> SeedAsyncWithout3MB(ScenarioSpecies species, GeoRect geoRect, Bathymetry bathymetry)
{
var bounds = new GeoArray(geoRect.NorthWest, geoRect.NorthEast, geoRect.SouthEast, geoRect.SouthWest, geoRect.NorthWest);
var result = new Animat { ScenarioSpecies = species };
var area = bounds.Area;
//Debug.WriteLine("Area: {0}",area);
var transformManyBlock = new TransformManyBlock<int, Geo<float>>(count =>
{
var geos = new List<Geo<float>>();
for (var i = 0; i < count; i++)
{
var location = bounds.RandomLocationWithinPerimeter();
var depth = bathymetry.Samples.GetNearestPointAsync(location).Result.Data;
if (depth < -50) geos.Add(new Geo<float>(location.Latitude, location.Longitude, (float)(depth * Random.NextDouble())));
}
return geos;
}, new ExecutionDataflowBlockOptions
{
TaskScheduler = TaskScheduler.Default,
BoundedCapacity = -1,
MaxDegreeOfParallelism = -1,
});
var bufferBlock = new BufferBlock<Geo<float>>();
transformManyBlock.LinkTo(bufferBlock);
var population = (int)Math.Round(area * species.PopulationDensity);
result.TotalAnimats = population;
const int blockSize = 100;
while (population > 0)
{
transformManyBlock.Post(population > blockSize ? blockSize : population);
population -= blockSize;
}
transformManyBlock.Complete();
await transformManyBlock.Completion;
IList<Geo<float>> animatGeos;
if (bufferBlock.TryReceiveAll(out animatGeos))
result.Locations.AddRange(animatGeos);
return result;
}
/// <summary>
/// Purges all blobs set to expire prior to the specified date.
/// </summary>
/// <param name="deleteBlobsExpiringBefore">
/// All blobs scheduled to expire prior to this date will be purged. The default value
/// is interpreted as <see cref="DateTime.UtcNow"/>.
/// </param>
/// <returns>The task representing the asynchronous operation.</returns>
public async Task PurgeBlobsExpiringBeforeAsync(DateTime deleteBlobsExpiringBefore = default(DateTime))
{
if (deleteBlobsExpiringBefore == default(DateTime))
{
deleteBlobsExpiringBefore = DateTime.UtcNow;
}
Requires.Argument(deleteBlobsExpiringBefore.Kind == DateTimeKind.Utc, "expirationUtc", "UTC required.");
var searchExpiredDirectoriesBlock = new TransformManyBlock<CloudBlobContainer, CloudBlobDirectory>(
async c =>
{
var results = await c.ListBlobsSegmentedAsync();
return from directory in results.OfType<CloudBlobDirectory>()
let expires = DateTime.Parse(directory.Uri.Segments[directory.Uri.Segments.Length - 1].TrimEnd('/'))
where expires < deleteBlobsExpiringBefore
select directory;
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 4,
});
var deleteDirectoryBlock = new TransformManyBlock<CloudBlobDirectory, CloudBlockBlob>(
async directory =>
{
var results = await directory.ListBlobsSegmentedAsync();
return results.OfType<CloudBlockBlob>();
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 2,
BoundedCapacity = 4,
});
var deleteBlobBlock = new ActionBlock<CloudBlockBlob>(
blob => blob.DeleteAsync(),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 4,
BoundedCapacity = 100,
});
searchExpiredDirectoriesBlock.LinkTo(deleteDirectoryBlock, new DataflowLinkOptions { PropagateCompletion = true });
deleteDirectoryBlock.LinkTo(deleteBlobBlock, new DataflowLinkOptions { PropagateCompletion = true });
searchExpiredDirectoriesBlock.Post(this.container);
searchExpiredDirectoriesBlock.Complete();
await deleteBlobBlock.Completion;
}
public static async Task PurgeExpiredAsync(CloudBlobContainer inboxContainer)
{
Requires.NotNull(inboxContainer, "inboxContainer");
var deleteBlobsExpiringBefore = DateTime.UtcNow;
int purgedBlobCount = 0;
var searchExpiredBlobs = new TransformManyBlock<CloudBlobContainer, ICloudBlob>(
async c =>
{
try
{
var results = await c.ListBlobsSegmentedAsync(
string.Empty,
useFlatBlobListing: true,
pageSize: 50,
details: BlobListingDetails.Metadata,
options: new BlobRequestOptions(),
operationContext: null);
return from blob in results.OfType<ICloudBlob>()
let expires = DateTime.Parse(blob.Metadata[ExpirationDateMetadataKey])
where expires < deleteBlobsExpiringBefore
select blob;
}
catch (StorageException ex)
{
var webException = ex.InnerException as WebException;
if (webException != null)
{
var httpResponse = (HttpWebResponse)webException.Response;
if (httpResponse.StatusCode == HttpStatusCode.NotFound)
{
// it's legit that some tests never created the container to begin with.
return Enumerable.Empty<ICloudBlob>();
}
}
throw;
}
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 4,
});
var deleteBlobBlock = new ActionBlock<ICloudBlob>(
blob =>
{
Interlocked.Increment(ref purgedBlobCount);
return blob.DeleteAsync();
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 4,
BoundedCapacity = 100,
});
searchExpiredBlobs.LinkTo(deleteBlobBlock, new DataflowLinkOptions { PropagateCompletion = true });
searchExpiredBlobs.Post(inboxContainer);
searchExpiredBlobs.Complete();
await deleteBlobBlock.Completion;
}
