private async Task Start(string path)
{
var uiScheduler = TaskScheduler.FromCurrentSynchronizationContext();
dirToFilesBlock = new TransformManyBlock <string, string>((Func <string, IEnumerable <string> >)(GetFileSystemItems), new ExecutionDataflowBlockOptions()
{
CancellationToken = ct
});
fileActionBlock = new ActionBlock <string>((Action <string>)ProcessFile, new ExecutionDataflowBlockOptions()
{
CancellationToken = ct, TaskScheduler = uiScheduler
});
// Order of LinkTo's important here!
dirToFilesBlock.LinkTo(dirToFilesBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
}, IsDirectory);
dirToFilesBlock.LinkTo(fileActionBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
}, IsRequiredDocType);
// Kick off the recursion.
dirToFilesBlock.Post(path);
await ProcessingIsComplete();
dirToFilesBlock.Complete();
await Task.WhenAll(dirToFilesBlock.Completion, fileActionBlock.Completion);
}
internal void start()
{
var consumerBlock = Consumer("consumer 1");
var consumer2Block = Consumer("\t\tconsumer 2");
var producerBlock = new TransformManyBlock <int, int>(x => Enumerable.Range(0, x));
producerBlock.LinkTo(consumerBlock, new DataflowLinkOptions
{
PropagateCompletion = true,
});
producerBlock.LinkTo(consumer2Block, new DataflowLinkOptions
{
PropagateCompletion = true
});
if (!producerBlock.Post(10))
{
Console.WriteLine("Failed Post");
}
producerBlock.Complete();
producerBlock.Completion.ContinueWith(p => print_status(p, "producer"));
consumerBlock.Completion.ContinueWith(p => print_status(p, "consumer 1"));
consumer2Block.Completion.ContinueWith(p => print_status(p, "\t\tconsumer 2"));
}
public LoopDataflow1()
{
InputMessageBlock = new TransformBlock <Message, Message>(async msg => await InputMessage(msg));
HandleMessageBlock = new TransformManyBlock <Message, Message>(async msg => await HandleMessage(msg));
OutputMessageBlock = new ActionBlock <Message>(msg => OutputMessage(msg));
var linkOptions = new DataflowLinkOptions()
{
PropagateCompletion = false
};
InputMessageBlock.LinkTo(HandleMessageBlock, linkOptions);
HandleMessageBlock.LinkTo(OutputMessageBlock, linkOptions, msg => msg.WasProcessed == true);
HandleMessageBlock.LinkTo(HandleMessageBlock, linkOptions, msg => msg.WasProcessed == false);
InputMessageBlock.Completion.ContinueWith(async tsk => {
await HandleMessageIsComplete();
HandleMessageBlock.Complete();
});
HandleMessageBlock.Completion.ContinueWith(tsk => {
OutputMessageBlock.Complete();
});
//DebuggingLoop();
}
public ImageProcessor()
{
imageCollectionBlock =
new TransformManyBlock <string, string>(
(Func <string, IEnumerable <string> >)FindImagesInDirectory);
loadAndToGreyBlock = new TransformBlock <string, BitmapSource>((Func <string, BitmapSource>)LoadAndToGrayScale,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 4,
});
publishImageBlock = new ActionBlock <BitmapSource>((Action <BitmapSource>)PublishImage,
new ExecutionDataflowBlockOptions()
{
TaskScheduler = TaskScheduler.FromCurrentSynchronizationContext()
});
//zipBlock.LinkTo(zipBlock, u => u.AbsolutePath.EndsWith(".zip"));
//zipBlock.LinkTo(loadAndToGreyBlock);
//imageCollectionBlock.LinkTo(zipBlock,u => u.AbsoluteUri.EndsWith(".zip"));
imageCollectionBlock.LinkTo(loadAndToGreyBlock, u => u.EndsWith(".jpg"));
imageCollectionBlock.LinkTo(imageCollectionBlock);
loadAndToGreyBlock.LinkTo(publishImageBlock);
}
public DirectoryWalkerLite(Action <string> fileAction)
{
directoryBrowseBlock = new TransformManyBlock <string, string>((Func <string, IEnumerable <string> >)(GetFilesInDirectory));
fileActionBlock = new ActionBlock <string>(fileAction);
directoryBrowseBlock.LinkTo(directoryBrowseBlock, Directory.Exists);
directoryBrowseBlock.LinkTo(fileActionBlock);
}
public DirectoryWalker(Action <string> fileAction)
{
_directoryBrowseBlock =
new TransformManyBlock <string, string>((Func <string, IEnumerable <string> >)GetFilesInDirectory);
_fileActionBlock = new ActionBlock <string>(fileAction);
_directoryBrowseBlock.LinkTo(_directoryBrowseBlock, Directory.Exists);
_directoryBrowseBlock.LinkTo(_fileActionBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
}
static async Task Main(string[] args)
{
var executionOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
};
var loader = new TransformBlock <string, string?>(LoadPage, executionOptions);
var searcher = new TransformManyBlock <string?, Uri>(SearchForReferences, executionOptions);
var fetcher = new TransformBlock <Uri, byte[]?>(FetchReferences, executionOptions);
var hasher = new TransformBlock <byte[]?, byte[]>(HashAlgorithm.ComputeHash, executionOptions);
var converter = new TransformBlock <byte[], string>(TransformBytesToHexidecimal, executionOptions);
var printer = new ActionBlock <string>(PrintHash);
var linkOptions = new DataflowLinkOptions
{
PropagateCompletion = true
};
loader.LinkTo(searcher, linkOptions, content => content is object);
loader.LinkTo(DataflowBlock.NullTarget <string?>());
searcher.LinkTo(fetcher, linkOptions, uri => Regex.IsMatch(uri.Scheme, "^https?"));
searcher.LinkTo(DataflowBlock.NullTarget <Uri>());
fetcher.LinkTo(hasher, linkOptions, content => content is object);
fetcher.LinkTo(DataflowBlock.NullTarget <byte[]?>());
hasher.LinkTo(converter, linkOptions);
converter.LinkTo(printer, linkOptions);
var pages = new string[]
{
"https://arstechnica.com/",
"https://www.reddit.com/",
"https://www.anandtech.com/",
"https://www.theverge.com/",
"https://stigvoss.dk"
};
foreach (var page in pages)
{
await loader.SendAsync(page);
}
loader.Complete();
await printer.Completion;
}
public CsvImporter()
{
databaseQueryBlock = new TransformManyBlock <string, object[]>((Func <string, IEnumerable <object[]> >)ExecuteQuery);
rowTGoLedgerBlock = new TransformManyBlock <object[], ILedgerEntry>((Func <object[], IEnumerable <ILedgerEntry> >)MapDatabaseRowToObject);
debitBlock = new ActionBlock <ILedgerEntry>((Action <ILedgerEntry>)WriteDebitEntry);
creditBlock = new ActionBlock <ILedgerEntry>((Action <ILedgerEntry>)WriteCreditEntry);
unknownLedgerEntryBlock = new ActionBlock <ILedgerEntry>((Action <ILedgerEntry>)LogUnknownLedgerEntryType);
databaseQueryBlock.LinkTo(rowTGoLedgerBlock);
rowTGoLedgerBlock.LinkTo(debitBlock, le => le.IsDebit); //if IsDebit
rowTGoLedgerBlock.LinkTo(creditBlock, le => le.IsCredit); // else if IsCredit
rowTGoLedgerBlock.LinkTo(unknownLedgerEntryBlock); // else
}
public void Start(CoreValues cv)
{
CoreValues = cv;
MyCamera = new Camera(cv.Origin, cv.LookAt, cv.VecUp, cv.FovY, cv.AspectRatio, cv.Aperture, cv.DistToFocus);
Image = new Vec3[cv.Width * cv.Height];
World = HitableList.DefaultWorld;
Count = cv.Width * cv.Height;
PixelFloats = new TransformManyBlock <int, InfoStruct>(GetSampleRays);
MergeColors = new ActionBlock <InfoStruct>(DoMergeColors, new ExecutionDataflowBlockOptions {
EnsureOrdered = false, MaxDegreeOfParallelism = -1
});
//writeImage = new ActionBlock<InfoStruct>(DoWriteImage, new ExecutionDataflowBlockOptions { EnsureOrdered = false, MaxDegreeOfParallelism = -1 });
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
PixelFloats.LinkTo(MergeColors, linkOptions);
//MergeColors.LinkTo(writeImage, linkOptions);
PixelFloats.Post(0);
PixelFloats.Complete();
MergeColors.Completion.Wait();
DoSaveImage();
}
private (ITargetBlock <string>, Task) CreatePipeline()
{
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
TransformManyBlock <object, IImage> fetchBlock = new TransformManyBlock <object, IImage>(_ => _functions.Fetch(_options.Value.Location), new ExecutionDataflowBlockOptions {
BoundedCapacity = 2, MaxDegreeOfParallelism = 1
});
TransformManyBlock <IImage, IImage> detectFacesBlock = new TransformManyBlock <IImage, IImage>(source => _functions.ExtractFaces(source));
TransformManyBlock <IImage, IRecognition> recogniseFacesBlock = new TransformManyBlock <IImage, IRecognition>(source => _functions.RecogniseFaces(source));
TransformBlock <IRecognition, IPersistedRecognition> persistRecognitionBlock = new TransformBlock <IRecognition, IPersistedRecognition>(recognition => _functions.PersistRecognition(recognition), new ExecutionDataflowBlockOptions {
BoundedCapacity = 10, MaxDegreeOfParallelism = 1
});
ActionBlock <IPersistedRecognition> notifyRecognitionBlock = new ActionBlock <IPersistedRecognition>(recognition => _functions.NotifyRecognition(recognition), new ExecutionDataflowBlockOptions {
BoundedCapacity = 10, MaxDegreeOfParallelism = 1
});
fetchBlock.LinkTo(detectFacesBlock, linkOptions);
detectFacesBlock.LinkTo(recogniseFacesBlock, linkOptions);
recogniseFacesBlock.LinkTo(persistRecognitionBlock, linkOptions);
persistRecognitionBlock.LinkTo(notifyRecognitionBlock, linkOptions);
return(fetchBlock, notifyRecognitionBlock.Completion);
}
protected override async Task ExecuteNodeTreeAsync(ExecutionContext context, ObjectExecutionNode rootNode)
{
//Options
var blockOptions = new ExecutionDataflowBlockOptions
{
CancellationToken = context.CancellationToken,
MaxDegreeOfParallelism = 1024
};
//Execute Block
var block = new TransformManyBlock <Job, Job>(
async job =>
{
var node = await ExecuteNodeAsync(job.Context, job.Node).ConfigureAwait(false);
var childJobs = (job.Node as IParentExecutionNode)?
.GetChildNodes()
.Select(child => job.CreateChildJob(job.Context, child))
.ToArray();
job.Complete();
return(childJobs);
},
blockOptions);
//Link to self
block.LinkTo(block);
//Start
var rootJob = new Job(context, rootNode);
await block.SendAsync(rootJob);
//Wait until done
await rootJob.Completion;
}
public FaceDecisionHelper(int maxDegreeOfParallelism, bool dropWhenFull)
{
ProcessingBlock = new TransformManyBlock <IFaceStream, bool>(
transform: Mock.DetermineAccessAsync,
dataflowBlockOptions: new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = maxDegreeOfParallelism,
BoundedCapacity = dropWhenFull ? maxDegreeOfParallelism : DataflowBlockOptions.Unbounded,
});
DecisionBlock = new ActionBlock <bool>(decision =>
{
bool?finalDecision = null;
if (!decision)
{
++_failureCount;
}
if (decision || _failureCount > 7)
{
finalDecision = decision;
}
if (finalDecision.HasValue)
{
DecisionAvailable?.Invoke(this, finalDecision.Value);
ProcessingBlock.Complete();
}
});
ProcessingBlock.LinkTo(DecisionBlock);
ProcessingBlock.Completion.ContinueWith(delegate { DecisionBlock.Complete(); });
}
public IEnumerable <IMetaData> TransformManyBlockUsage(string stringToSplit)
{
Console.WriteLine($"Inside {nameof(TplDataflow1ExecutionBlocksController)} - {nameof(TransformManyBlockUsage)}");
// Create the members of the pipeline.
var transformManyBlockSplitAnInputStringIntoArray = new TransformManyBlock <string, string>(input =>
Functions.SplitAnInputStringIntoArray(input, SplitterSeparator)
);
var transformBlockCreateASingleMedatadataFromAString = new TransformBlock <string, IMetaData>(stringInput =>
Functions.CreateASingleMedatadataFromAString(stringInput)
);
// Connect the dataflow blocks to form a pipeline.
transformManyBlockSplitAnInputStringIntoArray.LinkTo(transformBlockCreateASingleMedatadataFromAString, DataflowOptions.LinkOptions);
// Start TransformManyBlockUsage pipeline with the input values.
transformManyBlockSplitAnInputStringIntoArray.Post(stringToSplit);
// Mark the head of the pipeline as complete.
transformManyBlockSplitAnInputStringIntoArray.Complete();
// Equivalent of transformManyBlockSplitAnInputStringIntoArray.OutputCount
var ouputCount = stringToSplit?.Split(SplitterSeparator, StringSplitOptions.RemoveEmptyEntries).Length ?? 0;
for (var i = 0; i < ouputCount; i++)
{
yield return(transformBlockCreateASingleMedatadataFromAString.Receive());
}
}
public Task Generate(int maxDegreeOfParallelism, string destination, params string[] paths)
{
var options = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxDegreeOfParallelism
};
var openFile = new TransformBlock <string, string>(async filePath => await File.ReadAllTextAsync(filePath), options);
var generateTests = new TransformManyBlock <string, TestElement>(fileCode => TestGenerator.TestGenerator.GenerateTests(fileCode), options);
var writeToFile = new ActionBlock <TestElement>(async testElement => await File.WriteAllTextAsync(
Path.Combine(destination, testElement.TestName) + ".cs", testElement.TestCode));
var newOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
openFile.LinkTo(generateTests, newOptions);
generateTests.LinkTo(writeToFile, newOptions);
foreach (string path in paths)
{
openFile.Post(path);
}
openFile.Complete();
return(writeToFile.Completion);
}
public static async Task RunSampleOne(Func <int, Task <User[]> > getUsers, Func <User, Task> writeUser, int pageCount)
{
var getDataBlock = new TransformManyBlock <int, User>(
transform: async(pageNumber) => await getUsers(pageNumber),
dataflowBlockOptions: new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 3
});
var writeDataBlock = new ActionBlock <User>(
action: async(user) => await writeUser(user),
dataflowBlockOptions: new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = 1
});
getDataBlock.LinkTo(writeDataBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
for (var i = 1; i <= pageCount; i++)
{
getDataBlock.Post(i);
}
getDataBlock.Complete();
await writeDataBlock.Completion;
}
public CsvImporter()
{
var databaseQueryBlock =
new TransformManyBlock <string, object[]>((Func <string, IEnumerable <object[]> >)ExecuteQuery);
var rowToGoLedgerBlock = new TransformManyBlock <object[], ILedgerEntry>(
(Func <object[], IEnumerable <ILedgerEntry> >)MapDatabaseRowToObject);
var debitBlock = new ActionBlock <ILedgerEntry>((Action <ILedgerEntry>)WriteDebitEntry);
var creditBlock = new ActionBlock <ILedgerEntry>((Action <ILedgerEntry>)WriteCreditEntry);
var unknownLedgerEntryBlock = new ActionBlock <ILedgerEntry>((Action <ILedgerEntry>)LogUnknownLedgerEntryType);
databaseQueryBlock.LinkTo(rowToGoLedgerBlock);
rowToGoLedgerBlock.LinkTo(debitBlock, entry => entry.IsDebit); // If is debit
rowToGoLedgerBlock.LinkTo(creditBlock, entry => entry.IsCredit); // If is credit
rowToGoLedgerBlock.LinkTo(unknownLedgerEntryBlock); // Other cases
}
public async Task TestMultiplePublishersWithPropagateCompletion()
{
var block1 = new TransformManyBlock <string, char>(x => x.ToCharArray());
var block2 = new TransformManyBlock <string, char>(x => x.ToCharArray());
var target = new BufferBlock <char>();
var propagate = new DataflowLinkOptions {
PropagateCompletion = true
};
block1.LinkTo(target, propagate);
block2.LinkTo(target, propagate);
block1.Post("a");
// This propagates completion through to target
block1.Complete();
await AssertCompletes(block1.Completion);
Assert.IsTrue(await target.OutputAvailableAsync(), "target should have message waiting");
Assert.IsTrue(!target.Completion.IsCompleted, "target won't be complete until buffer empty");
// The target won't receive this, because it's already started completing (via propagation)
block2.Post("b");
block2.Complete();
// Pulling this out of the buffer allows the target to complete
Assert.AreEqual('a', target.Receive());
// This sholud happen pretty quickly
await AssertCompletes(target.Completion);
}
public async Task TestCircularLinking()
{
const int Iters = 200;
foreach (bool sync in DataflowTestHelpers.BooleanValues)
{
var tcs = new TaskCompletionSource <bool>();
Func <int, IEnumerable <int> > body = i => {
if (i >= Iters)
{
tcs.SetResult(true);
}
return(Enumerable.Repeat(i + 1, 1));
};
TransformManyBlock <int, int> tb = sync ?
new TransformManyBlock <int, int>(body) :
new TransformManyBlock <int, int>(i => Task.Run(() => body(i)));
using (tb.LinkTo(tb))
{
tb.Post(0);
await tcs.Task;
tb.Complete();
}
}
}
private static DataFlowPipeline <ITestGenerator> TestExecutionPipeline()
{
var copyBuffer = new BufferBlock <ITestGenerator>();
var expandBlock = new TransformManyBlock <ITestGenerator, ITest>(
generator => generator.GenerateTests(),
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1
});
var processBlock = new ActionBlock <ITest>(
test => test.PerformTest(),
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 1
});
copyBuffer.LinkTo(
expandBlock, new DataflowLinkOptions
{
PropagateCompletion = true
});
expandBlock.LinkTo(
processBlock, new DataflowLinkOptions
{
PropagateCompletion = true
});
return(new DataFlowPipeline <ITestGenerator>(copyBuffer, processBlock.Completion));
}
public Task Generate(List <string> inputFiles, string outputPath, int maxReadText, int maxGenerateTest, int maxWriteText)
{
var ExecReaxOption = new ExecutionDataflowBlockOptions();
ExecReaxOption.MaxDegreeOfParallelism = maxReadText;
var ExecGenerateOption = new ExecutionDataflowBlockOptions();
ExecGenerateOption.MaxDegreeOfParallelism = maxGenerateTest;
var ExecWriteOption = new ExecutionDataflowBlockOptions();
ExecWriteOption.MaxDegreeOfParallelism = maxWriteText;
var readText = new TransformBlock <string, string>(async textReadPath => await ReadTextAsync(textReadPath), ExecReaxOption);
var generateTests = new TransformManyBlock <string, Tests>(textCode => TestsGenerator.GenerateTests(textCode), ExecGenerateOption);
var writeText = new ActionBlock <Tests>(async test => await WriteTextAsync(Path.Combine(outputPath, test.FileName), test.TestCode), ExecWriteOption);
var linkOptions = new DataflowLinkOptions();
linkOptions.PropagateCompletion = true;
readText.LinkTo(generateTests, linkOptions);
generateTests.LinkTo(writeText, linkOptions);
foreach (string file in inputFiles)
{
readText.Post(file);
}
readText.Complete();
return(writeText.Completion);
}
public async Task Generate()
{
DataflowLinkOptions linkOptions = new DataflowLinkOptions
{
PropagateCompletion = true
};
ExecutionDataflowBlockOptions readOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = config.ReadThreadCount
};
ExecutionDataflowBlockOptions writeOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = config.WriteThreadCount
};
ExecutionDataflowBlockOptions processOptions = new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = config.ProcessThreadCount
};
var readTransform = new TransformBlock <string, Task <string> >((readPath) => config.Reader.ReadTextAsync(readPath), readOptions);
var sourceToTestfileTransform = new TransformManyBlock <Task <string>, PathContentPair>((readSourceTask) => config.TemplateGenerator.Generate(readSourceTask.Result), processOptions);
var writeAction = new ActionBlock <PathContentPair>((pathTextPair) => config.Writer.WriteTextAsync(pathTextPair).Wait(), writeOptions);
readTransform.LinkTo(sourceToTestfileTransform, linkOptions);
sourceToTestfileTransform.LinkTo(writeAction, linkOptions);
foreach (string readPath in config.ReadPaths)
{
await readTransform.SendAsync(readPath);
}
readTransform.Complete();
await writeAction.Completion;
}
public async Task TestLinkToOptionsAsyncEnumerable()
{
const int Messages = 1;
foreach (bool append in DataflowTestHelpers.BooleanValues)
{
var tb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToAsyncEnumerable);
var values = new int[Messages];
var targets = new ActionBlock <int> [Messages];
for (int i = 0; i < Messages; i++)
{
int slot = i;
targets[i] = new ActionBlock <int>(item => values[slot] = item);
tb.LinkTo(targets[i], new DataflowLinkOptions {
MaxMessages = 1, Append = append
});
}
tb.PostRange(0, Messages);
tb.Complete();
await tb.Completion;
for (int i = 0; i < Messages; i++)
{
Assert.Equal(
expected: append ? i : Messages - i - 1,
actual: values[i]);
}
}
}
internal static void Run()
{
const string w1 = "Hello";
const string w2 = "World";
// Create a TransformManyBlock<string, char> object that splits
// a string into its individual characters.
var transformManyBlock = new TransformManyBlock <string, char>(s => s.ToCharArray());
// Chained buffer block
var buffer = new BufferBlock <char>();
transformManyBlock.LinkTo(buffer);
transformManyBlock.Completion.ContinueWith(_ => buffer.Complete());
// Post two messages to the first block.
transformManyBlock.SendAsync(w1);
transformManyBlock.SendAsync(w2);
transformManyBlock.Complete();
transformManyBlock.Completion.Wait();
// Receive all output values from the last block.
IList <char> chars;
while (!buffer.TryReceiveAll(out chars))
{
// Needed if we don't wait for transformManyBlock.Completion
Console.Write(".");
Thread.SpinWait(10000);
}
Console.WriteLine(chars == null ? "null" : string.Join(", ", chars));
}
public async Task FindProjectJsonAsync()
{
TransformManyBlock <GitHubRepo, SearchResult> repoSearchBlock = new TransformManyBlock <GitHubRepo, SearchResult>(repo => SearchRepoAsync(repo),
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount * 4
//MaxDegreeOfParallelism = 1
});
ActionBlock <SearchResult> downloadFileBlock = new ActionBlock <SearchResult>(DownloadFileAsync, new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount * 4
//MaxDegreeOfParallelism = 1
});
repoSearchBlock.LinkTo(downloadFileBlock, new DataflowLinkOptions()
{
PropagateCompletion = true
});
foreach (var repo in _storage.GetAllRepos())
{
if (_cancelToken.IsCancellationRequested)
{
break;
}
repoSearchBlock.Post(repo);
}
repoSearchBlock.Complete();
await downloadFileBlock.Completion;
}
static void Main(string[] args)
{
var b1 = new TransformBlock <int, string>(i => new string((char)('a' + i), i));
var b2 = new TransformManyBlock <string, char>(i => i);
var b3 = new ActionBlock <char>(async c =>
{
await Task.Delay(1000).ConfigureAwait(false);
Console.Write($"{c}, ");
});
var b4 = new ActionBlock <char>(async c =>
{
await Task.Delay(500).ConfigureAwait(false);
Trace.Write($"{c}, ");
});
var b5 = new BroadcastBlock <char>(i => i);
b1.LinkTo(b2);
b2.LinkTo(b5);
b5.LinkTo(b3);
b5.LinkTo(b4);
for (int i = 1; i <= 10; i++)
{
b1.Post(i);
}
Console.WriteLine($@"
b1: in {b1.InputCount} out {b1.OutputCount}
b2: in {b2.InputCount} out {b2.OutputCount}
b3: in {b3.InputCount}
b4: in {b4.InputCount}
");
Console.ReadKey();
}
public override async Task ProcessAsync()
{
var settings = new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 10,
};
var listFilesBlock = new TransformManyBlock <string, string>(_fileSystem.GetFileNames, settings);
var getVehiclesBlock = new TransformManyBlock <string, IVehicle>(GetVehicles, settings);
var transformBlock = new TransformBlock <IVehicle, Truck>(TransformAsync, settings);
var doubleBlock = new TransformBlock <Truck, Truck>(DoubleDoorsAsync, settings);
var batchBlock = new BatchBlock <Truck>(10);
var saveBlock = new ActionBlock <IEnumerable <Truck> >(SaveTrucksAsync, settings);
DataflowLinkOptions linkOptions = new DataflowLinkOptions()
{
PropagateCompletion = true
};
listFilesBlock.LinkTo(getVehiclesBlock, linkOptions);
getVehiclesBlock.LinkTo(transformBlock, linkOptions);
transformBlock.LinkTo(doubleBlock, linkOptions);
doubleBlock.LinkTo(batchBlock, linkOptions);
batchBlock.LinkTo(saveBlock, linkOptions);
await listFilesBlock.SendAsync(_directory);
listFilesBlock.Complete();
await saveBlock.Completion;
}
public async Task Start()
{
Reader fileReader = new Reader();
Writer fileWriter = new Writer(fFolder);
var reader = new TransformBlock <string, string>(filename => fileReader.ReadFromFile(filename),
new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = fMaxDegreeOfParallelism
});
var transformer = new TransformManyBlock <string, string>(data => GeneratorTestClasses.Start(data),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = fMaxDegreeOfParallelism
});
var writer = new ActionBlock <string>(text => fileWriter.WritetoFile(text),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = fMaxDegreeOfParallelism
});
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
reader.LinkTo(transformer, linkOptions);
transformer.LinkTo(writer, linkOptions);
foreach (string filename in fFilenames)
{
await reader.SendAsync(filename);
}
reader.Complete();
writer.Completion.Wait();
}
public async Task TestPrecanceled()
{
var bb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable,
new ExecutionDataflowBlockOptions {
CancellationToken = new CancellationToken(canceled: true)
});
int ignoredValue;
IList <int> ignoredValues;
IDisposable link = bb.LinkTo(DataflowBlock.NullTarget <int>());
Assert.NotNull(link);
link.Dispose();
Assert.False(bb.Post(42));
var t = bb.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
Assert.False(bb.TryReceiveAll(out ignoredValues));
Assert.False(bb.TryReceive(out ignoredValue));
Assert.NotNull(bb.Completion);
await Assert.ThrowsAnyAsync <OperationCanceledException>(() => bb.Completion);
bb.Complete(); // just make sure it doesn't throw
}
public Task Generate(int maxDegreeOfParallelism, string resultPath, params string[] files)
{
var execOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = maxDegreeOfParallelism
};
var openFile = new TransformBlock <string, string>(async path => await File.ReadAllTextAsync(path), execOptions);
var generateTests = new TransformManyBlock <string, TestUnit>(file => TestsGenerator.Generate(file), execOptions);
var writeFile = new ActionBlock <TestUnit>(
async testUnit => await File.WriteAllTextAsync(
Path.Combine(resultPath, testUnit.Name) + ".cs", testUnit.Test),
execOptions);
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
openFile.LinkTo(generateTests, linkOptions);
generateTests.LinkTo(writeFile, linkOptions);
foreach (string path in files)
{
openFile.Post(path);
}
openFile.Complete();
return(writeFile.Completion);
}
public ITargetBlock <string> Build(DataflowBuilderConfiguration config)
{
var sinkCompletions = new List <Task>();
var fileParser =
new TransformManyBlock <string, UnparsedRecord>(fileLocation => config.FileParser.Parse(fileLocation));
var invalidParsedRecordExtractor = new TransformBlock <Validity <ParsedRecord>, InvalidRecord>(validity =>
new InvalidRecord(validity.Data.Id, validity.InvalidReason));
ITargetBlock <InvalidRecord> invalidRecordSink =
DataflowExtensions.CreateBatchedActionBlock <InvalidRecord>(BatchSize, config.InvalidRecordSink.Persist);
invalidParsedRecordExtractor.LinkTo(invalidRecordSink);
sinkCompletions.Add(invalidRecordSink.Completion);
foreach (RecordPipeline recordPipeline in config.RecordPipelines)
{
ITargetBlock <Validity <ParsedRecord> > sink = DataflowExtensions.ToTargetPipeline(
new TransformBlock <Validity <ParsedRecord>, ParsedRecord>(v => v.Data),
DataflowExtensions.CreateBatchedActionBlock <ParsedRecord>(BatchSize,
recordPipeline.ValidRecordSink.Persist));
fileParser.LinkTo(recordPipeline.Pipeline, DataflowExtensions.PropagateCompletion,
r => recordPipeline.RoutingPredicate(r));
recordPipeline.Pipeline.LinkTo(sink, DataflowExtensions.PropagateCompletion, validity => validity.IsValid);
recordPipeline.Pipeline.LinkTo(invalidParsedRecordExtractor, validity => !validity.IsValid);
sinkCompletions.Add(sink.Completion);
}
// We need to add unroutable last since it will greedily accept all messages. This won't work if any of
// the record pipeline have a limited capacity since they could overflow into this link
var unroutable = new TransformBlock <UnparsedRecord, InvalidRecord>(unparsed =>
new InvalidRecord(unparsed.Id, "Record did not match any of the pipelines' predicates."));
fileParser.LinkTo(unroutable, DataflowExtensions.PropagateCompletion);
invalidParsedRecordExtractor.CompleteWhenAllOrAnyFaulted(config.RecordPipelines.Select(p => p.Pipeline.Completion).ToList());
invalidRecordSink.CompleteWhenAllOrAnyFaulted(new List <Task> {
invalidParsedRecordExtractor.Completion, unroutable.Completion
});
return(DataflowExtensions.EncapsulateTarget(fileParser, Task.WhenAll(sinkCompletions)));
}
static public void ProcessingByTPL_StraightForwardImplementation()
{
const string pathToFiles = @"..\..\..\..\DataFiles";
string[] files = Directory.GetFiles(pathToFiles, "*.txt", SearchOption.AllDirectories);
var loadDataFromFileBlock = new TransformBlock<string[], List<CustomerTextData>>(fileItems =>
{
var factory = new CustomerTextDataFactory();
return new List<CustomerTextData>(Array.ConvertAll(fileItems, factory.LoadFromFile));
});
var filterBlock = new TransformBlock<List<CustomerTextData>, List<CustomerTextData>>(textDataList =>
{
var filter = new FilterTextData(5);
return textDataList.Where(filter.Run).ToList();
});
var toListBlock = new TransformManyBlock<List<CustomerTextData>, CustomerTextData>(textDataList =>
{
var queue = new ConcurrentQueue<CustomerTextData>();
textDataList.ForEach(queue.Enqueue);
return queue;
});
var action = new ActionBlock<CustomerTextData>(textData =>
{
var weight = new WeightTextData();
int result = weight.Run(textData);
Trace.WriteLine(result);
Console.WriteLine(result);
});
loadDataFromFileBlock.LinkTo(filterBlock);
filterBlock.LinkTo(toListBlock);
toListBlock.LinkTo(action);
loadDataFromFileBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)filterBlock).Fault(t.Exception);
else filterBlock.Complete();
});
filterBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)toListBlock).Fault(t.Exception);
else toListBlock.Complete();
});
toListBlock.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)action).Fault(t.Exception);
else action.Complete();
});
loadDataFromFileBlock.Post(files);
loadDataFromFileBlock.Complete();
action.Completion.Wait();
}
public void NullResultTest ()
{
bool received = false;
var transformMany =
new TransformManyBlock<int, int> (i => (IEnumerable<int>)null);
var action = new ActionBlock<int> (i => received = true);
transformMany.LinkTo (action);
Assert.IsTrue (transformMany.Post (1));
transformMany.Complete ();
Assert.IsTrue (transformMany.Completion.Wait (100));
Assert.IsFalse (received);
}
public void DeferredUsageTest ()
{
int insIndex = -1;
int[] array = new int[5 + 3];
var evt = new CountdownEvent (array.Length);
var block = new ActionBlock<int> (i => { array[Interlocked.Increment (ref insIndex)] = i; evt.Signal (); });
var trsm = new TransformManyBlock<int, int> (i => Enumerable.Range (0, i));
trsm.Post (5);
trsm.Post (3);
trsm.LinkTo (block);
evt.Wait ();
CollectionAssert.AreEquivalent (new int[] { 0, 1, 2, 3, 4, 0, 1, 2 }, array);
}
private static IPropagatorBlock<string, KeyValuePair<string, long>> CreateCustomBlock()
{
var directoryBrowserBlock = new TransformManyBlock<string, string>(path =>
{
var dir = new DirectoryInfo(path);
return dir.EnumerateFileSystemInfos()
.Select(fi => fi.FullName);
});
var fileSizeCalculator = new TransformBlock<string, KeyValuePair<string, long>>(fileName =>
{
var fi = new FileInfo(fileName);
return new KeyValuePair<string, long>(fileName, fi.Length);
});
directoryBrowserBlock.LinkTo(fileSizeCalculator, new DataflowLinkOptions { PropagateCompletion = true }, File.Exists);
var customBlock = DataflowBlock.Encapsulate(directoryBrowserBlock, fileSizeCalculator);
return customBlock;
}
public async static Task<Animat> SeedAsyncWithout3MB(ScenarioSpecies species, GeoRect geoRect, Bathymetry bathymetry)
{
var bounds = new GeoArray(geoRect.NorthWest, geoRect.NorthEast, geoRect.SouthEast, geoRect.SouthWest, geoRect.NorthWest);
var result = new Animat { ScenarioSpecies = species };
var area = bounds.Area;
//Debug.WriteLine("Area: {0}",area);
var transformManyBlock = new TransformManyBlock<int, Geo<float>>(count =>
{
var geos = new List<Geo<float>>();
for (var i = 0; i < count; i++)
{
var location = bounds.RandomLocationWithinPerimeter();
var depth = bathymetry.Samples.GetNearestPointAsync(location).Result.Data;
if (depth < -50) geos.Add(new Geo<float>(location.Latitude, location.Longitude, (float)(depth * Random.NextDouble())));
}
return geos;
}, new ExecutionDataflowBlockOptions
{
TaskScheduler = TaskScheduler.Default,
BoundedCapacity = -1,
MaxDegreeOfParallelism = -1,
});
var bufferBlock = new BufferBlock<Geo<float>>();
transformManyBlock.LinkTo(bufferBlock);
var population = (int)Math.Round(area * species.PopulationDensity);
result.TotalAnimats = population;
const int blockSize = 100;
while (population > 0)
{
transformManyBlock.Post(population > blockSize ? blockSize : population);
population -= blockSize;
}
transformManyBlock.Complete();
await transformManyBlock.Completion;
IList<Geo<float>> animatGeos;
if (bufferBlock.TryReceiveAll(out animatGeos))
result.Locations.AddRange(animatGeos);
return result;
}
private static void Main(string[] args)
{
var generatorBlock = new TransformManyBlock<int, string>(num => GenerateStrings(num));
var writerBlock = new TransformBlock<string, string>(str => WriteString(str), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5 });
var finishingBlock = new ActionBlock<string>(str =>
{
writerBlock.Completion.Wait();
Console.WriteLine(Thread.CurrentThread.ManagedThreadId + ": finished - " + str);
});
generatorBlock.LinkTo(writerBlock, new DataflowLinkOptions{PropagateCompletion = true});
writerBlock.LinkTo(finishingBlock, new DataflowLinkOptions { PropagateCompletion = true });
for (var i = 1; i <= 3; i++)
{
Console.WriteLine("Posted " + i*10);
generatorBlock.Post(i*10);
}
generatorBlock.Complete();
finishingBlock.Completion.Wait();
Console.WriteLine("Pipeline is finished");
Console.ReadKey();
}
public void BuildPipeline(CancellationToken token)
{
// create the blocks
getFileUrls = new TransformManyBlock<string, string>(
(url) =>
{
List<string> fileUrls = new List<string>();
GetFileUrls(url, fileUrls);
return fileUrls;
}
, new ExecutionDataflowBlockOptions { CancellationToken = token }
);
var downloadFile = new TransformBlock<string, string>(
(fileUrl) =>
{
string fileName = Regex.Match(fileUrl, @"([^/]+$)").Value;
using (var client = new WebClient())
{
client.DownloadFile(fileUrl, FolderPath + "\\" + fileName);
}
Interlocked.Increment(ref currentFileCount);
UpdateProgress((double)currentFileCount / (double)fileCount);
return FolderPath + "\\" + fileName;
}
,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
CancellationToken = token,
}
);
var convertFile = new TransformBlock<string, string>(
(fileName) =>
{
string newFileName = ConvertedFilesFolder + @"\" + Regex.Match(fileName, @"([^\\]+$)").Value + "x";
Process process = new Process();
process.StartInfo.Arguments = string.Format(@" -nme -oice {0} {1}", fileName, newFileName);
process.StartInfo.FileName = @"c:\Program Files (x86)\Microsoft Office\Office12\excelcnv.exe";
process.Start();
while (!process.WaitForExit(15000))
{
process.Kill();
process.Start();
}
Interlocked.Increment(ref convertedFilesCount);
UpdateConversionProgress((double)convertedFilesCount / (double)fileCount);
return newFileName;
}
,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
CancellationToken = token
}
);
var parseCitizenInfo = new ActionBlock<string>(
(fileName) =>
{
var referendumProcessor = new ReferendumProcessor();
citizensOfArmenia.Post(referendumProcessor.ProcessFile(fileName));
}
,
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = Environment.ProcessorCount,
CancellationToken = token
}
);
// Link the blocks
citizensOfArmenia = new BufferBlock<List<Citizen>>();
getFileUrls.LinkTo(downloadFile, new DataflowLinkOptions { PropagateCompletion = true });
downloadFile.LinkTo(convertFile, new DataflowLinkOptions { PropagateCompletion = true });
convertFile.LinkTo(parseCitizenInfo, new DataflowLinkOptions { PropagateCompletion = true });
parseCitizenInfo.Completion.ContinueWith((tsk) => citizensOfArmenia.Complete());
}
public async void Crawl()
{
var scraper = new TransformManyBlock<LinkInfo, LinkInfo>(
async linkInfo =>
{
var linkInfos = new List<LinkInfo>();
try
{
var response = await linkInfo.Uri.GetResponse();
if (!IsSuccessStatus(response, linkInfo))
return linkInfos;
var html = await response.Content.ReadAsStringAsync();
var doc = new HtmlDocument();
doc.LoadHtml(html);
linkInfos = doc.ToLinkInfos(linkInfo.Uri);
Log(Context.GoodHTML, linkInfo.Uri.AbsoluteUri);
}
catch (Exception error)
{
Log(Context.BadHTML, "Error: {0} (URL: {1})", error.Message, linkInfo.Uri);
}
return linkInfos;
},
new ExecutionDataflowBlockOptions()
{
CancellationToken = cts.Token
});
var fetcher = new ActionBlock<LinkInfo>(
async linkInfo =>
{
try
{
var fileName = linkInfo.GetFileName(linkInfo.Uri, "Downloads");
if (File.Exists(fileName))
{
Log(Context.DupMedia, linkInfo.Uri.AbsoluteUri);
return;
}
var response = await linkInfo.Uri.GetResponse();
if (!IsSuccessStatus(response, linkInfo))
return;
var webStream = await response.Content.ReadAsStreamAsync();
fileName.EnsurePathExists();
using (var fileStream = File.OpenWrite(fileName))
await webStream.CopyToAsync(fileStream);
Log(Context.GoodMedia, linkInfo.Uri.AbsoluteUri);
}
catch (Exception error)
{
Log(Context.BadMedia, "Error: {0} (URL: {1})", error.Message, linkInfo.Uri);
}
},
new ExecutionDataflowBlockOptions()
{
CancellationToken = cts.Token,
MaxDegreeOfParallelism = Environment.ProcessorCount * 12
});
scraper.Completion.SetOnlyOnFaultedCompletion(error => HandleErrors(error));
fetcher.Completion.SetOnlyOnFaultedCompletion(error => HandleErrors(error));
scraper.LinkTo(scraper, new Predicate<LinkInfo>(li => li.Kind == LinkKind.HTML));
scraper.LinkTo(fetcher, new Predicate<LinkInfo>(li => li.Kind == LinkKind.Media));
scraper.Post(new LinkInfo(new Uri("http://www.bbc.com/news/")));
try
{
await Task.WhenAll(scraper.Completion, fetcher.Completion);
}
catch (OperationCanceledException)
{
}
catch (Exception error)
{
Log(Context.Failure, "Error: " + error.Message);
}
if (OnFinished != null)
OnFinished(this, EventArgs.Empty);
}
public async Task TransformManyEnumerableToAction()
{
var data = new[] { 1 };
var tm = new TransformManyBlock<int, int>(i => data);
int completedCount = 0;
var c = new ActionBlock<int>(i => completedCount++);
tm.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true });
tm.PostRange(0, Iterations);
tm.Complete();
await c.Completion;
Assert.Equal(expected: Iterations, actual: completedCount);
}
public static async Task PurgeExpiredAsync(CloudBlobContainer inboxContainer)
{
Requires.NotNull(inboxContainer, "inboxContainer");
var deleteBlobsExpiringBefore = DateTime.UtcNow;
int purgedBlobCount = 0;
var searchExpiredBlobs = new TransformManyBlock<CloudBlobContainer, ICloudBlob>(
async c =>
{
try
{
var results = await c.ListBlobsSegmentedAsync(
string.Empty,
useFlatBlobListing: true,
pageSize: 50,
details: BlobListingDetails.Metadata,
options: new BlobRequestOptions(),
operationContext: null);
return from blob in results.OfType<ICloudBlob>()
let expires = DateTime.Parse(blob.Metadata[ExpirationDateMetadataKey])
where expires < deleteBlobsExpiringBefore
select blob;
}
catch (StorageException ex)
{
var webException = ex.InnerException as WebException;
if (webException != null)
{
var httpResponse = (HttpWebResponse)webException.Response;
if (httpResponse.StatusCode == HttpStatusCode.NotFound)
{
// it's legit that some tests never created the container to begin with.
return Enumerable.Empty<ICloudBlob>();
}
}
throw;
}
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 4,
});
var deleteBlobBlock = new ActionBlock<ICloudBlob>(
blob =>
{
Interlocked.Increment(ref purgedBlobCount);
return blob.DeleteAsync();
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 4,
BoundedCapacity = 100,
});
searchExpiredBlobs.LinkTo(deleteBlobBlock, new DataflowLinkOptions { PropagateCompletion = true });
searchExpiredBlobs.Post(inboxContainer);
searchExpiredBlobs.Complete();
await deleteBlobBlock.Completion;
}
public static IPropagatorBlock<FileWord, FileWord> GetFileWordSaverBlock(Func<IRepository> repositoryFactory)
{
var inputBuffer = new BufferBlock<FileWord>();
var outputBuffer = new BufferBlock<FileWord>();
var batchBlockWithoutParents = new BatchBlock<FileWord>(1000);
inputBuffer.LinkTo(batchBlockWithoutParents, i => i.File.FileId > 0 && i.Word.WordId > 0);
inputBuffer.PropagateCompleted(batchBlockWithoutParents);
var saveWithoutParentsBlock = new TransformManyBlock<FileWord[], FileWord>(
async fileWords =>
{
return await SaveEntities(repositoryFactory, fileWords, true);
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism
});
batchBlockWithoutParents.LinkToAndPropagateCompleted(saveWithoutParentsBlock);
saveWithoutParentsBlock.LinkTo(outputBuffer);
var batchBlockWithParents = new BatchBlock<FileWord>(300);
// MaxMessages value means inputBuffer will unlink automatically from batchBlockWithParents after 300 messages.
// This is required to prohibit "stealing" of workload from saveWithoutParentsBlock
inputBuffer.LinkTo(batchBlockWithParents, new DataflowLinkOptions { MaxMessages = 300 });
inputBuffer.PropagateCompleted(batchBlockWithParents);
var saveWithParentsBlock = new TransformManyBlock<FileWord[], FileWord>(
async fileWords =>
{
var res = await SaveEntities(repositoryFactory, fileWords, false);
FileWord fileWord;
if (inputBuffer.TryReceive(out fileWord)) // This unblocks inputBuffer due to unlinking from batchBlockWithParents
{
if (fileWord.File.FileId > 0 && fileWord.Word.WordId > 0)
{
batchBlockWithoutParents.Post(fileWord);
}
else
{
batchBlockWithParents.Post(fileWord);
}
}
// Link again for another 300 messages
inputBuffer.LinkTo(batchBlockWithParents, new DataflowLinkOptions { MaxMessages = 300 });
return res;
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1 // If we are saving files/words also we cannot do this concurrently or duplicates of files/words may be inserted
});
batchBlockWithParents.LinkToAndPropagateCompleted(saveWithParentsBlock);
saveWithParentsBlock.LinkTo(outputBuffer);
saveWithoutParentsBlock.Completion.ContinueWith(
t => ((IDataflowBlock)outputBuffer).Fault(t.Exception),
TaskContinuationOptions.OnlyOnFaulted);
saveWithParentsBlock.Completion.ContinueWith(
t => ((IDataflowBlock)outputBuffer).Fault(t.Exception),
TaskContinuationOptions.OnlyOnFaulted);
Task.WhenAll(saveWithoutParentsBlock.Completion, saveWithParentsBlock.Completion)
.ContinueWith(t => outputBuffer.Complete(), TaskContinuationOptions.NotOnFaulted);
return DataflowBlock.Encapsulate(inputBuffer, outputBuffer);
}
public Task Execute(CancellationToken token)
{
var options = new DataflowBlockOptions {CancellationToken = token};
_buffer = new BufferBlock<long>(options);
var hydrate = new TransformBlock<long, Summoner>(id =>
{
var summoner = _lookup.Hydrate(id);
return summoner;
}, new ExecutionDataflowBlockOptions { CancellationToken = token, MaxDegreeOfParallelism = 2 });
var store = new TransformBlock<Summoner, Summoner>(summoner =>
{
if (summoner != null)
_storage.Store(summoner);
return summoner;
}, new ExecutionDataflowBlockOptions {CancellationToken = token, MaxDegreeOfParallelism = 2});
var crawl = new TransformManyBlock<Summoner, FellowPlayerInfo>(async summoner =>
{
var summoners = new List<FellowPlayerInfo>();
var games = new List<PlayerGameStats>();
if (summoner != null)
{
await _crawler.Crawl(summoner, summoners.Add, games.Add);
}
return summoners;
}, new ExecutionDataflowBlockOptions {CancellationToken = token, MaxDegreeOfParallelism = 2});
var storeNextBatch = new ActionBlock<FellowPlayerInfo>(async info =>
{
if (info != null)
{
var data = await _lookup.Lookup(info.summonerId);
_storage.StoreWhenMissing(data);
}
}, new ExecutionDataflowBlockOptions {CancellationToken = token, MaxDegreeOfParallelism = 2});
_buffer.LinkTo(hydrate, new DataflowLinkOptions {PropagateCompletion = true});
hydrate.LinkTo(store, new DataflowLinkOptions {PropagateCompletion = true});
store.LinkTo(crawl, new DataflowLinkOptions {PropagateCompletion = true});
crawl.LinkTo(storeNextBatch, new DataflowLinkOptions {PropagateCompletion = true});
return Task.Run(async () =>
{
while (!token.IsCancellationRequested)
{
try
{
var batch = _producer.Produce((int) TimeSpan.FromDays(1).TotalMinutes, 30);
foreach (var id in batch)
await _buffer.SendAsync(id, token);
// Start the chain
_buffer.Complete();
// Wait until the chain is complete before iterating again
await storeNextBatch.Completion;
}
catch (Exception ex)
{
throw;
}
}
}, token);
}
public void RunTransformManyBlockConformanceTests()
{
bool passed = true;
#region Sync
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory =
options =>
{
TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => new[] { i }, (ExecutionDataflowBlockOptions)options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformManyBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token };
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16;
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 }));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test multiple yielded results
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
return Enumerable.Range(0, 10);
});
t.Post(42);
t.Complete();
for (int i = 0; i < 10; i++)
{
localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return new[] { i };
});
for (int i = 0; i < 10; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test using a precanceled token
{
bool localPassed = true;
try
{
var cts = new CancellationTokenSource();
cts.Cancel();
var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token };
var t = new TransformManyBlock<int, int>(i => new[] { i }, dbo);
int ignoredValue;
IList<int> ignoredValues;
localPassed &= t.LinkTo(new ActionBlock<int>(delegate { })) != null;
localPassed &= t.SendAsync(42).Result == false;
localPassed &= t.TryReceiveAll(out ignoredValues) == false;
localPassed &= t.Post(42) == false;
localPassed &= t.OutputCount == 0;
localPassed &= t.TryReceive(out ignoredValue) == false;
localPassed &= t.Completion != null;
t.Complete();
}
catch (Exception)
{
localPassed = false;
}
Console.WriteLine(" > {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, IEnumerable<int>>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test reuse of a list and array
{
bool localPassed = true;
foreach (bool bounded in new[] { false, true })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (IList<int> list in new IList<int>[]
{
new int[1],
new List<int>() { 0 },
new Collection<int>() { 0 }
})
{
int nextExpectedValue = 1;
TransformManyBlock<int, int> tmb1 = null;
tmb1 = new TransformManyBlock<int, int>(i =>
{
if (i == 1000)
{
tmb1.Complete();
return (IEnumerable<int>)null;
}
else if (dop == 1)
{
list[0] = i + 1;
return (IEnumerable<int>)list;
}
else if (list is int[])
{
return new int[1] { i + 1 };
}
else if (list is List<int>)
{
return new List<int>() { i + 1 };
}
else return new Collection<int>() { i + 1 };
}, dbo);
TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i =>
{
if (i != nextExpectedValue)
{
localPassed = false;
tmb1.Complete();
}
nextExpectedValue++;
return i;
});
tmb1.LinkTo(tmb2);
tmb2.LinkTo(tmb1);
tmb1.SendAsync(0).Wait();
tmb1.Completion.Wait();
}
}
}
Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test throwing an OCE
{
bool localPassed = true;
foreach (bool bounded in new[] { true, false })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (int mode in new[] { 0, 1, 2 })
{
const int ITERS = 50;
var mres = new ManualResetEventSlim();
var tmb = new TransformManyBlock<int, int>(i =>
{
if (i < ITERS - 1) throw new OperationCanceledException();
if (mode == 0) return new int[] { i };
else if (mode == 1) return new List<int>() { i };
else return Enumerable.Repeat(i, 1);
}, dbo);
var ab = new ActionBlock<int>(i =>
{
if (i != ITERS - 1) localPassed = false;
mres.Set();
});
tmb.LinkTo(ab);
for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait();
mres.Wait();
}
}
}
Console.WriteLine("{0}: Canceled invocation", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
#region Async
{
// Do everything twice - once through OfferMessage and Once through Post
for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++)
{
Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory =
options =>
{
TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }), (ExecutionDataflowBlockOptions)options);
ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options);
transformManyBlock.LinkTo(actionBlock);
return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false };
};
CancellationTokenSource cancellationSource = new CancellationTokenSource();
var defaultOptions = new ExecutionDataflowBlockOptions();
var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount };
var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 };
var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token };
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true);
passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true);
}
// Test chained Post/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
for (int i = 0; i < ITERS; i++)
{
network.Post(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync/Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
for (int i = 0; i < ITERS; i++)
{
network.SendAsync(i);
localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16);
}
Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained Post all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true;
for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16;
Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test chained SendAsync all then Receive
{
bool localPassed = true;
const int ITERS = 2;
var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 })));
var tasks = new Task[ITERS];
for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i);
Task.WaitAll(tasks);
int total = 0;
for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive();
localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16));
Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test multiple yielded results
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)Enumerable.Range(0, 10).ToArray()));
t.Post(42);
t.Complete();
for (int i = 0; i < 10; i++)
{
localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that OperationCanceledExceptions are ignored
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
if ((i % 2) == 0) throw new OperationCanceledException();
return new[] { i };
});
for (int i = 0; i < 10; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(i =>
{
if ((i % 2) == 0) return null;
return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i });
});
for (int i = 0; i < 10; i++) t.Post(i);
t.Complete();
for (int i = 0; i < 10; i++)
{
if ((i % 2) != 0) localPassed &= t.Receive() == i;
}
t.Completion.Wait();
Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test that null tasks are ignored when a reordering buffer is in place
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i =>
{
if (i == 0)
{
Task.Delay(1000).Wait();
return null;
}
return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i });
}), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 });
t.Post(0);
t.Post(1);
try
{
localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1;
}
catch
{
localPassed = false;
}
Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the delegate
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => { throw new InvalidOperationException(); }));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test faulting from the task
{
bool localPassed = true;
var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => Task<IEnumerable<int>>.Factory.StartNew(() => { throw new InvalidOperationException(); })));
t.Post(42);
t.Post(1);
t.Post(2);
t.Post(3);
try { t.Completion.Wait(); }
catch { }
localPassed &= t.Completion.IsFaulted;
localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500);
localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500);
localPassed &= t.Post(4) == false;
Console.WriteLine(" > {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test reuse of a list and array
{
bool localPassed = true;
foreach (bool bounded in new[] { false, true })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (IList<int> list in new IList<int>[]
{
new int[1],
new List<int>() { 0 },
new Collection<int>() { 0 }
})
{
int nextExpectedValue = 1;
TransformManyBlock<int, int> tmb1 = null;
tmb1 = new TransformManyBlock<int, int>(i =>
{
return Task.Factory.StartNew(() =>
{
if (i == 1000)
{
tmb1.Complete();
return (IEnumerable<int>)null;
}
else if (dop == 1)
{
list[0] = i + 1;
return (IEnumerable<int>)list;
}
else if (list is int[])
{
return new int[1] { i + 1 };
}
else if (list is List<int>)
{
return new List<int>() { i + 1 };
}
else return new Collection<int>() { i + 1 };
});
}, dbo);
TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i =>
{
if (i != nextExpectedValue)
{
localPassed = false;
tmb1.Complete();
}
nextExpectedValue++;
return i;
});
tmb1.LinkTo(tmb2);
tmb2.LinkTo(tmb1);
tmb1.SendAsync(0).Wait();
tmb1.Completion.Wait();
}
}
}
Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
// Test throwing an OCE
{
bool localPassed = true;
foreach (bool bounded in new[] { true, false })
{
for (int dop = 1; dop < Environment.ProcessorCount; dop++)
{
var dbo = bounded ?
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } :
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop };
foreach (int mode in new[] { 0, 1, 2 })
{
const int ITERS = 50;
var mres = new ManualResetEventSlim();
var tmb = new TransformManyBlock<int, int>(i =>
{
var cts = new CancellationTokenSource();
return Task.Factory.StartNew(() =>
{
if (i < ITERS - 1)
{
cts.Cancel();
cts.Token.ThrowIfCancellationRequested();
}
if (mode == 0) return new int[] { i };
else if (mode == 1) return new List<int>() { i };
else return Enumerable.Repeat(i, 1);
}, cts.Token);
}, dbo);
var ab = new ActionBlock<int>(i =>
{
if (i != ITERS - 1) localPassed = false;
mres.Set();
});
tmb.LinkTo(ab);
for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait();
mres.Wait();
}
}
}
Console.WriteLine(" > {0}: Canceled invocation", localPassed ? "Success" : "Failure");
passed &= localPassed;
}
}
#endregion
Assert.True(passed, "Test failed.");
}
static void Main(string[] args)
{
//
// Create the members of the pipeline.
//
// Downloads the requested resource as a string.
var downloadString = new TransformBlock<string, string>(uri =>
{
Console.WriteLine("Downloading '{0}'...", uri);
return new WebClient().DownloadString(uri);
});
// Separates the specified text into an array of words.
var createWordList = new TransformBlock<string, string[]>(text =>
{
Console.WriteLine("Creating word list...");
// Remove common punctuation by replacing all non-letter characters
// with a space character to.
char[] tokens = text.ToArray();
for (int i = 0; i < tokens.Length; i++)
{
if (!char.IsLetter(tokens[i]))
tokens[i] = ' ';
}
text = new string(tokens);
// Separate the text into an array of words.
return text.Split(new char[] { ' ' },
StringSplitOptions.RemoveEmptyEntries);
});
// Removes short words, orders the resulting words alphabetically,
// and then remove duplicates.
var filterWordList = new TransformBlock<string[], string[]>(words =>
{
Console.WriteLine("Filtering word list...");
return words.Where(word => word.Length > 3).OrderBy(word => word)
.Distinct().ToArray();
});
// Finds all words in the specified collection whose reverse also
// exists in the collection.
var findReversedWords = new TransformManyBlock<string[], string>(words =>
{
Console.WriteLine("Finding reversed words...");
// Holds reversed words.
var reversedWords = new ConcurrentQueue<string>();
// Add each word in the original collection to the result whose
// reversed word also exists in the collection.
Parallel.ForEach(words, word =>
{
// Reverse the work.
string reverse = new string(word.Reverse().ToArray());
// Enqueue the word if the reversed version also exists
// in the collection.
if (Array.BinarySearch<string>(words, reverse) >= 0 &&
word != reverse)
{
reversedWords.Enqueue(word);
}
});
return reversedWords;
});
// Prints the provided reversed words to the console.
var printReversedWords = new ActionBlock<string>(reversedWord =>
{
Console.WriteLine("Found reversed words {0}/{1}",
reversedWord, new string(reversedWord.Reverse().ToArray()));
});
//
// Connect the dataflow blocks to form a pipeline.
//
downloadString.LinkTo(createWordList);
createWordList.LinkTo(filterWordList);
filterWordList.LinkTo(findReversedWords);
findReversedWords.LinkTo(printReversedWords);
//
// For each completion task in the pipeline, create a continuation task
// that marks the next block in the pipeline as completed.
// A completed dataflow block processes any buffered elements, but does
// not accept new elements.
//
downloadString.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)createWordList).Fault(t.Exception);
else createWordList.Complete();
});
createWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)filterWordList).Fault(t.Exception);
else filterWordList.Complete();
});
filterWordList.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)findReversedWords).Fault(t.Exception);
else findReversedWords.Complete();
});
findReversedWords.Completion.ContinueWith(t =>
{
if (t.IsFaulted) ((IDataflowBlock)printReversedWords).Fault(t.Exception);
else printReversedWords.Complete();
});
// Process "The Iliad of Homer" by Homer.
downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt");
// Mark the head of the pipeline as complete. The continuation tasks
// propagate completion through the pipeline as each part of the
// pipeline finishes.
downloadString.Complete();
// Wait for the last block in the pipeline to process all messages.
printReversedWords.Completion.Wait();
Console.ReadLine();
}
public async Task TestPrecanceled()
{
var bb = new TransformManyBlock<int, int>(DataflowTestHelpers.ToEnumerable,
new ExecutionDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true) });
int ignoredValue;
IList<int> ignoredValues;
IDisposable link = bb.LinkTo(DataflowBlock.NullTarget<int>());
Assert.NotNull(link);
link.Dispose();
Assert.False(bb.Post(42));
var t = bb.SendAsync(42);
Assert.True(t.IsCompleted);
Assert.False(t.Result);
Assert.False(bb.TryReceiveAll(out ignoredValues));
Assert.False(bb.TryReceive(out ignoredValue));
Assert.NotNull(bb.Completion);
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => bb.Completion);
bb.Complete(); // just make sure it doesn't throw
}
public void Fetch(Link initialLink)
{
urls = new ConcurrentDictionary<string, bool>();
cts = new CancellationTokenSource();
var cancellable = new ExecutionDataflowBlockOptions()
{
CancellationToken = cts.Token
};
var hrefRegex = new Regex(@"(?<=<img\s.*?src\s*?=\s*?"").*?(?= "".*?>)",
RegexOptions.Compiled | RegexOptions.IgnoreCase);
var srcRegex = new Regex(@"(?<=<a\s.*?href\s*?=\s*?"").*?(?= "".*?>)",
RegexOptions.Compiled | RegexOptions.IgnoreCase);
var scraper = new TransformBlock<Link, Tuple<Link, HtmlDocument>>(
async link =>
{
if (!urls.TryAdd(link.Uri.AbsoluteUri, true))
{
Log(LogKind.DupHTML, link.Uri.AbsoluteUri);
return new Tuple<Link, HtmlDocument>(link, null);
}
HtmlDocument doc;
try
{
var response = await link.Uri.GetResponse();
if (!IsSuccessStatus(response, link))
return new Tuple<Link, HtmlDocument>(link, null);
var html = await response.Content.ReadAsStringAsync();
doc = new HtmlDocument();
doc.LoadHtml(html);
Log(LogKind.Scrape, link.Uri.AbsoluteUri);
}
catch (Exception error)
{
doc = null;
LogError(link, error);
}
return new Tuple<Link, HtmlDocument>(link, doc);
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = 4,
CancellationToken = cts.Token
});
var docCaster = new BroadcastBlock<Tuple<Link, HtmlDocument>>(
content => content,
cancellable);
//(?<=<a\s.*?href\s*?=.*?)".*?"(?=.*?>.*?</a>)
var hrefParser = new TransformManyBlock<Tuple<Link, HtmlDocument>, Link>(
tuple =>
{
return (from link in tuple.Item2.DocumentNode.Descendants("a")
where link.Attributes.Contains("href")
let href = link.GetAttributeValue("href", "")
where href.IsLink(tuple.Item1.Uri, LinkKind.HTML, UriFilter.None)
select new Link(tuple.Item1.Uri, href, LinkKind.HTML)).Distinct();
},
cancellable);
var srcParser = new TransformManyBlock<Tuple<Link, HtmlDocument>, Link>(
tuple =>
{
return (from img in tuple.Item2.DocumentNode.Descendants("img")
where img.Attributes.Contains("src")
let src = img.GetAttributeValue("src", "")
where src.IsLink(tuple.Item1.Uri, LinkKind.Image, UriFilter.None)
select new Link(tuple.Item1.Uri, src, LinkKind.Image)).Distinct();
},
cancellable);
var fetcher = new ActionBlock<Link>(
async link =>
{
try
{
var fileName = link.GetFileName("Downloads", false);
if (File.Exists(fileName))
{
Log(LogKind.DupMedia, link.Uri.AbsoluteUri);
return;
}
var response = await link.Uri.GetResponse();
if (!IsSuccessStatus(response, link))
return;
EnsurePathExists(fileName);
var webStream = await response.Content.ReadAsStreamAsync();
using (var fileStream = File.OpenWrite(fileName))
await webStream.CopyToAsync(fileStream);
Log(LogKind.Fetch, Path.GetFileName(fileName));
}
catch (Exception error)
{
LogError(link, error);
}
},
cancellable);
scraper.Completion.SetOnlyOnFaultedCompletion(error => HandleFailure(error));
docCaster.Completion.SetOnlyOnFaultedCompletion(error => HandleFailure(error));
hrefParser.Completion.SetOnlyOnFaultedCompletion(error => HandleFailure(error));
srcParser.Completion.SetOnlyOnFaultedCompletion(error => HandleFailure(error));
fetcher.Completion.SetOnlyOnFaultedCompletion(error => HandleFailure(error));
scraper.LinkTo(docCaster);
docCaster.LinkTo(hrefParser, tuple => tuple.Item2 != null);
docCaster.LinkTo(srcParser, tuple => tuple.Item2 != null);
docCaster.LinkTo(DataflowBlock.NullTarget<Tuple<Link, HtmlDocument>>());
hrefParser.LinkTo(scraper);
srcParser.LinkTo(fetcher);
tasks.Add(scraper.Completion);
tasks.Add(docCaster.Completion);
tasks.Add(hrefParser.Completion);
tasks.Add(srcParser.Completion);
tasks.Add(fetcher.Completion);
scraper.Post(initialLink);
}
/// <summary>
/// Purges all blobs set to expire prior to the specified date.
/// </summary>
/// <param name="deleteBlobsExpiringBefore">
/// All blobs scheduled to expire prior to this date will be purged. The default value
/// is interpreted as <see cref="DateTime.UtcNow"/>.
/// </param>
/// <returns>The task representing the asynchronous operation.</returns>
public async Task PurgeBlobsExpiringBeforeAsync(DateTime deleteBlobsExpiringBefore = default(DateTime))
{
if (deleteBlobsExpiringBefore == default(DateTime))
{
deleteBlobsExpiringBefore = DateTime.UtcNow;
}
Requires.Argument(deleteBlobsExpiringBefore.Kind == DateTimeKind.Utc, "expirationUtc", "UTC required.");
var searchExpiredDirectoriesBlock = new TransformManyBlock<CloudBlobContainer, CloudBlobDirectory>(
async c =>
{
var results = await c.ListBlobsSegmentedAsync();
return from directory in results.OfType<CloudBlobDirectory>()
let expires = DateTime.Parse(directory.Uri.Segments[directory.Uri.Segments.Length - 1].TrimEnd('/'))
where expires < deleteBlobsExpiringBefore
select directory;
},
new ExecutionDataflowBlockOptions
{
BoundedCapacity = 4,
});
var deleteDirectoryBlock = new TransformManyBlock<CloudBlobDirectory, CloudBlockBlob>(
async directory =>
{
var results = await directory.ListBlobsSegmentedAsync();
return results.OfType<CloudBlockBlob>();
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 2,
BoundedCapacity = 4,
});
var deleteBlobBlock = new ActionBlock<CloudBlockBlob>(
blob => blob.DeleteAsync(),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 4,
BoundedCapacity = 100,
});
searchExpiredDirectoriesBlock.LinkTo(deleteDirectoryBlock, new DataflowLinkOptions { PropagateCompletion = true });
deleteDirectoryBlock.LinkTo(deleteBlobBlock, new DataflowLinkOptions { PropagateCompletion = true });
searchExpiredDirectoriesBlock.Post(this.container);
searchExpiredDirectoriesBlock.Complete();
await deleteBlobBlock.Completion;
}
public async static Task<Animat> SeedAsync(ScenarioSpecies species, GeoRect geoRect, Bathymetry bathymetry)
{
var yxzFileName = Path.Combine(Path.GetTempPath(), Path.GetFileNameWithoutExtension(Path.GetRandomFileName()) + ".txt");
bathymetry.ToYXZ(yxzFileName, -1);
var mbs = new C3mbs();
mbsRESULT mbsResult;
if (mbsRESULT.OK != (mbsResult = mbs.SetOutputDirectory(Path.GetTempPath())))
throw new AnimatInterfaceMMBSException("SetOutputDirectory Error:" + mbs.ResultToTc(mbsResult));
var config = mbs.GetConfiguration();
config.enabled = false; // binary output enabled/disabled
config.durationLess = true; // make sure we're in durationless mode.
mbs.SetConfiguration(config);
mbsResult = mbs.LoadBathymetryFromTextFile(yxzFileName);
if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException("Bathymetry failed to load: " + mbs.ResultToTc(mbsResult));
mbsResult = mbs.AddSpecies(species.SpeciesDefinitionFilePath);
if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException(string.Format("C3mbs::AddSpecies FATAL error {0} for species {1}", mbs.ResultToTc(mbsResult), species.SpeciesDefinitionFilePath));
var bounds = new GeoArray(geoRect.NorthWest, geoRect.NorthEast, geoRect.SouthEast, geoRect.SouthWest, geoRect.NorthWest);
var result = new Animat { ScenarioSpecies = species };
var area = bounds.Area;
//Debug.WriteLine("Area: {0}",area);
var transformManyBlock = new TransformManyBlock<int, Geo<float>>(count =>
{
var geos = new List<Geo<float>>();
for (var i = 0; i < count; i++)
{
var location = bounds.RandomLocationWithinPerimeter();
var depth = bathymetry.Samples.GetNearestPointAsync(location).Result.Data;
mbsRESULT retval;
lock (mbs) retval = mbs.AddIndividualAnimat(0, new mbsPosition { latitude = location.Latitude, longitude = location.Longitude, depth = 0 });
if (mbsRESULT.OK == retval) geos.Add(new Geo<float>(location.Latitude, location.Longitude, (float)(depth * Random.NextDouble())));
}
return geos;
}, new ExecutionDataflowBlockOptions
{
TaskScheduler = TaskScheduler.Default,
BoundedCapacity = -1,
MaxDegreeOfParallelism = -1,
});
var bufferBlock = new BufferBlock<Geo<float>>();
transformManyBlock.LinkTo(bufferBlock);
var population = (int)Math.Round(area * species.PopulationDensity);
result.TotalAnimats = population;
const int blockSize = 100;
while (population > 0)
{
transformManyBlock.Post(population > blockSize ? blockSize : population);
population -= blockSize;
}
transformManyBlock.Complete();
await transformManyBlock.Completion;
//mbsResult = mbs.InitializeRun();
//if (mbsRESULT.OK == mbsResult) while (mbsRUNSTATE.INITIALIZING == mbs.GetRunState()) Thread.Sleep(1);
//else throw new AnimatInterfaceMMBSException("C3mbs::Initialize FATAL error " + mbs.ResultToTc(mbsResult));
mbsResult = mbs.FinishRun();
if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException("C3mbs::FinishRun FATAL error " + mbs.ResultToTc(mbsResult));
IList<Geo<float>> animatGeos;
if (bufferBlock.TryReceiveAll(out animatGeos))
result.Locations.AddRange(animatGeos);
return result;
}
