private async Task Start(string path) { var uiScheduler = TaskScheduler.FromCurrentSynchronizationContext(); dirToFilesBlock = new TransformManyBlock <string, string>((Func <string, IEnumerable <string> >)(GetFileSystemItems), new ExecutionDataflowBlockOptions() { CancellationToken = ct }); fileActionBlock = new ActionBlock <string>((Action <string>)ProcessFile, new ExecutionDataflowBlockOptions() { CancellationToken = ct, TaskScheduler = uiScheduler }); // Order of LinkTo's important here! dirToFilesBlock.LinkTo(dirToFilesBlock, new DataflowLinkOptions() { PropagateCompletion = true }, IsDirectory); dirToFilesBlock.LinkTo(fileActionBlock, new DataflowLinkOptions() { PropagateCompletion = true }, IsRequiredDocType); // Kick off the recursion. dirToFilesBlock.Post(path); await ProcessingIsComplete(); dirToFilesBlock.Complete(); await Task.WhenAll(dirToFilesBlock.Completion, fileActionBlock.Completion); }
internal void start() { var consumerBlock = Consumer("consumer 1"); var consumer2Block = Consumer("\t\tconsumer 2"); var producerBlock = new TransformManyBlock <int, int>(x => Enumerable.Range(0, x)); producerBlock.LinkTo(consumerBlock, new DataflowLinkOptions { PropagateCompletion = true, }); producerBlock.LinkTo(consumer2Block, new DataflowLinkOptions { PropagateCompletion = true }); if (!producerBlock.Post(10)) { Console.WriteLine("Failed Post"); } producerBlock.Complete(); producerBlock.Completion.ContinueWith(p => print_status(p, "producer")); consumerBlock.Completion.ContinueWith(p => print_status(p, "consumer 1")); consumer2Block.Completion.ContinueWith(p => print_status(p, "\t\tconsumer 2")); }
public LoopDataflow1() { InputMessageBlock = new TransformBlock <Message, Message>(async msg => await InputMessage(msg)); HandleMessageBlock = new TransformManyBlock <Message, Message>(async msg => await HandleMessage(msg)); OutputMessageBlock = new ActionBlock <Message>(msg => OutputMessage(msg)); var linkOptions = new DataflowLinkOptions() { PropagateCompletion = false }; InputMessageBlock.LinkTo(HandleMessageBlock, linkOptions); HandleMessageBlock.LinkTo(OutputMessageBlock, linkOptions, msg => msg.WasProcessed == true); HandleMessageBlock.LinkTo(HandleMessageBlock, linkOptions, msg => msg.WasProcessed == false); InputMessageBlock.Completion.ContinueWith(async tsk => { await HandleMessageIsComplete(); HandleMessageBlock.Complete(); }); HandleMessageBlock.Completion.ContinueWith(tsk => { OutputMessageBlock.Complete(); }); //DebuggingLoop(); }
public ImageProcessor() { imageCollectionBlock = new TransformManyBlock <string, string>( (Func <string, IEnumerable <string> >)FindImagesInDirectory); loadAndToGreyBlock = new TransformBlock <string, BitmapSource>((Func <string, BitmapSource>)LoadAndToGrayScale, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 4, }); publishImageBlock = new ActionBlock <BitmapSource>((Action <BitmapSource>)PublishImage, new ExecutionDataflowBlockOptions() { TaskScheduler = TaskScheduler.FromCurrentSynchronizationContext() }); //zipBlock.LinkTo(zipBlock, u => u.AbsolutePath.EndsWith(".zip")); //zipBlock.LinkTo(loadAndToGreyBlock); //imageCollectionBlock.LinkTo(zipBlock,u => u.AbsoluteUri.EndsWith(".zip")); imageCollectionBlock.LinkTo(loadAndToGreyBlock, u => u.EndsWith(".jpg")); imageCollectionBlock.LinkTo(imageCollectionBlock); loadAndToGreyBlock.LinkTo(publishImageBlock); }
public DirectoryWalkerLite(Action <string> fileAction) { directoryBrowseBlock = new TransformManyBlock <string, string>((Func <string, IEnumerable <string> >)(GetFilesInDirectory)); fileActionBlock = new ActionBlock <string>(fileAction); directoryBrowseBlock.LinkTo(directoryBrowseBlock, Directory.Exists); directoryBrowseBlock.LinkTo(fileActionBlock); }
public DirectoryWalker(Action <string> fileAction) { _directoryBrowseBlock = new TransformManyBlock <string, string>((Func <string, IEnumerable <string> >)GetFilesInDirectory); _fileActionBlock = new ActionBlock <string>(fileAction); _directoryBrowseBlock.LinkTo(_directoryBrowseBlock, Directory.Exists); _directoryBrowseBlock.LinkTo(_fileActionBlock, new DataflowLinkOptions { PropagateCompletion = true }); }
static async Task Main(string[] args) { var executionOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var loader = new TransformBlock <string, string?>(LoadPage, executionOptions); var searcher = new TransformManyBlock <string?, Uri>(SearchForReferences, executionOptions); var fetcher = new TransformBlock <Uri, byte[]?>(FetchReferences, executionOptions); var hasher = new TransformBlock <byte[]?, byte[]>(HashAlgorithm.ComputeHash, executionOptions); var converter = new TransformBlock <byte[], string>(TransformBytesToHexidecimal, executionOptions); var printer = new ActionBlock <string>(PrintHash); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; loader.LinkTo(searcher, linkOptions, content => content is object); loader.LinkTo(DataflowBlock.NullTarget <string?>()); searcher.LinkTo(fetcher, linkOptions, uri => Regex.IsMatch(uri.Scheme, "^https?")); searcher.LinkTo(DataflowBlock.NullTarget <Uri>()); fetcher.LinkTo(hasher, linkOptions, content => content is object); fetcher.LinkTo(DataflowBlock.NullTarget <byte[]?>()); hasher.LinkTo(converter, linkOptions); converter.LinkTo(printer, linkOptions); var pages = new string[] { "https://arstechnica.com/", "https://www.reddit.com/", "https://www.anandtech.com/", "https://www.theverge.com/", "https://stigvoss.dk" }; foreach (var page in pages) { await loader.SendAsync(page); } loader.Complete(); await printer.Completion; }
public CsvImporter() { databaseQueryBlock = new TransformManyBlock <string, object[]>((Func <string, IEnumerable <object[]> >)ExecuteQuery); rowTGoLedgerBlock = new TransformManyBlock <object[], ILedgerEntry>((Func <object[], IEnumerable <ILedgerEntry> >)MapDatabaseRowToObject); debitBlock = new ActionBlock <ILedgerEntry>((Action <ILedgerEntry>)WriteDebitEntry); creditBlock = new ActionBlock <ILedgerEntry>((Action <ILedgerEntry>)WriteCreditEntry); unknownLedgerEntryBlock = new ActionBlock <ILedgerEntry>((Action <ILedgerEntry>)LogUnknownLedgerEntryType); databaseQueryBlock.LinkTo(rowTGoLedgerBlock); rowTGoLedgerBlock.LinkTo(debitBlock, le => le.IsDebit); //if IsDebit rowTGoLedgerBlock.LinkTo(creditBlock, le => le.IsCredit); // else if IsCredit rowTGoLedgerBlock.LinkTo(unknownLedgerEntryBlock); // else }
public void Start(CoreValues cv) { CoreValues = cv; MyCamera = new Camera(cv.Origin, cv.LookAt, cv.VecUp, cv.FovY, cv.AspectRatio, cv.Aperture, cv.DistToFocus); Image = new Vec3[cv.Width * cv.Height]; World = HitableList.DefaultWorld; Count = cv.Width * cv.Height; PixelFloats = new TransformManyBlock <int, InfoStruct>(GetSampleRays); MergeColors = new ActionBlock <InfoStruct>(DoMergeColors, new ExecutionDataflowBlockOptions { EnsureOrdered = false, MaxDegreeOfParallelism = -1 }); //writeImage = new ActionBlock<InfoStruct>(DoWriteImage, new ExecutionDataflowBlockOptions { EnsureOrdered = false, MaxDegreeOfParallelism = -1 }); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; PixelFloats.LinkTo(MergeColors, linkOptions); //MergeColors.LinkTo(writeImage, linkOptions); PixelFloats.Post(0); PixelFloats.Complete(); MergeColors.Completion.Wait(); DoSaveImage(); }
private (ITargetBlock <string>, Task) CreatePipeline() { var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; TransformManyBlock <object, IImage> fetchBlock = new TransformManyBlock <object, IImage>(_ => _functions.Fetch(_options.Value.Location), new ExecutionDataflowBlockOptions { BoundedCapacity = 2, MaxDegreeOfParallelism = 1 }); TransformManyBlock <IImage, IImage> detectFacesBlock = new TransformManyBlock <IImage, IImage>(source => _functions.ExtractFaces(source)); TransformManyBlock <IImage, IRecognition> recogniseFacesBlock = new TransformManyBlock <IImage, IRecognition>(source => _functions.RecogniseFaces(source)); TransformBlock <IRecognition, IPersistedRecognition> persistRecognitionBlock = new TransformBlock <IRecognition, IPersistedRecognition>(recognition => _functions.PersistRecognition(recognition), new ExecutionDataflowBlockOptions { BoundedCapacity = 10, MaxDegreeOfParallelism = 1 }); ActionBlock <IPersistedRecognition> notifyRecognitionBlock = new ActionBlock <IPersistedRecognition>(recognition => _functions.NotifyRecognition(recognition), new ExecutionDataflowBlockOptions { BoundedCapacity = 10, MaxDegreeOfParallelism = 1 }); fetchBlock.LinkTo(detectFacesBlock, linkOptions); detectFacesBlock.LinkTo(recogniseFacesBlock, linkOptions); recogniseFacesBlock.LinkTo(persistRecognitionBlock, linkOptions); persistRecognitionBlock.LinkTo(notifyRecognitionBlock, linkOptions); return(fetchBlock, notifyRecognitionBlock.Completion); }
protected override async Task ExecuteNodeTreeAsync(ExecutionContext context, ObjectExecutionNode rootNode) { //Options var blockOptions = new ExecutionDataflowBlockOptions { CancellationToken = context.CancellationToken, MaxDegreeOfParallelism = 1024 }; //Execute Block var block = new TransformManyBlock <Job, Job>( async job => { var node = await ExecuteNodeAsync(job.Context, job.Node).ConfigureAwait(false); var childJobs = (job.Node as IParentExecutionNode)? .GetChildNodes() .Select(child => job.CreateChildJob(job.Context, child)) .ToArray(); job.Complete(); return(childJobs); }, blockOptions); //Link to self block.LinkTo(block); //Start var rootJob = new Job(context, rootNode); await block.SendAsync(rootJob); //Wait until done await rootJob.Completion; }
public FaceDecisionHelper(int maxDegreeOfParallelism, bool dropWhenFull) { ProcessingBlock = new TransformManyBlock <IFaceStream, bool>( transform: Mock.DetermineAccessAsync, dataflowBlockOptions: new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, BoundedCapacity = dropWhenFull ? maxDegreeOfParallelism : DataflowBlockOptions.Unbounded, }); DecisionBlock = new ActionBlock <bool>(decision => { bool?finalDecision = null; if (!decision) { ++_failureCount; } if (decision || _failureCount > 7) { finalDecision = decision; } if (finalDecision.HasValue) { DecisionAvailable?.Invoke(this, finalDecision.Value); ProcessingBlock.Complete(); } }); ProcessingBlock.LinkTo(DecisionBlock); ProcessingBlock.Completion.ContinueWith(delegate { DecisionBlock.Complete(); }); }
public IEnumerable <IMetaData> TransformManyBlockUsage(string stringToSplit) { Console.WriteLine($"Inside {nameof(TplDataflow1ExecutionBlocksController)} - {nameof(TransformManyBlockUsage)}"); // Create the members of the pipeline. var transformManyBlockSplitAnInputStringIntoArray = new TransformManyBlock <string, string>(input => Functions.SplitAnInputStringIntoArray(input, SplitterSeparator) ); var transformBlockCreateASingleMedatadataFromAString = new TransformBlock <string, IMetaData>(stringInput => Functions.CreateASingleMedatadataFromAString(stringInput) ); // Connect the dataflow blocks to form a pipeline. transformManyBlockSplitAnInputStringIntoArray.LinkTo(transformBlockCreateASingleMedatadataFromAString, DataflowOptions.LinkOptions); // Start TransformManyBlockUsage pipeline with the input values. transformManyBlockSplitAnInputStringIntoArray.Post(stringToSplit); // Mark the head of the pipeline as complete. transformManyBlockSplitAnInputStringIntoArray.Complete(); // Equivalent of transformManyBlockSplitAnInputStringIntoArray.OutputCount var ouputCount = stringToSplit?.Split(SplitterSeparator, StringSplitOptions.RemoveEmptyEntries).Length ?? 0; for (var i = 0; i < ouputCount; i++) { yield return(transformBlockCreateASingleMedatadataFromAString.Receive()); } }
public Task Generate(int maxDegreeOfParallelism, string destination, params string[] paths) { var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }; var openFile = new TransformBlock <string, string>(async filePath => await File.ReadAllTextAsync(filePath), options); var generateTests = new TransformManyBlock <string, TestElement>(fileCode => TestGenerator.TestGenerator.GenerateTests(fileCode), options); var writeToFile = new ActionBlock <TestElement>(async testElement => await File.WriteAllTextAsync( Path.Combine(destination, testElement.TestName) + ".cs", testElement.TestCode)); var newOptions = new DataflowLinkOptions { PropagateCompletion = true }; openFile.LinkTo(generateTests, newOptions); generateTests.LinkTo(writeToFile, newOptions); foreach (string path in paths) { openFile.Post(path); } openFile.Complete(); return(writeToFile.Completion); }
public static async Task RunSampleOne(Func <int, Task <User[]> > getUsers, Func <User, Task> writeUser, int pageCount) { var getDataBlock = new TransformManyBlock <int, User>( transform: async(pageNumber) => await getUsers(pageNumber), dataflowBlockOptions: new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 3 }); var writeDataBlock = new ActionBlock <User>( action: async(user) => await writeUser(user), dataflowBlockOptions: new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 }); getDataBlock.LinkTo(writeDataBlock, new DataflowLinkOptions { PropagateCompletion = true }); for (var i = 1; i <= pageCount; i++) { getDataBlock.Post(i); } getDataBlock.Complete(); await writeDataBlock.Completion; }
public CsvImporter() { var databaseQueryBlock = new TransformManyBlock <string, object[]>((Func <string, IEnumerable <object[]> >)ExecuteQuery); var rowToGoLedgerBlock = new TransformManyBlock <object[], ILedgerEntry>( (Func <object[], IEnumerable <ILedgerEntry> >)MapDatabaseRowToObject); var debitBlock = new ActionBlock <ILedgerEntry>((Action <ILedgerEntry>)WriteDebitEntry); var creditBlock = new ActionBlock <ILedgerEntry>((Action <ILedgerEntry>)WriteCreditEntry); var unknownLedgerEntryBlock = new ActionBlock <ILedgerEntry>((Action <ILedgerEntry>)LogUnknownLedgerEntryType); databaseQueryBlock.LinkTo(rowToGoLedgerBlock); rowToGoLedgerBlock.LinkTo(debitBlock, entry => entry.IsDebit); // If is debit rowToGoLedgerBlock.LinkTo(creditBlock, entry => entry.IsCredit); // If is credit rowToGoLedgerBlock.LinkTo(unknownLedgerEntryBlock); // Other cases }
public async Task TestMultiplePublishersWithPropagateCompletion() { var block1 = new TransformManyBlock <string, char>(x => x.ToCharArray()); var block2 = new TransformManyBlock <string, char>(x => x.ToCharArray()); var target = new BufferBlock <char>(); var propagate = new DataflowLinkOptions { PropagateCompletion = true }; block1.LinkTo(target, propagate); block2.LinkTo(target, propagate); block1.Post("a"); // This propagates completion through to target block1.Complete(); await AssertCompletes(block1.Completion); Assert.IsTrue(await target.OutputAvailableAsync(), "target should have message waiting"); Assert.IsTrue(!target.Completion.IsCompleted, "target won't be complete until buffer empty"); // The target won't receive this, because it's already started completing (via propagation) block2.Post("b"); block2.Complete(); // Pulling this out of the buffer allows the target to complete Assert.AreEqual('a', target.Receive()); // This sholud happen pretty quickly await AssertCompletes(target.Completion); }
public async Task TestCircularLinking() { const int Iters = 200; foreach (bool sync in DataflowTestHelpers.BooleanValues) { var tcs = new TaskCompletionSource <bool>(); Func <int, IEnumerable <int> > body = i => { if (i >= Iters) { tcs.SetResult(true); } return(Enumerable.Repeat(i + 1, 1)); }; TransformManyBlock <int, int> tb = sync ? new TransformManyBlock <int, int>(body) : new TransformManyBlock <int, int>(i => Task.Run(() => body(i))); using (tb.LinkTo(tb)) { tb.Post(0); await tcs.Task; tb.Complete(); } } }
private static DataFlowPipeline <ITestGenerator> TestExecutionPipeline() { var copyBuffer = new BufferBlock <ITestGenerator>(); var expandBlock = new TransformManyBlock <ITestGenerator, ITest>( generator => generator.GenerateTests(), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 }); var processBlock = new ActionBlock <ITest>( test => test.PerformTest(), new ExecutionDataflowBlockOptions { BoundedCapacity = 1 }); copyBuffer.LinkTo( expandBlock, new DataflowLinkOptions { PropagateCompletion = true }); expandBlock.LinkTo( processBlock, new DataflowLinkOptions { PropagateCompletion = true }); return(new DataFlowPipeline <ITestGenerator>(copyBuffer, processBlock.Completion)); }
public Task Generate(List <string> inputFiles, string outputPath, int maxReadText, int maxGenerateTest, int maxWriteText) { var ExecReaxOption = new ExecutionDataflowBlockOptions(); ExecReaxOption.MaxDegreeOfParallelism = maxReadText; var ExecGenerateOption = new ExecutionDataflowBlockOptions(); ExecGenerateOption.MaxDegreeOfParallelism = maxGenerateTest; var ExecWriteOption = new ExecutionDataflowBlockOptions(); ExecWriteOption.MaxDegreeOfParallelism = maxWriteText; var readText = new TransformBlock <string, string>(async textReadPath => await ReadTextAsync(textReadPath), ExecReaxOption); var generateTests = new TransformManyBlock <string, Tests>(textCode => TestsGenerator.GenerateTests(textCode), ExecGenerateOption); var writeText = new ActionBlock <Tests>(async test => await WriteTextAsync(Path.Combine(outputPath, test.FileName), test.TestCode), ExecWriteOption); var linkOptions = new DataflowLinkOptions(); linkOptions.PropagateCompletion = true; readText.LinkTo(generateTests, linkOptions); generateTests.LinkTo(writeText, linkOptions); foreach (string file in inputFiles) { readText.Post(file); } readText.Complete(); return(writeText.Completion); }
public async Task Generate() { DataflowLinkOptions linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; ExecutionDataflowBlockOptions readOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = config.ReadThreadCount }; ExecutionDataflowBlockOptions writeOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = config.WriteThreadCount }; ExecutionDataflowBlockOptions processOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = config.ProcessThreadCount }; var readTransform = new TransformBlock <string, Task <string> >((readPath) => config.Reader.ReadTextAsync(readPath), readOptions); var sourceToTestfileTransform = new TransformManyBlock <Task <string>, PathContentPair>((readSourceTask) => config.TemplateGenerator.Generate(readSourceTask.Result), processOptions); var writeAction = new ActionBlock <PathContentPair>((pathTextPair) => config.Writer.WriteTextAsync(pathTextPair).Wait(), writeOptions); readTransform.LinkTo(sourceToTestfileTransform, linkOptions); sourceToTestfileTransform.LinkTo(writeAction, linkOptions); foreach (string readPath in config.ReadPaths) { await readTransform.SendAsync(readPath); } readTransform.Complete(); await writeAction.Completion; }
public async Task TestLinkToOptionsAsyncEnumerable() { const int Messages = 1; foreach (bool append in DataflowTestHelpers.BooleanValues) { var tb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToAsyncEnumerable); var values = new int[Messages]; var targets = new ActionBlock <int> [Messages]; for (int i = 0; i < Messages; i++) { int slot = i; targets[i] = new ActionBlock <int>(item => values[slot] = item); tb.LinkTo(targets[i], new DataflowLinkOptions { MaxMessages = 1, Append = append }); } tb.PostRange(0, Messages); tb.Complete(); await tb.Completion; for (int i = 0; i < Messages; i++) { Assert.Equal( expected: append ? i : Messages - i - 1, actual: values[i]); } } }
internal static void Run() { const string w1 = "Hello"; const string w2 = "World"; // Create a TransformManyBlock<string, char> object that splits // a string into its individual characters. var transformManyBlock = new TransformManyBlock <string, char>(s => s.ToCharArray()); // Chained buffer block var buffer = new BufferBlock <char>(); transformManyBlock.LinkTo(buffer); transformManyBlock.Completion.ContinueWith(_ => buffer.Complete()); // Post two messages to the first block. transformManyBlock.SendAsync(w1); transformManyBlock.SendAsync(w2); transformManyBlock.Complete(); transformManyBlock.Completion.Wait(); // Receive all output values from the last block. IList <char> chars; while (!buffer.TryReceiveAll(out chars)) { // Needed if we don't wait for transformManyBlock.Completion Console.Write("."); Thread.SpinWait(10000); } Console.WriteLine(chars == null ? "null" : string.Join(", ", chars)); }
public async Task FindProjectJsonAsync() { TransformManyBlock <GitHubRepo, SearchResult> repoSearchBlock = new TransformManyBlock <GitHubRepo, SearchResult>(repo => SearchRepoAsync(repo), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount * 4 //MaxDegreeOfParallelism = 1 }); ActionBlock <SearchResult> downloadFileBlock = new ActionBlock <SearchResult>(DownloadFileAsync, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount * 4 //MaxDegreeOfParallelism = 1 }); repoSearchBlock.LinkTo(downloadFileBlock, new DataflowLinkOptions() { PropagateCompletion = true }); foreach (var repo in _storage.GetAllRepos()) { if (_cancelToken.IsCancellationRequested) { break; } repoSearchBlock.Post(repo); } repoSearchBlock.Complete(); await downloadFileBlock.Completion; }
static void Main(string[] args) { var b1 = new TransformBlock <int, string>(i => new string((char)('a' + i), i)); var b2 = new TransformManyBlock <string, char>(i => i); var b3 = new ActionBlock <char>(async c => { await Task.Delay(1000).ConfigureAwait(false); Console.Write($"{c}, "); }); var b4 = new ActionBlock <char>(async c => { await Task.Delay(500).ConfigureAwait(false); Trace.Write($"{c}, "); }); var b5 = new BroadcastBlock <char>(i => i); b1.LinkTo(b2); b2.LinkTo(b5); b5.LinkTo(b3); b5.LinkTo(b4); for (int i = 1; i <= 10; i++) { b1.Post(i); } Console.WriteLine($@" b1: in {b1.InputCount} out {b1.OutputCount} b2: in {b2.InputCount} out {b2.OutputCount} b3: in {b3.InputCount} b4: in {b4.InputCount} "); Console.ReadKey(); }
public override async Task ProcessAsync() { var settings = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 10, }; var listFilesBlock = new TransformManyBlock <string, string>(_fileSystem.GetFileNames, settings); var getVehiclesBlock = new TransformManyBlock <string, IVehicle>(GetVehicles, settings); var transformBlock = new TransformBlock <IVehicle, Truck>(TransformAsync, settings); var doubleBlock = new TransformBlock <Truck, Truck>(DoubleDoorsAsync, settings); var batchBlock = new BatchBlock <Truck>(10); var saveBlock = new ActionBlock <IEnumerable <Truck> >(SaveTrucksAsync, settings); DataflowLinkOptions linkOptions = new DataflowLinkOptions() { PropagateCompletion = true }; listFilesBlock.LinkTo(getVehiclesBlock, linkOptions); getVehiclesBlock.LinkTo(transformBlock, linkOptions); transformBlock.LinkTo(doubleBlock, linkOptions); doubleBlock.LinkTo(batchBlock, linkOptions); batchBlock.LinkTo(saveBlock, linkOptions); await listFilesBlock.SendAsync(_directory); listFilesBlock.Complete(); await saveBlock.Completion; }
public async Task Start() { Reader fileReader = new Reader(); Writer fileWriter = new Writer(fFolder); var reader = new TransformBlock <string, string>(filename => fileReader.ReadFromFile(filename), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = fMaxDegreeOfParallelism }); var transformer = new TransformManyBlock <string, string>(data => GeneratorTestClasses.Start(data), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = fMaxDegreeOfParallelism }); var writer = new ActionBlock <string>(text => fileWriter.WritetoFile(text), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = fMaxDegreeOfParallelism }); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; reader.LinkTo(transformer, linkOptions); transformer.LinkTo(writer, linkOptions); foreach (string filename in fFilenames) { await reader.SendAsync(filename); } reader.Complete(); writer.Completion.Wait(); }
public async Task TestPrecanceled() { var bb = new TransformManyBlock <int, int>(DataflowTestHelpers.ToEnumerable, new ExecutionDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true) }); int ignoredValue; IList <int> ignoredValues; IDisposable link = bb.LinkTo(DataflowBlock.NullTarget <int>()); Assert.NotNull(link); link.Dispose(); Assert.False(bb.Post(42)); var t = bb.SendAsync(42); Assert.True(t.IsCompleted); Assert.False(t.Result); Assert.False(bb.TryReceiveAll(out ignoredValues)); Assert.False(bb.TryReceive(out ignoredValue)); Assert.NotNull(bb.Completion); await Assert.ThrowsAnyAsync <OperationCanceledException>(() => bb.Completion); bb.Complete(); // just make sure it doesn't throw }
public Task Generate(int maxDegreeOfParallelism, string resultPath, params string[] files) { var execOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }; var openFile = new TransformBlock <string, string>(async path => await File.ReadAllTextAsync(path), execOptions); var generateTests = new TransformManyBlock <string, TestUnit>(file => TestsGenerator.Generate(file), execOptions); var writeFile = new ActionBlock <TestUnit>( async testUnit => await File.WriteAllTextAsync( Path.Combine(resultPath, testUnit.Name) + ".cs", testUnit.Test), execOptions); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; openFile.LinkTo(generateTests, linkOptions); generateTests.LinkTo(writeFile, linkOptions); foreach (string path in files) { openFile.Post(path); } openFile.Complete(); return(writeFile.Completion); }
public ITargetBlock <string> Build(DataflowBuilderConfiguration config) { var sinkCompletions = new List <Task>(); var fileParser = new TransformManyBlock <string, UnparsedRecord>(fileLocation => config.FileParser.Parse(fileLocation)); var invalidParsedRecordExtractor = new TransformBlock <Validity <ParsedRecord>, InvalidRecord>(validity => new InvalidRecord(validity.Data.Id, validity.InvalidReason)); ITargetBlock <InvalidRecord> invalidRecordSink = DataflowExtensions.CreateBatchedActionBlock <InvalidRecord>(BatchSize, config.InvalidRecordSink.Persist); invalidParsedRecordExtractor.LinkTo(invalidRecordSink); sinkCompletions.Add(invalidRecordSink.Completion); foreach (RecordPipeline recordPipeline in config.RecordPipelines) { ITargetBlock <Validity <ParsedRecord> > sink = DataflowExtensions.ToTargetPipeline( new TransformBlock <Validity <ParsedRecord>, ParsedRecord>(v => v.Data), DataflowExtensions.CreateBatchedActionBlock <ParsedRecord>(BatchSize, recordPipeline.ValidRecordSink.Persist)); fileParser.LinkTo(recordPipeline.Pipeline, DataflowExtensions.PropagateCompletion, r => recordPipeline.RoutingPredicate(r)); recordPipeline.Pipeline.LinkTo(sink, DataflowExtensions.PropagateCompletion, validity => validity.IsValid); recordPipeline.Pipeline.LinkTo(invalidParsedRecordExtractor, validity => !validity.IsValid); sinkCompletions.Add(sink.Completion); } // We need to add unroutable last since it will greedily accept all messages. This won't work if any of // the record pipeline have a limited capacity since they could overflow into this link var unroutable = new TransformBlock <UnparsedRecord, InvalidRecord>(unparsed => new InvalidRecord(unparsed.Id, "Record did not match any of the pipelines' predicates.")); fileParser.LinkTo(unroutable, DataflowExtensions.PropagateCompletion); invalidParsedRecordExtractor.CompleteWhenAllOrAnyFaulted(config.RecordPipelines.Select(p => p.Pipeline.Completion).ToList()); invalidRecordSink.CompleteWhenAllOrAnyFaulted(new List <Task> { invalidParsedRecordExtractor.Completion, unroutable.Completion }); return(DataflowExtensions.EncapsulateTarget(fileParser, Task.WhenAll(sinkCompletions))); }
static public void ProcessingByTPL_StraightForwardImplementation() { const string pathToFiles = @"..\..\..\..\DataFiles"; string[] files = Directory.GetFiles(pathToFiles, "*.txt", SearchOption.AllDirectories); var loadDataFromFileBlock = new TransformBlock<string[], List<CustomerTextData>>(fileItems => { var factory = new CustomerTextDataFactory(); return new List<CustomerTextData>(Array.ConvertAll(fileItems, factory.LoadFromFile)); }); var filterBlock = new TransformBlock<List<CustomerTextData>, List<CustomerTextData>>(textDataList => { var filter = new FilterTextData(5); return textDataList.Where(filter.Run).ToList(); }); var toListBlock = new TransformManyBlock<List<CustomerTextData>, CustomerTextData>(textDataList => { var queue = new ConcurrentQueue<CustomerTextData>(); textDataList.ForEach(queue.Enqueue); return queue; }); var action = new ActionBlock<CustomerTextData>(textData => { var weight = new WeightTextData(); int result = weight.Run(textData); Trace.WriteLine(result); Console.WriteLine(result); }); loadDataFromFileBlock.LinkTo(filterBlock); filterBlock.LinkTo(toListBlock); toListBlock.LinkTo(action); loadDataFromFileBlock.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)filterBlock).Fault(t.Exception); else filterBlock.Complete(); }); filterBlock.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)toListBlock).Fault(t.Exception); else toListBlock.Complete(); }); toListBlock.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)action).Fault(t.Exception); else action.Complete(); }); loadDataFromFileBlock.Post(files); loadDataFromFileBlock.Complete(); action.Completion.Wait(); }
public void NullResultTest () { bool received = false; var transformMany = new TransformManyBlock<int, int> (i => (IEnumerable<int>)null); var action = new ActionBlock<int> (i => received = true); transformMany.LinkTo (action); Assert.IsTrue (transformMany.Post (1)); transformMany.Complete (); Assert.IsTrue (transformMany.Completion.Wait (100)); Assert.IsFalse (received); }
public void DeferredUsageTest () { int insIndex = -1; int[] array = new int[5 + 3]; var evt = new CountdownEvent (array.Length); var block = new ActionBlock<int> (i => { array[Interlocked.Increment (ref insIndex)] = i; evt.Signal (); }); var trsm = new TransformManyBlock<int, int> (i => Enumerable.Range (0, i)); trsm.Post (5); trsm.Post (3); trsm.LinkTo (block); evt.Wait (); CollectionAssert.AreEquivalent (new int[] { 0, 1, 2, 3, 4, 0, 1, 2 }, array); }
private static IPropagatorBlock<string, KeyValuePair<string, long>> CreateCustomBlock() { var directoryBrowserBlock = new TransformManyBlock<string, string>(path => { var dir = new DirectoryInfo(path); return dir.EnumerateFileSystemInfos() .Select(fi => fi.FullName); }); var fileSizeCalculator = new TransformBlock<string, KeyValuePair<string, long>>(fileName => { var fi = new FileInfo(fileName); return new KeyValuePair<string, long>(fileName, fi.Length); }); directoryBrowserBlock.LinkTo(fileSizeCalculator, new DataflowLinkOptions { PropagateCompletion = true }, File.Exists); var customBlock = DataflowBlock.Encapsulate(directoryBrowserBlock, fileSizeCalculator); return customBlock; }
public async static Task<Animat> SeedAsyncWithout3MB(ScenarioSpecies species, GeoRect geoRect, Bathymetry bathymetry) { var bounds = new GeoArray(geoRect.NorthWest, geoRect.NorthEast, geoRect.SouthEast, geoRect.SouthWest, geoRect.NorthWest); var result = new Animat { ScenarioSpecies = species }; var area = bounds.Area; //Debug.WriteLine("Area: {0}",area); var transformManyBlock = new TransformManyBlock<int, Geo<float>>(count => { var geos = new List<Geo<float>>(); for (var i = 0; i < count; i++) { var location = bounds.RandomLocationWithinPerimeter(); var depth = bathymetry.Samples.GetNearestPointAsync(location).Result.Data; if (depth < -50) geos.Add(new Geo<float>(location.Latitude, location.Longitude, (float)(depth * Random.NextDouble()))); } return geos; }, new ExecutionDataflowBlockOptions { TaskScheduler = TaskScheduler.Default, BoundedCapacity = -1, MaxDegreeOfParallelism = -1, }); var bufferBlock = new BufferBlock<Geo<float>>(); transformManyBlock.LinkTo(bufferBlock); var population = (int)Math.Round(area * species.PopulationDensity); result.TotalAnimats = population; const int blockSize = 100; while (population > 0) { transformManyBlock.Post(population > blockSize ? blockSize : population); population -= blockSize; } transformManyBlock.Complete(); await transformManyBlock.Completion; IList<Geo<float>> animatGeos; if (bufferBlock.TryReceiveAll(out animatGeos)) result.Locations.AddRange(animatGeos); return result; }
private static void Main(string[] args) { var generatorBlock = new TransformManyBlock<int, string>(num => GenerateStrings(num)); var writerBlock = new TransformBlock<string, string>(str => WriteString(str), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 5 }); var finishingBlock = new ActionBlock<string>(str => { writerBlock.Completion.Wait(); Console.WriteLine(Thread.CurrentThread.ManagedThreadId + ": finished - " + str); }); generatorBlock.LinkTo(writerBlock, new DataflowLinkOptions{PropagateCompletion = true}); writerBlock.LinkTo(finishingBlock, new DataflowLinkOptions { PropagateCompletion = true }); for (var i = 1; i <= 3; i++) { Console.WriteLine("Posted " + i*10); generatorBlock.Post(i*10); } generatorBlock.Complete(); finishingBlock.Completion.Wait(); Console.WriteLine("Pipeline is finished"); Console.ReadKey(); }
public void BuildPipeline(CancellationToken token) { // create the blocks getFileUrls = new TransformManyBlock<string, string>( (url) => { List<string> fileUrls = new List<string>(); GetFileUrls(url, fileUrls); return fileUrls; } , new ExecutionDataflowBlockOptions { CancellationToken = token } ); var downloadFile = new TransformBlock<string, string>( (fileUrl) => { string fileName = Regex.Match(fileUrl, @"([^/]+$)").Value; using (var client = new WebClient()) { client.DownloadFile(fileUrl, FolderPath + "\\" + fileName); } Interlocked.Increment(ref currentFileCount); UpdateProgress((double)currentFileCount / (double)fileCount); return FolderPath + "\\" + fileName; } , new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount, CancellationToken = token, } ); var convertFile = new TransformBlock<string, string>( (fileName) => { string newFileName = ConvertedFilesFolder + @"\" + Regex.Match(fileName, @"([^\\]+$)").Value + "x"; Process process = new Process(); process.StartInfo.Arguments = string.Format(@" -nme -oice {0} {1}", fileName, newFileName); process.StartInfo.FileName = @"c:\Program Files (x86)\Microsoft Office\Office12\excelcnv.exe"; process.Start(); while (!process.WaitForExit(15000)) { process.Kill(); process.Start(); } Interlocked.Increment(ref convertedFilesCount); UpdateConversionProgress((double)convertedFilesCount / (double)fileCount); return newFileName; } , new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount, CancellationToken = token } ); var parseCitizenInfo = new ActionBlock<string>( (fileName) => { var referendumProcessor = new ReferendumProcessor(); citizensOfArmenia.Post(referendumProcessor.ProcessFile(fileName)); } , new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount, CancellationToken = token } ); // Link the blocks citizensOfArmenia = new BufferBlock<List<Citizen>>(); getFileUrls.LinkTo(downloadFile, new DataflowLinkOptions { PropagateCompletion = true }); downloadFile.LinkTo(convertFile, new DataflowLinkOptions { PropagateCompletion = true }); convertFile.LinkTo(parseCitizenInfo, new DataflowLinkOptions { PropagateCompletion = true }); parseCitizenInfo.Completion.ContinueWith((tsk) => citizensOfArmenia.Complete()); }
public async void Crawl() { var scraper = new TransformManyBlock<LinkInfo, LinkInfo>( async linkInfo => { var linkInfos = new List<LinkInfo>(); try { var response = await linkInfo.Uri.GetResponse(); if (!IsSuccessStatus(response, linkInfo)) return linkInfos; var html = await response.Content.ReadAsStringAsync(); var doc = new HtmlDocument(); doc.LoadHtml(html); linkInfos = doc.ToLinkInfos(linkInfo.Uri); Log(Context.GoodHTML, linkInfo.Uri.AbsoluteUri); } catch (Exception error) { Log(Context.BadHTML, "Error: {0} (URL: {1})", error.Message, linkInfo.Uri); } return linkInfos; }, new ExecutionDataflowBlockOptions() { CancellationToken = cts.Token }); var fetcher = new ActionBlock<LinkInfo>( async linkInfo => { try { var fileName = linkInfo.GetFileName(linkInfo.Uri, "Downloads"); if (File.Exists(fileName)) { Log(Context.DupMedia, linkInfo.Uri.AbsoluteUri); return; } var response = await linkInfo.Uri.GetResponse(); if (!IsSuccessStatus(response, linkInfo)) return; var webStream = await response.Content.ReadAsStreamAsync(); fileName.EnsurePathExists(); using (var fileStream = File.OpenWrite(fileName)) await webStream.CopyToAsync(fileStream); Log(Context.GoodMedia, linkInfo.Uri.AbsoluteUri); } catch (Exception error) { Log(Context.BadMedia, "Error: {0} (URL: {1})", error.Message, linkInfo.Uri); } }, new ExecutionDataflowBlockOptions() { CancellationToken = cts.Token, MaxDegreeOfParallelism = Environment.ProcessorCount * 12 }); scraper.Completion.SetOnlyOnFaultedCompletion(error => HandleErrors(error)); fetcher.Completion.SetOnlyOnFaultedCompletion(error => HandleErrors(error)); scraper.LinkTo(scraper, new Predicate<LinkInfo>(li => li.Kind == LinkKind.HTML)); scraper.LinkTo(fetcher, new Predicate<LinkInfo>(li => li.Kind == LinkKind.Media)); scraper.Post(new LinkInfo(new Uri("http://www.bbc.com/news/"))); try { await Task.WhenAll(scraper.Completion, fetcher.Completion); } catch (OperationCanceledException) { } catch (Exception error) { Log(Context.Failure, "Error: " + error.Message); } if (OnFinished != null) OnFinished(this, EventArgs.Empty); }
public async Task TransformManyEnumerableToAction() { var data = new[] { 1 }; var tm = new TransformManyBlock<int, int>(i => data); int completedCount = 0; var c = new ActionBlock<int>(i => completedCount++); tm.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true }); tm.PostRange(0, Iterations); tm.Complete(); await c.Completion; Assert.Equal(expected: Iterations, actual: completedCount); }
public static async Task PurgeExpiredAsync(CloudBlobContainer inboxContainer) { Requires.NotNull(inboxContainer, "inboxContainer"); var deleteBlobsExpiringBefore = DateTime.UtcNow; int purgedBlobCount = 0; var searchExpiredBlobs = new TransformManyBlock<CloudBlobContainer, ICloudBlob>( async c => { try { var results = await c.ListBlobsSegmentedAsync( string.Empty, useFlatBlobListing: true, pageSize: 50, details: BlobListingDetails.Metadata, options: new BlobRequestOptions(), operationContext: null); return from blob in results.OfType<ICloudBlob>() let expires = DateTime.Parse(blob.Metadata[ExpirationDateMetadataKey]) where expires < deleteBlobsExpiringBefore select blob; } catch (StorageException ex) { var webException = ex.InnerException as WebException; if (webException != null) { var httpResponse = (HttpWebResponse)webException.Response; if (httpResponse.StatusCode == HttpStatusCode.NotFound) { // it's legit that some tests never created the container to begin with. return Enumerable.Empty<ICloudBlob>(); } } throw; } }, new ExecutionDataflowBlockOptions { BoundedCapacity = 4, }); var deleteBlobBlock = new ActionBlock<ICloudBlob>( blob => { Interlocked.Increment(ref purgedBlobCount); return blob.DeleteAsync(); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 4, BoundedCapacity = 100, }); searchExpiredBlobs.LinkTo(deleteBlobBlock, new DataflowLinkOptions { PropagateCompletion = true }); searchExpiredBlobs.Post(inboxContainer); searchExpiredBlobs.Complete(); await deleteBlobBlock.Completion; }
public static IPropagatorBlock<FileWord, FileWord> GetFileWordSaverBlock(Func<IRepository> repositoryFactory) { var inputBuffer = new BufferBlock<FileWord>(); var outputBuffer = new BufferBlock<FileWord>(); var batchBlockWithoutParents = new BatchBlock<FileWord>(1000); inputBuffer.LinkTo(batchBlockWithoutParents, i => i.File.FileId > 0 && i.Word.WordId > 0); inputBuffer.PropagateCompleted(batchBlockWithoutParents); var saveWithoutParentsBlock = new TransformManyBlock<FileWord[], FileWord>( async fileWords => { return await SaveEntities(repositoryFactory, fileWords, true); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Utils.GlobalMaxDegreeOfParallelism }); batchBlockWithoutParents.LinkToAndPropagateCompleted(saveWithoutParentsBlock); saveWithoutParentsBlock.LinkTo(outputBuffer); var batchBlockWithParents = new BatchBlock<FileWord>(300); // MaxMessages value means inputBuffer will unlink automatically from batchBlockWithParents after 300 messages. // This is required to prohibit "stealing" of workload from saveWithoutParentsBlock inputBuffer.LinkTo(batchBlockWithParents, new DataflowLinkOptions { MaxMessages = 300 }); inputBuffer.PropagateCompleted(batchBlockWithParents); var saveWithParentsBlock = new TransformManyBlock<FileWord[], FileWord>( async fileWords => { var res = await SaveEntities(repositoryFactory, fileWords, false); FileWord fileWord; if (inputBuffer.TryReceive(out fileWord)) // This unblocks inputBuffer due to unlinking from batchBlockWithParents { if (fileWord.File.FileId > 0 && fileWord.Word.WordId > 0) { batchBlockWithoutParents.Post(fileWord); } else { batchBlockWithParents.Post(fileWord); } } // Link again for another 300 messages inputBuffer.LinkTo(batchBlockWithParents, new DataflowLinkOptions { MaxMessages = 300 }); return res; }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 // If we are saving files/words also we cannot do this concurrently or duplicates of files/words may be inserted }); batchBlockWithParents.LinkToAndPropagateCompleted(saveWithParentsBlock); saveWithParentsBlock.LinkTo(outputBuffer); saveWithoutParentsBlock.Completion.ContinueWith( t => ((IDataflowBlock)outputBuffer).Fault(t.Exception), TaskContinuationOptions.OnlyOnFaulted); saveWithParentsBlock.Completion.ContinueWith( t => ((IDataflowBlock)outputBuffer).Fault(t.Exception), TaskContinuationOptions.OnlyOnFaulted); Task.WhenAll(saveWithoutParentsBlock.Completion, saveWithParentsBlock.Completion) .ContinueWith(t => outputBuffer.Complete(), TaskContinuationOptions.NotOnFaulted); return DataflowBlock.Encapsulate(inputBuffer, outputBuffer); }
public Task Execute(CancellationToken token) { var options = new DataflowBlockOptions {CancellationToken = token}; _buffer = new BufferBlock<long>(options); var hydrate = new TransformBlock<long, Summoner>(id => { var summoner = _lookup.Hydrate(id); return summoner; }, new ExecutionDataflowBlockOptions { CancellationToken = token, MaxDegreeOfParallelism = 2 }); var store = new TransformBlock<Summoner, Summoner>(summoner => { if (summoner != null) _storage.Store(summoner); return summoner; }, new ExecutionDataflowBlockOptions {CancellationToken = token, MaxDegreeOfParallelism = 2}); var crawl = new TransformManyBlock<Summoner, FellowPlayerInfo>(async summoner => { var summoners = new List<FellowPlayerInfo>(); var games = new List<PlayerGameStats>(); if (summoner != null) { await _crawler.Crawl(summoner, summoners.Add, games.Add); } return summoners; }, new ExecutionDataflowBlockOptions {CancellationToken = token, MaxDegreeOfParallelism = 2}); var storeNextBatch = new ActionBlock<FellowPlayerInfo>(async info => { if (info != null) { var data = await _lookup.Lookup(info.summonerId); _storage.StoreWhenMissing(data); } }, new ExecutionDataflowBlockOptions {CancellationToken = token, MaxDegreeOfParallelism = 2}); _buffer.LinkTo(hydrate, new DataflowLinkOptions {PropagateCompletion = true}); hydrate.LinkTo(store, new DataflowLinkOptions {PropagateCompletion = true}); store.LinkTo(crawl, new DataflowLinkOptions {PropagateCompletion = true}); crawl.LinkTo(storeNextBatch, new DataflowLinkOptions {PropagateCompletion = true}); return Task.Run(async () => { while (!token.IsCancellationRequested) { try { var batch = _producer.Produce((int) TimeSpan.FromDays(1).TotalMinutes, 30); foreach (var id in batch) await _buffer.SendAsync(id, token); // Start the chain _buffer.Complete(); // Wait until the chain is complete before iterating again await storeNextBatch.Completion; } catch (Exception ex) { throw; } } }, token); }
public void RunTransformManyBlockConformanceTests() { bool passed = true; #region Sync { // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++) { Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory = options => { TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => new[] { i }, (ExecutionDataflowBlockOptions)options); ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); transformManyBlock.LinkTo(actionBlock); return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false }; }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token }; passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true); } // Test chained Post/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained Post all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true; for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16; Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => new[] { i * 2 })); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i); Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive(); localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16)); Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test multiple yielded results { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { return Enumerable.Range(0, 10); }); t.Post(42); t.Complete(); for (int i = 0; i < 10; i++) { localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that OperationCanceledExceptions are ignored { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { if ((i % 2) == 0) throw new OperationCanceledException(); return new[] { i }; }); for (int i = 0; i < 10; i++) t.Post(i); t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test using a precanceled token { bool localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new ExecutionDataflowBlockOptions { CancellationToken = cts.Token }; var t = new TransformManyBlock<int, int>(i => new[] { i }, dbo); int ignoredValue; IList<int> ignoredValues; localPassed &= t.LinkTo(new ActionBlock<int>(delegate { })) != null; localPassed &= t.SendAsync(42).Result == false; localPassed &= t.TryReceiveAll(out ignoredValues) == false; localPassed &= t.Post(42) == false; localPassed &= t.OutputCount == 0; localPassed &= t.TryReceive(out ignoredValue) == false; localPassed &= t.Completion != null; t.Complete(); } catch (Exception) { localPassed = false; } Console.WriteLine(" > {0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, IEnumerable<int>>(i => { throw new InvalidOperationException(); })); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test reuse of a list and array { bool localPassed = true; foreach (bool bounded in new[] { false, true }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (IList<int> list in new IList<int>[] { new int[1], new List<int>() { 0 }, new Collection<int>() { 0 } }) { int nextExpectedValue = 1; TransformManyBlock<int, int> tmb1 = null; tmb1 = new TransformManyBlock<int, int>(i => { if (i == 1000) { tmb1.Complete(); return (IEnumerable<int>)null; } else if (dop == 1) { list[0] = i + 1; return (IEnumerable<int>)list; } else if (list is int[]) { return new int[1] { i + 1 }; } else if (list is List<int>) { return new List<int>() { i + 1 }; } else return new Collection<int>() { i + 1 }; }, dbo); TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i => { if (i != nextExpectedValue) { localPassed = false; tmb1.Complete(); } nextExpectedValue++; return i; }); tmb1.LinkTo(tmb2); tmb2.LinkTo(tmb1); tmb1.SendAsync(0).Wait(); tmb1.Completion.Wait(); } } } Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test throwing an OCE { bool localPassed = true; foreach (bool bounded in new[] { true, false }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (int mode in new[] { 0, 1, 2 }) { const int ITERS = 50; var mres = new ManualResetEventSlim(); var tmb = new TransformManyBlock<int, int>(i => { if (i < ITERS - 1) throw new OperationCanceledException(); if (mode == 0) return new int[] { i }; else if (mode == 1) return new List<int>() { i }; else return Enumerable.Repeat(i, 1); }, dbo); var ab = new ActionBlock<int>(i => { if (i != ITERS - 1) localPassed = false; mres.Set(); }); tmb.LinkTo(ab); for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait(); mres.Wait(); } } } Console.WriteLine("{0}: Canceled invocation", localPassed ? "Success" : "Failure"); passed &= localPassed; } } #endregion #region Async { // Do everything twice - once through OfferMessage and Once through Post for (FeedMethod feedMethod = FeedMethod._First; passed & feedMethod < FeedMethod._Count; feedMethod++) { Func<DataflowBlockOptions, TargetProperties<int>> transformManyBlockFactory = options => { TransformManyBlock<int, int> transformManyBlock = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }), (ExecutionDataflowBlockOptions)options); ActionBlock<int> actionBlock = new ActionBlock<int>(i => TrackCaptures(i), (ExecutionDataflowBlockOptions)options); transformManyBlock.LinkTo(actionBlock); return new TargetProperties<int> { Target = transformManyBlock, Capturer = actionBlock, ErrorVerifyable = false }; }; CancellationTokenSource cancellationSource = new CancellationTokenSource(); var defaultOptions = new ExecutionDataflowBlockOptions(); var dopOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; var mptOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 10 }; var cancellationOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, MaxMessagesPerTask = 100, CancellationToken = cancellationSource.Token }; passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 1, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, defaultOptions, 10, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, dopOptions, 1000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.None, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, mptOptions, 10000, Intervention.Complete, null, feedMethod, true); passed &= FeedTarget(transformManyBlockFactory, cancellationOptions, 10000, Intervention.Cancel, cancellationSource, feedMethod, true); } // Test chained Post/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) { network.Post(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained Post/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync/Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) { network.SendAsync(i); localPassed &= (((IReceivableSourceBlock<int>)network).Receive() == i * 16); } Console.WriteLine("{0}: Chained SendAsync/Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained Post all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); for (int i = 0; i < ITERS; i++) localPassed &= network.Post(i) == true; for (int i = 0; i < ITERS; i++) localPassed &= ((IReceivableSourceBlock<int>)network).Receive() == i * 16; Console.WriteLine("{0}: Chained Post all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test chained SendAsync all then Receive { bool localPassed = true; const int ITERS = 2; var network = Chain<TransformManyBlock<int, int>, int>(4, () => new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i * 2 }))); var tasks = new Task[ITERS]; for (int i = 1; i <= ITERS; i++) tasks[i - 1] = network.SendAsync(i); Task.WaitAll(tasks); int total = 0; for (int i = 1; i <= ITERS; i++) total += ((IReceivableSourceBlock<int>)network).Receive(); localPassed &= (total == ((ITERS * (ITERS + 1)) / 2 * 16)); Console.WriteLine("{0}: Chained SendAsync all then Receive", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test multiple yielded results { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => Task.Factory.StartNew(() => (IEnumerable<int>)Enumerable.Range(0, 10).ToArray())); t.Post(42); t.Complete(); for (int i = 0; i < 10; i++) { localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: Test multiple yielded results", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that OperationCanceledExceptions are ignored { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { if ((i % 2) == 0) throw new OperationCanceledException(); return new[] { i }; }); for (int i = 0; i < 10; i++) t.Post(i); t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that null tasks are ignored { bool localPassed = true; var t = new TransformManyBlock<int, int>(i => { if ((i % 2) == 0) return null; return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }); }); for (int i = 0; i < 10; i++) t.Post(i); t.Complete(); for (int i = 0; i < 10; i++) { if ((i % 2) != 0) localPassed &= t.Receive() == i; } t.Completion.Wait(); Console.WriteLine("{0}: OperationCanceledExceptions are ignored", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test that null tasks are ignored when a reordering buffer is in place { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => { if (i == 0) { Task.Delay(1000).Wait(); return null; } return Task.Factory.StartNew(() => (IEnumerable<int>)new[] { i }); }), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2 }); t.Post(0); t.Post(1); try { localPassed &= t.Receive(TimeSpan.FromSeconds(4)) == 1; } catch { localPassed = false; } Console.WriteLine("{0}: null tasks are ignored with reordering buffer", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting from the delegate { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => { throw new InvalidOperationException(); })); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted from delegate handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test faulting from the task { bool localPassed = true; var t = new TransformManyBlock<int, int>(new Func<int, Task<IEnumerable<int>>>(i => Task<IEnumerable<int>>.Factory.StartNew(() => { throw new InvalidOperationException(); }))); t.Post(42); t.Post(1); t.Post(2); t.Post(3); try { t.Completion.Wait(); } catch { } localPassed &= t.Completion.IsFaulted; localPassed &= SpinWait.SpinUntil(() => t.InputCount == 0, 500); localPassed &= SpinWait.SpinUntil(() => t.OutputCount == 0, 500); localPassed &= t.Post(4) == false; Console.WriteLine(" > {0}: Faulted from task handled correctly", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test reuse of a list and array { bool localPassed = true; foreach (bool bounded in new[] { false, true }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (IList<int> list in new IList<int>[] { new int[1], new List<int>() { 0 }, new Collection<int>() { 0 } }) { int nextExpectedValue = 1; TransformManyBlock<int, int> tmb1 = null; tmb1 = new TransformManyBlock<int, int>(i => { return Task.Factory.StartNew(() => { if (i == 1000) { tmb1.Complete(); return (IEnumerable<int>)null; } else if (dop == 1) { list[0] = i + 1; return (IEnumerable<int>)list; } else if (list is int[]) { return new int[1] { i + 1 }; } else if (list is List<int>) { return new List<int>() { i + 1 }; } else return new Collection<int>() { i + 1 }; }); }, dbo); TransformBlock<int, int> tmb2 = new TransformBlock<int, int>(i => { if (i != nextExpectedValue) { localPassed = false; tmb1.Complete(); } nextExpectedValue++; return i; }); tmb1.LinkTo(tmb2); tmb2.LinkTo(tmb1); tmb1.SendAsync(0).Wait(); tmb1.Completion.Wait(); } } } Console.WriteLine(" > {0}: Reuse of a list and array", localPassed ? "Success" : "Failure"); passed &= localPassed; } // Test throwing an OCE { bool localPassed = true; foreach (bool bounded in new[] { true, false }) { for (int dop = 1; dop < Environment.ProcessorCount; dop++) { var dbo = bounded ? new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop, BoundedCapacity = 2 } : new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = dop }; foreach (int mode in new[] { 0, 1, 2 }) { const int ITERS = 50; var mres = new ManualResetEventSlim(); var tmb = new TransformManyBlock<int, int>(i => { var cts = new CancellationTokenSource(); return Task.Factory.StartNew(() => { if (i < ITERS - 1) { cts.Cancel(); cts.Token.ThrowIfCancellationRequested(); } if (mode == 0) return new int[] { i }; else if (mode == 1) return new List<int>() { i }; else return Enumerable.Repeat(i, 1); }, cts.Token); }, dbo); var ab = new ActionBlock<int>(i => { if (i != ITERS - 1) localPassed = false; mres.Set(); }); tmb.LinkTo(ab); for (int i = 0; i < ITERS; i++) tmb.SendAsync(i).Wait(); mres.Wait(); } } } Console.WriteLine(" > {0}: Canceled invocation", localPassed ? "Success" : "Failure"); passed &= localPassed; } } #endregion Assert.True(passed, "Test failed."); }
static void Main(string[] args) { // // Create the members of the pipeline. // // Downloads the requested resource as a string. var downloadString = new TransformBlock<string, string>(uri => { Console.WriteLine("Downloading '{0}'...", uri); return new WebClient().DownloadString(uri); }); // Separates the specified text into an array of words. var createWordList = new TransformBlock<string, string[]>(text => { Console.WriteLine("Creating word list..."); // Remove common punctuation by replacing all non-letter characters // with a space character to. char[] tokens = text.ToArray(); for (int i = 0; i < tokens.Length; i++) { if (!char.IsLetter(tokens[i])) tokens[i] = ' '; } text = new string(tokens); // Separate the text into an array of words. return text.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); }); // Removes short words, orders the resulting words alphabetically, // and then remove duplicates. var filterWordList = new TransformBlock<string[], string[]>(words => { Console.WriteLine("Filtering word list..."); return words.Where(word => word.Length > 3).OrderBy(word => word) .Distinct().ToArray(); }); // Finds all words in the specified collection whose reverse also // exists in the collection. var findReversedWords = new TransformManyBlock<string[], string>(words => { Console.WriteLine("Finding reversed words..."); // Holds reversed words. var reversedWords = new ConcurrentQueue<string>(); // Add each word in the original collection to the result whose // reversed word also exists in the collection. Parallel.ForEach(words, word => { // Reverse the work. string reverse = new string(word.Reverse().ToArray()); // Enqueue the word if the reversed version also exists // in the collection. if (Array.BinarySearch<string>(words, reverse) >= 0 && word != reverse) { reversedWords.Enqueue(word); } }); return reversedWords; }); // Prints the provided reversed words to the console. var printReversedWords = new ActionBlock<string>(reversedWord => { Console.WriteLine("Found reversed words {0}/{1}", reversedWord, new string(reversedWord.Reverse().ToArray())); }); // // Connect the dataflow blocks to form a pipeline. // downloadString.LinkTo(createWordList); createWordList.LinkTo(filterWordList); filterWordList.LinkTo(findReversedWords); findReversedWords.LinkTo(printReversedWords); // // For each completion task in the pipeline, create a continuation task // that marks the next block in the pipeline as completed. // A completed dataflow block processes any buffered elements, but does // not accept new elements. // downloadString.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)createWordList).Fault(t.Exception); else createWordList.Complete(); }); createWordList.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)filterWordList).Fault(t.Exception); else filterWordList.Complete(); }); filterWordList.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)findReversedWords).Fault(t.Exception); else findReversedWords.Complete(); }); findReversedWords.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)printReversedWords).Fault(t.Exception); else printReversedWords.Complete(); }); // Process "The Iliad of Homer" by Homer. downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt"); // Mark the head of the pipeline as complete. The continuation tasks // propagate completion through the pipeline as each part of the // pipeline finishes. downloadString.Complete(); // Wait for the last block in the pipeline to process all messages. printReversedWords.Completion.Wait(); Console.ReadLine(); }
public async Task TestPrecanceled() { var bb = new TransformManyBlock<int, int>(DataflowTestHelpers.ToEnumerable, new ExecutionDataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true) }); int ignoredValue; IList<int> ignoredValues; IDisposable link = bb.LinkTo(DataflowBlock.NullTarget<int>()); Assert.NotNull(link); link.Dispose(); Assert.False(bb.Post(42)); var t = bb.SendAsync(42); Assert.True(t.IsCompleted); Assert.False(t.Result); Assert.False(bb.TryReceiveAll(out ignoredValues)); Assert.False(bb.TryReceive(out ignoredValue)); Assert.NotNull(bb.Completion); await Assert.ThrowsAnyAsync<OperationCanceledException>(() => bb.Completion); bb.Complete(); // just make sure it doesn't throw }
public void Fetch(Link initialLink) { urls = new ConcurrentDictionary<string, bool>(); cts = new CancellationTokenSource(); var cancellable = new ExecutionDataflowBlockOptions() { CancellationToken = cts.Token }; var hrefRegex = new Regex(@"(?<=<img\s.*?src\s*?=\s*?"").*?(?= "".*?>)", RegexOptions.Compiled | RegexOptions.IgnoreCase); var srcRegex = new Regex(@"(?<=<a\s.*?href\s*?=\s*?"").*?(?= "".*?>)", RegexOptions.Compiled | RegexOptions.IgnoreCase); var scraper = new TransformBlock<Link, Tuple<Link, HtmlDocument>>( async link => { if (!urls.TryAdd(link.Uri.AbsoluteUri, true)) { Log(LogKind.DupHTML, link.Uri.AbsoluteUri); return new Tuple<Link, HtmlDocument>(link, null); } HtmlDocument doc; try { var response = await link.Uri.GetResponse(); if (!IsSuccessStatus(response, link)) return new Tuple<Link, HtmlDocument>(link, null); var html = await response.Content.ReadAsStringAsync(); doc = new HtmlDocument(); doc.LoadHtml(html); Log(LogKind.Scrape, link.Uri.AbsoluteUri); } catch (Exception error) { doc = null; LogError(link, error); } return new Tuple<Link, HtmlDocument>(link, doc); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 4, CancellationToken = cts.Token }); var docCaster = new BroadcastBlock<Tuple<Link, HtmlDocument>>( content => content, cancellable); //(?<=<a\s.*?href\s*?=.*?)".*?"(?=.*?>.*?</a>) var hrefParser = new TransformManyBlock<Tuple<Link, HtmlDocument>, Link>( tuple => { return (from link in tuple.Item2.DocumentNode.Descendants("a") where link.Attributes.Contains("href") let href = link.GetAttributeValue("href", "") where href.IsLink(tuple.Item1.Uri, LinkKind.HTML, UriFilter.None) select new Link(tuple.Item1.Uri, href, LinkKind.HTML)).Distinct(); }, cancellable); var srcParser = new TransformManyBlock<Tuple<Link, HtmlDocument>, Link>( tuple => { return (from img in tuple.Item2.DocumentNode.Descendants("img") where img.Attributes.Contains("src") let src = img.GetAttributeValue("src", "") where src.IsLink(tuple.Item1.Uri, LinkKind.Image, UriFilter.None) select new Link(tuple.Item1.Uri, src, LinkKind.Image)).Distinct(); }, cancellable); var fetcher = new ActionBlock<Link>( async link => { try { var fileName = link.GetFileName("Downloads", false); if (File.Exists(fileName)) { Log(LogKind.DupMedia, link.Uri.AbsoluteUri); return; } var response = await link.Uri.GetResponse(); if (!IsSuccessStatus(response, link)) return; EnsurePathExists(fileName); var webStream = await response.Content.ReadAsStreamAsync(); using (var fileStream = File.OpenWrite(fileName)) await webStream.CopyToAsync(fileStream); Log(LogKind.Fetch, Path.GetFileName(fileName)); } catch (Exception error) { LogError(link, error); } }, cancellable); scraper.Completion.SetOnlyOnFaultedCompletion(error => HandleFailure(error)); docCaster.Completion.SetOnlyOnFaultedCompletion(error => HandleFailure(error)); hrefParser.Completion.SetOnlyOnFaultedCompletion(error => HandleFailure(error)); srcParser.Completion.SetOnlyOnFaultedCompletion(error => HandleFailure(error)); fetcher.Completion.SetOnlyOnFaultedCompletion(error => HandleFailure(error)); scraper.LinkTo(docCaster); docCaster.LinkTo(hrefParser, tuple => tuple.Item2 != null); docCaster.LinkTo(srcParser, tuple => tuple.Item2 != null); docCaster.LinkTo(DataflowBlock.NullTarget<Tuple<Link, HtmlDocument>>()); hrefParser.LinkTo(scraper); srcParser.LinkTo(fetcher); tasks.Add(scraper.Completion); tasks.Add(docCaster.Completion); tasks.Add(hrefParser.Completion); tasks.Add(srcParser.Completion); tasks.Add(fetcher.Completion); scraper.Post(initialLink); }
/// <summary> /// Purges all blobs set to expire prior to the specified date. /// </summary> /// <param name="deleteBlobsExpiringBefore"> /// All blobs scheduled to expire prior to this date will be purged. The default value /// is interpreted as <see cref="DateTime.UtcNow"/>. /// </param> /// <returns>The task representing the asynchronous operation.</returns> public async Task PurgeBlobsExpiringBeforeAsync(DateTime deleteBlobsExpiringBefore = default(DateTime)) { if (deleteBlobsExpiringBefore == default(DateTime)) { deleteBlobsExpiringBefore = DateTime.UtcNow; } Requires.Argument(deleteBlobsExpiringBefore.Kind == DateTimeKind.Utc, "expirationUtc", "UTC required."); var searchExpiredDirectoriesBlock = new TransformManyBlock<CloudBlobContainer, CloudBlobDirectory>( async c => { var results = await c.ListBlobsSegmentedAsync(); return from directory in results.OfType<CloudBlobDirectory>() let expires = DateTime.Parse(directory.Uri.Segments[directory.Uri.Segments.Length - 1].TrimEnd('/')) where expires < deleteBlobsExpiringBefore select directory; }, new ExecutionDataflowBlockOptions { BoundedCapacity = 4, }); var deleteDirectoryBlock = new TransformManyBlock<CloudBlobDirectory, CloudBlockBlob>( async directory => { var results = await directory.ListBlobsSegmentedAsync(); return results.OfType<CloudBlockBlob>(); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, BoundedCapacity = 4, }); var deleteBlobBlock = new ActionBlock<CloudBlockBlob>( blob => blob.DeleteAsync(), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 4, BoundedCapacity = 100, }); searchExpiredDirectoriesBlock.LinkTo(deleteDirectoryBlock, new DataflowLinkOptions { PropagateCompletion = true }); deleteDirectoryBlock.LinkTo(deleteBlobBlock, new DataflowLinkOptions { PropagateCompletion = true }); searchExpiredDirectoriesBlock.Post(this.container); searchExpiredDirectoriesBlock.Complete(); await deleteBlobBlock.Completion; }
public async static Task<Animat> SeedAsync(ScenarioSpecies species, GeoRect geoRect, Bathymetry bathymetry) { var yxzFileName = Path.Combine(Path.GetTempPath(), Path.GetFileNameWithoutExtension(Path.GetRandomFileName()) + ".txt"); bathymetry.ToYXZ(yxzFileName, -1); var mbs = new C3mbs(); mbsRESULT mbsResult; if (mbsRESULT.OK != (mbsResult = mbs.SetOutputDirectory(Path.GetTempPath()))) throw new AnimatInterfaceMMBSException("SetOutputDirectory Error:" + mbs.ResultToTc(mbsResult)); var config = mbs.GetConfiguration(); config.enabled = false; // binary output enabled/disabled config.durationLess = true; // make sure we're in durationless mode. mbs.SetConfiguration(config); mbsResult = mbs.LoadBathymetryFromTextFile(yxzFileName); if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException("Bathymetry failed to load: " + mbs.ResultToTc(mbsResult)); mbsResult = mbs.AddSpecies(species.SpeciesDefinitionFilePath); if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException(string.Format("C3mbs::AddSpecies FATAL error {0} for species {1}", mbs.ResultToTc(mbsResult), species.SpeciesDefinitionFilePath)); var bounds = new GeoArray(geoRect.NorthWest, geoRect.NorthEast, geoRect.SouthEast, geoRect.SouthWest, geoRect.NorthWest); var result = new Animat { ScenarioSpecies = species }; var area = bounds.Area; //Debug.WriteLine("Area: {0}",area); var transformManyBlock = new TransformManyBlock<int, Geo<float>>(count => { var geos = new List<Geo<float>>(); for (var i = 0; i < count; i++) { var location = bounds.RandomLocationWithinPerimeter(); var depth = bathymetry.Samples.GetNearestPointAsync(location).Result.Data; mbsRESULT retval; lock (mbs) retval = mbs.AddIndividualAnimat(0, new mbsPosition { latitude = location.Latitude, longitude = location.Longitude, depth = 0 }); if (mbsRESULT.OK == retval) geos.Add(new Geo<float>(location.Latitude, location.Longitude, (float)(depth * Random.NextDouble()))); } return geos; }, new ExecutionDataflowBlockOptions { TaskScheduler = TaskScheduler.Default, BoundedCapacity = -1, MaxDegreeOfParallelism = -1, }); var bufferBlock = new BufferBlock<Geo<float>>(); transformManyBlock.LinkTo(bufferBlock); var population = (int)Math.Round(area * species.PopulationDensity); result.TotalAnimats = population; const int blockSize = 100; while (population > 0) { transformManyBlock.Post(population > blockSize ? blockSize : population); population -= blockSize; } transformManyBlock.Complete(); await transformManyBlock.Completion; //mbsResult = mbs.InitializeRun(); //if (mbsRESULT.OK == mbsResult) while (mbsRUNSTATE.INITIALIZING == mbs.GetRunState()) Thread.Sleep(1); //else throw new AnimatInterfaceMMBSException("C3mbs::Initialize FATAL error " + mbs.ResultToTc(mbsResult)); mbsResult = mbs.FinishRun(); if (mbsRESULT.OK != mbsResult) throw new AnimatInterfaceMMBSException("C3mbs::FinishRun FATAL error " + mbs.ResultToTc(mbsResult)); IList<Geo<float>> animatGeos; if (bufferBlock.TryReceiveAll(out animatGeos)) result.Locations.AddRange(animatGeos); return result; }