public async Task Can_TransformBlock_Be_Observable_And_Linked() { var totalValuesRecievedByActionBlock = new List <int>(); var totalValuesRecievedInStream = new List <int>(); var streamCompletionSource = new TaskCompletionSource <bool>(); var transformBlock = new TransformBlock <int, int>(_ => _); var actionBlock = new ActionBlock <int>(x => Task.Delay(1000).ContinueWith(_ => totalValuesRecievedByActionBlock.Add(x)), new ExecutionDataflowBlockOptions() { BoundedCapacity = 1 }); var subscription = transformBlock.AsObservable().Subscribe(x => totalValuesRecievedInStream.Add(x), () => streamCompletionSource.SetResult(true)); var actionLink = transformBlock.LinkTo(actionBlock, new DataflowLinkOptions() { PropagateCompletion = true }); foreach (var item in Enumerable.Range(0, 100)) { transformBlock.Post(item); } await Task.Delay(2000); transformBlock.Complete(); await actionBlock.Completion; await streamCompletionSource.Task; Assert.AreEqual(totalValuesRecievedByActionBlock, totalValuesRecievedInStream); }
public void ExampleBlockAsObservable() { var fizz = new TransformBlock <int, string>( async value => value % 3 == 0 ? "Fizz" : ""); var assert = fizz.AsObservable().Subscribe(result => Assert.Equal("Fizz", result)); fizz.Post(3); fizz.Post(9); fizz.Post(33); fizz.Complete(); fizz.Completion.Wait(); assert.Dispose(); }
public ItemHandler() { var options = new ExecutionDataflowBlockOptions() { BoundedCapacity = DataflowBlockOptions.Unbounded, MaxDegreeOfParallelism = Environment.ProcessorCount, EnsureOrdered = true }; PostProcessBlock = new TransformBlock <int, int>((Func <int, Task <int> >)PostProcess, options); var output = PostProcessBlock.AsObservable().Subscribe(x => { Console.WriteLine($"Value Output: {x}"); OutputValues.Add(x); }); }
private void LinkPipeline() { var linkOptions = new DataflowLinkOptions() { PropagateCompletion = true }; buffer.LinkTo(step1, linkOptions); step1.LinkTo(step2, linkOptions); finalStep = step2.AsObservable(); finalStepSubscription = finalStep.Subscribe( x => { Task.Delay(rnd.Next(100, 500)).Wait(); Console.WriteLine($"Final,{x.Id}"); FinalList.Enqueue(x); }, () => { tcs.SetResult(true); }); }
public void CreatePipeline() { var options = new ExecutionDataflowBlockOptions() { BoundedCapacity = 13, MaxDegreeOfParallelism = 10, EnsureOrdered = true }; buffer = new BufferBlock <Message>(); xForm1 = new TransformBlock <Message, Message>(x => { Console.WriteLine($"{DateTime.Now.TimeOfDay} - Started Id: {x.Id}"); Task.Delay(rnd.Next(1000, 3000)).Wait(); Console.WriteLine($"{DateTime.Now.TimeOfDay} - Finished Id: {x.Id}"); return(x); }, options); output = xForm1.AsObservable <Message>(); areWeDoneYet = new TaskCompletionSource <bool>(); }
public async Task TestAsObservableAndAsObserver_ErrorPropagation() { // Test that exceptional data flows when exception occurs before and after subscription foreach (bool beforeSubscription in DataflowTestHelpers.BooleanValues) { var tb = new TransformBlock<int, int>(i => { if (i == 42) throw new InvalidOperationException("uh oh"); return i; }); if (beforeSubscription) { tb.Post(42); await Assert.ThrowsAsync<InvalidOperationException>(async () => await tb.Completion); } ITargetBlock<int>[] targets = Enumerable.Range(0, 3).Select(_ => new WriteOnceBlock<int>(i => i)).ToArray(); foreach (var target in targets) { tb.AsObservable().Subscribe(target.AsObserver()); } if (!beforeSubscription) { tb.Post(42); await Assert.ThrowsAsync<InvalidOperationException>(() => tb.Completion); } foreach (var target in targets) { await Assert.ThrowsAsync<AggregateException>(() => target.Completion); } } }
public static void Main(string[] args) { var testObs = new TransformBlock <int, int>(item => item * item); var o = testObs.AsObservable(); o.Subscribe(i => Console.WriteLine("Test Obs received {0}", i.ToString())); for (int i = 0; i < 5; i++) { testObs.Post(i); } testObs.Completion.Wait(); Console.ReadKey(); var buffer = new BufferBlock <int>(); IObservable <int> integers = buffer.AsObservable(); integers.Subscribe(data => Console.WriteLine(data), ex => Console.WriteLine(ex), () => Console.WriteLine("Done")); buffer.Post(13); buffer.Post(14); buffer.Post(15); buffer.Post(16); Console.ReadKey(); IObservable <DateTimeOffset> ticks = Observable.Interval(TimeSpan.FromSeconds(1)) .Timestamp() .Select(x => x.Timestamp) .Take(5); var display = new ActionBlock <DateTimeOffset>(x => Console.WriteLine(x)); ticks.Subscribe(display.AsObserver()); Console.ReadKey(); try { display.Completion.Wait(); Trace.WriteLine("Done."); } catch (Exception ex) { Trace.WriteLine(ex); } try { var multiplyBlock = new TransformBlock <int, int>(item => { // if (item == 1) // throw new InvalidOperationException("Blech."); return(item * 2); }); var subtractBlock = new TransformBlock <int, int>(item => item - 2); multiplyBlock.LinkTo(subtractBlock); var printing = new ActionBlock <int>(i => Console.WriteLine("Received {0}", i.ToString())); subtractBlock.LinkTo(printing, new DataflowLinkOptions { PropagateCompletion = true }); IObservable <int> obs = subtractBlock.AsObservable(); obs.Subscribe(i => Console.WriteLine("Received Observable {0}", i.ToString())); for (int i = 0; i < 10; i++) { multiplyBlock.Post(i); subtractBlock.Post(i); } Console.ReadLine(); //printing.Completion.Wait(); } catch (AggregateException exception) { AggregateException ex = exception.Flatten(); System.Diagnostics.Trace.WriteLine(ex.InnerException); } Console.WriteLine("Hello World!"); Console.ReadLine(); }
public async Task CompressAndEncrypt( Stream streamSource, Stream streamDestination, CancellationTokenSource cts = null) { cts = cts ?? new CancellationTokenSource(); var compressorOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, BoundedCapacity = _boundedCapacity, CancellationToken = cts.Token }; var inputBuffer = new BufferBlock <CompressingDetails>( new DataflowBlockOptions { CancellationToken = cts.Token, BoundedCapacity = _boundedCapacity }); var compressor = new TransformBlock <CompressingDetails, CompressedDetails>( async details => { ChunkBytes compressedData = await IOUtils.Compress(details.Bytes); return(new CompressedDetails { Bytes = compressedData, ChunkSize = details.ChunkSize, Sequence = details.Sequence, CompressedDataSize = new ChunkBytes(BitConverter.GetBytes(compressedData.Length)) }); }, compressorOptions); var encryptor = new TransformBlock <CompressedDetails, EncryptDetails>( async details => { var data = IOUtils.CombineByteArrays(details.CompressedDataSize, details.ChunkSize, details.Bytes); var encryptedData = await IOUtils.Encrypt(data); return(new EncryptDetails { Bytes = encryptedData, Sequence = details.Sequence, EncryptedDataSize = new ChunkBytes(BitConverter.GetBytes(encryptedData.Length)) }); }, compressorOptions); // Enables Rx integration with TPL Dataflow encryptor.AsObservable() .Scan((new Dictionary <int, EncryptDetails>(), 0), (state, msg) => // Runs Rx Scan operation asynchronously Observable.FromAsync(async() => { (Dictionary <int, EncryptDetails> details, int lastIndexProc) = state; details.Add(msg.Sequence, msg); while (details.ContainsKey(lastIndexProc + 1)) { msg = details[lastIndexProc + 1]; // Persists asynchronously the data; the stream could be replaced // with a network stream to send the data across the wire. await streamDestination.WriteAsync(msg.EncryptedDataSize.Bytes, 0, msg.EncryptedDataSize.Length); await streamDestination.WriteAsync(msg.Bytes.Bytes, 0, msg.Bytes.Length); lastIndexProc = msg.Sequence; // the chunk of data that is processed is removed // from the local state, keeping track of the items to perform details.Remove(lastIndexProc); } return(details, lastIndexProc); }) .SingleAsync().Wait()) // Rx subscribes to TaskPoolScheduler .SubscribeOn(TaskPoolScheduler.Default).Subscribe(); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; inputBuffer.LinkTo(compressor, linkOptions); compressor.LinkTo(encryptor, linkOptions); long sourceLength = streamSource.Length; byte[] size = BitConverter.GetBytes(sourceLength); await streamDestination.WriteAsync(size, 0, size.Length); var chunkSize = (_chunkSize < sourceLength) ? _chunkSize : (int)sourceLength; int indexSequence = 0; while (sourceLength > 0) { var bytes = await IOUtils.ReadFromStream(streamSource, chunkSize); var compressingDetails = new CompressingDetails { Bytes = bytes, ChunkSize = new ChunkBytes(BitConverter.GetBytes(bytes.Length)), Sequence = ++indexSequence }; await inputBuffer.SendAsync(compressingDetails); sourceLength -= bytes.Length; if (sourceLength < chunkSize) { chunkSize = (int)sourceLength; } if (sourceLength == 0) { inputBuffer.Complete(); } } await inputBuffer.Completion.ContinueWith(task => compressor.Complete()); await compressor.Completion.ContinueWith(task => encryptor.Complete()); await encryptor.Completion; await streamDestination.FlushAsync(); }
public async Task DecryptAndDecompress( Stream streamSource, Stream streamDestination, CancellationTokenSource cts = null) { cts = cts ?? new CancellationTokenSource(); var compressorOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, CancellationToken = cts.Token, BoundedCapacity = _boundedCapacity }; var inputBuffer = new BufferBlock <DecryptDetails>( new DataflowBlockOptions { CancellationToken = cts.Token, BoundedCapacity = _boundedCapacity }); var decryptor = new TransformBlock <DecryptDetails, DecompressionDetails>( async details => { var decryptedData = await IOUtils.Decrypt(details.Bytes); return(new DecompressionDetails { Bytes = decryptedData, Sequence = details.Sequence, }); }, compressorOptions); var decompressor = new TransformBlock <DecompressionDetails, DecompressionDetails>( async details => { var decompressedData = await IOUtils.Decompress(details.Bytes, sizeof(int) + sizeof(int)); return(new DecompressionDetails { Bytes = decompressedData, Sequence = details.Sequence }); }, compressorOptions); decompressor.AsObservable() .Scan((new Dictionary <int, DecompressionDetails>(), 0), (state, msg) => // Runs Rx Scan operation asynchronously Observable.FromAsync(async() => { (Dictionary <int, DecompressionDetails> details, int lastIndexProc) = state; details.Add(msg.Sequence, msg); while (details.ContainsKey(lastIndexProc + 1)) { msg = details[lastIndexProc + 1]; await streamDestination.WriteAsync(msg.Bytes.Bytes, 0, msg.Bytes.Length); lastIndexProc = msg.Sequence; details.Remove(lastIndexProc); } return(details, lastIndexProc); }) .SingleAsync().Wait()) // Rx subscribes to TaskPoolScheduler .SubscribeOn(TaskPoolScheduler.Default).Subscribe(); var linkOptions = new DataflowLinkOptions() { PropagateCompletion = true }; inputBuffer.LinkTo(decryptor, linkOptions); decryptor.LinkTo(decompressor, linkOptions); GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce; byte[] size = new byte[sizeof(long)]; await streamSource.ReadAsync(size, 0, size.Length); // convert the size to number long destinationLength = BitConverter.ToInt64(size, 0); streamDestination.SetLength(destinationLength); long sourceLength = streamSource.Length - sizeof(long); int index = 0; while (sourceLength > 0) { // read the encrypted chunk size size = new byte[sizeof(int)]; int sizeReadCount = await streamSource.ReadAsync(size, 0, size.Length); // convert the size back to number int storedSize = BitConverter.ToInt32(size, 0); var encryptedData = await IOUtils.ReadFromStream(streamSource, storedSize); var decryptDetails = new DecryptDetails { Bytes = encryptedData, EncryptedDataSize = new ChunkBytes(size), Sequence = ++index }; await inputBuffer.SendAsync(decryptDetails); sourceLength -= (sizeReadCount + encryptedData.Length); if (sourceLength == 0) { inputBuffer.Complete(); } } await inputBuffer.Completion.ContinueWith(task => decryptor.Complete()); await decryptor.Completion.ContinueWith(task => decompressor.Complete()); await decompressor.Completion; await streamDestination.FlushAsync(); }
public ProductTransformer() { Transformer = new TransformBlock<Worker, Product>(wrk => CreateProductAsync(wrk)); Products = Transformer.AsObservable(); }
ITargetBlock <Job> CreateJobProcessingPipeline() { var loadXml = new TransformBlock <Job, Job>(job => { try { if (ReadDocument(job)) { // For later error handling job.ReturnCode = 100; // success } else { job.ReturnCode = 200; } return(job); } catch (OperationCanceledException) { job.ReturnCode = 300; return(job); } }, TransformBlockOptions()); var validateXml = new TransformBlock <Job, Job>(job => { try { if (ValidateXml(job)) { // For later error handling job.ReturnCode = 100; } else { job.ReturnCode = 200; } return(job); } catch (OperationCanceledException) { job.ReturnCode = 300; return(job); } }, TransformBlockOptions()); var importJob = new TransformBlock <Job, Job>(job => { try { if (ProcessJob(job)) { // For later error handling job.ReturnCode = 100; // success } else { job.ReturnCode = 200; } return(job); } catch (OperationCanceledException) { job.ReturnCode = 300; return(job); } }, TransformBlockOptions()); var loadingFailed = new ActionBlock <Job>(job => CreateResponse(job), ActionBlockOptions()); var validationFailed = new ActionBlock <Job>(job => CreateResponse(job), ActionBlockOptions()); var reportImport = new ActionBlock <Job>(job => CreateResponse(job), ActionBlockOptions()); // // Connect the pipeline // loadXml.LinkTo(validateXml, job => job.ReturnCode == 100); loadXml.LinkTo(loadingFailed); validateXml.LinkTo(importJob, Job => Job.ReturnCode == 100); validateXml.LinkTo(validationFailed); //importJob.LinkTo(reportImport); var output = importJob.AsObservable(); var subscription = output.Subscribe(x => { if (x.ReturnCode == 100) { //job success Console.WriteLine($"SendToDataBase {DateTime.Now.TimeOfDay} JobId: {x.ID}"); } else { //handle fault Console.WriteLine($"Job Failed {DateTime.Now.TimeOfDay} JobId: {x.ID}"); } }); // Return the head of the network. return(loadXml); }
static void Main(string[] args) { // // Create the members of the pipeline. // // Downloads the requested resource as a string. var downloadString = new TransformBlock<string, string>(uri => { Console.WriteLine("Downloading '{0}'...", uri); return new WebClient().DownloadString(uri); }); // Separates the specified text into an array of words. var createWordList = new TransformBlock<string, string[]>(text => { Console.WriteLine("Creating word list..."); // Remove common punctuation by replacing all non-letter characters // with a space character to. char[] tokens = text.ToArray(); for (int i = 0; i < tokens.Length; i++) { if (!char.IsLetter(tokens[i])) tokens[i] = ' '; } text = new string(tokens); // Separate the text into an array of words. return text.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); }); // Removes short words, orders the resulting words alphabetically, // and then remove duplicates. var filterWordList = new TransformBlock<string[], string[]>(words => { Console.WriteLine("Filtering word list..."); return words.Where(word => word.Length > 3).OrderBy(word => word) .Distinct().ToArray(); }); // Finds all words in the specified collection whose reverse also // exists in the collection. var findReversedWords = new TransformManyBlock<string[], string>(words => { Console.WriteLine("Finding reversed words..."); // Holds reversed words. var reversedWords = new ConcurrentQueue<string>(); // Add each word in the original collection to the result whose // reversed word also exists in the collection. Parallel.ForEach(words, word => { // Reverse the work. string reverse = new string(word.Reverse().ToArray()); // Enqueue the word if the reversed version also exists // in the collection. if (Array.BinarySearch<string>(words, reverse) >= 0 && word != reverse) { reversedWords.Enqueue(word); } }); return reversedWords; }); // Prints the provided reversed words to the console. var printReversedWords = new ActionBlock<string>(reversedWord => { Console.WriteLine("Found reversed words {0}/{1}", reversedWord, new string(reversedWord.Reverse().ToArray())); }); // // Connect the dataflow blocks to form a pipeline. // downloadString.AsObservable().Subscribe(i => Console.WriteLine("Action Sub {0}", i)); var te= downloadString.AsObserver(); downloadString.LinkTo(createWordList); createWordList.LinkTo(filterWordList); filterWordList.LinkTo(findReversedWords); findReversedWords.LinkTo(printReversedWords); // // For each completion task in the pipeline, create a continuation task // that marks the next block in the pipeline as completed. // A completed dataflow block processes any buffered elements, but does // not accept new elements. // downloadString.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)createWordList).Fault(t.Exception); else createWordList.Complete(); }); createWordList.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)filterWordList).Fault(t.Exception); else filterWordList.Complete(); }); filterWordList.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)findReversedWords).Fault(t.Exception); else findReversedWords.Complete(); }); findReversedWords.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)printReversedWords).Fault(t.Exception); else printReversedWords.Complete(); }); // Process "The Iliad of Homer" by Homer. downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt"); // Mark the head of the pipeline as complete. The continuation tasks // propagate completion through the pipeline as each part of the // pipeline finishes. downloadString.Complete(); // Wait for the last block in the pipeline to process all messages. printReversedWords.Completion.Wait(); Console.ReadLine(); te.OnNext("ciao"); Console.ReadLine(); }
static void Main(string[] args) { // // Create the members of the pipeline. // //var nums = Enumerable.Range(0, 10).ToList();//.AsQueryable(); //var query = (from num in nums.AsQueryable() // where num % 2 == 0 // select num * num).Sum(); //int sum = 0; //for (int index = 0; index < nums.Count; index++) //{ // int num = nums[index]; // if (num % 2 == 0) // sum += num * num; //} // Downloads the requested resource as a string. var downloadString = new TransformBlock <string, string>(uri => { Console.WriteLine("Downloading '{0}'...", uri); return(new WebClient().DownloadString(uri)); }); // Separates the specified text into an array of words. var createWordList = new TransformBlock <string, string[]>(text => { Console.WriteLine("Creating word list..."); // Remove common punctuation by replacing all non-letter characters // with a space character to. char[] tokens = text.ToArray(); for (int i = 0; i < tokens.Length; i++) { if (!char.IsLetter(tokens[i])) { tokens[i] = ' '; } } text = new string(tokens); // Separate the text into an array of words. return(text.Split(new char[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)); }); // Removes short words, orders the resulting words alphabetically, // and then remove duplicates. var filterWordList = new TransformBlock <string[], string[]>(words => { Console.WriteLine("Filtering word list..."); return(words.Where(word => word.Length > 3).OrderBy(word => word) .Distinct().ToArray()); }); // Finds all words in the specified collection whose reverse also // exists in the collection. var findReversedWords = new TransformManyBlock <string[], string>(words => { Console.WriteLine("Finding reversed words..."); // Holds reversed words. var reversedWords = new ConcurrentQueue <string>(); // Add each word in the original collection to the result whose // reversed word also exists in the collection. Parallel.ForEach(words, word => { // Reverse the work. string reverse = new string(word.Reverse().ToArray()); // Enqueue the word if the reversed version also exists // in the collection. if (Array.BinarySearch <string>(words, reverse) >= 0 && word != reverse) { reversedWords.Enqueue(word); } }); return(reversedWords); }); // Prints the provided reversed words to the console. var printReversedWords = new ActionBlock <string>(reversedWord => { Console.WriteLine("Found reversed words {0}/{1}", reversedWord, new string(reversedWord.Reverse().ToArray())); }); // // Connect the dataflow blocks to form a pipeline. // downloadString.AsObservable().Subscribe(i => Console.WriteLine("Action Sub {0}", i)); var te = downloadString.AsObserver(); downloadString.LinkTo(createWordList); createWordList.LinkTo(filterWordList); filterWordList.LinkTo(findReversedWords); findReversedWords.LinkTo(printReversedWords); // // For each completion task in the pipeline, create a continuation task // that marks the next block in the pipeline as completed. // A completed dataflow block processes any buffered elements, but does // not accept new elements. // downloadString.Completion.ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)createWordList).Fault(t.Exception); } else { createWordList.Complete(); } }); createWordList.Completion.ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)filterWordList).Fault(t.Exception); } else { filterWordList.Complete(); } }); filterWordList.Completion.ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)findReversedWords).Fault(t.Exception); } else { findReversedWords.Complete(); } }); findReversedWords.Completion.ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)printReversedWords).Fault(t.Exception); } else { printReversedWords.Complete(); } }); // Process "The Iliad of Homer" by Homer. downloadString.Post("http://www.gutenberg.org/files/6130/6130-0.txt"); // Mark the head of the pipeline as complete. The continuation tasks // propagate completion through the pipeline as each part of the // pipeline finishes. downloadString.Complete(); // Wait for the last block in the pipeline to process all messages. printReversedWords.Completion.Wait(); Console.ReadLine(); te.OnNext("ciao"); Console.ReadLine(); }
public static void Main(string[] args) { var testObs = new TransformBlock<int, int>(item => item * item); var o = testObs.AsObservable (); o.Subscribe(i=> Console.WriteLine("Test Obs received {0}", i.ToString())); for (int i = 0; i < 5; i++) { testObs.Post (i); } testObs.Completion.Wait (); Console.ReadKey (); var buffer = new BufferBlock<int>(); IObservable<int> integers = buffer.AsObservable(); integers.Subscribe(data => Console.WriteLine(data), ex => Console.WriteLine(ex), () => Console.WriteLine("Done")); buffer.Post(13); buffer.Post(14); buffer.Post(15); buffer.Post(16); Console.ReadKey (); IObservable<DateTimeOffset> ticks = Observable.Interval(TimeSpan.FromSeconds(1)) .Timestamp() .Select(x => x.Timestamp) .Take(5); var display = new ActionBlock<DateTimeOffset>(x => Console.WriteLine(x)); ticks.Subscribe(display.AsObserver()); Console.ReadKey (); try { display.Completion.Wait(); Trace.WriteLine("Done."); } catch (Exception ex) { Trace.WriteLine(ex); } try { var multiplyBlock = new TransformBlock<int, int>(item => { // if (item == 1) // throw new InvalidOperationException("Blech."); return item * 2; }); var subtractBlock = new TransformBlock<int, int>(item => item - 2); multiplyBlock.LinkTo(subtractBlock); var printing = new ActionBlock<int>(i => Console.WriteLine("Received {0}", i.ToString())); subtractBlock.LinkTo(printing, new DataflowLinkOptions { PropagateCompletion = true }); IObservable<int> obs = subtractBlock.AsObservable(); obs.Subscribe(i => Console.WriteLine("Received Observable {0}", i.ToString())); for(int i = 0; i < 10;i++){ multiplyBlock.Post(i); subtractBlock.Post(i); } Console.ReadLine (); //printing.Completion.Wait(); } catch (AggregateException exception) { AggregateException ex = exception.Flatten(); System.Diagnostics.Trace.WriteLine(ex.InnerException); } Console.WriteLine ("Hello World!"); Console.ReadLine (); }