public void Run() { Console.WriteLine("Generating first {0} powers of 2.", MaxItems); var bufferBlock = new BufferBlock<int>(); Enumerable.Range(1, MaxItems) .ToList() .ForEach(i => bufferBlock.Post(i)); Console.WriteLine("Signaling completion to the source block."); bufferBlock.Complete(); Console.WriteLine("Done."); Console.WriteLine("Creating and linking the remaing blocks to the network."); var transformBlock = new TransformBlock<int, double>(i => { Thread.Sleep(200); return Math.Pow(2, i); }, new ExecutionDataflowBlockOptions { BoundedCapacity = 1 }); var actionBlock = new ActionBlock<double>(async i => { await Task.Delay(500); Console.WriteLine(i); }, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 }); bufferBlock.LinkTo(transformBlock, new DataflowLinkOptions { PropagateCompletion = true }); transformBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true }); Console.WriteLine("Waiting for the completion to be propagated through the network..."); actionBlock.Completion.ContinueWith(t => { Console.WriteLine("Finished processing."); Console.WriteLine(string.Format("Completion status: {0}.", t.Status)); }).Wait(); }
public BatchedInsert() { Output = new TransformBlock<object, List<object>>(obj => { if (objects.Count > BatchSize) objects.Clear(); if(objects.Count < BatchSize) { objects.Add(obj); return null; } return objects; }); DatabaseInserter = new ActionBlock<object>(objs => { if (objs == null) return; var toInsert = (List<object>)objs; toInsert = new List<object>(toInsert); Mapping.Insert(toInsert); }); (Output as TransformBlock<object, List<object>>).LinkTo(DatabaseInserter); }
public static async Task ValidateBlockAsync(ICoreStorage coreStorage, ICoreRules rules, Chain newChain, ISourceBlock<ValidatableTx> validatableTxes, CancellationToken cancelToken = default(CancellationToken)) { // tally transactions object finalTally = null; var txTallier = new TransformBlock<ValidatableTx, ValidatableTx>( validatableTx => { var runningTally = finalTally; rules.TallyTransaction(newChain, validatableTx, ref runningTally); finalTally = runningTally; return validatableTx; }); validatableTxes.LinkTo(txTallier, new DataflowLinkOptions { PropagateCompletion = true }); // validate transactions var txValidator = InitTxValidator(rules, newChain, cancelToken); // begin feeding the tx validator txTallier.LinkTo(txValidator, new DataflowLinkOptions { PropagateCompletion = true }); // validate scripts var scriptValidator = InitScriptValidator(rules, newChain, cancelToken); // begin feeding the script validator txValidator.LinkTo(scriptValidator, new DataflowLinkOptions { PropagateCompletion = true }); //TODO await PipelineCompletion.Create( new Task[] { }, new IDataflowBlock[] { validatableTxes, txTallier, txValidator, scriptValidator }); // validate overall block rules.PostValidateBlock(newChain, finalTally); }
public void Run() { var cts = new CancellationTokenSource(); Console.WriteLine("Generating first {0} powers of 2.", MaxItems); var bufferBlock = new BufferBlock<int>(new DataflowBlockOptions { CancellationToken = cts.Token }); Enumerable.Range(1, MaxItems) .ToList() .ForEach(i => bufferBlock.Post(i)); Console.WriteLine("Scheduling cancellation after 5 seconds."); cts.CancelAfter(TimeSpan.FromSeconds(5)); Console.WriteLine("Creating and linking the remaing blocks to the network."); var transformBlock = new TransformBlock<int, double>(i => { Thread.Sleep(500); return Math.Pow(2, i); }, new ExecutionDataflowBlockOptions { BoundedCapacity = 1, CancellationToken = cts.Token }); var actionBlock = new ActionBlock<double>(async i => { await Task.Delay(1000); Console.WriteLine(i); }, new ExecutionDataflowBlockOptions { BoundedCapacity = 10, CancellationToken = cts.Token }); bufferBlock.LinkTo(transformBlock, new DataflowLinkOptions { PropagateCompletion = true }); transformBlock.LinkTo(actionBlock, new DataflowLinkOptions { PropagateCompletion = true }); var t1 = bufferBlock.Completion.ContinueWith(t => Console.WriteLine("Buffer block status: {0}", t.Status)); var t2 = actionBlock.Completion.ContinueWith(t => Console.WriteLine("Action block status: {0}", t.Status)); Console.WriteLine("Waiting for the network to finish."); Task.WaitAll(t1, t2); }
public static ITargetBlock<string> SetupPipeline() { var fileNamesForPath = new TransformBlock<string, IEnumerable<string>>( path => { return GetFileNames(path); }); var lines = new TransformBlock<IEnumerable<string>, IEnumerable<string>>( fileNames => { return LoadLines(fileNames); }); var words = new TransformBlock<IEnumerable<string>, IEnumerable<string>>( lines2 => { return GetWords(lines2); }); var display = new ActionBlock<IEnumerable<string>>( coll => { foreach (var s in coll) { WriteLine(s); } }); fileNamesForPath.LinkTo(lines); lines.LinkTo(words); words.LinkTo(display); return fileNamesForPath; }
public MessagePipeline() { linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; buildMessage = new TransformBlock<object, Messaging.Message>( x => { Console.WriteLine("buildMessage| message: {0}", x); return new Messaging.Message { Body = x }; }); logMessage = new TransformBlock<Messaging.Message, Messaging.Message> (x => { Console.WriteLine("logMessage| MessageId: {0}. Body: {1}.", x.MessageId, x.Body); return x; }); sendMessage = new TransformBlock<Messaging.Message, Messaging.Message>( x => { Console.WriteLine("sendMessage| MessageId: {0}. Body: {1}.", x.MessageId, x.Body); return x; }); buildMessage.LinkTo(logMessage, linkOptions); logMessage.LinkTo(sendMessage, linkOptions); }
public void Start() { var sink = new ActionBlock<PageResultMessage>((Action<PageResultMessage>)Sink); var source = new BufferBlock<GetPageMessage>(); var linkOptions = new DataflowLinkOptions {PropagateCompletion = false}; for (int i = 0; i < 10; i++) { var options = new ExecutionDataflowBlockOptions { BoundedCapacity = 1 }; var worker = new TransformBlock<GetPageMessage, PageResultMessage>( (Func<GetPageMessage, PageResultMessage>)Worker, options); source.LinkTo(worker, linkOptions); worker.LinkTo(sink, linkOptions); } foreach (var url in UrlList.Urls) { source.Post(new GetPageMessage{ Url = url }); } source.Complete(); sink.Completion.Wait(); }
public void TestTransformBlockConstructor() { // SYNC { // without option var block = new TransformBlock<int, string>(i => i.ToString()); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock."); //with not cancelled token and default scheduler block = new TransformBlock<int, string>(i => i.ToString(), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1 }); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock."); //with a cancelled token and default scheduler var token = new CancellationToken(true); block = new TransformBlock<int, string>(i => i.ToString(), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token }); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock."); } // ASYNC (a copy of the sync but with constructors returning Task<T> instead of T { // without option var block = new TransformBlock<int, string>(i => Task.Factory.StartNew(() => i.ToString())); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock."); //with not cancelled token and default scheduler block = new TransformBlock<int, string>(i => Task.Factory.StartNew(() => i.ToString()), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1 }); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock."); //with a cancelled token and default scheduler var token = new CancellationToken(true); block = new TransformBlock<int, string>(i => Task.Factory.StartNew(() => i.ToString()), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token }); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock."); } }
public MatchPipeline(RiotApi riotApi, RiotQuerySettings querySettings, IMatchDetailProcessor matchDetailProcessor) { api = riotApi; this.querySettings = querySettings; this.matchDetailProcessor = matchDetailProcessor; queryQueues.Add(querySettings.Queue); // Create match producer if (!querySettings.NoDownload) { playerMatchProducer = new PlayerMatchProducer(api, querySettings, queryQueues, testSynchronizer); // Create blocks ConsumeMatchBlock = new TransformBlock<MatchSummary, MatchDetail>( async match => await ConsumeMatch(match), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 5 }); } ConsumeMatchDetailBlock = new ActionBlock<MatchDetail>( async match => await matchDetailProcessor.ConsumeMatchDetail(match), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = matchDetailProcessor.MaxDegreeOfParallelism }); // Link blocks if (!querySettings.NoDownload) { playerMatchProducer.MatchProducerBlock.LinkTo(ConsumeMatchBlock, new DataflowLinkOptions() { PropagateCompletion = false }); ConsumeMatchBlock.LinkTo(ConsumeMatchDetailBlock, new DataflowLinkOptions() { PropagateCompletion = false }, match => match != null); ConsumeMatchBlock.LinkTo(DataflowBlock.NullTarget<MatchDetail>(), new DataflowLinkOptions() { PropagateCompletion = false }); } }
public void Run() { Console.WriteLine("Generating first {0} powers of 2.", MaxItems); var bufferBlock = new BufferBlock<int>(); var transformBlock = new TransformBlock<int, double>(i => { Thread.Sleep(500); return Math.Pow(2, i); }, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 }); var actionBlock = new ActionBlock<double>(async i => { await Task.Delay(1000); Console.WriteLine(i); _waitHandle.Signal(); }, new ExecutionDataflowBlockOptions { BoundedCapacity = 10 }); bufferBlock.LinkTo(transformBlock); transformBlock.LinkTo(actionBlock); Enumerable.Range(1, MaxItems) .ToList() .ForEach(i => bufferBlock.Post(i)); _waitHandle.Wait(); }
/// <summary> /// Search Asynchrony many extension in all of Fixed and Removable Disks. /// </summary> /// <param name="targetExtensions">Some file extensions for use search pattern.</param> /// <example> /// FileExtension example: /// {".jpg", 646546Byte, 646Byte} /// {".pdf", 25464645546Byte, 60000Byte} /// </example> /// <returns>A sorted list of detected files</returns> public static async Task<List<FileInfo>> DiskParallelProbingAsync(List<FileExtensionOption> targetExtensions, System.Threading.CancellationTokenSource CTS) { return await Task.Run(() => { searchComplete = false; // Reporter("DiskProbing", new ReportEventArgs("DiskProbing", ReportCodes.DiskProbingStarted, "---{Search Disks Started}---")); List<FileInfo> _result = new List<FileInfo>(); // // Find specific folders from windows drives instead of the total drive. // FolderInfo[] SpecificsDirectory = CheckDirectoriesChanges.GetDirectoriesInformation(); // // Set Data-flow // TransformBlock<FolderInfo, List<FileInfo>> TB = new TransformBlock<FolderInfo, List<FileInfo>>(dir => { Reporter(dir, new ReportEventArgs("DiskProbing", ReportCodes.TheSearchBeginning, "Searching {0} ...", dir.FullName)); List<FileInfo> res = dir.GetDirectoryInfo.SearchDirectory(targetExtensions, CTS); Reporter(dir, new ReportEventArgs("DiskProbing", ReportCodes.TheSearchCompleted, "The Search {0} was completed!", dir.FullName)); return res; }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = Environment.ProcessorCount }); ActionBlock<List<FileInfo>> AB = new ActionBlock<List<FileInfo>>(lst => _result.AddRange(lst)); // // Search specific folders from windows drives instead of the total drive. // try { TB.LinkTo(AB); ParallelOptions opt = new ParallelOptions() { CancellationToken = CTS.Token, MaxDegreeOfParallelism = Environment.ProcessorCount }; var pLoop = Parallel.ForEach(SpecificsDirectory, opt, async dir => await TB.SendAsync(dir)); TB.Complete(); TB.Completion.Wait(); } catch (Exception ex) { Reporter(SpecificsDirectory, new ReportEventArgs("SearchEngine.DiskProbing.SpecificsDirectory", ex)); } searchComplete = true; Reporter("DiskProbing", new ReportEventArgs("DiskProbing", ReportCodes.DiskProbingFinished, "---{Search Disks Finished}---")); LastSearchResult = _result; return _result; }); }
public void Initialise() { _systemMetrics = new Mock<ISystemMetricsService>(); _log = new Mock<ILog>(); _block = MessageParserBlockFactory.CreateMessageParserBlock(new CancellationToken(), _systemMetrics.Object, _log.Object); }
public static ISourceBlock<ValidatableTx> ReplayRollbackUtxo(ICoreStorage coreStorage, IChainState chainState, ChainedHeader replayBlock, CancellationToken cancelToken = default(CancellationToken)) { // replaying rollback of an on-chain block, use the chainstate tx index for replay, same as replaying forward if (chainState.Chain.BlocksByHash.ContainsKey(replayBlock.Hash)) { return ReplayFromTxIndex(coreStorage, chainState, replayBlock, replayForward: false, cancelToken: cancelToken); } // replaying rollback of an off-chain (re-org) block, use the unminted information for replay else { IImmutableList<UnmintedTx> unmintedTxesList; if (!chainState.TryGetBlockUnmintedTxes(replayBlock.Hash, out unmintedTxesList)) { //TODO if a wallet/monitor were to see a chainstate block that wasn't flushed to disk yet, //TODO and if bitsharp crashed, and if the block was orphaned: then the orphaned block would //TODO not be present in the chainstate, and it would not get rolled back to generate unminted information. //TODO DeferredChainStateCursor should be used in order to re-org the chainstate in memory and calculate the unminted information throw new MissingDataException(replayBlock.Hash); } var unmintedTxes = ImmutableDictionary.CreateRange( unmintedTxesList.Select(x => new KeyValuePair<UInt256, UnmintedTx>(x.TxHash, x))); var lookupLoadingTx = new TransformBlock<DecodedBlockTx, ValidatableTx>( blockTx => { var tx = blockTx.Transaction; var txIndex = blockTx.Index; var prevTxOutputs = ImmutableArray.CreateBuilder<PrevTxOutput>(!blockTx.IsCoinbase ? tx.Inputs.Length : 0); if (!blockTx.IsCoinbase) { UnmintedTx unmintedTx; if (!unmintedTxes.TryGetValue(tx.Hash, out unmintedTx)) throw new MissingDataException(replayBlock.Hash); prevTxOutputs.AddRange(unmintedTx.PrevTxOutputs); } return new ValidatableTx(blockTx, replayBlock, prevTxOutputs.MoveToImmutable()); }); IEnumerator<BlockTx> blockTxes; if (!coreStorage.TryReadBlockTransactions(replayBlock.Hash, out blockTxes)) { throw new MissingDataException(replayBlock.Hash); } var blockTxesBuffer = new BufferBlock<DecodedBlockTx>(); blockTxesBuffer.LinkTo(lookupLoadingTx, new DataflowLinkOptions { PropagateCompletion = true }); blockTxesBuffer.SendAndCompleteAsync(blockTxes.UsingAsEnumerable().Select(x => x.Decode()).Reverse(), cancelToken).Forget(); return lookupLoadingTx; } }
static public void ProcessingByTPL_StraightForwardImplementation() { const string pathToFiles = @"..\..\..\..\DataFiles"; string[] files = Directory.GetFiles(pathToFiles, "*.txt", SearchOption.AllDirectories); var loadDataFromFileBlock = new TransformBlock<string[], List<CustomerTextData>>(fileItems => { var factory = new CustomerTextDataFactory(); return new List<CustomerTextData>(Array.ConvertAll(fileItems, factory.LoadFromFile)); }); var filterBlock = new TransformBlock<List<CustomerTextData>, List<CustomerTextData>>(textDataList => { var filter = new FilterTextData(5); return textDataList.Where(filter.Run).ToList(); }); var toListBlock = new TransformManyBlock<List<CustomerTextData>, CustomerTextData>(textDataList => { var queue = new ConcurrentQueue<CustomerTextData>(); textDataList.ForEach(queue.Enqueue); return queue; }); var action = new ActionBlock<CustomerTextData>(textData => { var weight = new WeightTextData(); int result = weight.Run(textData); Trace.WriteLine(result); Console.WriteLine(result); }); loadDataFromFileBlock.LinkTo(filterBlock); filterBlock.LinkTo(toListBlock); toListBlock.LinkTo(action); loadDataFromFileBlock.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)filterBlock).Fault(t.Exception); else filterBlock.Complete(); }); filterBlock.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)toListBlock).Fault(t.Exception); else toListBlock.Complete(); }); toListBlock.Completion.ContinueWith(t => { if (t.IsFaulted) ((IDataflowBlock)action).Fault(t.Exception); else action.Complete(); }); loadDataFromFileBlock.Post(files); loadDataFromFileBlock.Complete(); action.Completion.Wait(); }
static void Main(string[] args) { string s = "http://cn.bing.com/search?q=MD5CryptoServiceProvider+slow&qs=n&pq=md5cryptoserviceprovider+slow&sc=0-25&sp=-1&sk=&cvid=67d40cbd8c424d55a3db83e6e9868267&first=51&FORM=PERE4"; using (MD5CryptoServiceProvider md5 = new MD5CryptoServiceProvider()) { byte[] inBytes = Encoding.UTF8.GetBytes(s); var bytes = md5.ComputeHash(inBytes); Console.WriteLine(bytes.Length); } var splitter = new TransformBlock<string, KeyValuePair<string, int>>( input => { var splitted = input.Split('='); return new KeyValuePair<string, int>(splitted[0], int.Parse(splitted[1])); }); var dict = new Dictionary<string, int>(); var aggregater = new ActionBlock<KeyValuePair<string, int>>( pair => { int oldValue; dict[pair.Key] = dict.TryGetValue(pair.Key, out oldValue) ? oldValue + pair.Value : pair.Value; }); splitter.LinkTo(aggregater, new DataflowLinkOptions() { PropagateCompletion = true}); splitter.Post("a=1"); splitter.Post("b=2"); splitter.Post("a=5"); splitter.Complete(); aggregater.Completion.Wait(); Console.WriteLine("sum(a) = {0}", dict["a"]); //prints sum(a) = 6 //CalcAsync().Wait(); //SlowFlowAsync().Wait(); //FailDemoAsync().Wait(); //TransformAndLinkDemo().Wait(); //LinkLeftToDemo().Wait(); //CircularFlowAutoComplete().Wait(); //RecorderDemo().Wait(); BulkInserterDemo().Wait(); //BulkInserterDemo2().Wait(); //BroadcasterDemo().Wait(); //MyLoggerDemo().Wait(); //ETLLookupDemo().Wait(); }
private static TransformBlock<int, string> ConstructTransformWithNMessages(int messagesCount) { var block = new TransformBlock<int, string>(i => i.ToString()); for (int i = 0; i < messagesCount; i++) { block.Post(i); } // Spin until the messages have been properly buffered up. // Otherwise TryReceive fails. SpinWait.SpinUntil(() => block.OutputCount == messagesCount); return block; }
/// <summary> /// Initials the transmitter asynchronous. /// Check the server and then database existence and ... /// </summary> public static async Task InitialTransmitterAsync() { await ServerValidatorAsync(); ErrorListenerTransformBlock = new TransformBlock<ProxyError, Tuple<ProxyError, bool>>( async (e) => await TransmitOneError(e), new ExecutionDataflowBlockOptions() { MaxMessagesPerTask = 1, MaxDegreeOfParallelism = 1 }); ErrorListenerTransformBlock.LinkTo(CacheController.AcknowledgeActionBlock); }
public async Task TransformThroughFilterToAction() { int completedCount = 0; var t = new TransformBlock<int, int>(i => i); var c = new ActionBlock<int>(i => completedCount++); t.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true }, i => true); t.PostRange(0, Iterations); t.Complete(); await c.Completion; Assert.Equal(expected: Iterations, actual: completedCount); }
public void DeferredUsageTest () { int[] array = new int[10]; var action = new ActionBlock<int> (i => array[Math.Abs (i)] = i); var block = new TransformBlock<int, int> (i => -i); for (int i = 0; i < array.Length; ++i) Assert.IsTrue (block.Post (i), "Not accepted"); Thread.Sleep (300); block.LinkTo (action); Thread.Sleep (100); CollectionAssert.AreEqual (new[] { 0, -1, -2, -3, -4, -5, -6, -7, -8, -9 }, array); }
public void BasicUsageTest () { int[] array = new int[10]; var evt = new ManualResetEventSlim (false); ActionBlock<int> action = new ActionBlock<int> ((i) => { array[Math.Abs (i)] = i; evt.Set (); }); TransformBlock<int, int> block = new TransformBlock<int, int> (i => -i); block.LinkTo (action); for (int i = 0; i < array.Length; ++i) Assert.IsTrue (block.Post (i), "Not accepted"); evt.Wait (); CollectionAssert.AreEqual (new int[] { 0, -1, -2, -3, -4, -5, -6, -7, -8, -9 }, array); }
public Statsd(string serviceName = null) { _log.Info("statsd.net starting."); _tokenSource = new CancellationTokenSource(); _shutdownComplete = new ManualResetEvent(false); SuperCheapIOC.Add(_log); var systemInfoService = new SystemInfoService(); SuperCheapIOC.Add(systemInfoService as ISystemInfoService); serviceName = serviceName ?? systemInfoService.HostName; var systemMetricsService = new SystemMetricsService("statsdnet", serviceName); SuperCheapIOC.Add(systemMetricsService as ISystemMetricsService); /** * The flow is: * Listeners -> * Message Parser -> * router -> * Aggregator -> * Broadcaster -> * Backends */ // Initialise the core blocks _router = new StatsdMessageRouterBlock(); _messageParser = MessageParserBlockFactory.CreateMessageParserBlock(_tokenSource.Token, SuperCheapIOC.Resolve<ISystemMetricsService>(), _log); _messageParser.LinkTo(_router); _messageParser.Completion.LogAndContinueWith(_log, "MessageParser", () => { _log.Info("MessageParser: Completion signaled. Notifying the MessageBroadcaster."); _messageBroadcaster.Complete(); }); _messageBroadcaster = new BroadcastBlock<Bucket>(Bucket.Clone); _messageBroadcaster.Completion.LogAndContinueWith(_log, "MessageBroadcaster", () => { _log.Info("MessageBroadcaster: Completion signaled. Notifying all backends."); _backends.ForEach(q => q.Complete()); }); // Add the broadcaster to the IOC container SuperCheapIOC.Add<BroadcastBlock<Bucket>>(_messageBroadcaster); systemMetricsService.SetTarget(_messageBroadcaster); _backends = new List<IBackend>(); _listeners = new List<IListener>(); }
static ITargetBlock<string> SetupPipeline() { var fileNames = new TransformBlock<string, IEnumerable<string>>(path => { try { return GetFileNames(path); } catch (OperationCanceledException) { return Enumerable.Empty<string>(); } }); var lines = new TransformBlock<IEnumerable<string>, IEnumerable<string>>(input => { try { return LoadLines(input); } catch (OperationCanceledException) { return Enumerable.Empty<string>(); } }); var words = new TransformBlock<IEnumerable<string>, IEnumerable<string>>(lines2 => { return GetWords(lines2); }); var display = new ActionBlock<IEnumerable<string>>(coll => { foreach (var s in coll) { Console.WriteLine(s); } }); fileNames.LinkTo(lines); lines.LinkTo(words); words.LinkTo(display); // fileNames.LinkTo(loadLines, fn => fn.Count() > 0); return fileNames; }
public void ProcessFile(string fileName) { var inputBlock = new BufferBlock<string>(); var readBlock = new TransformBlock<string, string>( (input) => { Console.WriteLine("Loading " + input); return File.ReadAllText(input); }); var compileBlock = new TransformBlock<string, string>( (input) => { Console.WriteLine("Processing..."); var client = new WebClient(); client.Headers.Add("content-type", "application/x-www-form-urlencoded"); string apiData = string.Format(Data, HttpUtility.UrlEncode(input)); return client.UploadString(Url, apiData); }); var convertBlock = new TransformBlock<string, XDocument>( (input) => { Console.WriteLine("Converting to XDocument..."); var xml = XDocument.Parse(input); return xml; }); var outputBlock = new ActionBlock<XDocument>( (input) => { Console.WriteLine("Writing compressed-" + fileName); XElement compiledCode = input.Element("compilationResult").Element("compiledCode"); File.WriteAllText("compressed-" + fileName, compiledCode.Value); }); inputBlock.LinkTo(readBlock, new DataflowLinkOptions { PropagateCompletion = true }); readBlock.LinkTo(compileBlock, new DataflowLinkOptions { PropagateCompletion = true }); compileBlock.LinkTo(convertBlock, new DataflowLinkOptions { PropagateCompletion = true }); convertBlock.LinkTo(outputBlock, new DataflowLinkOptions { PropagateCompletion = true }); inputBlock.Post(fileName); inputBlock.Complete(); outputBlock.Completion.Wait(); }
public void AsyncNullTest () { var scheduler = new TestScheduler (); var block = new TransformBlock<int, int> ( i => null, new ExecutionDataflowBlockOptions { TaskScheduler = scheduler }); Assert.IsTrue (block.Post (1)); scheduler.ExecuteAll (); Assert.IsFalse (block.Completion.Wait (100)); block.Complete (); Assert.IsTrue (block.Completion.Wait (100)); }
public CompletionDemo1() { broadCaster = new BroadcastBlock<int>( i => { return i; }).ToDataflow(); transformBlock1 = new TransformBlock<int, string>( i => { Console.WriteLine("1 input count: " + transformBlock1.InputCount); Thread.Sleep(50); return ("1_" + i); }); transformBlock2 = new TransformBlock<int, string>( i => { Console.WriteLine("2 input count: " + transformBlock2.InputCount); Thread.Sleep(20); return ("2_" + i); }); processor = new ActionBlock<string>( i => { Console.WriteLine(i); }).ToDataflow(); /** rather than TPL linking broadCastBlock.LinkTo(transformBlock1, new DataflowLinkOptions { PropagateCompletion = true }); broadCastBlock.LinkTo(transformBlock2, new DataflowLinkOptions { PropagateCompletion = true }); transformBlock1.LinkTo(processorBlock, new DataflowLinkOptions { PropagateCompletion = true }); transformBlock2.LinkTo(processorBlock, new DataflowLinkOptions { PropagateCompletion = true }); **/ //Use DataflowEx linking var transform1 = transformBlock1.ToDataflow(); var transform2 = transformBlock2.ToDataflow(); broadCaster.LinkTo(transform1); broadCaster.LinkTo(transform2); transform1.LinkTo(processor); transform2.LinkTo(processor); }
internal static bool TransformThroughFilterToAction() { const int ITERS = 2; int completedCount = 0; var t = new TransformBlock<int, int>(i => i); var c = new ActionBlock<int>(i => completedCount++); t.LinkTo(c, i => true); t.Completion.ContinueWith(_ => c.Complete()); for (int i = 0; i < ITERS; i++) t.Post(i); t.Complete(); c.Completion.Wait(); return completedCount == ITERS; }
private static IPropagatorBlock<string, KeyValuePair<string, long>> CreateCustomBlock() { var directoryBrowserBlock = new TransformManyBlock<string, string>(path => { var dir = new DirectoryInfo(path); return dir.EnumerateFileSystemInfos() .Select(fi => fi.FullName); }); var fileSizeCalculator = new TransformBlock<string, KeyValuePair<string, long>>(fileName => { var fi = new FileInfo(fileName); return new KeyValuePair<string, long>(fileName, fi.Length); }); directoryBrowserBlock.LinkTo(fileSizeCalculator, new DataflowLinkOptions { PropagateCompletion = true }, File.Exists); var customBlock = DataflowBlock.Encapsulate(directoryBrowserBlock, fileSizeCalculator); return customBlock; }
public static void Example1() { var conf = new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 4 }; ActionBlock<int> a = new ActionBlock<int>(i => { Thread.Sleep(500); Console.WriteLine(i); }, conf); TransformBlock<int, int> t = new TransformBlock<int, int>(i => i * 3); t.LinkTo(a); for (int i = 0; i < 12; i++) { t.Post(i); } }
public void Run() { var options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 8 }; var tb = new TransformBlock<int, int>(i => i * 2); var ab = new ActionBlock<int>(i => this.Compute(i), options); tb.LinkTo(ab); for (var i = 0; i < 10; i++) { tb.Post(i); } tb.Complete(); tb.Completion.Wait(); Thread.Sleep(500); }
public WebSocketListener(IPEndPoint endpoint, WebSocketListenerOptions options) { if (options == null) throw new ArgumentNullException("options"); if (endpoint == null) throw new ArgumentNullException("endpoint"); _options = options.Clone(); _cancel = new CancellationTokenSource(); _listener = new TcpListener(endpoint); ConnectionExtensions = new WebSocketConnectionExtensionCollection(this); Standards = new WebSocketFactoryCollection(this); Func<Socket, Task<WebSocketNegotiationResult>> negotiate = NegotiateWebSocket; _negotiationQueue = new TransformBlock<Socket, WebSocketNegotiationResult>(negotiate, new ExecutionDataflowBlockOptions() { CancellationToken = _cancel.Token, MaxDegreeOfParallelism = options.ParallelNegotiations, BoundedCapacity = options.NegotiationQueueCapacity }); _handShaker = new WebSocketHandshaker(Standards, _options); }
/// <summary> /// Disposes internal members. /// </summary> /// <param name="disposing">Whether the object is being disposed.</param> protected virtual void Dispose(bool disposing) { if (disposing) { if (this.eventSource != null) { // Flush the data buffer to upload all remaining events. this.eventSource.Complete(); this.eventProcessor.Completion.Wait(); this.eventSource = null; } if (this.eventUnsubscriber != null) { this.eventUnsubscriber.Dispose(); this.eventUnsubscriber = null; } this.cancellationTokenSource.Cancel(); } }
public async Task TestCancellationExceptionsIgnored() { var t = new TransformBlock <int, int>(i => { if ((i % 2) == 0) { throw new OperationCanceledException(); } return(i); }); t.PostRange(0, 2); t.Complete(); for (int i = 0; i < 2; i++) { if ((i % 2) != 0) { Assert.Equal(expected: i, actual: await t.ReceiveAsync()); } } await t.Completion; }
public async Task TestCtorOverloading() { Func <object, Task <object> > f = x => Task.FromResult <object>(x); for (int test = 0; test < 2; test++) { TransformBlock <object, object> tf = test == 0 ? new TransformBlock <object, object>(f) : new TransformBlock <object, object>((Func <object, object>)f); var tcs = new TaskCompletionSource <bool>(); ActionBlock <object> a = new ActionBlock <object>(x => { Assert.Equal(expected: test == 1, actual: x is Task <object>); tcs.SetResult(true); }); tf.LinkTo(a); tf.Post(new object()); await tcs.Task; } }
public CalculationDataFlow(ILogger <CalculationDataFlow> logger, IResultProducer resultProducer) { _logger = logger; _resultProducer = resultProducer; _deserializeBlock = new TransformBlock <byte[], CalculationPack>(msg => MessagePackSerializer.Deserialize <CalculationPack>(msg)); _calculationBlock = new TransformBlock <CalculationPack, CalculationPack>(pack => { Parallel.ForEach(pack.Data, unit => { unit.Equation.Result = CalculationHelper.Calculate(unit.Equation.EqMethod, unit.Equation.Values); }); return(pack); }); _publishBlock = new ActionBlock <CalculationPack>(pack => { //TODO publish in kafka topic //_logger.LogInformation(pack.Id.ToString()); foreach (var eq in pack.Data) { _logger.LogInformation($"PackId: {eq.CalcPackId} EqId: {eq.Equation.Id} Result:{eq.Equation.Result}"); } _resultProducer.ProduceResultAsync(pack); }); var linkOptions = new DataflowLinkOptions() { PropagateCompletion = true }; _deserializeBlock.LinkTo(_calculationBlock, linkOptions); _calculationBlock.LinkTo(_publishBlock, linkOptions); }
public void Process_pipeline_one_input_one_output() { var result = string.Empty; var head = new TransformBlock <string, string>(str => str + "A"); var block2 = new TransformBlock <string, string>(str => str + "B"); var block3 = new TransformBlock <string, string>(str => str + "C"); var block4 = new ActionBlock <string>(str => result = str); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; head.LinkTo(block2, linkOptions); block2.LinkTo(block3, linkOptions); block3.LinkTo(block4, linkOptions); head.Post(string.Empty); head.Complete(); block4.Completion.Wait(); result.Should().Be("ABC"); }
static void Main(string[] args) { // TransformMany(); // Batching(); // Errors(); // DangerOfCondiionalLinking(); // SingleActionBlockLoadBalance(); var greedy = new ExecutionDataflowBlockOptions(); var nonGreedy = new ExecutionDataflowBlockOptions() { BoundedCapacity = 1 }; ExecutionDataflowBlockOptions options = nonGreedy; var firstBlock = new ActionBlock <int>(i => Do(i, 1, 2), options); var secondBlock = new ActionBlock <int>(i => Do(i, 2, 1), options); var thirdBlock = new ActionBlock <int>(i => Do(i, 3, 2), options); var transform = new TransformBlock <int, int>(i => i * 2); transform.LinkTo(firstBlock); transform.LinkTo(secondBlock); transform.LinkTo(thirdBlock); for (int i = 0; i <= 10; i++) { transform.Post(i); } Console.ReadLine(); }
public MultiThreadRunner( bool verbose, List <DiagnosticAnalyzer> analyzers, Func <ImmutableArray <Diagnostic>, ParsedOptions, ConcurrentDictionary <string, DiagnosticDescriptor>, SarifV2ErrorLogger, int> logDiagnostics, ParsedOptions parsedOptions, ConcurrentDictionary <string, DiagnosticDescriptor> descriptors, SarifV2ErrorLogger logger, int threads) : base(analyzers, logDiagnostics, parsedOptions, descriptors, logger) { _scanBlock = new TransformBlock <Project, ImmutableArray <Diagnostic> >(async project => { if (verbose) { Console.WriteLine($"Starting: {project.FilePath}"); } return(await GetDiagnostics(project).ConfigureAwait(false)); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = threads, EnsureOrdered = false, BoundedCapacity = 32 }); _resultsBlock = new ActionBlock <ImmutableArray <Diagnostic> >(diagnostics => { _count += logDiagnostics(diagnostics, _parsedOptions, descriptors, logger); }, new ExecutionDataflowBlockOptions { EnsureOrdered = false }); _scanBlock.LinkTo(_resultsBlock, new DataflowLinkOptions { PropagateCompletion = true }); }
public void TestTransformBlockConstructor() { // SYNC { // without option var block = new TransformBlock <int, string>(i => i.ToString()); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock."); //with not cancelled token and default scheduler block = new TransformBlock <int, string>(i => i.ToString(), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1 }); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock."); //with a cancelled token and default scheduler var token = new CancellationToken(true); block = new TransformBlock <int, string>(i => i.ToString(), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token }); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock."); } // ASYNC (a copy of the sync but with constructors returning Task<T> instead of T { // without option var block = new TransformBlock <int, string>(i => Task.Run(() => i.ToString())); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock."); //with not cancelled token and default scheduler block = new TransformBlock <int, string>(i => Task.Run(() => i.ToString()), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1 }); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock."); //with a cancelled token and default scheduler var token = new CancellationToken(true); block = new TransformBlock <int, string>(i => Task.Run(() => i.ToString()), new ExecutionDataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token }); Assert.False(block.InputCount != 0, "Constructor failed! InputCount returned a non zero value for a brand new TransformBlock."); } }
private static void SynchronousHappySites() { string[] urls = new string[] { "http://www.bbc.co.uk", "http://www.cia.gov", "http://www.theregister.co.uk" }; var happySites = new List <string>(); var isHappySiteBlock = new TransformBlock <string, Tuple <string, bool> >((Func <string, Tuple <string, bool> >)IsHappy, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1 }); var addToHappySitesBlock = new ActionBlock <Tuple <string, bool> >( (Action <Tuple <string, bool> >)(tuple => happySites.Add(tuple.Item1))); isHappySiteBlock.LinkTo(addToHappySitesBlock, new DataflowLinkOptions() { PropagateCompletion = true }, tuple => tuple.Item2); isHappySiteBlock.LinkTo(DataflowBlock.NullTarget <Tuple <string, bool> >()); foreach (string url in urls) { isHappySiteBlock.Post(url); } isHappySiteBlock.Complete(); addToHappySitesBlock.Completion.Wait(); happySites.ForEach(Console.WriteLine); }
public void start() { var consumerBlock = Consumer("consumer"); var transformBlock = new TransformBlock <int, int>(input => { // if (input == 1) throw new Exception("test problem"); return(input * 3); }); var producerBlock = new CustomAdvance(); transformBlock.LinkTo(producerBlock, new DataflowLinkOptions() { PropagateCompletion = true }); producerBlock.LinkTo(consumerBlock, new DataflowLinkOptions() { PropagateCompletion = true }); for (int i = 0; i < 10; i++) { if (!transformBlock.Post(i)) { Console.WriteLine("Failed Post"); } } transformBlock.Complete(); transformBlock.Completion.ContinueWith(p => print_status(p, "transformer")); consumerBlock.Completion.ContinueWith(p => print_status(p, "consumer")); Console.Read(); }
public TableLoggerProvider(AzureLoggerSettings settings) { if (settings == null) { throw new ArgumentNullException(nameof(settings)); } this.settings = settings; var cloud = CloudStorageAccount.Parse(settings.ConnectionString); this.client = cloud.CreateCloudTableClient(); this.table = client.GetTableReference(settings.Table); this.blobClient = cloud.CreateCloudBlobClient(); if (!string.IsNullOrEmpty(settings.OverflowContainer)) { this.container = this.blobClient.GetContainerReference(settings.OverflowContainer); } this.batchBlock = new BatchBlock <DynamicTableEntity>(25); Timer triggerBatchTimer = new Timer((_) => this.batchBlock.TriggerBatch(), null, Timeout.Infinite, Timeout.Infinite); pipeline = new TransformBlock <DynamicTableEntity, DynamicTableEntity>((value) => { triggerBatchTimer.Change(5000, Timeout.Infinite); return(value); }); pipelineEnd = new ActionBlock <DynamicTableEntity[]>(WriteBatch); pipeline.LinkTo(batchBlock); batchBlock.LinkTo(pipelineEnd); this.overflowBlock = new ActionBlock <KeyValuePair <string, string> >(WriteOverflow); }
public async Task Processing(IEnumerable <string> files, string outputDirectory) { var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; var readingBlock = new TransformBlock <string, string>( async path => await File.ReadAllTextAsync(path), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = _pipelineConfiguration.MaxReadingTasks }); var processingBlock = new TransformManyBlock <string, TestClassInfo>( async code => await Task.Run(() => TestGenerator.Generate(code)), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = _pipelineConfiguration.MaxTasks }); var writingBlock = new ActionBlock <TestClassInfo>( async fI => await File.WriteAllTextAsync(outputDirectory + fI.TestClassName + ".txt", fI.TestClassCode), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = _pipelineConfiguration.MaxWritingTasks }); readingBlock.LinkTo(processingBlock, linkOptions); processingBlock.LinkTo(writingBlock, linkOptions); foreach (var file in files) { readingBlock.Post(file); } readingBlock.Complete(); await writingBlock.Completion; }
public WebhookDequeuer(WebhookSender webhookSender, IWebhookEventRepository webhookEventRepository, IWebhookRepository webhookRepository, IClock clock, ISemanticLog log) { Guard.NotNull(webhookEventRepository, nameof(webhookEventRepository)); Guard.NotNull(webhookRepository, nameof(webhookRepository)); Guard.NotNull(webhookSender, nameof(webhookSender)); Guard.NotNull(clock, nameof(clock)); Guard.NotNull(log, nameof(log)); this.webhookEventRepository = webhookEventRepository; this.webhookRepository = webhookRepository; this.webhookSender = webhookSender; this.clock = clock; this.log = log; requestBlock = new ActionBlock <IWebhookEventEntity>(MakeRequestAsync, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 32, BoundedCapacity = 32 }); blockBlock = new TransformBlock <IWebhookEventEntity, IWebhookEventEntity>(x => BlockAsync(x), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1, BoundedCapacity = 1 }); blockBlock.LinkTo(requestBlock, new DataflowLinkOptions { PropagateCompletion = true }); timer = new CompletionTimer(5000, QueryAsync); }
static void Main(string[] args) { var transformBlock = new TransformBlock <int, string>(n => { if (n == 5) { throw new Exception("Something went wrong"); } // Removed all messages in input queue and goes into faulted state Console.WriteLine($"Received {n}"); return(n.ToString()); }); var printBlock = new ActionBlock <string>(n => Console.WriteLine(n)); transformBlock.LinkTo(printBlock, new DataflowLinkOptions { PropagateCompletion = true }); for (int i = 0; i < 10; i++) { transformBlock.Post(i); } transformBlock.Complete(); try { printBlock.Completion.Wait(); } catch (AggregateException ae) { throw ae.Flatten().InnerException; } Console.WriteLine("Finished!"); Console.ReadKey(); }
public SaveBitmapBlock(string outputPath, int initialZoomLevel, string fileFormat, ExecutionDataflowBlockOptions options, IGraphicsApi <TImage> graphics) { OutputPath = outputPath; this.fileFormat = fileFormat; this.graphics = graphics; Block = new TransformBlock <ImageInfo <TImage>, IEnumerable <SubChunkData> >(info => { if (info == null) { return(null); } try { /*if (r.Next(100) == 0) * { * throw new ArgumentOutOfRangeException("TestError in SaveBitmap"); * }*/ SaveBitmap(initialZoomLevel, info.X, info.Z, info.Image); ProcessedCount++; return(info.Cd); } catch (Exception ex) { Console.WriteLine("Error in SaveBitmapBlock: " + ex.Message); return(null); } finally { if (info != null) { graphics.ReturnImage(info.Image); info.Image = null; } } }, options); }
public static TransformBlock <string, string> CreateSimplePipeline( Action <bool> resultCallback) { var step1 = new TransformBlock <string, string>( sentence => Utils.FindMostCommon(sentence), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 3, BoundedCapacity = 5, } ); var step2 = new TransformBlock <string, int>( word => Utils.CountChars(word), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 1, BoundedCapacity = 13, } ); var step3 = new TransformBlock <int, bool>( length => Utils.IsOdd(length), new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 11, BoundedCapacity = 6, } ); var callbackStep = new ActionBlock <bool>(resultCallback); step1.LinkTo(step2, new DataflowLinkOptions()); step2.LinkTo(step3, new DataflowLinkOptions()); step3.LinkTo(callbackStep); return(step1); }
private void InitializeDataflowInternal() { _uriPropagatorBlock = new UriPropagatorBlock( CancellationToken, _logger, Events, ShouldAccept, EmitHeartbeat, TaskScheduler.Default); _getUriContentBlock = new TransformBlock <Uri, Resource>( GetUriContent, new ExecutionDataflowBlockOptions { CancellationToken = CancellationToken, EnsureOrdered = false, TaskScheduler = TaskScheduler.Default, MaxDegreeOfParallelism = _maxUriProcessingDegreeOfParallelism, MaxMessagesPerTask = 4 }); _uriPropagatorBlock.LinkTo( _getUriContentBlock, new DataflowLinkOptions { PropagateCompletion = true }); // ensure failed (null) http requests are discarded _getUriContentBlock.LinkTo( DataflowBlock.NullTarget <Resource>(), new DataflowLinkOptions { PropagateCompletion = true }, resource => resource == null); // allow derived implementations to configure their dataflow InitializeDataflow(); }
public static ISourceBlock <T> Create <T>(string file, Func <Line, T> transform) { var block = new TransformBlock <Line, T>( transform, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, BoundedCapacity = 128 }); new Thread(() => { var input = block.AsObserver(); try { using (var reader = new StreamReader(file)) { string line; int lineNr = 0; var batch = new List <Line>(); while ((line = reader.ReadLine()) != null) { input.OnNext(new Line { Content = line, Number = lineNr++ }); } input.OnCompleted(); } } catch (Exception ex) { input.OnError(ex); } }).Start(); return(block); }
public void BuildPipeline() { _square = new TransformBlock <int, int>(n => { Console.WriteLine($"N = {n}"); return(n * n); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 4 }); _printOdd = new ActionBlock <int>(n => { if (n % 2 == 0) { Console.BackgroundColor = ConsoleColor.Red; } Console.WriteLine($"Odd number is {n}"); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 8 }); _printEven = new ActionBlock <int>(n => { if (n % 2 == 1) { Console.BackgroundColor = ConsoleColor.Red; } Console.WriteLine($"Even number is {n}"); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 8 }); }
/// <summary> /// This method is implemented using the Task Parallel Library (TPL). //. https://docs.microsoft.com/en-us/dotnet/standard/parallel-programming/task-parallel-library-tpl /// </summary> private void GetPublications(List <Tool> tools) { Logger.LogDebug("Setting up TPL for downloading tools archives."); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; var downloader = new TransformBlock <Tool, Tool>( new Func <Tool, Tool>(Downloader), _downloadExeOptions); var extractXMLs = new TransformBlock <Tool, Tool>( new Func <Tool, Tool>(WrapperExtractor), _xmlExtractExeOptions); var extractPublications = new TransformBlock <Tool, Tool>( new Func <Tool, Tool>(ExtractPublications), _pubExtractExeOptions); var cleanup = new ActionBlock <Tool>( input => { Cleanup(input); }); downloader.LinkTo(extractXMLs, linkOptions); extractXMLs.LinkTo(extractPublications, linkOptions); extractPublications.LinkTo(cleanup, linkOptions); Logger.LogDebug($"Starting to process (download archive, extract XML, extract citation) {tools.Count} tools *asynchronously*."); foreach (var info in tools) { downloader.Post(info); } downloader.Complete(); cleanup.Completion.Wait(); Logger.LogDebug($"Serializing extract bibliographies to `{_publicationsFilename}`"); WriteToJson(JsonConvert.SerializeObject(Publications, SerializerSettings), _publicationsFilename); Logger.LogDebug($"Serialized extracted bibliographies."); }
static async Task Main(string[] args) { Generator testGen = new Generator(); var downloadFile = new TransformBlock <string, string>(async filepath => { return(await File.ReadAllTextAsync(filepath)); }); var generateTest = new TransformManyBlock <string, OutputFileInfo>(async testsFile => { return(await Task.Run(() => testGen.GenerateTestClasses(testsFile).ToArray())); }); var createOutputFiles = new ActionBlock <OutputFileInfo>(async testFile => { await File.WriteAllTextAsync(outputFilesPath + testFile.fileName, testFile.text); }); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; downloadFile.LinkTo(generateTest, linkOptions); generateTest.LinkTo(createOutputFiles, linkOptions); string[] filePaths = Directory.GetFiles(inputFilesPath); foreach (string filePath in filePaths) { if (filePath.EndsWith(".cs")) { downloadFile.Post(filePath); } } downloadFile.Complete(); createOutputFiles.Completion.Wait(); }
public static void FirstExample() { var ReadJsonIn = new TransformBlock <string, string>(async FilePath => { Console.WriteLine("Reading Json In From {0}", FilePath); return(await File.ReadAllTextAsync(FilePath)); }); var TransformToObject = new TransformBlock <string, Appraisal>(json => { Console.WriteLine("Transforming JSON to Object Type: {0}", typeof(Appraisal)); return(JsonConvert.DeserializeObject <Appraisal>(json)); }); var GenerateEvents = new ActionBlock <Appraisal>(appraisal => { var app = (Appraisal)appraisal; Console.WriteLine("Client Number: {0} recieved", app.ClientId); Console.WriteLine("Sending Notification to Lender: {0}", app.Lender); Console.WriteLine("Sending Notification to Client: {0}", app.ClientName); }); var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; ReadJsonIn.LinkTo(TransformToObject, linkOptions); TransformToObject.LinkTo(GenerateEvents, linkOptions); ReadJsonIn.Post(".\\EntrySchema.json"); ReadJsonIn.Complete(); GenerateEvents.Completion.Wait(); }
private static ITargetBlock <string> GetAggregatorFlow(out Dictionary <string, int> result) { var splitter = new TransformBlock <string, KeyValuePair <string, int> >(input => { string[] splitted = input.Split('='); return(new KeyValuePair <string, int>(splitted[0], int.Parse(splitted[1]))); }); var dict = new Dictionary <string, int>(); var aggregater = new ActionBlock <KeyValuePair <string, int> >(pair => { int oldValue; dict[pair.Key] = dict.TryGetValue(pair.Key, out oldValue) ? oldValue + pair.Value : pair.Value; }); var dataflow = _factory.FromPropagator(splitter) .LinkToTarget(aggregater) .Create(); result = dict; return(dataflow); }
private static IPropagatorBlock <long, long> Create2CrossNode_1(Func <long> f) { var target = new TransformBlock <long, long>(x => f()); // f = LongOperation var source = new BroadcastBlock <long>(x => x); // linking target.LinkTo(source); // completion target.Completion.ContinueWith(completion => { if (completion.IsFaulted) { ((IDataflowBlock)source).Fault(completion.Exception); } else { source.Complete(); } }); return(DataflowBlock.Encapsulate(target, source)); }
public void BasicUsageTest() { int[] array = new int[10]; var evt = new CountdownEvent(10); var action = new ActionBlock <int> (i => { array [Math.Abs(i)] = i; evt.Signal(); }); var block = new TransformBlock <int, int> (i => - i); block.LinkTo(action); for (int i = 0; i < array.Length; ++i) { Assert.IsTrue(block.Post(i), "Not accepted"); } evt.Wait(); CollectionAssert.AreEqual( new[] { 0, -1, -2, -3, -4, -5, -6, -7, -8, -9 }, array); }
/// <summary> /// Add in an enumerable of items. /// </summary> /// <param name="collection"></param> public Task AddAsync(IEnumerable <T> collection) { if (collection == null) { throw new ArgumentNullException("collection"); } var produce = new TransformBlock <T, T>(item => item, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }); var consume = new ActionBlock <T>(action: async obj => await this.AddAsync(obj), dataflowBlockOptions: new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }); produce.LinkTo(consume); return(Task.Factory.StartNew(async() => { collection.AsParallel().ForAll(item => produce.SendAsync(item)); produce.Complete(); await consume.Completion; })); }
public ITargetBlock <InternalChannelHandlerContext <TKey, TProtocol> > InitReceive() { ExecutionDataflowBlockOptions options = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 }; var upPakc = new TransformManyBlock <InternalChannelHandlerContext <TKey, TProtocol>, InternalChannelHandlerContext <TKey, TProtocol> >(UnPack, options); var decoder = new TransformBlock <InternalChannelHandlerContext <TKey, TProtocol>, InternalChannelHandlerContext <TKey, TProtocol> >(Decoder, options); var deserialize = new TransformBlock <InternalChannelHandlerContext <TKey, TProtocol>, InternalChannelHandlerContext <TKey, TProtocol> >(Deserialize, options); var messageHandle = new ActionBlock <InternalChannelHandlerContext <TKey, TProtocol> >(MessageHandle, options); var failHandle = new ActionBlock <InternalChannelHandlerContext <TKey, TProtocol> >(FailHandle, options); upPakc.LinkTo(decoder, context => context.PipeStatus == PipeStatus.Success); upPakc.LinkTo(failHandle); decoder.LinkTo(deserialize, context => context.PipeStatus == PipeStatus.Success); decoder.LinkTo(failHandle); deserialize.LinkTo(messageHandle, context => context.PipeStatus == PipeStatus.Success); deserialize.LinkTo(failHandle); return(upPakc); }
private static void Example1() // NOTE: Синхронный прием данных { Func <int, int> fn = n => { Thread.Sleep(TimeSpan.FromSeconds(1)); return(n * n); }; var transformBlock = new TransformBlock <int, int>(fn); for (var i = 0; i < 10; i++) { transformBlock.Post(i); } for (var i = 0; i < 10; i++) { var result = transformBlock.Receive(); Console.WriteLine(result); } Console.WriteLine("Done"); }
public async Task TransformToAction() { var t = new TransformBlock <int, int>(i => i * 2); int completedCount = 0; int prev = -2; var c = new ActionBlock <int>(i => { completedCount++; Assert.Equal(expected: i, actual: prev + 2); prev = i; }); t.LinkTo(c, new DataflowLinkOptions { PropagateCompletion = true }); t.PostRange(0, Iterations); t.Complete(); await c.Completion; Assert.True(completedCount == Iterations); }
public void CreatePipeline() { var options = new ExecutionDataflowBlockOptions() { BoundedCapacity = 2, MaxDegreeOfParallelism = 10, EnsureOrdered = true }; buffer = new BufferBlock <Message>(); xForm1 = new TransformBlock <Message, Message>(x => { Console.WriteLine($"{DateTime.Now.TimeOfDay} - Started Id: {x.Id}"); Task.Delay(rnd.Next(1000, 3000)).Wait(); Console.WriteLine($"{DateTime.Now.TimeOfDay} - Finished Id: {x.Id}"); return(x); }, options); action = new ActionBlock <Message>(x => { Console.WriteLine($"{DateTime.Now.TimeOfDay} - Output Id: {x.Id} Value: {x.Value}"); //this delay will cause the messages to be unordered Task.Delay(rnd.Next(1000, 3000)).Wait(); OutputMessages.Add(x); }, options); }