public void TestBroadcastBlockConstructor() { try { // without option var block = new BroadcastBlock<int>(i => i); block = new BroadcastBlock<int>(null); // There is no property to verify. If construction itself fails, there will be an exception. //with not cancelled token and default scheduler block = new BroadcastBlock<int>(i => i, new DataflowBlockOptions { MaxMessagesPerTask = 1 }); block = new BroadcastBlock<int>(null, new DataflowBlockOptions { MaxMessagesPerTask = 1 }); // There is no property to verify. If construction itself fails, there will be an exception. //with a cancelled token and default scheduler var token = new CancellationToken(true); block = new BroadcastBlock<int>(i => i, new DataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token }); block = new BroadcastBlock<int>(null, new DataflowBlockOptions { MaxMessagesPerTask = 1, CancellationToken = token }); // There is no property to verify. If construction itself fails, there will be an exception. } catch (Exception ex) { Assert.True(false, "Test Failed, exception thrown during construction: " + ex.Message); } }
public void BasicUsageTest () { bool act1 = false, act2 = false; var evt = new CountdownEvent (2); var broadcast = new BroadcastBlock<int> (null); var action1 = new ActionBlock<int> (i => { act1 = i == 42; evt.Signal (); }); var action2 = new ActionBlock<int> (i => { act2 = i == 42; evt.Signal (); }); broadcast.LinkTo (action1); broadcast.LinkTo (action2); Assert.IsTrue (broadcast.Post (42)); Assert.IsTrue (evt.Wait (100)); Assert.IsTrue (act1); Assert.IsTrue (act2); }
public async Task TestPost() { var bb = new BroadcastBlock<int>(i => i); Assert.True(bb.Post(1)); bb.Complete(); Assert.False(bb.Post(2)); await bb.Completion; }
public void RegisterHandlers() { // add all handlers to the broadcast block so they receive news of events _broadcastBlock = new BroadcastBlock<IGESEvent>(x => x); // pass the two methods of the handler to the block, one determines if we are interested in this event, the other processes it. _eventHandlers.ForEach(x => _broadcastBlock.LinkTo(x.ReturnActionBlock(), x.HandlesEvent)); _broadcastBlock.LinkTo(DataflowBlock.NullTarget<IGESEvent>()); }
public async Task TestReceiveThenPost() { var bb = new BroadcastBlock<int>(null); var ignored = Task.Run(() => bb.Post(42)); Assert.Equal(expected: 42, actual: await bb.ReceiveAsync()); // this should always pass, but due to race we may not test what we're hoping to bb = new BroadcastBlock<int>(null); Task<int> t = bb.ReceiveAsync(); Assert.False(t.IsCompleted); bb.Post(16); Assert.Equal(expected: 16, actual: await t); }
/// <summary> /// 初期化処理。 /// </summary> /// <param name="settingObject"></param> /// <param name="token"></param> public void Init(dynamic settingObject, CancellationToken token) { logger.Trace("Init Start"); var opt = new DataflowBlockOptions { CancellationToken = token, }; var buffer = new BufferBlock<PastaLog>(opt); var bloadcast = new BroadcastBlock<PastaLog>(CloneLog, opt); buffer.LinkTo(bloadcast); Target = buffer; Source = bloadcast; logger.Trace("Init End"); }
/// <summary> /// Initializes a new instance of the <see cref="LogManager"/> class. /// </summary> /// <param name="loggingConfiguration">The logging configuration.</param> public LogManager(LoggingConfiguration loggingConfiguration) { _LoggingConfiguration = loggingConfiguration; _Broadcast = new BroadcastBlock<LogEntry>( logEntry => logEntry.Clone(), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = loggingConfiguration.MaxDegreeOfParallelism <= 0 ? Environment.ProcessorCount : loggingConfiguration.MaxDegreeOfParallelism }); }
private static BroadcastBlock<int> ConstructBroadcastNewWithNMessages(int messagesCount) { var block = new BroadcastBlock<int>(i => i); for (int i = 0; i < messagesCount; i++) { block.Post(i); } // Wait until the messages have been properly buffered up. // Otherwise TryReceive fails. // Since there is no property to check on the BroadcastBlock, just sleep for 2 ms Task.Delay(2).Wait(); return block; }
public Statsd(string serviceName = null) { _log.Info("statsd.net starting."); _tokenSource = new CancellationTokenSource(); _shutdownComplete = new ManualResetEvent(false); SuperCheapIOC.Add(_log); var systemInfoService = new SystemInfoService(); SuperCheapIOC.Add(systemInfoService as ISystemInfoService); serviceName = serviceName ?? systemInfoService.HostName; var systemMetricsService = new SystemMetricsService("statsdnet", serviceName); SuperCheapIOC.Add(systemMetricsService as ISystemMetricsService); /** * The flow is: * Listeners -> * Message Parser -> * router -> * Aggregator -> * Broadcaster -> * Backends */ // Initialise the core blocks _router = new StatsdMessageRouterBlock(); _messageParser = MessageParserBlockFactory.CreateMessageParserBlock(_tokenSource.Token, SuperCheapIOC.Resolve<ISystemMetricsService>(), _log); _messageParser.LinkTo(_router); _messageParser.Completion.LogAndContinueWith(_log, "MessageParser", () => { _log.Info("MessageParser: Completion signaled. Notifying the MessageBroadcaster."); _messageBroadcaster.Complete(); }); _messageBroadcaster = new BroadcastBlock<Bucket>(Bucket.Clone); _messageBroadcaster.Completion.LogAndContinueWith(_log, "MessageBroadcaster", () => { _log.Info("MessageBroadcaster: Completion signaled. Notifying all backends."); _backends.ForEach(q => q.Complete()); }); // Add the broadcaster to the IOC container SuperCheapIOC.Add<BroadcastBlock<Bucket>>(_messageBroadcaster); systemMetricsService.SetTarget(_messageBroadcaster); _backends = new List<IBackend>(); _listeners = new List<IListener>(); }
async Task GenerateFileFingerprintsAsync(CollectionPath[] paths, Func<FileInfo, bool> filePredicate, ITargetBlock<Tuple<AnnotatedPath, IFileFingerprint>> joinedTargetBlock, CancellationToken cancellationToken) { try { var annotatedPathBroadcastBlock = new BroadcastBlock<AnnotatedPath[]>(aps => aps, new DataflowBlockOptions { CancellationToken = cancellationToken }); var joiner = new LinkFingerprintJoiner(joinedTargetBlock); annotatedPathBroadcastBlock.LinkTo(joiner.AnnotatedPathsBlock, new DataflowLinkOptions { PropagateCompletion = true }); var fileFingerprintBroadcastBlock = new BroadcastBlock<IFileFingerprint>(ff => ff, new DataflowBlockOptions { CancellationToken = cancellationToken }); fileFingerprintBroadcastBlock.LinkTo(joiner.FileFingerprintBlock, new DataflowLinkOptions { PropagateCompletion = true }); var fingerprintGeneratorTask = _fileFingerprintManager .GenerateFileFingerprintsAsync(annotatedPathBroadcastBlock, fileFingerprintBroadcastBlock, cancellationToken); try { await DirectoryScanner.GenerateAnnotatedPathsAsync(paths, filePredicate, annotatedPathBroadcastBlock, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { } catch (Exception ex) { Console.Write("Path scan failed: " + ex.Message); } await fingerprintGeneratorTask.ConfigureAwait(false); } catch (Exception ex) { Console.WriteLine("GenerateFileFingerprintsAsync() failed: " + ex.Message); } finally { Debug.WriteLine("BlobManager.GenerateFileFingerprintsAsync() is done"); } }
public async Task TestBroadcasting() { var bb = new BroadcastBlock<int>(i => i + 1); var targets = Enumerable.Range(0, 3).Select(_ => new TransformBlock<int, int>(i => i)).ToArray(); foreach (var target in targets) { bb.LinkTo(target); } const int Messages = 3; bb.PostRange(0, Messages); for (int i = 0; i < Messages; i++) { foreach (var target in targets) { Assert.Equal(expected: i + 1, actual: await target.ReceiveAsync()); } } }
public void LinkAfterPostTest () { bool act = false; var evt = new ManualResetEventSlim (); var broadcast = new BroadcastBlock<int> (null); var action = new ActionBlock<int> (i => { act = i == 42; evt.Set (); }); Assert.IsTrue (broadcast.Post (42)); broadcast.LinkTo (action); Assert.IsTrue (evt.Wait (100)); Assert.IsTrue (act); }
public MsBuildFileSystemWatcher(string directory, string filter, int delayMilliseconds, IFileSystem fileSystem, IMsBuildFileSystemFilter fileSystemFilter, TaskScheduler taskScheduler = null, IActionLog log = null) { Requires.NotNullOrWhiteSpace(directory, nameof(directory)); Requires.NotNullOrWhiteSpace(filter, nameof(filter)); Requires.Range(delayMilliseconds >= 0, nameof(delayMilliseconds)); Requires.NotNull(fileSystem, nameof(fileSystem)); Requires.NotNull(fileSystemFilter, nameof(fileSystemFilter)); _directory = directory; _filter = filter; _delayMilliseconds = delayMilliseconds; _fileSystem = fileSystem; _fileSystemFilter = fileSystemFilter; _taskScheduler = taskScheduler ?? TaskScheduler.Default; _log = log ?? ProjectSystemActionLog.Default; _queue = new ConcurrentQueue<IFileSystemChange>(); _broadcastBlock = new BroadcastBlock<Changeset>(b => b, new DataflowBlockOptions { TaskScheduler = _taskScheduler }); SourceBlock = _broadcastBlock.SafePublicize(); _fileSystemFilter.Seal(); }
public static void Example3() { // Scenario: one buffer and 3 actions connected to it - each number precessed is send to each action var conf = new ExecutionDataflowBlockOptions(); //conf.MaxDegreeOfParallelism = 4; var buffer = new BroadcastBlock<int>(i => i); var action1 = new ActionBlock<int>(a => { Thread.Sleep(50); Console.WriteLine("Action 1 value: {0}", a); }, conf); var action2 = new ActionBlock<int>(a => { Thread.Sleep(50); Console.WriteLine("Action 2 value: {0}", a); }, conf); var action3 = new ActionBlock<int>(a => { Thread.Sleep(50); Console.WriteLine("Action 3 value: {0}", a); }, conf); buffer.LinkTo(action1); buffer.LinkTo(action2); buffer.LinkTo(action3); var t = new Task(() => { for (int i = 0; i < 12; i++) { buffer.Post(i); } }); t.Start(); }
public void Run() { var propagateCompletion = new DataflowLinkOptions { PropagateCompletion = true }; var csep = new ConcurrentExclusiveSchedulerPair(); var concurrentOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 2, TaskScheduler = csep.ConcurrentScheduler }; var exclusiveOptions = new ExecutionDataflowBlockOptions { TaskScheduler = csep.ExclusiveScheduler }; var concurrent = new ActionBlock<int>(async i => { Console.WriteLine("Concurent print value: {0}", i); await Task.Delay(500); }, concurrentOptions); var exclusive = new ActionBlock<int>(i => { Console.WriteLine("Exclusive print value: {0}", i); Thread.Sleep(750); }, exclusiveOptions); var broadcaster = new BroadcastBlock<int>(i => i); broadcaster.LinkTo(concurrent, propagateCompletion); broadcaster.LinkTo(exclusive, propagateCompletion, i => i % 2 == 0); Enumerable .Range(1, 10) .ToList() .ForEach(i => broadcaster.Post(i)); broadcaster.Complete(); Task.WaitAll(concurrent.Completion, exclusive.Completion); }
/// <summary> /// Need to amke sure we cleanup the dataflow links and file watcher /// </summary> /// <param name="disposing"></param> protected override void Dispose(bool disposing) { if (disposing) { CleanupFileWatcher(); if (FileChangeScheduler != null) { FileChangeScheduler.Dispose(); FileChangeScheduler = null; } if (_broadcastBlock != null) { _broadcastBlock.Complete(); _broadcastBlock = null; } if (_projectRuleSubscriptionLink != null) { _projectRuleSubscriptionLink.Dispose(); _projectRuleSubscriptionLink = null; } } }
public void FaultConsumeBroadcast() { var scheduler = new TestScheduler(); var source = new BroadcastBlock <int> (null, new DataflowBlockOptions { TaskScheduler = scheduler }); var target = new TestTargetBlock <int> { Postpone = true }; Assert.IsNotNull(source.LinkTo(target)); Assert.IsTrue(source.Post(42)); scheduler.ExecuteAll(); Assert.IsTrue(target.HasPostponed); var exception = new Exception(); ((IDataflowBlock)source).Fault(exception); scheduler.ExecuteAll(); try { source.Completion.Wait(1000); Assert.Fail("Task must be faulted"); } catch (AggregateException ex) { Assert.AreEqual(exception, ex.InnerException, "#9"); } Assert.IsTrue(source.Completion.IsFaulted); int value; Assert.IsTrue(target.RetryPostponed(out value)); Assert.AreEqual(42, value); }
public void CloningTest () { object act1 = null, act2 = null; var evt = new CountdownEvent (2); object source = new object (); var broadcast = new BroadcastBlock<object> (o => new object ()); var action1 = new ActionBlock<object> (i => { act1 = i; evt.Signal (); }); var action2 = new ActionBlock<object> (i => { act2 = i; evt.Signal (); }); broadcast.LinkTo (action1); broadcast.LinkTo (action2); Assert.IsTrue (broadcast.Post (source)); evt.Wait (); Assert.IsNotNull (act1); Assert.IsNotNull (act2); Assert.IsFalse (source.Equals (act1)); Assert.IsFalse (source.Equals (act2)); Assert.IsFalse (act2.Equals (act1)); }
/// <summary> /// 新增观察者 /// </summary> /// <typeparam name="T"></typeparam> public void AddObserver(object observer, string notificationName, Action <object> action) { ConcurrentDictionary <object, ActionBlock <object> > dicActionBlock; if (!subscribeList.TryGetValue(notificationName, out dicActionBlock)) { dicActionBlock = new ConcurrentDictionary <object, ActionBlock <object> >(); subscribeList.TryAdd(notificationName, dicActionBlock); } ActionBlock <object> handler; if (dicActionBlock.TryGetValue(observer, out handler)) { return; } handler = new ActionBlock <object>(obj => { if (action != null) { action(obj); } }); dicActionBlock.TryAdd(observer, handler); BroadcastBlock <object> broadcast; if (!broadcastList.TryGetValue(notificationName, out broadcast)) { broadcast = new BroadcastBlock <object>(msg => msg); broadcastList.TryAdd(notificationName, broadcast); } broadcast.LinkTo(handler); }
/// <summary> /// The DebugProfileProvider sinks 2 sets of information /// 1, Changes to the launchsettings.json file on disk /// 2. Changes to the ActiveDebugProfile property in the .user file /// </summary> protected override void Initialize() { // Create our broadcast block for subscribers to get new ILaunchProfiles Information _broadcastBlock = new BroadcastBlock <ILaunchSettings>(s => s); _changedSourceBlock = _broadcastBlock.SafePublicize(); // Subscribe to changes to the broadcast block using the idle scheduler. This should filter out a lot of the intermediates // states that files can be in. if (ProjectSubscriptionService != null) { var projectChangesBlock = new ActionBlock <IProjectVersionedValue <IProjectSubscriptionUpdate> >(ProjectRuleBlock_ChangedAsync); _projectRuleSubscriptionLink = ProjectSubscriptionService.ProjectRuleSource.SourceBlock.LinkTo( projectChangesBlock, ruleNames: ProjectDebugger.SchemaName, linkOptions: new DataflowLinkOptions { PropagateCompletion = true }); } // Make sure we are watching the file at this point WatchLaunchSettingsFile(); }
/// <summary> /// Create service /// </summary> /// <param name="connectionString"></param> public IoTHubService(string connectionString = null) { if (string.IsNullOrEmpty(connectionString)) { connectionString = Environment.GetEnvironmentVariable("_HUB_CS"); } if (string.IsNullOrEmpty(connectionString)) { throw new ArgumentException("You must provide the iothubowner connection " + "string, which can be obtained from the Azure IoT Hub portal. You must " + "then either initialize the DefaultProvider class, or set the _HUB_CS " + "environment variable."); } _hubConnectionString = ConnectionString.Parse(connectionString); _updateQueue = new BufferBlock <Tuple <INameRecord, NameServiceOperation> >( new DataflowBlockOptions { NameFormat = "Update (in IoTHubService) Id={1}", EnsureOrdered = false, MaxMessagesPerTask = DataflowBlockOptions.Unbounded, CancellationToken = _open.Token }); _listeners = new BroadcastBlock <Tuple <INameRecord, NameServiceEvent> >(null, new DataflowBlockOptions { NameFormat = "Notify (in IoTHubService) Id={1}", EnsureOrdered = false, MaxMessagesPerTask = DataflowBlockOptions.Unbounded, CancellationToken = _open.Token }); Write = DataflowBlockEx.Encapsulate(_updateQueue, _listeners); _registryLoaded = new TaskCompletionSource <bool>(); _cache = new ConcurrentDictionary <Reference, IoTHubRecord>(); _cacheWorker = CacheWorker(); }
public Logger(int maxBufferCapacity = 100, params ILogSink[] sinks) { _cancellationSource = new CancellationTokenSource(); _buffer = new BufferBlock <ILogEntry>( new DataflowBlockOptions { BoundedCapacity = maxBufferCapacity, CancellationToken = _cancellationSource.Token }); _broadcast = new BroadcastBlock <ILogEntry>(entry => entry, new DataflowBlockOptions { CancellationToken = _cancellationSource.Token }); LinkDataflowBlocks(_buffer, _broadcast); _sinks = new List <ActionBlock <ILogEntry> >(); foreach (var sink in sinks) { AddLogSink(sink); } }
public void BoundedQueuedTest () { var scheduler = new TestScheduler (); var broadcast = new BroadcastBlock<int> ( null, new DataflowBlockOptions { TaskScheduler = scheduler, BoundedCapacity = 1 }); Assert.IsTrue (broadcast.Post (1)); Assert.IsFalse (broadcast.Post (2)); }
public void ReserveChangedTest () { var scheduler = new TestScheduler (); var broadcast = new BroadcastBlock<int> (null, new DataflowBlockOptions { TaskScheduler = scheduler }); var target = new TestTargetBlock<int> { Postpone = true }; broadcast.LinkTo (target); Assert.IsFalse (target.HasPostponed); Assert.IsTrue (broadcast.Post (1)); scheduler.ExecuteAll (); Assert.IsTrue (target.HasPostponed); Assert.IsTrue(broadcast.Post(2)); scheduler.ExecuteAll (); Assert.IsTrue (target.ReservePostponed ()); int value; Assert.IsTrue (target.RetryPostponed (out value)); Assert.AreEqual (2, value); }
static async void StartCrawling() { if (!Directory.Exists("Images")) { Directory.CreateDirectory("Images"); } try { #region Dataflow block Options var downloaderOptions = new ExecutionDataflowBlockOptions { // enforce fairness, after handling n messages // the block's task will be re-schedule. // this will give the opportunity for other block // to actively process there messages (to avoid over subscription // the Tpl dataflow does not schedule all task at once if the machine // does not have enough cores) MaxMessagesPerTask = DOWNLOADER_MAX_MESSAGE_PER_TASK, // by default Tpl dataflow assign a single task per block, // but you can control it by using the MaxDegreeOfParallelism MaxDegreeOfParallelism = 2, //DOWNLOADER_MAX_DEGREE_OF_PARALLELISM, // the size of the block input buffer BoundedCapacity = DOWNLOADER_BOUNDED_CAPACITY }; var transformerOptions = new ExecutionDataflowBlockOptions { MaxMessagesPerTask = MAX_MESSAGE_PER_TASK, }; var writerOptions = new ExecutionDataflowBlockOptions { MaxMessagesPerTask = WRITER_MAX_DEGREE_OF_PARALLELISM, MaxDegreeOfParallelism = DOWNLOADER_MAX_DEGREE_OF_PARALLELISM }; var linkOption = new DataflowLinkOptions { PropagateCompletion = true }; #endregion // Dataflow block Options #region Downloader var downloader = new TransformBlock <string, XElement>( async(url) => { try { #region Validation //if (_urls.ContainsKey(url)) // return null; if (!_urls.TryAdd(url, true)) { return(null); } #endregion // Validation // using IOCP the thread pool worker thread does return to the pool WebClient wc = new WebClient(); Task <string> download = wc.DownloadStringTaskAsync(url); Task cancel = Task.Delay(DOWNLOAD_TIMEOUT_SEC * 1000); Task any = await Task.WhenAny(download, cancel).ConfigureAwait(false); #region Timeout validation if (any == cancel) { wc.CancelAsync(); WriteToConsole("Cancel: [{0}]", ConsoleColor.Gray, url); return(null); } #endregion // Timeout validation string html = download.Result; WriteToConsole("Downloaded: {0}", ConsoleColor.White, url); XElement e = HtmlToXElement(html); return(e); } #region Exception Handling catch (WebException ex) { WriteToConsole("Error: [{0}]\r\n\t{1}", ConsoleColor.Red, url, ex.Message); } catch (AggregateException ex) { foreach (var exc in ex.Flatten().InnerExceptions) { WriteToConsole("Error: [{0}]\r\n\t{1}", ConsoleColor.Red, url, exc.Message); } } catch (Exception ex) { WriteToConsole("Unexpected error: {0}", ConsoleColor.Red, ex.Message); } #endregion // Exception Handling return(null); }, downloaderOptions); #endregion // Downloader #region Link Parser var linkParser = new TransformManyBlock <XElement, string>( (html) => { #region Validation if (html == null) { return(Enumerable.Empty <string>()); } #endregion // Validation var linkes = from item in html.Descendants() where item.Name.LocalName == "a" let href = item.Attribute("href") where href != null select href.Value; try { var result = linkes.ToArray(); return(result); } #region Exception Handling catch (Exception ex) { WriteToConsole("Unexpected error: {0}", ConsoleColor.Red, ex.Message); return(Enumerable.Empty <string>()); } #endregion // Exception Handling }, transformerOptions); #endregion // Link Parser #region Image Parser var imgParser = new TransformManyBlock <XElement, string>( (html) => { var images = from item in html.Descendants() where item.Name.LocalName == "img" let src = item.Attribute("src") where src != null select src.Value; try { var result = images.ToArray(); return(result); } #region Exception Handling catch (Exception ex) { WriteToConsole("Unexpected error: {0}", ConsoleColor.Red, ex.Message); return(Enumerable.Empty <string>()); } #endregion // Exception Handling }, transformerOptions); #endregion // Image Parser #region Writer var writer = new ActionBlock <string>(async url => { try { if (string.IsNullOrEmpty(url)) { return; } WebClient wc = new WebClient(); // using IOCP the thread pool worker thread does return to the pool byte[] buffer = await wc.DownloadDataTaskAsync(url).ConfigureAwait(false); string fileName = Path.GetFileName(url); #region Validation if (!_images.TryAdd(fileName, true)) { return; } #endregion // Validation string name = @"Images\" + fileName; using (var image = Image.FromStream(new MemoryStream(buffer))) { if (image.Width > MIN_SIZE.Width && image.Height > MIN_SIZE.Height) { using (Stream srm = OpenWriteAsync(name)) { await srm.WriteAsync(buffer, 0, buffer.Length).ConfigureAwait(false); WriteToConsole("{0}: Width:{1}, Height:{2}", ConsoleColor.Yellow, fileName, image.Width, image.Height); } } } } #region Exception Handling catch (WebException ex) { WriteToConsole("Error: [{0}]\r\n\t{1}", ConsoleColor.Red, url, ex.Message); } catch (Exception ex) { WriteToConsole("Unexpected error: {0}", ConsoleColor.Red, ex.Message); } #endregion // Exception Handling }, writerOptions); #endregion // Writer var garbageA = DataflowBlock.NullTarget <XElement>(); var garbageB = DataflowBlock.NullTarget <string>(); // use to broadcast the downloader output to the image and link parsers var contentBroadcaster = new BroadcastBlock <XElement>(s => s); var linkBroadcaster = new BroadcastBlock <string>(s => s); #region LinkTo /////////////////////////////////////////////////////////////////////// // garbage <------- downloader <------------------------- // // | | // // contentBroadcaster | // // / \ | // // imgParser linkParsers--->linkBroadcaster // // | | // // writer <----------------------------------------- // /////////////////////////////////////////////////////////////////////// downloader.LinkTo(contentBroadcaster, linkOption, html => html != null); downloader.LinkTo(garbageA); //, linkOption /*, html => html == null*/); // fallback (otherwise empty messages will be stack in the block buffer and the block will never complete) contentBroadcaster.LinkTo(imgParser, linkOption); contentBroadcaster.LinkTo(linkParser, linkOption); linkParser.LinkTo(linkBroadcaster, linkOption); StringComparison comparison = StringComparison.InvariantCultureIgnoreCase; Predicate <string> linkFilter = link => link.StartsWith("http://"); Predicate <string> imgFilter = url => url.StartsWith("http://") && (url.EndsWith(".jpg", comparison) || url.EndsWith(".png", comparison) || url.EndsWith(".gif", comparison)); // Predicate<string> imgToGarbageFilter = url => !imgFilter(url); imgParser.LinkTo(writer, linkOption, imgFilter); imgParser.LinkTo(garbageB);// , imgToGarbageFilter); linkBroadcaster.LinkTo(writer, linkOption, imgFilter); linkBroadcaster.LinkTo(downloader, linkOption, linkFilter); //linkBroadcaster.LinkTo(garbage); #endregion // LinkTo downloader.Post(URL_CRAWL_TARGET); Console.WriteLine("Crawling"); Thread.Sleep(COMPLETE_AFTER_SEC * 1000); #region Complete downloader.Complete(); #region WriteToConsole ("Try to Complete...") ConsoleColor color = ConsoleColor.Yellow; WriteToConsole( @"Try to Complete (items in the buffer = downloader: is completed = {0}, input={1} , output={2} writer: is completed = {3}, input ={4} linkParser: is completed = {5}, input={6} , output={7} imgParser: is completed = {8}, input={9} , output={10} linkBroadcaster: is completed = {11}, contentBroadcaster: is completed = {12}", color, downloader.Completion.IsCompleted, downloader.InputCount, downloader.OutputCount, writer.Completion.IsCompleted, writer.InputCount, linkParser.Completion.IsCompleted, linkParser.InputCount, linkParser.OutputCount, imgParser.Completion.IsCompleted, imgParser.InputCount, imgParser.OutputCount, linkBroadcaster.Completion.IsCompleted, contentBroadcaster.Completion.IsCompleted); #endregion // WriteToConsole ("Try to Complete...") Task completeAll = Task.WhenAll( downloader.Completion, linkParser.Completion, imgParser.Completion, contentBroadcaster.Completion, writer.Completion); await Task.Run(async() => { while (!completeAll.IsCompleted) { await Task.Delay(2000).ConfigureAwait(false); #region WriteToConsole (status) color = color == ConsoleColor.Magenta ? ConsoleColor.White : ConsoleColor.Yellow; WriteToConsole( @"Complete Status (items in the buffer = downloader: is completed = {0}, input={1} , output={2} writer: is completed = {3}, input ={4} linkParser: is completed = {5}, input={6} , output={7} imgParser: is completed = {8}, input={9} , output={10} linkBroadcaster: is completed = {11}, contentBroadcaster: is completed = {12} ", color, downloader.Completion.IsCompleted, downloader.InputCount, downloader.OutputCount, writer.Completion.IsCompleted, writer.InputCount, linkParser.Completion.IsCompleted, linkParser.InputCount, linkParser.OutputCount, imgParser.Completion.IsCompleted, imgParser.InputCount, imgParser.OutputCount, linkBroadcaster.Completion.IsCompleted, contentBroadcaster.Completion.IsCompleted); } #endregion // WriteToConsole (status) }).ConfigureAwait(false); WriteToConsole("Complete (items in the writer input buffer = {0})", ConsoleColor.Green, writer.InputCount); #endregion // Complete } catch (Exception ex) { WriteToConsole("EXCEPTION: {0}", ConsoleColor.DarkRed, ex); } }
public TPLDataflowSubscribers() { _jobs = new BroadcastBlock <IJob>(job => job); }
public async Task TestFaultyScheduler() { var bb = new BroadcastBlock<int>(null, new DataflowBlockOptions { BoundedCapacity = 1, TaskScheduler = new DelegateTaskScheduler { QueueTaskDelegate = delegate { throw new FormatException(); } } }); Task<bool> t1 = bb.SendAsync(1); Task<bool> t2 = bb.SendAsync(2); bb.LinkTo(DataflowBlock.NullTarget<int>()); await Assert.ThrowsAsync<TaskSchedulerException>(() => bb.Completion); Assert.True(await t1); Assert.False(await t2); }
public async Task TestBounding() { var bb = new BroadcastBlock<int>(null, new DataflowBlockOptions { BoundedCapacity = 1 }); var ab = new ActionBlock<int>(i => { }); bb.LinkTo(ab, new DataflowLinkOptions { PropagateCompletion = true }); Task<bool>[] sends = Enumerable.Range(0, 40).Select(i => bb.SendAsync(i)).ToArray(); bb.Complete(); await Task.WhenAll(sends); await ab.Completion; }
static void Main() { var rand = new Random(DateTime.Now.Millisecond); var broadcastBlock = new BroadcastBlock <int>(x => x); var transformPositive = new TransformBlock <int, int>(x => { Thread.Sleep(1000); return(x); }); var transformNegative = new TransformBlock <int, int>(x => { Thread.Sleep(2000); return(-x); }); var join = new JoinBlock <int, int>(); var batchBlock = new BatchBlock <Tuple <int, int> >(5); var sumBlock = new ActionBlock <Tuple <int, int>[]>(tuples => { foreach (var tuple in tuples) { Console.WriteLine($"{tuple.Item1}+({tuple.Item2})={tuple.Item1 + tuple.Item2}"); } }); broadcastBlock.LinkTo(transformPositive, new DataflowLinkOptions { PropagateCompletion = true }); broadcastBlock.LinkTo(transformNegative, new DataflowLinkOptions { PropagateCompletion = true }); transformPositive.LinkTo(join.Target1, new DataflowLinkOptions { PropagateCompletion = true }); transformNegative.LinkTo(join.Target2, new DataflowLinkOptions { PropagateCompletion = true }); join.LinkTo(batchBlock, new DataflowLinkOptions { PropagateCompletion = true }); batchBlock.LinkTo(sumBlock, new DataflowLinkOptions { PropagateCompletion = true }); for (int i = 0; i < 30; i++) { broadcastBlock.Post(rand.Next(100)); Thread.Sleep(1000); } broadcastBlock.Complete(); Console.ReadLine(); }
static void Main(string[] args) { StringComparison comparison = StringComparison.InvariantCultureIgnoreCase; Predicate <string> linkFilter = link => link.IndexOf(".aspx", comparison) != -1 || link.IndexOf(".php", comparison) != -1 || link.IndexOf(".htm", comparison) != -1 || link.IndexOf(".html", comparison) != -1 || link.EndsWith(".com", comparison) || link.EndsWith(".net", comparison); Predicate <string> imgFilter = url => url.EndsWith(".jpg", comparison) || url.EndsWith(".png", comparison) || url.EndsWith(".gif", comparison); var downloader = new TransformBlock <string, string>( async(url) => { // using IOCP the thread pool worker thread does return to the pool var client = new HttpClient(); string result = await client.GetStringAsync(url); return(result); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 2 }); var contentBroadcaster = new BroadcastBlock <string>(s => s); var linkParser = new TransformManyBlock <string, string>( (html) => { var output = new List <string>(); var links = _linkRegexHRef.Matches(html); foreach (Match item in links) { var value = item.Groups["LINK"].Value; output.Add(value); } return(output); }); var imgParser = new TransformManyBlock <string, string>( (html) => { var output = new List <string>(); var images = _imgRegex.Matches(html); foreach (Match item in images) { var value = item.Groups["IMG"].Value; output.Add(value); } return(output); }); var writer = new ActionBlock <string>(async url => { var client = new HttpClient(); // using IOCP the thread pool worker thread does return to the pool byte[] buffer = await client.GetByteArrayAsync(url); string fileName = Path.GetFileName(url); Directory.CreateDirectory(AppDomain.CurrentDomain.BaseDirectory + @"\Images\"); string name = AppDomain.CurrentDomain.BaseDirectory + @"\Images\" + fileName; using (Stream srm = File.OpenWrite(name)) { await srm.WriteAsync(buffer, 0, buffer.Length); } }); var linkBroadcaster = new BroadcastBlock <string>(s => s); IDisposable disposeAll = new CompositeDisposable( // from [downloader] to [contentBroadcaster] downloader.LinkTo(contentBroadcaster), // from [contentBroadcaster] to [imgParser] contentBroadcaster.LinkTo(imgParser), // from [contentBroadcaster] to [linkParserHRef] contentBroadcaster.LinkTo(linkParser), // from [linkParser] to [linkBroadcaster] linkParser.LinkTo(linkBroadcaster), // conditional link to from [linkBroadcaster] to [downloader] linkBroadcaster.LinkTo(downloader, linkFilter), // from [linkBroadcaster] to [writer] linkBroadcaster.LinkTo(writer, imgFilter), // from [imgParser] to [writer] imgParser.LinkTo(writer)); downloader.Post("http://www.reactivex.io"); Console.WriteLine("Press any key to exit..."); Console.ReadLine(); downloader.Complete(); }
public void Test() { var action = new ActionBlock <int[]>(x => { }, new ExecutionDataflowBlockOptions { }); var batch = new BatchBlock <int>(10, new GroupingDataflowBlockOptions { }); using var likBA = batch.LinkTo(action, new DataflowLinkOptions { }, x => x[0] > 1); //actionBlock gets 10 elements batch.TriggerBatch(); //for leftover elements var batchJoined = new BatchedJoinBlock <int[], string>(10, new GroupingDataflowBlockOptions { }); batch.LinkTo(batchJoined.Target1, new DataflowLinkOptions { }, ints => ints[0] > 0); var action2 = new ActionBlock <Tuple <IList <int[]>, IList <string> > >(x => { }); using var linkBJA = batchJoined.LinkTo(action2, new DataflowLinkOptions { }, x => x.Item1.Count == x.Item2.Count);//it's 10 total so 7-3 or 0-10 will propagate var broadcast = new BroadcastBlock <int>(cloningFunction: x => x, new DataflowBlockOptions { }); var action3 = new ActionBlock <int>(x => { }); var action4 = new ActionBlock <int>(x => { }); broadcast.LinkTo(action3, new DataflowLinkOptions { }, x => x > 0); broadcast.LinkTo(action4, new DataflowLinkOptions { }, x => x > 0); //both will gete same elements var buffer = new BufferBlock <int>(new DataflowBlockOptions { }); var action5 = new ActionBlock <int>(x => { }); buffer.LinkTo(action5, new DataflowLinkOptions { }, x => x == 0); var buffer1 = new BufferBlock <int>(); var join = new JoinBlock <int, string>(new GroupingDataflowBlockOptions { }); buffer1.LinkTo(join.Target1); var action6 = new ActionBlock <Tuple <int, string> >(tlp => { }); join.LinkTo(action6); var transform = new TransformBlock <int, string>(x => x.ToString(), new ExecutionDataflowBlockOptions { }); var action7 = new ActionBlock <string>(str => { }); transform.LinkTo(action7); var transformMany = new TransformManyBlock <int, string>( x => Enumerable.Range(0, x).Select(i => i.ToString()));//one recived to many output var action8 = new ActionBlock <string>(str => { }); var writeOnce = new WriteOnceBlock <int>(cloningFunction: x => x, new DataflowBlockOptions { }); // it gets and stores 1 element but gives clones to linked blocks? var action9 = new ActionBlock <int>(x => { }); writeOnce.LinkTo(action9); var bufferC1 = new BufferBlock <int>(); var bufferC2 = new BufferBlock <string>(); var actionIdx = DataflowBlock.Choose <int, string>( bufferC1, x => Console.WriteLine(x + 1), bufferC2, str => Console.WriteLine(str + "")); var actionE1 = new ActionBlock <int>(x => { }); var bufferE1 = new BufferBlock <string>(); var propagator = DataflowBlock.Encapsulate <int, string>(actionE1, bufferE1); var scheduler = new ConcurrentExclusiveSchedulerPair(TaskScheduler.Default, Environment.ProcessorCount, int.MaxValue); var concurrent = scheduler.ConcurrentScheduler; var exclusive = scheduler.ExclusiveScheduler; }
private static async Task Main(string[] args) { var baseUri = new Uri(args[0]); var outputSize = args.Length > 1 ? int.Parse(args[1]) : DefaultOutputSize; var downloadString = new TransformBlock <Uri, string>( async uri => { using var client = new HttpClient(); return(await client.GetStringAsync(uri)); }); var visitedUris = new HashSet <Uri>(); var findUris = new TransformManyBlock <string, Uri>( html => AnchorRef.Matches(html) .Cast <Match>() .Select(m => new Uri(m.Value, UriKind.RelativeOrAbsolute)) .Select(uri => uri.IsAbsoluteUri ? uri : new Uri(baseUri, uri)) .Where(uri => visitedUris.Add(uri)), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = Environment.ProcessorCount, }); var broadcastUris = new BroadcastBlock <Uri>(uri => uri); var collectResults = new BatchBlock <Uri>(outputSize); downloadString.LinkTo(findUris); findUris.LinkTo(broadcastUris); broadcastUris.LinkTo(collectResults); broadcastUris.LinkTo( downloadString, uri => baseUri.IsBaseOf(uri) && uri != baseUri); downloadString.Post(baseUri); try { var uris = await collectResults.ReceiveAsync(TimeSpan.FromSeconds(5)); PrintResults(uris); } catch (InvalidOperationException exception) // time out { PrintException(exception); } finally { downloadString.Complete(); } try { await downloadString.Completion; } catch (AggregateException exception) { PrintException(exception); } catch (Exception exception) { PrintException(exception); } }
public flow() { m_buffer = new BufferBlock <object>(); BroadCast = new BroadcastBlock <string>(str => str); }
public void TryReceiveTest() { var scheduler = new TestScheduler(); var block = new BroadcastBlock<int>(i => i * 10, new DataflowBlockOptions { TaskScheduler = scheduler }); int item; Assert.IsFalse(block.TryReceive(null, out item)); Assert.IsTrue(block.Post(1)); Assert.IsTrue(block.Post(2)); scheduler.ExecuteAll(); Assert.IsTrue(block.TryReceive(null, out item)); Assert.AreEqual(20, item); // predicate is tested on original value, but returned is cloned Assert.IsTrue(block.TryReceive(i => i < 10, out item)); Assert.AreEqual(20, item); }
public ProviderManagerFlow( ILogger <ProviderManagerFlow> logger, IMessageDumper messageDumper, HttpClientManager httpClientManager, IHttpRequestMessageFactory httpRequestMessageFactory, ITranslationDistributer translationDistributer ) { this.logger = logger; this.messageDumper = messageDumper; this.httpClientManager = httpClientManager; this.httpRequestMessageFactory = httpRequestMessageFactory; this.translationDistributer = translationDistributer; createHttpRequestBlock = new TransformManyBlock <MessageContext <TranslationsRequest>, MessageContext <HttpRequestMessage> >( CreateHttpRequestHandler, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = maxDegreeOfParallelism } ); execHttpRequestBlock = new TransformManyBlock <MessageContext <HttpRequestMessage>, MessageContext <HttpResponseMessage> >( ExecHttpRequestHandler, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = maxDegreeOfParallelism } ); parseHttpResponseBlock = new TransformManyBlock <MessageContext <HttpResponseMessage>, MessageContext <string> >( ParseHttpResponseHandler, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = maxDegreeOfParallelism } ); broadcastResponseBlock = new BroadcastBlock <MessageContext <string> >(t => t); dumpResponseBlock = new ActionBlock <MessageContext <string> >( DumpResponseHandler, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = maxDegreeOfParallelism } ); deserializeResponseBlock = new TransformManyBlock <MessageContext <string>, MessageContext <Translation> >( DeserializeResponseHandler, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = maxDegreeOfParallelism } ); translationDistributerBlock = new ActionBlock <MessageContext <Translation> >( TranslationDistributerHandler, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = maxDegreeOfParallelism } ); createHttpRequestBlock.LinkTo(execHttpRequestBlock); execHttpRequestBlock.LinkTo(parseHttpResponseBlock); parseHttpResponseBlock.LinkTo(broadcastResponseBlock); broadcastResponseBlock.LinkTo(dumpResponseBlock); broadcastResponseBlock.LinkTo(deserializeResponseBlock); deserializeResponseBlock.LinkTo(translationDistributerBlock); }
public Task StartAsync(CancellationToken cancellationToken) { this.m_logger.LogDebug("Starting imagegrabber service"); this.m_cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); this.m_cts.Token.Register(() => { this.m_logger.LogDebug("ImageGrabberService cancellation requested"); }); this.m_inputBuffer = new BufferBlock <ImageData>(new DataflowBlockOptions { CancellationToken = this.m_cts.Token }); this.m_transform = new TransformBlock <ImageData, ImageData>(new Func <ImageData, ImageData>(this.TransformImage), new ExecutionDataflowBlockOptions { CancellationToken = this.m_cts.Token }); this.m_output = new BroadcastBlock <ImageData>(img => img.Clone()); this.m_imageLogger = new ActionBlock <ImageData>(img => { }); this.m_inputBuffer.LinkTo(this.m_transform); this.m_inputBuffer.Completion.ContinueWith(t => { this.m_logger.LogDebug("input buffer completed"); if (t.IsFaulted) { ((IDataflowBlock)this.m_transform).Fault(t.Exception); } else { this.m_transform.Complete(); } }); this.m_transform.LinkTo(this.m_output); this.m_transform.Completion.ContinueWith(t => { this.m_logger.LogDebug("transform block completed"); if (t.IsFaulted) { ((IDataflowBlock)this.m_output).Fault(t.Exception); } else { this.m_output.Complete(); } }); this.m_output.LinkTo(this.m_imageLogger); this.m_output.Completion.ContinueWith(t => { this.m_logger.LogDebug("output block completed"); if (t.IsFaulted) { ((IDataflowBlock)this.m_imageLogger).Fault(t.Exception); } else { this.m_imageLogger.Complete(); } }); try { this.m_arial = SystemFonts.Find("arial"); } catch (FontFamilyNotFountException) { int count = SystemFonts.Collection.Families.Count(); this.m_logger.LogWarning($"Arial not found. {count} fonts available, taking first"); this.m_arial = SystemFonts.Collection.Families.FirstOrDefault(); if (count > 0) { foreach (var ff in SystemFonts.Collection.Families) { this.m_logger.LogInformation($"Font : {ff.Name}"); } } } this.m_th.Start(this.m_cts.Token); return(Task.CompletedTask); }
public async Task TestFaultingAndCancellation() { foreach (bool fault in DataflowTestHelpers.BooleanValues) { var cts = new CancellationTokenSource(); var bb = new BroadcastBlock<int>(null, new GroupingDataflowBlockOptions { CancellationToken = cts.Token, BoundedCapacity = 2 }); Task<bool>[] sends = Enumerable.Range(0, 4).Select(i => bb.SendAsync(i)).ToArray(); if (fault) { Assert.Throws<ArgumentNullException>(() => ((IDataflowBlock)bb).Fault(null)); ((IDataflowBlock)bb).Fault(new InvalidCastException()); await Assert.ThrowsAsync<InvalidCastException>(() => bb.Completion); } else { cts.Cancel(); await Assert.ThrowsAnyAsync<OperationCanceledException>(() => bb.Completion); } await Task.WhenAll(sends); } }
private static async Task Main(string[] args) { var bufferBlock = new BroadcastBlock <int>(a => a); var a1 = new TransformBlock <int, int>(a => { Console.WriteLine($"Message {a} was processed by Consumer 1"); if (a % 2 == 0) { Task.Delay(300).Wait(); } else { Task.Delay(50).Wait(); } return(a * -1); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 3 } ); var a2 = new TransformBlock <int, int>(a => { Console.WriteLine($"Message {a} was processed by Consumer 2"); if (a % 2 == 0) { Task.Delay(50).Wait(); } else { Task.Delay(300).Wait(); } return(a); } , new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 3 } ); bufferBlock.LinkTo(a1); bufferBlock.LinkTo(a2); var joinBlock = new BatchedJoinBlock <int, int>(3); a1.LinkTo(joinBlock.Target1); a2.LinkTo(joinBlock.Target2); var finalBlock = new ActionBlock <Tuple <IList <int>, IList <int> > >(a => { Console.WriteLine($"Message {a.Item1},{a.Item2} was processed by all consumers. Total was [A=={string.Join(",", a.Item1)}:::::B=={string.Join(",", a.Item2)}] "); }); joinBlock.LinkTo(finalBlock); for (var i = 0; i < 10; i++) { await bufferBlock.SendAsync(i); } Console.WriteLine("Finished!"); Console.ReadKey(); }
public GuaranteedDeliveryBroadcastBlock(Func <T, T> cloningFunction) { _BroadcastBlock = new BroadcastBlock <T>(cloningFunction); _Completion = _BroadcastBlock.Completion; }
public async Task TestCloning() { // Test cloning when a clone function is provided { int data = 42; var bb = new BroadcastBlock <int>(x => - x); Assert.True(bb.Post(data)); for (int i = 0; i < 3; i++) { Assert.Equal(expected: -data, actual: await bb.ReceiveAsync()); Assert.Equal(expected: -data, actual: bb.Receive()); Assert.Equal(expected: -data, actual: await bb.ReceiveAsync()); IList <int> items; Assert.True(((IReceivableSourceBlock <int>)bb).TryReceiveAll(out items)); Assert.Equal(expected: 1, actual: items.Count); Assert.Equal(expected: -data, actual: items[0]); } int result = 0; var target = new ActionBlock <int>(i => { Assert.Equal(expected: 0, actual: result); result = i; Assert.Equal(expected: -data, actual: i); }); bb.LinkTo(target, new DataflowLinkOptions { PropagateCompletion = true }); bb.Complete(); await target.Completion; } // Test successful processing when no clone function exists { var data = new object(); var bb = new BroadcastBlock <object>(null); Assert.True(bb.Post(data)); object result; for (int i = 0; i < 3; i++) { Assert.Equal(expected: data, actual: await bb.ReceiveAsync()); Assert.Equal(expected: data, actual: bb.Receive()); Assert.Equal(expected: data, actual: await bb.ReceiveAsync()); IList <object> items; Assert.True(((IReceivableSourceBlock <object>)bb).TryReceiveAll(out items)); Assert.Equal(expected: 1, actual: items.Count); Assert.Equal(expected: data, actual: items[0]); } result = null; var target = new ActionBlock <object>(o => { Assert.Null(result); result = o; Assert.Equal(expected: data, actual: o); }); bb.LinkTo(target, new DataflowLinkOptions { PropagateCompletion = true }); bb.Complete(); await target.Completion; } }
public Multicast() { TypeInfo = new TypeInfo(typeof(TInput)); ObjectCopy = new ObjectCopy <TInput>(TypeInfo); BroadcastBlock = new BroadcastBlock <TInput>(Clone); }
public async Task SyncPathsAsync(string bucket, IEnumerable <string> paths, Func <FileInfo, bool> filePredicate, CancellationToken cancellationToken) { // BEWARE: This could cause trouble if there are // any case-sensitive paths involved. var namedPaths = (from arg in paths let split = arg.IndexOf('=') let validSplit = split > 0 && split < arg.Length - 1 select new CollectionPath(validSplit ? arg.Substring(0, split) : null, PathUtil.ForceTrailingSlash(Path.GetFullPath(validSplit ? arg.Substring(split + 1) : arg)))) .Distinct() .ToArray(); using (var blobManager = new BlobManager(new FileFingerprintManager(new MessagePackFileFingerprintStore(), new StreamFingerprinter())) as IBlobManager) { try { using (var awsManager = AwsManagerFactory.Create(bucket)) { var uniqueFingerprintBlock = new BufferBlock <Tuple <FileFingerprint, AnnotatedPath> >(); var linkBlock = new BufferBlock <Tuple <AnnotatedPath, FileFingerprint> >(); var uniqueFingerprints = new HashSet <BlobFingerprint>(); var uniqueFingerprintFilterBlock = new ActionBlock <Tuple <AnnotatedPath, FileFingerprint> >( t => { if (!uniqueFingerprints.Add(t.Item2.Fingerprint)) { return(Task.CompletedTask); } return(uniqueFingerprintBlock.SendAsync(Tuple.Create(t.Item2, t.Item1), cancellationToken)); }, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken }); var uniqueCompletionTask = uniqueFingerprintFilterBlock .Completion.ContinueWith(_ => { uniqueFingerprintBlock.Complete(); }, CancellationToken.None); TaskCollector.Default.Add(uniqueCompletionTask, "Unique filter completion"); var joinedBroadcastBlock = new BroadcastBlock <Tuple <AnnotatedPath, FileFingerprint> >(t => t, new DataflowBlockOptions { CancellationToken = cancellationToken }); joinedBroadcastBlock.LinkTo(linkBlock, DataflowLinkOptionsPropagateEnabled); joinedBroadcastBlock.LinkTo(uniqueFingerprintFilterBlock, DataflowLinkOptionsPropagateEnabled); var tasks = new List <Task>(); var loadBlobTask = blobManager.LoadAsync(namedPaths, filePredicate, joinedBroadcastBlock, cancellationToken); tasks.Add(loadBlobTask); if (S3Settings.UpdateLinks) { var updateLinksTask = _s3LinkCreator.UpdateLinksAsync(awsManager, linkBlock, cancellationToken); tasks.Add(updateLinksTask); } Task uploadBlobsTask = null; var scanBlobAsync = Task.Run(async() => { // ReSharper disable once AccessToDisposedClosure var knownObjects = await awsManager.ScanAsync(cancellationToken).ConfigureAwait(false); if (S3Settings.UploadBlobs) { // ReSharper disable once AccessToDisposedClosure uploadBlobsTask = _s3BlobUploader.UploadBlobsAsync(awsManager, uniqueFingerprintBlock, knownObjects, cancellationToken); } }, cancellationToken); tasks.Add(scanBlobAsync); await WaitAllWithWake(tasks).ConfigureAwait(false); if (null != uploadBlobsTask) { await uploadBlobsTask.ConfigureAwait(false); } } } finally { await blobManager.ShutdownAsync(cancellationToken).ConfigureAwait(false); } } }
public void DontOfferTwiceTest() { var scheduler = new TestScheduler (); var block = new BroadcastBlock<int> (null, new DataflowBlockOptions { TaskScheduler = scheduler }); var target = new TestTargetBlock<int> { Postpone = true }; block.LinkTo (target); Assert.IsFalse (target.HasPostponed); Assert.IsTrue (block.Post (1)); scheduler.ExecuteAll(); Assert.IsTrue (target.HasPostponed); target.Postpone = false; int value; Assert.IsTrue(target.RetryPostponed(out value)); Assert.AreEqual(1, value); block.LinkTo(new BufferBlock<int>()); scheduler.ExecuteAll(); Assert.AreEqual(default(int), target.DirectlyAccepted); }
public void Exec() { Console.WriteLine("start"); // Downloads the requested resource as a string. var createNumbers = new TransformManyBlock <int, int>(count => CreateNumbersFunc(count)); var printNumber = new ActionBlock <int>(number => PrintNumberFunc(number)); var printSecondFormat = new ActionBlock <int>(number => { Thread.Sleep(2000); if (number == 4 || number == 5) { throw new Exception("new error"); } Console.WriteLine($" Second format {number}"); }); var errorReportBlock = new ActionBlock <int>(number => { Console.WriteLine($" Error on: {number}"); }); //1.work in parralel //var printNumber = new ActionBlock<int>(number => PrintNumberFunc(number),new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 3 }); //2.filter mesages //createNumbers.LinkTo(printNumber,n=>n%2==0 ); //createNumbers.LinkTo(printSecondFormat,n => n % 2 != 0); //3.work in 2 blocks var broadcast = new BroadcastBlock <int>(null); createNumbers.LinkTo(broadcast); broadcast.LinkTo(printSecondFormat); broadcast.LinkTo(printNumber); createNumbers.Completion.ContinueWith(t => broadcast.Complete()); broadcast.Completion.ContinueWith(t => printNumber.Complete()); createNumbers.Completion.ContinueWith(t => printSecondFormat.Complete()); createNumbers.Completion.ContinueWith(t => { if (t.IsFaulted) { ((IDataflowBlock)printSecondFormat).Fault(t.Exception); } else { printSecondFormat.Complete(); } }); Stopwatch sw = new Stopwatch(); sw.Start(); createNumbers.Post(1); try { createNumbers.Complete(); printSecondFormat.Completion.Wait(); printNumber.Completion.Wait(); sw.Stop(); Console.WriteLine("time " + sw.Elapsed.TotalSeconds); } catch (AggregateException ex) { Console.WriteLine(ex.Flatten().Message); } Console.Read(); }
private void LoadAggregators(StatsdnetConfiguration config, IntervalService intervalService, BroadcastBlock <Bucket> messageBroadcaster, ISystemMetricsService systemMetrics) { foreach (var aggregator in config.Aggregators) { switch (aggregator.Key) { case "counters": var counter = aggregator.Value as CounterAggregationConfig; AddAggregator(MessageType.Counter, TimedCounterAggregatorBlockFactory.CreateBlock(messageBroadcaster, counter.Namespace, intervalService), systemMetrics); break; case "gauges": var gauge = aggregator.Value as GaugeAggregatorConfig; AddAggregator(MessageType.Gauge, TimedGaugeAggregatorBlockFactory.CreateBlock(messageBroadcaster, gauge.Namespace, gauge.RemoveZeroGauges, intervalService), systemMetrics); break; case "calendargrams": var calendargram = aggregator.Value as CalendargramAggregationConfig; AddAggregator(MessageType.Calendargram, TimedCalendargramAggregatorBlockFactory.CreateBlock(messageBroadcaster, calendargram.Namespace, intervalService, new TimeWindowService()), systemMetrics); break; case "timers": var timer = aggregator.Value as TimersAggregationConfig; AddAggregator(MessageType.Timing, TimedLatencyAggregatorBlockFactory.CreateBlock(messageBroadcaster, timer.Namespace, intervalService, timer.CalculateSumSquares), systemMetrics); // Add Percentiles foreach (var percentile in timer.Percentiles) { AddAggregator(MessageType.Timing, TimedLatencyPercentileAggregatorBlockFactory.CreateBlock(messageBroadcaster, timer.Namespace, intervalService, percentile.Threshold, percentile.Name), systemMetrics); } break; } } // Add the Raw (pass-through) aggregator AddAggregator(MessageType.Raw, PassThroughBlockFactory.CreateBlock(messageBroadcaster, intervalService), systemMetrics); }
public TracingTPLPipeline(ILogger <TracingTPLPipeline> logger) { _BatchBlock = new BroadcastBlock <TracingBatch>(x => x); _Logger = logger; }
public async Task TestChoose_ConsumeToAccept() { var bb = new BroadcastBlock<int>(i => i * 2); Task<int> t = DataflowBlock.Choose(bb, i => { }, bb, i => { }); Assert.False(t.IsCompleted); bb.Post(42); await t; }
public static Tuple <IPropagatorBlock <long, long>[], IPropagatorBlock <long, long>[]> Generate3CrossGraph(int cntNodes, Func <long[], long> f) { int cntRows = cntNodes; int cntCols = cntNodes; IPropagatorBlock <long, long>[,] matrix = new IPropagatorBlock <long, long> [cntRows, cntCols]; Stopwatch sw = new Stopwatch(); sw.Start(); // creation for (int row = 0; row < cntRows; row++) { if (row == 0) { for (int col = 0; col < cntCols; col++) { matrix[row, col] = new BroadcastBlock <long>(x => x); } } if (row > 0) { for (int col = 0; col < cntCols; col++) { if (col == 0 || col == cntCols - 1) { matrix[row, col] = Create3CrossNode(2, f); } else { matrix[row, col] = Create3CrossNode(3, f); } } } } // linking for (int row = 0; row < cntRows - 1; row++) { for (int col = 0; col < cntCols; col++) { if (col == 0) { matrix[row, col].LinkTo(matrix[row + 1, col]); matrix[row, col].LinkTo(matrix[row + 1, col + 1]); } if (col > 0 && col < cntCols - 1) { matrix[row, col].LinkTo(matrix[row + 1, col - 1]); matrix[row, col].LinkTo(matrix[row + 1, col]); matrix[row, col].LinkTo(matrix[row + 1, col + 1]); } if (col == cntCols - 1) { matrix[row, col].LinkTo(matrix[row + 1, col - 1]); matrix[row, col].LinkTo(matrix[row + 1, col]); } } } // first layer var firstLayer = new IPropagatorBlock <long, long> [cntCols]; for (int i = 0; i < cntCols; i++) { firstLayer[i] = matrix[0, i]; } //last layer var lastLayer = new IPropagatorBlock <long, long> [cntCols]; for (int i = 0; i < cntCols; i++) { lastLayer[i] = matrix[cntRows - 1, i]; } sw.Stop(); Console.WriteLine("Creation time: {0}", sw.Elapsed); return(new Tuple <IPropagatorBlock <long, long>[], IPropagatorBlock <long, long>[]>(firstLayer, lastLayer)); }
public void QueuedMessagesTest () { var scheduler = new TestScheduler (); var broadcast = new BroadcastBlock<int> (null, new DataflowBlockOptions { TaskScheduler = scheduler }); var target = new BufferBlock<int> (); broadcast.LinkTo (target); Assert.IsTrue (broadcast.Post (1)); Assert.IsTrue (broadcast.Post (2)); AssertEx.Throws<TimeoutException> ( () => target.Receive (TimeSpan.FromMilliseconds (100))); scheduler.ExecuteAll (); int item; Assert.IsTrue (target.TryReceive (out item)); Assert.AreEqual (1, item); Assert.IsTrue (target.TryReceive (out item)); Assert.AreEqual (2, item); }
public async Task TestPrecancellation() { var b = new BroadcastBlock<int>(null, new DataflowBlockOptions { CancellationToken = new CancellationToken(canceled: true) }); Assert.NotNull(b.LinkTo(DataflowBlock.NullTarget<int>())); Assert.False(b.Post(42)); Task<bool> t = b.SendAsync(42); Assert.True(t.IsCompleted); Assert.False(t.Result); int ignoredValue; IList<int> ignoredValues; Assert.False(b.TryReceive(out ignoredValue)); Assert.False(((IReceivableSourceBlock<int>)b).TryReceiveAll(out ignoredValues)); Assert.NotNull(b.Completion); b.Complete(); // verify doesn't throw await Assert.ThrowsAnyAsync<OperationCanceledException>(() => b.Completion); }
public void BoundedPostponedTest () { var scheduler = new TestScheduler (); var broadcast = new BroadcastBlock<int> ( null, new DataflowBlockOptions { TaskScheduler = scheduler, BoundedCapacity = 1 }); ITargetBlock<int> target = broadcast; var source = new TestSourceBlock<int> (); Assert.IsTrue (broadcast.Post (1)); var header = new DataflowMessageHeader (1); source.AddMessage (header, 2); Assert.AreEqual (DataflowMessageStatus.Postponed, target.OfferMessage (header, 2, source, false)); Assert.IsFalse (source.WasConsumed (header)); scheduler.ExecuteAll (); Assert.IsTrue (source.WasConsumed (header)); }
public async Task TestReserveReleaseConsume() { var bb = new BroadcastBlock<int>(i => i * 2); bb.Post(1); await DataflowTestHelpers.TestReserveAndRelease(bb, reservationIsTargetSpecific: false); bb = new BroadcastBlock<int>(i => i * 2); bb.Post(2); await DataflowTestHelpers.TestReserveAndConsume(bb, reservationIsTargetSpecific: false); }
public void TryReceiveAllTest() { var scheduler = new TestScheduler(); var block = new BroadcastBlock<int>(null, new DataflowBlockOptions { TaskScheduler = scheduler }); IReceivableSourceBlock<int> source = block; Assert.IsTrue(block.Post(1)); Assert.IsTrue(block.Post(2)); scheduler.ExecuteAll(); IList<int> items; Assert.IsTrue(source.TryReceiveAll(out items)); CollectionAssert.AreEqual(new[] { 2 }, items); }
private ITargetBlock <T> _targetBroadcastBlock; // reduces casting hits keeping a casted version cached public NoLossBroadcastBlock(Func <T, T> cloningFunction) { _broadcastBlock = new BroadcastBlock <T>(cloningFunction); _targetBroadcastBlock = (ITargetBlock <T>)_broadcastBlock; _completion = _broadcastBlock.Completion; }
public Task StartPipeline() { cancellationTokenSource = new CancellationTokenSource(); ExecutionDataflowBlockOptions executionDataflowBlockOptions = new ExecutionDataflowBlockOptions { CancellationToken = cancellationTokenSource.Token, //MaxDegreeOfParallelism = MAXPARA }; broadcastSymbol = new BroadcastBlock <string>(symbol => symbol); var joinblock = new JoinBlock <List <decimal>, List <Dividend>, KeyStats>(new GroupingDataflowBlockOptions { Greedy = false }); GetCompanyInfo = new TransformBlock <string, CompanyInfo>(symbol => { return(RetrieveCompanyInfo(symbol)); }, executionDataflowBlockOptions); GetDividendReports = new TransformBlock <string, List <Dividend> >(symbol => { return(RetrieveDividendInfo(symbol)); }, executionDataflowBlockOptions); GetKeyStatInfo = new TransformBlock <string, KeyStats>(symbol => { return(RetrieveKeyStats(symbol)); }, executionDataflowBlockOptions); GetIntervalReports = new TransformBlock <string, List <Interval> >(symbol => { return(RetrieveIntervals(symbol, 30)); }, executionDataflowBlockOptions); GetChangesOverInterval = new TransformBlock <List <Interval>, List <decimal> >(intervals => { return(ConstructIntervalReport(intervals)); }, executionDataflowBlockOptions); GenerateXmlString = new TransformBlock <Tuple <List <decimal>, List <Dividend>, KeyStats>, string>(tup => { var ReportObj = new Report { changeIntervals = tup.Item1, dividends = tup.Item2, keyStats = tup.Item3 }; XmlSerializer ser = new XmlSerializer(typeof(Report)); var stringWriter = new StringWriter(); ser.Serialize(stringWriter, ReportObj); return(stringWriter.ToString()); }, executionDataflowBlockOptions); GenerateCompleteReport = new ActionBlock <string>(xml => { var str = Path.GetRandomFileName().Replace(".", "") + ".xml"; File.WriteAllText(str, xml); Console.WriteLine("Finished File"); }, executionDataflowBlockOptions); var options = new DataflowLinkOptions { PropagateCompletion = true }; var buffer = new BufferBlock <string>(); buffer.LinkTo(broadcastSymbol, options); //Broadcasts the symbol broadcastSymbol.LinkTo(GetIntervalReports, options); broadcastSymbol.LinkTo(GetDividendReports, options); broadcastSymbol.LinkTo(GetKeyStatInfo, options); //Second teir parallel GetIntervalReports.LinkTo(GetChangesOverInterval, options); //Joins the parallel blocks back together GetDividendReports.LinkTo(joinblock.Target2, options); GetKeyStatInfo.LinkTo(joinblock.Target3, options); GetChangesOverInterval.LinkTo(joinblock.Target1, options); joinblock.LinkTo(GenerateXmlString, options); GenerateXmlString.LinkTo(GenerateCompleteReport, options); buffer.Post("F"); buffer.Post("AGFS"); buffer.Post("BAC"); buffer.Post("FCF"); buffer.Complete(); broadcastSymbol.Completion.ContinueWith(tsk => { if (!tsk.IsFaulted) { GetIntervalReports.Complete(); GetDividendReports.Complete(); GetKeyStatInfo.Complete(); } else { ((IDataflowBlock)GetIntervalReports).Fault(tsk.Exception); ((IDataflowBlock)GetDividendReports).Fault(tsk.Exception); ((IDataflowBlock)GetKeyStatInfo).Fault(tsk.Exception); } }); //TODO need to finish pipeline and find better implementation GenerateCompleteReport.Completion.Wait(cancellationTokenSource.Token); return(Task.CompletedTask); }
public void RunBroadcastBlockConformanceTests() { bool localPassed = true; { // Test posting then receiving localPassed = true; var bb = new BroadcastBlock <int>(i => i); for (int i = 0; i < 2; i++) { bb.Post(i); } Task.Delay(1).Wait(); localPassed |= bb.Receive() == 1; Assert.True(localPassed, string.Format("{0}: Posting then receiving", localPassed ? "Success" : "Failure")); } { // Test receiving then posting localPassed = true; var bb = new BroadcastBlock <int>(i => i); Task.Run(() => { Task.Delay(1).Wait(); bb.Post(42); }); localPassed |= bb.Receive() == 42; Assert.True(localPassed, string.Format("{0}: Receiving then posting", localPassed ? "Success" : "Failure")); } { // Test broadcasting localPassed = true; var bb = new BroadcastBlock <int>(i => i + 1); var tb1 = new TransformBlock <int, int>(i => i); var tb2 = new TransformBlock <int, int>(i => i); var tb3 = new TransformBlock <int, int>(i => i); bb.LinkTo(tb1); bb.LinkTo(tb2); bb.LinkTo(tb3); for (int i = 0; i < 2; i++) { bb.Post(i); } for (int i = 0; i < 2; i++) { localPassed |= tb1.Receive() == i + 1; localPassed |= tb2.Receive() == i + 1; localPassed |= tb3.Receive() == i + 1; } Assert.True(localPassed, string.Format("{0}: Broadcasting", localPassed ? "Success" : "Failure")); } // Test using a precanceled token { localPassed = true; try { var cts = new CancellationTokenSource(); cts.Cancel(); var dbo = new DataflowBlockOptions { CancellationToken = cts.Token }; var bb = new BroadcastBlock <int>(i => i, dbo); int ignoredValue; IList <int> ignoredValues; localPassed &= bb.LinkTo(new ActionBlock <int>(delegate { })) != null; localPassed &= bb.SendAsync(42).Result == false; localPassed &= ((IReceivableSourceBlock <int>)bb).TryReceiveAll(out ignoredValues) == false; localPassed &= bb.Post(42) == false; localPassed &= bb.TryReceive(out ignoredValue) == false; localPassed &= bb.Completion != null; bb.Complete(); } catch (Exception) { localPassed = false; } Assert.True(localPassed, string.Format("{0}: Precanceled tokens work correctly", localPassed ? "Success" : "Failure")); } }
public void PostponedTest () { var broadcast = new BroadcastBlock<int> (null); var target = new BufferBlock<int> ( new DataflowBlockOptions { BoundedCapacity = 1 }); broadcast.LinkTo (target); Assert.IsTrue (target.Post (1)); Assert.IsTrue (broadcast.Post (2)); Assert.AreEqual (1, target.Receive (TimeSpan.FromMilliseconds (0))); Assert.AreEqual (2, target.Receive (TimeSpan.FromMilliseconds (100))); }
static void Main(string[] args) { var broadcastBlock = new BroadcastBlock <int>(a => a, new DataflowBlockOptions() { BoundedCapacity = 1 } ); var a1 = new TransformBlock <int, int>( a => { Console.WriteLine($"Message {a} was processed by consumer 1"); if (a % 2 == 0) { Task.Delay(300).Wait(); } else { Task.Delay(50).Wait(); } return(a * -1); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 3 }); var a2 = new TransformBlock <int, int>( a => { Console.WriteLine($"Message {a} was processed by consumer 2"); if (a % 2 != 0) { Task.Delay(300).Wait(); } else { Task.Delay(50).Wait(); } return(a); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = 3 }); broadcastBlock.LinkTo(a1); broadcastBlock.LinkTo(a2); var joinblock = new JoinBlock <int, int>(); a1.LinkTo(joinblock.Target1); a2.LinkTo(joinblock.Target2); var printBlock = new ActionBlock <Tuple <int, int> >( a => Console.WriteLine($"Message {a} was processed. Sum: {a.Item1 + a.Item2}") ); joinblock.LinkTo(printBlock); for (int i = 0; i < 10; i++) { var i1 = i; broadcastBlock.SendAsync(i) .ContinueWith(a => { Console.WriteLine(a.Result ? $"Messgae {i1} was accepted" : $"Messgae {i1} was rejected"); }); } Console.WriteLine("done"); Console.ReadLine(); }
public override void ReadFrames(BroadcastBlock <FrameData> broadcast) { _broadcast = broadcast; }
public DiagnosticPipeline( IHealthReporter healthReporter, IReadOnlyCollection <IObservable <EventData> > inputs, IReadOnlyCollection <IFilter> globalFilters, IReadOnlyCollection <EventSink> sinks, DiagnosticPipelineConfiguration pipelineConfiguration = null, bool disposeDependencies = false, TaskScheduler taskScheduler = null) { Requires.NotNull(healthReporter, nameof(healthReporter)); Requires.NotNull(inputs, nameof(inputs)); Requires.Argument(inputs.Count > 0, nameof(inputs), "There must be at least one input"); Requires.NotNull(sinks, nameof(sinks)); Requires.Argument(sinks.Count > 0, nameof(sinks), "There must be at least one sink"); this.batcherTimerDisposalLock = new object(); this.pipelineConfiguration = pipelineConfiguration ?? new DiagnosticPipelineConfiguration(); taskScheduler = taskScheduler ?? TaskScheduler.Current; // An estimatie how many batches of events to allow inside the pipeline. // We want to be able to process full buffer of events, but also have enough batches in play in case of high concurrency. int MaxNumberOfBatchesInProgress = Math.Max( 5 * this.pipelineConfiguration.MaxConcurrency, this.pipelineConfiguration.PipelineBufferSize / this.pipelineConfiguration.MaxEventBatchSize); this.Inputs = inputs; this.Sinks = sinks; // Just play nice and make sure there is always something to enumerate on this.GlobalFilters = globalFilters ?? new IFilter[0]; this.HealthReporter = healthReporter; this.cancellationTokenSource = new CancellationTokenSource(); var propagateCompletion = new DataflowLinkOptions() { PropagateCompletion = true }; // One disposable for each input subscription. this.inputSubscriptions = new List <IDisposable>(inputs.Count); var inputBuffer = new BufferBlock <EventData>( new DataflowBlockOptions() { BoundedCapacity = this.pipelineConfiguration.PipelineBufferSize, CancellationToken = this.cancellationTokenSource.Token, TaskScheduler = taskScheduler }); this.pipelineHead = inputBuffer; var batcher = new BatchBlock <EventData>( this.pipelineConfiguration.MaxEventBatchSize, new GroupingDataflowBlockOptions() { BoundedCapacity = this.pipelineConfiguration.PipelineBufferSize, CancellationToken = this.cancellationTokenSource.Token, TaskScheduler = taskScheduler } ); inputBuffer.LinkTo(batcher, propagateCompletion); ISourceBlock <EventData[]> sinkSource; FilterAction filterTransform; if (this.GlobalFilters.Count > 0) { filterTransform = new FilterAction( this.GlobalFilters, this.cancellationTokenSource.Token, MaxNumberOfBatchesInProgress, this.pipelineConfiguration.MaxConcurrency, healthReporter, taskScheduler); var globalFiltersBlock = filterTransform.GetFilterBlock(); batcher.LinkTo(globalFiltersBlock, propagateCompletion); sinkSource = globalFiltersBlock; } else { sinkSource = batcher; } bool usingBroadcastBlock = sinks.Count > 1; if (usingBroadcastBlock) { var broadcaster = new BroadcastBlock <EventData[]>( (events) => events?.Select((e) => e.DeepClone()).ToArray(), new DataflowBlockOptions() { BoundedCapacity = MaxNumberOfBatchesInProgress, CancellationToken = this.cancellationTokenSource.Token, TaskScheduler = taskScheduler }); sinkSource.LinkTo(broadcaster, propagateCompletion); sinkSource = broadcaster; } this.outputCompletionTasks = new List <Task>(sinks.Count); foreach (var sink in sinks) { ISourceBlock <EventData[]> outputSource = sinkSource; if (sink.Filters != null && sink.Filters.Count > 0) { filterTransform = new FilterAction( sink.Filters, this.cancellationTokenSource.Token, MaxNumberOfBatchesInProgress, this.pipelineConfiguration.MaxConcurrency, healthReporter, taskScheduler); var filterBlock = filterTransform.GetFilterBlock(); if (usingBroadcastBlock) { var lossReportingPropagator = new LossReportingPropagatorBlock <EventData[]>(this.HealthReporter); sinkSource.LinkTo(lossReportingPropagator, propagateCompletion); lossReportingPropagator.LinkTo(filterBlock, propagateCompletion); } else { sinkSource.LinkTo(filterBlock, propagateCompletion); } outputSource = filterBlock; } else if (usingBroadcastBlock) { var lossReportingPropagator = new LossReportingPropagatorBlock <EventData[]>(this.HealthReporter); sinkSource.LinkTo(lossReportingPropagator, propagateCompletion); outputSource = lossReportingPropagator; } OutputAction outputAction = new OutputAction( sink.Output, this.cancellationTokenSource.Token, MaxNumberOfBatchesInProgress, this.pipelineConfiguration.MaxConcurrency, healthReporter, taskScheduler); var outputBlock = outputAction.GetOutputBlock(); outputSource.LinkTo(outputBlock, propagateCompletion); this.outputCompletionTasks.Add(outputBlock.Completion); } IObserver <EventData> inputBufferObserver = new TargetBlockObserver <EventData>(inputBuffer, this.HealthReporter); foreach (var input in inputs) { this.inputSubscriptions.Add(input.Subscribe(inputBufferObserver)); } this.disposed = false; this.disposeDependencies = disposeDependencies; this.batcherTimer = new Timer( (_) => { try { lock (this.batcherTimerDisposalLock) { if (!this.disposed) { batcher.TriggerBatch(); this.batcherTimer.Change(dueTime: TimeSpan.FromMilliseconds(this.pipelineConfiguration.MaxBatchDelayMsec), period: Timeout.InfiniteTimeSpan); } } } catch { } }, state: null, dueTime: TimeSpan.FromMilliseconds(this.pipelineConfiguration.MaxBatchDelayMsec), period: Timeout.InfiniteTimeSpan); }