/// <summary> /// Initializes a new instance of the <see cref="LogTargetBase"/> class. /// </summary> /// <param name="maxDegreeOfParallelism"> /// The max degree of parallelism the target instance will log entries (ie. <see cref="Log"/> method). /// </param> protected LogTargetBase(Int32 maxDegreeOfParallelism) { _ActionBlock = new ActionBlock<LogEntry>( logEntry => Log(logEntry), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }); }
public override void FeedTarget(ITargetBlock<GraphiteLine> target) { foreach (var gauge in Gauges) { target.Post(new GraphiteLine(RootNamespace + gauge.Key, gauge.Value, Epoch)); } }
public void LinkTo(ITargetBlock<string> target, CancellationToken cancellationToken) { _cancellationToken = cancellationToken; _targetBlock = target; Task.Factory.StartNew(() => { try { var endpoint = new IPEndPoint(IPAddress.Any, _port); var udpClient = new UdpClient( endpoint ); udpClient.Client.ReceiveBufferSize = MAX_BUFFER_SIZE; //32k buffer while (true) { if (_cancellationToken.IsCancellationRequested) { return; } byte[] data = udpClient.Receive(ref endpoint); _preprocessorBlock.Post( data ); } } catch (ObjectDisposedException) { /* Eat it, socket was closed */ } finally { IsListening = false; } }, cancellationToken); IsListening = true; }
public override void FeedTarget(ITargetBlock<GraphiteLine> target) { foreach (var line in _rawLines) { target.Post(new GraphiteLine(line.Name, line.Value, line.Timestamp ?? Epoch)); } }
static Task PostAllFilePathsAsync(IEnumerable<CollectionPath> paths, Func<FileInfo, bool> filePredicate, ITargetBlock<AnnotatedPath> filePathTargetBlock, CancellationToken cancellationToken) { var scanTasks = paths .Select<CollectionPath, Task>(path => Task.Factory.StartNew( async () => { foreach (var file in PathUtil.ScanDirectory(path.Path, filePredicate)) { if (cancellationToken.IsCancellationRequested) break; var relativePath = PathUtil.MakeRelativePath(path.Path, file.FullName); var annotatedPath = new AnnotatedPath { FileInfo = file, Collection = path.Collection ?? path.Path, RelativePath = relativePath }; await filePathTargetBlock.SendAsync(annotatedPath, cancellationToken).ConfigureAwait(false); } }, cancellationToken, TaskCreationOptions.DenyChildAttach | TaskCreationOptions.LongRunning | TaskCreationOptions.RunContinuationsAsynchronously, TaskScheduler.Default)); var task = Task.WhenAll(scanTasks); var completeTask = task.ContinueWith(_ => filePathTargetBlock.Complete()); TaskCollector.Default.Add(completeTask, "PostAllFilePathsAsync"); return task; }
public async void LinkTo(ITargetBlock<string> target, CancellationToken token) { _target = target; _token = token; _tcpListener.Start(); IsListening = true; await Listen(); }
public Business(Guid id, int size, ITargetBlock<ConsumptionData> underservedBlock) { this.ID = id; this.ElectricityConsumer = new ResourceConsumer<Electricity>(id, size, ElementType.Electricity, underservedBlock); this.WaterConsumer = new ResourceConsumer<Water>(id, size, ElementType.Water, underservedBlock); this.PeopleConsumer = new ResourceConsumer<People>(id, size, ElementType.People, underservedBlock); this.TrashProducer = new ResourceProducer<Trash>(id, size); this.SewageProducer = new ResourceProducer<Sewage>(id, size); }
public Home(Guid id, int residents, ITargetBlock<ConsumptionData> underservedBlock) { this.ID = id; this.ElectricityConsumer = new ResourceConsumer<Electricity>(id, residents, ElementType.Electricity, underservedBlock); this.WaterConsumer = new ResourceConsumer<Water>(id, residents, ElementType.Water, underservedBlock); this.PeopleProducer = new ResourceProducer<People>(id, residents); this.TrashProducer = new ResourceProducer<Trash>(id, residents); this.SewageProducer = new ResourceProducer<Sewage>(id, residents); }
public async Task LoadAsync(CollectionPath[] paths, Func<FileInfo, bool> filePredicate, ITargetBlock<Tuple<AnnotatedPath, IFileFingerprint>> joinedTargetBlock, CancellationToken cancellationToken) { await _fileFingerprintManager.LoadAsync(cancellationToken).ConfigureAwait(false); await GenerateFileFingerprintsAsync(paths, filePredicate, joinedTargetBlock, cancellationToken) .ConfigureAwait(false); }
/// <summary> /// Initializes a new instance of the <see cref="AprsPacketSpout" /> class. /// </summary> /// <param name="pipeline">The pipeline.</param> public AprsPacketSpout(IPipeline<byte[]> pipeline) { _user = ConfigurationManager.AppSettings["user"]; var password = ConfigurationManager.AppSettings["password"] ?? "-1"; _hostname = ConfigurationManager.AppSettings["hostname"]; _port = int.Parse(ConfigurationManager.AppSettings["port"]); _logon = Encoding.UTF8.GetBytes("user " + _user + " pass " + password + "\n"); _pipeline = pipeline.Create(string.Empty); }
public override void FeedTarget(ITargetBlock<GraphiteLine> target) { foreach (var set in Sets) { foreach (var item in set.Value) { target.Post(new GraphiteLine(RootNamespace + set.Key, 1, Epoch)); } } }
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target, string rootNamespace, bool removeZeroGauges, IIntervalService intervalService, ILog log) { var gauges = new ConcurrentDictionary<string, double>(); var root = rootNamespace; var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + "."; var incoming = new ActionBlock<StatsdMessage>(p => { var gauge = p as Gauge; gauges.AddOrUpdate(gauge.Name, gauge.Value, (key, oldValue) => gauge.Value); }, Utility.UnboundedExecution()); intervalService.Elapsed += (sender, e) => { if (gauges.Count == 0) { return; } var items = gauges.ToArray(); var bucket = new GaugesBucket(items, e.Epoch, ns); if (removeZeroGauges) { // Get all zero-value gauges double placeholder; var zeroGauges = 0; for (int index = 0; index < items.Length; index++) { if (items[index].Value == 0) { gauges.TryRemove(items[index].Key, out placeholder); zeroGauges += 1; } } if (zeroGauges > 0) { log.InfoFormat("Removed {0} empty gauges.", zeroGauges); } } gauges.Clear(); target.Post(bucket); }; incoming.Completion.ContinueWith(p => { // Tell the upstream block that we're done target.Complete(); }); return incoming; }
public override void FeedTarget(ITargetBlock<GraphiteLine> target) { foreach (var latency in Latencies) { var lines = MakeGraphiteLines(latency); foreach (var line in lines) { target.Post(line); } } }
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target, string rootNamespace, IIntervalService intervalService, int percentile, string percentileName, ILog log, int maxItemsPerBucket = 1000) { var latencies = new ConcurrentDictionary<string, DatapointBox>(); var root = rootNamespace; var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + "."; var random = new Random(); percentileName = "." + ( percentileName ?? ( "p" + percentile ) ); var incoming = new ActionBlock<StatsdMessage>( p => { var latency = p as Timing; latencies.AddOrUpdate(latency.Name, (key) => { return new DatapointBox(maxItemsPerBucket, latency.ValueMS); }, (key, bag) => { bag.Add(latency.ValueMS); return bag; }); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded }); intervalService.Elapsed += (sender, e) => { if (latencies.Count == 0) { return; } var bucket = new PercentileBucket(latencies.ToArray(), e.Epoch, ns, percentileName, percentile); latencies.Clear(); target.Post(bucket); }; incoming.Completion.ContinueWith(p => { // Tell the upstream block that we're done target.Complete(); }); return incoming; }
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<SetsBucket> target, string rootNamespace, IIntervalService intervalService, ILog log) { var sets = new ConcurrentDictionary<string, ConcurrentDictionary<int, bool>>(); var root = rootNamespace; var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + "."; var incoming = new ActionBlock<StatsdMessage>(p => { var set = p as Set; sets.AddOrUpdate(set.Name, (key) => { var setDict = new ConcurrentDictionary<int, bool>(); setDict.AddOrUpdate( set.Value, ( key2 ) => true, ( key2, oldValue ) => true ); return setDict; }, (key, setDict) => { setDict.AddOrUpdate( set.Value, ( key2 ) => true, ( key2, oldValue ) => true ); return setDict; }); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded }); intervalService.Elapsed += (sender, e) => { if (sets.Count == 0) { return; } var rawData = sets.ToArray(); sets.Clear(); var bucket = new SetsBucket( rawData.Select(p => new KeyValuePair<string, List<KeyValuePair<int, bool>>>(p.Key, p.Value.ToList()) ).ToList(), e.Epoch, ns); target.Post(bucket); }; incoming.Completion.ContinueWith(p => { // Tell the upstream block that we're done target.Complete(); }); return incoming; }
static void skynetTplRecursion( ITargetBlock<long> src, long num, long size, long div ) { if( size == 1 ) { src.SendAsync( num ); return; } for( var i = 0; i < div; i++ ) { var sub_num = num + i * ( size / div ); skynetTplRecursion( src, sub_num, size / div, div ); } }
public override void FeedTarget(ITargetBlock<GraphiteLine> target) { int percentileValue; foreach (var measurements in Timings) { if (TryComputePercentile(measurements, out percentileValue)) { target.Post( new GraphiteLine( RootNamespace + measurements.Key + PercentileName, percentileValue, Epoch)); } } }
private void DownloadPage(ITargetBlock<Parse> targetAscTransform) { var totalPages = 0; foreach (var parse in _dataAccess.GetEmailQueries().FindAll(x => x.Email == true).Select(q => _dataAccess.GetHtmlToScrape(GenSqlStatementForHtmlParsing(q.Id))).SelectMany(cars => cars.Select(car => new Parse { CarId = car.Id, Link = car.Link, UserAgent = _dataAccess.GetRandomUserAgent() }))) { targetAscTransform.Post(parse); totalPages++; } Logger.Debug("Total for this run is {0}", totalPages); }
/// <summary> /// Initializes a new instance of the <see cref="BatchLogTargetBase"/> class. /// </summary> /// <param name="batchSize">Size of the log batch.</param> /// <param name="flushInterval">The flush interval. /// The interval with which to initiate a batching operation even if the /// number of currently queued logs is less than the <paramref name="batchSize"/>. /// </param> /// <param name="maxDegreeOfParallelism">The max degree of parallelism the target instance will log batch entries (ie. <see cref="Log"/> method).</param> protected BatchLogTargetBase(Int32 batchSize, TimeSpan flushInterval, Int32 maxDegreeOfParallelism) { _FlushInterval = flushInterval; _BatchBlock = new BatchBlock<LogEntry>(batchSize); _ActionBlock = new ActionBlock<IEnumerable<LogEntry>>( logEntries => { Log(logEntries); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }); _BatchBlock.LinkTo(_ActionBlock); _FlushTimer = new Timer(FlushTimerCallback, null, _FlushInterval, TimeSpan.FromMilliseconds(-1)); }
private void DownloadPage(ITargetBlock<Parse> targetAscTransform) { foreach (var q in _dataAccess.GetEmailQueries().FindAll(x => x.Email == true)) { var cars = _dataAccess.GetHtmlToScrape(GenSqlStatementForHtmlParsing(q.Id)); var emailss = cars.Where(e => e.HtmlDownloaded == false); foreach (var car in cars) { var parse = new Parse { CarId = car.Id, Link = car.Link, UserAgent = _dataAccess.GetRandomUserAgent() }; targetAscTransform.Post(parse); } } }
public LinkFingerprintJoiner(ITargetBlock<Tuple<AnnotatedPath, IFileFingerprint>> targetBlock) { if (null == targetBlock) throw new ArgumentNullException(nameof(targetBlock)); _targetBlock = targetBlock; AnnotatedPathsBlock = new ActionBlock<AnnotatedPath[]>( aps => Task.WhenAll(aps.Select(SetAnnotatedPathAsync)), new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded }); FileFingerprintBlock = new ActionBlock<IFileFingerprint>(SetFileFingerprintAsync, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded }); var task = Task.WhenAll(AnnotatedPathsBlock.Completion, FileFingerprintBlock.Completion) .ContinueWith(_ => { targetBlock.Complete(); }); TaskCollector.Default.Add(task, "Joiner completion"); }
async Task GenerateFileFingerprintsAsync(CollectionPath[] paths, Func<FileInfo, bool> filePredicate, ITargetBlock<Tuple<AnnotatedPath, IFileFingerprint>> joinedTargetBlock, CancellationToken cancellationToken) { try { var annotatedPathBroadcastBlock = new BroadcastBlock<AnnotatedPath[]>(aps => aps, new DataflowBlockOptions { CancellationToken = cancellationToken }); var joiner = new LinkFingerprintJoiner(joinedTargetBlock); annotatedPathBroadcastBlock.LinkTo(joiner.AnnotatedPathsBlock, new DataflowLinkOptions { PropagateCompletion = true }); var fileFingerprintBroadcastBlock = new BroadcastBlock<IFileFingerprint>(ff => ff, new DataflowBlockOptions { CancellationToken = cancellationToken }); fileFingerprintBroadcastBlock.LinkTo(joiner.FileFingerprintBlock, new DataflowLinkOptions { PropagateCompletion = true }); var fingerprintGeneratorTask = _fileFingerprintManager .GenerateFileFingerprintsAsync(annotatedPathBroadcastBlock, fileFingerprintBroadcastBlock, cancellationToken); try { await DirectoryScanner.GenerateAnnotatedPathsAsync(paths, filePredicate, annotatedPathBroadcastBlock, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { } catch (Exception ex) { Console.Write("Path scan failed: " + ex.Message); } await fingerprintGeneratorTask.ConfigureAwait(false); } catch (Exception ex) { Console.WriteLine("GenerateFileFingerprintsAsync() failed: " + ex.Message); } finally { Debug.WriteLine("BlobManager.GenerateFileFingerprintsAsync() is done"); } }
public NeverendingTask(Func<DateTimeOffset, CancellationToken, Task> action, CancellationToken cancellationToken) { if (action == null) { throw new ArgumentNullException("action"); } _inner = new ActionBlock<DateTimeOffset>(async now => { await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken) .ConfigureAwait(false); await action(now, cancellationToken); _inner.Post(DateTimeOffset.UtcNow); }, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken }); }
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target, IIntervalService intervalService) { var rawLines = new ConcurrentStack<Raw>(); var incoming = new ActionBlock<StatsdMessage>(p => { rawLines.Push(p as Raw); }, Utility.UnboundedExecution()); intervalService.Elapsed += (sender, e) => { if (rawLines.Count == 0) { return; } var lines = rawLines.ToArray(); rawLines.Clear(); var bucket = new RawBucket(lines, e.Epoch); target.Post(bucket); }; return incoming; }
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target, string rootNamespace, IIntervalService intervalService, ILog log) { var counters = new ConcurrentDictionary<string, double>(); var root = rootNamespace; var ns = String.IsNullOrEmpty(rootNamespace) ? "" : (rootNamespace + "."); var incoming = new ActionBlock<StatsdMessage>(p => { var counter = p as Counter; counters.AddOrUpdate(counter.Name, counter.Value, (key, oldValue) => oldValue + counter.Value); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded }); intervalService.Elapsed += (sender, e) => { if (counters.Count == 0) { return; } var bucket = new CounterBucket(counters.ToArray(), e.Epoch, ns); counters.Clear(); target.Post(bucket); }; incoming.Completion.ContinueWith(p => { log.Info("TimedCounterAggregatorBlock completing."); // Tell the upstream block that we're done target.Complete(); }); return incoming; }
public static Task GenerateAnnotatedPathsAsync(IEnumerable<CollectionPath> paths, Func<FileInfo, bool> filePredicate, ITargetBlock<AnnotatedPath[]> filePathTargetBlock, CancellationToken cancellationToken) { var shuffleBlock = new TransformBlock<AnnotatedPath[], AnnotatedPath[]>( filenames => { // Sequential names tend to fall into the same AWS S3 partition, so we // shuffle things around. RandomUtil.Shuffle(filenames); return filenames; }, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = Environment.ProcessorCount }); shuffleBlock.LinkTo(filePathTargetBlock, new DataflowLinkOptions { PropagateCompletion = true }); var batcher = new BatchBlock<AnnotatedPath>(2048, new GroupingDataflowBlockOptions { CancellationToken = cancellationToken }); batcher.LinkTo(shuffleBlock, new DataflowLinkOptions { PropagateCompletion = true }); return PostAllFilePathsAsync(paths, filePredicate, batcher, cancellationToken); }
public void LinkTo(ITargetBlock<string> target, CancellationToken token) { _target = target; IsListening = true; _scheduler = KayakScheduler.Factory.Create(new SchedulerDelegate()); var wsTask = Task.Factory.StartNew(() => { var server = KayakServer.Factory.CreateHttp( new RequestDelegate(this), _scheduler); using (server.Listen(new IPEndPoint(IPAddress.Any, _port))) { _scheduler.Start(); } IsListening = false; }); Task.Factory.StartNew(() => { token.WaitHandle.WaitOne(); _scheduler.Stop(); }); }
public PipelineMergeLinker(ISourceBlock <IPipelineJobElement <T1> > from1, ISourceBlock <IPipelineJobElement <T2> > from2, ITargetBlock <IPipelineJobElement <Tuple <T1, T2> > > to, Predicate <IPipelineJobElement <T1> > predicate1, Predicate <IPipelineJobElement <T2> > predicate2) { if (predicate1 == null) { predicate1 = e => true; } if (predicate2 == null) { predicate2 = e => true; } Input = new JoinBlock <IPipelineJobElement <T1>, IPipelineJobElement <T2> >(); Output = new ActionBlock <Tuple <IPipelineJobElement <T1>, IPipelineJobElement <T2> > >( elements => { var job = elements.Item1.Job; var mergedElement = job.MergeToSingleElement(elements); to.Post(mergedElement); }); from1.LinkTo(Input.Target1, predicate1); from2.LinkTo(Input.Target2, predicate2); Input.LinkTo(Output); }
public UploadPipeline(ISession session, ILogger log, IFileSystem fs) { var tree = new FolderTree(session); var fileNameStep = new TransformBlock <AssetModel, (AssetModel, FilePath)>(async asset => { FilePath path; if (FilePathProvider != null) { path = FilePathProvider(asset); } else if (FilePathProviderAsync != null) { path = await FilePathProviderAsync(asset); } else { path = new FilePath(asset.Id); } return(asset, path); }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1, MaxMessagesPerTask = 1, BoundedCapacity = 1 }); var maxDegreeOfParallelism = fs.CanAccessInParallel ? Environment.ProcessorCount * 2 : 1; var uploadStep = new ActionBlock <(AssetModel, FilePath)>(async item => { var(asset, path) = item; var process = $"Uploading {path}"; try { var assetFile = fs.GetFile(path); await using (var stream = assetFile.OpenRead()) { var file = new FileParameter(stream, asset.FileName, asset.MimeType); var result = await session.Assets.PostUpsertAssetAsync(session.App, asset.Id, null, true, file); log.ProcessCompleted(process); } } catch (Exception ex) { log.ProcessFailed(process, ex); } }, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, MaxMessagesPerTask = 1, BoundedCapacity = maxDegreeOfParallelism * 2 }); fileNameStep.LinkTo(uploadStep, new DataflowLinkOptions { PropagateCompletion = true }); pipelineStart = fileNameStep; pipelineEnd = uploadStep; }
public Tuple <T1, T2, T3> ConsumeMessage(DataflowMessageHeader messageHeader, ITargetBlock <Tuple <T1, T2, T3> > target, out bool messageConsumed) { return(vault.ConsumeMessage(messageHeader, target, out messageConsumed)); }
public async void LinkTo(ITargetBlock<string> target, CancellationToken token) { _target = target; _token = token; await Listen(); }
/// <include file='XmlDocs/CommonXmlDocComments.xml' path='CommonXmlDocComments/Sources/Member[@name="ReleaseReservation"]/*' /> void ISourceBlock <T> .ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock <T> target) { // Validate arguments if (!messageHeader.IsValid) { throw new ArgumentException(SR.Argument_InvalidMessageHeader, nameof(messageHeader)); } if (target == null) { throw new ArgumentNullException(nameof(target)); } Contract.EndContractBlock(); // As long as the message is the one we have, everything's fine. if (_header.Id != messageHeader.Id) { throw new InvalidOperationException(SR.InvalidOperation_MessageNotReservedByTarget); } // In other blocks, upon release we typically re-offer the message to all linked targets. // We need to do the same thing for WriteOnceBlock, in order to account for cases where the block // may be linked to a join or similar block, such that the join could never again be satisfied // if it didn't receive another offer from this source. However, since the message is broadcast // and all targets can get a copy, we don't need to broadcast to all targets, only to // the target that released the message. Note that we don't care whether it's accepted // or not, nor do we care about any exceptions which may emerge (they should just propagate). Debug.Assert(_header.IsValid, "A valid header is required."); bool useCloning = _cloningFunction != null; target.OfferMessage(_header, _value, this, consumeToAccept: useCloning); }
//public string ThreadName; private void AscBufferProducer(ITargetBlock<Parse> targetAscTransform) // This is using TPL DataBlock producer consumer pattern. { DownloadPage(targetAscTransform); }
/// <include file='XmlDocs/CommonXmlDocComments.xml' path='CommonXmlDocComments/Sources/Member[@name="LinkTo"]/*' /> IDisposable ISourceBlock <T> .LinkTo(ITargetBlock <T> target, DataflowLinkOptions linkOptions) { throw new NotSupportedException(SR.NotSupported_MemberNotNeeded); }
public void ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock <TOutput> target) { _block.ReleaseReservation(messageHeader, target); }
public bool ReserveMessage(DataflowMessageHeader messageHeader, ITargetBlock <TOutput> target) { return(_block.ReserveMessage(messageHeader, target)); }
public TOutput ConsumeMessage(DataflowMessageHeader messageHeader, ITargetBlock <TOutput> target, out bool messageConsumed) { return(_block.ConsumeMessage(messageHeader, target, out messageConsumed)); }
public IDisposable LinkTo(ITargetBlock <TOutput> target, DataflowLinkOptions linkOptions) { return(_block.LinkTo(target, linkOptions)); }
/// <inheritdoc /> public bool ReserveMessage(DataflowMessageHeader messageHeader, ITargetBlock <T[]> target) { return(((IPropagatorBlock <T, T[]>)_batchBlock).ReserveMessage(messageHeader, target)); }
/// <inheritdoc /> public void ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock <T[]> target) { ((IPropagatorBlock <T, T[]>)_batchBlock).ReleaseReservation(messageHeader, target); }
/// <include file='XmlDocs/CommonXmlDocComments.xml' path='CommonXmlDocComments/Sources/Member[@name="ConsumeMessage"]/*' /> T? ISourceBlock <T> .ConsumeMessage(DataflowMessageHeader messageHeader, ITargetBlock <T> target, out bool messageConsumed) { return(_source.ConsumeMessage(messageHeader, target, out messageConsumed)); }
/// <include file='XmlDocs/CommonXmlDocComments.xml' path='CommonXmlDocComments/Sources/Member[@name="LinkTo"]/*' /> public IDisposable LinkTo(ITargetBlock <T> target, DataflowLinkOptions linkOptions) { return(_source.LinkTo(target, linkOptions)); }
public void SetTarget(ITargetBlock <StatsdMessage> target) { _target = target; }
void ISourceBlock <T> .ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock <T> target) { if (_messagesByBlock.TryGetValue(messageHeader, out var block)) { block.ReleaseReservation(messageHeader, target); } }
/// <summary>Gets whether the registry contains a particular target.</summary> /// <param name="target">The target.</param> /// <returns>true if the registry contains the target; otherwise, false.</returns> internal bool Contains(ITargetBlock <T> target) { return(_targetInformation.ContainsKey(target)); }
/// <summary> /// This method is where we tell data flow which blocks we're interested in receiving updates for /// </summary> protected override IDisposable LinkExternalInput(ITargetBlock <IProjectVersionedValue <DesignTimeInputSnapshot> > targetBlock) { JoinUpstreamDataSources(_designTimeInputsChangeTracker); return(_designTimeInputsChangeTracker.SourceBlock.LinkTo(targetBlock, DataflowOption.PropagateCompletion)); }
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <GraphiteLine> target, string rootNamespace, IIntervalService intervalService, int percentile, string percentileName, ILog log, int maxItemsPerBucket = 1000) { var latencies = new ConcurrentDictionary <string, DatapointBox>(); var root = rootNamespace; var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + "."; var random = new Random(); percentileName = "." + (percentileName ?? ("p" + percentile)); var incoming = new ActionBlock <StatsdMessage>(p => { var latency = p as Timing; latencies.AddOrUpdate(latency.Name, (key) => { return(new DatapointBox(maxItemsPerBucket, latency.ValueMS)); }, (key, bag) => { bag.Add(latency.ValueMS); return(bag); }); }, new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded }); intervalService.Elapsed += (sender, e) => { if (latencies.Count == 0) { return; } var buckets = latencies.ToArray(); latencies.Clear(); int percentileValue; int numLinesPosted = 0; foreach (var measurements in buckets) { if (Percentile.TryCompute(measurements.Value.ToArray().ToList(), percentile, out percentileValue)) { target.Post(new GraphiteLine(ns + measurements.Key + percentileName, percentileValue, e.Epoch)); numLinesPosted++; } } log.InfoFormat("TimedLatencyPercentileAggregatorBlock - Posted {0} buckets and {1} lines.", buckets.Length, numLinesPosted); }; incoming.Completion.ContinueWith(p => { // Tell the upstream block that we're done target.Complete(); }); return(incoming); }
public static void PostAssert <T>(ITargetBlock <T> target, T item) { bool retval = target.Post(item); Contracts.Assert(retval); }
public IDisposable LinkTo(ITargetBlock <TOutput> target, DataflowLinkOptions linkOptions) { return(outgoing.AddTarget(target, linkOptions)); }
/// <inheritdoc /> public T[] ConsumeMessage(DataflowMessageHeader messageHeader, ITargetBlock <T[]> target, out bool messageConsumed) { return(((IPropagatorBlock <T, T[]>)_batchBlock).ConsumeMessage(messageHeader, target, out messageConsumed)); }
public void ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock <Tuple <T1, T2, T3> > target) { vault.ReleaseReservation(messageHeader, target); }
public MergeJoinTarget(ITargetBlock <TInput> joinTarget) { TargetBlock = joinTarget; }
public bool ReserveMessage(DataflowMessageHeader messageHeader, ITargetBlock <Tuple <T1, T2, T3> > target) { return(vault.ReserveMessage(messageHeader, target)); }
public void SetTarget(ITargetBlock <Bucket> target) { _target = target; }
private async Task ProduceObjects(IAsyncEnumerable <IUser> users, ITargetBlock <IUser> target) { await users.ForEachAsync(t => target.Post(t)); target.Complete(); }
public static IDisposable LinkWithPropagationTo <T>(this ISourceBlock <T> source, ITargetBlock <T> target) { return(source.LinkTo(target, new DataflowLinkOptions { PropagateCompletion = true })); }
/// <include file='XmlDocs/CommonXmlDocComments.xml' path='CommonXmlDocComments/Sources/Member[@name="ReserveMessage"]/*' /> bool ISourceBlock <T> .ReserveMessage(DataflowMessageHeader messageHeader, ITargetBlock <T> target) { return(_source.ReserveMessage(messageHeader, target)); }
/// <summary> /// Creates a new instance of ChannelBase /// </summary> /// <param name="transport">The transport.</param> /// <param name="sendTimeout">The channel send timeout.</param> /// <param name="consumeTimeout">The channel consume timeout. Each envelope received from the transport must be consumed in the specified timeout or it will cause the channel to be closed.</param> /// <param name="closeTimeout">The channel close timeout.</param> /// <param name="envelopeBufferSize">Size of the envelope buffer.</param> /// <param name="fillEnvelopeRecipients">Indicates if the from and to properties of sent and received envelopes should be filled with the session information if not defined.</param> /// <param name="autoReplyPings">Indicates if the channel should reply automatically to ping request commands. In this case, the ping command are not returned by the ReceiveCommandAsync method.</param> /// <param name="remotePingInterval">The interval to ping the remote party.</param> /// <param name="remoteIdleTimeout">The timeout to close the channel due to inactivity.</param> /// <param name="channelCommandProcessor">The channel command processor.</param> /// <exception cref="System.ArgumentNullException"></exception> /// <exception cref="System.ArgumentException"> /// Invalid send timeout /// or /// Invalid consume timeout /// or /// Invalid close timeout /// </exception> /// <exception cref="System.ArgumentOutOfRangeException"></exception> protected ChannelBase( ITransport transport, TimeSpan sendTimeout, TimeSpan?consumeTimeout, TimeSpan closeTimeout, int envelopeBufferSize, bool fillEnvelopeRecipients, bool autoReplyPings, TimeSpan?remotePingInterval, TimeSpan?remoteIdleTimeout, IChannelCommandProcessor channelCommandProcessor) { if (transport == null) { throw new ArgumentNullException(nameof(transport)); } if (sendTimeout == default(TimeSpan)) { throw new ArgumentException("Invalid send timeout", nameof(sendTimeout)); } if (consumeTimeout != null && consumeTimeout.Value == default(TimeSpan)) { throw new ArgumentException("Invalid consume timeout", nameof(consumeTimeout)); } if (closeTimeout == default(TimeSpan)) { throw new ArgumentException("Invalid close timeout", nameof(closeTimeout)); } if (envelopeBufferSize <= 0) { throw new ArgumentOutOfRangeException(nameof(envelopeBufferSize)); } Transport = transport; Transport.Closing += Transport_Closing; _sendTimeout = sendTimeout; _consumeTimeout = consumeTimeout; _closeTimeout = closeTimeout; _consumerCts = new CancellationTokenSource(); _syncRoot = new object(); _transportBuffer = new BufferBlock <Envelope>(SingleBoundedCapacityBlockOptions); _messageConsumerBlock = new TransformBlock <Envelope, Message>(e => ConsumeMessageAsync(e), SingleBoundedCapacityBlockOptions); _commandConsumerBlock = new TransformBlock <Envelope, Command>(e => ConsumeCommandAsync(e), SingleBoundedCapacityBlockOptions); _notificationConsumerBlock = new TransformBlock <Envelope, Notification>(e => ConsumeNotificationAsync(e), SingleBoundedCapacityBlockOptions); _sessionConsumerBlock = new TransformBlock <Envelope, Session>(e => ConsumeSession(e), SingleBoundedCapacityBlockOptions); var options = new DataflowBlockOptions() { BoundedCapacity = envelopeBufferSize }; _messageBuffer = new BufferBlock <Message>(options); _commandBuffer = new BufferBlock <Command>(options); _notificationBuffer = new BufferBlock <Notification>(options); _sessionBuffer = new BufferBlock <Session>(SingleBoundedCapacityBlockOptions); _drainEnvelopeBlock = DataflowBlock.NullTarget <Envelope>(); _transportBuffer.LinkTo(_messageConsumerBlock, PropagateCompletionLinkOptions, e => e is Message); _transportBuffer.LinkTo(_commandConsumerBlock, PropagateCompletionLinkOptions, e => e is Command); _transportBuffer.LinkTo(_notificationConsumerBlock, PropagateCompletionLinkOptions, e => e is Notification); _transportBuffer.LinkTo(_sessionConsumerBlock, PropagateCompletionLinkOptions, e => e is Session); _messageConsumerBlock.LinkTo(_messageBuffer, PropagateCompletionLinkOptions, e => e != null); _messageConsumerBlock.LinkTo(_drainEnvelopeBlock, e => e == null); _commandConsumerBlock.LinkTo(_commandBuffer, PropagateCompletionLinkOptions, e => e != null); _commandConsumerBlock.LinkTo(_drainEnvelopeBlock, e => e == null); _notificationConsumerBlock.LinkTo(_notificationBuffer, PropagateCompletionLinkOptions, e => e != null); _notificationConsumerBlock.LinkTo(_drainEnvelopeBlock, e => e == null); _sessionConsumerBlock.LinkTo(_sessionBuffer, PropagateCompletionLinkOptions, e => e != null); _sessionConsumerBlock.LinkTo(_drainEnvelopeBlock, e => e == null); _channelCommandProcessor = channelCommandProcessor ?? new ChannelCommandProcessor(); MessageModules = new List <IChannelModule <Message> >(); NotificationModules = new List <IChannelModule <Notification> >(); CommandModules = new List <IChannelModule <Command> >(); if (autoReplyPings) { CommandModules.Add(new ReplyPingChannelModule(this)); } if (fillEnvelopeRecipients) { FillEnvelopeRecipientsChannelModule.CreateAndRegister(this); } if (remotePingInterval != null) { RemotePingChannelModule.CreateAndRegister(this, remotePingInterval.Value, remoteIdleTimeout); } }
public async Task TestAsObservableAndAsObserver_BroadcastFaultyTarget() { var targets = new ITargetBlock<int>[] { new BufferBlock<int>(), new DelegatePropagator<int, int>() { OfferMessageDelegate = delegate { throw new InvalidOperationException(); } }, new BufferBlock<int>() }; var source = new BufferBlock<int>(); foreach (var target in targets) source.AsObservable().Subscribe(target.AsObserver()); source.Post(1); await Task.WhenAll( Assert.ThrowsAsync<AggregateException>(() => targets[0].Completion), Assert.ThrowsAsync<AggregateException>(() => targets[2].Completion)); }
/// <include file='XmlDocs/CommonXmlDocComments.xml' path='CommonXmlDocComments/Sources/Member[@name="ReleaseReservation"]/*' /> void ISourceBlock <T> .ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock <T> target) { _source.ReleaseReservation(messageHeader, target); }