/// <summary>
/// Initializes a new instance of the <see cref="LogTargetBase"/> class.
/// </summary>
/// <param name="maxDegreeOfParallelism">
/// The max degree of parallelism the target instance will log entries (ie. <see cref="Log"/> method).
/// </param>
protected LogTargetBase(Int32 maxDegreeOfParallelism)
{
_ActionBlock = new ActionBlock<LogEntry>(
logEntry => Log(logEntry),
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = maxDegreeOfParallelism
});
}
public override void FeedTarget(ITargetBlock<GraphiteLine> target)
{
foreach (var gauge in Gauges)
{
target.Post(new GraphiteLine(RootNamespace + gauge.Key, gauge.Value, Epoch));
}
}
public void LinkTo(ITargetBlock<string> target, CancellationToken cancellationToken)
{
_cancellationToken = cancellationToken;
_targetBlock = target;
Task.Factory.StartNew(() =>
{
try
{
var endpoint = new IPEndPoint(IPAddress.Any, _port);
var udpClient = new UdpClient( endpoint );
udpClient.Client.ReceiveBufferSize = MAX_BUFFER_SIZE; //32k buffer
while (true)
{
if (_cancellationToken.IsCancellationRequested)
{
return;
}
byte[] data = udpClient.Receive(ref endpoint);
_preprocessorBlock.Post( data );
}
}
catch (ObjectDisposedException) { /* Eat it, socket was closed */ }
finally { IsListening = false; }
},
cancellationToken);
IsListening = true;
}
public override void FeedTarget(ITargetBlock<GraphiteLine> target)
{
foreach (var line in _rawLines)
{
target.Post(new GraphiteLine(line.Name, line.Value, line.Timestamp ?? Epoch));
}
}
static Task PostAllFilePathsAsync(IEnumerable<CollectionPath> paths, Func<FileInfo, bool> filePredicate,
ITargetBlock<AnnotatedPath> filePathTargetBlock, CancellationToken cancellationToken)
{
var scanTasks = paths
.Select<CollectionPath, Task>(path =>
Task.Factory.StartNew(
async () =>
{
foreach (var file in PathUtil.ScanDirectory(path.Path, filePredicate))
{
if (cancellationToken.IsCancellationRequested)
break;
var relativePath = PathUtil.MakeRelativePath(path.Path, file.FullName);
var annotatedPath = new AnnotatedPath { FileInfo = file, Collection = path.Collection ?? path.Path, RelativePath = relativePath };
await filePathTargetBlock.SendAsync(annotatedPath, cancellationToken).ConfigureAwait(false);
}
},
cancellationToken,
TaskCreationOptions.DenyChildAttach | TaskCreationOptions.LongRunning | TaskCreationOptions.RunContinuationsAsynchronously,
TaskScheduler.Default));
var task = Task.WhenAll(scanTasks);
var completeTask = task.ContinueWith(_ => filePathTargetBlock.Complete());
TaskCollector.Default.Add(completeTask, "PostAllFilePathsAsync");
return task;
}
public async void LinkTo(ITargetBlock<string> target, CancellationToken token)
{
_target = target;
_token = token;
_tcpListener.Start();
IsListening = true;
await Listen();
}
public Business(Guid id, int size, ITargetBlock<ConsumptionData> underservedBlock)
{
this.ID = id;
this.ElectricityConsumer = new ResourceConsumer<Electricity>(id, size, ElementType.Electricity, underservedBlock);
this.WaterConsumer = new ResourceConsumer<Water>(id, size, ElementType.Water, underservedBlock);
this.PeopleConsumer = new ResourceConsumer<People>(id, size, ElementType.People, underservedBlock);
this.TrashProducer = new ResourceProducer<Trash>(id, size);
this.SewageProducer = new ResourceProducer<Sewage>(id, size);
}
public Home(Guid id, int residents, ITargetBlock<ConsumptionData> underservedBlock)
{
this.ID = id;
this.ElectricityConsumer = new ResourceConsumer<Electricity>(id, residents, ElementType.Electricity, underservedBlock);
this.WaterConsumer = new ResourceConsumer<Water>(id, residents, ElementType.Water, underservedBlock);
this.PeopleProducer = new ResourceProducer<People>(id, residents);
this.TrashProducer = new ResourceProducer<Trash>(id, residents);
this.SewageProducer = new ResourceProducer<Sewage>(id, residents);
}
public async Task LoadAsync(CollectionPath[] paths, Func<FileInfo, bool> filePredicate,
ITargetBlock<Tuple<AnnotatedPath, IFileFingerprint>> joinedTargetBlock,
CancellationToken cancellationToken)
{
await _fileFingerprintManager.LoadAsync(cancellationToken).ConfigureAwait(false);
await GenerateFileFingerprintsAsync(paths, filePredicate, joinedTargetBlock, cancellationToken)
.ConfigureAwait(false);
}
/// <summary>
/// Initializes a new instance of the <see cref="AprsPacketSpout" /> class.
/// </summary>
/// <param name="pipeline">The pipeline.</param>
public AprsPacketSpout(IPipeline<byte[]> pipeline)
{
_user = ConfigurationManager.AppSettings["user"];
var password = ConfigurationManager.AppSettings["password"] ?? "-1";
_hostname = ConfigurationManager.AppSettings["hostname"];
_port = int.Parse(ConfigurationManager.AppSettings["port"]);
_logon = Encoding.UTF8.GetBytes("user " + _user + " pass " + password + "\n");
_pipeline = pipeline.Create(string.Empty);
}
public override void FeedTarget(ITargetBlock<GraphiteLine> target)
{
foreach (var set in Sets)
{
foreach (var item in set.Value)
{
target.Post(new GraphiteLine(RootNamespace + set.Key, 1, Epoch));
}
}
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
bool removeZeroGauges,
IIntervalService intervalService,
ILog log)
{
var gauges = new ConcurrentDictionary<string, double>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var gauge = p as Gauge;
gauges.AddOrUpdate(gauge.Name, gauge.Value, (key, oldValue) => gauge.Value);
},
Utility.UnboundedExecution());
intervalService.Elapsed += (sender, e) =>
{
if (gauges.Count == 0)
{
return;
}
var items = gauges.ToArray();
var bucket = new GaugesBucket(items, e.Epoch, ns);
if (removeZeroGauges)
{
// Get all zero-value gauges
double placeholder;
var zeroGauges = 0;
for (int index = 0; index < items.Length; index++)
{
if (items[index].Value == 0)
{
gauges.TryRemove(items[index].Key, out placeholder);
zeroGauges += 1;
}
}
if (zeroGauges > 0)
{
log.InfoFormat("Removed {0} empty gauges.", zeroGauges);
}
}
gauges.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public override void FeedTarget(ITargetBlock<GraphiteLine> target)
{
foreach (var latency in Latencies)
{
var lines = MakeGraphiteLines(latency);
foreach (var line in lines)
{
target.Post(line);
}
}
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
IIntervalService intervalService,
int percentile,
string percentileName,
ILog log,
int maxItemsPerBucket = 1000)
{
var latencies = new ConcurrentDictionary<string, DatapointBox>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var random = new Random();
percentileName = "." + ( percentileName ?? ( "p" + percentile ) );
var incoming = new ActionBlock<StatsdMessage>( p =>
{
var latency = p as Timing;
latencies.AddOrUpdate(latency.Name,
(key) =>
{
return new DatapointBox(maxItemsPerBucket, latency.ValueMS);
},
(key, bag) =>
{
bag.Add(latency.ValueMS);
return bag;
});
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (latencies.Count == 0)
{
return;
}
var bucket = new PercentileBucket(latencies.ToArray(),
e.Epoch,
ns,
percentileName,
percentile);
latencies.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<SetsBucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var sets = new ConcurrentDictionary<string, ConcurrentDictionary<int, bool>>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var set = p as Set;
sets.AddOrUpdate(set.Name,
(key) =>
{
var setDict = new ConcurrentDictionary<int, bool>();
setDict.AddOrUpdate( set.Value, ( key2 ) => true, ( key2, oldValue ) => true );
return setDict;
},
(key, setDict) =>
{
setDict.AddOrUpdate( set.Value, ( key2 ) => true, ( key2, oldValue ) => true );
return setDict;
});
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (sets.Count == 0)
{
return;
}
var rawData = sets.ToArray();
sets.Clear();
var bucket = new SetsBucket(
rawData.Select(p =>
new KeyValuePair<string, List<KeyValuePair<int, bool>>>(p.Key, p.Value.ToList())
).ToList(),
e.Epoch,
ns);
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
static void skynetTplRecursion( ITargetBlock<long> src, long num, long size, long div )
{
if( size == 1 )
{
src.SendAsync( num );
return;
}
for( var i = 0; i < div; i++ )
{
var sub_num = num + i * ( size / div );
skynetTplRecursion( src, sub_num, size / div, div );
}
}
public override void FeedTarget(ITargetBlock<GraphiteLine> target)
{
int percentileValue;
foreach (var measurements in Timings)
{
if (TryComputePercentile(measurements, out percentileValue))
{
target.Post(
new GraphiteLine(
RootNamespace + measurements.Key + PercentileName,
percentileValue,
Epoch));
}
}
}
private void DownloadPage(ITargetBlock<Parse> targetAscTransform)
{
var totalPages = 0;
foreach (var parse in _dataAccess.GetEmailQueries().FindAll(x => x.Email == true).Select(q => _dataAccess.GetHtmlToScrape(GenSqlStatementForHtmlParsing(q.Id))).SelectMany(cars => cars.Select(car => new Parse
{
CarId = car.Id,
Link = car.Link,
UserAgent = _dataAccess.GetRandomUserAgent()
})))
{
targetAscTransform.Post(parse);
totalPages++;
}
Logger.Debug("Total for this run is {0}", totalPages);
}
/// <summary>
/// Initializes a new instance of the <see cref="BatchLogTargetBase"/> class.
/// </summary>
/// <param name="batchSize">Size of the log batch.</param>
/// <param name="flushInterval">The flush interval.
/// The interval with which to initiate a batching operation even if the
/// number of currently queued logs is less than the <paramref name="batchSize"/>.
/// </param>
/// <param name="maxDegreeOfParallelism">The max degree of parallelism the target instance will log batch entries (ie. <see cref="Log"/> method).</param>
protected BatchLogTargetBase(Int32 batchSize, TimeSpan flushInterval, Int32 maxDegreeOfParallelism)
{
_FlushInterval = flushInterval;
_BatchBlock = new BatchBlock<LogEntry>(batchSize);
_ActionBlock = new ActionBlock<IEnumerable<LogEntry>>(
logEntries =>
{
Log(logEntries);
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = maxDegreeOfParallelism
});
_BatchBlock.LinkTo(_ActionBlock);
_FlushTimer = new Timer(FlushTimerCallback, null, _FlushInterval, TimeSpan.FromMilliseconds(-1));
}
private void DownloadPage(ITargetBlock<Parse> targetAscTransform)
{
foreach (var q in _dataAccess.GetEmailQueries().FindAll(x => x.Email == true))
{
var cars = _dataAccess.GetHtmlToScrape(GenSqlStatementForHtmlParsing(q.Id));
var emailss = cars.Where(e => e.HtmlDownloaded == false);
foreach (var car in cars)
{
var parse = new Parse
{
CarId = car.Id,
Link = car.Link,
UserAgent = _dataAccess.GetRandomUserAgent()
};
targetAscTransform.Post(parse);
}
}
}
public LinkFingerprintJoiner(ITargetBlock<Tuple<AnnotatedPath, IFileFingerprint>> targetBlock)
{
if (null == targetBlock)
throw new ArgumentNullException(nameof(targetBlock));
_targetBlock = targetBlock;
AnnotatedPathsBlock = new ActionBlock<AnnotatedPath[]>(
aps => Task.WhenAll(aps.Select(SetAnnotatedPathAsync)),
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
FileFingerprintBlock = new ActionBlock<IFileFingerprint>(SetFileFingerprintAsync,
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
var task = Task.WhenAll(AnnotatedPathsBlock.Completion, FileFingerprintBlock.Completion)
.ContinueWith(_ => { targetBlock.Complete(); });
TaskCollector.Default.Add(task, "Joiner completion");
}
async Task GenerateFileFingerprintsAsync(CollectionPath[] paths, Func<FileInfo, bool> filePredicate,
ITargetBlock<Tuple<AnnotatedPath, IFileFingerprint>> joinedTargetBlock,
CancellationToken cancellationToken)
{
try
{
var annotatedPathBroadcastBlock = new BroadcastBlock<AnnotatedPath[]>(aps => aps,
new DataflowBlockOptions { CancellationToken = cancellationToken });
var joiner = new LinkFingerprintJoiner(joinedTargetBlock);
annotatedPathBroadcastBlock.LinkTo(joiner.AnnotatedPathsBlock, new DataflowLinkOptions { PropagateCompletion = true });
var fileFingerprintBroadcastBlock = new BroadcastBlock<IFileFingerprint>(ff => ff,
new DataflowBlockOptions { CancellationToken = cancellationToken });
fileFingerprintBroadcastBlock.LinkTo(joiner.FileFingerprintBlock, new DataflowLinkOptions { PropagateCompletion = true });
var fingerprintGeneratorTask = _fileFingerprintManager
.GenerateFileFingerprintsAsync(annotatedPathBroadcastBlock, fileFingerprintBroadcastBlock, cancellationToken);
try
{
await DirectoryScanner.GenerateAnnotatedPathsAsync(paths, filePredicate, annotatedPathBroadcastBlock, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{ }
catch (Exception ex)
{
Console.Write("Path scan failed: " + ex.Message);
}
await fingerprintGeneratorTask.ConfigureAwait(false);
}
catch (Exception ex)
{
Console.WriteLine("GenerateFileFingerprintsAsync() failed: " + ex.Message);
}
finally
{
Debug.WriteLine("BlobManager.GenerateFileFingerprintsAsync() is done");
}
}
public NeverendingTask(Func<DateTimeOffset, CancellationToken, Task> action, CancellationToken cancellationToken)
{
if (action == null)
{
throw new ArgumentNullException("action");
}
_inner = new ActionBlock<DateTimeOffset>(async now =>
{
await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken)
.ConfigureAwait(false);
await action(now, cancellationToken);
_inner.Post(DateTimeOffset.UtcNow);
},
new ExecutionDataflowBlockOptions
{
CancellationToken = cancellationToken
});
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
IIntervalService intervalService)
{
var rawLines = new ConcurrentStack<Raw>();
var incoming = new ActionBlock<StatsdMessage>(p =>
{
rawLines.Push(p as Raw);
},
Utility.UnboundedExecution());
intervalService.Elapsed += (sender, e) =>
{
if (rawLines.Count == 0)
{
return;
}
var lines = rawLines.ToArray();
rawLines.Clear();
var bucket = new RawBucket(lines, e.Epoch);
target.Post(bucket);
};
return incoming;
}
public static ActionBlock<StatsdMessage> CreateBlock(ITargetBlock<Bucket> target,
string rootNamespace,
IIntervalService intervalService,
ILog log)
{
var counters = new ConcurrentDictionary<string, double>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : (rootNamespace + ".");
var incoming = new ActionBlock<StatsdMessage>(p =>
{
var counter = p as Counter;
counters.AddOrUpdate(counter.Name, counter.Value, (key, oldValue) => oldValue + counter.Value);
},
new ExecutionDataflowBlockOptions() { MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded });
intervalService.Elapsed += (sender, e) =>
{
if (counters.Count == 0)
{
return;
}
var bucket = new CounterBucket(counters.ToArray(), e.Epoch, ns);
counters.Clear();
target.Post(bucket);
};
incoming.Completion.ContinueWith(p =>
{
log.Info("TimedCounterAggregatorBlock completing.");
// Tell the upstream block that we're done
target.Complete();
});
return incoming;
}
public static Task GenerateAnnotatedPathsAsync(IEnumerable<CollectionPath> paths, Func<FileInfo, bool> filePredicate,
ITargetBlock<AnnotatedPath[]> filePathTargetBlock, CancellationToken cancellationToken)
{
var shuffleBlock = new TransformBlock<AnnotatedPath[], AnnotatedPath[]>(
filenames =>
{
// Sequential names tend to fall into the same AWS S3 partition, so we
// shuffle things around.
RandomUtil.Shuffle(filenames);
return filenames;
}, new ExecutionDataflowBlockOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = Environment.ProcessorCount });
shuffleBlock.LinkTo(filePathTargetBlock, new DataflowLinkOptions { PropagateCompletion = true });
var batcher = new BatchBlock<AnnotatedPath>(2048, new GroupingDataflowBlockOptions { CancellationToken = cancellationToken });
batcher.LinkTo(shuffleBlock, new DataflowLinkOptions
{
PropagateCompletion = true
});
return PostAllFilePathsAsync(paths, filePredicate, batcher, cancellationToken);
}
public void LinkTo(ITargetBlock<string> target, CancellationToken token)
{
_target = target;
IsListening = true;
_scheduler = KayakScheduler.Factory.Create(new SchedulerDelegate());
var wsTask = Task.Factory.StartNew(() =>
{
var server = KayakServer.Factory.CreateHttp(
new RequestDelegate(this),
_scheduler);
using (server.Listen(new IPEndPoint(IPAddress.Any, _port)))
{
_scheduler.Start();
}
IsListening = false;
});
Task.Factory.StartNew(() =>
{
token.WaitHandle.WaitOne();
_scheduler.Stop();
});
}
public PipelineMergeLinker(ISourceBlock <IPipelineJobElement <T1> > from1, ISourceBlock <IPipelineJobElement <T2> > from2, ITargetBlock <IPipelineJobElement <Tuple <T1, T2> > > to, Predicate <IPipelineJobElement <T1> > predicate1, Predicate <IPipelineJobElement <T2> > predicate2)
{
if (predicate1 == null)
{
predicate1 = e => true;
}
if (predicate2 == null)
{
predicate2 = e => true;
}
Input = new JoinBlock <IPipelineJobElement <T1>, IPipelineJobElement <T2> >();
Output = new ActionBlock <Tuple <IPipelineJobElement <T1>, IPipelineJobElement <T2> > >(
elements =>
{
var job = elements.Item1.Job;
var mergedElement = job.MergeToSingleElement(elements);
to.Post(mergedElement);
});
from1.LinkTo(Input.Target1, predicate1);
from2.LinkTo(Input.Target2, predicate2);
Input.LinkTo(Output);
}
public UploadPipeline(ISession session, ILogger log, IFileSystem fs)
{
var tree = new FolderTree(session);
var fileNameStep = new TransformBlock <AssetModel, (AssetModel, FilePath)>(async asset =>
{
FilePath path;
if (FilePathProvider != null)
{
path = FilePathProvider(asset);
}
else if (FilePathProviderAsync != null)
{
path = await FilePathProviderAsync(asset);
}
else
{
path = new FilePath(asset.Id);
}
return(asset, path);
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = 1,
MaxMessagesPerTask = 1,
BoundedCapacity = 1
});
var maxDegreeOfParallelism = fs.CanAccessInParallel ? Environment.ProcessorCount * 2 : 1;
var uploadStep = new ActionBlock <(AssetModel, FilePath)>(async item =>
{
var(asset, path) = item;
var process = $"Uploading {path}";
try
{
var assetFile = fs.GetFile(path);
await using (var stream = assetFile.OpenRead())
{
var file = new FileParameter(stream, asset.FileName, asset.MimeType);
var result = await session.Assets.PostUpsertAssetAsync(session.App, asset.Id, null, true, file);
log.ProcessCompleted(process);
}
}
catch (Exception ex)
{
log.ProcessFailed(process, ex);
}
}, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = maxDegreeOfParallelism,
MaxMessagesPerTask = 1,
BoundedCapacity = maxDegreeOfParallelism * 2
});
fileNameStep.LinkTo(uploadStep, new DataflowLinkOptions
{
PropagateCompletion = true
});
pipelineStart = fileNameStep;
pipelineEnd = uploadStep;
}
public Tuple <T1, T2, T3> ConsumeMessage(DataflowMessageHeader messageHeader, ITargetBlock <Tuple <T1, T2, T3> > target, out bool messageConsumed)
{
return(vault.ConsumeMessage(messageHeader, target, out messageConsumed));
}
public async void LinkTo(ITargetBlock<string> target, CancellationToken token)
{
_target = target;
_token = token;
await Listen();
}
/// <include file='XmlDocs/CommonXmlDocComments.xml' path='CommonXmlDocComments/Sources/Member[@name="ReleaseReservation"]/*' />
void ISourceBlock <T> .ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock <T> target)
{
// Validate arguments
if (!messageHeader.IsValid)
{
throw new ArgumentException(SR.Argument_InvalidMessageHeader, nameof(messageHeader));
}
if (target == null)
{
throw new ArgumentNullException(nameof(target));
}
Contract.EndContractBlock();
// As long as the message is the one we have, everything's fine.
if (_header.Id != messageHeader.Id)
{
throw new InvalidOperationException(SR.InvalidOperation_MessageNotReservedByTarget);
}
// In other blocks, upon release we typically re-offer the message to all linked targets.
// We need to do the same thing for WriteOnceBlock, in order to account for cases where the block
// may be linked to a join or similar block, such that the join could never again be satisfied
// if it didn't receive another offer from this source. However, since the message is broadcast
// and all targets can get a copy, we don't need to broadcast to all targets, only to
// the target that released the message. Note that we don't care whether it's accepted
// or not, nor do we care about any exceptions which may emerge (they should just propagate).
Debug.Assert(_header.IsValid, "A valid header is required.");
bool useCloning = _cloningFunction != null;
target.OfferMessage(_header, _value, this, consumeToAccept: useCloning);
}
//public string ThreadName;
private void AscBufferProducer(ITargetBlock<Parse> targetAscTransform)
// This is using TPL DataBlock producer consumer pattern.
{
DownloadPage(targetAscTransform);
}
/// <include file='XmlDocs/CommonXmlDocComments.xml' path='CommonXmlDocComments/Sources/Member[@name="LinkTo"]/*' />
IDisposable ISourceBlock <T> .LinkTo(ITargetBlock <T> target, DataflowLinkOptions linkOptions)
{
throw new NotSupportedException(SR.NotSupported_MemberNotNeeded);
}
public void ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock <TOutput> target)
{
_block.ReleaseReservation(messageHeader, target);
}
public bool ReserveMessage(DataflowMessageHeader messageHeader, ITargetBlock <TOutput> target)
{
return(_block.ReserveMessage(messageHeader, target));
}
public TOutput ConsumeMessage(DataflowMessageHeader messageHeader, ITargetBlock <TOutput> target, out bool messageConsumed)
{
return(_block.ConsumeMessage(messageHeader, target, out messageConsumed));
}
public IDisposable LinkTo(ITargetBlock <TOutput> target, DataflowLinkOptions linkOptions)
{
return(_block.LinkTo(target, linkOptions));
}
/// <inheritdoc />
public bool ReserveMessage(DataflowMessageHeader messageHeader, ITargetBlock <T[]> target)
{
return(((IPropagatorBlock <T, T[]>)_batchBlock).ReserveMessage(messageHeader, target));
}
/// <inheritdoc />
public void ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock <T[]> target)
{
((IPropagatorBlock <T, T[]>)_batchBlock).ReleaseReservation(messageHeader, target);
}
/// <include file='XmlDocs/CommonXmlDocComments.xml' path='CommonXmlDocComments/Sources/Member[@name="ConsumeMessage"]/*' />
T? ISourceBlock <T> .ConsumeMessage(DataflowMessageHeader messageHeader, ITargetBlock <T> target, out bool messageConsumed)
{
return(_source.ConsumeMessage(messageHeader, target, out messageConsumed));
}
/// <include file='XmlDocs/CommonXmlDocComments.xml' path='CommonXmlDocComments/Sources/Member[@name="LinkTo"]/*' />
public IDisposable LinkTo(ITargetBlock <T> target, DataflowLinkOptions linkOptions)
{
return(_source.LinkTo(target, linkOptions));
}
public void SetTarget(ITargetBlock <StatsdMessage> target)
{
_target = target;
}
void ISourceBlock <T> .ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock <T> target)
{
if (_messagesByBlock.TryGetValue(messageHeader, out var block))
{
block.ReleaseReservation(messageHeader, target);
}
}
/// <summary>Gets whether the registry contains a particular target.</summary>
/// <param name="target">The target.</param>
/// <returns>true if the registry contains the target; otherwise, false.</returns>
internal bool Contains(ITargetBlock <T> target)
{
return(_targetInformation.ContainsKey(target));
}
/// <summary>
/// This method is where we tell data flow which blocks we're interested in receiving updates for
/// </summary>
protected override IDisposable LinkExternalInput(ITargetBlock <IProjectVersionedValue <DesignTimeInputSnapshot> > targetBlock)
{
JoinUpstreamDataSources(_designTimeInputsChangeTracker);
return(_designTimeInputsChangeTracker.SourceBlock.LinkTo(targetBlock, DataflowOption.PropagateCompletion));
}
public static ActionBlock <StatsdMessage> CreateBlock(ITargetBlock <GraphiteLine> target,
string rootNamespace,
IIntervalService intervalService,
int percentile,
string percentileName,
ILog log,
int maxItemsPerBucket = 1000)
{
var latencies = new ConcurrentDictionary <string, DatapointBox>();
var root = rootNamespace;
var ns = String.IsNullOrEmpty(rootNamespace) ? "" : rootNamespace + ".";
var random = new Random();
percentileName = "." + (percentileName ?? ("p" + percentile));
var incoming = new ActionBlock <StatsdMessage>(p =>
{
var latency = p as Timing;
latencies.AddOrUpdate(latency.Name,
(key) =>
{
return(new DatapointBox(maxItemsPerBucket, latency.ValueMS));
},
(key, bag) =>
{
bag.Add(latency.ValueMS);
return(bag);
});
},
new ExecutionDataflowBlockOptions()
{
MaxDegreeOfParallelism = DataflowBlockOptions.Unbounded
});
intervalService.Elapsed += (sender, e) =>
{
if (latencies.Count == 0)
{
return;
}
var buckets = latencies.ToArray();
latencies.Clear();
int percentileValue;
int numLinesPosted = 0;
foreach (var measurements in buckets)
{
if (Percentile.TryCompute(measurements.Value.ToArray().ToList(), percentile, out percentileValue))
{
target.Post(new GraphiteLine(ns + measurements.Key + percentileName, percentileValue, e.Epoch));
numLinesPosted++;
}
}
log.InfoFormat("TimedLatencyPercentileAggregatorBlock - Posted {0} buckets and {1} lines.", buckets.Length, numLinesPosted);
};
incoming.Completion.ContinueWith(p =>
{
// Tell the upstream block that we're done
target.Complete();
});
return(incoming);
}
public static void PostAssert <T>(ITargetBlock <T> target, T item)
{
bool retval = target.Post(item);
Contracts.Assert(retval);
}
public IDisposable LinkTo(ITargetBlock <TOutput> target, DataflowLinkOptions linkOptions)
{
return(outgoing.AddTarget(target, linkOptions));
}
/// <inheritdoc />
public T[] ConsumeMessage(DataflowMessageHeader messageHeader, ITargetBlock <T[]> target, out bool messageConsumed)
{
return(((IPropagatorBlock <T, T[]>)_batchBlock).ConsumeMessage(messageHeader, target, out messageConsumed));
}
public void ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock <Tuple <T1, T2, T3> > target)
{
vault.ReleaseReservation(messageHeader, target);
}
public MergeJoinTarget(ITargetBlock <TInput> joinTarget)
{
TargetBlock = joinTarget;
}
public bool ReserveMessage(DataflowMessageHeader messageHeader, ITargetBlock <Tuple <T1, T2, T3> > target)
{
return(vault.ReserveMessage(messageHeader, target));
}
public void SetTarget(ITargetBlock <Bucket> target)
{
_target = target;
}
private async Task ProduceObjects(IAsyncEnumerable <IUser> users, ITargetBlock <IUser> target)
{
await users.ForEachAsync(t => target.Post(t));
target.Complete();
}
public static IDisposable LinkWithPropagationTo <T>(this ISourceBlock <T> source, ITargetBlock <T> target)
{
return(source.LinkTo(target, new DataflowLinkOptions
{
PropagateCompletion = true
}));
}
/// <include file='XmlDocs/CommonXmlDocComments.xml' path='CommonXmlDocComments/Sources/Member[@name="ReserveMessage"]/*' />
bool ISourceBlock <T> .ReserveMessage(DataflowMessageHeader messageHeader, ITargetBlock <T> target)
{
return(_source.ReserveMessage(messageHeader, target));
}
/// <summary>
/// Creates a new instance of ChannelBase
/// </summary>
/// <param name="transport">The transport.</param>
/// <param name="sendTimeout">The channel send timeout.</param>
/// <param name="consumeTimeout">The channel consume timeout. Each envelope received from the transport must be consumed in the specified timeout or it will cause the channel to be closed.</param>
/// <param name="closeTimeout">The channel close timeout.</param>
/// <param name="envelopeBufferSize">Size of the envelope buffer.</param>
/// <param name="fillEnvelopeRecipients">Indicates if the from and to properties of sent and received envelopes should be filled with the session information if not defined.</param>
/// <param name="autoReplyPings">Indicates if the channel should reply automatically to ping request commands. In this case, the ping command are not returned by the ReceiveCommandAsync method.</param>
/// <param name="remotePingInterval">The interval to ping the remote party.</param>
/// <param name="remoteIdleTimeout">The timeout to close the channel due to inactivity.</param>
/// <param name="channelCommandProcessor">The channel command processor.</param>
/// <exception cref="System.ArgumentNullException"></exception>
/// <exception cref="System.ArgumentException">
/// Invalid send timeout
/// or
/// Invalid consume timeout
/// or
/// Invalid close timeout
/// </exception>
/// <exception cref="System.ArgumentOutOfRangeException"></exception>
protected ChannelBase(
ITransport transport,
TimeSpan sendTimeout,
TimeSpan?consumeTimeout,
TimeSpan closeTimeout,
int envelopeBufferSize,
bool fillEnvelopeRecipients,
bool autoReplyPings,
TimeSpan?remotePingInterval,
TimeSpan?remoteIdleTimeout,
IChannelCommandProcessor channelCommandProcessor)
{
if (transport == null)
{
throw new ArgumentNullException(nameof(transport));
}
if (sendTimeout == default(TimeSpan))
{
throw new ArgumentException("Invalid send timeout", nameof(sendTimeout));
}
if (consumeTimeout != null && consumeTimeout.Value == default(TimeSpan))
{
throw new ArgumentException("Invalid consume timeout", nameof(consumeTimeout));
}
if (closeTimeout == default(TimeSpan))
{
throw new ArgumentException("Invalid close timeout", nameof(closeTimeout));
}
if (envelopeBufferSize <= 0)
{
throw new ArgumentOutOfRangeException(nameof(envelopeBufferSize));
}
Transport = transport;
Transport.Closing += Transport_Closing;
_sendTimeout = sendTimeout;
_consumeTimeout = consumeTimeout;
_closeTimeout = closeTimeout;
_consumerCts = new CancellationTokenSource();
_syncRoot = new object();
_transportBuffer = new BufferBlock <Envelope>(SingleBoundedCapacityBlockOptions);
_messageConsumerBlock = new TransformBlock <Envelope, Message>(e => ConsumeMessageAsync(e), SingleBoundedCapacityBlockOptions);
_commandConsumerBlock = new TransformBlock <Envelope, Command>(e => ConsumeCommandAsync(e), SingleBoundedCapacityBlockOptions);
_notificationConsumerBlock = new TransformBlock <Envelope, Notification>(e => ConsumeNotificationAsync(e), SingleBoundedCapacityBlockOptions);
_sessionConsumerBlock = new TransformBlock <Envelope, Session>(e => ConsumeSession(e), SingleBoundedCapacityBlockOptions);
var options = new DataflowBlockOptions()
{
BoundedCapacity = envelopeBufferSize
};
_messageBuffer = new BufferBlock <Message>(options);
_commandBuffer = new BufferBlock <Command>(options);
_notificationBuffer = new BufferBlock <Notification>(options);
_sessionBuffer = new BufferBlock <Session>(SingleBoundedCapacityBlockOptions);
_drainEnvelopeBlock = DataflowBlock.NullTarget <Envelope>();
_transportBuffer.LinkTo(_messageConsumerBlock, PropagateCompletionLinkOptions, e => e is Message);
_transportBuffer.LinkTo(_commandConsumerBlock, PropagateCompletionLinkOptions, e => e is Command);
_transportBuffer.LinkTo(_notificationConsumerBlock, PropagateCompletionLinkOptions, e => e is Notification);
_transportBuffer.LinkTo(_sessionConsumerBlock, PropagateCompletionLinkOptions, e => e is Session);
_messageConsumerBlock.LinkTo(_messageBuffer, PropagateCompletionLinkOptions, e => e != null);
_messageConsumerBlock.LinkTo(_drainEnvelopeBlock, e => e == null);
_commandConsumerBlock.LinkTo(_commandBuffer, PropagateCompletionLinkOptions, e => e != null);
_commandConsumerBlock.LinkTo(_drainEnvelopeBlock, e => e == null);
_notificationConsumerBlock.LinkTo(_notificationBuffer, PropagateCompletionLinkOptions, e => e != null);
_notificationConsumerBlock.LinkTo(_drainEnvelopeBlock, e => e == null);
_sessionConsumerBlock.LinkTo(_sessionBuffer, PropagateCompletionLinkOptions, e => e != null);
_sessionConsumerBlock.LinkTo(_drainEnvelopeBlock, e => e == null);
_channelCommandProcessor = channelCommandProcessor ?? new ChannelCommandProcessor();
MessageModules = new List <IChannelModule <Message> >();
NotificationModules = new List <IChannelModule <Notification> >();
CommandModules = new List <IChannelModule <Command> >();
if (autoReplyPings)
{
CommandModules.Add(new ReplyPingChannelModule(this));
}
if (fillEnvelopeRecipients)
{
FillEnvelopeRecipientsChannelModule.CreateAndRegister(this);
}
if (remotePingInterval != null)
{
RemotePingChannelModule.CreateAndRegister(this, remotePingInterval.Value, remoteIdleTimeout);
}
}
public async Task TestAsObservableAndAsObserver_BroadcastFaultyTarget()
{
var targets = new ITargetBlock<int>[]
{
new BufferBlock<int>(),
new DelegatePropagator<int, int>() { OfferMessageDelegate = delegate { throw new InvalidOperationException(); } },
new BufferBlock<int>()
};
var source = new BufferBlock<int>();
foreach (var target in targets) source.AsObservable().Subscribe(target.AsObserver());
source.Post(1);
await Task.WhenAll(
Assert.ThrowsAsync<AggregateException>(() => targets[0].Completion),
Assert.ThrowsAsync<AggregateException>(() => targets[2].Completion));
}
/// <include file='XmlDocs/CommonXmlDocComments.xml' path='CommonXmlDocComments/Sources/Member[@name="ReleaseReservation"]/*' />
void ISourceBlock <T> .ReleaseReservation(DataflowMessageHeader messageHeader, ITargetBlock <T> target)
{
_source.ReleaseReservation(messageHeader, target);
}
