public void ReactiveTest()
{
IPropagatorBlock <int, int> block = null;
AssertEx.Throws <ArgumentNullException> (() => block.AsObservable());
AssertEx.Throws <ArgumentNullException> (() => block.AsObserver());
}
public void LinkToTest()
{
IPropagatorBlock <int, int> nullBlock = null;
var realBlock = new BufferBlock <int> ();
AssertEx.Throws <ArgumentNullException> (
() => DataflowBlock.LinkTo(nullBlock, realBlock));
AssertEx.Throws <ArgumentNullException> (
() => DataflowBlock.LinkTo(realBlock, nullBlock));
AssertEx.Throws <ArgumentNullException> (
() => DataflowBlock.LinkTo(nullBlock, realBlock, i => true));
AssertEx.Throws <ArgumentNullException> (
() => DataflowBlock.LinkTo(realBlock, nullBlock, i => true));
AssertEx.Throws <ArgumentNullException> (
() => DataflowBlock.LinkTo(realBlock, realBlock, null));
AssertEx.Throws <ArgumentNullException> (
() => DataflowBlock.LinkTo(nullBlock, realBlock, new DataflowLinkOptions(), i => true));
AssertEx.Throws <ArgumentNullException> (
() => DataflowBlock.LinkTo(realBlock, nullBlock, new DataflowLinkOptions(), i => true));
AssertEx.Throws <ArgumentNullException> (
() => DataflowBlock.LinkTo(realBlock, realBlock, null, i => true));
AssertEx.Throws <ArgumentNullException> (
() => DataflowBlock.LinkTo(realBlock, realBlock, new DataflowLinkOptions(), null));
}
public ThrottledProducerConsumer(TimeSpan Interval, int MaxPerInterval,
Int32 QueueBoundedMax = 5, Action <T> ConsumerAction = null, Int32 MaxConsumers = 1,
Int32 MaxThrottleBuffer = 20, Int32 MaxDegreeOfParallelism = 10)
{
//-- Probably best to link MaxPerInterval and MaxThrottleBuffer
// and MaxConsumers with MaxDegreeOfParallelism
var consumerOptions = new ExecutionDataflowBlockOptions {
BoundedCapacity = 1,
};
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true,
};
//-- Create the Queue
_queue = new BufferBlock <T>(new DataflowBlockOptions {
BoundedCapacity = QueueBoundedMax,
});
//-- Create and link the throttle block
_throttleBlock = CreateThrottleBlock <T>(Interval, MaxPerInterval);
_queue.LinkTo(_throttleBlock, linkOptions);
//-- Create and link the consumer(s) to the throttle block
var consumerAction = (ConsumerAction != null) ? ConsumerAction : new Action <T>(ConsumeItem);
_consumers = new List <Task>();
for (int i = 0; i < MaxConsumers; i++)
{
var consumer = new ActionBlock <T>(consumerAction, consumerOptions);
_throttleBlock.LinkTo(consumer, linkOptions);
_consumers.Add(consumer.Completion);
}
//-- TODO: Add some cancellation tokens to shut this thing down
}
/// <summary>
/// Attach send and receive blocks
/// </summary>
/// <param name="send"></param>
/// <param name="receive"></param>
public virtual void Attach(
IPropagatorBlock <Message, Message> send,
IPropagatorBlock <Message, Message> receive)
{
_socketSend = send.ConnectTo(SendBlock);
_socketReceive = ReceiveBlock.ConnectTo(receive);
}
protected RetriableTransformBlock(ExecutionDataflowBlockOptions options, RetryPolicy retryPolicy, bool throwOnException = true)
{
if (retryPolicy == null)
{
throw new ArgumentNullException("retryPolicy");
}
retryPolicy.Retrying += (sender, args) => OnRetry(args);
_block = new TransformManyBlock <TInput, TOutput>(input =>
{
try
{
return(retryPolicy.ExecuteAction(() => new[] { Handle(input) }));
}
catch
{
if (throwOnException)
{
throw;
}
return(Enumerable.Empty <TOutput>());
}
}, options);
}
public Actor(BufferBlock <object> buffer, CancellationTokenSource cts)
{
_cts = cts;
token = _cts.Token;
var config = new ExecutionDataflowBlockOptions
{
CancellationToken = token,
SingleProducerConstrained = true
};
this._buffer = buffer;
_action = new ActionBlock <object>(message => Process(message), config);
this._buffer.LinkTo(_action, new DataflowLinkOptions()
{
PropagateCompletion = true
});
this._buffer.Completion.ContinueWith(t =>
{
while (_action.InputCount > 0)
{
Thread.Sleep(1000);
}
_action.Complete();
_action.Completion.GetAwaiter().GetResult();
});
}
public static Tuple <IPropagatorBlock <long, long>[], IPropagatorBlock <long, long>[]> GenerateSumSoFarGraph(int depth,
Func <long[], long> f)
{
Stopwatch sw = new Stopwatch();
sw.Restart();
// create
var firstLayer = new BroadcastBlock <long> [depth];
var secondLayer = new IPropagatorBlock <long, long> [depth];
for (int i = 0; i < depth; i++)
{
firstLayer[i] = new BroadcastBlock <long>(null);
secondLayer[i] = CreateSumSoFarNode(i + 1, f);
}
// linking
for (int i = 0; i < depth; i++)
{
for (int k = i; k < depth; k++)
{
firstLayer[i].LinkTo(secondLayer[k]);
}
}
sw.Stop();
Console.WriteLine("Creation: {0}", sw.Elapsed);
return(new Tuple <IPropagatorBlock <long, long>[], IPropagatorBlock <long, long>[]>(firstLayer, secondLayer));
}
public void SetUp()
{
_fakeBlock = MakeFakePropogatorBlock();
_fakeTarget = new Mock <ITargetBlock <int> >();
_block = CreateBlock(_fakeBlock.Object);
}
public S3BatchRemover(IAmazonS3 s3Client, string bucketName, int batchSize,
int maxDegreeOfParallelism, CancellationToken cancellationToken = default)
{
_s3Client = s3Client ?? throw new ArgumentNullException(nameof(s3Client));
_bucketName = bucketName ?? throw new ArgumentNullException(nameof(bucketName));
_cancellationToken = cancellationToken;
_targetBlock = new ActionBlock <string[]>(RemoveFiles,
new ExecutionDataflowBlockOptions
{
CancellationToken = cancellationToken,
MaxDegreeOfParallelism = maxDegreeOfParallelism
});
_propagatorBlock = new BatchBlock <string>(batchSize,
new GroupingDataflowBlockOptions
{
CancellationToken = cancellationToken,
BoundedCapacity = batchSize * Environment.ProcessorCount
});
_ = _propagatorBlock.LinkTo(_targetBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
}
/// <summary>
/// Initializes the DataflowBlock
/// </summary>
/// <param name="importJobInformation"><see cref="ImportJobInformation"/> of the ImportJob this DataflowBlock belongs to</param>
/// <param name="inputBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="InputBlock"/></param>
/// <param name="innerBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="InnerBlock"/></param>
/// <param name="outputBlockOptions"><see cref="ExecutionDataflowBlockOptions"/> for the <see cref="OutputBlock"/></param>
/// <param name="blockname">Name of this DataflowBlock (must be unique in a given chain of DataflowBlocks)</param>
/// <param name="isRestorePointAfterDeserialization">
/// <c>true</c>, if after deserialiization from disk, <see cref="PendingImportResourceNewGen"/>s are restored to
/// this block. If <c>false</c>, they are restored to the last passed DataflowBlock having this parameter set to <c>true</c>
/// </param>
/// <param name="parentImportJobController">ImportJobController to which this DataflowBlock belongs</param>
protected ImporterWorkerDataflowBlockBase(ImportJobInformation importJobInformation, ExecutionDataflowBlockOptions inputBlockOptions, ExecutionDataflowBlockOptions innerBlockOptions, ExecutionDataflowBlockOptions outputBlockOptions, String blockname, bool isRestorePointAfterDeserialization, ImportJobController parentImportJobController, CancellationToken ct)
{
_blockName = blockname;
_isRestorePointAfterDeserialization = isRestorePointAfterDeserialization;
_ct = ct;
ImportJobInformation = importJobInformation;
InputBlockOptions = inputBlockOptions;
InnerBlockOptions = innerBlockOptions;
OutputBlockOptions = outputBlockOptions;
ParentImportJobController = parentImportJobController;
_stopWatch = new Stopwatch();
_tcs = new TaskCompletionSource <object>();
Activated = new AsyncManualResetEvent(InnerBlockOptions.CancellationToken);
InputBlock = new TransformBlock <PendingImportResourceNewGen, PendingImportResourceNewGen>(p => InputBlockMethod(p), InputBlockOptions);
OutputBlock = new TransformManyBlock <PendingImportResourceNewGen, PendingImportResourceNewGen>(p => OutputBlockMethod(p), OutputBlockOptions);
// ReSharper disable once DoNotCallOverridableMethodsInConstructor
InnerBlock = CreateInnerBlock();
InnerBlock.LinkTo(OutputBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
InputBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted);
InnerBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted);
OutputBlock.Completion.ContinueWith(OnAnyBlockFaulted, TaskContinuationOptions.OnlyOnFaulted);
Task.WhenAll(InputBlock.Completion, InnerBlock.Completion, OutputBlock.Completion).ContinueWith(OnAllBlocksFinished);
}
public TargetWrapperDelegate(string name, IPropagatorBlock <TInput, TOutput> block)
{
Name = name;
Block = block;
Target = block;
Source = block;
}
private MessagePipeline(
Func <Func <TSource, Task>, IMessageSubscription> subscription,
IPropagatorBlock <TSource, TDestination> pipeline = null)
{
_pipeline = pipeline;
_subscription = subscription;
}
public static Tuple <IPropagatorBlock <long, long>[], IPropagatorBlock <long, long>[]> GenerateManyDepPrefixSumGraph(int width,
Func <long[], long> f, out string creationTime)
{
Stopwatch sw = new Stopwatch();
sw.Restart();
// create
var firstLayer = new BroadcastBlock <long> [width];
var secondLayer = new IPropagatorBlock <long, long> [width];
for (int i = 0; i < width; i++)
{
firstLayer[i] = new BroadcastBlock <long>(null);
secondLayer[i] = CreateManyDepPrefixSumNode(i + 1, f);
}
// linking
for (int i = 0; i < width; i++)
{
for (int k = i; k < width; k++)
{
firstLayer[i].LinkTo(secondLayer[k]);
}
}
sw.Stop();
const string format = "{0,10:F3}";
creationTime = string.Format(format, sw.Elapsed.Milliseconds);
return(new Tuple <IPropagatorBlock <long, long>[], IPropagatorBlock <long, long>[]>(firstLayer, secondLayer));
}
public static Tuple <IPropagatorBlock <long, long>, IPropagatorBlock <long, long> > GenerateLinearGraph(int cntNodes,
Func <long> f, out string creationTime)
{
Stopwatch sw = new Stopwatch();
sw.Start();
// creation
IPropagatorBlock <long, long>[] vertNodes = new IPropagatorBlock <long, long> [cntNodes];
vertNodes[0] = new BufferBlock <long>();
for (int i = 1; i < cntNodes; i++)
{
vertNodes[i] = new TransformBlock <long, long>(x => f()); // f = LongOperation() or some other
}
var linkOpt = new DataflowLinkOptions()
{
PropagateCompletion = true
};
// connections
for (int i = 1; i < cntNodes; i++)
{
vertNodes[i - 1].LinkTo(vertNodes[i], linkOpt);
}
sw.Stop();
//Console.WriteLine("Creation creationTime: " + sw.Elapsed);
const string format = "{0,10:F3}";
creationTime = string.Format(format, sw.Elapsed.Milliseconds);
return(new Tuple <IPropagatorBlock <long, long>, IPropagatorBlock <long, long> >(vertNodes[0], vertNodes[vertNodes.Length - 1]));
}
public RtpPipeBlock(RtpPipeOptions options) : base(options?.MiddlewareOptions)
{
responseBuffer = new BufferBlock <RtpMessage>(new DataflowBlockOptions
{
BoundedCapacity = options.ResponseBufferCapacity
});
}
public static Tuple <IPropagatorBlock <long, long>[], IPropagatorBlock <long, long>[]> Generate3AverageGraph(int depth,
Func <long[], long> f)
{
Stopwatch sw = new Stopwatch();
sw.Restart();
// create nodes
IPropagatorBlock <long, long>[] firstLayer = new IPropagatorBlock <long, long> [depth];
for (int i = 0; i < depth; i++)
{
firstLayer[i] = new BroadcastBlock <long>(x => x);
}
IPropagatorBlock <long, long>[] secondLayer = new IPropagatorBlock <long, long> [depth - 2];
for (int i = 0; i < depth - 2; i++)
{
secondLayer[i] = Create3AverageNode(f);
}
// link nodes - iterate over the 2nd layer
for (int i = 0; i < depth - 2; i++)
{
firstLayer[i].LinkTo(secondLayer[i]);
firstLayer[i + 1].LinkTo(secondLayer[i]);
firstLayer[i + 2].LinkTo(secondLayer[i]);
}
sw.Stop();
Console.WriteLine("Creation: {0}", sw.Elapsed);
return(new Tuple <IPropagatorBlock <long, long>[], IPropagatorBlock <long, long>[]>(firstLayer, secondLayer));
}
/// <summary>
/// Links source to a target block
/// </summary>
/// <typeparam name="TInput">The input</typeparam>
/// <typeparam name="TOutput">The output</typeparam>
/// <param name="source">The source</param>
/// <param name="target">The target</param>
/// <returns>ISourceBlock</returns>
public static ISourceBlock <TOutput> Next <TInput, TOutput>(
this ISourceBlock <TInput> source,
IPropagatorBlock <TInput, TOutput> target)
{
source.LinkTo(target);
return(target);
}
/// <summary>
/// Links to a propagator block.
/// </summary>
/// <typeparam name="TOutput"></typeparam>
/// <typeparam name="TOutput2"></typeparam>
/// <param name="builder"></param>
/// <param name="propagatorBlock"></param>
/// <param name="predicate"></param>
/// <param name="declinedTargetBlock"></param>
/// <returns></returns>
public static ISourceDataflowBuilder <TOutput2> LinkToPropagator <TOutput, TOutput2>(
this ISourceDataflowBuilder <TOutput> builder
, IPropagatorBlock <TOutput, TOutput2> propagatorBlock
, Predicate <TOutput> predicate
, ITargetBlock <TOutput> declinedTargetBlock = null)
{
return(builder.LinkToPropagator(propagatorBlock, DataflowDefaultOptions.DefaultLinkOptions, predicate, declinedTargetBlock, DataflowDefaultOptions.DefaultLinkOptions));
}
public IBuilder <TOrigin, TNewTarget> Then <TNewTarget>(IPropagatorBlock <TTarget, TNewTarget> block)
{
_current.LinkTo(block, new DataflowLinkOptions {
PropagateCompletion = true
});
return(new MiddleBuilder <TOrigin, TNewTarget>(_start, block));
}
public void Connect()
{
first = init.Value.Create();
last = slicer.Value.Create();
first.LinkTo(last, new DataflowLinkOptions {
PropagateCompletion = true
});
}
// generic way to trigger refresh (can be block delegate wrapper or Observable.Interval + scheduler)
#region Ctor
#region Overloads
/// <summary>
/// Initializes a new instance of the <see cref="PropogateHook{TIn, TOut}"/> class.
/// </summary>
/// <param name="local">The local.</param>
/// <param name="name">The name.</param>
/// <param name="visitor">Central information repository</param>
/// <param name="color"></param>
public PropogateHook(
IPropagatorBlock <TIn, TOut> local,
string name,
DataflowVisitor visitor,
Color?color = null)
: this(local, local, name, visitor, color)
{
}
public KeyProcessor(IActiveWord activeWord)
{
_activeWord = activeWord;
_incomingKeyPresses = new BroadcastBlock <KeyData>(ProcessKey);
_incomingCompletedWords = new TransformManyBlock <KeyData, CompletedWord>(ProcessWord);
_incomingKeyPresses.LinkTo(_incomingCompletedWords);
}
public Scraper(ScraperOptions options, Func <IDocument, Task <ScrapeResult> > transform = null)
{
transform = transform ?? Scrape;
_inner = new TransformBlock <IDocument, ScrapeResult>(transform, new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = -1
});
}
private TcpTransport(string host, int port)
{
_host = host;
_port = port;
_bufferBlock = new BufferBlock <string>();
_senderBlock = CreateSenderBlock();
_bufferBlock.LinkTo(_senderBlock);
}
protected override async Task StartTask(CancellationToken token)
{
_consumerBlock = new ActionBlock <T>(item => { OnItemAvailable?.Invoke(item); });
_propagationBlock = CreatePropagationBlock();
_propagationBlock.LinkTo(_consumerBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
await DoWork(token);
}
internal RetriableTransformBlock(IPropagatorBlock <TInput, TOutput> block)
{
if (block == null)
{
throw new ArgumentNullException("block");
}
_block = block;
}
public static ISourceBlock <TOutput> LinkWith <TInput, TOutput>(this ISourceBlock <TInput> sourceBlock,
IPropagatorBlock <TInput, TOutput> targetBlock)
{
sourceBlock.LinkTo(targetBlock,
new DataflowLinkOptions {
PropagateCompletion = true
});
return(targetBlock);
}
/// <summary>
/// Constructs a simple dataflow from an propagator block
/// </summary>
/// <typeparam name="TIn">The input type of the Dataflow</typeparam>
/// <typeparam name="TOut">The output type of the Dataflow</typeparam>
/// <param name="block">an propagator block to wrap</param>
/// <param name="name">Name of the new dataflow instance</param>
/// <returns>A dataflow that wraps the propagator block</returns>
public static Dataflow <TIn, TOut> FromBlock <TIn, TOut>(this IPropagatorBlock <TIn, TOut> block, string name = null)
{
var flow = new PropagatorDataflow <TIn, TOut>(block)
{
Name = name
};
return(flow);
}
public RecordPipeline(
Func <UnparsedRecord, bool> routingPredicate,
IPropagatorBlock <UnparsedRecord, Validity <ParsedRecord> > pipeline,
ISink <ParsedRecord> validRecordSink)
{
RoutingPredicate = routingPredicate;
Pipeline = pipeline;
ValidRecordSink = validRecordSink;
}
public DataFlowMessageBus()
{
_broadcast = new BroadcastBlock <T>(null);
_mailBox = new BufferBlock <T>();
_cleanup.Add(_mailBox.LinkTo(_broadcast, new DataflowLinkOptions
{
PropagateCompletion = true
}));
}
/// <summary>
/// Initializes a new instance of the <see cref="BatchLogTargetBase"/> class.
/// </summary>
/// <param name="batchSize">Size of the log batch.</param>
/// <param name="flushInterval">The flush interval.
/// The interval with which to initiate a batching operation even if the
/// number of currently queued logs is less than the <paramref name="batchSize"/>.
/// </param>
/// <param name="maxDegreeOfParallelism">The max degree of parallelism the target instance will log batch entries (ie. <see cref="Log"/> method).</param>
protected BatchLogTargetBase(Int32 batchSize, TimeSpan flushInterval, Int32 maxDegreeOfParallelism)
{
_FlushInterval = flushInterval;
_BatchBlock = new BatchBlock<LogEntry>(batchSize);
_ActionBlock = new ActionBlock<IEnumerable<LogEntry>>(
logEntries =>
{
Log(logEntries);
},
new ExecutionDataflowBlockOptions
{
MaxDegreeOfParallelism = maxDegreeOfParallelism
});
_BatchBlock.LinkTo(_ActionBlock);
_FlushTimer = new Timer(FlushTimerCallback, null, _FlushInterval, TimeSpan.FromMilliseconds(-1));
}
