public PipeLine(
DataflowOptions dataflowOptions,
int maxDegreeOfParallelism,
HttpClient client)
: base(dataflowOptions)
{
_client = client;
_dataflowOptions = dataflowOptions;
_options = dataflowOptions.ToExecutionBlockOption(true);
_options.MaxDegreeOfParallelism = maxDegreeOfParallelism;
_options.BoundedCapacity = 10000;
_options.MaxMessagesPerTask = 100;
_options.SingleProducerConstrained = true;
_requestBlock = new TransformBlock <string, HttpResponseMessage>(x =>
{
Interlocked.Increment(ref _totalOut);
Console.WriteLine("lines " + _totalOut);
var response = _client.GetAsync("http://localhost:3000/endpoint/info").Result;
return(default(HttpResponseMessage));
}, _options);
_flowBlock = new ActionBlock <HttpResponseMessage>(x =>
{
}, _options);
_request = _requestBlock.ToDataflow(_dataflowOptions, "Request");
_flow = _flowBlock.ToDataflow(_dataflowOptions, "Flow");
_request.LinkTo(_flow);
}
public CrawlersFlow(IEnumerable <Func <IReadOnlyList <string>, IReadOnlyList <IData> > > crawlers)
: base(DataflowOptions.Default)
{
if (crawlers is null)
{
throw new ArgumentNullException(nameof(crawlers));
}
_inputBroadcaster = new DataBroadcaster <IReadOnlyList <string> >(filteredData =>
{
Console.WriteLine($"Broadcasts all filtered inputters data. {filteredData.Count.ToString()}");
return(filteredData);
}, DataflowOptions.Default);
_resultConsumer = new TransformBlock <IReadOnlyList <IData>, IReadOnlyList <IData> >(
crawlersData => crawlersData
).ToDataflow(DataflowOptions.Default);
var crawlerFlows = crawlers
.Select(crawler => DataflowUtils.FromDelegate(crawler, DataflowOptions.Default));
foreach (var crawlerFlow in crawlerFlows)
{
_inputBroadcaster.LinkTo(crawlerFlow);
crawlerFlow.LinkTo(_resultConsumer);
_resultConsumer.RegisterDependency(crawlerFlow);
RegisterChild(crawlerFlow);
}
RegisterChild(_inputBroadcaster);
RegisterChild(_resultConsumer);
}
public OutputtersFlow(
IEnumerable <Action <IReadOnlyList <string> > > outputters)
: base(DataflowOptions.Default)
{
if (outputters is null)
{
throw new ArgumentNullException(nameof(outputters));
}
_resultList = new ConcurrentBag <string>();
_inputConsumer = new DataBroadcaster <IReadOnlyList <string> >(appraisersData =>
{
Console.WriteLine($"Consuming all appraised data. {appraisersData.Count.ToString()}\n");
foreach (string datum in appraisersData)
{
_resultList.Add(datum);
}
return(appraisersData);
}, DataflowOptions.Default);
var outputterFlows = outputters
.Select(outputters => DataflowUtils.FromDelegate(outputters, DataflowOptions.Default));
foreach (var outputterFlow in outputterFlows)
{
_inputConsumer.LinkTo(outputterFlow);
RegisterChild(outputterFlow);
}
RegisterChild(_inputConsumer);
}
public async Task Start()
{
if (_token.IsCancellationRequested)
{
return;
}
RegisterChild(_dispatcher);
var count = _buffer.BufferedCount;
// _log.Debug($"{count} streams in buffer", this);
if (count > 0)
{
var obs = _buffer.OutputBlock.AsObservable()
.Take(count).Select(async s =>
{
await _dispatcher.SendAsync(s);
if (!_token.IsCancellationRequested)
{
return(await s.Task);
}
return(false);
});
await obs.LastOrDefaultAsync();
}
_buffer.LinkTo(_dispatcher, s => !_token.IsCancellationRequested);
_buffer.LinkLeftToNull();
}
public MyDataflow() : base(DataflowOptions.Default)
{
m_block1 = new BufferBlock <T>().ToDataflow(name: "innerflow1");
m_block2 = new BatchBlock <T>(100).ToDataflow(name: "innerflow2");
m_block1.LinkTo(m_block2);
RegisterChild(this.m_block1);
RegisterChild(this.m_block2);
}
public CompletionDemo1()
{
broadCaster = new BroadcastBlock <int>(
i =>
{
return(i);
}).ToDataflow();
transformBlock1 = new TransformBlock <int, string>(
i =>
{
Console.WriteLine("1 input count: " + transformBlock1.InputCount);
Thread.Sleep(50);
return("1_" + i);
});
transformBlock2 = new TransformBlock <int, string>(
i =>
{
Console.WriteLine("2 input count: " + transformBlock2.InputCount);
Thread.Sleep(20);
return("2_" + i);
});
processor = new ActionBlock <string>(
i =>
{
Console.WriteLine(i);
}).ToDataflow();
/** rather than TPL linking
* broadCastBlock.LinkTo(transformBlock1, new DataflowLinkOptions { PropagateCompletion = true });
* broadCastBlock.LinkTo(transformBlock2, new DataflowLinkOptions { PropagateCompletion = true });
* transformBlock1.LinkTo(processorBlock, new DataflowLinkOptions { PropagateCompletion = true });
* transformBlock2.LinkTo(processorBlock, new DataflowLinkOptions { PropagateCompletion = true });
**/
//Use DataflowEx linking
var transform1 = transformBlock1.ToDataflow();
var transform2 = transformBlock2.ToDataflow();
broadCaster.LinkTo(transform1);
broadCaster.LinkTo(transform2);
transform1.LinkTo(processor);
transform2.LinkTo(processor);
}
public CompletionDemo1()
{
broadCaster = new BroadcastBlock<int>(
i =>
{
return i;
}).ToDataflow();
transformBlock1 = new TransformBlock<int, string>(
i =>
{
Console.WriteLine("1 input count: " + transformBlock1.InputCount);
Thread.Sleep(50);
return ("1_" + i);
});
transformBlock2 = new TransformBlock<int, string>(
i =>
{
Console.WriteLine("2 input count: " + transformBlock2.InputCount);
Thread.Sleep(20);
return ("2_" + i);
});
processor = new ActionBlock<string>(
i =>
{
Console.WriteLine(i);
}).ToDataflow();
/** rather than TPL linking
broadCastBlock.LinkTo(transformBlock1, new DataflowLinkOptions { PropagateCompletion = true });
broadCastBlock.LinkTo(transformBlock2, new DataflowLinkOptions { PropagateCompletion = true });
transformBlock1.LinkTo(processorBlock, new DataflowLinkOptions { PropagateCompletion = true });
transformBlock2.LinkTo(processorBlock, new DataflowLinkOptions { PropagateCompletion = true });
**/
//Use DataflowEx linking
var transform1 = transformBlock1.ToDataflow();
var transform2 = transformBlock2.ToDataflow();
broadCaster.LinkTo(transform1);
broadCaster.LinkTo(transform2);
transform1.LinkTo(processor);
transform2.LinkTo(processor);
}
public ProjectionBufferedDispatcher(DataflowOptions dataflowOptions, ProjectionBase <TState> projection)
: base(dataflowOptions)
{
_buffer = new BufferBlock <Tracked <IStream> >(dataflowOptions.ToDataflowBlockOptions(false)).ToDataflow(dataflowOptions);
_dispatcher = new ProjectionDispatcher <TState>(dataflowOptions, projection);
_log = projection.Log;
_token = projection.CancellationToken;
_token.Register(() => _buffer.LinkTo(DataflowBlock.NullTarget <Tracked <IStream> >().ToDataflow(dataflowOptions)));
RegisterChild(_buffer);
}
private void InitFlow(IEnumerable <Action <RatingDataContainer> > outputters)
{
var outputterFlows = outputters.Select(outputters =>
DataflowUtils.FromDelegate(outputters, DataflowOptions.Default)
);
foreach (Dataflow <RatingDataContainer> outputterFlow in outputterFlows)
{
_inputConsumer.LinkTo(outputterFlow);
RegisterChild(outputterFlow);
}
RegisterChild(_inputConsumer);
}
public ComplexIntFlow() : base(DataflowOptions.Default)
{
Dataflow <int, int> node2 = DataflowUtils.FromDelegate <int, int>(i => i);
Dataflow <int, int> node3 = DataflowUtils.FromDelegate <int, int>(i => i * -1);
Dataflow <int, int> node1 = DataflowUtils.FromDelegate <int, int>(
i => {
if (i % 2 == 0)
{
node2.Post(i);
}
else
{
node3.Post(i);
}
return(999);
});
Dataflow <int> printer = DataflowUtils.FromDelegate <int>(Console.WriteLine);
node1.Name = "node1";
node2.Name = "node2";
node3.Name = "node3";
printer.Name = "printer";
this.RegisterChild(node1);
this.RegisterChild(node2);
this.RegisterChild(node3);
this.RegisterChild(printer, t =>
{
if (t.Status == TaskStatus.RanToCompletion)
{
Console.WriteLine("Printer done!");
}
});
node1.LinkTo(printer);
node2.LinkTo(printer);
node3.LinkTo(printer);
//Completion propagation: node1 ---> node2
node2.RegisterDependency(node1);
//Completion propagation: node1 + node2 ---> node3
node3.RegisterDependency(node1);
node3.RegisterDependency(node2);
this._headBlock = node1.InputBlock;
}
public SlowFlow(DataflowOptions dataflowOptions)
: base(dataflowOptions)
{
_splitter = new TransformManyBlock <string, char>(new Func <string, IEnumerable <char> >(this.SlowSplit),
dataflowOptions.ToExecutionBlockOption())
.ToDataflow(dataflowOptions, "SlowSplitter");
_printer = new ActionBlock <char>(c => Console.WriteLine(c),
dataflowOptions.ToExecutionBlockOption())
.ToDataflow(dataflowOptions, "Printer");
RegisterChild(_splitter);
RegisterChild(_printer);
_splitter.LinkTo(_printer);
}
/// <summary>
/// Constructs a DbDataJoiner instance
/// </summary>
/// <param name="joinOn">Property path from the root object down to the lookup key</param>
/// <param name="dimTableTarget">Table information of the remote dimension table</param>
/// <param name="option">Option to use for this dataflow</param>
/// <param name="batchSize">The batch size for a batched remote look up</param>
/// <param name="cacheSize">The local cache item count (part of the remote table)</param>
public DbDataJoiner(Expression <Func <TIn, TLookupKey> > joinOn, TargetTable dimTableTarget, DataflowOptions option, int batchSize = 8 * 1024, int cacheSize = 1024 * 1024)
: base(option)
{
m_dimTableTarget = dimTableTarget;
m_batchSize = batchSize;
m_batcher = new BatchBlock <TIn>(batchSize, option.ToGroupingBlockOption()).ToDataflow(option);
m_batcher.Name = "Batcher";
m_lookupNode = new TransformManyDataflow <JoinBatch <TIn>, TIn>(this.JoinBatch, option);
m_lookupNode.Name = "LookupNode";
m_typeAccessor = TypeAccessorManager <TIn> .GetAccessorForTable(dimTableTarget);
m_keyComparer = typeof(TLookupKey) == typeof(byte[])
? (IEqualityComparer <TLookupKey>)((object)new ByteArrayEqualityComparer())
: EqualityComparer <TLookupKey> .Default;
m_rowCache = new RowCache <TLookupKey>(cacheSize, m_keyComparer);
m_logger = Utils.GetNamespaceLogger();
m_joinOnMapping = m_typeAccessor.DbColumnMappings.First(m => m.Host.PropertyInfo == this.ExtractPropertyInfo(joinOn));
var transformer =
new TransformBlock <TIn[], JoinBatch <TIn> >(
array => new JoinBatch <TIn>(array, CacheLookupStrategy.RemoteLookup), option.ToExecutionBlockOption()).ToDataflow(option);
transformer.Name = "ArrayToJoinBatchConverter";
m_batcher.LinkTo(transformer);
transformer.LinkTo(m_lookupNode);
RegisterChild(m_batcher);
RegisterChild(transformer);
RegisterChild(m_lookupNode);
m_dimInserter = new DimTableInserter(this, dimTableTarget, joinOn, option)
{
Name = "DimInserter"
};
var hb = new HeartbeatNode <JoinBatch <TIn> >(option);
m_dimInserter.RegisterDependency(m_lookupNode);
m_dimInserter.LinkTo(hb);
hb.LinkTo(m_lookupNode);
RegisterChild(m_dimInserter);
RegisterChild(hb);
RegisterChildRing(transformer.CompletionTask, m_lookupNode, m_dimInserter, hb);
}
public CircularFlow(DataflowOptions dataflowOptions) : base(dataflowOptions)
{
//a no-op node to demonstrate the usage of preTask param in RegisterChildRing
_buffer = new BufferBlock <int>().ToDataflow(name: "NoOpBuffer");
var heater = new TransformManyDataflow <int, int>(async i =>
{
await Task.Delay(200);
Console.WriteLine("Heated to {0}", i + 1);
return(new [] { i + 1 });
}, dataflowOptions);
heater.Name = "Heater";
var cooler = new TransformManyDataflow <int, int>(async i =>
{
await Task.Delay(200);
int cooled = i - 2;
Console.WriteLine("Cooled to {0}", cooled);
if (cooled < 0) //time to stop
{
return(Enumerable.Empty <int>());
}
return(new [] { cooled });
}, dataflowOptions);
cooler.Name = "Cooler";
var heartbeat = new HeartbeatNode <int>(dataflowOptions)
{
Name = "HeartBeat"
};
_buffer.LinkTo(heater);
//circular
heater.LinkTo(cooler);
cooler.LinkTo(heartbeat);
heartbeat.LinkTo(heater);
RegisterChildren(_buffer, heater, cooler, heartbeat);
//ring registration
RegisterChildRing(_buffer.Completion, heater, cooler, heartbeat);
}
private void InitFlow(IEnumerable <Func <string, IEnumerable <string> > > inputters)
{
var inputFlows = inputters.Select(inputter =>
DataflowUtils.FromDelegate(inputter, DataflowOptions.Default)
);
foreach (Dataflow <string, string> inputFlow in inputFlows)
{
_inputBroadcaster.LinkTo(inputFlow);
inputFlow.LinkTo(_resultTransformer, FilterInputData);
_resultTransformer.RegisterDependency(inputFlow);
RegisterChild(inputFlow);
}
RegisterChild(_inputBroadcaster);
RegisterChild(_resultTransformer);
}
private void InitFlow(IEnumerable <Func <string, IAsyncEnumerable <BasicInfo> > > crawlers)
{
var crawlerFlows = crawlers.Select(crawler =>
new TransformManyBlock <string, BasicInfo>(
async entity => await crawler(entity).AsEnumerable().LogIfErrorOccured()
).ToDataflow(DataflowOptions.Default)
);
foreach (Dataflow <string, BasicInfo> crawlerFlow in crawlerFlows)
{
_inputBroadcaster.LinkTo(crawlerFlow);
crawlerFlow.LinkTo(_resultConsumer);
_resultConsumer.RegisterDependency(crawlerFlow);
RegisterChild(crawlerFlow);
}
RegisterChild(_inputBroadcaster);
RegisterChild(_resultConsumer);
}
public InputtersFlow(IEnumerable <Func <string, IReadOnlyList <string> > > inputters)
: base(DataflowOptions.Default)
{
if (inputters is null)
{
throw new ArgumentNullException(nameof(inputters));
}
_filteringSet = new ConcurrentDictionary <string, byte>();
_inputBroadcaster = new DataBroadcaster <string>(input =>
{
Console.WriteLine("Broadcasts input to further blocks.");
return(input);
}, DataflowOptions.Default);
_resultTransformer =
DataflowUtils.FromDelegate <IReadOnlyList <string>, IReadOnlyList <string> >(
FilterInputData, DataflowOptions.Default
);
var inputFlows = inputters
.Select(inputter => DataflowUtils.FromDelegate(inputter, DataflowOptions.Default));
foreach (var inputFlow in inputFlows)
{
_inputBroadcaster.LinkTo(inputFlow);
inputFlow.LinkTo(_resultTransformer);
_resultTransformer.RegisterDependency(inputFlow);
RegisterChild(inputFlow);
}
RegisterChild(_inputBroadcaster);
RegisterChild(_resultTransformer);
}
private void InitFlow(IEnumerable <Funcotype> appraisers)
{
var usedTypes = new Dictionary <Type, DataBroadcaster <BasicInfo> >();
foreach (Funcotype appraiser in appraisers)
{
if (!usedTypes.TryGetValue(appraiser.DataType, out var broadcaster))
{
broadcaster = new DataBroadcaster <BasicInfo>(
crawlersData => crawlersData, DataflowOptions.Default
);
usedTypes.Add(appraiser.DataType, broadcaster);
_inputConsumer.TransformAndLink(
broadcaster,
info => info,
info => info.GetType().IsAssignableFrom(appraiser.DataType)
);
RegisterChild(broadcaster);
}
Dataflow <BasicInfo, RatingDataContainer> appraiserFlow = DataflowUtils.FromDelegate(
appraiser.Func, DataflowOptions.Default
);
broadcaster.LinkTo(appraiserFlow);
appraiserFlow.LinkTo(_resultConsumer);
_resultConsumer.RegisterDependency(appraiserFlow);
RegisterChild(appraiserFlow);
}
RegisterChild(_inputConsumer);
RegisterChild(_resultConsumer);
}
// Constructor
public CalculateAndStoreFromInputAndAsyncTerms(CalculateAndStoreFromInputAndAsyncTermsObservableData calculateAndStoreFromInputAndAsyncTermsObservableData, IWebGet webGet, CalculateAndStoreFromInputAndAsyncTermsOptions calculateAndStoreFromInputAndAsyncTermsOptions) : base(calculateAndStoreFromInputAndAsyncTermsOptions)
{
Log.Trace("Constructor starting");
_calculateAndStoreFromInputAndAsyncTermsObservableData = calculateAndStoreFromInputAndAsyncTermsObservableData;
_webGet = webGet;
_calculateAndStoreFromInputAndAsyncTermsOptions = calculateAndStoreFromInputAndAsyncTermsOptions;
// Create a place to store the TransientBuffers that buffer messages while waiting For all elements that make up the ElementSets Of Term1 to finish fetching
_transientBuffersForElementSetsOfTerm1 = new ConcurrentDictionary <string, Dataflow <IInternalMessage <string>, IInternalMessage <string> > >();
// foreach IInputMessage<TKeyTerm1,TValueTerm1>, create an internal message that adds the hashset for the terms1 and the bool used by the routing predicate
Log.Trace("Creating _bAccepter");
_bAccepter = new TransformBlock <IInputMessage <string, double>, IInternalMessage <string> >(_input => { Log.Trace("Accepter received IInputMessage");
// ToDo also check on the async tasks check when an upstream completion occurs
// ToDo add exception handling to ensure the tasks, as well as the async method's resources, are released if any blocks in the dataflow fault
// create a HashSet from the set of keys found in terms1
KeySignature <string> sig = new KeySignature <string>(_input.Value.terms1.Keys);
// Is the sig.largest in the ElementSetsOfTerm1Ready dictionary? set the output bool accordingly
bool isReadyToCalculate = _calculateAndStoreFromInputAndAsyncTermsObservableData.ElementSetsOfTerm1Ready.ContainsKey(sig.Longest());
// Pass the message along to the next block, which will be either the _bSolveStore, or the _bDynamicBuffers
return(new InternalMessage <string>((_input.Value.k1, _input.Value.k2, _input.Value.terms1, sig, isReadyToCalculate))); }).ToDataflow();
// this block accepts messages where isReadyToCalculate is false, and buffers them
Log.Trace("Creating _bDynamicBuffers");
_bDynamicBuffers = new DynamicBuffers(this);
// The terminator block performs both the Solve and the Store operations
Log.Trace("Creating _bSolveStore");
_bSolveStore = new ActionBlock <IInternalMessage <string> >(_input => { Log.Trace($"_bSolveStore received InternalMessage having signature {_input.Value.sig.Longest()}");
// solve the equation for the input and all terms
var r1 = 0.0;
foreach (var kvp in _input.Value.terms1)
{
r1 += kvp.Value /
_calculateAndStoreFromInputAndAsyncTermsObservableData.FetchedIndividualElementsOfTerm1[kvp.Key];
}
// Store the results value
calculateAndStoreFromInputAndAsyncTermsObservableData.RecordR(_input.Value.k1,
_input.Value.k2,
Convert.ToDecimal(r1)); }).ToDataflow(calculateAndStoreFromInputAndAsyncTermsOptions);
#region create asyncFetchCheckTimer and connect callback
// Create a timer that is used to check on the async fetch tasks, the async fetch tasks check-for-completion loop timer
// the timer has its interval from the options passed into this constructor, it will restart and the event handler will stop the timer and start the timer each time
// ToDo add the timer that checks on the health of the async fetch tasks check-for-completion loop every DefaultAsyncFetchTimeout interval, expecting it to provide a heartbeat,
// The Cleanup method will call this timers Dispose method
// the event handler's job is to call CheckAsyncTasks which will check for completed fetches and link the child Transient buffers to the _bSolveStore
Log.Trace("creating and starting the asyncFetchCheckTimer");
asyncFetchCheckTimer = new Timer(_calculateAndStoreFromInputAndAsyncTermsOptions.AsyncFetchTimeInterval.TotalMilliseconds);
asyncFetchCheckTimer.AutoReset = true;
// set the event handler (callback) for this timer to the function for async fetch tasks check-for-completion loop
asyncFetchCheckTimer.Elapsed += new ElapsedEventHandler(asyncFetchCheckTimer_Elapsed);
asyncFetchCheckTimer.Start();
#endregion create asyncFetchCheckTimer and connect callback
_bAccepter.Name = "_bAccepter";
_bSolveStore.Name = "_bSolveStore";
_bDynamicBuffers.Name = "_bDynamicBuffers";
// Link the data flow
Log.Trace("Linking dataflow between blocks");
// Link _bAccepter to _bSolveStore when the InternalMessage.Value has isReadyToCalculate = true
_bAccepter.LinkTo(_bSolveStore,
internalMessage => internalMessage.Value.isReadyToCalculate);
// Link _bAccepter to _bDynamicBuffers when the InternalMessage.Value has isReadyToCalculate = false
_bAccepter.LinkTo(_bDynamicBuffers,
internalMessage => !internalMessage.Value.isReadyToCalculate);
// data flow linkage of the dynamically created TransientBuffer children to the _bSolveStore is complex and handled elsewhere
// Link the completion tasks
Log.Trace("Linking completion between blocks");
_bDynamicBuffers.RegisterDependency(_bAccepter);
_bSolveStore.RegisterDependency(_bAccepter);
_bSolveStore.RegisterDependency(_bDynamicBuffers);
// Completion linkage of the dynamically created TransientBuffer children to the _bSolveStore is complex and handled elsewhere
Log.Trace("Registering Children");
this.RegisterChild(_bAccepter);
this.RegisterChild(_bSolveStore);
this.RegisterChild(_bDynamicBuffers);
// set the InputBlock for this dataflow graph to be the InputBlock of the _acceptor
this._headBlock = _bAccepter.InputBlock;
// ToDo: add an optional constructor parameter that supplies an initial list of elements that can start pre-fetching for each term
Log.Trace("Constructor Finished");
}
