public ComplexIntFlow() : base(DataflowOptions.Default)
{
Dataflow <int, int> node2 = DataflowUtils.FromDelegate <int, int>(i => i);
Dataflow <int, int> node3 = DataflowUtils.FromDelegate <int, int>(i => i * -1);
Dataflow <int, int> node1 = DataflowUtils.FromDelegate <int, int>(
i => {
if (i % 2 == 0)
{
node2.Post(i);
}
else
{
node3.Post(i);
}
return(999);
});
Dataflow <int> printer = DataflowUtils.FromDelegate <int>(Console.WriteLine);
node1.Name = "node1";
node2.Name = "node2";
node3.Name = "node3";
printer.Name = "printer";
this.RegisterChild(node1);
this.RegisterChild(node2);
this.RegisterChild(node3);
this.RegisterChild(printer, t =>
{
if (t.Status == TaskStatus.RanToCompletion)
{
Console.WriteLine("Printer done!");
}
});
node1.LinkTo(printer);
node2.LinkTo(printer);
node3.LinkTo(printer);
//Completion propagation: node1 ---> node2
node2.RegisterDependency(node1);
//Completion propagation: node1 + node2 ---> node3
node3.RegisterDependency(node1);
node3.RegisterDependency(node2);
this._headBlock = node1.InputBlock;
}
public CrawlersFlow(IEnumerable <Func <IReadOnlyList <string>, IReadOnlyList <IData> > > crawlers)
: base(DataflowOptions.Default)
{
if (crawlers is null)
{
throw new ArgumentNullException(nameof(crawlers));
}
_inputBroadcaster = new DataBroadcaster <IReadOnlyList <string> >(filteredData =>
{
Console.WriteLine($"Broadcasts all filtered inputters data. {filteredData.Count.ToString()}");
return(filteredData);
}, DataflowOptions.Default);
_resultConsumer = new TransformBlock <IReadOnlyList <IData>, IReadOnlyList <IData> >(
crawlersData => crawlersData
).ToDataflow(DataflowOptions.Default);
var crawlerFlows = crawlers
.Select(crawler => DataflowUtils.FromDelegate(crawler, DataflowOptions.Default));
foreach (var crawlerFlow in crawlerFlows)
{
_inputBroadcaster.LinkTo(crawlerFlow);
crawlerFlow.LinkTo(_resultConsumer);
_resultConsumer.RegisterDependency(crawlerFlow);
RegisterChild(crawlerFlow);
}
RegisterChild(_inputBroadcaster);
RegisterChild(_resultConsumer);
}
public AppraisersFlow(IEnumerable <Appraiser> appraisers)
: base(DataflowOptions.Default)
{
if (appraisers is null)
{
throw new ArgumentNullException(nameof(appraisers));
}
_inputConsumer = new DataBroadcaster <IReadOnlyList <IData> >(crawlersData =>
{
Console.WriteLine($"Broadcasts all crawlers data. {crawlersData.Count.ToString()}");
return(crawlersData);
}, DataflowOptions.Default);
_resultConsumer = new TransformBlock <IReadOnlyList <string>, IReadOnlyList <string> >(
appraisedData => appraisedData
).ToDataflow(DataflowOptions.Default);
var usedTypes = new Dictionary <Type, DataBroadcaster <IReadOnlyList <IData> > >();
foreach (var appraiser in appraisers)
{
if (!usedTypes.TryGetValue(appraiser.DataType, out var broadcaster))
{
broadcaster = new DataBroadcaster <IReadOnlyList <IData> >(crawlersData =>
{
Console.WriteLine($"Broadcasts specified data of type {appraiser.DataType.Name}. {crawlersData.Count.ToString()}");
return(crawlersData);
}, DataflowOptions.Default);
usedTypes.Add(appraiser.DataType, broadcaster);
_inputConsumer.TransformAndLink(
broadcaster,
l => l,
l => l.All(d => d.GetType().IsAssignableFrom(appraiser.DataType))
);
RegisterChild(broadcaster);
}
var appraiserFlow = DataflowUtils.FromDelegate(appraiser.Func, DataflowOptions.Default);
broadcaster.LinkTo(appraiserFlow);
appraiserFlow.LinkTo(_resultConsumer);
_resultConsumer.RegisterDependency(appraiserFlow);
RegisterChild(appraiserFlow);
}
RegisterChild(_inputConsumer);
RegisterChild(_resultConsumer);
}
private void InitFlow(IEnumerable <Func <string, IEnumerable <string> > > inputters)
{
var inputFlows = inputters.Select(inputter =>
DataflowUtils.FromDelegate(inputter, DataflowOptions.Default)
);
foreach (Dataflow <string, string> inputFlow in inputFlows)
{
_inputBroadcaster.LinkTo(inputFlow);
inputFlow.LinkTo(_resultTransformer, FilterInputData);
_resultTransformer.RegisterDependency(inputFlow);
RegisterChild(inputFlow);
}
RegisterChild(_inputBroadcaster);
RegisterChild(_resultTransformer);
}
private void InitFlow(IEnumerable <Func <string, IAsyncEnumerable <BasicInfo> > > crawlers)
{
var crawlerFlows = crawlers.Select(crawler =>
new TransformManyBlock <string, BasicInfo>(
async entity => await crawler(entity).AsEnumerable().LogIfErrorOccured()
).ToDataflow(DataflowOptions.Default)
);
foreach (Dataflow <string, BasicInfo> crawlerFlow in crawlerFlows)
{
_inputBroadcaster.LinkTo(crawlerFlow);
crawlerFlow.LinkTo(_resultConsumer);
_resultConsumer.RegisterDependency(crawlerFlow);
RegisterChild(crawlerFlow);
}
RegisterChild(_inputBroadcaster);
RegisterChild(_resultConsumer);
}
public DimTableInserter(DbDataJoiner <TIn, TLookupKey> host, TargetTable targetTable, Expression <Func <TIn, TLookupKey> > joinBy, DataflowOptions option)
: base(targetTable, option, host.m_batchSize)
{
this.m_host = host;
m_keyGetter = joinBy.Compile();
m_keyComparer = m_host.m_keyComparer;
m_outputBuffer = new BufferBlock <JoinBatch <TIn> >(option.ToGroupingBlockOption()).ToDataflow(option);
m_outputBuffer.Name = "OutputBuffer";
RegisterChild(m_outputBuffer);
m_outputBuffer.RegisterDependency(m_actionBlock);
m_tmpTargetTable = new TargetTable(
targetTable.DestLabel,
targetTable.ConnectionString,
targetTable.TableName + "_tmp");
//create tmp table
m_createTmpTable = string.Format(
"if OBJECT_ID('{0}', 'U') is not null drop table {0};"
+ "select * into {0} from {1} where 0 = 1",
this.m_tmpTargetTable.TableName,
targetTable.TableName);
//merge
m_mergeTmpToDimTable =
string.Format(
"MERGE INTO {0} as TGT "
+ "USING {1} as SRC on TGT.[{2}] = SRC.[{2}] "
+ "WHEN MATCHED THEN UPDATE SET TGT.[{2}] = TGT.[{2}] "
+ "WHEN NOT MATCHED THEN INSERT {3} VALUES {4} "
+ "OUTPUT inserted.[{5}], inserted.[{2}] ;",
targetTable.TableName,
this.m_tmpTargetTable.TableName,
m_host.m_joinOnMapping.DestColumnName,
Utils.FlattenColumnNames(m_typeAccessor.SchemaTable.Columns, ""),
Utils.FlattenColumnNames(m_typeAccessor.SchemaTable.Columns, "SRC"),
Utils.GetAutoIncrementColumn(m_typeAccessor.SchemaTable.Columns)
);
}
private void InitFlow(IEnumerable <Funcotype> appraisers)
{
var usedTypes = new Dictionary <Type, DataBroadcaster <BasicInfo> >();
foreach (Funcotype appraiser in appraisers)
{
if (!usedTypes.TryGetValue(appraiser.DataType, out var broadcaster))
{
broadcaster = new DataBroadcaster <BasicInfo>(
crawlersData => crawlersData, DataflowOptions.Default
);
usedTypes.Add(appraiser.DataType, broadcaster);
_inputConsumer.TransformAndLink(
broadcaster,
info => info,
info => info.GetType().IsAssignableFrom(appraiser.DataType)
);
RegisterChild(broadcaster);
}
Dataflow <BasicInfo, RatingDataContainer> appraiserFlow = DataflowUtils.FromDelegate(
appraiser.Func, DataflowOptions.Default
);
broadcaster.LinkTo(appraiserFlow);
appraiserFlow.LinkTo(_resultConsumer);
_resultConsumer.RegisterDependency(appraiserFlow);
RegisterChild(appraiserFlow);
}
RegisterChild(_inputConsumer);
RegisterChild(_resultConsumer);
}
public InputtersFlow(IEnumerable <Func <string, IReadOnlyList <string> > > inputters)
: base(DataflowOptions.Default)
{
if (inputters is null)
{
throw new ArgumentNullException(nameof(inputters));
}
_filteringSet = new ConcurrentDictionary <string, byte>();
_inputBroadcaster = new DataBroadcaster <string>(input =>
{
Console.WriteLine("Broadcasts input to further blocks.");
return(input);
}, DataflowOptions.Default);
_resultTransformer =
DataflowUtils.FromDelegate <IReadOnlyList <string>, IReadOnlyList <string> >(
FilterInputData, DataflowOptions.Default
);
var inputFlows = inputters
.Select(inputter => DataflowUtils.FromDelegate(inputter, DataflowOptions.Default));
foreach (var inputFlow in inputFlows)
{
_inputBroadcaster.LinkTo(inputFlow);
inputFlow.LinkTo(_resultTransformer);
_resultTransformer.RegisterDependency(inputFlow);
RegisterChild(inputFlow);
}
RegisterChild(_inputBroadcaster);
RegisterChild(_resultTransformer);
}
// Constructor
public CalculateAndStoreFromInputAndAsyncTerms(CalculateAndStoreFromInputAndAsyncTermsObservableData calculateAndStoreFromInputAndAsyncTermsObservableData, IWebGet webGet, CalculateAndStoreFromInputAndAsyncTermsOptions calculateAndStoreFromInputAndAsyncTermsOptions) : base(calculateAndStoreFromInputAndAsyncTermsOptions)
{
Log.Trace("Constructor starting");
_calculateAndStoreFromInputAndAsyncTermsObservableData = calculateAndStoreFromInputAndAsyncTermsObservableData;
_webGet = webGet;
_calculateAndStoreFromInputAndAsyncTermsOptions = calculateAndStoreFromInputAndAsyncTermsOptions;
// Create a place to store the TransientBuffers that buffer messages while waiting For all elements that make up the ElementSets Of Term1 to finish fetching
_transientBuffersForElementSetsOfTerm1 = new ConcurrentDictionary <string, Dataflow <IInternalMessage <string>, IInternalMessage <string> > >();
// foreach IInputMessage<TKeyTerm1,TValueTerm1>, create an internal message that adds the hashset for the terms1 and the bool used by the routing predicate
Log.Trace("Creating _bAccepter");
_bAccepter = new TransformBlock <IInputMessage <string, double>, IInternalMessage <string> >(_input => { Log.Trace("Accepter received IInputMessage");
// ToDo also check on the async tasks check when an upstream completion occurs
// ToDo add exception handling to ensure the tasks, as well as the async method's resources, are released if any blocks in the dataflow fault
// create a HashSet from the set of keys found in terms1
KeySignature <string> sig = new KeySignature <string>(_input.Value.terms1.Keys);
// Is the sig.largest in the ElementSetsOfTerm1Ready dictionary? set the output bool accordingly
bool isReadyToCalculate = _calculateAndStoreFromInputAndAsyncTermsObservableData.ElementSetsOfTerm1Ready.ContainsKey(sig.Longest());
// Pass the message along to the next block, which will be either the _bSolveStore, or the _bDynamicBuffers
return(new InternalMessage <string>((_input.Value.k1, _input.Value.k2, _input.Value.terms1, sig, isReadyToCalculate))); }).ToDataflow();
// this block accepts messages where isReadyToCalculate is false, and buffers them
Log.Trace("Creating _bDynamicBuffers");
_bDynamicBuffers = new DynamicBuffers(this);
// The terminator block performs both the Solve and the Store operations
Log.Trace("Creating _bSolveStore");
_bSolveStore = new ActionBlock <IInternalMessage <string> >(_input => { Log.Trace($"_bSolveStore received InternalMessage having signature {_input.Value.sig.Longest()}");
// solve the equation for the input and all terms
var r1 = 0.0;
foreach (var kvp in _input.Value.terms1)
{
r1 += kvp.Value /
_calculateAndStoreFromInputAndAsyncTermsObservableData.FetchedIndividualElementsOfTerm1[kvp.Key];
}
// Store the results value
calculateAndStoreFromInputAndAsyncTermsObservableData.RecordR(_input.Value.k1,
_input.Value.k2,
Convert.ToDecimal(r1)); }).ToDataflow(calculateAndStoreFromInputAndAsyncTermsOptions);
#region create asyncFetchCheckTimer and connect callback
// Create a timer that is used to check on the async fetch tasks, the async fetch tasks check-for-completion loop timer
// the timer has its interval from the options passed into this constructor, it will restart and the event handler will stop the timer and start the timer each time
// ToDo add the timer that checks on the health of the async fetch tasks check-for-completion loop every DefaultAsyncFetchTimeout interval, expecting it to provide a heartbeat,
// The Cleanup method will call this timers Dispose method
// the event handler's job is to call CheckAsyncTasks which will check for completed fetches and link the child Transient buffers to the _bSolveStore
Log.Trace("creating and starting the asyncFetchCheckTimer");
asyncFetchCheckTimer = new Timer(_calculateAndStoreFromInputAndAsyncTermsOptions.AsyncFetchTimeInterval.TotalMilliseconds);
asyncFetchCheckTimer.AutoReset = true;
// set the event handler (callback) for this timer to the function for async fetch tasks check-for-completion loop
asyncFetchCheckTimer.Elapsed += new ElapsedEventHandler(asyncFetchCheckTimer_Elapsed);
asyncFetchCheckTimer.Start();
#endregion create asyncFetchCheckTimer and connect callback
_bAccepter.Name = "_bAccepter";
_bSolveStore.Name = "_bSolveStore";
_bDynamicBuffers.Name = "_bDynamicBuffers";
// Link the data flow
Log.Trace("Linking dataflow between blocks");
// Link _bAccepter to _bSolveStore when the InternalMessage.Value has isReadyToCalculate = true
_bAccepter.LinkTo(_bSolveStore,
internalMessage => internalMessage.Value.isReadyToCalculate);
// Link _bAccepter to _bDynamicBuffers when the InternalMessage.Value has isReadyToCalculate = false
_bAccepter.LinkTo(_bDynamicBuffers,
internalMessage => !internalMessage.Value.isReadyToCalculate);
// data flow linkage of the dynamically created TransientBuffer children to the _bSolveStore is complex and handled elsewhere
// Link the completion tasks
Log.Trace("Linking completion between blocks");
_bDynamicBuffers.RegisterDependency(_bAccepter);
_bSolveStore.RegisterDependency(_bAccepter);
_bSolveStore.RegisterDependency(_bDynamicBuffers);
// Completion linkage of the dynamically created TransientBuffer children to the _bSolveStore is complex and handled elsewhere
Log.Trace("Registering Children");
this.RegisterChild(_bAccepter);
this.RegisterChild(_bSolveStore);
this.RegisterChild(_bDynamicBuffers);
// set the InputBlock for this dataflow graph to be the InputBlock of the _acceptor
this._headBlock = _bAccepter.InputBlock;
// ToDo: add an optional constructor parameter that supplies an initial list of elements that can start pre-fetching for each term
Log.Trace("Constructor Finished");
}
