/// <summary> /// Constructs a DbDataJoiner instance /// </summary> /// <param name="joinOn">Property path from the root object down to the lookup key</param> /// <param name="dimTableTarget">Table information of the remote dimension table</param> /// <param name="option">Option to use for this dataflow</param> /// <param name="batchSize">The batch size for a batched remote look up</param> /// <param name="cacheSize">The local cache item count (part of the remote table)</param> public DbDataJoiner(Expression <Func <TIn, TLookupKey> > joinOn, TargetTable dimTableTarget, DataflowOptions option, int batchSize = 8 * 1024, int cacheSize = 1024 * 1024) : base(option) { m_dimTableTarget = dimTableTarget; m_batchSize = batchSize; m_batcher = new BatchBlock <TIn>(batchSize, option.ToGroupingBlockOption()).ToDataflow(option); m_batcher.Name = "Batcher"; m_lookupNode = new TransformManyDataflow <JoinBatch <TIn>, TIn>(this.JoinBatch, option); m_lookupNode.Name = "LookupNode"; m_typeAccessor = TypeAccessorManager <TIn> .GetAccessorForTable(dimTableTarget); m_keyComparer = typeof(TLookupKey) == typeof(byte[]) ? (IEqualityComparer <TLookupKey>)((object)new ByteArrayEqualityComparer()) : EqualityComparer <TLookupKey> .Default; m_rowCache = new RowCache <TLookupKey>(cacheSize, m_keyComparer); m_logger = Utils.GetNamespaceLogger(); m_joinOnMapping = m_typeAccessor.DbColumnMappings.First(m => m.Host.PropertyInfo == this.ExtractPropertyInfo(joinOn)); var transformer = new TransformBlock <TIn[], JoinBatch <TIn> >( array => new JoinBatch <TIn>(array, CacheLookupStrategy.RemoteLookup), option.ToExecutionBlockOption()).ToDataflow(option); transformer.Name = "ArrayToJoinBatchConverter"; m_batcher.LinkTo(transformer); transformer.LinkTo(m_lookupNode); RegisterChild(m_batcher); RegisterChild(transformer); RegisterChild(m_lookupNode); m_dimInserter = new DimTableInserter(this, dimTableTarget, joinOn, option) { Name = "DimInserter" }; var hb = new HeartbeatNode <JoinBatch <TIn> >(option); m_dimInserter.RegisterDependency(m_lookupNode); m_dimInserter.LinkTo(hb); hb.LinkTo(m_lookupNode); RegisterChild(m_dimInserter); RegisterChild(hb); RegisterChildRing(transformer.CompletionTask, m_lookupNode, m_dimInserter, hb); }
public CircularFlow(DataflowOptions dataflowOptions) : base(dataflowOptions) { //a no-op node to demonstrate the usage of preTask param in RegisterChildRing _buffer = new BufferBlock <int>().ToDataflow(name: "NoOpBuffer"); var heater = new TransformManyDataflow <int, int>(async i => { await Task.Delay(200); Console.WriteLine("Heated to {0}", i + 1); return(new [] { i + 1 }); }, dataflowOptions); heater.Name = "Heater"; var cooler = new TransformManyDataflow <int, int>(async i => { await Task.Delay(200); int cooled = i - 2; Console.WriteLine("Cooled to {0}", cooled); if (cooled < 0) //time to stop { return(Enumerable.Empty <int>()); } return(new [] { cooled }); }, dataflowOptions); cooler.Name = "Cooler"; var heartbeat = new HeartbeatNode <int>(dataflowOptions) { Name = "HeartBeat" }; _buffer.LinkTo(heater); //circular heater.LinkTo(cooler); cooler.LinkTo(heartbeat); heartbeat.LinkTo(heater); RegisterChildren(_buffer, heater, cooler, heartbeat); //ring registration RegisterChildRing(_buffer.Completion, heater, cooler, heartbeat); }