public PipeLine(
DataflowOptions dataflowOptions,
int maxDegreeOfParallelism,
HttpClient client)
: base(dataflowOptions)
{
_client = client;
_dataflowOptions = dataflowOptions;
_options = dataflowOptions.ToExecutionBlockOption(true);
_options.MaxDegreeOfParallelism = maxDegreeOfParallelism;
_options.BoundedCapacity = 10000;
_options.MaxMessagesPerTask = 100;
_options.SingleProducerConstrained = true;
_requestBlock = new TransformBlock <string, HttpResponseMessage>(x =>
{
Interlocked.Increment(ref _totalOut);
Console.WriteLine("lines " + _totalOut);
var response = _client.GetAsync("http://localhost:3000/endpoint/info").Result;
return(default(HttpResponseMessage));
}, _options);
_flowBlock = new ActionBlock <HttpResponseMessage>(x =>
{
}, _options);
_request = _requestBlock.ToDataflow(_dataflowOptions, "Request");
_flow = _flowBlock.ToDataflow(_dataflowOptions, "Flow");
_request.LinkTo(_flow);
}
/// <summary>Initializes a new instance of the <see cref="ParallelDataDispatcherWithOutput{TKey, TIn}"/> class.Construct an DataDispatcher instance</summary>
/// <param name="dispatchFunc">The dispatch function</param>
/// <param name="keyComparer">The key comparer for this dataflow</param>
/// <param name="option">Option for this dataflow</param>
/// <param name="token">Cancellation token</param>
/// <param name="declaringType">Runtime type of the owner of this dispatcher</param>
private ParallelDataDispatcherWithOutput(
Func <TIn, TKey> dispatchFunc,
IEqualityComparer <TKey> keyComparer,
DataflowOptions option,
CancellationToken token,
Type declaringType)
: base(option)
{
_token = token;
_dispatchFunc = dispatchFunc;
_declaringType = declaringType;
_destinations = new ConcurrentDictionary <TKey, Lazy <Dataflow <TIn, TIn> > >(keyComparer);
_initer = key => new Lazy <Dataflow <TIn, TIn> >(() =>
{
var childFlow = CreateChildFlow(key);
RegisterChild(childFlow);
childFlow.RegisterDependency(_dispatcherBlock);
return(childFlow);
});
_completedBlock = new BufferBlock <TIn>();
_dispatcherBlock = new ActionBlock <TIn>(async input => await Dispatch(input), option.ToDataflowBlockOptions(true)); // .ToExecutionBlockOption(true));
RegisterChild(_dispatcherBlock);
RegisterChild(_completedBlock);
}
public PeopleFlow(DataflowOptions dataflowOptions)
: base(dataflowOptions)
{
m_peopleRecorder = new StatisticsRecorder(this)
{
Name = "PeopleRecorder"
};
m_converter = new TransformBlock <string, Person>(s => JsonConvert.DeserializeObject <Person>(s));
m_recorder = new TransformBlock <Person, Person>(
p =>
{
//record every person
m_peopleRecorder.Record(p);
return(p);
});
m_converter.LinkTo(m_recorder, new DataflowLinkOptions()
{
PropagateCompletion = true
});
RegisterChild(m_converter);
RegisterChild(m_recorder);
}
public DbBulkInserter(TargetTable targetTable, DataflowOptions options, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null)
: base(targetTable, options, bulkSize, dbBulkInserterName, postBulkInsert)
{
m_longConnection = new SqlConnection(targetTable.ConnectionString);
m_longConnection.Open();
m_transaction = m_longConnection.BeginTransaction();
}
/// <summary>
/// Initializes a new instance of the <see cref="EncodeFlow{T}"/> class.
/// </summary>
/// <param name="dataflowOptions">Dataflow options</param>
/// <param name="serializer">Event serializer</param>
public EncodeFlow(DataflowOptions dataflowOptions, ISerializer <IEvent> serializer)
: base(dataflowOptions)
{
var block = new TransformBlock <IEvent, T>(
e => serializer.Encode <T>(e), dataflowOptions.ToDataflowBlockOptions(true)); // dataflowOptions.ToExecutionBlockOption(true) );
RegisterChild(block);
InputBlock = block;
OutputBlock = block;
}
/// <summary>
/// Constructs an instance of DbBulkInserter
/// </summary>
/// <param name="connectionString">The connection string to the output database</param>
/// <param name="destTable">The table name in database to bulk insert into</param>
/// <param name="options">Options to use for this dataflow</param>
/// <param name="destLabel">The mapping label to help choose among all column mappings</param>
/// <param name="bulkSize">The bulk size to insert in a batch. Default to 8192.</param>
/// <param name="dbBulkInserterName">A given name of this bulk inserter (would be nice for logging)</param>
/// <param name="postBulkInsert">A delegate that enables you to inject some customized work whenever a bulk insert is done</param>
public DbBulkInserterBase(
string connectionString,
string destTable,
DataflowOptions options,
string destLabel,
int bulkSize = 4096 * 2,
string dbBulkInserterName = null,
PostBulkInsertDelegate <T> postBulkInsert = null)
: this(new TargetTable(destLabel, connectionString, destTable), options, bulkSize, dbBulkInserterName, postBulkInsert)
{
}
public ProjectionBufferedDispatcher(DataflowOptions dataflowOptions, ProjectionBase <TState> projection)
: base(dataflowOptions)
{
_buffer = new BufferBlock <Tracked <IStream> >(dataflowOptions.ToDataflowBlockOptions(false)).ToDataflow(dataflowOptions);
_dispatcher = new ProjectionDispatcher <TState>(dataflowOptions, projection);
_log = projection.Log;
_token = projection.CancellationToken;
_token.Register(() => _buffer.LinkTo(DataflowBlock.NullTarget <Tracked <IStream> >().ToDataflow(dataflowOptions)));
RegisterChild(_buffer);
}
/// <summary>
/// Constructs an instance of DbBulkInserter
/// </summary>
/// <param name="targetTable">Information about the database target and mapping label</param>
/// <param name="options">Options to use for this dataflow</param>
/// <param name="bulkSize">The bulk size to insert in a batch. Default to 8192.</param>
/// <param name="dbBulkInserterName">A given name of this bulk inserter (would be nice for logging)</param>
/// <param name="postBulkInsert">A delegate that enables you to inject some customized work whenever a bulk insert is done</param>
public DbBulkInserterBase(TargetTable targetTable,
DataflowOptions options,
int bulkSize = 4096 * 2,
string dbBulkInserterName = null,
PostBulkInsertDelegate <T> postBulkInsert = null)
: base(options)
{
this.m_targetTable = targetTable;
this.m_typeAccessor = TypeAccessorManager <T> .GetAccessorForTable(targetTable);
this.m_bulkSize = bulkSize;
this.m_dbBulkInserterName = dbBulkInserterName;
this.m_postBulkInsert = postBulkInsert;
this.m_batchBlock = new BatchBlock <T>(bulkSize, options.ToGroupingBlockOption());
var bulkInsertOption = options.ToExecutionBlockOption();
//Action block deal with array references
if (bulkInsertOption.BoundedCapacity != DataflowBlockOptions.Unbounded)
{
bulkInsertOption.BoundedCapacity = bulkInsertOption.BoundedCapacity / bulkSize;
}
this.m_logger = Utils.GetNamespaceLogger();
this.m_actionBlock = new ActionBlock <T[]>(
async array =>
{
m_logger.Debug(h => h("{3} starts bulk-inserting {0} {1} to db table {2}", array.Length, typeof(T).Name, targetTable.TableName, this.FullName));
try
{
await this.DumpToDBAsync(array, targetTable);
}
catch (Exception e)
{
m_logger.Error($"{this.FullName} failed bulk-inserting {array.Length} {typeof(T).Name} to db table {targetTable.TableName}", e);
throw;
}
m_logger.Debug(h => h("{3} bulk-inserted {0} {1} to db table {2}", array.Length, typeof(T).Name, targetTable.TableName, this.FullName));
}
, bulkInsertOption);
this.m_batchBlock.LinkTo(this.m_actionBlock, this.m_defaultLinkOption);
this.RegisterChild(this.m_batchBlock);
this.RegisterChild(this.m_actionBlock);
this.m_timer = new Timer(
state =>
{
this.TriggerBatch();
}, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10));
}
public ConnectorFlow(DataflowOptions dataflowOptions, JsonConnector connector)
: base(dataflowOptions)
{
var block = new ActionBlock <TrackedResult <string, string> >(
async url =>
{
var r = await connector.GetAsync(url.Value);
url.SetResult(r);
}, dataflowOptions.ToDataflowBlockOptions(true)); // .ToExecutionBlockOption(true));
RegisterChild(block);
InputBlock = block;
}
/// <summary>
/// Initializes a new instance of the <see cref="DeserializeFlow{TStreamMessage, TEvent}"/> class.
/// </summary>
/// <param name="dataflowOptions">Dataflow options</param>
/// <param name="serializer">Event deserializer</param>
public DeserializeFlow(DataflowOptions dataflowOptions, ISerializer <IEvent> serializer)
: base(dataflowOptions)
{
TransformBlock <TStreamMessage, TEvent> block = null;
block = new TransformBlock <TStreamMessage, TEvent>(
async m => await serializer.Decode <TStreamMessage, TEvent>(m),
dataflowOptions.ToDataflowBlockOptions(true)); // dataflowOptions.ToExecutionBlockOption(true));
RegisterChild(block);
InputBlock = block;
OutputBlock = block;
}
public QueryFlow(QueryHandlerDecorator <TQuery, TResult> handler, DataflowOptions dataflowOptions)
: base(dataflowOptions)
{
var block = new ActionBlock <TrackedResult <TQuery, TResult> >(
async q =>
{
var result = await handler.HandleEx(q.Value);
q.SetResult(result);
},
Configuration.DataflowOptions.ToDataflowBlockOptions(false)); // new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 1 });
RegisterChild(block);
InputBlock = block;
}
public ProjectionFlow(DataflowOptions dataflowOptions, ProjectionBase <TState> projection)
: base(dataflowOptions)
{
_projection = projection;
_eventStore = projection.EventStore;
_log = projection.Log;
_token = projection.CancellationToken;
_versions = projection.Versions;
var block = new ActionBlock <Tracked <IStream> >(Process, dataflowOptions.ToDataflowBlockOptions(true));
RegisterChild(block);
InputBlock = block;
}
public ProjectionDispatcher(DataflowOptions options, ProjectionBase <TState> projection)
: base(s => s.Value.Key, options, projection.CancellationToken, projection.GetType())
{
_projection = projection;
Log = projection.Log;
CompletionTask.ContinueWith(t =>
{
if (t.IsFaulted)
{
Log.Errors.Add(t.Exception);
}
});
}
public HeartbeatNode(DataflowOptions options) : base(options)
{
m_beats = 0;
Func <T, T> f = arg =>
{
IsBusy = true;
m_beats++;
IsBusy = false;
return(arg);
};
m_block = new TransformBlock <T, T>(f, options.ToExecutionBlockOption());
RegisterChild(m_block);
}
public SlowFlow(DataflowOptions dataflowOptions)
: base(dataflowOptions)
{
_splitter = new TransformManyBlock <string, char>(new Func <string, IEnumerable <char> >(this.SlowSplit),
dataflowOptions.ToExecutionBlockOption())
.ToDataflow(dataflowOptions, "SlowSplitter");
_printer = new ActionBlock <char>(c => Console.WriteLine(c),
dataflowOptions.ToExecutionBlockOption())
.ToDataflow(dataflowOptions, "Printer");
RegisterChild(_splitter);
RegisterChild(_printer);
_splitter.LinkTo(_printer);
}
/// <summary>
/// Initializes a new instance of the <see cref="SagaFlow"/> class.
/// </summary>
/// <param name="repository">Saga repository</param>
/// <param name="log">Log service</param>
/// <param name="messageQueue">Message queue service</param>
public SagaFlow(IEsRepository <ISaga> repository, ILog log, IMessageQueue messageQueue)
: base(Configuration.DataflowOptions)
{
_repository = repository;
_log = log;
var block = new ActionBlock <IEvent>(
async e =>
{
await Handle(e);
await messageQueue.CompleteMessage(e);
}, DataflowOptions.ToDataflowBlockOptions(false)); // .ToExecutionBlockOption());
RegisterChild(block);
InputBlock = block;
}
public CircularFlow(DataflowOptions dataflowOptions) : base(dataflowOptions)
{
//a no-op node to demonstrate the usage of preTask param in RegisterChildRing
_buffer = new BufferBlock <int>().ToDataflow(name: "NoOpBuffer");
var heater = new TransformManyDataflow <int, int>(async i =>
{
await Task.Delay(200);
Console.WriteLine("Heated to {0}", i + 1);
return(new [] { i + 1 });
}, dataflowOptions);
heater.Name = "Heater";
var cooler = new TransformManyDataflow <int, int>(async i =>
{
await Task.Delay(200);
int cooled = i - 2;
Console.WriteLine("Cooled to {0}", cooled);
if (cooled < 0) //time to stop
{
return(Enumerable.Empty <int>());
}
return(new [] { cooled });
}, dataflowOptions);
cooler.Name = "Cooler";
var heartbeat = new HeartbeatNode <int>(dataflowOptions)
{
Name = "HeartBeat"
};
_buffer.LinkTo(heater);
//circular
heater.LinkTo(cooler);
cooler.LinkTo(heartbeat);
heartbeat.LinkTo(heater);
RegisterChildren(_buffer, heater, cooler, heartbeat);
//ring registration
RegisterChildRing(_buffer.Completion, heater, cooler, heartbeat);
}
/// <summary>
/// Initializes a new instance of the <see cref="ProjectionBase{TState}"/> class.
/// </summary>
/// <param name="eventStore">Event store service</param>
/// <param name="log">Log service</param>
/// <param name="activeTimeline">Timeline service</param>
/// <param name="streamLocator">Stream locator</param>
public ProjectionBase(IEventStore <IAggregate> eventStore, ILog log, ITimeline activeTimeline, IStreamLocator streamLocator)
{
EventStore = eventStore;
Log = log;
ActiveTimeline = activeTimeline;
_streamLocator = streamLocator;
CancellationSource = new RepeatableCancellationTokenSource();
_start = new Lazy <Task>(() => Task.Run(Start));
var options = new DataflowOptions {
RecommendedParallelismIfMultiThreaded = 1
};
Build = new BuildFlow(options, this);
StatusSubject.Where(s => s != Sleeping)
.Subscribe(s => Log?.Info($"[{Timeline}]{GetType().GetFriendlyName()}/{RuntimeHelpers.GetHashCode(this)}/ : {s.ToString()}"));
}
/// <summary>
/// Constructs a MultiDbBulkInserter instance.
/// </summary>
/// <param name="options">The option for this dataflow</param>
/// <param name="dispatchFunc">The dispatch function to decide which child flow the incoming objects will be delivered to</param>
/// <param name="connectionGetter">Connection string generator for child flows</param>
/// <param name="destTable">The table name in database to bulk insert into for every child</param>
/// <param name="destLabel">The mapping label to help choose among all column mappings</param>
/// <param name="bulkSize">The bulk size to insert in a batch. Default to 8192.</param>
/// <param name="displayName">A given name of this multi bulk inserter (would be nice for logging)</param>
/// <param name="postBulkInsert">A delegate that enables you to inject some customized work whenever a bulk insert is done</param>
public MultiDbBulkInserter(DataflowOptions options,
Func <T, int> dispatchFunc,
Func <int, string> connectionGetter,
string destTable,
string destLabel,
int bulkSize = 4096 * 2,
string displayName = null,
PostBulkInsertDelegate <T> postBulkInsert = null)
: base(dispatchFunc, options)
{
m_options = options;
m_connectionGetter = connectionGetter;
m_destTable = destTable;
m_destLabel = destLabel;
m_bulkSize = bulkSize;
m_displayName = displayName;
this.m_postBulkInsert = postBulkInsert;
}
public IEnumerable <string> BufferBlockUsageWithBoundedCapacity(int numberOfIteration, int capacity = 1000)
{
Console.WriteLine($"Inside {nameof(TplDataflow2BufferingBlocksController)} - {nameof(BufferBlockUsageWithBoundedCapacity)}");
var strings = new BlockingCollection <string>();
// Create the members of the pipeline.
var bufferBlockGivenInputToSubscribers = new BufferBlock <string>();
var actionBlockSubscriber1 = new ActionBlock <string>(stringInput =>
Functions.AddInputIntoTheGivenList(strings, stringInput, "Sub 1")
, DataflowOptions.CreateBlockOptionsWithBoundedBoundedCapacity(capacity)
);
var actionBlockSubscriber2 = new ActionBlock <string>(stringInput =>
Functions.AddInputIntoTheGivenList(strings, stringInput, "Sub 2")
, DataflowOptions.CreateBlockOptionsWithBoundedBoundedCapacity(capacity)
);
var actionBlockSubscriber3 = new ActionBlock <string>(stringInput =>
Functions.AddInputIntoTheGivenList(strings, stringInput, "Sub 3")
, DataflowOptions.CreateBlockOptionsWithBoundedBoundedCapacity(capacity)
);
// Connect the dataflow blocks to form a pipeline.
bufferBlockGivenInputToSubscribers.LinkTo(actionBlockSubscriber1, DataflowOptions.LinkOptions);
bufferBlockGivenInputToSubscribers.LinkTo(actionBlockSubscriber2, DataflowOptions.LinkOptions);
bufferBlockGivenInputToSubscribers.LinkTo(actionBlockSubscriber3, DataflowOptions.LinkOptions);
// Start BufferBlockUsageWithBoundedCapacity pipeline with the input values.
for (var i = 1; i <= numberOfIteration; i++)
{
bufferBlockGivenInputToSubscribers.Post($"Value = {i}");
}
// Mark the head of the pipeline as complete.
bufferBlockGivenInputToSubscribers.Complete();
// Waiting block to receive all post input.
Task.WaitAll(actionBlockSubscriber1.Completion,
actionBlockSubscriber2.Completion,
actionBlockSubscriber3.Completion);
return(strings);
}
public SimpleLogReader(DataflowOptions dataflowOptions)
: base(dataflowOptions)
{
this.m_parsingBlock = new TransformManyBlock <string, string>(
s =>
{
if (string.IsNullOrEmpty(s))
{
return(Enumerable.Empty <string>());
}
if (s == "ERROR")
{
throw new InvalidDataException("evil data :)");
}
return(s.Split(Splitor));
});
this.m_recorder = new StatisticsRecorder(this);
this.m_recordBlock = new ActionBlock <string>(
s =>
{
if (s == "ERROR")
{
throw new InvalidDataException("evil data :)");
}
this.m_recorder.RecordEvent(s);
});
var df1 = DataflowUtils.FromBlock(m_parsingBlock);
var df2 = DataflowUtils.FromBlock(m_recordBlock);
df1.LinkTo(df2);
RegisterChild(df1);
RegisterChild(df2);
}
public SimpleLogReader(DataflowOptions dataflowOptions)
: base(dataflowOptions)
{
this.m_parsingBlock = new TransformManyBlock<string, string>(
s =>
{
if (string.IsNullOrEmpty(s))
{
return Enumerable.Empty<string>();
}
if (s == "ERROR")
{
throw new InvalidDataException("evil data :)");
}
return s.Split(Splitor);
});
this.m_recorder = new StatisticsRecorder(this);
this.m_recordBlock = new ActionBlock<string>(
s =>
{
if (s == "ERROR")
{
throw new InvalidDataException("evil data :)");
}
this.m_recorder.RecordEvent(s);
});
var df1 = DataflowUtils.FromBlock(m_parsingBlock);
var df2 = DataflowUtils.FromBlock(m_recordBlock);
df1.LinkTo(df2);
RegisterChild(df1);
RegisterChild(df2);
}
/// <summary>
/// Gets the default execution dataflow block options
/// </summary>
/// <param name="options">Base options</param>
/// <param name="isBlockMultiThreaded">Can block run multiple threads</param>
/// <param name="useScheduler">Use limited concurrency scheduler</param>
/// <param name="maxMessagesPerTask">Maximum messages per task</param>
/// <returns>Execution dataflow block options</returns>
public static ExecutionDataflowBlockOptions ToDataflowBlockOptions(
this DataflowOptions options,
bool isBlockMultiThreaded = false,
bool useScheduler = false,
int maxMessagesPerTask = -1)
{
var executionOptions = options.ToExecutionBlockOption(isBlockMultiThreaded);
if (Configuration.MaxMessagesPerTask > 0)
{
maxMessagesPerTask = Configuration.MaxMessagesPerTask;
}
if (maxMessagesPerTask > 0)
{
executionOptions.MaxMessagesPerTask = maxMessagesPerTask;
}
if (useScheduler || Configuration.UseLimitedScheduler)
{
executionOptions.TaskScheduler = Configuration.LimitedTaskScheduler;
}
return(executionOptions);
}
/// <summary>
/// Constructs an instance of DbBulkInserter
/// </summary>
/// <param name="targetTable">Information about the database target and mapping label</param>
/// <param name="options">Options to use for this dataflow</param>
/// <param name="bulkSize">The bulk size to insert in a batch. Default to 8192.</param>
/// <param name="dbBulkInserterName">A given name of this bulk inserter (would be nice for logging)</param>
/// <param name="postBulkInsert">A delegate that enables you to inject some customized work whenever a bulk insert is done</param>
public DbBulkInserterBase(TargetTable targetTable,
DataflowOptions options,
int bulkSize = 4096 * 2,
string dbBulkInserterName = null,
PostBulkInsertDelegate <T> postBulkInsert = null)
: base(options)
{
this.m_targetTable = targetTable;
this.m_typeAccessor = TypeAccessorManager <T> .GetAccessorForTable(targetTable);
this.m_bulkSize = bulkSize;
this.m_dbBulkInserterName = dbBulkInserterName;
this.m_postBulkInsert = postBulkInsert;
this.m_batchBlock = new BatchBlock <T>(bulkSize, options.ToGroupingBlockOption());
var bulkInsertOption = options.ToExecutionBlockOption();
//Action block deal with array references
if (bulkInsertOption.BoundedCapacity != DataflowBlockOptions.Unbounded)
{
bulkInsertOption.BoundedCapacity = bulkInsertOption.BoundedCapacity / bulkSize;
}
this.m_actionBlock = new ActionBlock <T[]>(array => this.DumpToDBAsync(array, targetTable), bulkInsertOption);
this.m_batchBlock.LinkTo(this.m_actionBlock, this.m_defaultLinkOption);
this.m_logger = Utils.GetNamespaceLogger();
this.RegisterChild(this.m_batchBlock);
this.RegisterChild(this.m_actionBlock);
this.m_timer = new Timer(
state =>
{
this.TriggerBatch();
}, null, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(10));
}
public CalculateAndStoreFromInputAndAsyncTermsOptions(TimeSpan asyncFetchTimeout, TimeSpan asyncFetchTimeInterval, DataflowOptions dataFlowOptions) : base()
{
_asyncFetchTimeout = asyncFetchTimeout;
base.BlockMonitorEnabled = dataFlowOptions.BlockMonitorEnabled;
base.FlowMonitorEnabled = dataFlowOptions.FlowMonitorEnabled;
base.MonitorInterval = dataFlowOptions.MonitorInterval;
base.PerformanceMonitorMode = dataFlowOptions.PerformanceMonitorMode;
base.RecommendedCapacity = dataFlowOptions.RecommendedCapacity;
base.RecommendedParallelismIfMultiThreaded = dataFlowOptions.RecommendedParallelismIfMultiThreaded;
}
public CalculateAndStoreFromInputAndAsyncTermsOptions(DataflowOptions dataFlowOptions) : this(DefaultAsyncFetchTimeout, DefaultAsyncFetchTimeInterval, dataFlowOptions)
{
}
public AutoCompleteWrapper(Dataflow <TIn, TOut> dataflow, TimeSpan processTimeout, DataflowOptions options) : base(options)
{
m_Dataflow = dataflow;
m_processTimeout = processTimeout;
m_timer = new Timer();
m_timer.Interval = m_processTimeout.TotalMilliseconds;
m_timer.Elapsed += OnTimerElapsed;
var before = new TransformBlock <TIn, TIn>(@in =>
{
if (m_last == null || @in.UniqueId == m_last.Value)
{
//The last one is back, so there is nothing else in the pipeline.
//Set a timer: if nothing new produced when timer expires, the whole loop ends.
m_timer.Start();
}
return(@in);
});
m_before = DataflowUtils.FromBlock(before);
var after = new TransformBlock <TOut, TOut>(@out =>
{
if (@out.UniqueId != Guid.Empty)
{
m_last = @out.UniqueId;
m_timer.Stop();
}
else
{
LogHelper.Logger.WarnFormat("Empty guid found in output. You may have forgotten to set it.");
}
return(@out);
});
m_after = DataflowUtils.FromBlock(after);
m_before.GoTo(dataflow).GoTo(m_after);
RegisterChild(m_before);
RegisterChild(dataflow);
RegisterChild(m_after);
}
public CommandDispatcher(Func <ICommand, Task> handler, string timeline, DataflowOptions options)
: base(c => $"{timeline}:{c.Target}", options, CancellationToken.None)
{
_handler = handler;
_options = options;
}
public EagerDbBulkInserter(TargetTable targetTable, DataflowOptions options, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null)
: base(targetTable, options, bulkSize, dbBulkInserterName, postBulkInsert)
{
}
public EagerDbBulkInserter(string connectionString, string destTable, DataflowOptions options, string destLabel, int bulkSize = 8192, string dbBulkInserterName = null, PostBulkInsertDelegate <T> postBulkInsert = null)
: base(connectionString, destTable, options, destLabel, bulkSize, dbBulkInserterName, postBulkInsert)
{
}
public DimTableInserter(DbDataJoiner <TIn, TLookupKey> host, TargetTable targetTable, Expression <Func <TIn, TLookupKey> > joinBy, DataflowOptions option)
: base(targetTable, option, host.m_batchSize)
{
this.m_host = host;
m_keyGetter = joinBy.Compile();
m_keyComparer = m_host.m_keyComparer;
m_outputBuffer = new BufferBlock <JoinBatch <TIn> >(option.ToGroupingBlockOption()).ToDataflow(option);
m_outputBuffer.Name = "OutputBuffer";
RegisterChild(m_outputBuffer);
m_outputBuffer.RegisterDependency(m_actionBlock);
m_tmpTargetTable = new TargetTable(
targetTable.DestLabel,
targetTable.ConnectionString,
targetTable.TableName + "_tmp");
//create tmp table
m_createTmpTable = string.Format(
"if OBJECT_ID('{0}', 'U') is not null drop table {0};"
+ "select * into {0} from {1} where 0 = 1",
this.m_tmpTargetTable.TableName,
targetTable.TableName);
//merge
m_mergeTmpToDimTable =
string.Format(
"MERGE INTO {0} as TGT "
+ "USING {1} as SRC on TGT.[{2}] = SRC.[{2}] "
+ "WHEN MATCHED THEN UPDATE SET TGT.[{2}] = TGT.[{2}] "
+ "WHEN NOT MATCHED THEN INSERT {3} VALUES {4} "
+ "OUTPUT inserted.[{5}], inserted.[{2}] ;",
targetTable.TableName,
this.m_tmpTargetTable.TableName,
m_host.m_joinOnMapping.DestColumnName,
Utils.FlattenColumnNames(m_typeAccessor.SchemaTable.Columns, ""),
Utils.FlattenColumnNames(m_typeAccessor.SchemaTable.Columns, "SRC"),
Utils.GetAutoIncrementColumn(m_typeAccessor.SchemaTable.Columns)
);
}
