public EPDataFlowInstanceImpl(
object dataFlowInstanceUserObject,
string dataFlowInstanceId,
OperatorStatisticsProvider statistics,
IDictionary<int, object> operators,
IList<GraphSourceRunnable> sourceRunnables,
DataflowDesc dataflowDesc,
AgentInstanceContext agentInstanceContext,
EPDataFlowInstanceStatistics statisticsProvider,
IDictionary<string, object> parametersURIs)
{
UserObject = dataFlowInstanceUserObject;
InstanceId = dataFlowInstanceId;
_statistics = statistics;
_dataflowDesc = dataflowDesc;
_agentInstanceContext = agentInstanceContext;
_sourceRunnables = sourceRunnables;
Statistics = statisticsProvider;
Parameters = parametersURIs;
State = EPDataFlowState.INSTANTIATED;
_operators = new OrderedListDictionary<int, Pair<object, bool>>();
foreach (var entry in operators) {
_operators.Put(entry.Key, new Pair<object, bool>(entry.Value, false));
}
}
private static object InstantiateOperator(
IContainer container,
int operatorNum,
DataflowDesc dataflow,
EPDataFlowInstantiationOptions options,
AgentInstanceContext agentInstanceContext)
{
var operatorFactory = dataflow.OperatorFactories.Get(operatorNum);
var metadata = dataflow.OperatorMetadata.Get(operatorNum);
// see if the operator is already provided by options
var operatorX = options.OperatorProvider?.Provide(
new EPDataFlowOperatorProviderContext(
dataflow.DataflowName,
metadata.OperatorName,
operatorFactory));
if (operatorX != null) {
return operatorX;
}
IDictionary<string, object> additionalParameters = null;
if (options.ParametersURIs != null) {
var prefix = metadata.OperatorName + "/";
foreach (var entry in options.ParametersURIs) {
if (!entry.Key.StartsWith(prefix)) {
continue;
}
if (additionalParameters == null) {
additionalParameters = new Dictionary<string, object>();
}
additionalParameters.Put(entry.Key.Substring(prefix.Length), entry.Value);
}
}
object @operator;
try {
@operator = operatorFactory.Operator(
new DataFlowOpInitializeContext(
container,
dataflow.DataflowName,
metadata.OperatorName,
operatorNum,
agentInstanceContext,
additionalParameters,
options.DataFlowInstanceId,
options.ParameterProvider,
operatorFactory,
options.DataFlowInstanceUserObject));
}
catch (Exception t) {
var meta = dataflow.OperatorMetadata.Get(operatorNum);
throw new EPException(
"Failed to obtain operator instance for '" + meta.OperatorName + "': " + t.Message,
t);
}
return @operator;
}
public void Add(
string dataflowName,
DataflowDesc metadata)
{
DataflowDesc existing = dataflows.Get(dataflowName);
if (existing != null) {
throw new IllegalStateException("Dataflow already found for name '" + dataflowName + "'");
}
dataflows.Put(dataflowName, metadata);
}
public void RemoveDataflow(
string deploymentId,
DataflowDesc dataflow)
{
lock (this) {
var deployment = _deployments.Get(deploymentId);
if (deployment == null) {
return;
}
deployment.Remove(dataflow.DataflowName);
}
}
public void AddDataflow(
string deploymentId,
DataflowDesc dataflow)
{
lock (this) {
var deployment = _deployments.Get(deploymentId);
if (deployment == null) {
deployment = new DataflowDeployment();
_deployments.Put(deploymentId, deployment);
}
deployment.Add(dataflow.DataflowName, dataflow);
}
}
private static IDictionary<int, object> InstantiateOperators(
IContainer container,
AgentInstanceContext agentInstanceContext,
EPDataFlowInstantiationOptions options,
DataflowDesc dataflow)
{
IDictionary<int, object> operators = new Dictionary<int, object>();
foreach (var operatorNum in dataflow.OperatorMetadata.Keys) {
var @operator = InstantiateOperator(
container, operatorNum, dataflow, options, agentInstanceContext);
operators.Put(operatorNum, @operator);
}
return operators;
}
public static OperatorStatisticsProvider Realize(
DataflowDesc dataflow,
IDictionary<int, object> operators,
IList<LogicalChannelBinding> bindings,
DataFlowSignalManager dataFlowSignalManager,
EPDataFlowInstantiationOptions options,
AgentInstanceContext agentInstanceContext)
{
IDictionary<int, OperatorMetadataDescriptor> operatorMetadata = dataflow.OperatorMetadata;
// First pass: inject runtime context
IDictionary<int, EPDataFlowEmitter> runtimeContexts = new Dictionary<int, EPDataFlowEmitter>();
OperatorStatisticsProvider statisticsProvider = null;
if (options.IsOperatorStatistics) {
statisticsProvider = new OperatorStatisticsProvider(operatorMetadata);
}
foreach (int producerOpNum in dataflow.OperatorBuildOrder) {
var operatorPrettyPrint = operatorMetadata.Get(producerOpNum).OperatorPrettyPrint;
if (Log.IsDebugEnabled) {
Log.Debug("Generating runtime context for " + operatorPrettyPrint);
}
// determine the number of output streams
var producingOp = operators.Get(producerOpNum);
var numOutputStreams = operatorMetadata.Get(producerOpNum).NumOutputPorts;
var targets = GetOperatorConsumersPerStream(
numOutputStreams,
producerOpNum,
operators,
operatorMetadata,
bindings);
var runtimeContext = GenerateRuntimeContext(
agentInstanceContext,
dataflow,
options.DataFlowInstanceId,
producerOpNum,
operatorPrettyPrint,
dataFlowSignalManager,
targets,
options);
if (options.IsOperatorStatistics) {
runtimeContext = new EPDataFlowEmitterWrapperWStatistics(
runtimeContext,
producerOpNum,
statisticsProvider,
options.IsCpuStatistics);
}
TypeHelper.SetFieldForAnnotation(producingOp, typeof(DataFlowContextAttribute), runtimeContext);
runtimeContexts.Put(producerOpNum, runtimeContext);
}
// Second pass: hook punctuation such that it gets forwarded
foreach (int producerOpNum in dataflow.OperatorBuildOrder) {
var operatorPrettyPrint = operatorMetadata.Get(producerOpNum).OperatorPrettyPrint;
if (Log.IsDebugEnabled) {
Log.Debug("Handling signals for " + operatorPrettyPrint);
}
// determine consumers that receive punctuation
ISet<int> consumingOperatorsWithPunctuation = new HashSet<int>();
foreach (var binding in bindings) {
if (!binding.LogicalChannel.OutputPort.HasPunctuation ||
binding.LogicalChannel.OutputPort.ProducingOpNum != producerOpNum) {
continue;
}
consumingOperatorsWithPunctuation.Add(binding.LogicalChannel.ConsumingOpNum);
}
// hook up a listener for each
foreach (var consumerPunc in consumingOperatorsWithPunctuation) {
var context = runtimeContexts.Get(consumerPunc);
if (context == null) {
continue;
}
dataFlowSignalManager.AddSignalListener(
producerOpNum,
new ProxyDataFlowSignalListener {
ProcProcessSignal = signal => context.SubmitSignal(signal)
});
}
}
return statisticsProvider;
}
private static EPDataFlowEmitter GenerateRuntimeContext(
AgentInstanceContext agentInstanceContext,
DataflowDesc dataflow,
string instanceId,
int producerOpNum,
string operatorPrettyPrint,
DataFlowSignalManager dataFlowSignalManager,
IList<ObjectBindingPair>[] targetsPerStream,
EPDataFlowInstantiationOptions options)
{
// handle no targets
if (targetsPerStream == null) {
return new EPDataFlowEmitterNoTarget(producerOpNum, dataFlowSignalManager);
}
var dataflowName = dataflow.DataflowName;
var classpathImportService = agentInstanceContext.ImportServiceRuntime;
// handle single-stream case
if (targetsPerStream.Length == 1) {
var targets = targetsPerStream[0];
// handle single-stream single target case
if (targets.Count == 1) {
var target = targets[0];
return GetSubmitHandler(
agentInstanceContext,
dataflow.DataflowName,
instanceId,
producerOpNum,
operatorPrettyPrint,
dataFlowSignalManager,
target,
options.ExceptionHandler,
classpathImportService);
}
var handlers = new SubmitHandler[targets.Count];
for (var i = 0; i < handlers.Length; i++) {
handlers[i] = GetSubmitHandler(
agentInstanceContext,
dataflowName,
instanceId,
producerOpNum,
operatorPrettyPrint,
dataFlowSignalManager,
targets[i],
options.ExceptionHandler,
classpathImportService);
}
return new EPDataFlowEmitter1StreamNTarget(producerOpNum, dataFlowSignalManager, handlers);
}
// handle multi-stream case
var handlersPerStream = new SubmitHandler[targetsPerStream.Length][];
for (var streamNum = 0; streamNum < targetsPerStream.Length; streamNum++) {
var handlers = new SubmitHandler[targetsPerStream[streamNum].Count];
handlersPerStream[streamNum] = handlers;
for (var i = 0; i < handlers.Length; i++) {
handlers[i] = GetSubmitHandler(
agentInstanceContext,
dataflowName,
instanceId,
producerOpNum,
operatorPrettyPrint,
dataFlowSignalManager,
targetsPerStream[streamNum][i],
options.ExceptionHandler,
classpathImportService);
}
}
return new EPDataFlowEmitterNStreamNTarget(producerOpNum, dataFlowSignalManager, handlersPerStream);
}
public static EPDataFlowInstance Instantiate(
IContainer container,
int agentInstanceId,
DataflowDesc dataflow,
EPDataFlowInstantiationOptions options)
{
var statementContext = dataflow.StatementContext;
// allocate agent instance context
var @lock = statementContext.StatementAgentInstanceLockFactory.GetStatementLock(
statementContext.StatementName,
statementContext.Annotations,
statementContext.IsStatelessSelect,
statementContext.StatementType);
var handle = new EPStatementAgentInstanceHandle(statementContext.EpStatementHandle, agentInstanceId, @lock);
var auditProvider = statementContext.StatementInformationals.AuditProvider;
var instrumentationProvider = statementContext.StatementInformationals.InstrumentationProvider;
var agentInstanceContext = new AgentInstanceContext(statementContext, handle, null, null, auditProvider, instrumentationProvider);
// assure variables
statementContext.VariableManagementService.SetLocalVersion();
// instantiate operators
var operators = InstantiateOperators(container, agentInstanceContext, options, dataflow);
// determine binding of each channel to input methods (ports)
IList<LogicalChannelBinding> operatorChannelBindings = new List<LogicalChannelBinding>();
foreach (var channel in dataflow.LogicalChannels) {
var targetClass = operators.Get(channel.ConsumingOpNum).GetType();
var consumingMethod = FindMatchingMethod(channel.ConsumingOpPrettyPrint, targetClass, channel, false);
LogicalChannelBindingMethodDesc onSignalMethod = null;
if (channel.OutputPort.HasPunctuation) {
onSignalMethod = FindMatchingMethod(channel.ConsumingOpPrettyPrint, targetClass, channel, true);
}
operatorChannelBindings.Add(new LogicalChannelBinding(channel, consumingMethod, onSignalMethod));
}
// obtain realization
var dataFlowSignalManager = new DataFlowSignalManager();
var statistics = DataflowInstantiatorHelper.Realize(
dataflow,
operators,
operatorChannelBindings,
dataFlowSignalManager,
options,
agentInstanceContext);
// For each GraphSource add runnable
IList<GraphSourceRunnable> sourceRunnables = new List<GraphSourceRunnable>();
var audit = AuditEnum.DATAFLOW_SOURCE.GetAudit(statementContext.Annotations) != null;
foreach (var operatorEntry in operators) {
if (!(operatorEntry.Value is DataFlowSourceOperator)) {
continue;
}
var meta = dataflow.OperatorMetadata.Get(operatorEntry.Key);
var graphSource = (DataFlowSourceOperator) operatorEntry.Value;
var runnable = new GraphSourceRunnable(
agentInstanceContext,
graphSource,
dataflow.DataflowName,
options.DataFlowInstanceId,
meta.OperatorName,
operatorEntry.Key,
meta.OperatorPrettyPrint,
options.ExceptionHandler,
audit);
sourceRunnables.Add(runnable);
dataFlowSignalManager.AddSignalListener(operatorEntry.Key, runnable);
}
return new EPDataFlowInstanceImpl(
options.DataFlowInstanceUserObject,
options.DataFlowInstanceId,
statistics,
operators,
sourceRunnables,
dataflow,
agentInstanceContext,
statistics,
options.ParametersURIs);
}
