/// <summary>
/// Walks the chain of lookups and constructs lookup strategy and plan specification based
/// on the index specifications.
/// </summary>
/// <param name="lookupStream">the stream to construct the query plan for</param>
/// <param name="bestChain">the chain that the lookup follows to make best use of indexes</param>
/// <param name="queryGraph">the repository for key properties to indexes</param>
/// <param name="indexSpecsPerStream">specifications of indexes</param>
/// <param name="typesPerStream">event types for each stream</param>
/// <param name="isHistorical">indicator for each stream if it is a historical streams or not</param>
/// <param name="historicalStreamIndexLists">index management, populated for the query plan</param>
/// <param name="tablesPerStream">tables</param>
/// <param name="streamJoinAnalysisResult">stream join analysis</param>
/// <param name="raw">raw statement information</param>
/// <param name="serdeResolver">serde resolver</param>
/// <returns>NestedIterationNode with lookups attached underneath</returns>
public static QueryPlanNodeForgeDesc CreateStreamPlan(
int lookupStream,
int[] bestChain,
QueryGraphForge queryGraph,
QueryPlanIndexForge[] indexSpecsPerStream,
EventType[] typesPerStream,
bool[] isHistorical,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo raw,
SerdeCompileTimeResolver serdeResolver)
{
var nestedIterNode = new NestedIterationNodeForge(bestChain);
var currentLookupStream = lookupStream;
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Walk through each successive lookup
for (var i = 0; i < bestChain.Length; i++) {
var indexedStream = bestChain[i];
QueryPlanNodeForge node;
if (isHistorical[indexedStream]) {
if (historicalStreamIndexLists[indexedStream] == null) {
historicalStreamIndexLists[indexedStream] = new HistoricalStreamIndexListForge(
indexedStream,
typesPerStream,
queryGraph);
}
historicalStreamIndexLists[indexedStream].AddIndex(currentLookupStream);
node = new HistoricalDataPlanNodeForge(
indexedStream,
lookupStream,
currentLookupStream,
typesPerStream.Length,
null);
}
else {
var tableLookupPlan = CreateLookupPlan(
queryGraph,
currentLookupStream,
indexedStream,
streamJoinAnalysisResult.IsVirtualDW(indexedStream),
indexSpecsPerStream[indexedStream],
typesPerStream,
tablesPerStream[indexedStream],
raw,
serdeResolver);
node = new TableLookupNodeForge(tableLookupPlan.Forge);
additionalForgeables.AddAll(tableLookupPlan.AdditionalForgeables);
}
nestedIterNode.AddChildNode(node);
currentLookupStream = bestChain[i];
}
return new QueryPlanNodeForgeDesc(nestedIterNode, additionalForgeables);
}
public JoinSetComposerPrototypeGeneralForge(
EventType[] streamTypes,
ExprNode postJoinEvaluator,
bool outerJoins,
QueryPlanForge queryPlan,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
string[] streamNames,
bool joinRemoveStream,
bool hasHistorical)
: base(streamTypes, postJoinEvaluator, outerJoins)
{
this.queryPlan = queryPlan;
this.streamJoinAnalysisResult = streamJoinAnalysisResult;
this.streamNames = streamNames;
this.joinRemoveStream = joinRemoveStream;
this.hasHistorical = hasHistorical;
}
// Remove plans for non-unidirectional streams
private static void RemoveUnidirectionalAndTable(
QueryPlanForge queryPlan,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult)
{
bool allUnidirectional = streamJoinAnalysisResult.IsUnidirectionalAll;
for (int streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) {
if (allUnidirectional) {
queryPlan.ExecNodeSpecs[streamNum] = new QueryPlanNodeForgeAllUnidirectionalOuter(streamNum);
}
else {
bool unidirectional = streamJoinAnalysisResult.IsUnidirectional &&
!streamJoinAnalysisResult.UnidirectionalInd[streamNum];
bool table = streamJoinAnalysisResult.TablesPerStream[streamNum] != null;
if (unidirectional || table) {
queryPlan.ExecNodeSpecs[streamNum] = QueryPlanNodeNoOpForge.INSTANCE;
}
}
}
}
public static QueryPlanForgeDesc Build(
QueryGraphForge queryGraph,
EventType[] typesPerStream,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
bool hasForceNestedIter,
string[][][] indexedStreamsUniqueProps,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo raw,
SerdeCompileTimeResolver serdeResolver)
{
if (Log.IsDebugEnabled) {
Log.Debug(".build filterQueryGraph=" + queryGraph);
}
var numStreams = queryGraph.NumStreams;
var additionalForgeables = new List<StmtClassForgeableFactory>();
var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(
queryGraph,
typesPerStream,
indexedStreamsUniqueProps);
if (Log.IsDebugEnabled) {
Log.Debug(".build Index build completed, indexes=" + QueryPlanIndexForge.Print(indexSpecs));
}
// any historical streams don't get indexes, the lookup strategy accounts for cached indexes
if (historicalViewableDesc.IsHistorical) {
for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) {
if (historicalViewableDesc.Historical[i]) {
indexSpecs[i] = null;
}
}
}
var planNodeSpecs = new QueryPlanNodeForge[numStreams];
var worstDepth = int.MaxValue;
for (var streamNo = 0; streamNo < numStreams; streamNo++) {
// no plan for historical streams that are dependent upon other streams
if (historicalViewableDesc.Historical[streamNo] && dependencyGraph.HasDependency(streamNo)) {
planNodeSpecs[streamNo] = new QueryPlanNodeNoOpForge();
continue;
}
var bestChainResult = ComputeBestPath(streamNo, queryGraph, dependencyGraph);
var bestChain = bestChainResult.Chain;
if (Log.IsDebugEnabled) {
Log.Debug(".build For stream " + streamNo + " bestChain=" + bestChain.RenderAny());
}
if (bestChainResult.Depth < worstDepth) {
worstDepth = bestChainResult.Depth;
}
var planDesc = CreateStreamPlan(
streamNo,
bestChain,
queryGraph,
indexSpecs,
typesPerStream,
historicalViewableDesc.Historical,
historicalStreamIndexLists,
tablesPerStream,
streamJoinAnalysisResult,
raw,
serdeResolver);
planNodeSpecs[streamNo] = planDesc.Forge;
additionalForgeables.AddAll(planDesc.AdditionalForgeables);
if (Log.IsDebugEnabled) {
Log.Debug(".build spec=" + planNodeSpecs[streamNo]);
}
}
// We use the merge/nested (outer) join algorithm instead.
if (worstDepth < numStreams - 1 && !hasForceNestedIter) {
return null;
}
// build historical index and lookup strategies
for (var i = 0; i < numStreams; i++) {
var plan = planNodeSpecs[i];
QueryPlanNodeForgeVisitor visitor = new ProxyQueryPlanNodeForgeVisitor {
ProcVisit = node => {
if (node is HistoricalDataPlanNodeForge) {
var historical = (HistoricalDataPlanNodeForge) node;
JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum].GetStrategy(
historical.LookupStreamNum, raw, serdeResolver);
historical.PollResultIndexingStrategy = desc.IndexingForge;
historical.HistoricalIndexLookupStrategy = desc.LookupForge;
additionalForgeables.AddAll(desc.AdditionalForgeables);
}
}
};
plan.Accept(visitor);
}
var forge = new QueryPlanForge(indexSpecs, planNodeSpecs);
return new QueryPlanForgeDesc(forge, additionalForgeables);
}
/// <summary>
/// Build query plan.
/// </summary>
/// <param name="queryGraph">navigability info</param>
/// <param name="optionalOuterJoinType">outer join type, null if not an outer join</param>
/// <param name="typesPerStream">event types for each stream</param>
/// <param name="streamJoinAnalysisResult">stream join analysis</param>
/// <returns>query plan</returns>
public static QueryPlanForgeDesc Build(
EventType[] typesPerStream,
QueryGraphForge queryGraph,
OuterJoinType? optionalOuterJoinType,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo raw)
{
var uniqueIndexProps = streamJoinAnalysisResult.UniqueKeys;
var tablesPerStream = streamJoinAnalysisResult.TablesPerStream;
var additionalForgeable = new List<StmtClassForgeableFactory>();
var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(
queryGraph,
typesPerStream,
uniqueIndexProps);
var execNodeSpecs = new QueryPlanNodeForge[2];
var lookupPlans = new TableLookupPlanForge[2];
// plan lookup from 1 to zero
TableLookupPlanDesc plan1to0 = NStreamQueryPlanBuilder.CreateLookupPlan(
queryGraph,
1,
0,
streamJoinAnalysisResult.IsVirtualDW(0),
indexSpecs[0],
typesPerStream,
tablesPerStream[0],
raw,
SerdeCompileTimeResolverNonHA.INSTANCE);
lookupPlans[1] = plan1to0.Forge;
additionalForgeable.AddAll(plan1to0.AdditionalForgeables);
// plan lookup from zero to 1
TableLookupPlanDesc plan0to1 = NStreamQueryPlanBuilder.CreateLookupPlan(
queryGraph,
0,
1,
streamJoinAnalysisResult.IsVirtualDW(1),
indexSpecs[1],
typesPerStream,
tablesPerStream[1],
raw,
SerdeCompileTimeResolverNonHA.INSTANCE);
lookupPlans[0] = plan0to1.Forge;
additionalForgeable.AddAll(plan0to1.AdditionalForgeables);
execNodeSpecs[0] = new TableLookupNodeForge(lookupPlans[0]);
execNodeSpecs[1] = new TableLookupNodeForge(lookupPlans[1]);
if (optionalOuterJoinType != null) {
if ((optionalOuterJoinType.Equals(OuterJoinType.LEFT)) ||
(optionalOuterJoinType.Equals(OuterJoinType.FULL))) {
execNodeSpecs[0] = new TableOuterLookupNodeForge(lookupPlans[0]);
}
if ((optionalOuterJoinType.Equals(OuterJoinType.RIGHT)) ||
(optionalOuterJoinType.Equals(OuterJoinType.FULL))) {
execNodeSpecs[1] = new TableOuterLookupNodeForge(lookupPlans[1]);
}
}
var forge = new QueryPlanForge(indexSpecs, execNodeSpecs);
return new QueryPlanForgeDesc(forge, additionalForgeable);
}
public static QueryPlanForgeDesc GetPlan(
EventType[] typesPerStream,
OuterJoinDesc[] outerJoinDescList,
QueryGraphForge queryGraph,
string[] streamNames,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
bool isQueryPlanLogging,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
string methodName = ".getPlan ";
int numStreams = typesPerStream.Length;
if (numStreams < 2) {
throw new ArgumentException("Number of join stream types is less then 2");
}
if (outerJoinDescList.Length >= numStreams) {
throw new ArgumentException("Too many outer join descriptors found");
}
if (numStreams == 2) {
OuterJoinType? outerJoinType = null;
if (outerJoinDescList.Length > 0) {
outerJoinType = outerJoinDescList[0].OuterJoinType;
}
QueryPlanForgeDesc queryPlan = TwoStreamQueryPlanBuilder.Build(
typesPerStream,
queryGraph,
outerJoinType,
streamJoinAnalysisResult,
statementRawInfo);
RemoveUnidirectionalAndTable(queryPlan.Forge, streamJoinAnalysisResult);
if (Log.IsDebugEnabled) {
Log.Debug(methodName + "2-Stream queryPlan=" + queryPlan);
}
return queryPlan;
}
bool hasPreferMergeJoin = HintEnum.PREFER_MERGE_JOIN.GetHint(statementRawInfo.Annotations) != null;
bool hasForceNestedIter = HintEnum.FORCE_NESTED_ITER.GetHint(statementRawInfo.Annotations) != null;
bool isAllInnerJoins = outerJoinDescList.Length == 0 ||
OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList);
if (isAllInnerJoins && !hasPreferMergeJoin) {
QueryPlanForgeDesc queryPlan = NStreamQueryPlanBuilder.Build(
queryGraph,
typesPerStream,
historicalViewableDesc,
dependencyGraph,
historicalStreamIndexLists,
hasForceNestedIter,
streamJoinAnalysisResult.UniqueKeys,
streamJoinAnalysisResult.TablesPerStream,
streamJoinAnalysisResult,
statementRawInfo,
services.SerdeResolver);
if (queryPlan != null) {
RemoveUnidirectionalAndTable(queryPlan.Forge, streamJoinAnalysisResult);
if (Log.IsDebugEnabled) {
Log.Debug(methodName + "N-Stream inner-join queryPlan=" + queryPlan);
}
return queryPlan;
}
if (isQueryPlanLogging && QUERY_PLAN_LOG.IsInfoEnabled) {
Log.Info("Switching to Outer-NStream algorithm for query plan");
}
}
QueryPlanForgeDesc queryPlanX = NStreamOuterQueryPlanBuilder.Build(
queryGraph,
outerJoinDescList,
streamNames,
typesPerStream,
historicalViewableDesc,
dependencyGraph,
historicalStreamIndexLists,
streamJoinAnalysisResult.UniqueKeys,
streamJoinAnalysisResult.TablesPerStream,
streamJoinAnalysisResult,
statementRawInfo,
services);
RemoveUnidirectionalAndTable(queryPlanX.Forge, streamJoinAnalysisResult);
return queryPlanX;
}
private StmtForgeMethodResult Build(
string @namespace,
string classPostfix,
StatementCompileTimeServices services)
{
var compileResult = CreateWindowUtil.HandleCreateWindow(@base, services);
var namedWindowType = compileResult.FilterSpecCompiled.FilterForEventType;
// view must be non-empty list
var createWindowDesc = @base.StatementSpec.Raw.CreateWindowDesc;
if (createWindowDesc.ViewSpecs.IsEmpty()) {
throw new ExprValidationException(NamedWindowManagementServiceConstants.ERROR_MSG_DATAWINDOWS);
}
if (services.NamedWindowCompileTimeResolver.Resolve(createWindowDesc.WindowName) != null) {
throw new ExprValidationException(
"Named window named '" + createWindowDesc.WindowName + "' has already been declared");
}
// build forge
var activator = new ViewableActivatorFilterForge(compileResult.FilterSpecCompiled, false, 0, false, -1);
var viewSpecs = createWindowDesc.ViewSpecs;
var viewArgs = new ViewFactoryForgeArgs(
0,
false,
-1,
createWindowDesc.StreamSpecOptions,
createWindowDesc.WindowName,
@base.StatementRawInfo,
services);
var viewForges = ViewFactoryForgeUtil.CreateForges(viewSpecs.ToArray(), viewArgs, namedWindowType);
IList<ScheduleHandleCallbackProvider> schedules = new List<ScheduleHandleCallbackProvider>();
ViewFactoryForgeUtil.DetermineViewSchedules(viewForges, schedules);
VerifyDataWindowViewFactoryChain(viewForges);
var optionalUniqueKeyProps =
StreamJoinAnalysisResultCompileTime.GetUniqueCandidateProperties(
viewForges,
@base.StatementSpec.Annotations);
var uniqueKeyProArray = optionalUniqueKeyProps == null ? null : optionalUniqueKeyProps.ToArray();
NamedWindowMetaData insertFromNamedWindow = null;
ExprNode insertFromFilter = null;
if (createWindowDesc.IsInsert || createWindowDesc.InsertFilter != null) {
var name = createWindowDesc.AsEventTypeName;
insertFromNamedWindow = services.NamedWindowCompileTimeResolver.Resolve(name);
if (insertFromNamedWindow == null) {
throw new ExprValidationException(
"A named window by name '" +
name +
"' could not be located, the insert-keyword requires an existing named window");
}
insertFromFilter = createWindowDesc.InsertFilter;
if (insertFromFilter != null) {
var checkMinimal = ExprNodeUtilityValidate.IsMinimalExpression(insertFromFilter);
if (checkMinimal != null) {
throw new ExprValidationException("Create window where-clause may not have " + checkMinimal);
}
StreamTypeService streamTypeService = new StreamTypeServiceImpl(
insertFromNamedWindow.EventType,
name,
true);
var validationContext = new ExprValidationContextBuilder(
streamTypeService,
@base.StatementRawInfo,
services).Build();
insertFromFilter = ExprNodeUtilityValidate.GetValidatedSubtree(
ExprNodeOrigin.CREATEWINDOWFILTER,
insertFromFilter,
validationContext);
}
}
// handle output format
var defaultSelectAllSpec = new StatementSpecCompiled();
defaultSelectAllSpec.SelectClauseCompiled.WithSelectExprList(new SelectClauseElementWildcard());
defaultSelectAllSpec.Raw.SelectStreamDirEnum = SelectClauseStreamSelectorEnum.RSTREAM_ISTREAM_BOTH;
StreamTypeService typeService = new StreamTypeServiceImpl(
new[] {namedWindowType},
new[] {createWindowDesc.WindowName},
new[] {true},
false,
false);
var resultSetProcessor = ResultSetProcessorFactoryFactory.GetProcessorPrototype(
new ResultSetSpec(defaultSelectAllSpec),
typeService,
null,
new bool[1],
false,
@base.ContextPropertyRegistry,
false,
false,
@base.StatementRawInfo,
services);
var classNameRSP = CodeGenerationIDGenerator.GenerateClassNameSimple(
typeof(ResultSetProcessorFactoryProvider),
classPostfix);
var selectSubscriberDescriptor = resultSetProcessor.SelectSubscriberDescriptor;
var forge = new StatementAgentInstanceFactoryCreateNWForge(
activator,
createWindowDesc.WindowName,
viewForges,
insertFromNamedWindow,
insertFromFilter,
compileResult.AsEventType,
classNameRSP);
// add named window
var isBatchingDataWindow = DetermineBatchingDataWindow(viewForges);
var virtualDataWindow = viewForges[0] is VirtualDWViewFactoryForge;
var isEnableIndexShare = virtualDataWindow ||
HintEnum.ENABLE_WINDOW_SUBQUERY_INDEXSHARE.GetHint(
@base.StatementSpec.Annotations) !=
null;
var metaData = new NamedWindowMetaData(
namedWindowType,
@base.ModuleName,
@base.ContextName,
uniqueKeyProArray,
isBatchingDataWindow,
isEnableIndexShare,
compileResult.AsEventType,
virtualDataWindow);
services.NamedWindowCompileTimeRegistry.NewNamedWindow(metaData);
// build forge list
IList<StmtClassForgable> forgables = new List<StmtClassForgable>(2);
var statementFieldsClassName =
CodeGenerationIDGenerator.GenerateClassNameSimple(typeof(StatementFields), classPostfix);
var packageScope = new CodegenNamespaceScope(
@namespace,
statementFieldsClassName,
services.IsInstrumented);
forgables.Add(
new StmtClassForgableRSPFactoryProvider(
classNameRSP,
resultSetProcessor,
packageScope,
@base.StatementRawInfo));
var aiFactoryProviderClassName = CodeGenerationIDGenerator.GenerateClassNameSimple(
typeof(StatementAIFactoryProvider),
classPostfix);
var aiFactoryForgable = new StmtClassForgableAIFactoryProviderCreateNW(
aiFactoryProviderClassName,
packageScope,
forge,
createWindowDesc.WindowName);
forgables.Add(aiFactoryForgable);
var statementProviderClassName =
CodeGenerationIDGenerator.GenerateClassNameSimple(typeof(StatementProvider), classPostfix);
var informationals = StatementInformationalsUtil.GetInformationals(
@base,
Collections.SingletonList(compileResult.FilterSpecCompiled),
schedules,
new EmptyList<NamedWindowConsumerStreamSpec>(),
true,
selectSubscriberDescriptor,
packageScope,
services);
forgables.Add(
new StmtClassForgableStmtProvider(
aiFactoryProviderClassName,
statementProviderClassName,
informationals,
packageScope));
forgables.Add(new StmtClassForgableStmtFields(statementFieldsClassName, packageScope, 1));
return new StmtForgeMethodResult(
forgables,
Collections.SingletonList(compileResult.FilterSpecCompiled),
schedules,
new EmptyList<NamedWindowConsumerStreamSpec>(),
new EmptyList<FilterSpecParamExprNodeForge>());
}
internal static QueryPlanForgeDesc Build(
QueryGraphForge queryGraph,
OuterJoinDesc[] outerJoinDescList,
string[] streamNames,
EventType[] typesPerStream,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
string[][][] indexedStreamsUniqueProps,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
if (Log.IsDebugEnabled) {
Log.Debug(".build filterQueryGraph=" + queryGraph);
}
var numStreams = queryGraph.NumStreams;
var planNodeSpecs = new QueryPlanNodeForge[numStreams];
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Build index specifications
var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(
queryGraph,
typesPerStream,
indexedStreamsUniqueProps);
// any historical streams don't get indexes, the lookup strategy accounts for cached indexes
if (historicalViewableDesc.IsHistorical) {
for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) {
if (historicalViewableDesc.Historical[i]) {
indexSpecs[i] = null;
}
}
}
// Build graph of the outer join to inner table relationships.
// Build a map of inner joins.
OuterInnerDirectionalGraph outerInnerGraph;
InnerJoinGraph innerJoinGraph;
if (outerJoinDescList.Length > 0) {
outerInnerGraph = GraphOuterJoins(numStreams, outerJoinDescList);
innerJoinGraph = InnerJoinGraph.GraphInnerJoins(numStreams, outerJoinDescList);
}
else {
// all inner joins - thereby no (or empty) directional graph
outerInnerGraph = new OuterInnerDirectionalGraph(numStreams);
innerJoinGraph = new InnerJoinGraph(numStreams, true);
}
if (Log.IsDebugEnabled) {
Log.Debug(".build directional graph=" + outerInnerGraph.Print());
}
// For each stream determine the query plan
for (var streamNo = 0; streamNo < numStreams; streamNo++) {
// no plan for historical streams that are dependent upon other streams
if (historicalViewableDesc.Historical[streamNo] && dependencyGraph.HasDependency(streamNo)) {
planNodeSpecs[streamNo] = new QueryPlanNodeNoOpForge();
continue;
}
QueryPlanNodeForgeDesc desc = BuildPlanNode(
numStreams,
streamNo,
streamNames,
queryGraph,
outerInnerGraph,
outerJoinDescList,
innerJoinGraph,
indexSpecs,
typesPerStream,
historicalViewableDesc.Historical,
dependencyGraph,
historicalStreamIndexLists,
tablesPerStream,
streamJoinAnalysisResult,
statementRawInfo,
services);
QueryPlanNodeForge queryPlanNode = desc.Forge;
additionalForgeables.AddAll(desc.AdditionalForgeables);
if (Log.IsDebugEnabled) {
Log.Debug(
".build spec for stream '" +
streamNames[streamNo] +
"' number " +
streamNo +
" is " +
queryPlanNode);
}
planNodeSpecs[streamNo] = queryPlanNode;
}
var queryPlan = new QueryPlanForge(indexSpecs, planNodeSpecs);
if (Log.IsDebugEnabled) {
Log.Debug(".build query plan=" + queryPlan);
}
return new QueryPlanForgeDesc(queryPlan, additionalForgeables);
}
private static LookupInstructionPlanDesc BuildLookupInstructions(
int rootStreamNum,
IDictionary<int, int[]> substreamsPerStream,
bool[] requiredPerStream,
string[] streamNames,
QueryGraphForge queryGraph,
QueryPlanIndexForge[] indexSpecs,
EventType[] typesPerStream,
OuterJoinDesc[] outerJoinDescList,
bool[] isHistorical,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
var result = new List<LookupInstructionPlanForge>();
var additionalForgeables = new List<StmtClassForgeableFactory>();
foreach (var fromStream in substreamsPerStream.Keys) {
var substreams = substreamsPerStream.Get(fromStream);
// for streams with no substreams we don't need to look up
if (substreams.Length == 0) {
continue;
}
var plans = new TableLookupPlanForge[substreams.Length];
var historicalPlans = new HistoricalDataPlanNodeForge[substreams.Length];
for (var i = 0; i < substreams.Length; i++) {
var toStream = substreams[i];
if (isHistorical[toStream]) {
// There may not be an outer-join descriptor, use if provided to build the associated expression
ExprNode outerJoinExpr = null;
if (outerJoinDescList.Length > 0) {
OuterJoinDesc outerJoinDesc;
if (toStream == 0) {
outerJoinDesc = outerJoinDescList[0];
}
else {
outerJoinDesc = outerJoinDescList[toStream - 1];
}
outerJoinExpr = outerJoinDesc.MakeExprNode(statementRawInfo, services);
}
if (historicalStreamIndexLists[toStream] == null) {
historicalStreamIndexLists[toStream] = new HistoricalStreamIndexListForge(
toStream,
typesPerStream,
queryGraph);
}
historicalStreamIndexLists[toStream].AddIndex(fromStream);
historicalPlans[i] = new HistoricalDataPlanNodeForge(
toStream,
rootStreamNum,
fromStream,
typesPerStream.Length,
outerJoinExpr == null ? null : outerJoinExpr.Forge);
}
else {
TableLookupPlanDesc planDesc = NStreamQueryPlanBuilder.CreateLookupPlan(
queryGraph,
fromStream,
toStream,
streamJoinAnalysisResult.IsVirtualDW(toStream),
indexSpecs[toStream],
typesPerStream,
tablesPerStream[toStream],
statementRawInfo,
SerdeCompileTimeResolverNonHA.INSTANCE);
plans[i] = planDesc.Forge;
additionalForgeables.AddAll(planDesc.AdditionalForgeables);
}
}
var fromStreamName = streamNames[fromStream];
var instruction = new LookupInstructionPlanForge(
fromStream,
fromStreamName,
substreams,
plans,
historicalPlans,
requiredPerStream);
result.Add(instruction);
}
return new LookupInstructionPlanDesc(result, additionalForgeables);
}
private static QueryPlanNodeForgeDesc BuildPlanNode(
int numStreams,
int streamNo,
string[] streamNames,
QueryGraphForge queryGraph,
OuterInnerDirectionalGraph outerInnerGraph,
OuterJoinDesc[] outerJoinDescList,
InnerJoinGraph innerJoinGraph,
QueryPlanIndexForge[] indexSpecs,
EventType[] typesPerStream,
bool[] isHistorical,
DependencyGraph dependencyGraph,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
// For each stream build an array of substreams, considering required streams (inner joins) first
// The order is relevant therefore preserving order via a LinkedHashMap.
var substreamsPerStream = new LinkedHashMap<int, int[]>();
var requiredPerStream = new bool[numStreams];
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Recursive populating the required (outer) and optional (inner) relationships
// of this stream and the substream
ISet<int> completedStreams = new HashSet<int>();
// keep track of tree path as only those stream events are always available to historical streams
var streamCallStack = new Stack<int>();
streamCallStack.Push(streamNo);
// For all inner-joins, the algorithm is slightly different
if (innerJoinGraph.IsAllInnerJoin) {
requiredPerStream.Fill(true);
RecursiveBuildInnerJoin(
streamNo,
streamCallStack,
queryGraph,
completedStreams,
substreamsPerStream,
dependencyGraph);
// compute a best chain to see if all streams are handled and add the remaining
var bestChain = NStreamQueryPlanBuilder.ComputeBestPath(streamNo, queryGraph, dependencyGraph);
AddNotYetNavigated(streamNo, numStreams, substreamsPerStream, bestChain);
}
else {
RecursiveBuild(
streamNo,
streamCallStack,
queryGraph,
outerInnerGraph,
innerJoinGraph,
completedStreams,
substreamsPerStream,
requiredPerStream,
dependencyGraph);
}
// verify the substreamsPerStream, all streams must exists and be linked
VerifyJoinedPerStream(streamNo, substreamsPerStream);
// build list of instructions for lookup
LookupInstructionPlanDesc lookupDesc = BuildLookupInstructions(
streamNo,
substreamsPerStream,
requiredPerStream,
streamNames,
queryGraph,
indexSpecs,
typesPerStream,
outerJoinDescList,
isHistorical,
historicalStreamIndexLists,
tablesPerStream,
streamJoinAnalysisResult,
statementRawInfo,
services);
var lookupInstructions = lookupDesc.Forges;
additionalForgeables.AddAll(lookupDesc.AdditionalForgeables);
// build historical index and lookup strategies
foreach (var lookups in lookupInstructions) {
foreach (var historical in lookups.HistoricalPlans) {
if (historical == null) {
continue;
}
JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum]
.GetStrategy(historical.LookupStreamNum, statementRawInfo, services.SerdeResolver);
historical.HistoricalIndexLookupStrategy = desc.LookupForge;
historical.PollResultIndexingStrategy = desc.IndexingForge;
additionalForgeables.AddAll(desc.AdditionalForgeables);
}
}
// build strategy tree for putting the result back together
var assemblyTopNodeFactory = AssemblyStrategyTreeBuilder.Build(
streamNo,
substreamsPerStream,
requiredPerStream);
var assemblyInstructionFactories =
BaseAssemblyNodeFactory.GetDescendentNodesBottomUp(assemblyTopNodeFactory);
var forge = new LookupInstructionQueryPlanNodeForge(
streamNo,
streamNames[streamNo],
numStreams,
requiredPerStream,
lookupInstructions,
assemblyInstructionFactories);
return new QueryPlanNodeForgeDesc(forge, additionalForgeables);
}
public static JoinSetComposerPrototypeDesc MakeComposerPrototype(
StatementSpecCompiled spec,
StreamJoinAnalysisResultCompileTime joinAnalysisResult,
StreamTypeService typeService,
HistoricalViewableDesc historicalViewableDesc,
bool isOnDemandQuery,
bool hasAggregations,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices compileTimeServices)
{
var streamTypes = typeService.EventTypes;
var streamNames = typeService.StreamNames;
var whereClause = spec.Raw.WhereClause;
var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan;
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Determine if there is a historical stream, and what dependencies exist
var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false);
for (var i = 0; i < streamTypes.Length; i++) {
if (historicalViewableDesc.Historical[i]) {
var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i];
historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn);
}
}
if (Log.IsDebugEnabled) {
Log.Debug("Dependency graph: " + historicalDependencyGraph);
}
// Handle a join with a database or other historical data source for 2 streams
var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList);
if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) {
var desc = MakeComposerHistorical2Stream(
outerJoinDescs,
whereClause,
streamTypes,
streamNames,
historicalViewableDesc,
queryPlanLogging,
statementRawInfo,
compileTimeServices);
return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables);
}
var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs);
// Query graph for graph relationships between streams/historicals
// For outer joins the query graph will just contain outer join relationships
var hint = ExcludePlanHint.GetHint(
typeService.StreamNames,
statementRawInfo,
compileTimeServices);
var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false);
if (outerJoinDescs.Length > 0) {
OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph);
}
}
// Let the query graph reflect the where-clause
if (whereClause != null) {
// Analyze relationships between streams using the optional filter expression.
// Relationships are properties in AND and EQUALS nodes of joins.
FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph);
}
// Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c)
QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph);
}
}
// Historical index lists
var historicalStreamIndexLists =
new HistoricalStreamIndexListForge[streamTypes.Length];
var queryPlanDesc = QueryPlanBuilder.GetPlan(
streamTypes,
outerJoinDescs,
queryGraph,
typeService.StreamNames,
historicalViewableDesc,
historicalDependencyGraph,
historicalStreamIndexLists,
joinAnalysisResult,
queryPlanLogging,
statementRawInfo,
compileTimeServices);
QueryPlanForge queryPlan = queryPlanDesc.Forge;
additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables);
// remove unused indexes - consider all streams or all unidirectional
var usedIndexes = new HashSet<TableLookupIndexReqKey>();
var indexSpecs = queryPlan.IndexSpecs;
for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) {
var planNode = queryPlan.ExecNodeSpecs[streamNum];
planNode?.AddIndexes(usedIndexes);
}
foreach (var indexSpec in indexSpecs) {
if (indexSpec == null) {
continue;
}
var items = indexSpec.Items;
var indexNames = items.Keys.ToArray();
foreach (var indexName in indexNames) {
if (!usedIndexes.Contains(indexName)) {
items.Remove(indexName);
}
}
}
// plan multikeys
IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys(
indexSpecs, statementRawInfo, compileTimeServices);
additionalForgeables.AddAll(multikeyForgeables);
QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook(
spec.Annotations,
compileTimeServices.ImportServiceCompileTime);
if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) {
QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan());
hook?.Join(queryPlan);
}
var selectsRemoveStream =
spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null;
var joinRemoveStream = selectsRemoveStream || hasAggregations;
ExprNode postJoinEvaluator;
if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf(
isOuterJoins,
joinAnalysisResult.IsUnidirectional,
joinAnalysisResult.IsPureSelfJoin)) {
postJoinEvaluator = GetFilterExpressionInclOnClause(
spec.Raw.WhereClause,
outerJoinDescs,
statementRawInfo,
compileTimeServices);
}
else {
postJoinEvaluator = spec.Raw.WhereClause;
}
JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge(
typeService.EventTypes,
postJoinEvaluator,
outerJoinDescs.Length > 0,
queryPlan,
joinAnalysisResult,
typeService.StreamNames,
joinRemoveStream,
historicalViewableDesc.IsHistorical);
return new JoinSetComposerPrototypeDesc(forge, additionalForgeables);
}
public FAFQueryMethodSelectDesc(
StatementSpecCompiled statementSpec,
Compilable compilable,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
Annotations = statementSpec.Annotations;
ContextName = statementSpec.Raw.OptionalContextName;
var queryPlanLogging = services.Configuration.Common.Logging.IsEnableQueryPlan;
if (queryPlanLogging) {
QUERY_PLAN_LOG.Info("Query plans for Fire-and-forget query '" + compilable.ToEPL() + "'");
}
HasTableAccess = statementSpec.TableAccessNodes != null && statementSpec.TableAccessNodes.Count > 0;
foreach (var streamSpec in statementSpec.StreamSpecs) {
HasTableAccess |= streamSpec is TableQueryStreamSpec;
}
HasTableAccess |= StatementLifecycleSvcUtil.IsSubqueryWithTable(
statementSpec.SubselectNodes, services.TableCompileTimeResolver);
IsDistinct = statementSpec.SelectClauseCompiled.IsDistinct;
FAFQueryMethodHelper.ValidateFAFQuery(statementSpec);
var numStreams = statementSpec.StreamSpecs.Length;
var typesPerStream = new EventType[numStreams];
var namesPerStream = new string[numStreams];
var eventTypeNames = new string[numStreams];
Processors = new FireAndForgetProcessorForge[numStreams];
ConsumerFilters = new ExprNode[numStreams];
// check context partition use
if (statementSpec.Raw.OptionalContextName != null) {
if (numStreams > 1) {
throw new ExprValidationException(
"Joins in runtime queries for context partitions are not supported");
}
}
// resolve types and processors
for (var i = 0; i < numStreams; i++) {
var streamSpec = statementSpec.StreamSpecs[i];
Processors[i] = FireAndForgetProcessorForgeFactory.ValidateResolveProcessor(streamSpec);
if (numStreams > 1 && Processors[i].ContextName != null) {
throw new ExprValidationException(
"Joins against named windows that are under context are not supported");
}
var streamName = Processors[i].NamedWindowOrTableName;
if (streamSpec.OptionalStreamName != null) {
streamName = streamSpec.OptionalStreamName;
}
namesPerStream[i] = streamName;
typesPerStream[i] = Processors[i].EventTypeRspInputEvents;
eventTypeNames[i] = typesPerStream[i].Name;
IList<ExprNode> consumerFilterExprs;
if (streamSpec is NamedWindowConsumerStreamSpec) {
var namedSpec = (NamedWindowConsumerStreamSpec) streamSpec;
consumerFilterExprs = namedSpec.FilterExpressions;
}
else {
var tableSpec = (TableQueryStreamSpec) streamSpec;
consumerFilterExprs = tableSpec.FilterExpressions;
}
ConsumerFilters[i] = ExprNodeUtilityMake.ConnectExpressionsByLogicalAndWhenNeeded(consumerFilterExprs);
}
// compile filter to optimize access to named window
var optionalStreamsIfAny = OuterJoinAnalyzer.OptionalStreamsIfAny(statementSpec.Raw.OuterJoinDescList);
var types = new StreamTypeServiceImpl(
typesPerStream,
namesPerStream,
new bool[numStreams],
false,
optionalStreamsIfAny);
var excludePlanHint = ExcludePlanHint.GetHint(types.StreamNames, statementRawInfo, services);
QueryGraph = new QueryGraphForge(numStreams, excludePlanHint, false);
if (statementSpec.Raw.WhereClause != null) {
for (var i = 0; i < numStreams; i++) {
try {
var validationContext = new ExprValidationContextBuilder(types, statementRawInfo, services)
.WithAllowBindingConsumption(true)
.WithIsFilterExpression(true)
.Build();
var validated = ExprNodeUtilityValidate.GetValidatedSubtree(
ExprNodeOrigin.FILTER,
statementSpec.Raw.WhereClause,
validationContext);
FilterExprAnalyzer.Analyze(validated, QueryGraph, false);
}
catch (Exception ex) {
Log.Warn("Unexpected exception analyzing filter paths: " + ex.Message, ex);
}
}
}
// handle subselects
// first we create streams for subselects, if there are any
var @base = new StatementBaseInfo(compilable, statementSpec, null, statementRawInfo, null);
var subqueryNamedWindowConsumers = new List<NamedWindowConsumerStreamSpec>();
SubSelectActivationDesc subSelectActivationDesc = SubSelectHelperActivations.CreateSubSelectActivation(
EmptyList<FilterSpecCompiled>.Instance, subqueryNamedWindowConsumers, @base, services);
IDictionary<ExprSubselectNode, SubSelectActivationPlan> subselectActivation = subSelectActivationDesc.Subselects;
AdditionalForgeables.AddAll(subSelectActivationDesc.AdditionalForgeables);
SubSelectHelperForgePlan subSelectForgePlan = SubSelectHelperForgePlanner.PlanSubSelect(
@base, subselectActivation, namesPerStream, typesPerStream, eventTypeNames, services);
SubselectForges = subSelectForgePlan.Subselects;
AdditionalForgeables.AddAll(subSelectForgePlan.AdditionalForgeables);
// obtain result set processor
var isIStreamOnly = new bool[namesPerStream.Length];
isIStreamOnly.Fill(true);
StreamTypeService typeService = new StreamTypeServiceImpl(
typesPerStream,
namesPerStream,
isIStreamOnly,
true,
optionalStreamsIfAny);
WhereClause = EPStatementStartMethodHelperValidate.ValidateNodes(
statementSpec.Raw,
typeService,
null,
statementRawInfo,
services);
var resultSetSpec = new ResultSetSpec(statementSpec);
ResultSetProcessor = ResultSetProcessorFactoryFactory.GetProcessorPrototype(
resultSetSpec,
typeService,
null,
new bool[0],
true,
null,
true,
false,
statementRawInfo,
services);
AdditionalForgeables.AddAll(ResultSetProcessor.AdditionalForgeables);
// plan table access
TableAccessForges = ExprTableEvalHelperPlan.PlanTableAccess(statementSpec.Raw.TableExpressions);
// plan joins or simple queries
if (numStreams > 1) {
var streamJoinAnalysisResult = new StreamJoinAnalysisResultCompileTime(numStreams);
CompatExtensions.Fill(streamJoinAnalysisResult.NamedWindowsPerStream, (NamedWindowMetaData) null);
for (var i = 0; i < numStreams; i++) {
var uniqueIndexes = Processors[i].UniqueIndexes;
streamJoinAnalysisResult.UniqueKeys[i] = uniqueIndexes;
}
var hasAggregations = ResultSetProcessor.ResultSetProcessorType.IsAggregated();
var desc = JoinSetComposerPrototypeForgeFactory.MakeComposerPrototype(
statementSpec,
streamJoinAnalysisResult,
types,
new HistoricalViewableDesc(numStreams),
true,
hasAggregations,
statementRawInfo,
services);
AdditionalForgeables.AddAll(desc.AdditionalForgeables);
Joins = desc.Forge;
}
else {
Joins = null;
}
var multiKeyPlan = MultiKeyPlanner.PlanMultiKeyDistinct(
IsDistinct, ResultSetProcessor.ResultEventType, statementRawInfo, SerdeCompileTimeResolverNonHA.INSTANCE);
AdditionalForgeables.AddAll(multiKeyPlan.MultiKeyForgeables);
DistinctMultiKey = multiKeyPlan.ClassRef;
}
