/// <summary>
/// Build index specification from navigability INFO.
/// <para/>
/// Looks at each stream and determines which properties in the stream must be indexed
/// in order for other streams to look up into the stream. Determines the unique set of
/// properties to avoid building duplicate indexes on the same set of properties.
/// </summary>
/// <param name="queryGraph">navigability INFO</param>
/// <param name="typePerStream">The type per stream.</param>
/// <param name="indexedStreamsUniqueProps">The indexed streams unique props.</param>
/// <returns>query index specs for each stream</returns>
public static QueryPlanIndex[] BuildIndexSpec(QueryGraph queryGraph, EventType[] typePerStream, String[][][] indexedStreamsUniqueProps)
{
var numStreams = queryGraph.NumStreams;
var indexSpecs = new QueryPlanIndex[numStreams];
// For each stream compile a list of index property sets.
for (int streamIndexed = 0; streamIndexed < numStreams; streamIndexed++)
{
var indexesSet = new List <QueryPlanIndexItem>();
// Look at the index from the viewpoint of the stream looking up in the index
for (int streamLookup = 0; streamLookup < numStreams; streamLookup++)
{
if (streamIndexed == streamLookup)
{
continue;
}
var value = queryGraph.GetGraphValue(streamLookup, streamIndexed);
var hashKeyAndIndexProps = value.HashKeyProps;
// Sort index properties, but use the sorted properties only to eliminate duplicates
var hashIndexProps = hashKeyAndIndexProps.Indexed;
var hashKeyProps = hashKeyAndIndexProps.Keys;
var indexCoercionTypes = CoercionUtil.GetCoercionTypesHash(typePerStream, streamLookup, streamIndexed, hashKeyProps, hashIndexProps);
var hashCoercionTypeArr = indexCoercionTypes.CoercionTypes;
var rangeAndIndexProps = value.RangeProps;
var rangeIndexProps = rangeAndIndexProps.Indexed;
var rangeKeyProps = rangeAndIndexProps.Keys;
var rangeCoercionTypes = CoercionUtil.GetCoercionTypesRange(typePerStream, streamIndexed, rangeIndexProps, rangeKeyProps);
var rangeCoercionTypeArr = rangeCoercionTypes.CoercionTypes;
if (hashIndexProps.Count == 0 && rangeIndexProps.Count == 0)
{
QueryGraphValuePairInKWSingleIdx singles = value.InKeywordSingles;
if (!singles.Key.IsEmpty())
{
String indexedProp = singles.Indexed[0];
QueryPlanIndexItem indexItem = new QueryPlanIndexItem(new String[] { indexedProp }, null, null, null, false, null);
CheckDuplicateOrAdd(indexItem, indexesSet);
}
IList <QueryGraphValuePairInKWMultiIdx> multis = value.InKeywordMulti;
if (!multis.IsEmpty())
{
QueryGraphValuePairInKWMultiIdx multi = multis[0];
foreach (ExprNode propIndexed in multi.Indexed)
{
ExprIdentNode identNode = (ExprIdentNode)propIndexed;
QueryPlanIndexItem indexItem = new QueryPlanIndexItem(new String[] { identNode.ResolvedPropertyName }, null, null, null, false, null);
CheckDuplicateOrAdd(indexItem, indexesSet);
}
}
continue;
}
// reduce to any unique index if applicable
var unique = false;
var reduced = QueryPlanIndexUniqueHelper.ReduceToUniqueIfPossible(hashIndexProps, hashCoercionTypeArr, hashKeyProps, indexedStreamsUniqueProps[streamIndexed]);
if (reduced != null)
{
hashIndexProps = reduced.PropertyNames;
hashCoercionTypeArr = reduced.CoercionTypes;
unique = true;
rangeIndexProps = new String[0];
rangeCoercionTypeArr = new Type[0];
}
var proposed = new QueryPlanIndexItem(
hashIndexProps,
hashCoercionTypeArr,
rangeIndexProps,
rangeCoercionTypeArr,
unique, null);
CheckDuplicateOrAdd(proposed, indexesSet);
}
// create full-table-scan
if (indexesSet.IsEmpty())
{
indexesSet.Add(new QueryPlanIndexItem(null, null, null, null, false, null));
}
indexSpecs[streamIndexed] = QueryPlanIndex.MakeIndex(indexesSet);
}
return(indexSpecs);
}
private static void Add(QueryGraph queryGraph, ExprIdentNode identNodeLeft, ExprIdentNode identNodeRight)
{
queryGraph.AddStrictEquals(identNodeLeft.StreamId, identNodeLeft.ResolvedPropertyName, identNodeLeft,
identNodeRight.StreamId, identNodeRight.ResolvedPropertyName, identNodeRight);
}
/// <summary>
/// Build query plan using the filter.
/// </summary>
/// <param name="typesPerStream">event types for each stream</param>
/// <param name="outerJoinDescList">list of outer join criteria, or null if there are no outer joins</param>
/// <param name="queryGraph">relationships between streams based on filter expressions and outer-join on-criteria</param>
/// <param name="streamNames">names of streams</param>
/// <param name="historicalViewableDesc">The historical viewable desc.</param>
/// <param name="dependencyGraph">dependencies between historical streams</param>
/// <param name="historicalStreamIndexLists">index management, populated for the query plan</param>
/// <param name="streamJoinAnalysisResult">The stream join analysis result.</param>
/// <param name="isQueryPlanLogging">if set to <c>true</c> [is query plan logging].</param>
/// <param name="annotations">The annotations.</param>
/// <param name="exprEvaluatorContext">The expr evaluator context.</param>
/// <returns>
/// query plan
/// </returns>
/// <exception cref="System.ArgumentException">
/// Number of join stream types is less then 2
/// or
/// Too many outer join descriptors found
/// </exception>
/// <throws>ExprValidationException if the query plan fails</throws>
public static QueryPlan GetPlan(EventType[] typesPerStream,
OuterJoinDesc[] outerJoinDescList,
QueryGraph queryGraph,
string[] streamNames,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexList[] historicalStreamIndexLists,
StreamJoinAnalysisResult streamJoinAnalysisResult,
bool isQueryPlanLogging,
Attribute[] annotations,
ExprEvaluatorContext exprEvaluatorContext)
{
string methodName = ".getPlan ";
int numStreams = typesPerStream.Length;
if (numStreams < 2)
{
throw new ArgumentException("Number of join stream types is less then 2");
}
if (outerJoinDescList.Length >= numStreams)
{
throw new ArgumentException("Too many outer join descriptors found");
}
if (numStreams == 2)
{
OuterJoinType?outerJoinType = null;
if (outerJoinDescList.Length > 0)
{
outerJoinType = outerJoinDescList[0].OuterJoinType;
}
QueryPlan queryPlanX = TwoStreamQueryPlanBuilder.Build(typesPerStream, queryGraph, outerJoinType, streamJoinAnalysisResult.UniqueKeys, streamJoinAnalysisResult.TablesPerStream);
RemoveUnidirectionalAndTable(queryPlanX, streamJoinAnalysisResult);
if (log.IsDebugEnabled)
{
log.Debug(methodName + "2-Stream queryPlan=" + queryPlanX);
}
return(queryPlanX);
}
bool hasPreferMergeJoin = HintEnum.PREFER_MERGE_JOIN.GetHint(annotations) != null;
bool hasForceNestedIter = HintEnum.FORCE_NESTED_ITER.GetHint(annotations) != null;
bool isAllInnerJoins = outerJoinDescList.Length == 0 || OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList);
if (isAllInnerJoins && !hasPreferMergeJoin)
{
QueryPlan queryPlanX = NStreamQueryPlanBuilder.Build(queryGraph, typesPerStream,
historicalViewableDesc, dependencyGraph, historicalStreamIndexLists,
hasForceNestedIter, streamJoinAnalysisResult.UniqueKeys,
streamJoinAnalysisResult.TablesPerStream);
if (queryPlanX != null)
{
RemoveUnidirectionalAndTable(queryPlanX, streamJoinAnalysisResult);
if (log.IsDebugEnabled)
{
log.Debug(methodName + "Count-Stream inner-join queryPlan=" + queryPlanX);
}
return(queryPlanX);
}
if (isQueryPlanLogging && queryPlanLog.IsInfoEnabled)
{
log.Info("Switching to Outer-NStream algorithm for query plan");
}
}
QueryPlan queryPlan = NStreamOuterQueryPlanBuilder.Build(queryGraph, outerJoinDescList, streamNames, typesPerStream,
historicalViewableDesc, dependencyGraph, historicalStreamIndexLists, exprEvaluatorContext, streamJoinAnalysisResult.UniqueKeys,
streamJoinAnalysisResult.TablesPerStream);
RemoveUnidirectionalAndTable(queryPlan, streamJoinAnalysisResult);
return(queryPlan);
}
public static SubordPropPlan GetJoinProps(ExprNode filterExpr, int outsideStreamCount, EventType[] allStreamTypesZeroIndexed, ExcludePlanHint excludePlanHint)
{
// No filter expression means full table scan
if (filterExpr == null)
{
return(new SubordPropPlan());
}
// analyze query graph
var queryGraph = new QueryGraph(outsideStreamCount + 1, excludePlanHint, true);
FilterExprAnalyzer.Analyze(filterExpr, queryGraph, false);
// Build a list of streams and indexes
var joinProps = new LinkedHashMap <String, SubordPropHashKey>();
var rangeProps = new LinkedHashMap <String, SubordPropRangeKey>();
var customIndexOps = Collections.GetEmptyMap <QueryGraphValueEntryCustomKey, QueryGraphValueEntryCustomOperation>();
for (int stream = 0; stream < outsideStreamCount; stream++)
{
int lookupStream = stream + 1;
QueryGraphValue queryGraphValue = queryGraph.GetGraphValue(lookupStream, 0);
QueryGraphValuePairHashKeyIndex hashKeysAndIndexes = queryGraphValue.HashKeyProps;
// determine application functions
foreach (QueryGraphValueDesc item in queryGraphValue.Items)
{
if (item.Entry is QueryGraphValueEntryCustom)
{
if (customIndexOps.IsEmpty())
{
customIndexOps = new Dictionary <QueryGraphValueEntryCustomKey, QueryGraphValueEntryCustomOperation>();
}
QueryGraphValueEntryCustom custom = (QueryGraphValueEntryCustom)item.Entry;
custom.MergeInto(customIndexOps);
}
}
// handle key-lookups
var keyPropertiesJoin = hashKeysAndIndexes.Keys;
var indexPropertiesJoin = hashKeysAndIndexes.Indexed;
if (keyPropertiesJoin.IsNotEmpty())
{
if (keyPropertiesJoin.Count != indexPropertiesJoin.Count)
{
throw new IllegalStateException("Invalid query key and index property collection for stream " + stream);
}
for (int i = 0; i < keyPropertiesJoin.Count; i++)
{
QueryGraphValueEntryHashKeyed keyDesc = keyPropertiesJoin[i];
ExprNode compareNode = keyDesc.KeyExpr;
var keyPropType = compareNode.ExprEvaluator.ReturnType.GetBoxedType();
var indexedPropType = allStreamTypesZeroIndexed[0].GetPropertyType(indexPropertiesJoin[i]).GetBoxedType();
var coercionType = indexedPropType;
if (keyPropType != indexedPropType)
{
coercionType = keyPropType.GetCompareToCoercionType(indexedPropType);
}
SubordPropHashKey desc;
if (keyPropertiesJoin[i] is QueryGraphValueEntryHashKeyedExpr)
{
var keyExpr = (QueryGraphValueEntryHashKeyedExpr)keyPropertiesJoin[i];
var keyStreamNum = keyExpr.IsRequiresKey ? stream : (int?)null;
desc = new SubordPropHashKey(keyDesc, keyStreamNum, coercionType);
}
else
{
var prop = (QueryGraphValueEntryHashKeyedProp)keyDesc;
desc = new SubordPropHashKey(prop, stream, coercionType);
}
joinProps.Put(indexPropertiesJoin[i], desc);
}
}
// handle range lookups
QueryGraphValuePairRangeIndex rangeKeysAndIndexes = queryGraphValue.RangeProps;
var rangeIndexes = rangeKeysAndIndexes.Indexed;
var rangeDescs = rangeKeysAndIndexes.Keys;
if (rangeDescs.IsEmpty())
{
continue;
}
// get all ranges lookups
int count = -1;
foreach (QueryGraphValueEntryRange rangeDesc in rangeDescs)
{
count++;
String rangeIndexProp = rangeIndexes[count];
SubordPropRangeKey subqRangeDesc = rangeProps.Get(rangeIndexProp);
// other streams may specify the start or end endpoint of a range, therefore this operation can be additive
if (subqRangeDesc != null)
{
if (subqRangeDesc.RangeInfo.RangeType.IsRange())
{
continue;
}
// see if we can make this additive by using a range
var relOpOther = (QueryGraphValueEntryRangeRelOp)subqRangeDesc.RangeInfo;
var relOpThis = (QueryGraphValueEntryRangeRelOp)rangeDesc;
QueryGraphRangeConsolidateDesc opsDesc = QueryGraphRangeUtil.GetCanConsolidate(
relOpThis.RangeType,
relOpOther.RangeType);
if (opsDesc != null)
{
ExprNode start;
ExprNode end;
if (!opsDesc.IsReverse)
{
start = relOpOther.Expression;
end = relOpThis.Expression;
}
else
{
start = relOpThis.Expression;
end = relOpOther.Expression;
}
var allowRangeReversal = relOpOther.IsBetweenPart && relOpThis.IsBetweenPart;
var rangeIn = new QueryGraphValueEntryRangeIn(opsDesc.RangeType, start, end, allowRangeReversal);
var indexedPropType = allStreamTypesZeroIndexed[0].GetPropertyType(rangeIndexProp).GetBoxedType();
var coercionType = indexedPropType;
var proposedType = CoercionUtil.GetCoercionTypeRangeIn(indexedPropType, rangeIn.ExprStart, rangeIn.ExprEnd);
if (proposedType != null && proposedType != indexedPropType)
{
coercionType = proposedType;
}
subqRangeDesc = new SubordPropRangeKey(rangeIn, coercionType);
rangeProps.Put(rangeIndexProp, subqRangeDesc);
}
// ignore
continue;
}
// an existing entry has not been found
if (rangeDesc.RangeType.IsRange())
{
var rangeIn = (QueryGraphValueEntryRangeIn)rangeDesc;
var indexedPropType = allStreamTypesZeroIndexed[0].GetPropertyType(rangeIndexProp).GetBoxedType();
var coercionType = indexedPropType;
var proposedType = CoercionUtil.GetCoercionTypeRangeIn(indexedPropType, rangeIn.ExprStart, rangeIn.ExprEnd);
if (proposedType != null && proposedType != indexedPropType)
{
coercionType = proposedType;
}
subqRangeDesc = new SubordPropRangeKey(rangeDesc, coercionType);
}
else
{
var relOp = (QueryGraphValueEntryRangeRelOp)rangeDesc;
var keyPropType = relOp.Expression.ExprEvaluator.ReturnType;
var indexedPropType = allStreamTypesZeroIndexed[0].GetPropertyType(rangeIndexProp).GetBoxedType();
var coercionType = indexedPropType;
if (keyPropType != indexedPropType)
{
coercionType = keyPropType.GetCompareToCoercionType(indexedPropType);
}
subqRangeDesc = new SubordPropRangeKey(rangeDesc, coercionType);
}
rangeProps.Put(rangeIndexProp, subqRangeDesc);
}
}
SubordPropInKeywordSingleIndex inKeywordSingleIdxProp = null;
SubordPropInKeywordMultiIndex inKeywordMultiIdxProp = null;
if (joinProps.IsEmpty() && rangeProps.IsEmpty())
{
for (int stream = 0; stream < outsideStreamCount; stream++)
{
int lookupStream = stream + 1;
QueryGraphValue queryGraphValue = queryGraph.GetGraphValue(lookupStream, 0);
QueryGraphValuePairInKWSingleIdx inkwSingles = queryGraphValue.InKeywordSingles;
if (inkwSingles.Indexed.Length != 0)
{
var keys = inkwSingles.Key[0].KeyExprs;
var key = inkwSingles.Indexed[0];
if (inKeywordSingleIdxProp != null)
{
continue;
}
var coercionType = keys[0].ExprEvaluator.ReturnType; // for in-comparison the same type is required
inKeywordSingleIdxProp = new SubordPropInKeywordSingleIndex(key, coercionType, keys);
}
IList <QueryGraphValuePairInKWMultiIdx> inkwMultis = queryGraphValue.InKeywordMulti;
if (!inkwMultis.IsEmpty())
{
QueryGraphValuePairInKWMultiIdx multi = inkwMultis[0];
inKeywordMultiIdxProp = new SubordPropInKeywordMultiIndex(
ExprNodeUtility.GetIdentResolvedPropertyNames(multi.Indexed),
multi.Indexed[0].ExprEvaluator.ReturnType,
multi.Key.KeyExpr);
}
if (inKeywordSingleIdxProp != null && inKeywordMultiIdxProp != null)
{
inKeywordMultiIdxProp = null;
}
}
}
return(new SubordPropPlan(joinProps, rangeProps, inKeywordSingleIdxProp, inKeywordMultiIdxProp, customIndexOps));
}
/// <summary>
/// Recusivly builds a substream-per-stream ordered tree graph using the
/// join information supplied for outer joins and from the query graph (where clause).
/// <para />Required streams are considered first and their lookup is placed first in the list
/// to gain performance.
/// </summary>
/// <param name="streamNum">is the root stream number that supplies the incoming event to build the tree for</param>
/// <param name="queryGraph">contains where-clause stream relationship info</param>
/// <param name="completedStreams">is a temporary holder for streams already considered</param>
/// <param name="substreamsPerStream">is the ordered, tree-like structure to be filled</param>
/// <param name="streamCallStack">the query plan call stack of streams available via cursor</param>
/// <param name="dependencyGraph">dependencies between historical streams</param>
/// <throws>ExprValidationException if the query planning failed</throws>
internal static void RecursiveBuildInnerJoin(
int streamNum,
Stack <int> streamCallStack,
QueryGraph queryGraph,
ISet <int> completedStreams,
IDictionary <int, int[]> substreamsPerStream,
DependencyGraph dependencyGraph)
{
// add this stream to the set of completed streams
completedStreams.Add(streamNum);
// check if the dependencies have been satisfied
if (dependencyGraph.HasDependency(streamNum))
{
var dependencies = dependencyGraph.GetDependenciesForStream(streamNum);
foreach (var dependentStream in dependencies)
{
if (!streamCallStack.Contains(dependentStream))
{
throw new ExprValidationException("Historical stream " + streamNum + " parameter dependency originating in stream " + dependentStream + " cannot or may not be satisfied by the join");
}
}
}
// Determine the streams we can navigate to from this stream
var navigableStreams = queryGraph.GetNavigableStreams(streamNum);
// remove streams with a dependency on other streams not yet processed
var navigableStreamArr = navigableStreams.ToArray();
foreach (int navigableStream in navigableStreamArr)
{
if (dependencyGraph.HasUnsatisfiedDependency(navigableStream, completedStreams))
{
navigableStreams.Remove(navigableStream);
}
}
// remove those already done
navigableStreams.RemoveAll(completedStreams);
// if we are a leaf node, we are done
if (navigableStreams.IsEmpty())
{
substreamsPerStream.Put(streamNum, new int[0]);
return;
}
// First the outer (required) streams to this stream, then the inner (optional) streams
var substreams = new int[navigableStreams.Count];
substreamsPerStream.Put(streamNum, substreams);
var count = 0;
foreach (int stream in navigableStreams)
{
substreams[count++] = stream;
completedStreams.Add(stream);
}
foreach (int stream in navigableStreams)
{
streamCallStack.Push(stream);
RecursiveBuildInnerJoin(stream, streamCallStack, queryGraph, completedStreams, substreamsPerStream, dependencyGraph);
streamCallStack.Pop();
}
}
/// <summary>
/// Build a query plan based on the stream property relationships indicated in queryGraph.
/// </summary>
/// <param name="queryGraph">navigation info between streams</param>
/// <param name="outerJoinDescList">descriptors for all outer joins</param>
/// <param name="streamNames">stream names</param>
/// <param name="typesPerStream">event types for each stream</param>
/// <param name="historicalViewableDesc">The historical viewable desc.</param>
/// <param name="dependencyGraph">dependencies between historical streams</param>
/// <param name="historicalStreamIndexLists">index management, populated for the query plan</param>
/// <param name="exprEvaluatorContext">context for expression evalauation</param>
/// <param name="indexedStreamsUniqueProps">The indexed streams unique props.</param>
/// <param name="tablesPerStream">The tables per stream.</param>
/// <returns>
/// query plan
/// </returns>
/// <throws>ExprValidationException if the query planning failed</throws>
internal static QueryPlan Build(
QueryGraph queryGraph,
OuterJoinDesc[] outerJoinDescList,
string[] streamNames,
EventType[] typesPerStream,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexList[] historicalStreamIndexLists,
ExprEvaluatorContext exprEvaluatorContext,
string[][][] indexedStreamsUniqueProps,
TableMetadata[] tablesPerStream)
{
if (Log.IsDebugEnabled)
{
Log.Debug(".build queryGraph=" + queryGraph);
}
var numStreams = queryGraph.NumStreams;
var planNodeSpecs = new QueryPlanNode[numStreams];
// Build index specifications
var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(queryGraph, typesPerStream, indexedStreamsUniqueProps);
if (Log.IsDebugEnabled)
{
Log.Debug(".build Index build completed, indexes=" + QueryPlanIndex.Print(indexSpecs));
}
// any historical streams don't get indexes, the lookup strategy accounts for cached indexes
if (historicalViewableDesc.HasHistorical)
{
for (var i = 0; i < historicalViewableDesc.Historical.Length; i++)
{
if (historicalViewableDesc.Historical[i])
{
indexSpecs[i] = null;
}
}
}
// Build graph of the outer join to inner table relationships.
// Build a map of inner joins.
OuterInnerDirectionalGraph outerInnerGraph;
InnerJoinGraph innerJoinGraph;
if (outerJoinDescList.Length > 0)
{
outerInnerGraph = GraphOuterJoins(numStreams, outerJoinDescList);
innerJoinGraph = InnerJoinGraph.GraphInnerJoins(numStreams, outerJoinDescList);
}
else
{
// all inner joins - thereby no (or empty) directional graph
outerInnerGraph = new OuterInnerDirectionalGraph(numStreams);
innerJoinGraph = new InnerJoinGraph(numStreams, true);
}
if (Log.IsDebugEnabled)
{
Log.Debug(".build directional graph=" + outerInnerGraph.Print());
}
// For each stream determine the query plan
for (var streamNo = 0; streamNo < numStreams; streamNo++)
{
// no plan for historical streams that are dependent upon other streams
if ((historicalViewableDesc.Historical[streamNo]) && (dependencyGraph.HasDependency(streamNo)))
{
planNodeSpecs[streamNo] = new QueryPlanNodeNoOp();
continue;
}
var queryPlanNode = BuildPlanNode(numStreams, streamNo, streamNames, queryGraph, outerInnerGraph, outerJoinDescList, innerJoinGraph, indexSpecs, typesPerStream, historicalViewableDesc.Historical, dependencyGraph, historicalStreamIndexLists, exprEvaluatorContext, tablesPerStream);
if (Log.IsDebugEnabled)
{
Log.Debug(".build spec for stream '" + streamNames[streamNo] +
"' number " + streamNo + " is " + queryPlanNode);
}
planNodeSpecs[streamNo] = queryPlanNode;
}
var queryPlan = new QueryPlan(indexSpecs, planNodeSpecs);
if (Log.IsDebugEnabled)
{
Log.Debug(".build query plan=" + queryPlan);
}
return(queryPlan);
}
/// <summary>
/// Recusivly builds a substream-per-stream ordered tree graph using the
/// join information supplied for outer joins and from the query graph (where clause).
/// <para />Required streams are considered first and their lookup is placed first in the list
/// to gain performance.
/// </summary>
/// <param name="streamNum">is the root stream number that supplies the incoming event to build the tree for</param>
/// <param name="streamCallStack">the query plan call stack of streams available via cursor</param>
/// <param name="queryGraph">contains where-clause stream relationship info</param>
/// <param name="outerInnerGraph">contains the outer join stream relationship info</param>
/// <param name="innerJoinGraph">The inner join graph.</param>
/// <param name="completedStreams">is a temporary holder for streams already considered</param>
/// <param name="substreamsPerStream">is the ordered, tree-like structure to be filled</param>
/// <param name="requiredPerStream">indicates which streams are required and which are optional</param>
/// <param name="dependencyGraph">dependencies between historical streams</param>
/// <exception cref="ExprValidationException">Historical stream + streamNum + parameter dependency originating in stream + dependentStream + cannot or may not be satisfied by the join</exception>
/// <throws>ExprValidationException if the query planning failed</throws>
internal static void RecursiveBuild(
int streamNum,
Stack <int> streamCallStack,
QueryGraph queryGraph,
OuterInnerDirectionalGraph outerInnerGraph,
InnerJoinGraph innerJoinGraph,
ISet <int> completedStreams,
IDictionary <int, int[]> substreamsPerStream,
bool[] requiredPerStream,
DependencyGraph dependencyGraph)
{
// add this stream to the set of completed streams
completedStreams.Add(streamNum);
// check if the dependencies have been satisfied
if (dependencyGraph.HasDependency(streamNum))
{
var dependencies = dependencyGraph.GetDependenciesForStream(streamNum);
foreach (var dependentStream in dependencies)
{
if (!streamCallStack.Contains(dependentStream))
{
throw new ExprValidationException("Historical stream " + streamNum + " parameter dependency originating in stream " + dependentStream + " cannot or may not be satisfied by the join");
}
}
}
// Determine the streams we can navigate to from this stream
var navigableStreams = queryGraph.GetNavigableStreams(streamNum);
// add unqualified navigable streams (since on-expressions in outer joins are optional)
var unqualifiedNavigable = outerInnerGraph.UnqualifiedNavigableStreams.Get(streamNum);
if (unqualifiedNavigable != null)
{
navigableStreams.AddAll(unqualifiedNavigable);
}
// remove those already done
navigableStreams.RemoveAll(completedStreams);
// Which streams are inner streams to this stream (optional), which ones are outer to the stream (required)
var requiredStreams = GetOuterStreams(streamNum, navigableStreams, outerInnerGraph);
// Add inner joins, if any, unless already completed for this stream
innerJoinGraph.AddRequiredStreams(streamNum, requiredStreams, completedStreams);
var optionalStreams = GetInnerStreams(streamNum, navigableStreams, outerInnerGraph, innerJoinGraph, completedStreams);
// Remove from the required streams the optional streams which places 'full' joined streams
// into the optional stream category
requiredStreams.RemoveAll(optionalStreams);
// if we are a leaf node, we are done
if (navigableStreams.IsEmpty())
{
substreamsPerStream.Put(streamNum, new int[0]);
return;
}
// First the outer (required) streams to this stream, then the inner (optional) streams
var substreams = new int[requiredStreams.Count + optionalStreams.Count];
substreamsPerStream.Put(streamNum, substreams);
var count = 0;
foreach (int stream in requiredStreams)
{
substreams[count++] = stream;
requiredPerStream[stream] = true;
}
foreach (int stream in optionalStreams)
{
substreams[count++] = stream;
}
// next we look at all the required streams and add their dependent streams
foreach (int stream in requiredStreams)
{
completedStreams.Add(stream);
}
foreach (int stream in requiredStreams)
{
streamCallStack.Push(stream);
RecursiveBuild(stream, streamCallStack, queryGraph, outerInnerGraph, innerJoinGraph,
completedStreams, substreamsPerStream, requiredPerStream, dependencyGraph);
streamCallStack.Pop();
}
// look at all the optional streams and add their dependent streams
foreach (int stream in optionalStreams)
{
streamCallStack.Push(stream);
RecursiveBuild(stream, streamCallStack, queryGraph, outerInnerGraph, innerJoinGraph,
completedStreams, substreamsPerStream, requiredPerStream, dependencyGraph);
streamCallStack.Pop();
}
}
private static IList <LookupInstructionPlan> BuildLookupInstructions(
int rootStreamNum,
LinkedHashMap <int, int[]> substreamsPerStream,
bool[] requiredPerStream,
string[] streamNames,
QueryGraph queryGraph,
QueryPlanIndex[] indexSpecs,
EventType[] typesPerStream,
OuterJoinDesc[] outerJoinDescList,
bool[] isHistorical,
HistoricalStreamIndexList[] historicalStreamIndexLists,
ExprEvaluatorContext exprEvaluatorContext,
TableMetadata[] tablesPerStream)
{
IList <LookupInstructionPlan> result = new List <LookupInstructionPlan>();
foreach (int fromStream in substreamsPerStream.Keys)
{
var substreams = substreamsPerStream.Get(fromStream);
// for streams with no substreams we don't need to look up
if (substreams.Length == 0)
{
continue;
}
var plans = new TableLookupPlan[substreams.Length];
var historicalPlans = new HistoricalDataPlanNode[substreams.Length];
for (var i = 0; i < substreams.Length; i++)
{
var toStream = substreams[i];
if (isHistorical[toStream])
{
// There may not be an outer-join descriptor, use if provided to build the associated expression
ExprNode outerJoinExpr = null;
if (outerJoinDescList.Length > 0)
{
OuterJoinDesc outerJoinDesc;
if (toStream == 0)
{
outerJoinDesc = outerJoinDescList[0];
}
else
{
outerJoinDesc = outerJoinDescList[toStream - 1];
}
outerJoinExpr = outerJoinDesc.MakeExprNode(exprEvaluatorContext);
}
if (historicalStreamIndexLists[toStream] == null)
{
historicalStreamIndexLists[toStream] = new HistoricalStreamIndexList(toStream, typesPerStream, queryGraph);
}
historicalStreamIndexLists[toStream].AddIndex(fromStream);
historicalPlans[i] = new HistoricalDataPlanNode(toStream, rootStreamNum, fromStream, typesPerStream.Length, outerJoinExpr);
}
else
{
plans[i] = NStreamQueryPlanBuilder.CreateLookupPlan(queryGraph, fromStream, toStream, indexSpecs[toStream], typesPerStream, tablesPerStream[toStream]);
}
}
var fromStreamName = streamNames[fromStream];
var instruction = new LookupInstructionPlan(fromStream, fromStreamName, substreams, plans, historicalPlans, requiredPerStream);
result.Add(instruction);
}
return(result);
}
/// <summary>
/// Given a chain of streams to look up and indexing information, compute the index within the
/// chain of the first non-index lookup.
/// </summary>
/// <param name="lookupStream">stream to start lookup for</param>
/// <param name="nextStreams">list of stream numbers next in lookup</param>
/// <param name="queryGraph">indexing information</param>
/// <returns>value between 0 and (nextStreams.lenght - 1)</returns>
internal static int ComputeNavigableDepth(int lookupStream, int[] nextStreams, QueryGraph queryGraph)
{
var currentStream = lookupStream;
var currentDepth = 0;
for (var i = 0; i < nextStreams.Length; i++)
{
var nextStream = nextStreams[i];
var navigable = queryGraph.IsNavigableAtAll(currentStream, nextStream);
if (!navigable)
{
break;
}
currentStream = nextStream;
currentDepth++;
}
return(currentDepth);
}
/// <summary>
/// Create the table lookup plan for a from-stream to look up in an indexed stream
/// using the columns supplied in the query graph and looking at the actual indexes available
/// and their index number.
/// </summary>
/// <param name="queryGraph">contains properties joining the 2 streams</param>
/// <param name="currentLookupStream">stream to use key values from</param>
/// <param name="indexedStream">stream to look up in</param>
/// <param name="indexSpecs">index specification defining indexes to be created for stream</param>
/// <param name="typesPerStream">event types for each stream</param>
/// <param name="indexedStreamTableMeta">The indexed stream table meta.</param>
/// <returns>
/// plan for performing a lookup in a given table using one of the indexes supplied
/// </returns>
/// <exception cref="IllegalStateException">Failed to query plan as index for " + hashIndexProps.Render() + " and " + rangeIndexProps.Render() + " in the index specification</exception>
public static TableLookupPlan CreateLookupPlan(
QueryGraph queryGraph, int currentLookupStream, int indexedStream,
QueryPlanIndex indexSpecs, EventType[] typesPerStream,
TableMetadata indexedStreamTableMeta)
{
var queryGraphValue = queryGraph.GetGraphValue(currentLookupStream, indexedStream);
var hashKeyProps = queryGraphValue.HashKeyProps;
var hashPropsKeys = hashKeyProps.Keys;
var hashIndexProps = hashKeyProps.Indexed.ToArray();
var rangeProps = queryGraphValue.RangeProps;
var rangePropsKeys = rangeProps.Keys;
var rangeIndexProps = rangeProps.Indexed.ToArray();
var pairIndexHashRewrite = indexSpecs.GetIndexNum(hashIndexProps, rangeIndexProps);
var indexNum = pairIndexHashRewrite == null ? null : pairIndexHashRewrite.First;
// handle index redirection towards unique index
if (pairIndexHashRewrite != null && pairIndexHashRewrite.Second != null)
{
var indexes = pairIndexHashRewrite.Second;
var newHashIndexProps = new string[indexes.Length];
IList <QueryGraphValueEntryHashKeyed> newHashKeys = new List <QueryGraphValueEntryHashKeyed>();
for (var i = 0; i < indexes.Length; i++)
{
newHashIndexProps[i] = hashIndexProps[indexes[i]];
newHashKeys.Add(hashPropsKeys[indexes[i]]);
}
hashIndexProps = newHashIndexProps;
hashPropsKeys = newHashKeys;
rangeIndexProps = new string[0];
rangePropsKeys = Collections.GetEmptyList <QueryGraphValueEntryRange>();
}
// no direct hash or range lookups
if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 0)
{
// handle single-direction 'in' keyword
var singles = queryGraphValue.InKeywordSingles;
if (!singles.Key.IsEmpty())
{
QueryGraphValueEntryInKeywordSingleIdx single = null;
indexNum = null;
if (indexedStreamTableMeta != null)
{
var indexes = singles.Indexed;
var count = 0;
foreach (var index in indexes)
{
Pair <IndexMultiKey, EventTableIndexEntryBase> indexPairFound =
EventTableIndexUtil.FindIndexBestAvailable(
indexedStreamTableMeta.EventTableIndexMetadataRepo.IndexesAsBase,
Collections.SingletonSet(index),
Collections.GetEmptySet <string>(), null);
if (indexPairFound != null)
{
indexNum = new TableLookupIndexReqKey(indexPairFound.Second.OptionalIndexName, indexedStreamTableMeta.TableName);
single = singles.Key[count];
}
count++;
}
}
else
{
single = singles.Key[0];
var pairIndex = indexSpecs.GetIndexNum(new string[] { singles.Indexed[0] }, null);
indexNum = pairIndex.First;
}
if (indexNum != null)
{
return(new InKeywordTableLookupPlanSingleIdx(currentLookupStream, indexedStream, indexNum, single.KeyExprs));
}
}
// handle multi-direction 'in' keyword
var multis = queryGraphValue.InKeywordMulti;
if (!multis.IsEmpty())
{
if (indexedStreamTableMeta != null)
{
return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta));
}
QueryGraphValuePairInKWMultiIdx multi = multis[0];
var indexNameArray = new TableLookupIndexReqKey[multi.Indexed.Count];
var foundAll = true;
for (var i = 0; i < multi.Indexed.Count; i++)
{
var identNode = (ExprIdentNode)multi.Indexed[i];
var pairIndex = indexSpecs.GetIndexNum(new string[] { identNode.ResolvedPropertyName }, null);
if (pairIndex == null)
{
foundAll = false;
}
else
{
indexNameArray[i] = pairIndex.First;
}
}
if (foundAll)
{
return(new InKeywordTableLookupPlanMultiIdx(currentLookupStream, indexedStream, indexNameArray, multi.Key.KeyExpr));
}
}
// We don't use a keyed index but use the full stream set as the stream does not have any indexes
// If no such full set index exists yet, add to specs
if (indexedStreamTableMeta != null)
{
return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta));
}
if (indexNum == null)
{
indexNum = new TableLookupIndexReqKey(indexSpecs.AddIndex(null, null));
}
return(new FullTableScanLookupPlan(currentLookupStream, indexedStream, indexNum));
}
if (indexNum == null)
{
throw new IllegalStateException("Failed to query plan as index for " + hashIndexProps.Render() + " and " + rangeIndexProps.Render() + " in the index specification");
}
if (indexedStreamTableMeta != null)
{
var indexPairFound = EventTableIndexUtil.FindIndexBestAvailable(
indexedStreamTableMeta.EventTableIndexMetadataRepo.IndexesAsBase,
ToSet(hashIndexProps),
ToSet(rangeIndexProps),
null);
if (indexPairFound != null)
{
var indexKeyInfo = SubordinateQueryPlannerUtil.CompileIndexKeyInfo(indexPairFound.First, hashIndexProps, GetHashKeyFuncsAsSubProp(hashPropsKeys), rangeIndexProps, GetRangeFuncsAsSubProp(rangePropsKeys));
if (indexKeyInfo.OrderedKeyCoercionTypes.IsCoerce || indexKeyInfo.OrderedRangeCoercionTypes.IsCoerce)
{
return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta));
}
hashPropsKeys = ToHashKeyFuncs(indexKeyInfo.OrderedHashDesc);
hashIndexProps = IndexedPropDesc.GetIndexProperties(indexPairFound.First.HashIndexedProps);
rangePropsKeys = ToRangeKeyFuncs(indexKeyInfo.OrderedRangeDesc);
rangeIndexProps = IndexedPropDesc.GetIndexProperties(indexPairFound.First.RangeIndexedProps);
indexNum = new TableLookupIndexReqKey(indexPairFound.Second.OptionalIndexName, indexedStreamTableMeta.TableName);
// the plan will be created below
if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 0)
{
return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta));
}
}
else
{
return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta));
}
}
// straight keyed-index lookup
if (hashIndexProps.Length > 0 && rangeIndexProps.Length == 0)
{
TableLookupPlan tableLookupPlan;
if (hashPropsKeys.Count == 1)
{
tableLookupPlan = new IndexedTableLookupPlanSingle(currentLookupStream, indexedStream, indexNum, hashPropsKeys[0]);
}
else
{
tableLookupPlan = new IndexedTableLookupPlanMulti(currentLookupStream, indexedStream, indexNum, hashPropsKeys);
}
// Determine coercion required
var coercionTypes = CoercionUtil.GetCoercionTypesHash(typesPerStream, currentLookupStream, indexedStream, hashPropsKeys, hashIndexProps);
if (coercionTypes.IsCoerce)
{
// check if there already are coercion types for this index
var existCoercionTypes = indexSpecs.GetCoercionTypes(hashIndexProps);
if (existCoercionTypes != null)
{
for (var i = 0; i < existCoercionTypes.Length; i++)
{
coercionTypes.CoercionTypes[i] = TypeHelper.GetCompareToCoercionType(existCoercionTypes[i], coercionTypes.CoercionTypes[i]);
}
}
indexSpecs.SetCoercionTypes(hashIndexProps, coercionTypes.CoercionTypes);
}
return(tableLookupPlan);
}
// sorted index lookup
if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 1)
{
QueryGraphValueEntryRange range = rangePropsKeys[0];
return(new SortedTableLookupPlan(currentLookupStream, indexedStream, indexNum, range));
}
// composite range and index lookup
else
{
return(new CompositeTableLookupPlan(currentLookupStream, indexedStream, indexNum, hashPropsKeys, rangePropsKeys));
}
}
/// <summary>
/// Build a query plan based on the stream property relationships indicated in queryGraph.
/// </summary>
/// <param name="queryGraph">navigation info between streams</param>
/// <param name="typesPerStream">event types for each stream</param>
/// <param name="historicalViewableDesc">The historical viewable desc.</param>
/// <param name="dependencyGraph">dependencies between historical streams</param>
/// <param name="historicalStreamIndexLists">index management, populated for the query plan</param>
/// <param name="hasForceNestedIter">if set to <c>true</c> [has force nested iter].</param>
/// <param name="indexedStreamsUniqueProps">The indexed streams unique props.</param>
/// <param name="tablesPerStream">The tables per stream.</param>
/// <returns>
/// query plan
/// </returns>
internal static QueryPlan Build(
QueryGraph queryGraph,
EventType[] typesPerStream,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexList[] historicalStreamIndexLists,
bool hasForceNestedIter,
string[][][] indexedStreamsUniqueProps,
TableMetadata[] tablesPerStream)
{
if (Log.IsDebugEnabled)
{
Log.Debug(".build queryGraph=" + queryGraph);
}
var numStreams = queryGraph.NumStreams;
var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(queryGraph, typesPerStream, indexedStreamsUniqueProps);
if (Log.IsDebugEnabled)
{
Log.Debug(".build Index build completed, indexes=" + QueryPlanIndex.Print(indexSpecs));
}
// any historical streams don't get indexes, the lookup strategy accounts for cached indexes
if (historicalViewableDesc.HasHistorical)
{
for (var i = 0; i < historicalViewableDesc.Historical.Length; i++)
{
if (historicalViewableDesc.Historical[i])
{
indexSpecs[i] = null;
}
}
}
var planNodeSpecs = new QueryPlanNode[numStreams];
int worstDepth = int.MaxValue;
for (var streamNo = 0; streamNo < numStreams; streamNo++)
{
// no plan for historical streams that are dependent upon other streams
if ((historicalViewableDesc.Historical[streamNo]) && (dependencyGraph.HasDependency(streamNo)))
{
planNodeSpecs[streamNo] = new QueryPlanNodeNoOp();
continue;
}
var bestChainResult = ComputeBestPath(streamNo, queryGraph, dependencyGraph);
var bestChain = bestChainResult.Chain;
if (Log.IsDebugEnabled)
{
Log.Debug(".build For stream " + streamNo + " bestChain=" + bestChain.Render());
}
if (bestChainResult.Depth < worstDepth)
{
worstDepth = bestChainResult.Depth;
}
planNodeSpecs[streamNo] = CreateStreamPlan(streamNo, bestChain, queryGraph, indexSpecs, typesPerStream, historicalViewableDesc.Historical, historicalStreamIndexLists, tablesPerStream);
if (Log.IsDebugEnabled)
{
Log.Debug(".build spec=" + planNodeSpecs[streamNo]);
}
}
// We use the merge/nested (outer) join algorithm instead.
if ((worstDepth < numStreams - 1) && (!hasForceNestedIter))
{
return(null);
}
return(new QueryPlan(indexSpecs, planNodeSpecs));
}