public void SetUp() { QueryPlanIndexItem itemOne = new QueryPlanIndexItem(new String[] { "p01", "p02" }, null, null, null, false); QueryPlanIndexItem itemTwo = new QueryPlanIndexItem(new String[] { "p21" }, new Type[0], null, null, false); QueryPlanIndexItem itemThree = new QueryPlanIndexItem(new String[0], new Type[0], null, null, false); indexSpec = QueryPlanIndex.MakeIndex(itemOne, itemTwo, itemThree); }
public String ToQueryPlan() { var buffer = new StringBuilder(); buffer.Append("QueryPlanNode\n"); buffer.Append(QueryPlanIndex.Print(IndexSpecs)); buffer.Append(QueryPlanNode.Print(ExecNodeSpecs)); return(buffer.ToString()); }
/// <summary> /// Create the table lookup plan for a from-stream to look up in an indexed stream /// using the columns supplied in the query graph and looking at the actual indexes available /// and their index number. /// </summary> /// <param name="queryGraph">contains properties joining the 2 streams</param> /// <param name="currentLookupStream">stream to use key values from</param> /// <param name="indexedStream">stream to look up in</param> /// <param name="indexSpecs">index specification defining indexes to be created for stream</param> /// <param name="typesPerStream">event types for each stream</param> /// <returns>plan for performing a lookup in a given table using one of the indexes supplied</returns> public static TableLookupPlan CreateLookupPlan(QueryGraph queryGraph, int currentLookupStream, int indexedStream, QueryPlanIndex indexSpecs, EventType[] typesPerStream, TableMetadata indexedStreamTableMeta) { var queryGraphValue = queryGraph.GetGraphValue(currentLookupStream, indexedStream); var hashKeyProps = queryGraphValue.HashKeyProps; var hashPropsKeys = hashKeyProps.Keys; var hashIndexProps = hashKeyProps.Indexed.ToArray(); var rangeProps = queryGraphValue.RangeProps; var rangePropsKeys = rangeProps.Keys; var rangeIndexProps = rangeProps.Indexed.ToArray(); var pairIndexHashRewrite = indexSpecs.GetIndexNum(hashIndexProps, rangeIndexProps); var indexNum = pairIndexHashRewrite == null ? null : pairIndexHashRewrite.First; // handle index redirection towards unique index if (pairIndexHashRewrite != null && pairIndexHashRewrite.Second != null) { var indexes = pairIndexHashRewrite.Second; var newHashIndexProps = new string[indexes.Length]; IList <QueryGraphValueEntryHashKeyed> newHashKeys = new List <QueryGraphValueEntryHashKeyed>(); for (var i = 0; i < indexes.Length; i++) { newHashIndexProps[i] = hashIndexProps[indexes[i]]; newHashKeys.Add(hashPropsKeys[indexes[i]]); } hashIndexProps = newHashIndexProps; hashPropsKeys = newHashKeys; rangeIndexProps = new string[0]; rangePropsKeys = Collections.GetEmptyList <QueryGraphValueEntryRange>(); } // no direct hash or range lookups if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 0) { // handle single-direction 'in' keyword var singles = queryGraphValue.InKeywordSingles; if (!singles.Key.IsEmpty()) { QueryGraphValueEntryInKeywordSingleIdx single = null; indexNum = null; if (indexedStreamTableMeta != null) { var indexes = singles.Indexed; var count = 0; foreach (var index in indexes) { Pair <IndexMultiKey, EventTableIndexEntryBase> indexPairFound = EventTableIndexUtil.FindIndexBestAvailable( indexedStreamTableMeta.EventTableIndexMetadataRepo.Indexes, Collections.SingletonSet(index), Collections.GetEmptySet <string>(), null); if (indexPairFound != null) { indexNum = new TableLookupIndexReqKey(indexPairFound.Second.OptionalIndexName, indexedStreamTableMeta.TableName); single = singles.Key[count]; } count++; } } else { single = singles.Key[0]; var pairIndex = indexSpecs.GetIndexNum(new string[] { singles.Indexed[0] }, null); indexNum = pairIndex.First; } if (indexNum != null) { return(new InKeywordTableLookupPlanSingleIdx(currentLookupStream, indexedStream, indexNum, single.KeyExprs)); } } // handle multi-direction 'in' keyword var multis = queryGraphValue.InKeywordMulti; if (!multis.IsEmpty()) { if (indexedStreamTableMeta != null) { return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta)); } QueryGraphValuePairInKWMultiIdx multi = multis[0]; var indexNameArray = new TableLookupIndexReqKey[multi.Indexed.Length]; var foundAll = true; for (var i = 0; i < multi.Indexed.Length; i++) { var identNode = (ExprIdentNode)multi.Indexed[i]; var pairIndex = indexSpecs.GetIndexNum(new string[] { identNode.ResolvedPropertyName }, null); if (pairIndex == null) { foundAll = false; } else { indexNameArray[i] = pairIndex.First; } } if (foundAll) { return(new InKeywordTableLookupPlanMultiIdx(currentLookupStream, indexedStream, indexNameArray, multi.Key.KeyExpr)); } } // We don't use a keyed index but use the full stream set as the stream does not have any indexes // If no such full set index exists yet, add to specs if (indexedStreamTableMeta != null) { return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta)); } if (indexNum == null) { indexNum = new TableLookupIndexReqKey(indexSpecs.AddIndex(null, null)); } return(new FullTableScanLookupPlan(currentLookupStream, indexedStream, indexNum)); } if (indexNum == null) { throw new IllegalStateException("Failed to query plan as index for " + hashIndexProps.Render() + " and " + rangeIndexProps.Render() + " in the index specification"); } if (indexedStreamTableMeta != null) { var indexPairFound = EventTableIndexUtil.FindIndexBestAvailable(indexedStreamTableMeta.EventTableIndexMetadataRepo.Indexes, ToSet(hashIndexProps), ToSet(rangeIndexProps), null); if (indexPairFound != null) { var indexKeyInfo = SubordinateQueryPlannerUtil.CompileIndexKeyInfo(indexPairFound.First, hashIndexProps, GetHashKeyFuncsAsSubProp(hashPropsKeys), rangeIndexProps, GetRangeFuncsAsSubProp(rangePropsKeys)); if (indexKeyInfo.OrderedKeyCoercionTypes.IsCoerce || indexKeyInfo.OrderedRangeCoercionTypes.IsCoerce) { return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta)); } hashPropsKeys = ToHashKeyFuncs(indexKeyInfo.OrderedHashDesc); hashIndexProps = IndexedPropDesc.GetIndexProperties(indexPairFound.First.HashIndexedProps); rangePropsKeys = ToRangeKeyFuncs(indexKeyInfo.OrderedRangeDesc); rangeIndexProps = IndexedPropDesc.GetIndexProperties(indexPairFound.First.RangeIndexedProps); indexNum = new TableLookupIndexReqKey(indexPairFound.Second.OptionalIndexName, indexedStreamTableMeta.TableName); // the plan will be created below if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 0) { return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta)); } } else { return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta)); } } // straight keyed-index lookup if (hashIndexProps.Length > 0 && rangeIndexProps.Length == 0) { TableLookupPlan tableLookupPlan; if (hashPropsKeys.Count == 1) { tableLookupPlan = new IndexedTableLookupPlanSingle(currentLookupStream, indexedStream, indexNum, hashPropsKeys[0]); } else { tableLookupPlan = new IndexedTableLookupPlanMulti(currentLookupStream, indexedStream, indexNum, hashPropsKeys); } // Determine coercion required var coercionTypes = CoercionUtil.GetCoercionTypesHash(typesPerStream, currentLookupStream, indexedStream, hashPropsKeys, hashIndexProps); if (coercionTypes.IsCoerce) { // check if there already are coercion types for this index var existCoercionTypes = indexSpecs.GetCoercionTypes(hashIndexProps); if (existCoercionTypes != null) { for (var i = 0; i < existCoercionTypes.Length; i++) { coercionTypes.CoercionTypes[i] = TypeHelper.GetCompareToCoercionType(existCoercionTypes[i], coercionTypes.CoercionTypes[i]); } } indexSpecs.SetCoercionTypes(hashIndexProps, coercionTypes.CoercionTypes); } return(tableLookupPlan); } // sorted index lookup if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 1) { QueryGraphValueEntryRange range = rangePropsKeys[0]; return(new SortedTableLookupPlan(currentLookupStream, indexedStream, indexNum, range)); } // composite range and index lookup else { return(new CompositeTableLookupPlan(currentLookupStream, indexedStream, indexNum, hashPropsKeys, rangePropsKeys)); } }
/// <summary> /// Build a query plan based on the stream property relationships indicated in queryGraph. /// </summary> /// <param name="queryGraph">navigation info between streams</param> /// <param name="typesPerStream">event types for each stream</param> /// <param name="historicalViewableDesc">The historical viewable desc.</param> /// <param name="dependencyGraph">dependencies between historical streams</param> /// <param name="historicalStreamIndexLists">index management, populated for the query plan</param> /// <param name="hasForceNestedIter">if set to <c>true</c> [has force nested iter].</param> /// <param name="indexedStreamsUniqueProps">The indexed streams unique props.</param> /// <param name="tablesPerStream">The tables per stream.</param> /// <returns> /// query plan /// </returns> internal static QueryPlan Build( QueryGraph queryGraph, EventType[] typesPerStream, HistoricalViewableDesc historicalViewableDesc, DependencyGraph dependencyGraph, HistoricalStreamIndexList[] historicalStreamIndexLists, bool hasForceNestedIter, string[][][] indexedStreamsUniqueProps, TableMetadata[] tablesPerStream) { if (Log.IsDebugEnabled) { Log.Debug(".build queryGraph=" + queryGraph); } var numStreams = queryGraph.NumStreams; var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(queryGraph, typesPerStream, indexedStreamsUniqueProps); if (Log.IsDebugEnabled) { Log.Debug(".build Index build completed, indexes=" + QueryPlanIndex.Print(indexSpecs)); } // any historical streams don't get indexes, the lookup strategy accounts for cached indexes if (historicalViewableDesc.HasHistorical) { for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) { if (historicalViewableDesc.Historical[i]) { indexSpecs[i] = null; } } } var planNodeSpecs = new QueryPlanNode[numStreams]; int worstDepth = int.MaxValue; for (var streamNo = 0; streamNo < numStreams; streamNo++) { // no plan for historical streams that are dependent upon other streams if ((historicalViewableDesc.Historical[streamNo]) && (dependencyGraph.HasDependency(streamNo))) { planNodeSpecs[streamNo] = new QueryPlanNodeNoOp(); continue; } var bestChainResult = ComputeBestPath(streamNo, queryGraph, dependencyGraph); var bestChain = bestChainResult.Chain; if (Log.IsDebugEnabled) { Log.Debug(".build For stream " + streamNo + " bestChain=" + bestChain.Render()); } if (bestChainResult.Depth < worstDepth) { worstDepth = bestChainResult.Depth; } planNodeSpecs[streamNo] = CreateStreamPlan(streamNo, bestChain, queryGraph, indexSpecs, typesPerStream, historicalViewableDesc.Historical, historicalStreamIndexLists, tablesPerStream); if (Log.IsDebugEnabled) { Log.Debug(".build spec=" + planNodeSpecs[streamNo]); } } // We use the merge/nested (outer) join algorithm instead. if ((worstDepth < numStreams - 1) && (!hasForceNestedIter)) { return(null); } return(new QueryPlan(indexSpecs, planNodeSpecs)); }
/// <summary> /// Build a query plan based on the stream property relationships indicated in queryGraph. /// </summary> /// <param name="queryGraph">navigation info between streams</param> /// <param name="outerJoinDescList">descriptors for all outer joins</param> /// <param name="streamNames">stream names</param> /// <param name="typesPerStream">event types for each stream</param> /// <param name="historicalViewableDesc">The historical viewable desc.</param> /// <param name="dependencyGraph">dependencies between historical streams</param> /// <param name="historicalStreamIndexLists">index management, populated for the query plan</param> /// <param name="exprEvaluatorContext">context for expression evalauation</param> /// <param name="indexedStreamsUniqueProps">The indexed streams unique props.</param> /// <param name="tablesPerStream">The tables per stream.</param> /// <returns> /// query plan /// </returns> /// <throws>ExprValidationException if the query planning failed</throws> internal static QueryPlan Build( QueryGraph queryGraph, OuterJoinDesc[] outerJoinDescList, string[] streamNames, EventType[] typesPerStream, HistoricalViewableDesc historicalViewableDesc, DependencyGraph dependencyGraph, HistoricalStreamIndexList[] historicalStreamIndexLists, ExprEvaluatorContext exprEvaluatorContext, string[][][] indexedStreamsUniqueProps, TableMetadata[] tablesPerStream) { if (Log.IsDebugEnabled) { Log.Debug(".build queryGraph=" + queryGraph); } var numStreams = queryGraph.NumStreams; var planNodeSpecs = new QueryPlanNode[numStreams]; // Build index specifications var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(queryGraph, typesPerStream, indexedStreamsUniqueProps); if (Log.IsDebugEnabled) { Log.Debug(".build Index build completed, indexes=" + QueryPlanIndex.Print(indexSpecs)); } // any historical streams don't get indexes, the lookup strategy accounts for cached indexes if (historicalViewableDesc.HasHistorical) { for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) { if (historicalViewableDesc.Historical[i]) { indexSpecs[i] = null; } } } // Build graph of the outer join to inner table relationships. // Build a map of inner joins. OuterInnerDirectionalGraph outerInnerGraph; InnerJoinGraph innerJoinGraph; if (outerJoinDescList.Length > 0) { outerInnerGraph = GraphOuterJoins(numStreams, outerJoinDescList); innerJoinGraph = InnerJoinGraph.GraphInnerJoins(numStreams, outerJoinDescList); } else { // all inner joins - thereby no (or empty) directional graph outerInnerGraph = new OuterInnerDirectionalGraph(numStreams); innerJoinGraph = new InnerJoinGraph(numStreams, true); } if (Log.IsDebugEnabled) { Log.Debug(".build directional graph=" + outerInnerGraph.Print()); } // For each stream determine the query plan for (var streamNo = 0; streamNo < numStreams; streamNo++) { // no plan for historical streams that are dependent upon other streams if ((historicalViewableDesc.Historical[streamNo]) && (dependencyGraph.HasDependency(streamNo))) { planNodeSpecs[streamNo] = new QueryPlanNodeNoOp(); continue; } var queryPlanNode = BuildPlanNode(numStreams, streamNo, streamNames, queryGraph, outerInnerGraph, outerJoinDescList, innerJoinGraph, indexSpecs, typesPerStream, historicalViewableDesc.Historical, dependencyGraph, historicalStreamIndexLists, exprEvaluatorContext, tablesPerStream); if (Log.IsDebugEnabled) { Log.Debug(".build spec for stream '" + streamNames[streamNo] + "' number " + streamNo + " is " + queryPlanNode); } planNodeSpecs[streamNo] = queryPlanNode; } var queryPlan = new QueryPlan(indexSpecs, planNodeSpecs); if (Log.IsDebugEnabled) { Log.Debug(".build query plan=" + queryPlan); } return(queryPlan); }
/// <summary> /// Build index specification from navigability info. /// <para/> /// Looks at each stream and determines which properties in the stream must be indexed /// in order for other streams to look up into the stream. Determines the unique set of /// properties to avoid building duplicate indexes on the same set of properties. /// </summary> /// <param name="queryGraph">navigability info</param> /// <param name="typePerStream">The type per stream.</param> /// <param name="indexedStreamsUniqueProps">The indexed streams unique props.</param> /// <returns>query index specs for each stream</returns> public static QueryPlanIndex[] BuildIndexSpec(QueryGraph queryGraph, EventType[] typePerStream, String[][][] indexedStreamsUniqueProps) { var numStreams = queryGraph.NumStreams; var indexSpecs = new QueryPlanIndex[numStreams]; // For each stream compile a list of index property sets. for (int streamIndexed = 0; streamIndexed < numStreams; streamIndexed++) { var indexesSet = new List <QueryPlanIndexItem>(); // Look at the index from the viewpoint of the stream looking up in the index for (int streamLookup = 0; streamLookup < numStreams; streamLookup++) { if (streamIndexed == streamLookup) { continue; } var value = queryGraph.GetGraphValue(streamLookup, streamIndexed); var hashKeyAndIndexProps = value.HashKeyProps; // Sort index properties, but use the sorted properties only to eliminate duplicates var hashIndexProps = hashKeyAndIndexProps.Indexed; var hashKeyProps = hashKeyAndIndexProps.Keys; var indexCoercionTypes = CoercionUtil.GetCoercionTypesHash(typePerStream, streamLookup, streamIndexed, hashKeyProps, hashIndexProps); var hashCoercionTypeArr = indexCoercionTypes.CoercionTypes; var rangeAndIndexProps = value.RangeProps; var rangeIndexProps = rangeAndIndexProps.Indexed; var rangeKeyProps = rangeAndIndexProps.Keys; var rangeCoercionTypes = CoercionUtil.GetCoercionTypesRange(typePerStream, streamIndexed, rangeIndexProps, rangeKeyProps); var rangeCoercionTypeArr = rangeCoercionTypes.CoercionTypes; if (hashIndexProps.Count == 0 && rangeIndexProps.Count == 0) { QueryGraphValuePairInKWSingleIdx singles = value.InKeywordSingles; if (!singles.Key.IsEmpty()) { String indexedProp = singles.Indexed[0]; QueryPlanIndexItem indexItem = new QueryPlanIndexItem(new String[] { indexedProp }, null, null, null, false); CheckDuplicateOrAdd(indexItem, indexesSet); } IList <QueryGraphValuePairInKWMultiIdx> multis = value.InKeywordMulti; if (!multis.IsEmpty()) { QueryGraphValuePairInKWMultiIdx multi = multis[0]; foreach (ExprNode propIndexed in multi.Indexed) { ExprIdentNode identNode = (ExprIdentNode)propIndexed; QueryPlanIndexItem indexItem = new QueryPlanIndexItem(new String[] { identNode.ResolvedPropertyName }, null, null, null, false); CheckDuplicateOrAdd(indexItem, indexesSet); } } continue; } // reduce to any unique index if applicable var unique = false; var reduced = QueryPlanIndexUniqueHelper.ReduceToUniqueIfPossible(hashIndexProps, hashCoercionTypeArr, hashKeyProps, indexedStreamsUniqueProps[streamIndexed]); if (reduced != null) { hashIndexProps = reduced.PropertyNames; hashCoercionTypeArr = reduced.CoercionTypes; unique = true; rangeIndexProps = new String[0]; rangeCoercionTypeArr = new Type[0]; } var proposed = new QueryPlanIndexItem(hashIndexProps, hashCoercionTypeArr, rangeIndexProps, rangeCoercionTypeArr, unique); CheckDuplicateOrAdd(proposed, indexesSet); } // create full-table-scan if (indexesSet.IsEmpty()) { indexesSet.Add(new QueryPlanIndexItem(null, null, null, null, false)); } indexSpecs[streamIndexed] = QueryPlanIndex.MakeIndex(indexesSet); } return(indexSpecs); }