private static void CompareIndex( int streamNum, QueryPlanIndexForge expected, QueryPlanIndexForge actual, IDictionary<TableLookupIndexReqKey, TableLookupIndexReqKey> indexNameMapping) { var actualItems = actual.Items; var expectedItems = expected.Items; Assert.AreEqual( expectedItems.Count, actualItems.Count, "Number of indexes mismatch for stream " + streamNum ); var actualEnum = actualItems.GetEnumerator(); var expectedEnum = expectedItems.GetEnumerator(); var count = 0; while (true) { var actualNext = actualEnum.MoveNext(); var expectedNext = expectedEnum.MoveNext(); Assert.That(actualNext, Is.EqualTo(expectedNext)); if (!actualNext) { break; } var actualItem = actualEnum.Current; var expectedItem = expectedEnum.Current; CompareIndexItem(streamNum, count, expectedItem.Value, actualItem.Value); count++; indexNameMapping.Put(actualItem.Key, expectedItem.Key); } }
private SupportQueryPlanBuilder(int numStreams) { var indexes = new QueryPlanIndexForge[numStreams]; for (var i = 0; i < indexes.Length; i++) { indexes[i] = new QueryPlanIndexForge(new LinkedHashMap<TableLookupIndexReqKey, QueryPlanIndexItemForge>()); } queryPlan = new QueryPlanForge(indexes, new QueryPlanNodeForge[numStreams]); }
public CompositeTableLookupPlanForge( int lookupStream, int indexedStream, bool indexedStreamIsVDW, EventType[] typesPerStream, TableLookupIndexReqKey indexNum, IList<QueryGraphValueEntryHashKeyedForge> hashKeys, Type[] hashCoercionTypes, IList<QueryGraphValueEntryRangeForge> rangeKeyPairs, Type[] optRangeCoercionTypes, QueryPlanIndexForge indexSpecs, MultiKeyClassRef optionalEPLTableLookupMultiKey) : base(lookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, new[] {indexNum}) { _hashKeys = hashKeys; _hashCoercionTypes = hashCoercionTypes; _rangeKeyPairs = rangeKeyPairs; _optRangeCoercionTypes = optRangeCoercionTypes; _indexSpecs = indexSpecs; _optionalEPLTableLookupMultiKey = optionalEPLTableLookupMultiKey; }
public IndexedTableLookupPlanHashedOnlyForge( int lookupStream, int indexedStream, bool indexedStreamIsVDW, EventType[] typesPerStream, TableLookupIndexReqKey indexNum, QueryGraphValueEntryHashKeyedForge[] hashKeys, QueryPlanIndexForge indexSpecs, Type[] optionalCoercionTypes, MultiKeyClassRef optionalEPLTableLookupMultiKey) : base( lookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, new[] { indexNum }) { _hashKeys = hashKeys; _indexSpecs = indexSpecs; _optionalCoercionTypes = optionalCoercionTypes; _optionalEPLTableLookupMultiKey = optionalEPLTableLookupMultiKey; }
/// <summary> /// Build index specification from navigability info. /// <para /> /// Looks at each stream and determines which properties in the stream must be indexed /// in order for other streams to look up into the stream. Determines the unique set of properties /// to avoid building duplicate indexes on the same set of properties. /// </summary> /// <param name="queryGraph">navigability info</param> /// <param name="typePerStream">type info</param> /// <param name="indexedStreamsUniqueProps">per-stream unique props</param> /// <returns>query index specs for each stream</returns> public static QueryPlanIndexForge[] BuildIndexSpec( QueryGraphForge queryGraph, EventType[] typePerStream, string[][][] indexedStreamsUniqueProps) { var numStreams = queryGraph.NumStreams; var indexSpecs = new QueryPlanIndexForge[numStreams]; // For each stream compile a list of index property sets. for (var streamIndexed = 0; streamIndexed < numStreams; streamIndexed++) { IList<QueryPlanIndexItemForge> indexesSet = new List<QueryPlanIndexItemForge>(); // Look at the index from the viewpoint of the stream looking up in the index for (var streamLookup = 0; streamLookup < numStreams; streamLookup++) { if (streamIndexed == streamLookup) { continue; } var value = queryGraph.GetGraphValue(streamLookup, streamIndexed); var hashKeyAndIndexProps = value.HashKeyProps; // Sort index properties, but use the sorted properties only to eliminate duplicates var hashIndexProps = hashKeyAndIndexProps.Indexed; var hashKeyProps = hashKeyAndIndexProps.Keys; var indexCoercionTypes = CoercionUtil.GetCoercionTypesHash( typePerStream, streamLookup, streamIndexed, hashKeyProps, hashIndexProps); var hashCoercionTypeArr = indexCoercionTypes.CoercionTypes; var rangeAndIndexProps = value.RangeProps; var rangeIndexProps = rangeAndIndexProps.Indexed; var rangeKeyProps = rangeAndIndexProps.Keys; var rangeCoercionTypes = CoercionUtil.GetCoercionTypesRange( typePerStream, streamIndexed, rangeIndexProps, rangeKeyProps); var rangeCoercionTypeArr = rangeCoercionTypes.CoercionTypes; if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 0) { var singles = value.InKeywordSingles; if (!singles.Key.IsEmpty()) { var indexedProp = singles.Indexed[0]; var indexedType = typePerStream[streamIndexed].GetPropertyType(indexedProp); var indexItem = new QueryPlanIndexItemForge( new[] {indexedProp}, new[] {indexedType}, new string[0], new Type[0], false, null, typePerStream[streamIndexed]); CheckDuplicateOrAdd(indexItem, indexesSet); } var multis = value.InKeywordMulti; if (!multis.IsEmpty()) { QueryGraphValuePairInKWMultiIdx multi = multis[0]; foreach (var propIndexed in multi.Indexed) { var identNode = (ExprIdentNode) propIndexed; var type = identNode.Forge.EvaluationType; var indexItem = new QueryPlanIndexItemForge( new[] {identNode.ResolvedPropertyName}, new[] {type}, new string[0], new Type[0], false, null, typePerStream[streamIndexed]); CheckDuplicateOrAdd(indexItem, indexesSet); } } continue; } // reduce to any unique index if applicable var unique = false; var reduced = QueryPlanIndexUniqueHelper.ReduceToUniqueIfPossible( hashIndexProps, hashCoercionTypeArr, hashKeyProps, indexedStreamsUniqueProps[streamIndexed]); if (reduced != null) { hashIndexProps = reduced.PropertyNames; hashCoercionTypeArr = reduced.CoercionTypes; unique = true; rangeIndexProps = new string[0]; rangeCoercionTypeArr = new Type[0]; } var proposed = new QueryPlanIndexItemForge( hashIndexProps, hashCoercionTypeArr, rangeIndexProps, rangeCoercionTypeArr, unique, null, typePerStream[streamIndexed]); CheckDuplicateOrAdd(proposed, indexesSet); } // create full-table-scan if (indexesSet.IsEmpty()) { indexesSet.Add( new QueryPlanIndexItemForge( new string[0], new Type[0], new string[0], new Type[0], false, null, typePerStream[streamIndexed])); } indexSpecs[streamIndexed] = QueryPlanIndexForge.MakeIndex(indexesSet); } return indexSpecs; }
/// <summary> /// Create the table lookup plan for a from-stream to look up in an indexed stream /// using the columns supplied in the query graph and looking at the actual indexes available /// and their index number. /// </summary> /// <param name="queryGraph">contains properties joining the 2 streams</param> /// <param name="currentLookupStream">stream to use key values from</param> /// <param name="indexedStream">stream to look up in</param> /// <param name="indexSpecs">index specification defining indexes to be created for stream</param> /// <param name="typesPerStream">event types for each stream</param> /// <param name="indexedStreamTableMeta">table info</param> /// <param name="indexedStreamIsVDW">vdw indicators</param> /// <param name="raw">raw statement information</param> /// <param name="serdeResolver">serde resolver</param> /// <returns>plan for performing a lookup in a given table using one of the indexes supplied</returns> public static TableLookupPlanDesc CreateLookupPlan( QueryGraphForge queryGraph, int currentLookupStream, int indexedStream, bool indexedStreamIsVDW, QueryPlanIndexForge indexSpecs, EventType[] typesPerStream, TableMetaData indexedStreamTableMeta, StatementRawInfo raw, SerdeCompileTimeResolver serdeResolver) { var queryGraphValue = queryGraph.GetGraphValue(currentLookupStream, indexedStream); var hashKeyProps = queryGraphValue.HashKeyProps; var hashPropsKeys = hashKeyProps.Keys; var hashIndexProps = hashKeyProps.Indexed; var rangeProps = queryGraphValue.RangeProps; var rangePropsKeys = rangeProps.Keys; var rangeIndexProps = rangeProps.Indexed; var pairIndexHashRewrite = indexSpecs.GetIndexNum(hashIndexProps, rangeIndexProps); var indexNum = pairIndexHashRewrite == null ? null : pairIndexHashRewrite.First; // handle index redirection towards unique index if (pairIndexHashRewrite != null && pairIndexHashRewrite.Second != null) { var indexes = pairIndexHashRewrite.Second; var newHashIndexProps = new string[indexes.Length]; IList<QueryGraphValueEntryHashKeyedForge> newHashKeys = new List<QueryGraphValueEntryHashKeyedForge>(); for (var i = 0; i < indexes.Length; i++) { newHashIndexProps[i] = hashIndexProps[indexes[i]]; newHashKeys.Add(hashPropsKeys[indexes[i]]); } hashIndexProps = newHashIndexProps; hashPropsKeys = newHashKeys; rangeIndexProps = new string[0]; rangePropsKeys = Collections.GetEmptyList<QueryGraphValueEntryRangeForge>(); } // no direct hash or range lookups if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 0) { // handle single-direction 'in' keyword var singles = queryGraphValue.InKeywordSingles; if (!singles.Key.IsEmpty()) { QueryGraphValueEntryInKeywordSingleIdxForge single = null; indexNum = null; if (indexedStreamTableMeta != null) { var indexes = singles.Indexed; var count = 0; foreach (var index in indexes) { var indexPairFound = EventTableIndexUtil.FindIndexBestAvailable( indexedStreamTableMeta.IndexMetadata.Indexes, Collections.SingletonSet(index), Collections.GetEmptySet<string>(), null); if (indexPairFound != null) { indexNum = new TableLookupIndexReqKey( indexPairFound.Second.OptionalIndexName, indexPairFound.Second.OptionalIndexModuleName, indexedStreamTableMeta.TableName); single = singles.Key[count]; } count++; } } else { single = singles.Key[0]; var pairIndex = indexSpecs.GetIndexNum( new[] {singles.Indexed[0]}, new string[0]); indexNum = pairIndex.First; } if (indexNum != null) { var forge = new InKeywordTableLookupPlanSingleIdxForge( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexNum, single.KeyExprs); return new TableLookupPlanDesc(forge, EmptyList<StmtClassForgeableFactory>.Instance); } } // handle multi-direction 'in' keyword var multis = queryGraphValue.InKeywordMulti; if (!multis.IsEmpty()) { if (indexedStreamTableMeta != null) { return GetFullTableScanTable( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexedStreamTableMeta); } var multi = multis[0]; var indexNameArray = new TableLookupIndexReqKey[multi.Indexed.Length]; var foundAll = true; for (var i = 0; i < multi.Indexed.Length; i++) { var identNode = (ExprIdentNode) multi.Indexed[i]; var pairIndex = indexSpecs.GetIndexNum( new[] {identNode.ResolvedPropertyName}, new string[0]); if (pairIndex == null) { foundAll = false; } else { indexNameArray[i] = pairIndex.First; } } if (foundAll) { var forge = new InKeywordTableLookupPlanMultiIdxForge( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexNameArray, multi.Key.KeyExpr); return new TableLookupPlanDesc(forge, EmptyList<StmtClassForgeableFactory>.Instance); } } // We don't use a keyed index but use the full stream set as the stream does not have any indexes // If no such full set index exists yet, add to specs if (indexedStreamTableMeta != null) { return GetFullTableScanTable( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexedStreamTableMeta); } if (indexNum == null) { indexNum = new TableLookupIndexReqKey( indexSpecs.AddIndex(new string[0], new Type[0], typesPerStream[indexedStream]), null); } var forgeX = new FullTableScanLookupPlanForge(currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexNum); return new TableLookupPlanDesc(forgeX, EmptyList<StmtClassForgeableFactory>.Instance); } if (indexNum == null) { throw new IllegalStateException( "Failed to query plan as index for " + hashIndexProps.RenderAny() + " and " + rangeIndexProps.RenderAny() + " in the index specification"); } if (indexedStreamTableMeta != null) { var indexPairFound = EventTableIndexUtil.FindIndexBestAvailable( indexedStreamTableMeta.IndexMetadata.Indexes, ToSet(hashIndexProps), ToSet(rangeIndexProps), null); if (indexPairFound != null) { var indexKeyInfo = SubordinateQueryPlannerUtil.CompileIndexKeyInfo( indexPairFound.First, hashIndexProps, GetHashKeyFuncsAsSubProp(hashPropsKeys), rangeIndexProps, GetRangeFuncsAsSubProp(rangePropsKeys)); if (indexKeyInfo.OrderedKeyCoercionTypes.IsCoerce || indexKeyInfo.OrderedRangeCoercionTypes.IsCoerce) { return GetFullTableScanTable( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexedStreamTableMeta); } hashPropsKeys = ToHashKeyFuncs(indexKeyInfo.OrderedHashDesc); hashIndexProps = IndexedPropDesc.GetIndexProperties(indexPairFound.First.HashIndexedProps); rangePropsKeys = ToRangeKeyFuncs(indexKeyInfo.OrderedRangeDesc); rangeIndexProps = IndexedPropDesc.GetIndexProperties(indexPairFound.First.RangeIndexedProps); indexNum = new TableLookupIndexReqKey( indexPairFound.Second.OptionalIndexName, indexPairFound.Second.OptionalIndexModuleName, indexedStreamTableMeta.TableName); // the plan will be created below if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 0) { return GetFullTableScanTable( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexedStreamTableMeta); } } else { return GetFullTableScanTable( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexedStreamTableMeta); } } // straight keyed-index lookup if (hashIndexProps.Length > 0 && rangeIndexProps.Length == 0) { // Determine coercion required var coercionTypes = CoercionUtil.GetCoercionTypesHash( typesPerStream, currentLookupStream, indexedStream, hashPropsKeys, hashIndexProps); if (coercionTypes.IsCoerce) { // check if there already are coercion types for this index var existCoercionTypes = indexSpecs.GetCoercionTypes(hashIndexProps); if (existCoercionTypes != null) { for (var i = 0; i < existCoercionTypes.Length; i++) { coercionTypes.CoercionTypes[i] = existCoercionTypes[i] .GetCompareToCoercionType(coercionTypes.CoercionTypes[i]); } } if (!indexSpecs.Items.IsEmpty()) { indexSpecs.SetCoercionTypes(hashIndexProps, coercionTypes.CoercionTypes); } } var coercionTypesArray = coercionTypes.CoercionTypes; MultiKeyClassRef tableLookupMultiKey = null; IList<StmtClassForgeableFactory> additionalForgeables = EmptyList<StmtClassForgeableFactory>.Instance; if (indexNum.TableName != null) { var tableMultiKeyPlan = MultiKeyPlanner.PlanMultiKey(coercionTypesArray, true, raw, serdeResolver); tableLookupMultiKey = tableMultiKeyPlan.ClassRef; additionalForgeables = tableMultiKeyPlan.MultiKeyForgeables; } var forge = new IndexedTableLookupPlanHashedOnlyForge( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexNum, hashPropsKeys.ToArray(), indexSpecs, coercionTypesArray, tableLookupMultiKey); return new TableLookupPlanDesc(forge, additionalForgeables); } // sorted index lookup var coercionTypesRange = CoercionUtil.GetCoercionTypesRange( typesPerStream, indexedStream, rangeIndexProps, rangePropsKeys); var coercionTypesHash = CoercionUtil.GetCoercionTypesHash( typesPerStream, currentLookupStream, indexedStream, hashPropsKeys, hashIndexProps); if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 1) { var range = rangePropsKeys[0]; Type coercionType = null; if (coercionTypesRange.IsCoerce) { coercionType = coercionTypesRange.CoercionTypes[0]; } SortedTableLookupPlanForge forge = new SortedTableLookupPlanForge( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexNum, range, coercionType); return new TableLookupPlanDesc(forge, EmptyList<StmtClassForgeableFactory>.Instance); } else { MultiKeyClassRef tableLookupMultiKey = null; IList<StmtClassForgeableFactory> additionalForgeables = EmptyList<StmtClassForgeableFactory>.Instance; if (indexNum.TableName != null) { MultiKeyPlan tableMultiKeyPlan = MultiKeyPlanner.PlanMultiKey(coercionTypesHash.CoercionTypes, true, raw, serdeResolver); tableLookupMultiKey = tableMultiKeyPlan.ClassRef; additionalForgeables = tableMultiKeyPlan.MultiKeyForgeables; } // composite range and index lookup CompositeTableLookupPlanForge forge = new CompositeTableLookupPlanForge( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexNum, hashPropsKeys, coercionTypesHash.CoercionTypes, rangePropsKeys, coercionTypesRange.CoercionTypes, indexSpecs, tableLookupMultiKey); return new TableLookupPlanDesc(forge, additionalForgeables); } }
public static QueryPlanForgeDesc Build( QueryGraphForge queryGraph, EventType[] typesPerStream, HistoricalViewableDesc historicalViewableDesc, DependencyGraph dependencyGraph, HistoricalStreamIndexListForge[] historicalStreamIndexLists, bool hasForceNestedIter, string[][][] indexedStreamsUniqueProps, TableMetaData[] tablesPerStream, StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult, StatementRawInfo raw, SerdeCompileTimeResolver serdeResolver) { if (Log.IsDebugEnabled) { Log.Debug(".build filterQueryGraph=" + queryGraph); } var numStreams = queryGraph.NumStreams; var additionalForgeables = new List<StmtClassForgeableFactory>(); var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec( queryGraph, typesPerStream, indexedStreamsUniqueProps); if (Log.IsDebugEnabled) { Log.Debug(".build Index build completed, indexes=" + QueryPlanIndexForge.Print(indexSpecs)); } // any historical streams don't get indexes, the lookup strategy accounts for cached indexes if (historicalViewableDesc.IsHistorical) { for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) { if (historicalViewableDesc.Historical[i]) { indexSpecs[i] = null; } } } var planNodeSpecs = new QueryPlanNodeForge[numStreams]; var worstDepth = int.MaxValue; for (var streamNo = 0; streamNo < numStreams; streamNo++) { // no plan for historical streams that are dependent upon other streams if (historicalViewableDesc.Historical[streamNo] && dependencyGraph.HasDependency(streamNo)) { planNodeSpecs[streamNo] = new QueryPlanNodeNoOpForge(); continue; } var bestChainResult = ComputeBestPath(streamNo, queryGraph, dependencyGraph); var bestChain = bestChainResult.Chain; if (Log.IsDebugEnabled) { Log.Debug(".build For stream " + streamNo + " bestChain=" + bestChain.RenderAny()); } if (bestChainResult.Depth < worstDepth) { worstDepth = bestChainResult.Depth; } var planDesc = CreateStreamPlan( streamNo, bestChain, queryGraph, indexSpecs, typesPerStream, historicalViewableDesc.Historical, historicalStreamIndexLists, tablesPerStream, streamJoinAnalysisResult, raw, serdeResolver); planNodeSpecs[streamNo] = planDesc.Forge; additionalForgeables.AddAll(planDesc.AdditionalForgeables); if (Log.IsDebugEnabled) { Log.Debug(".build spec=" + planNodeSpecs[streamNo]); } } // We use the merge/nested (outer) join algorithm instead. if (worstDepth < numStreams - 1 && !hasForceNestedIter) { return null; } // build historical index and lookup strategies for (var i = 0; i < numStreams; i++) { var plan = planNodeSpecs[i]; QueryPlanNodeForgeVisitor visitor = new ProxyQueryPlanNodeForgeVisitor { ProcVisit = node => { if (node is HistoricalDataPlanNodeForge) { var historical = (HistoricalDataPlanNodeForge) node; JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum].GetStrategy( historical.LookupStreamNum, raw, serdeResolver); historical.PollResultIndexingStrategy = desc.IndexingForge; historical.HistoricalIndexLookupStrategy = desc.LookupForge; additionalForgeables.AddAll(desc.AdditionalForgeables); } } }; plan.Accept(visitor); } var forge = new QueryPlanForge(indexSpecs, planNodeSpecs); return new QueryPlanForgeDesc(forge, additionalForgeables); }