public static object[] GetIndexProperties( QueryGraphForge graph, int lookup, int indexed) { var val = graph.GetGraphValue(lookup, indexed); var pair = val.HashKeyProps; return(pair.Indexed); }
/// <summary> /// Build index specification from navigability info. /// <para /> /// Looks at each stream and determines which properties in the stream must be indexed /// in order for other streams to look up into the stream. Determines the unique set of properties /// to avoid building duplicate indexes on the same set of properties. /// </summary> /// <param name="queryGraph">navigability info</param> /// <param name="typePerStream">type info</param> /// <param name="indexedStreamsUniqueProps">per-stream unique props</param> /// <returns>query index specs for each stream</returns> public static QueryPlanIndexForge[] BuildIndexSpec( QueryGraphForge queryGraph, EventType[] typePerStream, string[][][] indexedStreamsUniqueProps) { var numStreams = queryGraph.NumStreams; var indexSpecs = new QueryPlanIndexForge[numStreams]; // For each stream compile a list of index property sets. for (var streamIndexed = 0; streamIndexed < numStreams; streamIndexed++) { IList<QueryPlanIndexItemForge> indexesSet = new List<QueryPlanIndexItemForge>(); // Look at the index from the viewpoint of the stream looking up in the index for (var streamLookup = 0; streamLookup < numStreams; streamLookup++) { if (streamIndexed == streamLookup) { continue; } var value = queryGraph.GetGraphValue(streamLookup, streamIndexed); var hashKeyAndIndexProps = value.HashKeyProps; // Sort index properties, but use the sorted properties only to eliminate duplicates var hashIndexProps = hashKeyAndIndexProps.Indexed; var hashKeyProps = hashKeyAndIndexProps.Keys; var indexCoercionTypes = CoercionUtil.GetCoercionTypesHash( typePerStream, streamLookup, streamIndexed, hashKeyProps, hashIndexProps); var hashCoercionTypeArr = indexCoercionTypes.CoercionTypes; var rangeAndIndexProps = value.RangeProps; var rangeIndexProps = rangeAndIndexProps.Indexed; var rangeKeyProps = rangeAndIndexProps.Keys; var rangeCoercionTypes = CoercionUtil.GetCoercionTypesRange( typePerStream, streamIndexed, rangeIndexProps, rangeKeyProps); var rangeCoercionTypeArr = rangeCoercionTypes.CoercionTypes; if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 0) { var singles = value.InKeywordSingles; if (!singles.Key.IsEmpty()) { var indexedProp = singles.Indexed[0]; var indexedType = typePerStream[streamIndexed].GetPropertyType(indexedProp); var indexItem = new QueryPlanIndexItemForge( new[] {indexedProp}, new[] {indexedType}, new string[0], new Type[0], false, null, typePerStream[streamIndexed]); CheckDuplicateOrAdd(indexItem, indexesSet); } var multis = value.InKeywordMulti; if (!multis.IsEmpty()) { QueryGraphValuePairInKWMultiIdx multi = multis[0]; foreach (var propIndexed in multi.Indexed) { var identNode = (ExprIdentNode) propIndexed; var type = identNode.Forge.EvaluationType; var indexItem = new QueryPlanIndexItemForge( new[] {identNode.ResolvedPropertyName}, new[] {type}, new string[0], new Type[0], false, null, typePerStream[streamIndexed]); CheckDuplicateOrAdd(indexItem, indexesSet); } } continue; } // reduce to any unique index if applicable var unique = false; var reduced = QueryPlanIndexUniqueHelper.ReduceToUniqueIfPossible( hashIndexProps, hashCoercionTypeArr, hashKeyProps, indexedStreamsUniqueProps[streamIndexed]); if (reduced != null) { hashIndexProps = reduced.PropertyNames; hashCoercionTypeArr = reduced.CoercionTypes; unique = true; rangeIndexProps = new string[0]; rangeCoercionTypeArr = new Type[0]; } var proposed = new QueryPlanIndexItemForge( hashIndexProps, hashCoercionTypeArr, rangeIndexProps, rangeCoercionTypeArr, unique, null, typePerStream[streamIndexed]); CheckDuplicateOrAdd(proposed, indexesSet); } // create full-table-scan if (indexesSet.IsEmpty()) { indexesSet.Add( new QueryPlanIndexItemForge( new string[0], new Type[0], new string[0], new Type[0], false, null, typePerStream[streamIndexed])); } indexSpecs[streamIndexed] = QueryPlanIndexForge.MakeIndex(indexesSet); } return indexSpecs; }
public static SubordPropPlan GetJoinProps( ExprNode filterExpr, int outsideStreamCount, EventType[] allStreamTypesZeroIndexed, ExcludePlanHint excludePlanHint) { // No filter expression means full table scan if (filterExpr == null) { return new SubordPropPlan(); } // analyze query graph var queryGraph = new QueryGraphForge(outsideStreamCount + 1, excludePlanHint, true); FilterExprAnalyzer.Analyze(filterExpr, queryGraph, false); // Build a list of streams and indexes var joinProps = new LinkedHashMap<string, SubordPropHashKeyForge>(); var rangeProps = new LinkedHashMap<string, SubordPropRangeKeyForge>(); IDictionary<QueryGraphValueEntryCustomKeyForge, QueryGraphValueEntryCustomOperationForge> customIndexOps = EmptyDictionary<QueryGraphValueEntryCustomKeyForge, QueryGraphValueEntryCustomOperationForge>.Instance; for (var stream = 0; stream < outsideStreamCount; stream++) { var lookupStream = stream + 1; var queryGraphValue = queryGraph.GetGraphValue(lookupStream, 0); var hashKeysAndIndexes = queryGraphValue.HashKeyProps; // determine application functions foreach (var item in queryGraphValue.Items) { if (item.Entry is QueryGraphValueEntryCustomForge) { if (customIndexOps.IsEmpty()) { customIndexOps = new Dictionary<QueryGraphValueEntryCustomKeyForge, QueryGraphValueEntryCustomOperationForge>(); } var custom = (QueryGraphValueEntryCustomForge) item.Entry; custom.MergeInto(customIndexOps); } } // handle key-lookups var keyPropertiesJoin = hashKeysAndIndexes.Keys; var indexPropertiesJoin = hashKeysAndIndexes.Indexed; if (!keyPropertiesJoin.IsEmpty()) { if (keyPropertiesJoin.Count != indexPropertiesJoin.Length) { throw new IllegalStateException( "Invalid query key and index property collection for stream " + stream); } for (var i = 0; i < keyPropertiesJoin.Count; i++) { QueryGraphValueEntryHashKeyedForge keyDesc = keyPropertiesJoin[i]; var compareNode = keyDesc.KeyExpr; var keyPropType = compareNode.Forge.EvaluationType.GetBoxedType(); var indexedPropType = allStreamTypesZeroIndexed[0] .GetPropertyType(indexPropertiesJoin[i]) .GetBoxedType(); var coercionType = indexedPropType; if (keyPropType != indexedPropType) { coercionType = keyPropType.GetCompareToCoercionType(indexedPropType); } SubordPropHashKeyForge desc; if (keyPropertiesJoin[i] is QueryGraphValueEntryHashKeyedForgeExpr) { var keyExpr = (QueryGraphValueEntryHashKeyedForgeExpr) keyPropertiesJoin[i]; var keyStreamNum = keyExpr.IsRequiresKey ? (int?) stream : null; desc = new SubordPropHashKeyForge(keyDesc, keyStreamNum, coercionType); } else { var prop = (QueryGraphValueEntryHashKeyedForgeProp) keyDesc; desc = new SubordPropHashKeyForge(prop, stream, coercionType); } joinProps.Put(indexPropertiesJoin[i], desc); } } // handle range lookups var rangeKeysAndIndexes = queryGraphValue.RangeProps; var rangeIndexes = rangeKeysAndIndexes.Indexed; var rangeDescs = rangeKeysAndIndexes.Keys; if (rangeDescs.IsEmpty()) { continue; } // get all ranges lookups var count = -1; foreach (var rangeDesc in rangeDescs) { count++; var rangeIndexProp = rangeIndexes[count]; var subqRangeDesc = rangeProps.Get(rangeIndexProp); // other streams may specify the start or end endpoint of a range, therefore this operation can be additive if (subqRangeDesc != null) { if (subqRangeDesc.RangeInfo.Type.IsRange()) { continue; } // see if we can make this additive by using a range var relOpOther = (QueryGraphValueEntryRangeRelOpForge) subqRangeDesc.RangeInfo; var relOpThis = (QueryGraphValueEntryRangeRelOpForge) rangeDesc; var opsDesc = QueryGraphRangeUtil.GetCanConsolidate(relOpThis.Type, relOpOther.Type); if (opsDesc != null) { ExprNode start; ExprNode end; if (!opsDesc.IsReverse) { start = relOpOther.Expression; end = relOpThis.Expression; } else { start = relOpThis.Expression; end = relOpOther.Expression; } var allowRangeReversal = relOpOther.IsBetweenPart && relOpThis.IsBetweenPart; var rangeIn = new QueryGraphValueEntryRangeInForge( opsDesc.Type, start, end, allowRangeReversal); var indexedPropType = allStreamTypesZeroIndexed[0] .GetPropertyType(rangeIndexProp) .GetBoxedType(); var coercionType = indexedPropType; var proposedType = CoercionUtil.GetCoercionTypeRangeIn( indexedPropType, rangeIn.ExprStart, rangeIn.ExprEnd); if (proposedType != null && proposedType != indexedPropType) { coercionType = proposedType; } subqRangeDesc = new SubordPropRangeKeyForge(rangeIn, coercionType); rangeProps.Put(rangeIndexProp, subqRangeDesc); } // ignore continue; } // an existing entry has not been found if (rangeDesc.Type.IsRange()) { var rangeIn = (QueryGraphValueEntryRangeInForge) rangeDesc; var indexedPropType = allStreamTypesZeroIndexed[0].GetPropertyType(rangeIndexProp).GetBoxedType(); var coercionType = indexedPropType; var proposedType = CoercionUtil.GetCoercionTypeRangeIn( indexedPropType, rangeIn.ExprStart, rangeIn.ExprEnd); if (proposedType != null && proposedType != indexedPropType) { coercionType = proposedType; } subqRangeDesc = new SubordPropRangeKeyForge(rangeDesc, coercionType); } else { var relOp = (QueryGraphValueEntryRangeRelOpForge) rangeDesc; var keyPropType = relOp.Expression.Forge.EvaluationType; var indexedPropType = allStreamTypesZeroIndexed[0].GetPropertyType(rangeIndexProp).GetBoxedType(); var coercionType = indexedPropType; if (keyPropType != indexedPropType) { coercionType = keyPropType.GetCompareToCoercionType(indexedPropType); } subqRangeDesc = new SubordPropRangeKeyForge(rangeDesc, coercionType); } rangeProps.Put(rangeIndexProp, subqRangeDesc); } } SubordPropInKeywordSingleIndex inKeywordSingleIdxProp = null; SubordPropInKeywordMultiIndex inKeywordMultiIdxProp = null; if (joinProps.IsEmpty() && rangeProps.IsEmpty()) { for (var stream = 0; stream < outsideStreamCount; stream++) { var lookupStream = stream + 1; var queryGraphValue = queryGraph.GetGraphValue(lookupStream, 0); var inkwSingles = queryGraphValue.InKeywordSingles; if (inkwSingles.Indexed.Length != 0) { ExprNode[] keys = inkwSingles.Key[0].KeyExprs; var key = inkwSingles.Indexed[0]; if (inKeywordSingleIdxProp != null) { continue; } var coercionType = keys[0].Forge.EvaluationType; // for in-comparison the same type is required inKeywordSingleIdxProp = new SubordPropInKeywordSingleIndex(key, coercionType, keys); } var inkwMultis = queryGraphValue.InKeywordMulti; if (!inkwMultis.IsEmpty()) { QueryGraphValuePairInKWMultiIdx multi = inkwMultis[0]; inKeywordMultiIdxProp = new SubordPropInKeywordMultiIndex( ExprNodeUtilityQuery.GetIdentResolvedPropertyNames(multi.Indexed), multi.Indexed[0].Forge.EvaluationType, multi.Key.KeyExpr); } if (inKeywordSingleIdxProp != null && inKeywordMultiIdxProp != null) { inKeywordMultiIdxProp = null; } } } return new SubordPropPlan( joinProps, rangeProps, inKeywordSingleIdxProp, inKeywordMultiIdxProp, customIndexOps); }
/// <summary> /// Create the table lookup plan for a from-stream to look up in an indexed stream /// using the columns supplied in the query graph and looking at the actual indexes available /// and their index number. /// </summary> /// <param name="queryGraph">contains properties joining the 2 streams</param> /// <param name="currentLookupStream">stream to use key values from</param> /// <param name="indexedStream">stream to look up in</param> /// <param name="indexSpecs">index specification defining indexes to be created for stream</param> /// <param name="typesPerStream">event types for each stream</param> /// <param name="indexedStreamTableMeta">table info</param> /// <param name="indexedStreamIsVDW">vdw indicators</param> /// <param name="raw">raw statement information</param> /// <param name="serdeResolver">serde resolver</param> /// <returns>plan for performing a lookup in a given table using one of the indexes supplied</returns> public static TableLookupPlanDesc CreateLookupPlan( QueryGraphForge queryGraph, int currentLookupStream, int indexedStream, bool indexedStreamIsVDW, QueryPlanIndexForge indexSpecs, EventType[] typesPerStream, TableMetaData indexedStreamTableMeta, StatementRawInfo raw, SerdeCompileTimeResolver serdeResolver) { var queryGraphValue = queryGraph.GetGraphValue(currentLookupStream, indexedStream); var hashKeyProps = queryGraphValue.HashKeyProps; var hashPropsKeys = hashKeyProps.Keys; var hashIndexProps = hashKeyProps.Indexed; var rangeProps = queryGraphValue.RangeProps; var rangePropsKeys = rangeProps.Keys; var rangeIndexProps = rangeProps.Indexed; var pairIndexHashRewrite = indexSpecs.GetIndexNum(hashIndexProps, rangeIndexProps); var indexNum = pairIndexHashRewrite == null ? null : pairIndexHashRewrite.First; // handle index redirection towards unique index if (pairIndexHashRewrite != null && pairIndexHashRewrite.Second != null) { var indexes = pairIndexHashRewrite.Second; var newHashIndexProps = new string[indexes.Length]; IList<QueryGraphValueEntryHashKeyedForge> newHashKeys = new List<QueryGraphValueEntryHashKeyedForge>(); for (var i = 0; i < indexes.Length; i++) { newHashIndexProps[i] = hashIndexProps[indexes[i]]; newHashKeys.Add(hashPropsKeys[indexes[i]]); } hashIndexProps = newHashIndexProps; hashPropsKeys = newHashKeys; rangeIndexProps = new string[0]; rangePropsKeys = Collections.GetEmptyList<QueryGraphValueEntryRangeForge>(); } // no direct hash or range lookups if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 0) { // handle single-direction 'in' keyword var singles = queryGraphValue.InKeywordSingles; if (!singles.Key.IsEmpty()) { QueryGraphValueEntryInKeywordSingleIdxForge single = null; indexNum = null; if (indexedStreamTableMeta != null) { var indexes = singles.Indexed; var count = 0; foreach (var index in indexes) { var indexPairFound = EventTableIndexUtil.FindIndexBestAvailable( indexedStreamTableMeta.IndexMetadata.Indexes, Collections.SingletonSet(index), Collections.GetEmptySet<string>(), null); if (indexPairFound != null) { indexNum = new TableLookupIndexReqKey( indexPairFound.Second.OptionalIndexName, indexPairFound.Second.OptionalIndexModuleName, indexedStreamTableMeta.TableName); single = singles.Key[count]; } count++; } } else { single = singles.Key[0]; var pairIndex = indexSpecs.GetIndexNum( new[] {singles.Indexed[0]}, new string[0]); indexNum = pairIndex.First; } if (indexNum != null) { var forge = new InKeywordTableLookupPlanSingleIdxForge( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexNum, single.KeyExprs); return new TableLookupPlanDesc(forge, EmptyList<StmtClassForgeableFactory>.Instance); } } // handle multi-direction 'in' keyword var multis = queryGraphValue.InKeywordMulti; if (!multis.IsEmpty()) { if (indexedStreamTableMeta != null) { return GetFullTableScanTable( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexedStreamTableMeta); } var multi = multis[0]; var indexNameArray = new TableLookupIndexReqKey[multi.Indexed.Length]; var foundAll = true; for (var i = 0; i < multi.Indexed.Length; i++) { var identNode = (ExprIdentNode) multi.Indexed[i]; var pairIndex = indexSpecs.GetIndexNum( new[] {identNode.ResolvedPropertyName}, new string[0]); if (pairIndex == null) { foundAll = false; } else { indexNameArray[i] = pairIndex.First; } } if (foundAll) { var forge = new InKeywordTableLookupPlanMultiIdxForge( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexNameArray, multi.Key.KeyExpr); return new TableLookupPlanDesc(forge, EmptyList<StmtClassForgeableFactory>.Instance); } } // We don't use a keyed index but use the full stream set as the stream does not have any indexes // If no such full set index exists yet, add to specs if (indexedStreamTableMeta != null) { return GetFullTableScanTable( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexedStreamTableMeta); } if (indexNum == null) { indexNum = new TableLookupIndexReqKey( indexSpecs.AddIndex(new string[0], new Type[0], typesPerStream[indexedStream]), null); } var forgeX = new FullTableScanLookupPlanForge(currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexNum); return new TableLookupPlanDesc(forgeX, EmptyList<StmtClassForgeableFactory>.Instance); } if (indexNum == null) { throw new IllegalStateException( "Failed to query plan as index for " + hashIndexProps.RenderAny() + " and " + rangeIndexProps.RenderAny() + " in the index specification"); } if (indexedStreamTableMeta != null) { var indexPairFound = EventTableIndexUtil.FindIndexBestAvailable( indexedStreamTableMeta.IndexMetadata.Indexes, ToSet(hashIndexProps), ToSet(rangeIndexProps), null); if (indexPairFound != null) { var indexKeyInfo = SubordinateQueryPlannerUtil.CompileIndexKeyInfo( indexPairFound.First, hashIndexProps, GetHashKeyFuncsAsSubProp(hashPropsKeys), rangeIndexProps, GetRangeFuncsAsSubProp(rangePropsKeys)); if (indexKeyInfo.OrderedKeyCoercionTypes.IsCoerce || indexKeyInfo.OrderedRangeCoercionTypes.IsCoerce) { return GetFullTableScanTable( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexedStreamTableMeta); } hashPropsKeys = ToHashKeyFuncs(indexKeyInfo.OrderedHashDesc); hashIndexProps = IndexedPropDesc.GetIndexProperties(indexPairFound.First.HashIndexedProps); rangePropsKeys = ToRangeKeyFuncs(indexKeyInfo.OrderedRangeDesc); rangeIndexProps = IndexedPropDesc.GetIndexProperties(indexPairFound.First.RangeIndexedProps); indexNum = new TableLookupIndexReqKey( indexPairFound.Second.OptionalIndexName, indexPairFound.Second.OptionalIndexModuleName, indexedStreamTableMeta.TableName); // the plan will be created below if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 0) { return GetFullTableScanTable( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexedStreamTableMeta); } } else { return GetFullTableScanTable( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexedStreamTableMeta); } } // straight keyed-index lookup if (hashIndexProps.Length > 0 && rangeIndexProps.Length == 0) { // Determine coercion required var coercionTypes = CoercionUtil.GetCoercionTypesHash( typesPerStream, currentLookupStream, indexedStream, hashPropsKeys, hashIndexProps); if (coercionTypes.IsCoerce) { // check if there already are coercion types for this index var existCoercionTypes = indexSpecs.GetCoercionTypes(hashIndexProps); if (existCoercionTypes != null) { for (var i = 0; i < existCoercionTypes.Length; i++) { coercionTypes.CoercionTypes[i] = existCoercionTypes[i] .GetCompareToCoercionType(coercionTypes.CoercionTypes[i]); } } if (!indexSpecs.Items.IsEmpty()) { indexSpecs.SetCoercionTypes(hashIndexProps, coercionTypes.CoercionTypes); } } var coercionTypesArray = coercionTypes.CoercionTypes; MultiKeyClassRef tableLookupMultiKey = null; IList<StmtClassForgeableFactory> additionalForgeables = EmptyList<StmtClassForgeableFactory>.Instance; if (indexNum.TableName != null) { var tableMultiKeyPlan = MultiKeyPlanner.PlanMultiKey(coercionTypesArray, true, raw, serdeResolver); tableLookupMultiKey = tableMultiKeyPlan.ClassRef; additionalForgeables = tableMultiKeyPlan.MultiKeyForgeables; } var forge = new IndexedTableLookupPlanHashedOnlyForge( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexNum, hashPropsKeys.ToArray(), indexSpecs, coercionTypesArray, tableLookupMultiKey); return new TableLookupPlanDesc(forge, additionalForgeables); } // sorted index lookup var coercionTypesRange = CoercionUtil.GetCoercionTypesRange( typesPerStream, indexedStream, rangeIndexProps, rangePropsKeys); var coercionTypesHash = CoercionUtil.GetCoercionTypesHash( typesPerStream, currentLookupStream, indexedStream, hashPropsKeys, hashIndexProps); if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 1) { var range = rangePropsKeys[0]; Type coercionType = null; if (coercionTypesRange.IsCoerce) { coercionType = coercionTypesRange.CoercionTypes[0]; } SortedTableLookupPlanForge forge = new SortedTableLookupPlanForge( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexNum, range, coercionType); return new TableLookupPlanDesc(forge, EmptyList<StmtClassForgeableFactory>.Instance); } else { MultiKeyClassRef tableLookupMultiKey = null; IList<StmtClassForgeableFactory> additionalForgeables = EmptyList<StmtClassForgeableFactory>.Instance; if (indexNum.TableName != null) { MultiKeyPlan tableMultiKeyPlan = MultiKeyPlanner.PlanMultiKey(coercionTypesHash.CoercionTypes, true, raw, serdeResolver); tableLookupMultiKey = tableMultiKeyPlan.ClassRef; additionalForgeables = tableMultiKeyPlan.MultiKeyForgeables; } // composite range and index lookup CompositeTableLookupPlanForge forge = new CompositeTableLookupPlanForge( currentLookupStream, indexedStream, indexedStreamIsVDW, typesPerStream, indexNum, hashPropsKeys, coercionTypesHash.CoercionTypes, rangePropsKeys, coercionTypesRange.CoercionTypes, indexSpecs, tableLookupMultiKey); return new TableLookupPlanDesc(forge, additionalForgeables); } }
/// <summary> /// Get the strategies to use for polling from a given stream. /// </summary> /// <param name="streamViewStreamNum">the stream providing the polling events</param> /// <returns>looking and indexing strategy</returns> public JoinSetComposerPrototypeHistoricalDesc GetStrategy( int streamViewStreamNum, StatementRawInfo raw, SerdeCompileTimeResolver serdeResolver) { // If there is only a single polling stream, then build a single index if (_pollingStreams.Count == 1) { return JoinSetComposerPrototypeForgeFactory.DetermineIndexing( _queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[streamViewStreamNum], _historicalStreamNum, streamViewStreamNum, raw, serdeResolver); } // If there are multiple polling streams, determine if a single index is appropriate. // An index can be reused if: // (a) indexed property names are the same // (b) indexed property types are the same // (c) key property types are the same (because of coercion) // A index lookup strategy is always specific to the providing stream. var additionalForgeables = new List<StmtClassForgeableFactory>(); if (_indexesUsedByStreams == null) { _indexesUsedByStreams = new LinkedHashMap<HistoricalStreamIndexDesc, IList<int>>(); foreach (var pollingStream in _pollingStreams) { var queryGraphValue = _queryGraph.GetGraphValue(pollingStream, _historicalStreamNum); var hashKeyProps = queryGraphValue.HashKeyProps; var indexProperties = hashKeyProps.Indexed; var keyTypes = GetPropertyTypes(hashKeyProps.Keys); var indexTypes = GetPropertyTypes(_typesPerStream[_historicalStreamNum], indexProperties); var desc = new HistoricalStreamIndexDesc( indexProperties, indexTypes, keyTypes); var usedByStreams = _indexesUsedByStreams.Get(desc); if (usedByStreams == null) { usedByStreams = new List<int>(); _indexesUsedByStreams.Put(desc, usedByStreams); } usedByStreams.Add(pollingStream); } // There are multiple indexes required: // Build a master indexing strategy that forms multiple indexes and numbers each. if (_indexesUsedByStreams.Count > 1) { var numIndexes = _indexesUsedByStreams.Count; var indexingStrategies = new PollResultIndexingStrategyForge[numIndexes]; // create an indexing strategy for each index var count = 0; foreach (var desc in _indexesUsedByStreams) { var sampleStreamViewStreamNum = desc.Value[0]; var indexingX = JoinSetComposerPrototypeForgeFactory.DetermineIndexing( _queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[sampleStreamViewStreamNum], _historicalStreamNum, sampleStreamViewStreamNum, raw, serdeResolver); indexingStrategies[count] = indexingX.IndexingForge; additionalForgeables.AddAll(indexingX.AdditionalForgeables); count++; } // create a master indexing strategy that utilizes each indexing strategy to create a set of indexes _masterIndexingStrategy = new PollResultIndexingStrategyMultiForge( streamViewStreamNum, indexingStrategies); } } // there is one type of index if (_indexesUsedByStreams.Count == 1) { return JoinSetComposerPrototypeForgeFactory.DetermineIndexing( _queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[streamViewStreamNum], _historicalStreamNum, streamViewStreamNum, raw, serdeResolver); } // determine which index number the polling stream must use var indexUsed = 0; var found = false; foreach (var desc in _indexesUsedByStreams.Values) { if (desc.Contains(streamViewStreamNum)) { found = true; break; } indexUsed++; } if (!found) { throw new IllegalStateException("Index not found for use by stream " + streamViewStreamNum); } // Use one of the indexes built by the master index and a lookup strategy JoinSetComposerPrototypeHistoricalDesc indexing = JoinSetComposerPrototypeForgeFactory.DetermineIndexing( _queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[streamViewStreamNum], _historicalStreamNum, streamViewStreamNum, raw, serdeResolver); HistoricalIndexLookupStrategyForge innerLookupStrategy = indexing.LookupForge; HistoricalIndexLookupStrategyForge lookupStrategy = new HistoricalIndexLookupStrategyMultiForge(indexUsed, innerLookupStrategy); additionalForgeables.AddAll(indexing.AdditionalForgeables); return new JoinSetComposerPrototypeHistoricalDesc(lookupStrategy, _masterIndexingStrategy, additionalForgeables); }
/// <summary> /// Get the strategies to use for polling from a given stream. /// </summary> /// <param name="streamViewStreamNum">the stream providing the polling events</param> /// <returns>looking and indexing strategy</returns> public Pair<HistoricalIndexLookupStrategyForge, PollResultIndexingStrategyForge> GetStrategy( int streamViewStreamNum) { // If there is only a single polling stream, then build a single index if (pollingStreams.Count == 1) { return JoinSetComposerPrototypeForgeFactory.DetermineIndexing( queryGraph, typesPerStream[historicalStreamNum], typesPerStream[streamViewStreamNum], historicalStreamNum, streamViewStreamNum); } // If there are multiple polling streams, determine if a single index is appropriate. // An index can be reused if: // (a) indexed property names are the same // (b) indexed property types are the same // (c) key property types are the same (because of coercion) // A index lookup strategy is always specific to the providing stream. if (indexesUsedByStreams == null) { indexesUsedByStreams = new LinkedHashMap<HistoricalStreamIndexDesc, IList<int>>(); foreach (int pollingStream in pollingStreams) { QueryGraphValueForge queryGraphValue = queryGraph.GetGraphValue(pollingStream, historicalStreamNum); QueryGraphValuePairHashKeyIndexForge hashKeyProps = queryGraphValue.HashKeyProps; string[] indexProperties = hashKeyProps.Indexed; Type[] keyTypes = GetPropertyTypes(hashKeyProps.Keys); Type[] indexTypes = GetPropertyTypes(typesPerStream[historicalStreamNum], indexProperties); HistoricalStreamIndexDesc desc = new HistoricalStreamIndexDesc( indexProperties, indexTypes, keyTypes); IList<int> usedByStreams = indexesUsedByStreams.Get(desc); if (usedByStreams == null) { usedByStreams = new List<int>(); indexesUsedByStreams.Put(desc, usedByStreams); } usedByStreams.Add(pollingStream); } // There are multiple indexes required: // Build a master indexing strategy that forms multiple indexes and numbers each. if (indexesUsedByStreams.Count > 1) { int numIndexes = indexesUsedByStreams.Count; PollResultIndexingStrategyForge[] indexingStrategies = new PollResultIndexingStrategyForge[numIndexes]; // create an indexing strategy for each index int count = 0; foreach (KeyValuePair<HistoricalStreamIndexDesc, IList<int>> desc in indexesUsedByStreams) { int sampleStreamViewStreamNum = desc.Value[0]; indexingStrategies[count] = JoinSetComposerPrototypeForgeFactory.DetermineIndexing( queryGraph, typesPerStream[historicalStreamNum], typesPerStream[sampleStreamViewStreamNum], historicalStreamNum, sampleStreamViewStreamNum) .Second; count++; } // create a master indexing strategy that utilizes each indexing strategy to create a set of indexes masterIndexingStrategy = new PollResultIndexingStrategyMultiForge( streamViewStreamNum, indexingStrategies); } } // there is one type of index if (indexesUsedByStreams.Count == 1) { return JoinSetComposerPrototypeForgeFactory.DetermineIndexing( queryGraph, typesPerStream[historicalStreamNum], typesPerStream[streamViewStreamNum], historicalStreamNum, streamViewStreamNum); } // determine which index number the polling stream must use int indexUsed = 0; bool found = false; foreach (IList<int> desc in indexesUsedByStreams.Values) { if (desc.Contains(streamViewStreamNum)) { found = true; break; } indexUsed++; } if (!found) { throw new IllegalStateException("Index not found for use by stream " + streamViewStreamNum); } // Use one of the indexes built by the master index and a lookup strategy HistoricalIndexLookupStrategyForge innerLookupStrategy = JoinSetComposerPrototypeForgeFactory .DetermineIndexing( queryGraph, typesPerStream[historicalStreamNum], typesPerStream[streamViewStreamNum], historicalStreamNum, streamViewStreamNum) .First; HistoricalIndexLookupStrategyForge lookupStrategy = new HistoricalIndexLookupStrategyMultiForge(indexUsed, innerLookupStrategy); return new Pair<HistoricalIndexLookupStrategyForge, PollResultIndexingStrategyForge>( lookupStrategy, masterIndexingStrategy); }
/// <summary> /// Constructs indexing and lookup strategy for a given relationship that a historical stream may have with another /// stream (historical or not) that looks up into results of a poll of a historical stream. /// <para /> /// The term "polled" refers to the assumed-historical stream. /// </summary> /// <param name="queryGraph">relationship representation of where-clause filter and outer join on-expressions</param> /// <param name="polledViewType">the event type of the historical that is indexed</param> /// <param name="streamViewType">the event type of the stream looking up in indexes</param> /// <param name="polledViewStreamNum">the stream number of the historical that is indexed</param> /// <param name="streamViewStreamNum">the stream number of the historical that is looking up</param> /// <param name="raw"></param> /// <param name="serdeResolver"></param> /// <returns>indexing and lookup strategy pair</returns> public static JoinSetComposerPrototypeHistoricalDesc DetermineIndexing( QueryGraphForge queryGraph, EventType polledViewType, EventType streamViewType, int polledViewStreamNum, int streamViewStreamNum, StatementRawInfo raw, SerdeCompileTimeResolver serdeResolver) { var queryGraphValue = queryGraph.GetGraphValue(streamViewStreamNum, polledViewStreamNum); var hashKeysAndIndes = queryGraphValue.HashKeyProps; var rangeKeysAndIndex = queryGraphValue.RangeProps; // index and key property names var hashKeys = hashKeysAndIndes.Keys; var hashIndexes = hashKeysAndIndes.Indexed; var rangeKeys = rangeKeysAndIndex.Keys; var rangeIndexes = rangeKeysAndIndex.Indexed; // If the analysis revealed no join columns, must use the brute-force full table scan if (hashKeys.IsEmpty() && rangeKeys.IsEmpty()) { var inKeywordSingles = queryGraphValue.InKeywordSingles; if (inKeywordSingles != null && inKeywordSingles.Indexed.Length != 0) { var indexed = inKeywordSingles.Indexed[0]; var lookup = inKeywordSingles.Key[0]; var strategy = new HistoricalIndexLookupStrategyInKeywordSingleForge(streamViewStreamNum, lookup.KeyExprs); var indexing = new PollResultIndexingStrategyHashForge( polledViewStreamNum, polledViewType, new string[]{ indexed }, null, null); return new JoinSetComposerPrototypeHistoricalDesc(strategy, indexing, EmptyList<StmtClassForgeableFactory>.Instance); } var multis = queryGraphValue.InKeywordMulti; if (!multis.IsEmpty()) { var multi = multis[0]; var strategy = new HistoricalIndexLookupStrategyInKeywordMultiForge(streamViewStreamNum, multi.Key.KeyExpr); var indexing = new PollResultIndexingStrategyInKeywordMultiForge( polledViewStreamNum, polledViewType, ExprNodeUtilityQuery.GetIdentResolvedPropertyNames(multi.Indexed)); return new JoinSetComposerPrototypeHistoricalDesc(strategy, indexing, EmptyList<StmtClassForgeableFactory>.Instance); } return new JoinSetComposerPrototypeHistoricalDesc( HistoricalIndexLookupStrategyNoIndexForge.INSTANCE, PollResultIndexingStrategyNoIndexForge.INSTANCE, EmptyList<StmtClassForgeableFactory>.Instance); } CoercionDesc keyCoercionTypes = CoercionUtil.GetCoercionTypesHash( new[] {streamViewType, polledViewType}, 0, 1, hashKeys, hashIndexes); if (rangeKeys.IsEmpty()) { var hashEvals = QueryGraphValueEntryHashKeyedForge.GetForges(hashKeys.ToArray()); var multiKeyPlan = MultiKeyPlanner.PlanMultiKey(hashEvals, false, raw, serdeResolver); var lookup = new HistoricalIndexLookupStrategyHashForge( streamViewStreamNum, hashEvals, keyCoercionTypes.CoercionTypes, multiKeyPlan.ClassRef); var indexing = new PollResultIndexingStrategyHashForge( polledViewStreamNum, polledViewType, hashIndexes, keyCoercionTypes.CoercionTypes, multiKeyPlan.ClassRef); return new JoinSetComposerPrototypeHistoricalDesc(lookup, indexing, multiKeyPlan.MultiKeyForgeables); } CoercionDesc rangeCoercionTypes = CoercionUtil.GetCoercionTypesRange( new[] {streamViewType, polledViewType}, 1, rangeIndexes, rangeKeys); if (rangeKeys.Count == 1 && hashKeys.Count == 0) { var rangeCoercionType = rangeCoercionTypes.CoercionTypes[0]; var indexing = new PollResultIndexingStrategySortedForge( polledViewStreamNum, polledViewType, rangeIndexes[0], rangeCoercionType); var lookup = new HistoricalIndexLookupStrategySortedForge( streamViewStreamNum, rangeKeys[0], rangeCoercionType); return new JoinSetComposerPrototypeHistoricalDesc(lookup, indexing, EmptyList<StmtClassForgeableFactory>.Instance); } else { var hashEvals = QueryGraphValueEntryHashKeyedForge.GetForges(hashKeys.ToArray()); var multiKeyPlan = MultiKeyPlanner.PlanMultiKey(hashEvals, false, raw, serdeResolver); var strategy = new HistoricalIndexLookupStrategyCompositeForge( streamViewStreamNum, hashEvals, multiKeyPlan.ClassRef, rangeKeys.ToArray()); var indexing = new PollResultIndexingStrategyCompositeForge( polledViewStreamNum, polledViewType, hashIndexes, keyCoercionTypes.CoercionTypes, multiKeyPlan.ClassRef, rangeIndexes, rangeCoercionTypes.CoercionTypes); return new JoinSetComposerPrototypeHistoricalDesc(strategy, indexing, multiKeyPlan.MultiKeyForgeables); } }