Exemplo n.º 1
0
        /// <summary>
        /// Build index specification from navigability info.
        /// <para/>
        /// Looks at each stream and determines which properties in the stream must be indexed
        /// in order for other streams to look up into the stream. Determines the unique set of
        /// properties to avoid building duplicate indexes on the same set of properties.
        /// </summary>
        /// <param name="queryGraph">navigability info</param>
        /// <param name="typePerStream">The type per stream.</param>
        /// <param name="indexedStreamsUniqueProps">The indexed streams unique props.</param>
        /// <returns>query index specs for each stream</returns>
        public static QueryPlanIndex[] BuildIndexSpec(QueryGraph queryGraph, EventType[] typePerStream, String[][][] indexedStreamsUniqueProps)
        {
            var numStreams = queryGraph.NumStreams;
            var indexSpecs = new QueryPlanIndex[numStreams];

            // For each stream compile a list of index property sets.
            for (int streamIndexed = 0; streamIndexed < numStreams; streamIndexed++)
            {
                var indexesSet = new List <QueryPlanIndexItem>();

                // Look at the index from the viewpoint of the stream looking up in the index
                for (int streamLookup = 0; streamLookup < numStreams; streamLookup++)
                {
                    if (streamIndexed == streamLookup)
                    {
                        continue;
                    }

                    var value = queryGraph.GetGraphValue(streamLookup, streamIndexed);
                    var hashKeyAndIndexProps = value.HashKeyProps;

                    // Sort index properties, but use the sorted properties only to eliminate duplicates
                    var hashIndexProps      = hashKeyAndIndexProps.Indexed;
                    var hashKeyProps        = hashKeyAndIndexProps.Keys;
                    var indexCoercionTypes  = CoercionUtil.GetCoercionTypesHash(typePerStream, streamLookup, streamIndexed, hashKeyProps, hashIndexProps);
                    var hashCoercionTypeArr = indexCoercionTypes.CoercionTypes;

                    var rangeAndIndexProps   = value.RangeProps;
                    var rangeIndexProps      = rangeAndIndexProps.Indexed;
                    var rangeKeyProps        = rangeAndIndexProps.Keys;
                    var rangeCoercionTypes   = CoercionUtil.GetCoercionTypesRange(typePerStream, streamIndexed, rangeIndexProps, rangeKeyProps);
                    var rangeCoercionTypeArr = rangeCoercionTypes.CoercionTypes;

                    if (hashIndexProps.Count == 0 && rangeIndexProps.Count == 0)
                    {
                        QueryGraphValuePairInKWSingleIdx singles = value.InKeywordSingles;
                        if (!singles.Key.IsEmpty())
                        {
                            String             indexedProp = singles.Indexed[0];
                            QueryPlanIndexItem indexItem   = new QueryPlanIndexItem(new String[] { indexedProp }, null, null, null, false);
                            CheckDuplicateOrAdd(indexItem, indexesSet);
                        }

                        IList <QueryGraphValuePairInKWMultiIdx> multis = value.InKeywordMulti;
                        if (!multis.IsEmpty())
                        {
                            QueryGraphValuePairInKWMultiIdx multi = multis[0];
                            foreach (ExprNode propIndexed in multi.Indexed)
                            {
                                ExprIdentNode      identNode = (ExprIdentNode)propIndexed;
                                QueryPlanIndexItem indexItem = new QueryPlanIndexItem(new String[] { identNode.ResolvedPropertyName }, null, null, null, false);
                                CheckDuplicateOrAdd(indexItem, indexesSet);
                            }
                        }

                        continue;
                    }

                    // reduce to any unique index if applicable
                    var unique  = false;
                    var reduced = QueryPlanIndexUniqueHelper.ReduceToUniqueIfPossible(hashIndexProps, hashCoercionTypeArr, hashKeyProps, indexedStreamsUniqueProps[streamIndexed]);
                    if (reduced != null)
                    {
                        hashIndexProps      = reduced.PropertyNames;
                        hashCoercionTypeArr = reduced.CoercionTypes;
                        unique               = true;
                        rangeIndexProps      = new String[0];
                        rangeCoercionTypeArr = new Type[0];
                    }

                    var proposed = new QueryPlanIndexItem(hashIndexProps, hashCoercionTypeArr, rangeIndexProps, rangeCoercionTypeArr, unique);
                    CheckDuplicateOrAdd(proposed, indexesSet);
                }

                // create full-table-scan
                if (indexesSet.IsEmpty())
                {
                    indexesSet.Add(new QueryPlanIndexItem(null, null, null, null, false));
                }

                indexSpecs[streamIndexed] = QueryPlanIndex.MakeIndex(indexesSet);
            }

            return(indexSpecs);
        }
Exemplo n.º 2
0
        /// <summary>
        /// Create the table lookup plan for a from-stream to look up in an indexed stream
        /// using the columns supplied in the query graph and looking at the actual indexes available
        /// and their index number.
        /// </summary>
        /// <param name="queryGraph">contains properties joining the 2 streams</param>
        /// <param name="currentLookupStream">stream to use key values from</param>
        /// <param name="indexedStream">stream to look up in</param>
        /// <param name="indexSpecs">index specification defining indexes to be created for stream</param>
        /// <param name="typesPerStream">event types for each stream</param>
        /// <returns>plan for performing a lookup in a given table using one of the indexes supplied</returns>
        public static TableLookupPlan CreateLookupPlan(QueryGraph queryGraph, int currentLookupStream, int indexedStream,
                                                       QueryPlanIndex indexSpecs, EventType[] typesPerStream,
                                                       TableMetadata indexedStreamTableMeta)
        {
            var queryGraphValue = queryGraph.GetGraphValue(currentLookupStream, indexedStream);
            var hashKeyProps    = queryGraphValue.HashKeyProps;
            var hashPropsKeys   = hashKeyProps.Keys;
            var hashIndexProps  = hashKeyProps.Indexed.ToArray();

            var rangeProps      = queryGraphValue.RangeProps;
            var rangePropsKeys  = rangeProps.Keys;
            var rangeIndexProps = rangeProps.Indexed.ToArray();

            var pairIndexHashRewrite = indexSpecs.GetIndexNum(hashIndexProps, rangeIndexProps);
            var indexNum             = pairIndexHashRewrite == null ? null : pairIndexHashRewrite.First;

            // handle index redirection towards unique index
            if (pairIndexHashRewrite != null && pairIndexHashRewrite.Second != null)
            {
                var indexes           = pairIndexHashRewrite.Second;
                var newHashIndexProps = new string[indexes.Length];
                IList <QueryGraphValueEntryHashKeyed> newHashKeys = new List <QueryGraphValueEntryHashKeyed>();
                for (var i = 0; i < indexes.Length; i++)
                {
                    newHashIndexProps[i] = hashIndexProps[indexes[i]];
                    newHashKeys.Add(hashPropsKeys[indexes[i]]);
                }
                hashIndexProps  = newHashIndexProps;
                hashPropsKeys   = newHashKeys;
                rangeIndexProps = new string[0];
                rangePropsKeys  = Collections.GetEmptyList <QueryGraphValueEntryRange>();
            }

            // no direct hash or range lookups
            if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 0)
            {
                // handle single-direction 'in' keyword
                var singles = queryGraphValue.InKeywordSingles;
                if (!singles.Key.IsEmpty())
                {
                    QueryGraphValueEntryInKeywordSingleIdx single = null;
                    indexNum = null;
                    if (indexedStreamTableMeta != null)
                    {
                        var indexes = singles.Indexed;
                        var count   = 0;
                        foreach (var index in indexes)
                        {
                            Pair <IndexMultiKey, EventTableIndexEntryBase> indexPairFound =
                                EventTableIndexUtil.FindIndexBestAvailable(
                                    indexedStreamTableMeta.EventTableIndexMetadataRepo.Indexes,
                                    Collections.SingletonSet(index),
                                    Collections.GetEmptySet <string>(), null);
                            if (indexPairFound != null)
                            {
                                indexNum = new TableLookupIndexReqKey(indexPairFound.Second.OptionalIndexName, indexedStreamTableMeta.TableName);
                                single   = singles.Key[count];
                            }
                            count++;
                        }
                    }
                    else
                    {
                        single = singles.Key[0];
                        var pairIndex = indexSpecs.GetIndexNum(new string[] { singles.Indexed[0] }, null);
                        indexNum = pairIndex.First;
                    }

                    if (indexNum != null)
                    {
                        return(new InKeywordTableLookupPlanSingleIdx(currentLookupStream, indexedStream, indexNum, single.KeyExprs));
                    }
                }

                // handle multi-direction 'in' keyword
                var multis = queryGraphValue.InKeywordMulti;
                if (!multis.IsEmpty())
                {
                    if (indexedStreamTableMeta != null)
                    {
                        return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta));
                    }
                    QueryGraphValuePairInKWMultiIdx multi = multis[0];
                    var indexNameArray = new TableLookupIndexReqKey[multi.Indexed.Length];
                    var foundAll       = true;
                    for (var i = 0; i < multi.Indexed.Length; i++)
                    {
                        var identNode = (ExprIdentNode)multi.Indexed[i];
                        var pairIndex = indexSpecs.GetIndexNum(new string[] { identNode.ResolvedPropertyName }, null);
                        if (pairIndex == null)
                        {
                            foundAll = false;
                        }
                        else
                        {
                            indexNameArray[i] = pairIndex.First;
                        }
                    }
                    if (foundAll)
                    {
                        return(new InKeywordTableLookupPlanMultiIdx(currentLookupStream, indexedStream, indexNameArray, multi.Key.KeyExpr));
                    }
                }

                // We don't use a keyed index but use the full stream set as the stream does not have any indexes

                // If no such full set index exists yet, add to specs
                if (indexedStreamTableMeta != null)
                {
                    return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta));
                }
                if (indexNum == null)
                {
                    indexNum = new TableLookupIndexReqKey(indexSpecs.AddIndex(null, null));
                }
                return(new FullTableScanLookupPlan(currentLookupStream, indexedStream, indexNum));
            }

            if (indexNum == null)
            {
                throw new IllegalStateException("Failed to query plan as index for " + hashIndexProps.Render() + " and " + rangeIndexProps.Render() + " in the index specification");
            }

            if (indexedStreamTableMeta != null)
            {
                var indexPairFound = EventTableIndexUtil.FindIndexBestAvailable(indexedStreamTableMeta.EventTableIndexMetadataRepo.Indexes, ToSet(hashIndexProps), ToSet(rangeIndexProps), null);
                if (indexPairFound != null)
                {
                    var indexKeyInfo = SubordinateQueryPlannerUtil.CompileIndexKeyInfo(indexPairFound.First, hashIndexProps, GetHashKeyFuncsAsSubProp(hashPropsKeys), rangeIndexProps, GetRangeFuncsAsSubProp(rangePropsKeys));
                    if (indexKeyInfo.OrderedKeyCoercionTypes.IsCoerce || indexKeyInfo.OrderedRangeCoercionTypes.IsCoerce)
                    {
                        return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta));
                    }
                    hashPropsKeys   = ToHashKeyFuncs(indexKeyInfo.OrderedHashDesc);
                    hashIndexProps  = IndexedPropDesc.GetIndexProperties(indexPairFound.First.HashIndexedProps);
                    rangePropsKeys  = ToRangeKeyFuncs(indexKeyInfo.OrderedRangeDesc);
                    rangeIndexProps = IndexedPropDesc.GetIndexProperties(indexPairFound.First.RangeIndexedProps);
                    indexNum        = new TableLookupIndexReqKey(indexPairFound.Second.OptionalIndexName, indexedStreamTableMeta.TableName);
                    // the plan will be created below
                    if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 0)
                    {
                        return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta));
                    }
                }
                else
                {
                    return(GetFullTableScanTable(currentLookupStream, indexedStream, indexedStreamTableMeta));
                }
            }

            // straight keyed-index lookup
            if (hashIndexProps.Length > 0 && rangeIndexProps.Length == 0)
            {
                TableLookupPlan tableLookupPlan;
                if (hashPropsKeys.Count == 1)
                {
                    tableLookupPlan = new IndexedTableLookupPlanSingle(currentLookupStream, indexedStream, indexNum, hashPropsKeys[0]);
                }
                else
                {
                    tableLookupPlan = new IndexedTableLookupPlanMulti(currentLookupStream, indexedStream, indexNum, hashPropsKeys);
                }

                // Determine coercion required
                var coercionTypes = CoercionUtil.GetCoercionTypesHash(typesPerStream, currentLookupStream, indexedStream, hashPropsKeys, hashIndexProps);
                if (coercionTypes.IsCoerce)
                {
                    // check if there already are coercion types for this index
                    var existCoercionTypes = indexSpecs.GetCoercionTypes(hashIndexProps);
                    if (existCoercionTypes != null)
                    {
                        for (var i = 0; i < existCoercionTypes.Length; i++)
                        {
                            coercionTypes.CoercionTypes[i] = TypeHelper.GetCompareToCoercionType(existCoercionTypes[i], coercionTypes.CoercionTypes[i]);
                        }
                    }
                    indexSpecs.SetCoercionTypes(hashIndexProps, coercionTypes.CoercionTypes);
                }

                return(tableLookupPlan);
            }

            // sorted index lookup
            if (hashIndexProps.Length == 0 && rangeIndexProps.Length == 1)
            {
                QueryGraphValueEntryRange range = rangePropsKeys[0];
                return(new SortedTableLookupPlan(currentLookupStream, indexedStream, indexNum, range));
            }
            // composite range and index lookup
            else
            {
                return(new CompositeTableLookupPlan(currentLookupStream, indexedStream, indexNum, hashPropsKeys, rangePropsKeys));
            }
        }
Exemplo n.º 3
0
        public static SubordPropPlan GetJoinProps(ExprNode filterExpr, int outsideStreamCount, EventType[] allStreamTypesZeroIndexed, ExcludePlanHint excludePlanHint)
        {
            // No filter expression means full table scan
            if (filterExpr == null)
            {
                return(new SubordPropPlan());
            }

            // analyze query graph
            var queryGraph = new QueryGraph(outsideStreamCount + 1, excludePlanHint, true);

            FilterExprAnalyzer.Analyze(filterExpr, queryGraph, false);

            // Build a list of streams and indexes
            var joinProps  = new LinkedHashMap <String, SubordPropHashKey>();
            var rangeProps = new LinkedHashMap <String, SubordPropRangeKey>();

            for (int stream = 0; stream < outsideStreamCount; stream++)
            {
                int lookupStream = stream + 1;

                QueryGraphValue queryGraphValue = queryGraph.GetGraphValue(lookupStream, 0);
                QueryGraphValuePairHashKeyIndex hashKeysAndIndexes = queryGraphValue.HashKeyProps;

                // handle key-lookups
                var keyPropertiesJoin   = hashKeysAndIndexes.Keys;
                var indexPropertiesJoin = hashKeysAndIndexes.Indexed;
                if (keyPropertiesJoin.IsNotEmpty())
                {
                    if (keyPropertiesJoin.Count != indexPropertiesJoin.Count)
                    {
                        throw new IllegalStateException("Invalid query key and index property collection for stream " + stream);
                    }

                    for (int i = 0; i < keyPropertiesJoin.Count; i++)
                    {
                        QueryGraphValueEntryHashKeyed keyDesc = keyPropertiesJoin[i];
                        ExprNode compareNode = keyDesc.KeyExpr;

                        var keyPropType     = compareNode.ExprEvaluator.ReturnType.GetBoxedType();
                        var indexedPropType = allStreamTypesZeroIndexed[0].GetPropertyType(indexPropertiesJoin[i]).GetBoxedType();
                        var coercionType    = indexedPropType;
                        if (keyPropType != indexedPropType)
                        {
                            coercionType = keyPropType.GetCompareToCoercionType(indexedPropType);
                        }

                        SubordPropHashKey desc;
                        if (keyPropertiesJoin[i] is QueryGraphValueEntryHashKeyedExpr)
                        {
                            var keyExpr      = (QueryGraphValueEntryHashKeyedExpr)keyPropertiesJoin[i];
                            var keyStreamNum = keyExpr.IsRequiresKey ? stream : (int?)null;
                            desc = new SubordPropHashKey(keyDesc, keyStreamNum, coercionType);
                        }
                        else
                        {
                            var prop = (QueryGraphValueEntryHashKeyedProp)keyDesc;
                            desc = new SubordPropHashKey(prop, stream, coercionType);
                        }
                        joinProps.Put(indexPropertiesJoin[i], desc);
                    }
                }

                // handle range lookups
                QueryGraphValuePairRangeIndex rangeKeysAndIndexes = queryGraphValue.RangeProps;
                var rangeIndexes = rangeKeysAndIndexes.Indexed;
                var rangeDescs   = rangeKeysAndIndexes.Keys;
                if (rangeDescs.IsEmpty())
                {
                    continue;
                }

                // get all ranges lookups
                int count = -1;
                foreach (QueryGraphValueEntryRange rangeDesc in rangeDescs)
                {
                    count++;
                    String rangeIndexProp = rangeIndexes[count];

                    SubordPropRangeKey subqRangeDesc = rangeProps.Get(rangeIndexProp);

                    // other streams may specify the start or end endpoint of a range, therefore this operation can be additive
                    if (subqRangeDesc != null)
                    {
                        if (subqRangeDesc.RangeInfo.RangeType.IsRange())
                        {
                            continue;
                        }

                        // see if we can make this additive by using a range
                        var relOpOther = (QueryGraphValueEntryRangeRelOp)subqRangeDesc.RangeInfo;
                        var relOpThis  = (QueryGraphValueEntryRangeRelOp)rangeDesc;

                        QueryGraphRangeConsolidateDesc opsDesc = QueryGraphRangeUtil.GetCanConsolidate(
                            relOpThis.RangeType,
                            relOpOther.RangeType);
                        if (opsDesc != null)
                        {
                            ExprNode start;
                            ExprNode end;
                            if (!opsDesc.IsReverse)
                            {
                                start = relOpOther.Expression;
                                end   = relOpThis.Expression;
                            }
                            else
                            {
                                start = relOpThis.Expression;
                                end   = relOpOther.Expression;
                            }
                            var allowRangeReversal = relOpOther.IsBetweenPart && relOpThis.IsBetweenPart;
                            var rangeIn            = new QueryGraphValueEntryRangeIn(opsDesc.RangeType, start, end, allowRangeReversal);

                            var indexedPropType = allStreamTypesZeroIndexed[0].GetPropertyType(rangeIndexProp).GetBoxedType();
                            var coercionType    = indexedPropType;
                            var proposedType    = CoercionUtil.GetCoercionTypeRangeIn(indexedPropType, rangeIn.ExprStart, rangeIn.ExprEnd);
                            if (proposedType != null && proposedType != indexedPropType)
                            {
                                coercionType = proposedType;
                            }

                            subqRangeDesc = new SubordPropRangeKey(rangeIn, coercionType);
                            rangeProps.Put(rangeIndexProp, subqRangeDesc);
                        }
                        // ignore
                        continue;
                    }

                    // an existing entry has not been found
                    if (rangeDesc.RangeType.IsRange())
                    {
                        var rangeIn         = (QueryGraphValueEntryRangeIn)rangeDesc;
                        var indexedPropType = allStreamTypesZeroIndexed[0].GetPropertyType(rangeIndexProp).GetBoxedType();
                        var coercionType    = indexedPropType;
                        var proposedType    = CoercionUtil.GetCoercionTypeRangeIn(indexedPropType, rangeIn.ExprStart, rangeIn.ExprEnd);
                        if (proposedType != null && proposedType != indexedPropType)
                        {
                            coercionType = proposedType;
                        }
                        subqRangeDesc = new SubordPropRangeKey(rangeDesc, coercionType);
                    }
                    else
                    {
                        var relOp           = (QueryGraphValueEntryRangeRelOp)rangeDesc;
                        var keyPropType     = relOp.Expression.ExprEvaluator.ReturnType;
                        var indexedPropType = allStreamTypesZeroIndexed[0].GetPropertyType(rangeIndexProp).GetBoxedType();
                        var coercionType    = indexedPropType;
                        if (keyPropType != indexedPropType)
                        {
                            coercionType = keyPropType.GetCompareToCoercionType(indexedPropType);
                        }
                        subqRangeDesc = new SubordPropRangeKey(rangeDesc, coercionType);
                    }
                    rangeProps.Put(rangeIndexProp, subqRangeDesc);
                }
            }

            SubordPropInKeywordSingleIndex inKeywordSingleIdxProp = null;
            SubordPropInKeywordMultiIndex  inKeywordMultiIdxProp  = null;

            if (joinProps.IsEmpty() && rangeProps.IsEmpty())
            {
                for (int stream = 0; stream < outsideStreamCount; stream++)
                {
                    int             lookupStream    = stream + 1;
                    QueryGraphValue queryGraphValue = queryGraph.GetGraphValue(lookupStream, 0);

                    QueryGraphValuePairInKWSingleIdx inkwSingles = queryGraphValue.InKeywordSingles;
                    if (inkwSingles.Indexed.Length != 0)
                    {
                        ExprNode[] keys = inkwSingles.Key[0].KeyExprs;
                        String     key  = inkwSingles.Indexed[0];
                        if (inKeywordSingleIdxProp != null)
                        {
                            continue;
                        }
                        var coercionType = keys[0].ExprEvaluator.ReturnType;  // for in-comparison the same type is required
                        inKeywordSingleIdxProp = new SubordPropInKeywordSingleIndex(key, coercionType, keys);
                    }

                    IList <QueryGraphValuePairInKWMultiIdx> inkwMultis = queryGraphValue.InKeywordMulti;
                    if (!inkwMultis.IsEmpty())
                    {
                        QueryGraphValuePairInKWMultiIdx multi = inkwMultis[0];
                        inKeywordMultiIdxProp = new SubordPropInKeywordMultiIndex(ExprNodeUtility.GetIdentResolvedPropertyNames(multi.Indexed), multi.Indexed[0].ExprEvaluator.ReturnType, multi.Key.KeyExpr);
                    }

                    if (inKeywordSingleIdxProp != null && inKeywordMultiIdxProp != null)
                    {
                        inKeywordMultiIdxProp = null;
                    }
                }
            }

            return(new SubordPropPlan(joinProps, rangeProps, inKeywordSingleIdxProp, inKeywordMultiIdxProp));
        }