Example #1
0
        public void TestCreateStreamPlan()
        {
            QueryPlanIndexForge[] indexes = QueryPlanIndexBuilder.BuildIndexSpec(queryGraph, typesPerStream, new string[queryGraph.NumStreams][][]);
            for (int i = 0; i < indexes.Length; i++)
            {
                log.Debug(".testCreateStreamPlan index " + i + " = " + indexes[i]);
            }

            QueryPlanNodeForge plan = NStreamQueryPlanBuilder.CreateStreamPlan(0, new int[] { 2, 4, 3, 1 }, queryGraph, indexes, typesPerStream, new bool[5], null, new TableMetaData[queryGraph.NumStreams], new StreamJoinAnalysisResultCompileTime(5));

            log.Debug(".testCreateStreamPlan plan=" + plan);

            Assert.IsTrue(plan is NestedIterationNodeForge);
            NestedIterationNodeForge nested          = (NestedIterationNodeForge)plan;
            TableLookupNodeForge     tableLookupSpec = (TableLookupNodeForge)nested.ChildNodes[0];

            // Check lookup strategy for first lookup
            IndexedTableLookupPlanHashedOnlyForge lookupStrategySpec = (IndexedTableLookupPlanHashedOnlyForge)tableLookupSpec.LookupStrategySpec;

            Assert.AreEqual("P01", ((ExprIdentNode)(lookupStrategySpec.HashKeys[0]).KeyExpr).ResolvedPropertyName);
            Assert.AreEqual(0, lookupStrategySpec.LookupStream);
            Assert.AreEqual(2, lookupStrategySpec.IndexedStream);
            Assert.IsNotNull(lookupStrategySpec.IndexNum);

            // Check lookup strategy for last lookup
            tableLookupSpec = (TableLookupNodeForge)nested.ChildNodes[3];
            FullTableScanLookupPlanForge unkeyedSpecScan = (FullTableScanLookupPlanForge)tableLookupSpec.LookupStrategySpec;

            Assert.AreEqual(1, unkeyedSpecScan.IndexedStream);
            Assert.IsNotNull(unkeyedSpecScan.IndexNum);
        }
        public void TestBuildDefaultNestingOrder()
        {
            int[] result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 0);
            Assert.IsTrue(Arrays.AreEqual(result, new int[] { 1, 2, 3 }));

            result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 1);
            Assert.IsTrue(Arrays.AreEqual(result, new int[] { 0, 2, 3 }));

            result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 2);
            Assert.IsTrue(Arrays.AreEqual(result, new int[] { 0, 1, 3 }));

            result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 3);
            Assert.IsTrue(Arrays.AreEqual(result, new int[] { 0, 1, 2 }));
        }
        public void TestBuild()
        {
            var plan = NStreamQueryPlanBuilder.Build(
                queryGraph,
                typesPerStream,
                new HistoricalViewableDesc(6),
                dependencyGraph,
                null,
                false,
                new string[queryGraph.NumStreams][][],
                new TableMetaData[queryGraph.NumStreams],
                new StreamJoinAnalysisResultCompileTime(5),
                null,
                SerdeCompileTimeResolverNonHA.INSTANCE);

            log.Debug(".testBuild plan=" + plan);
        }
        public void TestComputeBestPath()
        {
            NStreamQueryPlanBuilder.BestChainResult bestChain = NStreamQueryPlanBuilder.ComputeBestPath(0, queryGraph, dependencyGraph);
            Assert.AreEqual(3, bestChain.Depth);
            Assert.IsTrue(Arrays.AreEqual(bestChain.Chain, new int[] { 2, 4, 3, 1 }));

            bestChain = NStreamQueryPlanBuilder.ComputeBestPath(3, queryGraph, dependencyGraph);
            Assert.AreEqual(4, bestChain.Depth);
            Assert.IsTrue(Arrays.AreEqual(bestChain.Chain, new int[] { 4, 2, 0, 1 }));

            // try a stream that is not connected in any way
            queryGraph = new QueryGraphForge(6, null, false);
            bestChain  = NStreamQueryPlanBuilder.ComputeBestPath(5, queryGraph, dependencyGraph);
            log.Debug(".testComputeBestPath bestChain=" + bestChain);
            Assert.AreEqual(0, bestChain.Depth);
            Assert.IsTrue(Arrays.AreEqual(bestChain.Chain, new int[] { 0, 1, 2, 3, 4 }));
        }
        public void TestComputeNavigableDepth()
        {
            ExprIdentNode fake = supportExprNodeFactory.MakeIdentNode("TheString", "s0");

            queryGraph.AddStrictEquals(3, "P30", fake, 2, "P20", fake);
            queryGraph.AddStrictEquals(2, "P30", fake, 1, "P20", fake);

            int depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(0, new int[] { 1, 2, 3, 4 }, queryGraph);

            Assert.AreEqual(4, depth);

            depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(0, new int[] { 4, 2, 3, 1 }, queryGraph);
            Assert.AreEqual(0, depth);

            depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(4, new int[] { 3, 2, 1, 0 }, queryGraph);
            Assert.AreEqual(4, depth);

            depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(1, new int[] { 0, 3, 4, 2 }, queryGraph);
            Assert.AreEqual(1, depth);
        }
        public void TestIsDependencySatisfied()
        {
            DependencyGraph graph = new DependencyGraph(3, false);

            graph.AddDependency(1, 0);
            graph.AddDependency(2, 0);

            Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(0, new int[] { 1, 2 }, graph));
            Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 0, 2 }, graph));
            Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(2, new int[] { 0, 1 }, graph));

            graph = new DependencyGraph(5, false);
            graph.AddDependency(4, 1);
            graph.AddDependency(4, 2);
            graph.AddDependency(2, 0);

            Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(0, new int[] { 1, 2, 3, 4 }, graph));
            Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 0, 2, 3, 4 }, graph));
            Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 2, 0, 3, 4 }, graph));
            Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 4, 0, 3, 2 }, graph));
            Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(3, new int[] { 4, 0, 1, 2 }, graph));
            Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(2, new int[] { 3, 1, 4, 0 }, graph));
            Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(3, new int[] { 1, 0, 2, 4 }, graph));
        }
        /// <summary>
        /// Build query plan.
        /// </summary>
        /// <param name="queryGraph">navigability info</param>
        /// <param name="optionalOuterJoinType">outer join type, null if not an outer join</param>
        /// <param name="typesPerStream">event types for each stream</param>
        /// <param name="streamJoinAnalysisResult">stream join analysis</param>
        /// <returns>query plan</returns>
        public static QueryPlanForgeDesc Build(
            EventType[] typesPerStream,
            QueryGraphForge queryGraph,
            OuterJoinType? optionalOuterJoinType,
            StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
            StatementRawInfo raw)
        {
            var uniqueIndexProps = streamJoinAnalysisResult.UniqueKeys;
            var tablesPerStream = streamJoinAnalysisResult.TablesPerStream;
            var additionalForgeable = new List<StmtClassForgeableFactory>();

            var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(
                queryGraph,
                typesPerStream,
                uniqueIndexProps);

            var execNodeSpecs = new QueryPlanNodeForge[2];
            var lookupPlans = new TableLookupPlanForge[2];

            // plan lookup from 1 to zero
            TableLookupPlanDesc plan1to0 = NStreamQueryPlanBuilder.CreateLookupPlan(
                queryGraph,
                1,
                0,
                streamJoinAnalysisResult.IsVirtualDW(0),
                indexSpecs[0],
                typesPerStream,
                tablesPerStream[0],
                raw,
                SerdeCompileTimeResolverNonHA.INSTANCE);
            lookupPlans[1] = plan1to0.Forge;
            additionalForgeable.AddAll(plan1to0.AdditionalForgeables);

            // plan lookup from zero to 1
            TableLookupPlanDesc plan0to1 = NStreamQueryPlanBuilder.CreateLookupPlan(
                queryGraph,
                0,
                1,
                streamJoinAnalysisResult.IsVirtualDW(1),
                indexSpecs[1],
                typesPerStream,
                tablesPerStream[1],
                raw,
                SerdeCompileTimeResolverNonHA.INSTANCE);
            lookupPlans[0] = plan0to1.Forge;
            additionalForgeable.AddAll(plan0to1.AdditionalForgeables);

            execNodeSpecs[0] = new TableLookupNodeForge(lookupPlans[0]);
            execNodeSpecs[1] = new TableLookupNodeForge(lookupPlans[1]);

            if (optionalOuterJoinType != null) {
                if ((optionalOuterJoinType.Equals(OuterJoinType.LEFT)) ||
                    (optionalOuterJoinType.Equals(OuterJoinType.FULL))) {
                    execNodeSpecs[0] = new TableOuterLookupNodeForge(lookupPlans[0]);
                }

                if ((optionalOuterJoinType.Equals(OuterJoinType.RIGHT)) ||
                    (optionalOuterJoinType.Equals(OuterJoinType.FULL))) {
                    execNodeSpecs[1] = new TableOuterLookupNodeForge(lookupPlans[1]);
                }
            }

            var forge = new QueryPlanForge(indexSpecs, execNodeSpecs);
            return new QueryPlanForgeDesc(forge, additionalForgeable);
        }
Example #8
0
        public static QueryPlanForgeDesc GetPlan(
            EventType[] typesPerStream,
            OuterJoinDesc[] outerJoinDescList,
            QueryGraphForge queryGraph,
            string[] streamNames,
            HistoricalViewableDesc historicalViewableDesc,
            DependencyGraph dependencyGraph,
            HistoricalStreamIndexListForge[] historicalStreamIndexLists,
            StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
            bool isQueryPlanLogging,
            StatementRawInfo statementRawInfo,
            StatementCompileTimeServices services)
        {
            string methodName = ".getPlan ";

            int numStreams = typesPerStream.Length;
            if (numStreams < 2) {
                throw new ArgumentException("Number of join stream types is less then 2");
            }

            if (outerJoinDescList.Length >= numStreams) {
                throw new ArgumentException("Too many outer join descriptors found");
            }

            if (numStreams == 2) {
                OuterJoinType? outerJoinType = null;
                if (outerJoinDescList.Length > 0) {
                    outerJoinType = outerJoinDescList[0].OuterJoinType;
                }

                QueryPlanForgeDesc queryPlan = TwoStreamQueryPlanBuilder.Build(
                    typesPerStream,
                    queryGraph,
                    outerJoinType,
                    streamJoinAnalysisResult,
                    statementRawInfo);
                RemoveUnidirectionalAndTable(queryPlan.Forge, streamJoinAnalysisResult);

                if (Log.IsDebugEnabled) {
                    Log.Debug(methodName + "2-Stream queryPlan=" + queryPlan);
                }

                return queryPlan;
            }

            bool hasPreferMergeJoin = HintEnum.PREFER_MERGE_JOIN.GetHint(statementRawInfo.Annotations) != null;
            bool hasForceNestedIter = HintEnum.FORCE_NESTED_ITER.GetHint(statementRawInfo.Annotations) != null;
            bool isAllInnerJoins = outerJoinDescList.Length == 0 ||
                                   OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList);

            if (isAllInnerJoins && !hasPreferMergeJoin) {
                QueryPlanForgeDesc queryPlan = NStreamQueryPlanBuilder.Build(
                    queryGraph,
                    typesPerStream,
                    historicalViewableDesc,
                    dependencyGraph,
                    historicalStreamIndexLists,
                    hasForceNestedIter,
                    streamJoinAnalysisResult.UniqueKeys,
                    streamJoinAnalysisResult.TablesPerStream,
                    streamJoinAnalysisResult,
                    statementRawInfo,
                    services.SerdeResolver);

                if (queryPlan != null) {
                    RemoveUnidirectionalAndTable(queryPlan.Forge, streamJoinAnalysisResult);

                    if (Log.IsDebugEnabled) {
                        Log.Debug(methodName + "N-Stream inner-join queryPlan=" + queryPlan);
                    }

                    return queryPlan;
                }

                if (isQueryPlanLogging && QUERY_PLAN_LOG.IsInfoEnabled) {
                    Log.Info("Switching to Outer-NStream algorithm for query plan");
                }
            }

            QueryPlanForgeDesc queryPlanX = NStreamOuterQueryPlanBuilder.Build(
                queryGraph,
                outerJoinDescList,
                streamNames,
                typesPerStream,
                historicalViewableDesc,
                dependencyGraph,
                historicalStreamIndexLists,
                streamJoinAnalysisResult.UniqueKeys,
                streamJoinAnalysisResult.TablesPerStream,
                streamJoinAnalysisResult,
                statementRawInfo,
                services);
            RemoveUnidirectionalAndTable(queryPlanX.Forge, streamJoinAnalysisResult);
            return queryPlanX;
        }
        private static LookupInstructionPlanDesc BuildLookupInstructions(
            int rootStreamNum,
            IDictionary<int, int[]> substreamsPerStream,
            bool[] requiredPerStream,
            string[] streamNames,
            QueryGraphForge queryGraph,
            QueryPlanIndexForge[] indexSpecs,
            EventType[] typesPerStream,
            OuterJoinDesc[] outerJoinDescList,
            bool[] isHistorical,
            HistoricalStreamIndexListForge[] historicalStreamIndexLists,
            TableMetaData[] tablesPerStream,
            StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
            StatementRawInfo statementRawInfo,
            StatementCompileTimeServices services)
        {
            var result = new List<LookupInstructionPlanForge>();
            var additionalForgeables = new List<StmtClassForgeableFactory>();
            
            foreach (var fromStream in substreamsPerStream.Keys) {
                var substreams = substreamsPerStream.Get(fromStream);

                // for streams with no substreams we don't need to look up
                if (substreams.Length == 0) {
                    continue;
                }

                var plans = new TableLookupPlanForge[substreams.Length];
                var historicalPlans = new HistoricalDataPlanNodeForge[substreams.Length];

                for (var i = 0; i < substreams.Length; i++) {
                    var toStream = substreams[i];

                    if (isHistorical[toStream]) {
                        // There may not be an outer-join descriptor, use if provided to build the associated expression
                        ExprNode outerJoinExpr = null;
                        if (outerJoinDescList.Length > 0) {
                            OuterJoinDesc outerJoinDesc;
                            if (toStream == 0) {
                                outerJoinDesc = outerJoinDescList[0];
                            }
                            else {
                                outerJoinDesc = outerJoinDescList[toStream - 1];
                            }

                            outerJoinExpr = outerJoinDesc.MakeExprNode(statementRawInfo, services);
                        }

                        if (historicalStreamIndexLists[toStream] == null) {
                            historicalStreamIndexLists[toStream] = new HistoricalStreamIndexListForge(
                                toStream,
                                typesPerStream,
                                queryGraph);
                        }

                        historicalStreamIndexLists[toStream].AddIndex(fromStream);
                        historicalPlans[i] = new HistoricalDataPlanNodeForge(
                            toStream,
                            rootStreamNum,
                            fromStream,
                            typesPerStream.Length,
                            outerJoinExpr == null ? null : outerJoinExpr.Forge);
                    }
                    else {
                        TableLookupPlanDesc planDesc = NStreamQueryPlanBuilder.CreateLookupPlan(
                            queryGraph,
                            fromStream,
                            toStream,
                            streamJoinAnalysisResult.IsVirtualDW(toStream),
                            indexSpecs[toStream],
                            typesPerStream,
                            tablesPerStream[toStream],
                            statementRawInfo,
                            SerdeCompileTimeResolverNonHA.INSTANCE);
                        plans[i] = planDesc.Forge;
                        additionalForgeables.AddAll(planDesc.AdditionalForgeables);
                    }
                }

                var fromStreamName = streamNames[fromStream];
                var instruction = new LookupInstructionPlanForge(
                    fromStream,
                    fromStreamName,
                    substreams,
                    plans,
                    historicalPlans,
                    requiredPerStream);
                result.Add(instruction);
            }

            return new LookupInstructionPlanDesc(result, additionalForgeables);
        }
        private static QueryPlanNodeForgeDesc BuildPlanNode(
            int numStreams,
            int streamNo,
            string[] streamNames,
            QueryGraphForge queryGraph,
            OuterInnerDirectionalGraph outerInnerGraph,
            OuterJoinDesc[] outerJoinDescList,
            InnerJoinGraph innerJoinGraph,
            QueryPlanIndexForge[] indexSpecs,
            EventType[] typesPerStream,
            bool[] isHistorical,
            DependencyGraph dependencyGraph,
            HistoricalStreamIndexListForge[] historicalStreamIndexLists,
            TableMetaData[] tablesPerStream,
            StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
            StatementRawInfo statementRawInfo,
            StatementCompileTimeServices services)
        {
            // For each stream build an array of substreams, considering required streams (inner joins) first
            // The order is relevant therefore preserving order via a LinkedHashMap.
            var substreamsPerStream = new LinkedHashMap<int, int[]>();
            var requiredPerStream = new bool[numStreams];
            var additionalForgeables = new List<StmtClassForgeableFactory>();

            // Recursive populating the required (outer) and optional (inner) relationships
            // of this stream and the substream
            ISet<int> completedStreams = new HashSet<int>();
            // keep track of tree path as only those stream events are always available to historical streams
            var streamCallStack = new Stack<int>();
            streamCallStack.Push(streamNo);

            // For all inner-joins, the algorithm is slightly different
            if (innerJoinGraph.IsAllInnerJoin) {
                requiredPerStream.Fill(true);
                RecursiveBuildInnerJoin(
                    streamNo,
                    streamCallStack,
                    queryGraph,
                    completedStreams,
                    substreamsPerStream,
                    dependencyGraph);

                // compute a best chain to see if all streams are handled and add the remaining
                var bestChain = NStreamQueryPlanBuilder.ComputeBestPath(streamNo, queryGraph, dependencyGraph);
                AddNotYetNavigated(streamNo, numStreams, substreamsPerStream, bestChain);
            }
            else {
                RecursiveBuild(
                    streamNo,
                    streamCallStack,
                    queryGraph,
                    outerInnerGraph,
                    innerJoinGraph,
                    completedStreams,
                    substreamsPerStream,
                    requiredPerStream,
                    dependencyGraph);
            }

            // verify the substreamsPerStream, all streams must exists and be linked
            VerifyJoinedPerStream(streamNo, substreamsPerStream);

            // build list of instructions for lookup
            LookupInstructionPlanDesc lookupDesc = BuildLookupInstructions(
                streamNo,
                substreamsPerStream,
                requiredPerStream,
                streamNames,
                queryGraph,
                indexSpecs,
                typesPerStream,
                outerJoinDescList,
                isHistorical,
                historicalStreamIndexLists,
                tablesPerStream,
                streamJoinAnalysisResult,
                statementRawInfo,
                services);
            var lookupInstructions = lookupDesc.Forges;
            additionalForgeables.AddAll(lookupDesc.AdditionalForgeables);

            // build historical index and lookup strategies
            foreach (var lookups in lookupInstructions) {
                foreach (var historical in lookups.HistoricalPlans) {
                    if (historical == null) {
                        continue;
                    }

                    JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum]
                        .GetStrategy(historical.LookupStreamNum, statementRawInfo, services.SerdeResolver);
                    historical.HistoricalIndexLookupStrategy = desc.LookupForge;
                    historical.PollResultIndexingStrategy = desc.IndexingForge;
                    additionalForgeables.AddAll(desc.AdditionalForgeables);
                }
            }

            // build strategy tree for putting the result back together
            var assemblyTopNodeFactory = AssemblyStrategyTreeBuilder.Build(
                streamNo,
                substreamsPerStream,
                requiredPerStream);
            var assemblyInstructionFactories =
                BaseAssemblyNodeFactory.GetDescendentNodesBottomUp(assemblyTopNodeFactory);

            var forge = new LookupInstructionQueryPlanNodeForge(
                streamNo,
                streamNames[streamNo],
                numStreams,
                requiredPerStream,
                lookupInstructions,
                assemblyInstructionFactories);
            
            return new QueryPlanNodeForgeDesc(forge, additionalForgeables);
        }