public void TestCreateStreamPlan()
{
QueryPlanIndex[] indexes = QueryPlanIndexBuilder.BuildIndexSpec(_queryGraph, _typesPerStream, new string[_queryGraph.NumStreams][][]);
for (int i = 0; i < indexes.Length; i++)
{
Log.Debug(".testCreateStreamPlan index " + i + " = " + indexes[i]);
}
QueryPlanNode plan = NStreamQueryPlanBuilder.CreateStreamPlan(
0, new int[] { 2, 4, 3, 1 }, _queryGraph,
indexes, _typesPerStream, new bool[5], null,
new TableMetadata[_queryGraph.NumStreams]);
Log.Debug(".testCreateStreamPlan plan=" + plan);
Assert.IsTrue(plan is NestedIterationNode);
var nested = (NestedIterationNode)plan;
var tableLookupSpec = (TableLookupNode)nested.ChildNodes[0];
// Check lookup strategy for first lookup
var lookupStrategySpec = (IndexedTableLookupPlanSingle)tableLookupSpec.LookupStrategySpec;
Assert.AreEqual("P01", ((QueryGraphValueEntryHashKeyedProp)lookupStrategySpec.HashKey).KeyProperty);
Assert.AreEqual(0, lookupStrategySpec.LookupStream);
Assert.AreEqual(2, lookupStrategySpec.IndexedStream);
Assert.NotNull(lookupStrategySpec.IndexNum);
// Check lookup strategy for last lookup
tableLookupSpec = (TableLookupNode)nested.ChildNodes[3];
var unkeyedSpecScan = (FullTableScanLookupPlan)tableLookupSpec.LookupStrategySpec;
Assert.AreEqual(1, unkeyedSpecScan.IndexedStream);
Assert.NotNull(unkeyedSpecScan.IndexNum);
}
/// <summary>
/// Build query plan.
/// </summary>
/// <param name="typesPerStream">event types for each stream</param>
/// <param name="queryGraph">navigability info</param>
/// <param name="optionalOuterJoinType">outer join type, null if not an outer join</param>
/// <param name="uniqueIndexProps">The unique index props.</param>
/// <param name="tablesPerStream">The tables per stream.</param>
/// <returns>
/// query plan
/// </returns>
public static QueryPlan Build(EventType[] typesPerStream, QueryGraph queryGraph, OuterJoinType?optionalOuterJoinType, string[][][] uniqueIndexProps, TableMetadata[] tablesPerStream)
{
var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(queryGraph, typesPerStream, uniqueIndexProps);
var execNodeSpecs = new QueryPlanNode[2];
var lookupPlans = new TableLookupPlan[2];
// plan lookup from 1 to zero
lookupPlans[1] = NStreamQueryPlanBuilder.CreateLookupPlan(queryGraph, 1, 0, indexSpecs[0], typesPerStream, tablesPerStream[0]);
// plan lookup from zero to 1
lookupPlans[0] = NStreamQueryPlanBuilder.CreateLookupPlan(queryGraph, 0, 1, indexSpecs[1], typesPerStream, tablesPerStream[1]);
execNodeSpecs[0] = new TableLookupNode(lookupPlans[0]);
execNodeSpecs[1] = new TableLookupNode(lookupPlans[1]);
if (optionalOuterJoinType != null)
{
if ((optionalOuterJoinType.Equals(OuterJoinType.LEFT)) ||
(optionalOuterJoinType.Equals(OuterJoinType.FULL)))
{
execNodeSpecs[0] = new TableOuterLookupNode(lookupPlans[0]);
}
if ((optionalOuterJoinType.Equals(OuterJoinType.RIGHT)) ||
(optionalOuterJoinType.Equals(OuterJoinType.FULL)))
{
execNodeSpecs[1] = new TableOuterLookupNode(lookupPlans[1]);
}
}
return(new QueryPlan(indexSpecs, execNodeSpecs));
}
private static QueryPlanNode BuildPlanNode(
int numStreams,
int streamNo,
string[] streamNames,
QueryGraph queryGraph,
OuterInnerDirectionalGraph outerInnerGraph,
OuterJoinDesc[] outerJoinDescList,
InnerJoinGraph innerJoinGraph,
QueryPlanIndex[] indexSpecs,
EventType[] typesPerStream,
bool[] ishistorical,
DependencyGraph dependencyGraph,
HistoricalStreamIndexList[] historicalStreamIndexLists,
ExprEvaluatorContext exprEvaluatorContext,
TableMetadata[] tablesPerStream)
{
// For each stream build an array of substreams, considering required streams (inner joins) first
// The order is relevant therefore preserving order via a LinkedHashMap.
var substreamsPerStream = new LinkedHashMap <int, int[]>();
var requiredPerStream = new bool[numStreams];
// Recursive populating the required (outer) and optional (inner) relationships
// of this stream and the substream
ISet <int> completedStreams = new HashSet <int>();
// keep track of tree path as only those stream events are always available to historical streams
var streamCallStack = new Stack <int>();
streamCallStack.Push(streamNo);
// For all inner-joins, the algorithm is slightly different
if (innerJoinGraph.IsAllInnerJoin)
{
requiredPerStream.Fill(true);
RecursiveBuildInnerJoin(streamNo, streamCallStack, queryGraph, completedStreams, substreamsPerStream, dependencyGraph);
// compute a best chain to see if all streams are handled and add the remaining
var bestChain = NStreamQueryPlanBuilder.ComputeBestPath(streamNo, queryGraph, dependencyGraph);
AddNotYetNavigated(streamNo, numStreams, substreamsPerStream, bestChain);
}
else
{
RecursiveBuild(streamNo, streamCallStack, queryGraph, outerInnerGraph, innerJoinGraph, completedStreams, substreamsPerStream, requiredPerStream, dependencyGraph);
}
// verify the substreamsPerStream, all streams must exists and be linked
VerifyJoinedPerStream(streamNo, substreamsPerStream);
// build list of instructions for lookup
var lookupInstructions = BuildLookupInstructions(streamNo, substreamsPerStream, requiredPerStream,
streamNames, queryGraph, indexSpecs, typesPerStream, outerJoinDescList, ishistorical, historicalStreamIndexLists, exprEvaluatorContext, tablesPerStream);
// build strategy tree for putting the result back together
var assemblyTopNodeFactory = AssemblyStrategyTreeBuilder.Build(streamNo, substreamsPerStream, requiredPerStream);
var assemblyInstructionFactories = BaseAssemblyNodeFactory.GetDescendentNodesBottomUp(assemblyTopNodeFactory);
return(new LookupInstructionQueryPlanNode(streamNo, streamNames[streamNo], numStreams, requiredPerStream,
lookupInstructions, assemblyInstructionFactories));
}
public void TestBuildDefaultNestingOrder()
{
int[] result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 0);
Assert.IsTrue(Collections.AreEqual(result, new int[] { 1, 2, 3 }));
result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 1);
Assert.IsTrue(Collections.AreEqual(result, new int[] { 0, 2, 3 }));
result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 2);
Assert.IsTrue(Collections.AreEqual(result, new int[] { 0, 1, 3 }));
result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 3);
Assert.IsTrue(Collections.AreEqual(result, new int[] { 0, 1, 2 }));
}
public void TestComputeBestPath()
{
NStreamQueryPlanBuilder.BestChainResult bestChain = NStreamQueryPlanBuilder.ComputeBestPath(
0, _queryGraph, _dependencyGraph);
Assert.AreEqual(3, bestChain.Depth);
Assert.IsTrue(Collections.AreEqual(bestChain.Chain, new int[] { 2, 4, 3, 1 }));
bestChain = NStreamQueryPlanBuilder.ComputeBestPath(3, _queryGraph, _dependencyGraph);
Assert.AreEqual(4, bestChain.Depth);
Assert.IsTrue(Collections.AreEqual(bestChain.Chain, new int[] { 4, 2, 0, 1 }));
// try a stream that is not connected in any way
_queryGraph = new QueryGraph(6, null, false);
bestChain = NStreamQueryPlanBuilder.ComputeBestPath(5, _queryGraph, _dependencyGraph);
Log.Debug(".testComputeBestPath bestChain=" + bestChain);
Assert.AreEqual(0, bestChain.Depth);
Assert.IsTrue(Collections.AreEqual(bestChain.Chain, new int[] { 0, 1, 2, 3, 4 }));
}
public void TestComputeNavigableDepth()
{
_queryGraph.AddStrictEquals(3, "P30", null, 2, "P20", null);
_queryGraph.AddStrictEquals(2, "P30", null, 1, "P20", null);
int depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(0, new int[] { 1, 2, 3, 4 }, _queryGraph);
Assert.AreEqual(4, depth);
depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(0, new int[] { 4, 2, 3, 1 }, _queryGraph);
Assert.AreEqual(0, depth);
depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(4, new int[] { 3, 2, 1, 0 }, _queryGraph);
Assert.AreEqual(4, depth);
depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(1, new int[] { 0, 3, 4, 2 }, _queryGraph);
Assert.AreEqual(1, depth);
}
public void TestIsDependencySatisfied()
{
var graph = new DependencyGraph(3, false);
graph.AddDependency(1, 0);
graph.AddDependency(2, 0);
Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(0, new int[] { 1, 2 }, graph));
Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 0, 2 }, graph));
Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(2, new int[] { 0, 1 }, graph));
graph = new DependencyGraph(5, false);
graph.AddDependency(4, 1);
graph.AddDependency(4, 2);
graph.AddDependency(2, 0);
Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(0, new int[] { 1, 2, 3, 4 }, graph));
Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 0, 2, 3, 4 }, graph));
Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 2, 0, 3, 4 }, graph));
Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 4, 0, 3, 2 }, graph));
Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(3, new int[] { 4, 0, 1, 2 }, graph));
Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(2, new int[] { 3, 1, 4, 0 }, graph));
Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(3, new int[] { 1, 0, 2, 4 }, graph));
}
/// <summary>
/// Build query plan using the filter.
/// </summary>
/// <param name="typesPerStream">event types for each stream</param>
/// <param name="outerJoinDescList">list of outer join criteria, or null if there are no outer joins</param>
/// <param name="queryGraph">relationships between streams based on filter expressions and outer-join on-criteria</param>
/// <param name="streamNames">names of streams</param>
/// <param name="historicalViewableDesc">The historical viewable desc.</param>
/// <param name="dependencyGraph">dependencies between historical streams</param>
/// <param name="historicalStreamIndexLists">index management, populated for the query plan</param>
/// <param name="streamJoinAnalysisResult">The stream join analysis result.</param>
/// <param name="isQueryPlanLogging">if set to <c>true</c> [is query plan logging].</param>
/// <param name="annotations">The annotations.</param>
/// <param name="exprEvaluatorContext">The expr evaluator context.</param>
/// <returns>
/// query plan
/// </returns>
/// <exception cref="System.ArgumentException">
/// Number of join stream types is less then 2
/// or
/// Too many outer join descriptors found
/// </exception>
/// <throws>ExprValidationException if the query plan fails</throws>
public static QueryPlan GetPlan(EventType[] typesPerStream,
OuterJoinDesc[] outerJoinDescList,
QueryGraph queryGraph,
string[] streamNames,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexList[] historicalStreamIndexLists,
StreamJoinAnalysisResult streamJoinAnalysisResult,
bool isQueryPlanLogging,
Attribute[] annotations,
ExprEvaluatorContext exprEvaluatorContext)
{
string methodName = ".getPlan ";
int numStreams = typesPerStream.Length;
if (numStreams < 2)
{
throw new ArgumentException("Number of join stream types is less then 2");
}
if (outerJoinDescList.Length >= numStreams)
{
throw new ArgumentException("Too many outer join descriptors found");
}
if (numStreams == 2)
{
OuterJoinType?outerJoinType = null;
if (outerJoinDescList.Length > 0)
{
outerJoinType = outerJoinDescList[0].OuterJoinType;
}
QueryPlan queryPlanX = TwoStreamQueryPlanBuilder.Build(typesPerStream, queryGraph, outerJoinType, streamJoinAnalysisResult.UniqueKeys, streamJoinAnalysisResult.TablesPerStream);
RemoveUnidirectionalAndTable(queryPlanX, streamJoinAnalysisResult);
if (log.IsDebugEnabled)
{
log.Debug(methodName + "2-Stream queryPlan=" + queryPlanX);
}
return(queryPlanX);
}
bool hasPreferMergeJoin = HintEnum.PREFER_MERGE_JOIN.GetHint(annotations) != null;
bool hasForceNestedIter = HintEnum.FORCE_NESTED_ITER.GetHint(annotations) != null;
bool isAllInnerJoins = outerJoinDescList.Length == 0 || OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList);
if (isAllInnerJoins && !hasPreferMergeJoin)
{
QueryPlan queryPlanX = NStreamQueryPlanBuilder.Build(queryGraph, typesPerStream,
historicalViewableDesc, dependencyGraph, historicalStreamIndexLists,
hasForceNestedIter, streamJoinAnalysisResult.UniqueKeys,
streamJoinAnalysisResult.TablesPerStream);
if (queryPlanX != null)
{
RemoveUnidirectionalAndTable(queryPlanX, streamJoinAnalysisResult);
if (log.IsDebugEnabled)
{
log.Debug(methodName + "Count-Stream inner-join queryPlan=" + queryPlanX);
}
return(queryPlanX);
}
if (isQueryPlanLogging && queryPlanLog.IsInfoEnabled)
{
log.Info("Switching to Outer-NStream algorithm for query plan");
}
}
QueryPlan queryPlan = NStreamOuterQueryPlanBuilder.Build(queryGraph, outerJoinDescList, streamNames, typesPerStream,
historicalViewableDesc, dependencyGraph, historicalStreamIndexLists, exprEvaluatorContext, streamJoinAnalysisResult.UniqueKeys,
streamJoinAnalysisResult.TablesPerStream);
RemoveUnidirectionalAndTable(queryPlan, streamJoinAnalysisResult);
return(queryPlan);
}
public void TestBuild()
{
QueryPlan plan = NStreamQueryPlanBuilder.Build(_queryGraph, _typesPerStream, new HistoricalViewableDesc(6), _dependencyGraph, null, false, new string[_queryGraph.NumStreams][][], new TableMetadata[_queryGraph.NumStreams]);
Log.Debug(".testBuild plan=" + plan);
}
private static IList <LookupInstructionPlan> BuildLookupInstructions(
int rootStreamNum,
LinkedHashMap <int, int[]> substreamsPerStream,
bool[] requiredPerStream,
string[] streamNames,
QueryGraph queryGraph,
QueryPlanIndex[] indexSpecs,
EventType[] typesPerStream,
OuterJoinDesc[] outerJoinDescList,
bool[] isHistorical,
HistoricalStreamIndexList[] historicalStreamIndexLists,
ExprEvaluatorContext exprEvaluatorContext,
TableMetadata[] tablesPerStream)
{
IList <LookupInstructionPlan> result = new List <LookupInstructionPlan>();
foreach (int fromStream in substreamsPerStream.Keys)
{
var substreams = substreamsPerStream.Get(fromStream);
// for streams with no substreams we don't need to look up
if (substreams.Length == 0)
{
continue;
}
var plans = new TableLookupPlan[substreams.Length];
var historicalPlans = new HistoricalDataPlanNode[substreams.Length];
for (var i = 0; i < substreams.Length; i++)
{
var toStream = substreams[i];
if (isHistorical[toStream])
{
// There may not be an outer-join descriptor, use if provided to build the associated expression
ExprNode outerJoinExpr = null;
if (outerJoinDescList.Length > 0)
{
OuterJoinDesc outerJoinDesc;
if (toStream == 0)
{
outerJoinDesc = outerJoinDescList[0];
}
else
{
outerJoinDesc = outerJoinDescList[toStream - 1];
}
outerJoinExpr = outerJoinDesc.MakeExprNode(exprEvaluatorContext);
}
if (historicalStreamIndexLists[toStream] == null)
{
historicalStreamIndexLists[toStream] = new HistoricalStreamIndexList(toStream, typesPerStream, queryGraph);
}
historicalStreamIndexLists[toStream].AddIndex(fromStream);
historicalPlans[i] = new HistoricalDataPlanNode(toStream, rootStreamNum, fromStream, typesPerStream.Length, outerJoinExpr);
}
else
{
plans[i] = NStreamQueryPlanBuilder.CreateLookupPlan(queryGraph, fromStream, toStream, indexSpecs[toStream], typesPerStream, tablesPerStream[toStream]);
}
}
var fromStreamName = streamNames[fromStream];
var instruction = new LookupInstructionPlan(fromStream, fromStreamName, substreams, plans, historicalPlans, requiredPerStream);
result.Add(instruction);
}
return(result);
}