public void TestCreateStreamPlan()
{
QueryPlanIndexForge[] indexes = QueryPlanIndexBuilder.BuildIndexSpec(queryGraph, typesPerStream, new string[queryGraph.NumStreams][][]);
for (int i = 0; i < indexes.Length; i++)
{
log.Debug(".testCreateStreamPlan index " + i + " = " + indexes[i]);
}
QueryPlanNodeForge plan = NStreamQueryPlanBuilder.CreateStreamPlan(0, new int[] { 2, 4, 3, 1 }, queryGraph, indexes, typesPerStream, new bool[5], null, new TableMetaData[queryGraph.NumStreams], new StreamJoinAnalysisResultCompileTime(5));
log.Debug(".testCreateStreamPlan plan=" + plan);
Assert.IsTrue(plan is NestedIterationNodeForge);
NestedIterationNodeForge nested = (NestedIterationNodeForge)plan;
TableLookupNodeForge tableLookupSpec = (TableLookupNodeForge)nested.ChildNodes[0];
// Check lookup strategy for first lookup
IndexedTableLookupPlanHashedOnlyForge lookupStrategySpec = (IndexedTableLookupPlanHashedOnlyForge)tableLookupSpec.LookupStrategySpec;
Assert.AreEqual("P01", ((ExprIdentNode)(lookupStrategySpec.HashKeys[0]).KeyExpr).ResolvedPropertyName);
Assert.AreEqual(0, lookupStrategySpec.LookupStream);
Assert.AreEqual(2, lookupStrategySpec.IndexedStream);
Assert.IsNotNull(lookupStrategySpec.IndexNum);
// Check lookup strategy for last lookup
tableLookupSpec = (TableLookupNodeForge)nested.ChildNodes[3];
FullTableScanLookupPlanForge unkeyedSpecScan = (FullTableScanLookupPlanForge)tableLookupSpec.LookupStrategySpec;
Assert.AreEqual(1, unkeyedSpecScan.IndexedStream);
Assert.IsNotNull(unkeyedSpecScan.IndexNum);
}
public void TestBuildDefaultNestingOrder()
{
int[] result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 0);
Assert.IsTrue(Arrays.AreEqual(result, new int[] { 1, 2, 3 }));
result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 1);
Assert.IsTrue(Arrays.AreEqual(result, new int[] { 0, 2, 3 }));
result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 2);
Assert.IsTrue(Arrays.AreEqual(result, new int[] { 0, 1, 3 }));
result = NStreamQueryPlanBuilder.BuildDefaultNestingOrder(4, 3);
Assert.IsTrue(Arrays.AreEqual(result, new int[] { 0, 1, 2 }));
}
public void TestBuild()
{
var plan = NStreamQueryPlanBuilder.Build(
queryGraph,
typesPerStream,
new HistoricalViewableDesc(6),
dependencyGraph,
null,
false,
new string[queryGraph.NumStreams][][],
new TableMetaData[queryGraph.NumStreams],
new StreamJoinAnalysisResultCompileTime(5),
null,
SerdeCompileTimeResolverNonHA.INSTANCE);
log.Debug(".testBuild plan=" + plan);
}
public void TestComputeBestPath()
{
NStreamQueryPlanBuilder.BestChainResult bestChain = NStreamQueryPlanBuilder.ComputeBestPath(0, queryGraph, dependencyGraph);
Assert.AreEqual(3, bestChain.Depth);
Assert.IsTrue(Arrays.AreEqual(bestChain.Chain, new int[] { 2, 4, 3, 1 }));
bestChain = NStreamQueryPlanBuilder.ComputeBestPath(3, queryGraph, dependencyGraph);
Assert.AreEqual(4, bestChain.Depth);
Assert.IsTrue(Arrays.AreEqual(bestChain.Chain, new int[] { 4, 2, 0, 1 }));
// try a stream that is not connected in any way
queryGraph = new QueryGraphForge(6, null, false);
bestChain = NStreamQueryPlanBuilder.ComputeBestPath(5, queryGraph, dependencyGraph);
log.Debug(".testComputeBestPath bestChain=" + bestChain);
Assert.AreEqual(0, bestChain.Depth);
Assert.IsTrue(Arrays.AreEqual(bestChain.Chain, new int[] { 0, 1, 2, 3, 4 }));
}
public void TestComputeNavigableDepth()
{
ExprIdentNode fake = supportExprNodeFactory.MakeIdentNode("TheString", "s0");
queryGraph.AddStrictEquals(3, "P30", fake, 2, "P20", fake);
queryGraph.AddStrictEquals(2, "P30", fake, 1, "P20", fake);
int depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(0, new int[] { 1, 2, 3, 4 }, queryGraph);
Assert.AreEqual(4, depth);
depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(0, new int[] { 4, 2, 3, 1 }, queryGraph);
Assert.AreEqual(0, depth);
depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(4, new int[] { 3, 2, 1, 0 }, queryGraph);
Assert.AreEqual(4, depth);
depth = NStreamQueryPlanBuilder.ComputeNavigableDepth(1, new int[] { 0, 3, 4, 2 }, queryGraph);
Assert.AreEqual(1, depth);
}
public void TestIsDependencySatisfied()
{
DependencyGraph graph = new DependencyGraph(3, false);
graph.AddDependency(1, 0);
graph.AddDependency(2, 0);
Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(0, new int[] { 1, 2 }, graph));
Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 0, 2 }, graph));
Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(2, new int[] { 0, 1 }, graph));
graph = new DependencyGraph(5, false);
graph.AddDependency(4, 1);
graph.AddDependency(4, 2);
graph.AddDependency(2, 0);
Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(0, new int[] { 1, 2, 3, 4 }, graph));
Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 0, 2, 3, 4 }, graph));
Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 2, 0, 3, 4 }, graph));
Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(1, new int[] { 4, 0, 3, 2 }, graph));
Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(3, new int[] { 4, 0, 1, 2 }, graph));
Assert.IsFalse(NStreamQueryPlanBuilder.IsDependencySatisfied(2, new int[] { 3, 1, 4, 0 }, graph));
Assert.IsTrue(NStreamQueryPlanBuilder.IsDependencySatisfied(3, new int[] { 1, 0, 2, 4 }, graph));
}
/// <summary>
/// Build query plan.
/// </summary>
/// <param name="queryGraph">navigability info</param>
/// <param name="optionalOuterJoinType">outer join type, null if not an outer join</param>
/// <param name="typesPerStream">event types for each stream</param>
/// <param name="streamJoinAnalysisResult">stream join analysis</param>
/// <returns>query plan</returns>
public static QueryPlanForgeDesc Build(
EventType[] typesPerStream,
QueryGraphForge queryGraph,
OuterJoinType? optionalOuterJoinType,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo raw)
{
var uniqueIndexProps = streamJoinAnalysisResult.UniqueKeys;
var tablesPerStream = streamJoinAnalysisResult.TablesPerStream;
var additionalForgeable = new List<StmtClassForgeableFactory>();
var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(
queryGraph,
typesPerStream,
uniqueIndexProps);
var execNodeSpecs = new QueryPlanNodeForge[2];
var lookupPlans = new TableLookupPlanForge[2];
// plan lookup from 1 to zero
TableLookupPlanDesc plan1to0 = NStreamQueryPlanBuilder.CreateLookupPlan(
queryGraph,
1,
0,
streamJoinAnalysisResult.IsVirtualDW(0),
indexSpecs[0],
typesPerStream,
tablesPerStream[0],
raw,
SerdeCompileTimeResolverNonHA.INSTANCE);
lookupPlans[1] = plan1to0.Forge;
additionalForgeable.AddAll(plan1to0.AdditionalForgeables);
// plan lookup from zero to 1
TableLookupPlanDesc plan0to1 = NStreamQueryPlanBuilder.CreateLookupPlan(
queryGraph,
0,
1,
streamJoinAnalysisResult.IsVirtualDW(1),
indexSpecs[1],
typesPerStream,
tablesPerStream[1],
raw,
SerdeCompileTimeResolverNonHA.INSTANCE);
lookupPlans[0] = plan0to1.Forge;
additionalForgeable.AddAll(plan0to1.AdditionalForgeables);
execNodeSpecs[0] = new TableLookupNodeForge(lookupPlans[0]);
execNodeSpecs[1] = new TableLookupNodeForge(lookupPlans[1]);
if (optionalOuterJoinType != null) {
if ((optionalOuterJoinType.Equals(OuterJoinType.LEFT)) ||
(optionalOuterJoinType.Equals(OuterJoinType.FULL))) {
execNodeSpecs[0] = new TableOuterLookupNodeForge(lookupPlans[0]);
}
if ((optionalOuterJoinType.Equals(OuterJoinType.RIGHT)) ||
(optionalOuterJoinType.Equals(OuterJoinType.FULL))) {
execNodeSpecs[1] = new TableOuterLookupNodeForge(lookupPlans[1]);
}
}
var forge = new QueryPlanForge(indexSpecs, execNodeSpecs);
return new QueryPlanForgeDesc(forge, additionalForgeable);
}
public static QueryPlanForgeDesc GetPlan(
EventType[] typesPerStream,
OuterJoinDesc[] outerJoinDescList,
QueryGraphForge queryGraph,
string[] streamNames,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
bool isQueryPlanLogging,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
string methodName = ".getPlan ";
int numStreams = typesPerStream.Length;
if (numStreams < 2) {
throw new ArgumentException("Number of join stream types is less then 2");
}
if (outerJoinDescList.Length >= numStreams) {
throw new ArgumentException("Too many outer join descriptors found");
}
if (numStreams == 2) {
OuterJoinType? outerJoinType = null;
if (outerJoinDescList.Length > 0) {
outerJoinType = outerJoinDescList[0].OuterJoinType;
}
QueryPlanForgeDesc queryPlan = TwoStreamQueryPlanBuilder.Build(
typesPerStream,
queryGraph,
outerJoinType,
streamJoinAnalysisResult,
statementRawInfo);
RemoveUnidirectionalAndTable(queryPlan.Forge, streamJoinAnalysisResult);
if (Log.IsDebugEnabled) {
Log.Debug(methodName + "2-Stream queryPlan=" + queryPlan);
}
return queryPlan;
}
bool hasPreferMergeJoin = HintEnum.PREFER_MERGE_JOIN.GetHint(statementRawInfo.Annotations) != null;
bool hasForceNestedIter = HintEnum.FORCE_NESTED_ITER.GetHint(statementRawInfo.Annotations) != null;
bool isAllInnerJoins = outerJoinDescList.Length == 0 ||
OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescList);
if (isAllInnerJoins && !hasPreferMergeJoin) {
QueryPlanForgeDesc queryPlan = NStreamQueryPlanBuilder.Build(
queryGraph,
typesPerStream,
historicalViewableDesc,
dependencyGraph,
historicalStreamIndexLists,
hasForceNestedIter,
streamJoinAnalysisResult.UniqueKeys,
streamJoinAnalysisResult.TablesPerStream,
streamJoinAnalysisResult,
statementRawInfo,
services.SerdeResolver);
if (queryPlan != null) {
RemoveUnidirectionalAndTable(queryPlan.Forge, streamJoinAnalysisResult);
if (Log.IsDebugEnabled) {
Log.Debug(methodName + "N-Stream inner-join queryPlan=" + queryPlan);
}
return queryPlan;
}
if (isQueryPlanLogging && QUERY_PLAN_LOG.IsInfoEnabled) {
Log.Info("Switching to Outer-NStream algorithm for query plan");
}
}
QueryPlanForgeDesc queryPlanX = NStreamOuterQueryPlanBuilder.Build(
queryGraph,
outerJoinDescList,
streamNames,
typesPerStream,
historicalViewableDesc,
dependencyGraph,
historicalStreamIndexLists,
streamJoinAnalysisResult.UniqueKeys,
streamJoinAnalysisResult.TablesPerStream,
streamJoinAnalysisResult,
statementRawInfo,
services);
RemoveUnidirectionalAndTable(queryPlanX.Forge, streamJoinAnalysisResult);
return queryPlanX;
}
private static LookupInstructionPlanDesc BuildLookupInstructions(
int rootStreamNum,
IDictionary<int, int[]> substreamsPerStream,
bool[] requiredPerStream,
string[] streamNames,
QueryGraphForge queryGraph,
QueryPlanIndexForge[] indexSpecs,
EventType[] typesPerStream,
OuterJoinDesc[] outerJoinDescList,
bool[] isHistorical,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
var result = new List<LookupInstructionPlanForge>();
var additionalForgeables = new List<StmtClassForgeableFactory>();
foreach (var fromStream in substreamsPerStream.Keys) {
var substreams = substreamsPerStream.Get(fromStream);
// for streams with no substreams we don't need to look up
if (substreams.Length == 0) {
continue;
}
var plans = new TableLookupPlanForge[substreams.Length];
var historicalPlans = new HistoricalDataPlanNodeForge[substreams.Length];
for (var i = 0; i < substreams.Length; i++) {
var toStream = substreams[i];
if (isHistorical[toStream]) {
// There may not be an outer-join descriptor, use if provided to build the associated expression
ExprNode outerJoinExpr = null;
if (outerJoinDescList.Length > 0) {
OuterJoinDesc outerJoinDesc;
if (toStream == 0) {
outerJoinDesc = outerJoinDescList[0];
}
else {
outerJoinDesc = outerJoinDescList[toStream - 1];
}
outerJoinExpr = outerJoinDesc.MakeExprNode(statementRawInfo, services);
}
if (historicalStreamIndexLists[toStream] == null) {
historicalStreamIndexLists[toStream] = new HistoricalStreamIndexListForge(
toStream,
typesPerStream,
queryGraph);
}
historicalStreamIndexLists[toStream].AddIndex(fromStream);
historicalPlans[i] = new HistoricalDataPlanNodeForge(
toStream,
rootStreamNum,
fromStream,
typesPerStream.Length,
outerJoinExpr == null ? null : outerJoinExpr.Forge);
}
else {
TableLookupPlanDesc planDesc = NStreamQueryPlanBuilder.CreateLookupPlan(
queryGraph,
fromStream,
toStream,
streamJoinAnalysisResult.IsVirtualDW(toStream),
indexSpecs[toStream],
typesPerStream,
tablesPerStream[toStream],
statementRawInfo,
SerdeCompileTimeResolverNonHA.INSTANCE);
plans[i] = planDesc.Forge;
additionalForgeables.AddAll(planDesc.AdditionalForgeables);
}
}
var fromStreamName = streamNames[fromStream];
var instruction = new LookupInstructionPlanForge(
fromStream,
fromStreamName,
substreams,
plans,
historicalPlans,
requiredPerStream);
result.Add(instruction);
}
return new LookupInstructionPlanDesc(result, additionalForgeables);
}
private static QueryPlanNodeForgeDesc BuildPlanNode(
int numStreams,
int streamNo,
string[] streamNames,
QueryGraphForge queryGraph,
OuterInnerDirectionalGraph outerInnerGraph,
OuterJoinDesc[] outerJoinDescList,
InnerJoinGraph innerJoinGraph,
QueryPlanIndexForge[] indexSpecs,
EventType[] typesPerStream,
bool[] isHistorical,
DependencyGraph dependencyGraph,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
// For each stream build an array of substreams, considering required streams (inner joins) first
// The order is relevant therefore preserving order via a LinkedHashMap.
var substreamsPerStream = new LinkedHashMap<int, int[]>();
var requiredPerStream = new bool[numStreams];
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Recursive populating the required (outer) and optional (inner) relationships
// of this stream and the substream
ISet<int> completedStreams = new HashSet<int>();
// keep track of tree path as only those stream events are always available to historical streams
var streamCallStack = new Stack<int>();
streamCallStack.Push(streamNo);
// For all inner-joins, the algorithm is slightly different
if (innerJoinGraph.IsAllInnerJoin) {
requiredPerStream.Fill(true);
RecursiveBuildInnerJoin(
streamNo,
streamCallStack,
queryGraph,
completedStreams,
substreamsPerStream,
dependencyGraph);
// compute a best chain to see if all streams are handled and add the remaining
var bestChain = NStreamQueryPlanBuilder.ComputeBestPath(streamNo, queryGraph, dependencyGraph);
AddNotYetNavigated(streamNo, numStreams, substreamsPerStream, bestChain);
}
else {
RecursiveBuild(
streamNo,
streamCallStack,
queryGraph,
outerInnerGraph,
innerJoinGraph,
completedStreams,
substreamsPerStream,
requiredPerStream,
dependencyGraph);
}
// verify the substreamsPerStream, all streams must exists and be linked
VerifyJoinedPerStream(streamNo, substreamsPerStream);
// build list of instructions for lookup
LookupInstructionPlanDesc lookupDesc = BuildLookupInstructions(
streamNo,
substreamsPerStream,
requiredPerStream,
streamNames,
queryGraph,
indexSpecs,
typesPerStream,
outerJoinDescList,
isHistorical,
historicalStreamIndexLists,
tablesPerStream,
streamJoinAnalysisResult,
statementRawInfo,
services);
var lookupInstructions = lookupDesc.Forges;
additionalForgeables.AddAll(lookupDesc.AdditionalForgeables);
// build historical index and lookup strategies
foreach (var lookups in lookupInstructions) {
foreach (var historical in lookups.HistoricalPlans) {
if (historical == null) {
continue;
}
JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum]
.GetStrategy(historical.LookupStreamNum, statementRawInfo, services.SerdeResolver);
historical.HistoricalIndexLookupStrategy = desc.LookupForge;
historical.PollResultIndexingStrategy = desc.IndexingForge;
additionalForgeables.AddAll(desc.AdditionalForgeables);
}
}
// build strategy tree for putting the result back together
var assemblyTopNodeFactory = AssemblyStrategyTreeBuilder.Build(
streamNo,
substreamsPerStream,
requiredPerStream);
var assemblyInstructionFactories =
BaseAssemblyNodeFactory.GetDescendentNodesBottomUp(assemblyTopNodeFactory);
var forge = new LookupInstructionQueryPlanNodeForge(
streamNo,
streamNames[streamNo],
numStreams,
requiredPerStream,
lookupInstructions,
assemblyInstructionFactories);
return new QueryPlanNodeForgeDesc(forge, additionalForgeables);
}