public void TestRecursiveBuild()
{
var streamNum = 2;
var queryGraph = new QueryGraph(6, null, false);
var outerInnerGraph = new OuterInnerDirectionalGraph(6);
var completedStreams = new HashSet <int>();
var substreamsPerStream = new LinkedHashMap <int, int[]>();
var requiredPerStream = new bool[6];
outerInnerGraph.Add(3, 2).Add(2, 1).Add(4, 3).Add(1, 0).Add(3, 5);
queryGraph.AddStrictEquals(2, "", null, 3, "", null);
queryGraph.AddStrictEquals(3, "", null, 4, "", null);
queryGraph.AddStrictEquals(3, "", null, 5, "", null);
queryGraph.AddStrictEquals(2, "", null, 1, "", null);
queryGraph.AddStrictEquals(1, "", null, 0, "", null);
ICollection <InterchangeablePair <int, int> > innerJoins = new HashSet <InterchangeablePair <int, int> >();
var innerJoinGraph = new InnerJoinGraph(6, innerJoins);
var streamStack = new Stack <int>();
NStreamOuterQueryPlanBuilder.RecursiveBuild(streamNum, streamStack, queryGraph, outerInnerGraph, innerJoinGraph, completedStreams,
substreamsPerStream, requiredPerStream, new DependencyGraph(6, false));
Assert.AreEqual(6, substreamsPerStream.Count);
EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream[2], new int[] { 3, 1 });
EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(3), new int[] { 4, 5 });
EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream[1], new int[] { 0 });
EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(4), new int[] {});
EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(5), new int[] {});
EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream[0], new int[] {});
NStreamOuterQueryPlanBuilder.VerifyJoinedPerStream(2, substreamsPerStream);
EPAssertionUtil.AssertEqualsExactOrder(requiredPerStream, new bool[] { false, false, false, true, true, false }
);
}
private static bool RecursiveHasInnerJoin(
int toStream,
OuterInnerDirectionalGraph outerInnerGraph,
InnerJoinGraph innerJoinGraph,
ICollection <int> completedStreams)
{
// Check if the to-stream is in any of the inner joins
var hasInnerJoin = innerJoinGraph.HasInnerJoin(toStream);
if (hasInnerJoin)
{
return(true);
}
var innerToToStream = outerInnerGraph.GetInner(toStream);
if (innerToToStream != null)
{
foreach (int nextStream in innerToToStream)
{
if (completedStreams.Contains(nextStream))
{
continue;
}
var notConsider = new HashSet <int>(completedStreams);
notConsider.Add(toStream);
var result = RecursiveHasInnerJoin(nextStream, outerInnerGraph, innerJoinGraph, notConsider);
if (result)
{
return(true);
}
}
}
var outerToToStream = outerInnerGraph.GetOuter(toStream);
if (outerToToStream != null)
{
foreach (int nextStream in outerToToStream)
{
if (completedStreams.Contains(nextStream))
{
continue;
}
var notConsider = new HashSet <int>(completedStreams);
notConsider.Add(toStream);
var result = RecursiveHasInnerJoin(nextStream, outerInnerGraph, innerJoinGraph, notConsider);
if (result)
{
return(true);
}
}
}
return(false);
}
private void AssertOuters(int[][] outersPerStream, OuterInnerDirectionalGraph graph)
{
for (var i = 0; i < outersPerStream.Length; i++)
{
EPAssertionUtil.AssertEqualsAnyOrder(outersPerStream[i], graph.GetOuter(i));
}
}
private static ISet <int> GetInnerStreams(
int fromStream,
IEnumerable <int> toStreams,
OuterInnerDirectionalGraph outerInnerGraph,
InnerJoinGraph innerJoinGraph,
ISet <int> completedStreams)
{
ISet <int> innerStreams = new HashSet <int>();
foreach (int toStream in toStreams)
{
if (outerInnerGraph.IsInner(fromStream, toStream))
{
// if the to-stream, recursively, has an inner join itself, it becomes a required stream and not optional
var hasInnerJoin = false;
if (!innerJoinGraph.IsEmpty())
{
var doNotUseStreams = new HashSet <int>(completedStreams);
completedStreams.Add(fromStream);
hasInnerJoin = RecursiveHasInnerJoin(toStream, outerInnerGraph, innerJoinGraph, doNotUseStreams);
}
if (!hasInnerJoin)
{
innerStreams.Add(toStream);
}
}
}
return(innerStreams);
}
private static QueryPlanNode BuildPlanNode(
int numStreams,
int streamNo,
string[] streamNames,
QueryGraph queryGraph,
OuterInnerDirectionalGraph outerInnerGraph,
OuterJoinDesc[] outerJoinDescList,
InnerJoinGraph innerJoinGraph,
QueryPlanIndex[] indexSpecs,
EventType[] typesPerStream,
bool[] ishistorical,
DependencyGraph dependencyGraph,
HistoricalStreamIndexList[] historicalStreamIndexLists,
ExprEvaluatorContext exprEvaluatorContext,
TableMetadata[] tablesPerStream)
{
// For each stream build an array of substreams, considering required streams (inner joins) first
// The order is relevant therefore preserving order via a LinkedHashMap.
var substreamsPerStream = new LinkedHashMap <int, int[]>();
var requiredPerStream = new bool[numStreams];
// Recursive populating the required (outer) and optional (inner) relationships
// of this stream and the substream
ISet <int> completedStreams = new HashSet <int>();
// keep track of tree path as only those stream events are always available to historical streams
var streamCallStack = new Stack <int>();
streamCallStack.Push(streamNo);
// For all inner-joins, the algorithm is slightly different
if (innerJoinGraph.IsAllInnerJoin)
{
requiredPerStream.Fill(true);
RecursiveBuildInnerJoin(streamNo, streamCallStack, queryGraph, completedStreams, substreamsPerStream, dependencyGraph);
// compute a best chain to see if all streams are handled and add the remaining
var bestChain = NStreamQueryPlanBuilder.ComputeBestPath(streamNo, queryGraph, dependencyGraph);
AddNotYetNavigated(streamNo, numStreams, substreamsPerStream, bestChain);
}
else
{
RecursiveBuild(streamNo, streamCallStack, queryGraph, outerInnerGraph, innerJoinGraph, completedStreams, substreamsPerStream, requiredPerStream, dependencyGraph);
}
// verify the substreamsPerStream, all streams must exists and be linked
VerifyJoinedPerStream(streamNo, substreamsPerStream);
// build list of instructions for lookup
var lookupInstructions = BuildLookupInstructions(streamNo, substreamsPerStream, requiredPerStream,
streamNames, queryGraph, indexSpecs, typesPerStream, outerJoinDescList, ishistorical, historicalStreamIndexLists, exprEvaluatorContext, tablesPerStream);
// build strategy tree for putting the result back together
var assemblyTopNodeFactory = AssemblyStrategyTreeBuilder.Build(streamNo, substreamsPerStream, requiredPerStream);
var assemblyInstructionFactories = BaseAssemblyNodeFactory.GetDescendentNodesBottomUp(assemblyTopNodeFactory);
return(new LookupInstructionQueryPlanNode(streamNo, streamNames[streamNo], numStreams, requiredPerStream,
lookupInstructions, assemblyInstructionFactories));
}
public void SetUp()
{
graph = new OuterInnerDirectionalGraph(4);
}
/// <summary>
/// Builds a graph of outer joins given the outer join information from the statement.
/// Eliminates right and left joins and full joins by placing the information in a graph object.
/// </summary>
/// <param name="numStreams">is the number of streams</param>
/// <param name="outerJoinDescList">list of outer join stream numbers and property names</param>
/// <returns>graph object</returns>
internal static OuterInnerDirectionalGraph GraphOuterJoins(int numStreams, OuterJoinDesc[] outerJoinDescList)
{
if ((outerJoinDescList.Length + 1) != numStreams)
{
throw new ArgumentException("Number of outer join descriptors and number of streams not matching up");
}
var graph = new OuterInnerDirectionalGraph(numStreams);
for (var i = 0; i < outerJoinDescList.Length; i++)
{
var desc = outerJoinDescList[i];
var streamMax = i + 1; // the outer join must references streams less then streamMax
// Check outer join on-expression, if provided
int streamOne;
int streamTwo;
int lowerStream;
int higherStream;
if (desc.OptLeftNode != null)
{
streamOne = desc.OptLeftNode.StreamId;
streamTwo = desc.OptRightNode.StreamId;
if ((streamOne > streamMax) || (streamTwo > streamMax) ||
(streamOne == streamTwo))
{
throw new ArgumentException("Outer join descriptors reference future streams, or same streams");
}
// Determine who is the first stream in the streams listed
lowerStream = streamOne;
higherStream = streamTwo;
if (streamOne > streamTwo)
{
lowerStream = streamTwo;
higherStream = streamOne;
}
}
else
{
streamOne = i;
streamTwo = i + 1;
lowerStream = i;
higherStream = i + 1;
graph.AddUnqualifiedNavigable(streamOne, streamTwo);
}
// Add to graph
if (desc.OuterJoinType == OuterJoinType.FULL)
{
graph.Add(streamOne, streamTwo);
graph.Add(streamTwo, streamOne);
}
else if (desc.OuterJoinType == OuterJoinType.LEFT)
{
graph.Add(lowerStream, higherStream);
}
else if (desc.OuterJoinType == OuterJoinType.RIGHT)
{
graph.Add(higherStream, lowerStream);
}
else if (desc.OuterJoinType == OuterJoinType.INNER)
{
// no navigability for inner joins
}
else
{
throw new ArgumentException("Outer join descriptors join type not handled, type=" + desc.OuterJoinType);
}
}
return(graph);
}
// which streams are to this table an outer stream
private static ISet <int> GetOuterStreams(int fromStream, IEnumerable <int> toStreams, OuterInnerDirectionalGraph outerInnerGraph)
{
ISet <int> outerStreams = new HashSet <int>();
foreach (int toStream in toStreams)
{
if (outerInnerGraph.IsOuter(toStream, fromStream))
{
outerStreams.Add(toStream);
}
}
return(outerStreams);
}
/// <summary>
/// Build a query plan based on the stream property relationships indicated in queryGraph.
/// </summary>
/// <param name="queryGraph">navigation info between streams</param>
/// <param name="outerJoinDescList">descriptors for all outer joins</param>
/// <param name="streamNames">stream names</param>
/// <param name="typesPerStream">event types for each stream</param>
/// <param name="historicalViewableDesc">The historical viewable desc.</param>
/// <param name="dependencyGraph">dependencies between historical streams</param>
/// <param name="historicalStreamIndexLists">index management, populated for the query plan</param>
/// <param name="exprEvaluatorContext">context for expression evalauation</param>
/// <param name="indexedStreamsUniqueProps">The indexed streams unique props.</param>
/// <param name="tablesPerStream">The tables per stream.</param>
/// <returns>
/// query plan
/// </returns>
/// <throws>ExprValidationException if the query planning failed</throws>
internal static QueryPlan Build(
QueryGraph queryGraph,
OuterJoinDesc[] outerJoinDescList,
string[] streamNames,
EventType[] typesPerStream,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexList[] historicalStreamIndexLists,
ExprEvaluatorContext exprEvaluatorContext,
string[][][] indexedStreamsUniqueProps,
TableMetadata[] tablesPerStream)
{
if (Log.IsDebugEnabled)
{
Log.Debug(".build queryGraph=" + queryGraph);
}
var numStreams = queryGraph.NumStreams;
var planNodeSpecs = new QueryPlanNode[numStreams];
// Build index specifications
var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(queryGraph, typesPerStream, indexedStreamsUniqueProps);
if (Log.IsDebugEnabled)
{
Log.Debug(".build Index build completed, indexes=" + QueryPlanIndex.Print(indexSpecs));
}
// any historical streams don't get indexes, the lookup strategy accounts for cached indexes
if (historicalViewableDesc.HasHistorical)
{
for (var i = 0; i < historicalViewableDesc.Historical.Length; i++)
{
if (historicalViewableDesc.Historical[i])
{
indexSpecs[i] = null;
}
}
}
// Build graph of the outer join to inner table relationships.
// Build a map of inner joins.
OuterInnerDirectionalGraph outerInnerGraph;
InnerJoinGraph innerJoinGraph;
if (outerJoinDescList.Length > 0)
{
outerInnerGraph = GraphOuterJoins(numStreams, outerJoinDescList);
innerJoinGraph = InnerJoinGraph.GraphInnerJoins(numStreams, outerJoinDescList);
}
else
{
// all inner joins - thereby no (or empty) directional graph
outerInnerGraph = new OuterInnerDirectionalGraph(numStreams);
innerJoinGraph = new InnerJoinGraph(numStreams, true);
}
if (Log.IsDebugEnabled)
{
Log.Debug(".build directional graph=" + outerInnerGraph.Print());
}
// For each stream determine the query plan
for (var streamNo = 0; streamNo < numStreams; streamNo++)
{
// no plan for historical streams that are dependent upon other streams
if ((historicalViewableDesc.Historical[streamNo]) && (dependencyGraph.HasDependency(streamNo)))
{
planNodeSpecs[streamNo] = new QueryPlanNodeNoOp();
continue;
}
var queryPlanNode = BuildPlanNode(numStreams, streamNo, streamNames, queryGraph, outerInnerGraph, outerJoinDescList, innerJoinGraph, indexSpecs, typesPerStream, historicalViewableDesc.Historical, dependencyGraph, historicalStreamIndexLists, exprEvaluatorContext, tablesPerStream);
if (Log.IsDebugEnabled)
{
Log.Debug(".build spec for stream '" + streamNames[streamNo] +
"' number " + streamNo + " is " + queryPlanNode);
}
planNodeSpecs[streamNo] = queryPlanNode;
}
var queryPlan = new QueryPlan(indexSpecs, planNodeSpecs);
if (Log.IsDebugEnabled)
{
Log.Debug(".build query plan=" + queryPlan);
}
return(queryPlan);
}
/// <summary>
/// Recusivly builds a substream-per-stream ordered tree graph using the
/// join information supplied for outer joins and from the query graph (where clause).
/// <para />Required streams are considered first and their lookup is placed first in the list
/// to gain performance.
/// </summary>
/// <param name="streamNum">is the root stream number that supplies the incoming event to build the tree for</param>
/// <param name="streamCallStack">the query plan call stack of streams available via cursor</param>
/// <param name="queryGraph">contains where-clause stream relationship info</param>
/// <param name="outerInnerGraph">contains the outer join stream relationship info</param>
/// <param name="innerJoinGraph">The inner join graph.</param>
/// <param name="completedStreams">is a temporary holder for streams already considered</param>
/// <param name="substreamsPerStream">is the ordered, tree-like structure to be filled</param>
/// <param name="requiredPerStream">indicates which streams are required and which are optional</param>
/// <param name="dependencyGraph">dependencies between historical streams</param>
/// <exception cref="ExprValidationException">Historical stream + streamNum + parameter dependency originating in stream + dependentStream + cannot or may not be satisfied by the join</exception>
/// <throws>ExprValidationException if the query planning failed</throws>
internal static void RecursiveBuild(
int streamNum,
Stack <int> streamCallStack,
QueryGraph queryGraph,
OuterInnerDirectionalGraph outerInnerGraph,
InnerJoinGraph innerJoinGraph,
ISet <int> completedStreams,
IDictionary <int, int[]> substreamsPerStream,
bool[] requiredPerStream,
DependencyGraph dependencyGraph)
{
// add this stream to the set of completed streams
completedStreams.Add(streamNum);
// check if the dependencies have been satisfied
if (dependencyGraph.HasDependency(streamNum))
{
var dependencies = dependencyGraph.GetDependenciesForStream(streamNum);
foreach (var dependentStream in dependencies)
{
if (!streamCallStack.Contains(dependentStream))
{
throw new ExprValidationException("Historical stream " + streamNum + " parameter dependency originating in stream " + dependentStream + " cannot or may not be satisfied by the join");
}
}
}
// Determine the streams we can navigate to from this stream
var navigableStreams = queryGraph.GetNavigableStreams(streamNum);
// add unqualified navigable streams (since on-expressions in outer joins are optional)
var unqualifiedNavigable = outerInnerGraph.UnqualifiedNavigableStreams.Get(streamNum);
if (unqualifiedNavigable != null)
{
navigableStreams.AddAll(unqualifiedNavigable);
}
// remove those already done
navigableStreams.RemoveAll(completedStreams);
// Which streams are inner streams to this stream (optional), which ones are outer to the stream (required)
var requiredStreams = GetOuterStreams(streamNum, navigableStreams, outerInnerGraph);
// Add inner joins, if any, unless already completed for this stream
innerJoinGraph.AddRequiredStreams(streamNum, requiredStreams, completedStreams);
var optionalStreams = GetInnerStreams(streamNum, navigableStreams, outerInnerGraph, innerJoinGraph, completedStreams);
// Remove from the required streams the optional streams which places 'full' joined streams
// into the optional stream category
requiredStreams.RemoveAll(optionalStreams);
// if we are a leaf node, we are done
if (navigableStreams.IsEmpty())
{
substreamsPerStream.Put(streamNum, new int[0]);
return;
}
// First the outer (required) streams to this stream, then the inner (optional) streams
var substreams = new int[requiredStreams.Count + optionalStreams.Count];
substreamsPerStream.Put(streamNum, substreams);
var count = 0;
foreach (int stream in requiredStreams)
{
substreams[count++] = stream;
requiredPerStream[stream] = true;
}
foreach (int stream in optionalStreams)
{
substreams[count++] = stream;
}
// next we look at all the required streams and add their dependent streams
foreach (int stream in requiredStreams)
{
completedStreams.Add(stream);
}
foreach (int stream in requiredStreams)
{
streamCallStack.Push(stream);
RecursiveBuild(stream, streamCallStack, queryGraph, outerInnerGraph, innerJoinGraph,
completedStreams, substreamsPerStream, requiredPerStream, dependencyGraph);
streamCallStack.Pop();
}
// look at all the optional streams and add their dependent streams
foreach (int stream in optionalStreams)
{
streamCallStack.Push(stream);
RecursiveBuild(stream, streamCallStack, queryGraph, outerInnerGraph, innerJoinGraph,
completedStreams, substreamsPerStream, requiredPerStream, dependencyGraph);
streamCallStack.Pop();
}
}