private static ISet<int> GetInnerStreams(
int fromStream,
ISet<int> toStreams,
OuterInnerDirectionalGraph outerInnerGraph,
InnerJoinGraph innerJoinGraph,
ISet<int> completedStreams)
{
ISet<int> innerStreams = new HashSet<int>();
foreach (var toStream in toStreams) {
if (outerInnerGraph.IsInner(fromStream, toStream)) {
// if the to-stream, recursively, has an inner join itself, it becomes a required stream and not optional
var hasInnerJoin = false;
if (!innerJoinGraph.IsEmpty()) {
var doNotUseStreams = new HashSet<int>(completedStreams);
completedStreams.Add(fromStream);
hasInnerJoin = RecursiveHasInnerJoin(
toStream,
outerInnerGraph,
innerJoinGraph,
doNotUseStreams);
}
if (!hasInnerJoin) {
innerStreams.Add(toStream);
}
}
}
return innerStreams;
}
private void AssertOuters(
int[][] outersPerStream,
OuterInnerDirectionalGraph graph)
{
for (var i = 0; i < outersPerStream.Length; i++)
{
EPAssertionUtil.AssertEqualsAnyOrder(outersPerStream[i], graph.GetOuter(i));
}
}
// which streams are to this table an outer stream
private static ISet<int> GetOuterStreams(
int fromStream,
ISet<int> toStreams,
OuterInnerDirectionalGraph outerInnerGraph)
{
ISet<int> outerStreams = new HashSet<int>();
foreach (var toStream in toStreams) {
if (outerInnerGraph.IsOuter(toStream, fromStream)) {
outerStreams.Add(toStream);
}
}
return outerStreams;
}
private static bool RecursiveHasInnerJoin(
int toStream,
OuterInnerDirectionalGraph outerInnerGraph,
InnerJoinGraph innerJoinGraph,
ISet<int> completedStreams)
{
// Check if the to-stream is in any of the inner joins
var hasInnerJoin = innerJoinGraph.HasInnerJoin(toStream);
if (hasInnerJoin) {
return true;
}
var innerToToStream = outerInnerGraph.GetInner(toStream);
if (innerToToStream != null) {
foreach (var nextStream in innerToToStream) {
if (completedStreams.Contains(nextStream)) {
continue;
}
var notConsider = new HashSet<int>(completedStreams);
notConsider.Add(toStream);
var result = RecursiveHasInnerJoin(nextStream, outerInnerGraph, innerJoinGraph, notConsider);
if (result) {
return true;
}
}
}
var outerToToStream = outerInnerGraph.GetOuter(toStream);
if (outerToToStream != null) {
foreach (var nextStream in outerToToStream) {
if (completedStreams.Contains(nextStream)) {
continue;
}
var notConsider = new HashSet<int>(completedStreams);
notConsider.Add(toStream);
var result = RecursiveHasInnerJoin(nextStream, outerInnerGraph, innerJoinGraph, notConsider);
if (result) {
return true;
}
}
}
return false;
}
public void TestRecursiveBuild()
{
var streamNum = 2;
var queryGraph = new QueryGraphForge(6, null, false);
var outerInnerGraph = new OuterInnerDirectionalGraph(6);
var completedStreams = new HashSet <int>();
var substreamsPerStream = new LinkedHashMap <int, int[]>();
var requiredPerStream = new bool[6];
outerInnerGraph.Add(3, 2).Add(2, 1).Add(4, 3).Add(1, 0).Add(3, 5);
var fake = supportExprNodeFactory.MakeIdentNode("TheString", "s0");
queryGraph.AddStrictEquals(2, "", fake, 3, "", fake);
queryGraph.AddStrictEquals(3, "", fake, 4, "", fake);
queryGraph.AddStrictEquals(3, "", fake, 5, "", fake);
queryGraph.AddStrictEquals(2, "", fake, 1, "", fake);
queryGraph.AddStrictEquals(1, "", fake, 0, "", fake);
ISet <InterchangeablePair <int, int> > innerJoins = new HashSet <InterchangeablePair <int, int> >();
var innerJoinGraph = new InnerJoinGraph(6, innerJoins);
var streamStack = new Stack <int>();
NStreamOuterQueryPlanBuilder.RecursiveBuild(
streamNum,
streamStack,
queryGraph,
outerInnerGraph,
innerJoinGraph,
completedStreams,
substreamsPerStream,
requiredPerStream,
new DependencyGraph(6, false));
Assert.AreEqual(6, substreamsPerStream.Count);
EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(2), new[] { 3, 1 });
EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(3), new[] { 4, 5 });
EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(1), new[] { 0 });
EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(4), new int[] { });
EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(5), new int[] { });
EPAssertionUtil.AssertEqualsExactOrder(substreamsPerStream.Get(0), new int[] { });
NStreamOuterQueryPlanBuilder.VerifyJoinedPerStream(2, substreamsPerStream);
EPAssertionUtil.AssertEqualsExactOrder(
requiredPerStream,
new[] { false, false, false, true, true, false }
);
}
/// <summary>
/// Builds a graph of outer joins given the outer join information from the statement.
/// Eliminates right and left joins and full joins by placing the information in a graph object.
/// </summary>
/// <param name="numStreams">is the number of streams</param>
/// <param name="outerJoinDescList">list of outer join stream numbers and property names</param>
/// <returns>graph object</returns>
public static OuterInnerDirectionalGraph GraphOuterJoins(
int numStreams,
OuterJoinDesc[] outerJoinDescList)
{
if (outerJoinDescList.Length + 1 != numStreams) {
throw new ArgumentException("Number of outer join descriptors and number of streams not matching up");
}
var graph = new OuterInnerDirectionalGraph(numStreams);
for (var i = 0; i < outerJoinDescList.Length; i++) {
var desc = outerJoinDescList[i];
var streamMax = i + 1; // the outer join must references streams less then streamMax
// Check outer join on-expression, if provided
int streamOne;
int streamTwo;
int lowerStream;
int higherStream;
if (desc.OptLeftNode != null) {
streamOne = desc.OptLeftNode.StreamId;
streamTwo = desc.OptRightNode.StreamId;
if (streamOne > streamMax ||
streamTwo > streamMax ||
streamOne == streamTwo) {
throw new ArgumentException("Outer join descriptors reference future streams, or same streams");
}
// Determine who is the first stream in the streams listed
lowerStream = streamOne;
higherStream = streamTwo;
if (streamOne > streamTwo) {
lowerStream = streamTwo;
higherStream = streamOne;
}
}
else {
streamOne = i;
streamTwo = i + 1;
lowerStream = i;
higherStream = i + 1;
graph.AddUnqualifiedNavigable(streamOne, streamTwo);
}
// Add to graph
if (desc.OuterJoinType == OuterJoinType.FULL) {
graph.Add(streamOne, streamTwo);
graph.Add(streamTwo, streamOne);
}
else if (desc.OuterJoinType == OuterJoinType.LEFT) {
graph.Add(lowerStream, higherStream);
}
else if (desc.OuterJoinType == OuterJoinType.RIGHT) {
graph.Add(higherStream, lowerStream);
}
else if (desc.OuterJoinType == OuterJoinType.INNER) {
// no navigability for inner joins
}
else {
throw new ArgumentException(
"Outer join descriptors join type not handled, type=" + desc.OuterJoinType);
}
}
return graph;
}
/// <summary>
/// Recusivly builds a substream-per-stream ordered tree graph using the
/// join information supplied for outer joins and from the query graph (where clause).
/// <para />
/// Required streams are considered first and their lookup is placed first in the list
/// to gain performance.
/// </summary>
/// <param name="streamNum">is the root stream number that supplies the incoming event to build the tree for</param>
/// <param name="queryGraph">contains where-clause stream relationship info</param>
/// <param name="outerInnerGraph">contains the outer join stream relationship info</param>
/// <param name="completedStreams">is a temporary holder for streams already considered</param>
/// <param name="substreamsPerStream">is the ordered, tree-like structure to be filled</param>
/// <param name="requiredPerStream">indicates which streams are required and which are optional</param>
/// <param name="streamCallStack">the query plan call stack of streams available via cursor</param>
/// <param name="dependencyGraph">dependencies between historical streams</param>
/// <param name="innerJoinGraph">inner join graph</param>
/// <throws>ExprValidationException if the query planning failed</throws>
public static void RecursiveBuild(
int streamNum,
Stack<int> streamCallStack,
QueryGraphForge queryGraph,
OuterInnerDirectionalGraph outerInnerGraph,
InnerJoinGraph innerJoinGraph,
ISet<int> completedStreams,
IDictionary<int, int[]> substreamsPerStream,
bool[] requiredPerStream,
DependencyGraph dependencyGraph
)
{
// add this stream to the set of completed streams
completedStreams.Add(streamNum);
// check if the dependencies have been satisfied
if (dependencyGraph.HasDependency(streamNum)) {
var dependencies = dependencyGraph.GetDependenciesForStream(streamNum);
foreach (var dependentStream in dependencies) {
if (!streamCallStack.Contains(dependentStream)) {
throw new ExprValidationException(
"Historical stream " +
streamNum +
" parameter dependency originating in stream " +
dependentStream +
" cannot or may not be satisfied by the join");
}
}
}
// Determine the streams we can navigate to from this stream
var navigableStreams = queryGraph.GetNavigableStreams(streamNum);
// add unqualified navigable streams (since on-expressions in outer joins are optional)
var unqualifiedNavigable = outerInnerGraph.UnqualifiedNavigableStreams.Get(streamNum);
if (unqualifiedNavigable != null) {
navigableStreams.AddAll(unqualifiedNavigable);
}
// remove those already done
navigableStreams.RemoveAll(completedStreams);
// Which streams are inner streams to this stream (optional), which ones are outer to the stream (required)
var requiredStreams = GetOuterStreams(streamNum, navigableStreams, outerInnerGraph);
// Add inner joins, if any, unless already completed for this stream
innerJoinGraph.AddRequiredStreams(streamNum, requiredStreams, completedStreams);
var optionalStreams = GetInnerStreams(
streamNum,
navigableStreams,
outerInnerGraph,
innerJoinGraph,
completedStreams);
// Remove from the required streams the optional streams which places 'full' joined streams
// into the optional stream category
requiredStreams.RemoveAll(optionalStreams);
// if we are a leaf node, we are done
if (navigableStreams.IsEmpty()) {
substreamsPerStream.Put(streamNum, new int[0]);
return;
}
// First the outer (required) streams to this stream, then the inner (optional) streams
var substreams = new int[requiredStreams.Count + optionalStreams.Count];
substreamsPerStream.Put(streamNum, substreams);
var count = 0;
foreach (var stream in requiredStreams) {
substreams[count++] = stream;
requiredPerStream[stream] = true;
}
foreach (var stream in optionalStreams) {
substreams[count++] = stream;
}
// next we look at all the required streams and add their dependent streams
foreach (var stream in requiredStreams) {
completedStreams.Add(stream);
}
foreach (var stream in requiredStreams) {
streamCallStack.Push(stream);
RecursiveBuild(
stream,
streamCallStack,
queryGraph,
outerInnerGraph,
innerJoinGraph,
completedStreams,
substreamsPerStream,
requiredPerStream,
dependencyGraph);
streamCallStack.Pop();
}
// look at all the optional streams and add their dependent streams
foreach (var stream in optionalStreams) {
streamCallStack.Push(stream);
RecursiveBuild(
stream,
streamCallStack,
queryGraph,
outerInnerGraph,
innerJoinGraph,
completedStreams,
substreamsPerStream,
requiredPerStream,
dependencyGraph);
streamCallStack.Pop();
}
}
internal static QueryPlanForgeDesc Build(
QueryGraphForge queryGraph,
OuterJoinDesc[] outerJoinDescList,
string[] streamNames,
EventType[] typesPerStream,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
string[][][] indexedStreamsUniqueProps,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
if (Log.IsDebugEnabled) {
Log.Debug(".build filterQueryGraph=" + queryGraph);
}
var numStreams = queryGraph.NumStreams;
var planNodeSpecs = new QueryPlanNodeForge[numStreams];
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Build index specifications
var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(
queryGraph,
typesPerStream,
indexedStreamsUniqueProps);
// any historical streams don't get indexes, the lookup strategy accounts for cached indexes
if (historicalViewableDesc.IsHistorical) {
for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) {
if (historicalViewableDesc.Historical[i]) {
indexSpecs[i] = null;
}
}
}
// Build graph of the outer join to inner table relationships.
// Build a map of inner joins.
OuterInnerDirectionalGraph outerInnerGraph;
InnerJoinGraph innerJoinGraph;
if (outerJoinDescList.Length > 0) {
outerInnerGraph = GraphOuterJoins(numStreams, outerJoinDescList);
innerJoinGraph = InnerJoinGraph.GraphInnerJoins(numStreams, outerJoinDescList);
}
else {
// all inner joins - thereby no (or empty) directional graph
outerInnerGraph = new OuterInnerDirectionalGraph(numStreams);
innerJoinGraph = new InnerJoinGraph(numStreams, true);
}
if (Log.IsDebugEnabled) {
Log.Debug(".build directional graph=" + outerInnerGraph.Print());
}
// For each stream determine the query plan
for (var streamNo = 0; streamNo < numStreams; streamNo++) {
// no plan for historical streams that are dependent upon other streams
if (historicalViewableDesc.Historical[streamNo] && dependencyGraph.HasDependency(streamNo)) {
planNodeSpecs[streamNo] = new QueryPlanNodeNoOpForge();
continue;
}
QueryPlanNodeForgeDesc desc = BuildPlanNode(
numStreams,
streamNo,
streamNames,
queryGraph,
outerInnerGraph,
outerJoinDescList,
innerJoinGraph,
indexSpecs,
typesPerStream,
historicalViewableDesc.Historical,
dependencyGraph,
historicalStreamIndexLists,
tablesPerStream,
streamJoinAnalysisResult,
statementRawInfo,
services);
QueryPlanNodeForge queryPlanNode = desc.Forge;
additionalForgeables.AddAll(desc.AdditionalForgeables);
if (Log.IsDebugEnabled) {
Log.Debug(
".build spec for stream '" +
streamNames[streamNo] +
"' number " +
streamNo +
" is " +
queryPlanNode);
}
planNodeSpecs[streamNo] = queryPlanNode;
}
var queryPlan = new QueryPlanForge(indexSpecs, planNodeSpecs);
if (Log.IsDebugEnabled) {
Log.Debug(".build query plan=" + queryPlan);
}
return new QueryPlanForgeDesc(queryPlan, additionalForgeables);
}
private static QueryPlanNodeForgeDesc BuildPlanNode(
int numStreams,
int streamNo,
string[] streamNames,
QueryGraphForge queryGraph,
OuterInnerDirectionalGraph outerInnerGraph,
OuterJoinDesc[] outerJoinDescList,
InnerJoinGraph innerJoinGraph,
QueryPlanIndexForge[] indexSpecs,
EventType[] typesPerStream,
bool[] isHistorical,
DependencyGraph dependencyGraph,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
// For each stream build an array of substreams, considering required streams (inner joins) first
// The order is relevant therefore preserving order via a LinkedHashMap.
var substreamsPerStream = new LinkedHashMap<int, int[]>();
var requiredPerStream = new bool[numStreams];
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Recursive populating the required (outer) and optional (inner) relationships
// of this stream and the substream
ISet<int> completedStreams = new HashSet<int>();
// keep track of tree path as only those stream events are always available to historical streams
var streamCallStack = new Stack<int>();
streamCallStack.Push(streamNo);
// For all inner-joins, the algorithm is slightly different
if (innerJoinGraph.IsAllInnerJoin) {
requiredPerStream.Fill(true);
RecursiveBuildInnerJoin(
streamNo,
streamCallStack,
queryGraph,
completedStreams,
substreamsPerStream,
dependencyGraph);
// compute a best chain to see if all streams are handled and add the remaining
var bestChain = NStreamQueryPlanBuilder.ComputeBestPath(streamNo, queryGraph, dependencyGraph);
AddNotYetNavigated(streamNo, numStreams, substreamsPerStream, bestChain);
}
else {
RecursiveBuild(
streamNo,
streamCallStack,
queryGraph,
outerInnerGraph,
innerJoinGraph,
completedStreams,
substreamsPerStream,
requiredPerStream,
dependencyGraph);
}
// verify the substreamsPerStream, all streams must exists and be linked
VerifyJoinedPerStream(streamNo, substreamsPerStream);
// build list of instructions for lookup
LookupInstructionPlanDesc lookupDesc = BuildLookupInstructions(
streamNo,
substreamsPerStream,
requiredPerStream,
streamNames,
queryGraph,
indexSpecs,
typesPerStream,
outerJoinDescList,
isHistorical,
historicalStreamIndexLists,
tablesPerStream,
streamJoinAnalysisResult,
statementRawInfo,
services);
var lookupInstructions = lookupDesc.Forges;
additionalForgeables.AddAll(lookupDesc.AdditionalForgeables);
// build historical index and lookup strategies
foreach (var lookups in lookupInstructions) {
foreach (var historical in lookups.HistoricalPlans) {
if (historical == null) {
continue;
}
JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum]
.GetStrategy(historical.LookupStreamNum, statementRawInfo, services.SerdeResolver);
historical.HistoricalIndexLookupStrategy = desc.LookupForge;
historical.PollResultIndexingStrategy = desc.IndexingForge;
additionalForgeables.AddAll(desc.AdditionalForgeables);
}
}
// build strategy tree for putting the result back together
var assemblyTopNodeFactory = AssemblyStrategyTreeBuilder.Build(
streamNo,
substreamsPerStream,
requiredPerStream);
var assemblyInstructionFactories =
BaseAssemblyNodeFactory.GetDescendentNodesBottomUp(assemblyTopNodeFactory);
var forge = new LookupInstructionQueryPlanNodeForge(
streamNo,
streamNames[streamNo],
numStreams,
requiredPerStream,
lookupInstructions,
assemblyInstructionFactories);
return new QueryPlanNodeForgeDesc(forge, additionalForgeables);
}
