private static void HandleIndexDependencies(
QueryPlanForge queryPlan,
StatementCompileTimeServices services)
{
if (queryPlan == null) {
return;
}
var indexes = new HashSet<TableLookupIndexReqKey>();
for (var streamnum = 0; streamnum < queryPlan.ExecNodeSpecs.Length; streamnum++) {
var node = queryPlan.ExecNodeSpecs[streamnum];
indexes.Clear();
node.AddIndexes(indexes);
foreach (var index in indexes) {
if (index.TableName != null) {
var tableMeta = services.TableCompileTimeResolver.Resolve(index.TableName);
if (tableMeta.TableVisibility == NameAccessModifier.PUBLIC) {
services.ModuleDependenciesCompileTime.AddPathIndex(
false,
index.TableName,
tableMeta.TableModuleName,
index.IndexName,
index.IndexModuleName,
services.NamedWindowCompileTimeRegistry,
services.TableCompileTimeRegistry);
}
}
}
}
}
public static void CompareQueryPlans(
QueryPlanForge expectedPlan,
QueryPlanForge actualPlan)
{
IDictionary<TableLookupIndexReqKey, TableLookupIndexReqKey> indexNameMapping =
new Dictionary<TableLookupIndexReqKey, TableLookupIndexReqKey>();
CompareIndexes(expectedPlan.IndexSpecs, actualPlan.IndexSpecs, indexNameMapping);
CompareExecNodeSpecs(expectedPlan.ExecNodeSpecs, actualPlan.ExecNodeSpecs, indexNameMapping);
}
private SupportQueryPlanBuilder(int numStreams)
{
var indexes = new QueryPlanIndexForge[numStreams];
for (var i = 0; i < indexes.Length; i++) {
indexes[i] =
new QueryPlanIndexForge(new LinkedHashMap<TableLookupIndexReqKey, QueryPlanIndexItemForge>());
}
queryPlan = new QueryPlanForge(indexes, new QueryPlanNodeForge[numStreams]);
}
private static void TryAssertion(
RegressionEnvironment env,
string epl,
QueryPlanForge expectedPlan)
{
SupportQueryPlanIndexHook.Reset();
epl = INDEX_CALLBACK_HOOK + epl;
env.CompileDeploy(epl);
var actualPlan = SupportQueryPlanIndexHook.AssertJoinAndReset();
SupportQueryPlanIndexHelper.CompareQueryPlans(expectedPlan, actualPlan);
env.UndeployAll();
}
public JoinSetComposerPrototypeGeneralForge(
EventType[] streamTypes,
ExprNode postJoinEvaluator,
bool outerJoins,
QueryPlanForge queryPlan,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
string[] streamNames,
bool joinRemoveStream,
bool hasHistorical)
: base(streamTypes, postJoinEvaluator, outerJoins)
{
this.queryPlan = queryPlan;
this.streamJoinAnalysisResult = streamJoinAnalysisResult;
this.streamNames = streamNames;
this.joinRemoveStream = joinRemoveStream;
this.hasHistorical = hasHistorical;
}
// Remove plans for non-unidirectional streams
private static void RemoveUnidirectionalAndTable(
QueryPlanForge queryPlan,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult)
{
bool allUnidirectional = streamJoinAnalysisResult.IsUnidirectionalAll;
for (int streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) {
if (allUnidirectional) {
queryPlan.ExecNodeSpecs[streamNum] = new QueryPlanNodeForgeAllUnidirectionalOuter(streamNum);
}
else {
bool unidirectional = streamJoinAnalysisResult.IsUnidirectional &&
!streamJoinAnalysisResult.UnidirectionalInd[streamNum];
bool table = streamJoinAnalysisResult.TablesPerStream[streamNum] != null;
if (unidirectional || table) {
queryPlan.ExecNodeSpecs[streamNum] = QueryPlanNodeNoOpForge.INSTANCE;
}
}
}
}
private void AssertPlan(QueryPlanForge plan)
{
Assert.AreEqual(2, plan.ExecNodeSpecs.Length);
Assert.AreEqual(2, plan.ExecNodeSpecs.Length);
}
public static SupportQueryPlanBuilder Start(QueryPlanForge existing)
{
return new SupportQueryPlanBuilder(existing);
}
public SupportQueryPlanBuilder(QueryPlanForge queryPlan)
{
this.queryPlan = queryPlan;
}
public static QueryPlanForgeDesc Build(
QueryGraphForge queryGraph,
EventType[] typesPerStream,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
bool hasForceNestedIter,
string[][][] indexedStreamsUniqueProps,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo raw,
SerdeCompileTimeResolver serdeResolver)
{
if (Log.IsDebugEnabled) {
Log.Debug(".build filterQueryGraph=" + queryGraph);
}
var numStreams = queryGraph.NumStreams;
var additionalForgeables = new List<StmtClassForgeableFactory>();
var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(
queryGraph,
typesPerStream,
indexedStreamsUniqueProps);
if (Log.IsDebugEnabled) {
Log.Debug(".build Index build completed, indexes=" + QueryPlanIndexForge.Print(indexSpecs));
}
// any historical streams don't get indexes, the lookup strategy accounts for cached indexes
if (historicalViewableDesc.IsHistorical) {
for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) {
if (historicalViewableDesc.Historical[i]) {
indexSpecs[i] = null;
}
}
}
var planNodeSpecs = new QueryPlanNodeForge[numStreams];
var worstDepth = int.MaxValue;
for (var streamNo = 0; streamNo < numStreams; streamNo++) {
// no plan for historical streams that are dependent upon other streams
if (historicalViewableDesc.Historical[streamNo] && dependencyGraph.HasDependency(streamNo)) {
planNodeSpecs[streamNo] = new QueryPlanNodeNoOpForge();
continue;
}
var bestChainResult = ComputeBestPath(streamNo, queryGraph, dependencyGraph);
var bestChain = bestChainResult.Chain;
if (Log.IsDebugEnabled) {
Log.Debug(".build For stream " + streamNo + " bestChain=" + bestChain.RenderAny());
}
if (bestChainResult.Depth < worstDepth) {
worstDepth = bestChainResult.Depth;
}
var planDesc = CreateStreamPlan(
streamNo,
bestChain,
queryGraph,
indexSpecs,
typesPerStream,
historicalViewableDesc.Historical,
historicalStreamIndexLists,
tablesPerStream,
streamJoinAnalysisResult,
raw,
serdeResolver);
planNodeSpecs[streamNo] = planDesc.Forge;
additionalForgeables.AddAll(planDesc.AdditionalForgeables);
if (Log.IsDebugEnabled) {
Log.Debug(".build spec=" + planNodeSpecs[streamNo]);
}
}
// We use the merge/nested (outer) join algorithm instead.
if (worstDepth < numStreams - 1 && !hasForceNestedIter) {
return null;
}
// build historical index and lookup strategies
for (var i = 0; i < numStreams; i++) {
var plan = planNodeSpecs[i];
QueryPlanNodeForgeVisitor visitor = new ProxyQueryPlanNodeForgeVisitor {
ProcVisit = node => {
if (node is HistoricalDataPlanNodeForge) {
var historical = (HistoricalDataPlanNodeForge) node;
JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum].GetStrategy(
historical.LookupStreamNum, raw, serdeResolver);
historical.PollResultIndexingStrategy = desc.IndexingForge;
historical.HistoricalIndexLookupStrategy = desc.LookupForge;
additionalForgeables.AddAll(desc.AdditionalForgeables);
}
}
};
plan.Accept(visitor);
}
var forge = new QueryPlanForge(indexSpecs, planNodeSpecs);
return new QueryPlanForgeDesc(forge, additionalForgeables);
}
/// <summary>
/// Build query plan.
/// </summary>
/// <param name="queryGraph">navigability info</param>
/// <param name="optionalOuterJoinType">outer join type, null if not an outer join</param>
/// <param name="typesPerStream">event types for each stream</param>
/// <param name="streamJoinAnalysisResult">stream join analysis</param>
/// <returns>query plan</returns>
public static QueryPlanForgeDesc Build(
EventType[] typesPerStream,
QueryGraphForge queryGraph,
OuterJoinType? optionalOuterJoinType,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo raw)
{
var uniqueIndexProps = streamJoinAnalysisResult.UniqueKeys;
var tablesPerStream = streamJoinAnalysisResult.TablesPerStream;
var additionalForgeable = new List<StmtClassForgeableFactory>();
var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(
queryGraph,
typesPerStream,
uniqueIndexProps);
var execNodeSpecs = new QueryPlanNodeForge[2];
var lookupPlans = new TableLookupPlanForge[2];
// plan lookup from 1 to zero
TableLookupPlanDesc plan1to0 = NStreamQueryPlanBuilder.CreateLookupPlan(
queryGraph,
1,
0,
streamJoinAnalysisResult.IsVirtualDW(0),
indexSpecs[0],
typesPerStream,
tablesPerStream[0],
raw,
SerdeCompileTimeResolverNonHA.INSTANCE);
lookupPlans[1] = plan1to0.Forge;
additionalForgeable.AddAll(plan1to0.AdditionalForgeables);
// plan lookup from zero to 1
TableLookupPlanDesc plan0to1 = NStreamQueryPlanBuilder.CreateLookupPlan(
queryGraph,
0,
1,
streamJoinAnalysisResult.IsVirtualDW(1),
indexSpecs[1],
typesPerStream,
tablesPerStream[1],
raw,
SerdeCompileTimeResolverNonHA.INSTANCE);
lookupPlans[0] = plan0to1.Forge;
additionalForgeable.AddAll(plan0to1.AdditionalForgeables);
execNodeSpecs[0] = new TableLookupNodeForge(lookupPlans[0]);
execNodeSpecs[1] = new TableLookupNodeForge(lookupPlans[1]);
if (optionalOuterJoinType != null) {
if ((optionalOuterJoinType.Equals(OuterJoinType.LEFT)) ||
(optionalOuterJoinType.Equals(OuterJoinType.FULL))) {
execNodeSpecs[0] = new TableOuterLookupNodeForge(lookupPlans[0]);
}
if ((optionalOuterJoinType.Equals(OuterJoinType.RIGHT)) ||
(optionalOuterJoinType.Equals(OuterJoinType.FULL))) {
execNodeSpecs[1] = new TableOuterLookupNodeForge(lookupPlans[1]);
}
}
var forge = new QueryPlanForge(indexSpecs, execNodeSpecs);
return new QueryPlanForgeDesc(forge, additionalForgeable);
}
public void Join(QueryPlanForge join)
{
JOINS.Add(join);
}
public void TestBuild()
{
QueryPlanForge plan = NStreamQueryPlanBuilder.Build(queryGraph, typesPerStream, new HistoricalViewableDesc(6), dependencyGraph, null, false, new string[queryGraph.NumStreams][][], new TableMetaData[queryGraph.NumStreams], new StreamJoinAnalysisResultCompileTime(5));
log.Debug(".testBuild plan=" + plan);
}
internal static QueryPlanForgeDesc Build(
QueryGraphForge queryGraph,
OuterJoinDesc[] outerJoinDescList,
string[] streamNames,
EventType[] typesPerStream,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
string[][][] indexedStreamsUniqueProps,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
if (Log.IsDebugEnabled) {
Log.Debug(".build filterQueryGraph=" + queryGraph);
}
var numStreams = queryGraph.NumStreams;
var planNodeSpecs = new QueryPlanNodeForge[numStreams];
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Build index specifications
var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(
queryGraph,
typesPerStream,
indexedStreamsUniqueProps);
// any historical streams don't get indexes, the lookup strategy accounts for cached indexes
if (historicalViewableDesc.IsHistorical) {
for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) {
if (historicalViewableDesc.Historical[i]) {
indexSpecs[i] = null;
}
}
}
// Build graph of the outer join to inner table relationships.
// Build a map of inner joins.
OuterInnerDirectionalGraph outerInnerGraph;
InnerJoinGraph innerJoinGraph;
if (outerJoinDescList.Length > 0) {
outerInnerGraph = GraphOuterJoins(numStreams, outerJoinDescList);
innerJoinGraph = InnerJoinGraph.GraphInnerJoins(numStreams, outerJoinDescList);
}
else {
// all inner joins - thereby no (or empty) directional graph
outerInnerGraph = new OuterInnerDirectionalGraph(numStreams);
innerJoinGraph = new InnerJoinGraph(numStreams, true);
}
if (Log.IsDebugEnabled) {
Log.Debug(".build directional graph=" + outerInnerGraph.Print());
}
// For each stream determine the query plan
for (var streamNo = 0; streamNo < numStreams; streamNo++) {
// no plan for historical streams that are dependent upon other streams
if (historicalViewableDesc.Historical[streamNo] && dependencyGraph.HasDependency(streamNo)) {
planNodeSpecs[streamNo] = new QueryPlanNodeNoOpForge();
continue;
}
QueryPlanNodeForgeDesc desc = BuildPlanNode(
numStreams,
streamNo,
streamNames,
queryGraph,
outerInnerGraph,
outerJoinDescList,
innerJoinGraph,
indexSpecs,
typesPerStream,
historicalViewableDesc.Historical,
dependencyGraph,
historicalStreamIndexLists,
tablesPerStream,
streamJoinAnalysisResult,
statementRawInfo,
services);
QueryPlanNodeForge queryPlanNode = desc.Forge;
additionalForgeables.AddAll(desc.AdditionalForgeables);
if (Log.IsDebugEnabled) {
Log.Debug(
".build spec for stream '" +
streamNames[streamNo] +
"' number " +
streamNo +
" is " +
queryPlanNode);
}
planNodeSpecs[streamNo] = queryPlanNode;
}
var queryPlan = new QueryPlanForge(indexSpecs, planNodeSpecs);
if (Log.IsDebugEnabled) {
Log.Debug(".build query plan=" + queryPlan);
}
return new QueryPlanForgeDesc(queryPlan, additionalForgeables);
}
public static JoinSetComposerPrototypeDesc MakeComposerPrototype(
StatementSpecCompiled spec,
StreamJoinAnalysisResultCompileTime joinAnalysisResult,
StreamTypeService typeService,
HistoricalViewableDesc historicalViewableDesc,
bool isOnDemandQuery,
bool hasAggregations,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices compileTimeServices)
{
var streamTypes = typeService.EventTypes;
var streamNames = typeService.StreamNames;
var whereClause = spec.Raw.WhereClause;
var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan;
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Determine if there is a historical stream, and what dependencies exist
var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false);
for (var i = 0; i < streamTypes.Length; i++) {
if (historicalViewableDesc.Historical[i]) {
var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i];
historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn);
}
}
if (Log.IsDebugEnabled) {
Log.Debug("Dependency graph: " + historicalDependencyGraph);
}
// Handle a join with a database or other historical data source for 2 streams
var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList);
if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) {
var desc = MakeComposerHistorical2Stream(
outerJoinDescs,
whereClause,
streamTypes,
streamNames,
historicalViewableDesc,
queryPlanLogging,
statementRawInfo,
compileTimeServices);
return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables);
}
var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs);
// Query graph for graph relationships between streams/historicals
// For outer joins the query graph will just contain outer join relationships
var hint = ExcludePlanHint.GetHint(
typeService.StreamNames,
statementRawInfo,
compileTimeServices);
var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false);
if (outerJoinDescs.Length > 0) {
OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph);
}
}
// Let the query graph reflect the where-clause
if (whereClause != null) {
// Analyze relationships between streams using the optional filter expression.
// Relationships are properties in AND and EQUALS nodes of joins.
FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph);
}
// Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c)
QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph);
}
}
// Historical index lists
var historicalStreamIndexLists =
new HistoricalStreamIndexListForge[streamTypes.Length];
var queryPlanDesc = QueryPlanBuilder.GetPlan(
streamTypes,
outerJoinDescs,
queryGraph,
typeService.StreamNames,
historicalViewableDesc,
historicalDependencyGraph,
historicalStreamIndexLists,
joinAnalysisResult,
queryPlanLogging,
statementRawInfo,
compileTimeServices);
QueryPlanForge queryPlan = queryPlanDesc.Forge;
additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables);
// remove unused indexes - consider all streams or all unidirectional
var usedIndexes = new HashSet<TableLookupIndexReqKey>();
var indexSpecs = queryPlan.IndexSpecs;
for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) {
var planNode = queryPlan.ExecNodeSpecs[streamNum];
planNode?.AddIndexes(usedIndexes);
}
foreach (var indexSpec in indexSpecs) {
if (indexSpec == null) {
continue;
}
var items = indexSpec.Items;
var indexNames = items.Keys.ToArray();
foreach (var indexName in indexNames) {
if (!usedIndexes.Contains(indexName)) {
items.Remove(indexName);
}
}
}
// plan multikeys
IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys(
indexSpecs, statementRawInfo, compileTimeServices);
additionalForgeables.AddAll(multikeyForgeables);
QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook(
spec.Annotations,
compileTimeServices.ImportServiceCompileTime);
if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) {
QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan());
hook?.Join(queryPlan);
}
var selectsRemoveStream =
spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null;
var joinRemoveStream = selectsRemoveStream || hasAggregations;
ExprNode postJoinEvaluator;
if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf(
isOuterJoins,
joinAnalysisResult.IsUnidirectional,
joinAnalysisResult.IsPureSelfJoin)) {
postJoinEvaluator = GetFilterExpressionInclOnClause(
spec.Raw.WhereClause,
outerJoinDescs,
statementRawInfo,
compileTimeServices);
}
else {
postJoinEvaluator = spec.Raw.WhereClause;
}
JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge(
typeService.EventTypes,
postJoinEvaluator,
outerJoinDescs.Length > 0,
queryPlan,
joinAnalysisResult,
typeService.StreamNames,
joinRemoveStream,
historicalViewableDesc.IsHistorical);
return new JoinSetComposerPrototypeDesc(forge, additionalForgeables);
}
