/// <summary>
/// Walks the chain of lookups and constructs lookup strategy and plan specification based
/// on the index specifications.
/// </summary>
/// <param name="lookupStream">the stream to construct the query plan for</param>
/// <param name="bestChain">the chain that the lookup follows to make best use of indexes</param>
/// <param name="queryGraph">the repository for key properties to indexes</param>
/// <param name="indexSpecsPerStream">specifications of indexes</param>
/// <param name="typesPerStream">event types for each stream</param>
/// <param name="isHistorical">indicator for each stream if it is a historical streams or not</param>
/// <param name="historicalStreamIndexLists">index management, populated for the query plan</param>
/// <param name="tablesPerStream">tables</param>
/// <param name="streamJoinAnalysisResult">stream join analysis</param>
/// <param name="raw">raw statement information</param>
/// <param name="serdeResolver">serde resolver</param>
/// <returns>NestedIterationNode with lookups attached underneath</returns>
public static QueryPlanNodeForgeDesc CreateStreamPlan(
int lookupStream,
int[] bestChain,
QueryGraphForge queryGraph,
QueryPlanIndexForge[] indexSpecsPerStream,
EventType[] typesPerStream,
bool[] isHistorical,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo raw,
SerdeCompileTimeResolver serdeResolver)
{
var nestedIterNode = new NestedIterationNodeForge(bestChain);
var currentLookupStream = lookupStream;
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Walk through each successive lookup
for (var i = 0; i < bestChain.Length; i++) {
var indexedStream = bestChain[i];
QueryPlanNodeForge node;
if (isHistorical[indexedStream]) {
if (historicalStreamIndexLists[indexedStream] == null) {
historicalStreamIndexLists[indexedStream] = new HistoricalStreamIndexListForge(
indexedStream,
typesPerStream,
queryGraph);
}
historicalStreamIndexLists[indexedStream].AddIndex(currentLookupStream);
node = new HistoricalDataPlanNodeForge(
indexedStream,
lookupStream,
currentLookupStream,
typesPerStream.Length,
null);
}
else {
var tableLookupPlan = CreateLookupPlan(
queryGraph,
currentLookupStream,
indexedStream,
streamJoinAnalysisResult.IsVirtualDW(indexedStream),
indexSpecsPerStream[indexedStream],
typesPerStream,
tablesPerStream[indexedStream],
raw,
serdeResolver);
node = new TableLookupNodeForge(tableLookupPlan.Forge);
additionalForgeables.AddAll(tableLookupPlan.AdditionalForgeables);
}
nestedIterNode.AddChildNode(node);
currentLookupStream = bestChain[i];
}
return new QueryPlanNodeForgeDesc(nestedIterNode, additionalForgeables);
}
private static LookupInstructionPlanDesc BuildLookupInstructions(
int rootStreamNum,
IDictionary<int, int[]> substreamsPerStream,
bool[] requiredPerStream,
string[] streamNames,
QueryGraphForge queryGraph,
QueryPlanIndexForge[] indexSpecs,
EventType[] typesPerStream,
OuterJoinDesc[] outerJoinDescList,
bool[] isHistorical,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
var result = new List<LookupInstructionPlanForge>();
var additionalForgeables = new List<StmtClassForgeableFactory>();
foreach (var fromStream in substreamsPerStream.Keys) {
var substreams = substreamsPerStream.Get(fromStream);
// for streams with no substreams we don't need to look up
if (substreams.Length == 0) {
continue;
}
var plans = new TableLookupPlanForge[substreams.Length];
var historicalPlans = new HistoricalDataPlanNodeForge[substreams.Length];
for (var i = 0; i < substreams.Length; i++) {
var toStream = substreams[i];
if (isHistorical[toStream]) {
// There may not be an outer-join descriptor, use if provided to build the associated expression
ExprNode outerJoinExpr = null;
if (outerJoinDescList.Length > 0) {
OuterJoinDesc outerJoinDesc;
if (toStream == 0) {
outerJoinDesc = outerJoinDescList[0];
}
else {
outerJoinDesc = outerJoinDescList[toStream - 1];
}
outerJoinExpr = outerJoinDesc.MakeExprNode(statementRawInfo, services);
}
if (historicalStreamIndexLists[toStream] == null) {
historicalStreamIndexLists[toStream] = new HistoricalStreamIndexListForge(
toStream,
typesPerStream,
queryGraph);
}
historicalStreamIndexLists[toStream].AddIndex(fromStream);
historicalPlans[i] = new HistoricalDataPlanNodeForge(
toStream,
rootStreamNum,
fromStream,
typesPerStream.Length,
outerJoinExpr == null ? null : outerJoinExpr.Forge);
}
else {
TableLookupPlanDesc planDesc = NStreamQueryPlanBuilder.CreateLookupPlan(
queryGraph,
fromStream,
toStream,
streamJoinAnalysisResult.IsVirtualDW(toStream),
indexSpecs[toStream],
typesPerStream,
tablesPerStream[toStream],
statementRawInfo,
SerdeCompileTimeResolverNonHA.INSTANCE);
plans[i] = planDesc.Forge;
additionalForgeables.AddAll(planDesc.AdditionalForgeables);
}
}
var fromStreamName = streamNames[fromStream];
var instruction = new LookupInstructionPlanForge(
fromStream,
fromStreamName,
substreams,
plans,
historicalPlans,
requiredPerStream);
result.Add(instruction);
}
return new LookupInstructionPlanDesc(result, additionalForgeables);
}
public static JoinSetComposerPrototypeDesc MakeComposerPrototype(
StatementSpecCompiled spec,
StreamJoinAnalysisResultCompileTime joinAnalysisResult,
StreamTypeService typeService,
HistoricalViewableDesc historicalViewableDesc,
bool isOnDemandQuery,
bool hasAggregations,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices compileTimeServices)
{
var streamTypes = typeService.EventTypes;
var streamNames = typeService.StreamNames;
var whereClause = spec.Raw.WhereClause;
var queryPlanLogging = compileTimeServices.Configuration.Common.Logging.IsEnableQueryPlan;
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Determine if there is a historical stream, and what dependencies exist
var historicalDependencyGraph = new DependencyGraph(streamTypes.Length, false);
for (var i = 0; i < streamTypes.Length; i++) {
if (historicalViewableDesc.Historical[i]) {
var streamsThisStreamDependsOn = historicalViewableDesc.DependenciesPerHistorical[i];
historicalDependencyGraph.AddDependency(i, streamsThisStreamDependsOn);
}
}
if (Log.IsDebugEnabled) {
Log.Debug("Dependency graph: " + historicalDependencyGraph);
}
// Handle a join with a database or other historical data source for 2 streams
var outerJoinDescs = OuterJoinDesc.ToArray(spec.Raw.OuterJoinDescList);
if (historicalViewableDesc.IsHistorical && streamTypes.Length == 2) {
var desc = MakeComposerHistorical2Stream(
outerJoinDescs,
whereClause,
streamTypes,
streamNames,
historicalViewableDesc,
queryPlanLogging,
statementRawInfo,
compileTimeServices);
return new JoinSetComposerPrototypeDesc(desc.Forge, desc.AdditionalForgeables);
}
var isOuterJoins = !OuterJoinDesc.ConsistsOfAllInnerJoins(outerJoinDescs);
// Query graph for graph relationships between streams/historicals
// For outer joins the query graph will just contain outer join relationships
var hint = ExcludePlanHint.GetHint(
typeService.StreamNames,
statementRawInfo,
compileTimeServices);
var queryGraph = new QueryGraphForge(streamTypes.Length, hint, false);
if (outerJoinDescs.Length > 0) {
OuterJoinAnalyzer.Analyze(outerJoinDescs, queryGraph);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After outer join filterQueryGraph=\n" + queryGraph);
}
}
// Let the query graph reflect the where-clause
if (whereClause != null) {
// Analyze relationships between streams using the optional filter expression.
// Relationships are properties in AND and EQUALS nodes of joins.
FilterExprAnalyzer.Analyze(whereClause, queryGraph, isOuterJoins);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After filter expression filterQueryGraph=\n" + queryGraph);
}
// Add navigation entries based on key and index property equivalency (a=b, b=c follows a=c)
QueryGraphForge.FillEquivalentNav(streamTypes, queryGraph);
if (Log.IsDebugEnabled) {
Log.Debug(".makeComposer After fill equiv. nav. filterQueryGraph=\n" + queryGraph);
}
}
// Historical index lists
var historicalStreamIndexLists =
new HistoricalStreamIndexListForge[streamTypes.Length];
var queryPlanDesc = QueryPlanBuilder.GetPlan(
streamTypes,
outerJoinDescs,
queryGraph,
typeService.StreamNames,
historicalViewableDesc,
historicalDependencyGraph,
historicalStreamIndexLists,
joinAnalysisResult,
queryPlanLogging,
statementRawInfo,
compileTimeServices);
QueryPlanForge queryPlan = queryPlanDesc.Forge;
additionalForgeables.AddAll(queryPlanDesc.AdditionalForgeables);
// remove unused indexes - consider all streams or all unidirectional
var usedIndexes = new HashSet<TableLookupIndexReqKey>();
var indexSpecs = queryPlan.IndexSpecs;
for (var streamNum = 0; streamNum < queryPlan.ExecNodeSpecs.Length; streamNum++) {
var planNode = queryPlan.ExecNodeSpecs[streamNum];
planNode?.AddIndexes(usedIndexes);
}
foreach (var indexSpec in indexSpecs) {
if (indexSpec == null) {
continue;
}
var items = indexSpec.Items;
var indexNames = items.Keys.ToArray();
foreach (var indexName in indexNames) {
if (!usedIndexes.Contains(indexName)) {
items.Remove(indexName);
}
}
}
// plan multikeys
IList<StmtClassForgeableFactory> multikeyForgeables = PlanMultikeys(
indexSpecs, statementRawInfo, compileTimeServices);
additionalForgeables.AddAll(multikeyForgeables);
QueryPlanIndexHook hook = QueryPlanIndexHookUtil.GetHook(
spec.Annotations,
compileTimeServices.ImportServiceCompileTime);
if (queryPlanLogging && (QUERY_PLAN_LOG.IsInfoEnabled || hook != null)) {
QUERY_PLAN_LOG.Info("Query plan: " + queryPlan.ToQueryPlan());
hook?.Join(queryPlan);
}
var selectsRemoveStream =
spec.Raw.SelectStreamSelectorEnum.IsSelectsRStream() || spec.Raw.OutputLimitSpec != null;
var joinRemoveStream = selectsRemoveStream || hasAggregations;
ExprNode postJoinEvaluator;
if (JoinSetComposerUtil.IsNonUnidirectionalNonSelf(
isOuterJoins,
joinAnalysisResult.IsUnidirectional,
joinAnalysisResult.IsPureSelfJoin)) {
postJoinEvaluator = GetFilterExpressionInclOnClause(
spec.Raw.WhereClause,
outerJoinDescs,
statementRawInfo,
compileTimeServices);
}
else {
postJoinEvaluator = spec.Raw.WhereClause;
}
JoinSetComposerPrototypeGeneralForge forge = new JoinSetComposerPrototypeGeneralForge(
typeService.EventTypes,
postJoinEvaluator,
outerJoinDescs.Length > 0,
queryPlan,
joinAnalysisResult,
typeService.StreamNames,
joinRemoveStream,
historicalViewableDesc.IsHistorical);
return new JoinSetComposerPrototypeDesc(forge, additionalForgeables);
}