/// <summary>
/// Get the strategies to use for polling from a given stream.
/// </summary>
/// <param name="streamViewStreamNum">the stream providing the polling events</param>
/// <returns>looking and indexing strategy</returns>
public JoinSetComposerPrototypeHistoricalDesc GetStrategy(
int streamViewStreamNum,
StatementRawInfo raw,
SerdeCompileTimeResolver serdeResolver)
{
// If there is only a single polling stream, then build a single index
if (_pollingStreams.Count == 1) {
return JoinSetComposerPrototypeForgeFactory.DetermineIndexing(
_queryGraph,
_typesPerStream[_historicalStreamNum],
_typesPerStream[streamViewStreamNum],
_historicalStreamNum,
streamViewStreamNum,
raw,
serdeResolver);
}
// If there are multiple polling streams, determine if a single index is appropriate.
// An index can be reused if:
// (a) indexed property names are the same
// (b) indexed property types are the same
// (c) key property types are the same (because of coercion)
// A index lookup strategy is always specific to the providing stream.
var additionalForgeables = new List<StmtClassForgeableFactory>();
if (_indexesUsedByStreams == null) {
_indexesUsedByStreams = new LinkedHashMap<HistoricalStreamIndexDesc, IList<int>>();
foreach (var pollingStream in _pollingStreams) {
var queryGraphValue = _queryGraph.GetGraphValue(pollingStream, _historicalStreamNum);
var hashKeyProps = queryGraphValue.HashKeyProps;
var indexProperties = hashKeyProps.Indexed;
var keyTypes = GetPropertyTypes(hashKeyProps.Keys);
var indexTypes = GetPropertyTypes(_typesPerStream[_historicalStreamNum], indexProperties);
var desc = new HistoricalStreamIndexDesc(
indexProperties,
indexTypes,
keyTypes);
var usedByStreams = _indexesUsedByStreams.Get(desc);
if (usedByStreams == null) {
usedByStreams = new List<int>();
_indexesUsedByStreams.Put(desc, usedByStreams);
}
usedByStreams.Add(pollingStream);
}
// There are multiple indexes required:
// Build a master indexing strategy that forms multiple indexes and numbers each.
if (_indexesUsedByStreams.Count > 1) {
var numIndexes = _indexesUsedByStreams.Count;
var indexingStrategies =
new PollResultIndexingStrategyForge[numIndexes];
// create an indexing strategy for each index
var count = 0;
foreach (var desc in _indexesUsedByStreams) {
var sampleStreamViewStreamNum = desc.Value[0];
var indexingX = JoinSetComposerPrototypeForgeFactory.DetermineIndexing(
_queryGraph,
_typesPerStream[_historicalStreamNum],
_typesPerStream[sampleStreamViewStreamNum],
_historicalStreamNum,
sampleStreamViewStreamNum,
raw,
serdeResolver);
indexingStrategies[count] = indexingX.IndexingForge;
additionalForgeables.AddAll(indexingX.AdditionalForgeables);
count++;
}
// create a master indexing strategy that utilizes each indexing strategy to create a set of indexes
_masterIndexingStrategy = new PollResultIndexingStrategyMultiForge(
streamViewStreamNum,
indexingStrategies);
}
}
// there is one type of index
if (_indexesUsedByStreams.Count == 1) {
return JoinSetComposerPrototypeForgeFactory.DetermineIndexing(
_queryGraph,
_typesPerStream[_historicalStreamNum],
_typesPerStream[streamViewStreamNum],
_historicalStreamNum,
streamViewStreamNum,
raw,
serdeResolver);
}
// determine which index number the polling stream must use
var indexUsed = 0;
var found = false;
foreach (var desc in _indexesUsedByStreams.Values) {
if (desc.Contains(streamViewStreamNum)) {
found = true;
break;
}
indexUsed++;
}
if (!found) {
throw new IllegalStateException("Index not found for use by stream " + streamViewStreamNum);
}
// Use one of the indexes built by the master index and a lookup strategy
JoinSetComposerPrototypeHistoricalDesc indexing = JoinSetComposerPrototypeForgeFactory.DetermineIndexing(
_queryGraph,
_typesPerStream[_historicalStreamNum],
_typesPerStream[streamViewStreamNum],
_historicalStreamNum,
streamViewStreamNum,
raw,
serdeResolver);
HistoricalIndexLookupStrategyForge innerLookupStrategy = indexing.LookupForge;
HistoricalIndexLookupStrategyForge lookupStrategy = new HistoricalIndexLookupStrategyMultiForge(indexUsed, innerLookupStrategy);
additionalForgeables.AddAll(indexing.AdditionalForgeables);
return new JoinSetComposerPrototypeHistoricalDesc(lookupStrategy, _masterIndexingStrategy, additionalForgeables);
}
public static QueryPlanForgeDesc Build(
QueryGraphForge queryGraph,
EventType[] typesPerStream,
HistoricalViewableDesc historicalViewableDesc,
DependencyGraph dependencyGraph,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
bool hasForceNestedIter,
string[][][] indexedStreamsUniqueProps,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo raw,
SerdeCompileTimeResolver serdeResolver)
{
if (Log.IsDebugEnabled) {
Log.Debug(".build filterQueryGraph=" + queryGraph);
}
var numStreams = queryGraph.NumStreams;
var additionalForgeables = new List<StmtClassForgeableFactory>();
var indexSpecs = QueryPlanIndexBuilder.BuildIndexSpec(
queryGraph,
typesPerStream,
indexedStreamsUniqueProps);
if (Log.IsDebugEnabled) {
Log.Debug(".build Index build completed, indexes=" + QueryPlanIndexForge.Print(indexSpecs));
}
// any historical streams don't get indexes, the lookup strategy accounts for cached indexes
if (historicalViewableDesc.IsHistorical) {
for (var i = 0; i < historicalViewableDesc.Historical.Length; i++) {
if (historicalViewableDesc.Historical[i]) {
indexSpecs[i] = null;
}
}
}
var planNodeSpecs = new QueryPlanNodeForge[numStreams];
var worstDepth = int.MaxValue;
for (var streamNo = 0; streamNo < numStreams; streamNo++) {
// no plan for historical streams that are dependent upon other streams
if (historicalViewableDesc.Historical[streamNo] && dependencyGraph.HasDependency(streamNo)) {
planNodeSpecs[streamNo] = new QueryPlanNodeNoOpForge();
continue;
}
var bestChainResult = ComputeBestPath(streamNo, queryGraph, dependencyGraph);
var bestChain = bestChainResult.Chain;
if (Log.IsDebugEnabled) {
Log.Debug(".build For stream " + streamNo + " bestChain=" + bestChain.RenderAny());
}
if (bestChainResult.Depth < worstDepth) {
worstDepth = bestChainResult.Depth;
}
var planDesc = CreateStreamPlan(
streamNo,
bestChain,
queryGraph,
indexSpecs,
typesPerStream,
historicalViewableDesc.Historical,
historicalStreamIndexLists,
tablesPerStream,
streamJoinAnalysisResult,
raw,
serdeResolver);
planNodeSpecs[streamNo] = planDesc.Forge;
additionalForgeables.AddAll(planDesc.AdditionalForgeables);
if (Log.IsDebugEnabled) {
Log.Debug(".build spec=" + planNodeSpecs[streamNo]);
}
}
// We use the merge/nested (outer) join algorithm instead.
if (worstDepth < numStreams - 1 && !hasForceNestedIter) {
return null;
}
// build historical index and lookup strategies
for (var i = 0; i < numStreams; i++) {
var plan = planNodeSpecs[i];
QueryPlanNodeForgeVisitor visitor = new ProxyQueryPlanNodeForgeVisitor {
ProcVisit = node => {
if (node is HistoricalDataPlanNodeForge) {
var historical = (HistoricalDataPlanNodeForge) node;
JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum].GetStrategy(
historical.LookupStreamNum, raw, serdeResolver);
historical.PollResultIndexingStrategy = desc.IndexingForge;
historical.HistoricalIndexLookupStrategy = desc.LookupForge;
additionalForgeables.AddAll(desc.AdditionalForgeables);
}
}
};
plan.Accept(visitor);
}
var forge = new QueryPlanForge(indexSpecs, planNodeSpecs);
return new QueryPlanForgeDesc(forge, additionalForgeables);
}
private static QueryPlanNodeForgeDesc BuildPlanNode(
int numStreams,
int streamNo,
string[] streamNames,
QueryGraphForge queryGraph,
OuterInnerDirectionalGraph outerInnerGraph,
OuterJoinDesc[] outerJoinDescList,
InnerJoinGraph innerJoinGraph,
QueryPlanIndexForge[] indexSpecs,
EventType[] typesPerStream,
bool[] isHistorical,
DependencyGraph dependencyGraph,
HistoricalStreamIndexListForge[] historicalStreamIndexLists,
TableMetaData[] tablesPerStream,
StreamJoinAnalysisResultCompileTime streamJoinAnalysisResult,
StatementRawInfo statementRawInfo,
StatementCompileTimeServices services)
{
// For each stream build an array of substreams, considering required streams (inner joins) first
// The order is relevant therefore preserving order via a LinkedHashMap.
var substreamsPerStream = new LinkedHashMap<int, int[]>();
var requiredPerStream = new bool[numStreams];
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Recursive populating the required (outer) and optional (inner) relationships
// of this stream and the substream
ISet<int> completedStreams = new HashSet<int>();
// keep track of tree path as only those stream events are always available to historical streams
var streamCallStack = new Stack<int>();
streamCallStack.Push(streamNo);
// For all inner-joins, the algorithm is slightly different
if (innerJoinGraph.IsAllInnerJoin) {
requiredPerStream.Fill(true);
RecursiveBuildInnerJoin(
streamNo,
streamCallStack,
queryGraph,
completedStreams,
substreamsPerStream,
dependencyGraph);
// compute a best chain to see if all streams are handled and add the remaining
var bestChain = NStreamQueryPlanBuilder.ComputeBestPath(streamNo, queryGraph, dependencyGraph);
AddNotYetNavigated(streamNo, numStreams, substreamsPerStream, bestChain);
}
else {
RecursiveBuild(
streamNo,
streamCallStack,
queryGraph,
outerInnerGraph,
innerJoinGraph,
completedStreams,
substreamsPerStream,
requiredPerStream,
dependencyGraph);
}
// verify the substreamsPerStream, all streams must exists and be linked
VerifyJoinedPerStream(streamNo, substreamsPerStream);
// build list of instructions for lookup
LookupInstructionPlanDesc lookupDesc = BuildLookupInstructions(
streamNo,
substreamsPerStream,
requiredPerStream,
streamNames,
queryGraph,
indexSpecs,
typesPerStream,
outerJoinDescList,
isHistorical,
historicalStreamIndexLists,
tablesPerStream,
streamJoinAnalysisResult,
statementRawInfo,
services);
var lookupInstructions = lookupDesc.Forges;
additionalForgeables.AddAll(lookupDesc.AdditionalForgeables);
// build historical index and lookup strategies
foreach (var lookups in lookupInstructions) {
foreach (var historical in lookups.HistoricalPlans) {
if (historical == null) {
continue;
}
JoinSetComposerPrototypeHistoricalDesc desc = historicalStreamIndexLists[historical.StreamNum]
.GetStrategy(historical.LookupStreamNum, statementRawInfo, services.SerdeResolver);
historical.HistoricalIndexLookupStrategy = desc.LookupForge;
historical.PollResultIndexingStrategy = desc.IndexingForge;
additionalForgeables.AddAll(desc.AdditionalForgeables);
}
}
// build strategy tree for putting the result back together
var assemblyTopNodeFactory = AssemblyStrategyTreeBuilder.Build(
streamNo,
substreamsPerStream,
requiredPerStream);
var assemblyInstructionFactories =
BaseAssemblyNodeFactory.GetDescendentNodesBottomUp(assemblyTopNodeFactory);
var forge = new LookupInstructionQueryPlanNodeForge(
streamNo,
streamNames[streamNo],
numStreams,
requiredPerStream,
lookupInstructions,
assemblyInstructionFactories);
return new QueryPlanNodeForgeDesc(forge, additionalForgeables);
}
