/// <summary>Get the strategies to use for polling from a given stream. </summary> /// <param name="streamViewStreamNum">the stream providing the polling events</param> /// <returns>looking and indexing strategy</returns> public Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy> GetStrategy(int streamViewStreamNum) { // If there is only a single polling stream, then build a single index if (_pollingStreams.Count == 1) { return(JoinSetComposerPrototypeFactory.DetermineIndexing(_queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[streamViewStreamNum], _historicalStreamNum, streamViewStreamNum)); } // If there are multiple polling streams, determine if a single index is appropriate. // An index can be reused if: // (a) indexed property names are the same // (b) indexed property types are the same // (c) key property types are the same (because of coercion) // A index lookup strategy is always specific to the providing stream. if (_indexesUsedByStreams == null) { _indexesUsedByStreams = new LinkedHashMap <HistoricalStreamIndexDesc, IList <int> >(); foreach (var pollingStream in _pollingStreams) { var queryGraphValue = _queryGraph.GetGraphValue(pollingStream, _historicalStreamNum); var hashKeyProps = queryGraphValue.HashKeyProps; var indexProperties = hashKeyProps.Indexed; var keyTypes = GetPropertyTypes(hashKeyProps.Keys); var indexTypes = GetPropertyTypes(_typesPerStream[_historicalStreamNum], indexProperties); var desc = new HistoricalStreamIndexDesc(indexProperties, indexTypes, keyTypes); var usedByStreams = _indexesUsedByStreams.Get(desc); if (usedByStreams == null) { usedByStreams = new List <int>(); _indexesUsedByStreams.Put(desc, usedByStreams); } usedByStreams.Add(pollingStream); } // There are multiple indexes required: // Build a master indexing strategy that forms multiple indexes and numbers each. if (_indexesUsedByStreams.Count > 1) { var numIndexes = _indexesUsedByStreams.Count; var indexingStrategies = new PollResultIndexingStrategy[numIndexes]; // create an indexing strategy for each index var count = 0; foreach (var desc in _indexesUsedByStreams) { var sampleStreamViewStreamNum = desc.Value[0]; indexingStrategies[count] = JoinSetComposerPrototypeFactory.DetermineIndexing(_queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[sampleStreamViewStreamNum], _historicalStreamNum, sampleStreamViewStreamNum).Second; count++; } // create a master indexing strategy that utilizes each indexing strategy to create a set of indexes var streamNum = streamViewStreamNum; _masterIndexingStrategy = new ProxyPollResultIndexingStrategy { ProcIndex = (pollResult, isActiveCache, statementContext) => { var tables = new EventTable[numIndexes]; for (var i = 0; i < numIndexes; i++) { tables[i] = indexingStrategies[i].Index(pollResult, isActiveCache, statementContext)[0]; } var organization = new EventTableOrganization(null, false, false, streamNum, null, EventTableOrganizationType.MULTIINDEX); return(new EventTable[] { new MultiIndexEventTable(tables, organization) }); }, ProcToQueryPlan = () => { var writer = new StringWriter(); var delimiter = ""; foreach (var strategy in indexingStrategies) { writer.Write(delimiter); writer.Write(strategy.ToQueryPlan()); delimiter = ", "; } return(GetType().FullName + " " + writer); } }; } } // there is one type of index if (_indexesUsedByStreams.Count == 1) { return(JoinSetComposerPrototypeFactory.DetermineIndexing( _queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[streamViewStreamNum], _historicalStreamNum, streamViewStreamNum)); } // determine which index number the polling stream must use var indexUsed = 0; var found = false; foreach (var desc in _indexesUsedByStreams.Values) { if (desc.Contains(streamViewStreamNum)) { found = true; break; } indexUsed++; } if (!found) { throw new IllegalStateException("MapIndex not found for use by stream " + streamViewStreamNum); } // Use one of the indexes built by the master index and a lookup strategy var indexNumber = indexUsed; HistoricalIndexLookupStrategy innerLookupStrategy = JoinSetComposerPrototypeFactory.DetermineIndexing(_queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[streamViewStreamNum], _historicalStreamNum, streamViewStreamNum).First; var lookupStrategy = new ProxyHistoricalIndexLookupStrategy { ProcLookup = (lookupEvent, index, context) => { var multiIndex = (MultiIndexEventTable)index[0]; var indexToUse = multiIndex.Tables[indexNumber]; return(innerLookupStrategy.Lookup(lookupEvent, new EventTable[] { indexToUse }, context)); }, ProcToQueryPlan = () => GetType().FullName + " inner: " + innerLookupStrategy.ToQueryPlan() }; return(new Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy> (lookupStrategy, _masterIndexingStrategy)); }
/// <summary> /// Constructs indexing and lookup strategy for a given relationship that a historical stream may have with another /// stream (historical or not) that looks up into results of a poll of a historical stream. /// <para />The term "polled" refers to the assumed-historical stream. /// </summary> /// <param name="queryGraph">relationship representation of where-clause filter and outer join on-expressions</param> /// <param name="polledViewType">the event type of the historical that is indexed</param> /// <param name="streamViewType">the event type of the stream looking up in indexes</param> /// <param name="polledViewStreamNum">the stream number of the historical that is indexed</param> /// <param name="streamViewStreamNum">the stream number of the historical that is looking up</param> /// <returns>indexing and lookup strategy pair</returns> public static Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy> DetermineIndexing(QueryGraph queryGraph, EventType polledViewType, EventType streamViewType, int polledViewStreamNum, int streamViewStreamNum) { var queryGraphValue = queryGraph.GetGraphValue(streamViewStreamNum, polledViewStreamNum); var hashKeysAndIndes = queryGraphValue.HashKeyProps; var rangeKeysAndIndex = queryGraphValue.RangeProps; // index and key property names var hashKeys = hashKeysAndIndes.Keys; var hashIndexes = hashKeysAndIndes.Indexed.ToArray(); var rangeKeys = rangeKeysAndIndex.Keys; var rangeIndexes = rangeKeysAndIndex.Indexed.ToArray(); // If the analysis revealed no join columns, must use the brute-force full table scan if (hashKeys.IsEmpty() && rangeKeys.IsEmpty()) { var inKeywordSingles = queryGraphValue.InKeywordSingles; if (inKeywordSingles != null && inKeywordSingles.Indexed.Length != 0) { var indexed = inKeywordSingles.Indexed[0]; var lookup = inKeywordSingles.Key[0]; var strategy = new HistoricalIndexLookupStrategyInKeywordSingle(streamViewStreamNum, lookup.KeyExprs); var indexing = new PollResultIndexingStrategyIndexSingle(polledViewStreamNum, polledViewType, indexed); return(new Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(strategy, indexing)); } var multis = queryGraphValue.InKeywordMulti; if (!multis.IsEmpty()) { var multi = multis[0]; var strategy = new HistoricalIndexLookupStrategyInKeywordMulti(streamViewStreamNum, multi.Key.KeyExpr); var indexing = new PollResultIndexingStrategyIndexSingleArray(polledViewStreamNum, polledViewType, ExprNodeUtility.GetIdentResolvedPropertyNames(multi.Indexed)); return(new Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(strategy, indexing)); } return(new Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy>( new HistoricalIndexLookupStrategyNoIndex(), new PollResultIndexingStrategyNoIndex())); } var keyCoercionTypes = CoercionUtil.GetCoercionTypesHash(new EventType[] { streamViewType, polledViewType }, 0, 1, hashKeys, hashIndexes); if (rangeKeys.IsEmpty()) { // No coercion if (!keyCoercionTypes.IsCoerce) { if (hashIndexes.Length == 1) { var indexing = new PollResultIndexingStrategyIndexSingle(polledViewStreamNum, polledViewType, hashIndexes[0]); HistoricalIndexLookupStrategy strategy = new HistoricalIndexLookupStrategyIndexSingle(streamViewStreamNum, hashKeys[0]); return(new Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(strategy, indexing)); } else { var indexing = new PollResultIndexingStrategyIndex(polledViewStreamNum, polledViewType, hashIndexes); HistoricalIndexLookupStrategy strategy = new HistoricalIndexLookupStrategyIndex(streamViewType, streamViewStreamNum, hashKeys); return(new Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(strategy, indexing)); } } // With coercion, same lookup strategy as the index coerces if (hashIndexes.Length == 1) { PollResultIndexingStrategy indexing = new PollResultIndexingStrategyIndexCoerceSingle(polledViewStreamNum, polledViewType, hashIndexes[0], keyCoercionTypes.CoercionTypes[0]); HistoricalIndexLookupStrategy strategy = new HistoricalIndexLookupStrategyIndexSingle(streamViewStreamNum, hashKeys[0]); return(new Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(strategy, indexing)); } else { PollResultIndexingStrategy indexing = new PollResultIndexingStrategyIndexCoerce(polledViewStreamNum, polledViewType, hashIndexes, keyCoercionTypes.CoercionTypes); HistoricalIndexLookupStrategy strategy = new HistoricalIndexLookupStrategyIndex(streamViewType, streamViewStreamNum, hashKeys); return(new Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(strategy, indexing)); } } else { var rangeCoercionTypes = CoercionUtil.GetCoercionTypesRange(new EventType[] { streamViewType, polledViewType }, 1, rangeIndexes, rangeKeys); if (rangeKeys.Count == 1 && hashKeys.Count == 0) { var rangeCoercionType = rangeCoercionTypes.IsCoerce ? rangeCoercionTypes.CoercionTypes[0] : null; var indexing = new PollResultIndexingStrategySorted(polledViewStreamNum, polledViewType, rangeIndexes[0], rangeCoercionType); HistoricalIndexLookupStrategy strategy = new HistoricalIndexLookupStrategySorted(streamViewStreamNum, rangeKeys[0]); return(new Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(strategy, indexing)); } else { var indexing = new PollResultIndexingStrategyComposite(polledViewStreamNum, polledViewType, hashIndexes, keyCoercionTypes.CoercionTypes, rangeIndexes, rangeCoercionTypes.CoercionTypes); HistoricalIndexLookupStrategy strategy = new HistoricalIndexLookupStrategyComposite(streamViewStreamNum, hashKeys, keyCoercionTypes.CoercionTypes, rangeKeys, rangeCoercionTypes.CoercionTypes); return(new Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy>(strategy, indexing)); } } }
/// <summary> /// The snapshot /// </summary> /// <param name="queryGraph">The <see cref="QueryGraph"/></param> /// <param name="attributes">The <see cref="Attribute" /> array</param> /// <param name="virtualDataWindow">The <see cref="VirtualDWView"/></param> /// <param name="indexRepository">The <see cref="EventTableIndexRepository"/></param> /// <param name="queryPlanLogging">The <see cref="bool"/></param> /// <param name="queryPlanLogDestination">The <see cref="ILog"/></param> /// <param name="objectName">The <see cref="string"/></param> /// <param name="agentInstanceContext">The <see cref="AgentInstanceContext"/></param> /// <returns>The <see cref="ICollection{EventBean}"/></returns> public static ICollection <EventBean> Snapshot( QueryGraph queryGraph, Attribute[] attributes, VirtualDWView virtualDataWindow, EventTableIndexRepository indexRepository, bool queryPlanLogging, ILog queryPlanLogDestination, string objectName, AgentInstanceContext agentInstanceContext) { var queryGraphValue = queryGraph == null ? null : queryGraph.GetGraphValue(QueryGraph.SELF_STREAM, 0); if (queryGraphValue == null || queryGraphValue.Items.IsEmpty()) { if (virtualDataWindow != null) { var pair = virtualDataWindow.GetFireAndForgetDesc(Collections.GetEmptySet <string>(), Collections.GetEmptySet <string>()); return(virtualDataWindow.GetFireAndForgetData(pair.Second, new Object[0], new RangeIndexLookupValue[0], attributes)); } return(null); } // determine custom index var customResult = SnapshotCustomIndex( queryGraphValue, indexRepository, attributes, agentInstanceContext, queryPlanLogging, queryPlanLogDestination, objectName); if (customResult != null) { return(customResult.Value); } // determine lookup based on hash-keys and ranges var keysAvailable = queryGraphValue.HashKeyProps; var keyNamesAvailable = keysAvailable.Indexed.Count == 0 ? Collections.GetEmptySet <string>() : new HashSet <string>(keysAvailable.Indexed); var rangesAvailable = queryGraphValue.RangeProps; var rangeNamesAvailable = rangesAvailable.Indexed.Count == 0 ? Collections.GetEmptySet <string>() : new HashSet <string>(rangesAvailable.Indexed); // find index that matches the needs var tablePair = FindIndex(keyNamesAvailable, rangeNamesAvailable, indexRepository, virtualDataWindow, attributes); // regular index lookup if (tablePair != null) { return(SnapshotIndex(keysAvailable, rangesAvailable, tablePair, virtualDataWindow, attributes, agentInstanceContext, queryPlanLogging, queryPlanLogDestination, objectName)); } // in-keyword lookup var inkwResult = SnapshotInKeyword(queryGraphValue, indexRepository, virtualDataWindow, attributes, agentInstanceContext, queryPlanLogging, queryPlanLogDestination, objectName); if (inkwResult != null) { return(inkwResult.Value); } QueryPlanReportTableScan(attributes, agentInstanceContext, queryPlanLogging, queryPlanLogDestination, objectName); return(null); }
public static ICollection<EventBean> Snapshot( QueryGraph filterQueryGraph, Attribute[] annotations, VirtualDWView virtualDataWindow, EventTableIndexRepository indexRepository, string objectName, AgentInstanceContext agentInstanceContext) { var queryGraphValue = filterQueryGraph == null ? null : filterQueryGraph.GetGraphValue(QueryGraphForge.SELF_STREAM, 0); if (queryGraphValue == null || queryGraphValue.Items.IsEmpty()) { if (virtualDataWindow != null) { Pair<IndexMultiKey, EventTable> pair = VirtualDWQueryPlanUtil.GetFireAndForgetDesc( virtualDataWindow.EventType, new EmptySet<string>(), new EmptySet<string>()); return virtualDataWindow.GetFireAndForgetData( pair.Second, CollectionUtil.OBJECTARRAY_EMPTY, new RangeIndexLookupValue[0], annotations); } return null; } // determine custom index var customResult = SnapshotCustomIndex( queryGraphValue, indexRepository, annotations, agentInstanceContext, objectName); if (customResult != null) { return customResult.Value; } // determine lookup based on hash-keys and ranges var keysAvailable = queryGraphValue.HashKeyProps; ISet<string> keyNamesAvailable = keysAvailable.Indexed.Length == 0 ? (ISet<string>) new EmptySet<string>() : (ISet<string>) new HashSet<string>(keysAvailable.Indexed); var rangesAvailable = queryGraphValue.RangeProps; ISet<string> rangeNamesAvailable = rangesAvailable.Indexed.Length == 0 ? (ISet<string>) new EmptySet<string>() : (ISet<string>) new HashSet<string>(rangesAvailable.Indexed); Pair<IndexMultiKey, EventTableAndNamePair> tablePair; // find index that matches the needs tablePair = FindIndex( keyNamesAvailable, rangeNamesAvailable, indexRepository, virtualDataWindow, annotations); // regular index lookup if (tablePair != null) { return SnapshotIndex( keysAvailable, rangesAvailable, tablePair, virtualDataWindow, annotations, agentInstanceContext, objectName); } // in-keyword lookup var inkwResult = SnapshotInKeyword( queryGraphValue, indexRepository, virtualDataWindow, annotations, agentInstanceContext, objectName); if (inkwResult != null) { return inkwResult.Value; } QueryPlanReportTableScan(annotations, agentInstanceContext, objectName); return null; }