public SubordTableLookupStrategy MakeStrategy(EventTable[] eventTable, VirtualDWView vdw)
{
Pair <IndexMultiKey, EventTable> tableVW = vdw.GetSubordinateQueryDesc(false, hashAndRanges.HashedProps, hashAndRanges.BtreeProps);
return(vdw.GetSubordinateLookupStrategy(statementName,
statementId, annotations,
outerStreams, hashKeys, hashKeyCoercionTypes, rangeKeys, rangeKeyCoercionTypes, nwOnTrigger,
tableVW.Second, joinDesc, forceTableScan));
}
public SubordWMatchExprLookupStrategy Realize(
EventTable[] indexes,
AgentInstanceContext agentInstanceContext,
IEnumerable<EventBean> scanIterable,
VirtualDWView virtualDataWindow)
{
var strategy = OptionalInnerStrategy.MakeStrategy(indexes, agentInstanceContext, virtualDataWindow);
return new SubordWMatchExprLookupStrategyIndexedFiltered(exprEvaluator, strategy);
}
public NamedWindowConsumerView AddConsumer(NamedWindowConsumerDesc consumerDesc, bool isSubselect)
{
var consumerCallback = new ProxyNamedWindowConsumerCallback()
{
ProcGetEnumerator = () => {
NamedWindowProcessorInstance instance = _namedWindowProcessor.GetProcessorInstance(_agentInstanceContext);
if (instance == null)
{
// this can happen on context-partition "output when terminated"
return(GetEnumerator());
}
return(instance.TailViewInstance.GetEnumerator());
},
ProcStopped = (namedWindowConsumerView) => {
RemoveConsumer(namedWindowConsumerView);
}
};
// Construct consumer view, allow a callback to this view to remove the consumer
bool audit = AuditEnum.STREAM.GetAudit(consumerDesc.AgentInstanceContext.StatementContext.Annotations) != null;
var consumerView = new NamedWindowConsumerView(ExprNodeUtility.GetEvaluators(consumerDesc.FilterList), consumerDesc.OptPropertyEvaluator, _tailView.EventType, consumerCallback, consumerDesc.AgentInstanceContext, audit);
// indicate to virtual data window that a consumer was added
VirtualDWView virtualDWView = _rootViewInstance.VirtualDataWindow;
if (virtualDWView != null)
{
virtualDWView.VirtualDataWindow.HandleEvent(
new VirtualDataWindowEventConsumerAdd(_tailView.EventType.Name, consumerView, consumerDesc.AgentInstanceContext.StatementName, consumerDesc.AgentInstanceContext.AgentInstanceId, ExprNodeUtility.ToArray(consumerDesc.FilterList), _agentInstanceContext));
}
// Keep a list of consumer views per statement to accommodate joins and subqueries
var viewsPerStatements = _consumersInContext.Get(consumerDesc.AgentInstanceContext.EpStatementAgentInstanceHandle);
if (viewsPerStatements == null)
{
viewsPerStatements = new CopyOnWriteList <NamedWindowConsumerView>();
// avoid concurrent modification as a thread may currently iterate over consumers as its dispatching
// without the engine lock
var newConsumers = NamedWindowUtil.CreateConsumerMap(_tailView.IsPrioritized);
newConsumers.PutAll(_consumersInContext);
newConsumers.Put(consumerDesc.AgentInstanceContext.EpStatementAgentInstanceHandle, viewsPerStatements);
_consumersInContext = newConsumers;
}
if (isSubselect)
{
viewsPerStatements.Insert(0, consumerView);
}
else
{
viewsPerStatements.Add(consumerView);
}
return(consumerView);
}
public SubordTableLookupStrategy MakeStrategy(
EventTable[] eventTable,
AgentInstanceContext agentInstanceContext,
VirtualDWView vdw)
{
if (isNWOnTrigger) {
return new SubordInKeywordSingleTableLookupStrategyNW(this, (PropertyHashedEventTable) eventTable[0]);
}
return new SubordInKeywordSingleTableLookupStrategy(this, (PropertyHashedEventTable) eventTable[0]);
}
public SubordTableLookupStrategy MakeStrategy(EventTable[] eventTable, VirtualDWView vdw)
{
if (eventTable[0] is PropertyIndexedEventTableSingleUnique)
{
return(new SubordIndexedTableLookupStrategySinglePropUnique(_keyStreamNum, _propertyGetter, (PropertyIndexedEventTableSingleUnique)eventTable[0],
new LookupStrategyDesc(LookupStrategyType.SINGLEPROPUNIQUE, new String[] { _property })));
}
LookupStrategyDesc desc = new LookupStrategyDesc(LookupStrategyType.SINGLEPROPNONUNIQUE, new String[] { _property });
return(new SubordIndexedTableLookupStrategySingleProp(_keyStreamNum, _propertyGetter, (PropertyIndexedEventTableSingle)eventTable[0], desc));
}
public SubordTableLookupStrategy MakeStrategy(EventTable[] eventTable, VirtualDWView vdw)
{
if (IsNWOnTrigger)
{
return(new SubordInKeywordSingleTableLookupStrategyNW(Evaluators, (PropertyIndexedEventTableSingle)eventTable[0], StrategyDesc));
}
else
{
return(new SubordInKeywordSingleTableLookupStrategy(StreamCountOuter, Evaluators, (PropertyIndexedEventTableSingle)eventTable[0], StrategyDesc));
}
}
public SubordTableLookupStrategy MakeStrategy(EventTable[] eventTable, VirtualDWView vdw)
{
if (IsNWOnTrigger)
{
return(new SubordInKeywordMultiTableLookupStrategyNW(Evaluator, eventTable, StrategyDesc));
}
else
{
return(new SubordInKeywordMultiTableLookupStrategy(StreamCountOuter, Evaluator, eventTable, StrategyDesc));
}
}
/// <summary>
/// Return a snapshot using index lookup filters.
/// </summary>
/// <param name="optionalFilter">to index lookup</param>
/// <returns>events</returns>
public ICollection <EventBean> Snapshot(FilterSpecCompiled optionalFilter, Attribute[] annotations)
{
VirtualDWView virtualDataWindow = null;
if (IsVirtualDataWindow)
{
virtualDataWindow = VirtualDataWindow;
}
return(FireAndForgetQueryExec.Snapshot(optionalFilter, annotations, virtualDataWindow,
_indexRepository, _rootView.IsQueryPlanLogging, NamedWindowRootView.QueryPlanLog,
_rootView.EventType.Name, _agentInstanceContext));
}
public SubordTableLookupStrategy MakeStrategy(
EventTable[] eventTable,
AgentInstanceContext agentInstanceContext,
VirtualDWView vdw)
{
if (IsNwOnTrigger) {
return new SubordTableLookupStrategyQuadTreeNW((EventTableQuadTree) eventTable[0], this);
}
return new SubordTableLookupStrategyQuadTreeSubq(
(EventTableQuadTree) eventTable[0],
this,
StreamCountOuter);
}
/// <summary>
/// The FafTableLookup
/// </summary>
/// <param name="virtualDataWindow">The <see cref="VirtualDWView"/></param>
/// <param name="indexMultiKey">The <see cref="IndexMultiKey"/></param>
/// <param name="eventTable">The <see cref="EventTable"/></param>
/// <param name="keyValues">The <see cref="Object"/> array</param>
/// <param name="rangeValues">The <see cref="RangeIndexLookupValue" /> array</param>
/// <param name="attributes">The <see cref="Attribute" /> array</param>
/// <returns>The <see cref="ICollection{EventBean}"/></returns>
private static ICollection <EventBean> FafTableLookup(
VirtualDWView virtualDataWindow,
IndexMultiKey indexMultiKey,
EventTable eventTable,
Object[] keyValues,
RangeIndexLookupValue[] rangeValues,
Attribute[] attributes)
{
if (virtualDataWindow != null)
{
return(virtualDataWindow.GetFireAndForgetData(eventTable, keyValues, rangeValues, attributes));
}
ISet <EventBean> result;
if (indexMultiKey.HashIndexedProps.Length > 0 && indexMultiKey.RangeIndexedProps.Length == 0)
{
if (indexMultiKey.HashIndexedProps.Length == 1)
{
var table = (PropertyIndexedEventTableSingle)eventTable;
result = table.Lookup(keyValues[0]);
}
else
{
var table = (PropertyIndexedEventTable)eventTable;
result = table.Lookup(keyValues);
}
}
else if (indexMultiKey.HashIndexedProps.Length == 0 && indexMultiKey.RangeIndexedProps.Length == 1)
{
var table = (PropertySortedEventTable)eventTable;
result = table.LookupConstants(rangeValues[0]);
}
else
{
var table = (PropertyCompositeEventTable)eventTable;
var rangeCoercion = table.OptRangeCoercedTypes;
var lookup = CompositeIndexLookupFactory.Make(keyValues, rangeValues, rangeCoercion);
result = new HashSet <EventBean>();
lookup.Lookup(table.MapIndex, result, table.PostProcessor);
}
if (result != null)
{
return(result);
}
return(Collections.GetEmptyList <EventBean>());
}
public SubordTableLookupStrategy MakeStrategy(
EventTable[] eventTable,
AgentInstanceContext agentInstanceContext,
VirtualDWView vdw)
{
var indexes = new PropertyHashedEventTable[eventTable.Length];
for (var i = 0; i < eventTable.Length; i++) {
indexes[i] = (PropertyHashedEventTable) eventTable[i];
}
if (isNWOnTrigger) {
return new SubordInKeywordMultiTableLookupStrategyNW(this, indexes);
}
return new SubordInKeywordMultiTableLookupStrategy(this, indexes);
}
/// <summary>
/// Return a snapshot using index lookup filters.
/// </summary>
/// <param name="annotations">annotations</param>
/// <param name="queryGraph">query graph</param>
/// <returns>events</returns>
public ICollection<EventBean> Snapshot(
QueryGraph queryGraph,
Attribute[] annotations)
{
VirtualDWView virtualDataWindow = null;
if (IsVirtualDataWindow) {
virtualDataWindow = VirtualDataWindow;
}
return FireAndForgetQueryExec.Snapshot(
queryGraph,
annotations,
virtualDataWindow,
IndexRepository,
rootView.EventType.Name,
AgentInstanceContext);
}
/// <summary>
/// The FindIndex
/// </summary>
/// <param name="keyNamesAvailable">The key names available.</param>
/// <param name="rangeNamesAvailable">The range names available.</param>
/// <param name="indexRepository">The <see cref="EventTableIndexRepository" /></param>
/// <param name="virtualDataWindow">The <see cref="VirtualDWView" /></param>
/// <param name="attributes">The <see cref="Attribute" /> array</param>
/// <returns></returns>
private static Pair <IndexMultiKey, EventTableAndNamePair> FindIndex(
ISet <string> keyNamesAvailable,
ISet <string> rangeNamesAvailable,
EventTableIndexRepository indexRepository,
VirtualDWView virtualDataWindow,
Attribute[] attributes)
{
if (virtualDataWindow != null)
{
var tablePairNoName = virtualDataWindow.GetFireAndForgetDesc(keyNamesAvailable, rangeNamesAvailable);
return(new Pair <IndexMultiKey, EventTableAndNamePair>(tablePairNoName.First, new EventTableAndNamePair(tablePairNoName.Second, null)));
}
var indexHint = IndexHint.GetIndexHint(attributes);
var optionalIndexHintInstructions = indexHint != null ? indexHint.InstructionsFireAndForget : null;
return(indexRepository.FindTable(keyNamesAvailable, rangeNamesAvailable, optionalIndexHintInstructions));
}
public StatementAgentInstanceFactoryResult NewContext(AgentInstanceContext agentInstanceContext, bool isRecoveringResilient)
{
StopCallback stopCallback;
if (namedWindowProcessor != null)
{
// handle named window index
NamedWindowProcessorInstance processorInstance = namedWindowProcessor.GetProcessorInstance(agentInstanceContext);
if (namedWindowProcessor.IsVirtualDataWindow)
{
VirtualDWView virtualDWView = processorInstance.RootViewInstance.VirtualDataWindow;
virtualDWView.HandleStartIndex(spec);
stopCallback = () => {
virtualDWView.HandleStopIndex(spec);
};
}
else
{
try {
processorInstance.RootViewInstance.AddExplicitIndex(spec.IsUnique, spec.IndexName, spec.Columns);
}
catch (ExprValidationException e) {
throw new EPException("Failed to create index: " + e.Message, e);
}
stopCallback = () => {
};
}
}
else
{
// handle table access
try {
TableStateInstance instance = services.TableService.GetState(tableName, agentInstanceContext.AgentInstanceId);
instance.AddExplicitIndex(spec);
}
catch (ExprValidationException ex) {
throw new EPException("Failed to create index: " + ex.Message, ex);
}
stopCallback = () => {
};
}
return(new StatementAgentInstanceFactoryCreateIndexResult(finalView, stopCallback, agentInstanceContext));
}
private static Pair<IndexMultiKey, EventTableAndNamePair> FindIndex(
ISet<string> keyNamesAvailable,
ISet<string> rangeNamesAvailable,
EventTableIndexRepository indexRepository,
VirtualDWView virtualDataWindow,
Attribute[] annotations)
{
if (virtualDataWindow != null) {
var tablePairNoName = VirtualDWQueryPlanUtil.GetFireAndForgetDesc(
virtualDataWindow.EventType,
keyNamesAvailable,
rangeNamesAvailable);
return new Pair<IndexMultiKey, EventTableAndNamePair>(
tablePairNoName.First,
new EventTableAndNamePair(tablePairNoName.Second, null));
}
var indexHint = IndexHint.GetIndexHint(annotations);
IList<IndexHintInstruction> optionalIndexHintInstructions =
indexHint != null ? indexHint.GetInstructionsFireAndForget() : null;
return indexRepository.FindTable(keyNamesAvailable, rangeNamesAvailable, optionalIndexHintInstructions);
}
private static ICollection<EventBean> FafTableLookup(
VirtualDWView virtualDataWindow,
IndexMultiKey indexMultiKey,
EventTable eventTable,
object[] keyValues,
RangeIndexLookupValue[] rangeValues,
Attribute[] annotations,
AgentInstanceContext agentInstanceContext)
{
if (virtualDataWindow != null) {
return virtualDataWindow.GetFireAndForgetData(eventTable, keyValues, rangeValues, annotations);
}
ISet<EventBean> result;
if (indexMultiKey.HashIndexedProps.Length > 0 && indexMultiKey.RangeIndexedProps.Length == 0) {
var table = (PropertyHashedEventTable) eventTable;
var lookupKey = table.MultiKeyTransform.From(keyValues);
result = table.Lookup(lookupKey);
}
else if (indexMultiKey.HashIndexedProps.Length == 0 && indexMultiKey.RangeIndexedProps.Length == 1) {
var table = (PropertySortedEventTable) eventTable;
result = table.LookupConstants(rangeValues[0]);
}
else {
var table = (PropertyCompositeEventTable) eventTable;
var rangeCoercion = table.OptRangeCoercedTypes;
var lookup = CompositeIndexLookupFactory.Make(keyValues, table.MultiKeyTransform, rangeValues, rangeCoercion);
result = new HashSet<EventBean>();
lookup.Lookup(table.Index, result, table.PostProcessor);
}
if (result != null) {
return result;
}
return EmptyList<EventBean>.Instance;
}
public SubordTableLookupStrategy MakeStrategy(EventTable[] eventTable, VirtualDWView vdw)
{
return(new SubordIndexedTableLookupStrategyProp(_keyStreamNums, _propertyGetters, (PropertyIndexedEventTable)eventTable[0], _strategyDesc));
}
public SubordWMatchExprLookupStrategy Realize(EventTable[] indexes, AgentInstanceContext agentInstanceContext, IEnumerable <EventBean> scanIterable, VirtualDWView virtualDataWindow)
{
return(new SubordWMatchExprLookupStrategyAllFiltered(_exprEvaluator, scanIterable));
}
public static ICollection <EventBean> Snapshot(
FilterSpecCompiled optionalFilter,
Attribute[] annotations,
VirtualDWView virtualDataWindow,
EventTableIndexRepository indexRepository,
bool queryPlanLogging,
ILog queryPlanLogDestination,
string objectName,
AgentInstanceContext agentInstanceContext)
{
if (optionalFilter == null || optionalFilter.Parameters.Length == 0)
{
if (virtualDataWindow != null)
{
var pair = virtualDataWindow.GetFireAndForgetDesc(Collections.GetEmptySet <string>(), Collections.GetEmptySet <string>());
return(virtualDataWindow.GetFireAndForgetData(pair.Second, new object[0], new RangeIndexLookupValue[0], annotations));
}
return(null);
}
// Determine what straight-equals keys and which ranges are available.
// Widening/Coercion is part of filter spec compile.
ISet <string> keysAvailable = new HashSet <string>();
ISet <string> rangesAvailable = new HashSet <string>();
if (optionalFilter.Parameters.Length == 1)
{
foreach (FilterSpecParam param in optionalFilter.Parameters[0])
{
if (!(param is FilterSpecParamConstant ||
param is FilterSpecParamRange ||
param is FilterSpecParamIn))
{
continue;
}
if (param.FilterOperator == FilterOperator.EQUAL ||
param.FilterOperator == FilterOperator.IS ||
param.FilterOperator == FilterOperator.IN_LIST_OF_VALUES)
{
keysAvailable.Add(param.Lookupable.Expression);
}
else if (param.FilterOperator.IsRangeOperator() ||
param.FilterOperator.IsInvertedRangeOperator() ||
param.FilterOperator.IsComparisonOperator())
{
rangesAvailable.Add(param.Lookupable.Expression);
}
else if (param.FilterOperator.IsRangeOperator())
{
rangesAvailable.Add(param.Lookupable.Expression);
}
}
}
// Find an index that matches the needs
Pair <IndexMultiKey, EventTableAndNamePair> tablePair;
if (virtualDataWindow != null)
{
var tablePairNoName = virtualDataWindow.GetFireAndForgetDesc(keysAvailable, rangesAvailable);
tablePair = new Pair <IndexMultiKey, EventTableAndNamePair>(tablePairNoName.First, new EventTableAndNamePair(tablePairNoName.Second, null));
}
else
{
var indexHint = IndexHint.GetIndexHint(annotations);
IList <IndexHintInstruction> optionalIndexHintInstructions = null;
if (indexHint != null)
{
optionalIndexHintInstructions = indexHint.InstructionsFireAndForget;
}
tablePair = indexRepository.FindTable(keysAvailable, rangesAvailable, optionalIndexHintInstructions);
}
var hook = QueryPlanIndexHookUtil.GetHook(annotations);
if (queryPlanLogging && (queryPlanLogDestination.IsInfoEnabled || hook != null))
{
var prefix = "Fire-and-forget from " + objectName + " ";
var indexName = tablePair != null && tablePair.Second != null ? tablePair.Second.IndexName : null;
var indexText = indexName != null ? "index " + indexName + " " : "full table scan ";
indexText += "(snapshot only, for join see separate query plan)";
if (tablePair == null)
{
queryPlanLogDestination.Info(prefix + indexText);
}
else
{
queryPlanLogDestination.Info(prefix + indexText + tablePair.Second.EventTable.ToQueryPlan());
}
if (hook != null)
{
hook.FireAndForget(new QueryPlanIndexDescFAF(
new IndexNameAndDescPair[] {
new IndexNameAndDescPair(indexName, tablePair != null ?
tablePair.Second.EventTable.ProviderClass.Name : null)
}));
}
}
if (tablePair == null)
{
return(null); // indicates table scan
}
// Compile key sets which contain key index lookup values
var keyIndexProps = IndexedPropDesc.GetIndexProperties(tablePair.First.HashIndexedProps);
var hasKeyWithInClause = false;
var keyValues = new object[keyIndexProps.Length];
for (var keyIndex = 0; keyIndex < keyIndexProps.Length; keyIndex++)
{
foreach (var param in optionalFilter.Parameters[0])
{
if (param.Lookupable.Expression.Equals(keyIndexProps[keyIndex]))
{
if (param.FilterOperator == FilterOperator.IN_LIST_OF_VALUES)
{
var keyValuesList = ((MultiKeyUntyped)param.GetFilterValue(null, agentInstanceContext)).Keys;
if (keyValuesList.Length == 0)
{
continue;
}
else if (keyValuesList.Length == 1)
{
keyValues[keyIndex] = keyValuesList[0];
}
else
{
keyValues[keyIndex] = keyValuesList;
hasKeyWithInClause = true;
}
}
else
{
keyValues[keyIndex] = param.GetFilterValue(null, agentInstanceContext);
}
break;
}
}
}
// Analyze ranges - these may include key lookup value (EQUALS semantics)
var rangeIndexProps = IndexedPropDesc.GetIndexProperties(tablePair.First.RangeIndexedProps);
RangeIndexLookupValue[] rangeValues;
if (rangeIndexProps.Length > 0)
{
rangeValues = CompileRangeLookupValues(rangeIndexProps, optionalFilter.Parameters[0], agentInstanceContext);
}
else
{
rangeValues = new RangeIndexLookupValue[0];
}
var eventTable = tablePair.Second.EventTable;
var indexMultiKey = tablePair.First;
// table lookup without in-clause
if (!hasKeyWithInClause)
{
return(FafTableLookup(virtualDataWindow, indexMultiKey, eventTable, keyValues, rangeValues, annotations));
}
// table lookup with in-clause: determine combinations
var combinations = new object[keyIndexProps.Length][];
for (var i = 0; i < keyValues.Length; i++)
{
if (keyValues[i] is object[])
{
combinations[i] = (object[])keyValues[i];
}
else
{
combinations[i] = new object[] { keyValues[i] };
}
}
// enumerate combinations
var enumeration = new CombinationEnumeration(combinations);
var events = new HashSet <EventBean>();
for (; enumeration.MoveNext();)
{
object[] keys = enumeration.Current;
var result = FafTableLookup(virtualDataWindow, indexMultiKey, eventTable, keys, rangeValues, annotations);
events.AddAll(result);
}
return(events);
}
public SubordTableLookupStrategy MakeStrategy(EventTable[] eventTable, VirtualDWView vdw)
{
return(new SubordSortedTableLookupStrategy(_strategy, (PropertySortedEventTable)eventTable[0], _strategyDesc));
}
/// <summary>
/// The SnapshotInKeyword
/// </summary>
/// <param name="queryGraphValue">The <see cref="QueryGraphValue"/></param>
/// <param name="indexRepository">The <see cref="EventTableIndexRepository"/></param>
/// <param name="virtualDataWindow">The <see cref="VirtualDWView"/></param>
/// <param name="attributes">The <see cref="Attribute" /> array</param>
/// <param name="agentInstanceContext">The <see cref="AgentInstanceContext"/></param>
/// <param name="queryPlanLogging">The <see cref="bool"/></param>
/// <param name="queryPlanLogDestination">The <see cref="ILog"/></param>
/// <param name="objectName">The <see cref="string"/></param>
/// <returns>The collection of event beans</returns>
private static NullableObject <ICollection <EventBean> > SnapshotInKeyword(
QueryGraphValue queryGraphValue,
EventTableIndexRepository indexRepository,
VirtualDWView virtualDataWindow,
Attribute[] attributes,
AgentInstanceContext agentInstanceContext,
bool queryPlanLogging,
ILog queryPlanLogDestination,
string objectName)
{
var inkwSingles = queryGraphValue.InKeywordSingles;
if (inkwSingles.Indexed.Length == 0)
{
return(null);
}
var tablePair = FindIndex(
new HashSet <string>(inkwSingles.Indexed),
Collections.GetEmptySet <string>(),
indexRepository,
virtualDataWindow,
attributes);
if (tablePair == null)
{
return(null);
}
queryPlanReport(tablePair.Second.IndexName, tablePair.Second.EventTable, attributes, agentInstanceContext, queryPlanLogging, queryPlanLogDestination, objectName);
var evaluateParamsTrue = new EvaluateParams(null, true, agentInstanceContext);
// table lookup with in-clause: determine combinations
var tableHashProps = tablePair.First.HashIndexedProps;
var combinations = new Object[tableHashProps.Length][];
for (var tableHashPropNum = 0; tableHashPropNum < tableHashProps.Length; tableHashPropNum++)
{
for (var i = 0; i < inkwSingles.Indexed.Length; i++)
{
if (inkwSingles.Indexed[i].Equals(tableHashProps[tableHashPropNum].IndexPropName))
{
var keysExpressions = inkwSingles.Key[i];
var values = new Object[keysExpressions.KeyExprs.Count];
combinations[tableHashPropNum] = values;
for (var j = 0; j < keysExpressions.KeyExprs.Count; j++)
{
values[j] = keysExpressions.KeyExprs[j].ExprEvaluator.Evaluate(evaluateParamsTrue);
}
}
}
}
// enumerate combinations
var enumeration = CombinationEnumeration.New(combinations);
var events = new HashSet <EventBean>();
foreach (Object[] keys in enumeration)
{
var result = FafTableLookup(virtualDataWindow, tablePair.First, tablePair.Second.EventTable, keys, null, attributes);
events.AddAll(result);
}
return(new NullableObject <ICollection <EventBean> >(events));
}
public StatementAgentInstanceFactoryResult NewContext(AgentInstanceContext agentInstanceContext, bool isRecoveringResilient)
{
StopCallback stopCallback;
int agentInstanceId = agentInstanceContext.AgentInstanceId;
if (namedWindowProcessor != null)
{
// handle named window index
NamedWindowProcessorInstance processorInstance = namedWindowProcessor.GetProcessorInstance(agentInstanceContext);
if (namedWindowProcessor.IsVirtualDataWindow)
{
VirtualDWView virtualDWView = processorInstance.RootViewInstance.VirtualDataWindow;
virtualDWView.HandleStartIndex(spec);
stopCallback = new ProxyStopCallback(() => virtualDWView.HandleStopIndex(spec));
}
else
{
try {
processorInstance.RootViewInstance.AddExplicitIndex(spec.IsUnique, spec.IndexName, spec.Columns, isRecoveringResilient);
}
catch (ExprValidationException e) {
throw new EPException("Failed to create index: " + e.Message, e);
}
stopCallback = new ProxyStopCallback(() => {
// we remove the index when context partitioned.
// when not context partition the index gets removed when the last reference to the named window gets destroyed.
if (contextName != null)
{
var instance = namedWindowProcessor.GetProcessorInstance(agentInstanceId);
if (instance != null)
{
instance.RemoveExplicitIndex(spec.IndexName);
}
}
});
}
}
else
{
// handle table access
try {
TableStateInstance instance = services.TableService.GetState(tableName, agentInstanceContext.AgentInstanceId);
instance.AddExplicitIndex(spec, isRecoveringResilient, contextName != null);
}
catch (ExprValidationException ex) {
throw new EPException("Failed to create index: " + ex.Message, ex);
}
stopCallback = new ProxyStopCallback(() => {
// we remove the index when context partitioned.
// when not context partition the index gets removed when the last reference to the table gets destroyed.
if (contextName != null)
{
TableStateInstance instance = services.TableService.GetState(tableName, agentInstanceId);
if (instance != null)
{
instance.RemoveExplicitIndex(spec.IndexName);
}
}
});
}
return(new StatementAgentInstanceFactoryCreateIndexResult(finalView, stopCallback, agentInstanceContext));
}
public SubordWMatchExprLookupStrategy Realize(EventTable[] indexes, AgentInstanceContext agentInstanceContext, IEnumerable <EventBean> scanIterable, VirtualDWView virtualDataWindow)
{
SubordTableLookupStrategy strategy = _lookupStrategyFactory.MakeStrategy(indexes, virtualDataWindow);
return(new SubordWMatchExprLookupStrategyIndexedUnfiltered(strategy));
}
public virtual SubordTableLookupStrategy MakeStrategy(EventTable[] eventTable, VirtualDWView vdw)
{
if (_isNWOnTrigger)
{
return(new SubordIndexedTableLookupStrategyExprNW(_evaluators, (PropertyIndexedEventTable)eventTable[0], _strategyDesc));
}
else
{
return(new SubordIndexedTableLookupStrategyExpr(_numStreamsOuter, _evaluators, (PropertyIndexedEventTable)eventTable[0], _strategyDesc));
}
}
public JoinSetComposerDesc Create(Viewable[] streamViews, bool isFireAndForget, AgentInstanceContext agentInstanceContext)
{
// Build indexes
var indexesPerStream = new IDictionary <TableLookupIndexReqKey, EventTable> [_indexSpecs.Length];
var tableSecondaryIndexLocks = new ILockable[_indexSpecs.Length];
var hasTable = false;
for (var streamNo = 0; streamNo < _indexSpecs.Length; streamNo++)
{
if (_indexSpecs[streamNo] == null)
{
continue;
}
var items = _indexSpecs[streamNo].Items;
indexesPerStream[streamNo] = new LinkedHashMap <TableLookupIndexReqKey, EventTable>();
if (_streamJoinAnalysisResult.TablesPerStream[streamNo] != null)
{
// build for tables
var metadata = _streamJoinAnalysisResult.TablesPerStream[streamNo];
var state = _tableService.GetState(metadata.TableName, agentInstanceContext.AgentInstanceId);
foreach (var indexName in state.SecondaryIndexes) // add secondary indexes
{
var eventTable = state.GetIndex(indexName);
indexesPerStream[streamNo].Put(new TableLookupIndexReqKey(indexName, metadata.TableName), eventTable);
}
var index = state.GetIndex(metadata.TableName); // add primary index
indexesPerStream[streamNo].Put(new TableLookupIndexReqKey(metadata.TableName, metadata.TableName), index);
hasTable = true;
tableSecondaryIndexLocks[streamNo] = agentInstanceContext.StatementContext.IsWritesToTables
? state.TableLevelRWLock.WriteLock
: state.TableLevelRWLock.ReadLock;
}
else
{
// build tables for implicit indexes
foreach (var entry in items)
{
EventTable index;
if (_streamJoinAnalysisResult.ViewExternal[streamNo] != null)
{
var view = _streamJoinAnalysisResult.ViewExternal[streamNo].Invoke(agentInstanceContext);
index = view.GetJoinIndexTable(items.Get(entry.Key));
}
else
{
index = EventTableUtil.BuildIndex(streamNo, items.Get(entry.Key), _streamTypes[streamNo], false, entry.Value.IsUnique, null);
}
indexesPerStream[streamNo].Put(entry.Key, index);
}
}
}
// obtain any external views
var externalViewProviders = _streamJoinAnalysisResult.ViewExternal;
var externalViews = new VirtualDWView[externalViewProviders.Length];
for (var i = 0; i < externalViews.Length; i++)
{
if (externalViewProviders[i] != null)
{
externalViews[i] = _streamJoinAnalysisResult.ViewExternal[i].Invoke(agentInstanceContext);
}
}
// Build strategies
var queryExecSpecs = _queryPlan.ExecNodeSpecs;
var queryStrategies = new QueryStrategy[queryExecSpecs.Length];
for (var i = 0; i < queryExecSpecs.Length; i++)
{
var planNode = queryExecSpecs[i];
if (planNode == null)
{
Log.Debug(".makeComposer No execution node for stream " + i + " '" + _streamNames[i] + "'");
continue;
}
var executionNode = planNode.MakeExec(_statementName, _statementId, _annotations, indexesPerStream, _streamTypes, streamViews, _historicalStreamIndexLists, externalViews, tableSecondaryIndexLocks);
if (Log.IsDebugEnabled)
{
Log.Debug(".makeComposer Execution nodes for stream " + i + " '" + _streamNames[i] +
"' : \n" + ExecNode.Print(executionNode));
}
queryStrategies[i] = new ExecNodeQueryStrategy(i, _streamTypes.Length, executionNode);
}
// Remove indexes that are from tables as these are only available to query strategies
if (hasTable)
{
indexesPerStream = RemoveTableIndexes(indexesPerStream, _streamJoinAnalysisResult.TablesPerStream);
}
// If this is not unidirectional and not a self-join (excluding self-outer-join)
JoinSetComposerDesc joinSetComposerDesc;
if ((!_streamJoinAnalysisResult.IsUnidirectional) &&
(!_streamJoinAnalysisResult.IsPureSelfJoin || _outerJoinDescList.Length > 0))
{
JoinSetComposer composer;
if (_historicalViewableDesc.HasHistorical)
{
composer = new JoinSetComposerHistoricalImpl(indexesPerStream, queryStrategies, streamViews, _exprEvaluatorContext);
}
else
{
if (isFireAndForget)
{
composer = new JoinSetComposerFAFImpl(indexesPerStream, queryStrategies, _streamJoinAnalysisResult.IsPureSelfJoin, _exprEvaluatorContext, _joinRemoveStream, _isOuterJoins);
}
else
{
composer = new JoinSetComposerImpl(indexesPerStream, queryStrategies, _streamJoinAnalysisResult.IsPureSelfJoin, _exprEvaluatorContext, _joinRemoveStream);
}
}
// rewrite the filter expression for all-inner joins in case "on"-clause outer join syntax was used to include those expressions
var filterExpression = GetFilterExpressionInclOnClause(_optionalFilterNode, _outerJoinDescList);
var postJoinEval = filterExpression == null ? null : filterExpression.ExprEvaluator;
joinSetComposerDesc = new JoinSetComposerDesc(composer, postJoinEval);
}
else
{
QueryStrategy driver;
int unidirectionalStream;
if (_streamJoinAnalysisResult.UnidirectionalStreamNumber != -1)
{
unidirectionalStream = _streamJoinAnalysisResult.UnidirectionalStreamNumber;
driver = queryStrategies[unidirectionalStream];
}
else
{
unidirectionalStream = 0;
driver = queryStrategies[0];
}
JoinSetComposer composer = new JoinSetComposerStreamToWinImpl(indexesPerStream, _streamJoinAnalysisResult.IsPureSelfJoin,
unidirectionalStream, driver, _streamJoinAnalysisResult.UnidirectionalNonDriving);
var postJoinEval = _optionalFilterNode == null ? null : _optionalFilterNode.ExprEvaluator;
joinSetComposerDesc = new JoinSetComposerDesc(composer, postJoinEval);
}
// compile prior events per stream to preload any indexes
var eventsPerStream = new EventBean[_streamNames.Length][];
var events = new List <EventBean>();
for (var i = 0; i < eventsPerStream.Length; i++)
{
// For named windows and tables, we don't need to preload indexes from the iterators as this is always done already
if (_streamJoinAnalysisResult.NamedWindow[i] || _streamJoinAnalysisResult.TablesPerStream[i] != null)
{
continue;
}
IEnumerator <EventBean> en = null;
if (!(streamViews[i] is HistoricalEventViewable) && !(streamViews[i] is DerivedValueView))
{
try
{
en = streamViews[i].GetEnumerator();
}
catch (UnsupportedOperationException)
{
// Joins do not support the iterator
}
}
if (en != null)
{
while (en.MoveNext())
{
events.Add(en.Current);
}
eventsPerStream[i] = events.ToArray();
events.Clear();
}
else
{
eventsPerStream[i] = new EventBean[0];
}
}
// init
joinSetComposerDesc.JoinSetComposer.Init(eventsPerStream);
return(joinSetComposerDesc);
}
/// <summary>
/// The snapshot
/// </summary>
/// <param name="queryGraph">The <see cref="QueryGraph"/></param>
/// <param name="attributes">The <see cref="Attribute" /> array</param>
/// <param name="virtualDataWindow">The <see cref="VirtualDWView"/></param>
/// <param name="indexRepository">The <see cref="EventTableIndexRepository"/></param>
/// <param name="queryPlanLogging">The <see cref="bool"/></param>
/// <param name="queryPlanLogDestination">The <see cref="ILog"/></param>
/// <param name="objectName">The <see cref="string"/></param>
/// <param name="agentInstanceContext">The <see cref="AgentInstanceContext"/></param>
/// <returns>The <see cref="ICollection{EventBean}"/></returns>
public static ICollection <EventBean> Snapshot(
QueryGraph queryGraph,
Attribute[] attributes,
VirtualDWView virtualDataWindow,
EventTableIndexRepository indexRepository,
bool queryPlanLogging,
ILog queryPlanLogDestination,
string objectName,
AgentInstanceContext agentInstanceContext)
{
var queryGraphValue = queryGraph == null ? null : queryGraph.GetGraphValue(QueryGraph.SELF_STREAM, 0);
if (queryGraphValue == null || queryGraphValue.Items.IsEmpty())
{
if (virtualDataWindow != null)
{
var pair = virtualDataWindow.GetFireAndForgetDesc(Collections.GetEmptySet <string>(), Collections.GetEmptySet <string>());
return(virtualDataWindow.GetFireAndForgetData(pair.Second, new Object[0], new RangeIndexLookupValue[0], attributes));
}
return(null);
}
// determine custom index
var customResult = SnapshotCustomIndex(
queryGraphValue, indexRepository, attributes, agentInstanceContext, queryPlanLogging, queryPlanLogDestination, objectName);
if (customResult != null)
{
return(customResult.Value);
}
// determine lookup based on hash-keys and ranges
var keysAvailable = queryGraphValue.HashKeyProps;
var keyNamesAvailable = keysAvailable.Indexed.Count == 0
? Collections.GetEmptySet <string>()
: new HashSet <string>(keysAvailable.Indexed);
var rangesAvailable = queryGraphValue.RangeProps;
var rangeNamesAvailable = rangesAvailable.Indexed.Count == 0
? Collections.GetEmptySet <string>()
: new HashSet <string>(rangesAvailable.Indexed);
// find index that matches the needs
var tablePair = FindIndex(keyNamesAvailable, rangeNamesAvailable, indexRepository, virtualDataWindow, attributes);
// regular index lookup
if (tablePair != null)
{
return(SnapshotIndex(keysAvailable, rangesAvailable, tablePair, virtualDataWindow, attributes, agentInstanceContext, queryPlanLogging, queryPlanLogDestination, objectName));
}
// in-keyword lookup
var inkwResult = SnapshotInKeyword(queryGraphValue, indexRepository, virtualDataWindow, attributes, agentInstanceContext, queryPlanLogging, queryPlanLogDestination, objectName);
if (inkwResult != null)
{
return(inkwResult.Value);
}
QueryPlanReportTableScan(attributes, agentInstanceContext, queryPlanLogging, queryPlanLogDestination, objectName);
return(null);
}
public SubordTableLookupStrategy MakeStrategy(EventTable[] eventTable, VirtualDWView vdw)
{
return(new SubordFullTableScanLookupStrategy((UnindexedEventTable)eventTable[0]));
}
public override SubordTableLookupStrategy MakeStrategy(EventTable[] eventTable, VirtualDWView vdw)
{
if (IsNWOnTrigger)
{
return(new SubordIndexedTableLookupStrategySingleCoercingNW(Evaluator, (PropertyIndexedEventTableSingle)eventTable[0], _coercionType, StrategyDesc));
}
else
{
return(new SubordIndexedTableLookupStrategySingleCoercing(StreamCountOuter, Evaluator, (PropertyIndexedEventTableSingle)eventTable[0], _coercionType, StrategyDesc));
}
}
/// <summary>
/// The SnapshotIndex
/// </summary>
/// <param name="keysAvailable">The <see cref="QueryGraphValuePairHashKeyIndex"/></param>
/// <param name="rangesAvailable">The <see cref="QueryGraphValuePairRangeIndex"/></param>
/// <param name="tablePair">The <see cref="Pair{IndexMultiKey, EventTableAndNamePair}"/></param>
/// <param name="virtualDataWindow">The <see cref="VirtualDWView"/></param>
/// <param name="attributes">The <see cref="Attribute"/> array</param>
/// <param name="agentInstanceContext">The <see cref="AgentInstanceContext"/></param>
/// <param name="queryPlanLogging">The <see cref="bool"/></param>
/// <param name="queryPlanLogDestination">The <see cref="ILog"/></param>
/// <param name="objectName">The <see cref="string"/></param>
/// <returns>The <see cref="ICollection{EventBean}"/></returns>
private static ICollection <EventBean> SnapshotIndex(
QueryGraphValuePairHashKeyIndex keysAvailable,
QueryGraphValuePairRangeIndex rangesAvailable,
Pair <IndexMultiKey, EventTableAndNamePair> tablePair,
VirtualDWView virtualDataWindow,
Attribute[] attributes,
AgentInstanceContext agentInstanceContext,
bool queryPlanLogging,
ILog queryPlanLogDestination,
string objectName)
{
var evaluateParamsTrue = new EvaluateParams(null, true, agentInstanceContext);
// report plan
queryPlanReport(tablePair.Second.IndexName, tablePair.Second.EventTable, attributes, agentInstanceContext, queryPlanLogging, queryPlanLogDestination, objectName);
// compile hash lookup values
var tableHashProps = tablePair.First.HashIndexedProps;
var keyValues = new Object[tableHashProps.Length];
for (var tableHashPropNum = 0; tableHashPropNum < tableHashProps.Length; tableHashPropNum++)
{
var tableHashProp = tableHashProps[tableHashPropNum];
for (var i = 0; i < keysAvailable.Indexed.Count; i++)
{
if (keysAvailable.Indexed[i].Equals(tableHashProp.IndexPropName))
{
var key = keysAvailable.Keys[i];
var value = key.KeyExpr.ExprEvaluator.Evaluate(evaluateParamsTrue);
if (value != null)
{
value = MayCoerceNonNull(value, tableHashProp.CoercionType);
keyValues[tableHashPropNum] = value;
}
}
}
}
// compile range lookup values
var tableRangeProps = tablePair.First.RangeIndexedProps;
var rangeValues = new RangeIndexLookupValue[tableRangeProps.Length];
for (var tableRangePropNum = 0; tableRangePropNum < tableRangeProps.Length; tableRangePropNum++)
{
var tableRangeProp = tableRangeProps[tableRangePropNum];
for (var i = 0; i < rangesAvailable.Indexed.Count; i++)
{
if (rangesAvailable.Indexed[i].Equals(tableRangeProp.IndexPropName))
{
var range = rangesAvailable.Keys[i];
if (range is QueryGraphValueEntryRangeIn)
{
var between = (QueryGraphValueEntryRangeIn)range;
var start = between.ExprStart.ExprEvaluator.Evaluate(evaluateParamsTrue);
var end = between.ExprEnd.ExprEvaluator.Evaluate(evaluateParamsTrue);
Range rangeValue;
if (tableRangeProp.CoercionType.IsNumeric())
{
double?startDouble = null;
if (start != null)
{
startDouble = start.AsDouble();
}
double?endDouble = null;
if (end != null)
{
endDouble = end.AsDouble();
}
rangeValue = new DoubleRange(startDouble, endDouble);
}
else
{
rangeValue = new StringRange(start == null ? null : start.ToString(), end == null ? null : end.ToString());
}
rangeValues[tableRangePropNum] = new RangeIndexLookupValueRange(rangeValue, between.RangeType, between.IsAllowRangeReversal);
}
else
{
var relOp = (QueryGraphValueEntryRangeRelOp)range;
var value = relOp.Expression.ExprEvaluator.Evaluate(evaluateParamsTrue);
if (value != null)
{
value = MayCoerceNonNull(value, tableRangeProp.CoercionType);
}
rangeValues[tableRangePropNum] = new RangeIndexLookupValueRange(value, relOp.RangeType, true);
}
}
}
}
// perform lookup
return(FafTableLookup(virtualDataWindow, tablePair.First, tablePair.Second.EventTable, keyValues, rangeValues, attributes));
}
public SubordTableLookupStrategy MakeStrategy(EventTable[] eventTable, VirtualDWView vdw)
{
return(new SubordCompositeTableLookupStrategy(
_innerIndexQuery, (PropertyCompositeEventTable)eventTable[0], _strategyDesc));
}