public void SetUp()
{
var inputProperties = new[] { "s0.IntPrimitive" };
var dataCache = new DataCacheLRUImpl(100);
var resultProperties = new Dictionary <String, Object>();
resultProperties["myvarchar"] = typeof(string);
var resultEventType = SupportEventAdapterService.Service.CreateAnonymousMapType("test", resultProperties, true);
var pollResults = new Dictionary <MultiKey <Object>, IList <EventBean> >();
pollResults.Put(new MultiKey <Object>(new Object[] { -1 }), new List <EventBean>());
pollResults.Put(new MultiKey <Object>(new Object[] { 500 }), new List <EventBean>());
var supportPollingStrategy = new SupportPollingStrategy(pollResults);
_pollingViewable = new DatabasePollingViewable(1, inputProperties, supportPollingStrategy, dataCache, resultEventType);
var sqlParameters = new Dictionary <int, IList <ExprNode> >();
sqlParameters.Put(1, ((ExprNode) new ExprIdentNodeImpl("IntPrimitive", "s0")).AsSingleton());
_pollingViewable.Validate(null, new SupportStreamTypeSvc3Stream(), null, null, null, null, null, null, null, null, sqlParameters, null, SupportStatementContextFactory.MakeContext());
_indexingStrategy = new ProxyPollResultIndexingStrategy
{
ProcIndex = (pollResult, isActiveCache, statementContext) => new EventTable[] { new UnindexedEventTableList(pollResult, -1) },
ProcToQueryPlan = () => GetType().Name + " unindexed"
};
}
/// <summary>Ctor. </summary>
/// <param name="historicalEventViewable">the view of the historical</param>
/// <param name="indexingStrategy">the strategy to index poll result for future use</param>
/// <param name="indexLookupStrategy">the strategy to use past indexed results</param>
/// <param name="numStreams">the number of streams in the join</param>
/// <param name="historicalStreamNumber">the stream number of the historical</param>
public HistoricalDataExecNode(HistoricalEventViewable historicalEventViewable, PollResultIndexingStrategy indexingStrategy, HistoricalIndexLookupStrategy indexLookupStrategy, int numStreams, int historicalStreamNumber)
{
_historicalEventViewable = historicalEventViewable;
_indexingStrategy = indexingStrategy;
_indexLookupStrategy = indexLookupStrategy;
_numStreams = numStreams;
_historicalStreamNumber = historicalStreamNumber;
_lookupRows1Event = new EventBean[1][];
_lookupRows1Event[0] = new EventBean[numStreams];
}
/// <summary>Ctor. </summary>
/// <param name="viewable">providing the polling access</param>
/// <param name="indexingStrategy">strategy for indexing results</param>
/// <param name="lookupStrategy">strategy for using indexed results</param>
/// <param name="numStreams">number of streams</param>
/// <param name="streamNum">stream number of the historical stream</param>
/// <param name="rootStreamNum">the query plan root stream number</param>
/// <param name="outerJoinExprNode">an optional outer join expression</param>
public HistoricalTableLookupStrategy(HistoricalEventViewable viewable, PollResultIndexingStrategy indexingStrategy, HistoricalIndexLookupStrategy lookupStrategy, int numStreams, int streamNum, int rootStreamNum, ExprEvaluator outerJoinExprNode)
{
_viewable = viewable;
_indexingStrategy = indexingStrategy;
_lookupStrategy = lookupStrategy;
_streamNum = streamNum;
_rootStreamNum = rootStreamNum;
_outerJoinExprNode = outerJoinExprNode;
_lookupEventsPerStream = new EventBean[][]
{
new EventBean[numStreams]
};
}
/// <summary>Ctor. </summary>
/// <param name="myStreamNumber">is the strategy's stream number</param>
/// <param name="historicalStreamNumber">is the stream number of the view to be polled</param>
/// <param name="historicalEventViewable">is the view to be polled from</param>
/// <param name="isOuterJoin">is this is an outer join</param>
/// <param name="outerJoinCompareNode">is the node to perform the on-comparison for outer joins</param>
/// <param name="indexLookupStrategy">the strategy to use for limiting the cache result setto only those rows that match filter criteria </param>
/// <param name="pollResultIndexingStrategy">the strategy for indexing poll-results such that astrategy can use the index instead of a full table scan to resolve rows </param>
public HistoricalDataQueryStrategy(int myStreamNumber,
int historicalStreamNumber,
HistoricalEventViewable historicalEventViewable,
bool isOuterJoin,
ExprEvaluator outerJoinCompareNode,
HistoricalIndexLookupStrategy indexLookupStrategy,
PollResultIndexingStrategy pollResultIndexingStrategy)
{
_myStreamNumber = myStreamNumber;
_historicalStreamNumber = historicalStreamNumber;
_historicalEventViewable = historicalEventViewable;
_isOuterJoin = isOuterJoin;
_outerJoinCompareNode = outerJoinCompareNode;
_lookupRows1Event = new EventBean[1][];
_lookupRows1Event[0] = new EventBean[2];
_indexLookupStrategy = indexLookupStrategy;
_pollResultIndexingStrategy = pollResultIndexingStrategy;
}
public EventTable[][] Poll(
EventBean[][] lookupEventsPerStream,
PollResultIndexingStrategy indexingStrategy,
ExprEvaluatorContext exprEvaluatorContext)
{
var localDataCache = _factory.DataCacheThreadLocal.GetOrCreate();
var strategyStarted = false;
var resultPerInputRow = new EventTable[lookupEventsPerStream.Length][];
// Get input parameters for each row
EventBean[] eventsPerStream;
for (var row = 0; row < lookupEventsPerStream.Length; row++) {
// Build lookup keys
eventsPerStream = lookupEventsPerStream[row];
var lookupValue = _factory.Evaluator.Evaluate(eventsPerStream, true, exprEvaluatorContext);
EventTable[] result = null;
// try the threadlocal iteration cache, if set
object cacheMultiKey = null;
if (localDataCache != null || DataCache.IsActive) {
cacheMultiKey = _factory.LookupValueToMultiKey.Invoke(lookupValue);
}
if (localDataCache != null) {
result = localDataCache.GetCached(cacheMultiKey);
}
// try the connection cache
if (result == null) {
var multi = DataCache.GetCached(cacheMultiKey);
if (multi != null) {
result = multi;
localDataCache?.Put(cacheMultiKey, multi);
}
}
// use the result from cache
if (result != null) {
// found in cache
resultPerInputRow[row] = result;
}
else {
// not found in cache, get from actual polling (db query)
try {
if (!strategyStarted) {
_pollExecStrategy.Start();
strategyStarted = true;
}
// Poll using the polling execution strategy and lookup values
IList<EventBean> pollResult = _pollExecStrategy.Poll(lookupValue, _agentInstanceContext);
// index the result, if required, using an indexing strategy
var indexTable = indexingStrategy.Index(pollResult, DataCache.IsActive, _agentInstanceContext);
// assign to row
resultPerInputRow[row] = indexTable;
// save in cache
DataCache.Put(cacheMultiKey, indexTable);
localDataCache?.Put(cacheMultiKey, indexTable);
}
catch (EPException) {
if (strategyStarted) {
_pollExecStrategy.Done();
}
throw;
}
}
}
if (strategyStarted) {
_pollExecStrategy.Done();
}
return resultPerInputRow;
}
public EventTable[][] Poll(EventBean[][] lookupEventsPerStream, PollResultIndexingStrategy indexingStrategy, ExprEvaluatorContext exprEvaluatorContext)
{
var localDataCache = _dataCacheThreadLocal.GetOrCreate();
var strategyStarted = false;
var resultPerInputRow = new EventTable[lookupEventsPerStream.Length][];
// Get input parameters for each row
for (var row = 0; row < lookupEventsPerStream.Length; row++)
{
var methodParams = new Object[_validatedExprNodes.Length];
var evaluateParams = new EvaluateParams(lookupEventsPerStream[row], true, exprEvaluatorContext);
// Build lookup keys
for (var valueNum = 0; valueNum < _validatedExprNodes.Length; valueNum++)
{
var parameterValue = _validatedExprNodes[valueNum].Evaluate(evaluateParams);
methodParams[valueNum] = parameterValue;
}
EventTable[] result = null;
// try the threadlocal iteration cache, if set
if (localDataCache != null)
{
result = localDataCache.GetCached(methodParams, _methodStreamSpec.Expressions.Count);
}
// try the connection cache
if (result == null)
{
result = _dataCache.GetCached(methodParams, _methodStreamSpec.Expressions.Count);
if ((result != null) && (localDataCache != null))
{
localDataCache.PutCached(methodParams, _methodStreamSpec.Expressions.Count, result);
}
}
if (result != null)
{
// found in cache
resultPerInputRow[row] = result;
}
else
{
// not found in cache, get from actual polling (db query)
try {
if (!strategyStarted)
{
_pollExecStrategy.Start();
strategyStarted = true;
}
// Poll using the polling execution strategy and lookup values
var pollResult = _pollExecStrategy.Poll(methodParams, exprEvaluatorContext);
// index the result, if required, using an indexing strategy
var indexTable = indexingStrategy.Index(pollResult, _dataCache.IsActive, _statementContext);
// assign to row
resultPerInputRow[row] = indexTable;
// save in cache
_dataCache.PutCached(methodParams, _methodStreamSpec.Expressions.Count, indexTable);
if (localDataCache != null)
{
localDataCache.PutCached(methodParams, _methodStreamSpec.Expressions.Count, indexTable);
}
} catch (EPException ex) {
if (strategyStarted)
{
_pollExecStrategy.Done();
}
throw;
}
}
}
if (strategyStarted)
{
_pollExecStrategy.Done();
}
return(resultPerInputRow);
}
/// <summary>Get the strategies to use for polling from a given stream. </summary>
/// <param name="streamViewStreamNum">the stream providing the polling events</param>
/// <returns>looking and indexing strategy</returns>
public Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy> GetStrategy(int streamViewStreamNum)
{
// If there is only a single polling stream, then build a single index
if (_pollingStreams.Count == 1)
{
return(JoinSetComposerPrototypeFactory.DetermineIndexing(_queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[streamViewStreamNum], _historicalStreamNum, streamViewStreamNum));
}
// If there are multiple polling streams, determine if a single index is appropriate.
// An index can be reused if:
// (a) indexed property names are the same
// (b) indexed property types are the same
// (c) key property types are the same (because of coercion)
// A index lookup strategy is always specific to the providing stream.
if (_indexesUsedByStreams == null)
{
_indexesUsedByStreams = new LinkedHashMap <HistoricalStreamIndexDesc, IList <int> >();
foreach (var pollingStream in _pollingStreams)
{
var queryGraphValue = _queryGraph.GetGraphValue(pollingStream, _historicalStreamNum);
var hashKeyProps = queryGraphValue.HashKeyProps;
var indexProperties = hashKeyProps.Indexed;
var keyTypes = GetPropertyTypes(hashKeyProps.Keys);
var indexTypes = GetPropertyTypes(_typesPerStream[_historicalStreamNum], indexProperties);
var desc = new HistoricalStreamIndexDesc(indexProperties, indexTypes, keyTypes);
var usedByStreams = _indexesUsedByStreams.Get(desc);
if (usedByStreams == null)
{
usedByStreams = new List <int>();
_indexesUsedByStreams.Put(desc, usedByStreams);
}
usedByStreams.Add(pollingStream);
}
// There are multiple indexes required:
// Build a master indexing strategy that forms multiple indexes and numbers each.
if (_indexesUsedByStreams.Count > 1)
{
var numIndexes = _indexesUsedByStreams.Count;
var indexingStrategies = new PollResultIndexingStrategy[numIndexes];
// create an indexing strategy for each index
var count = 0;
foreach (var desc in _indexesUsedByStreams)
{
var sampleStreamViewStreamNum = desc.Value[0];
indexingStrategies[count] = JoinSetComposerPrototypeFactory.DetermineIndexing(_queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[sampleStreamViewStreamNum], _historicalStreamNum, sampleStreamViewStreamNum).Second;
count++;
}
// create a master indexing strategy that utilizes each indexing strategy to create a set of indexes
var streamNum = streamViewStreamNum;
_masterIndexingStrategy = new ProxyPollResultIndexingStrategy
{
ProcIndex = (pollResult, isActiveCache, statementContext) =>
{
var tables = new EventTable[numIndexes];
for (var i = 0; i < numIndexes; i++)
{
tables[i] = indexingStrategies[i].Index(pollResult, isActiveCache, statementContext)[0];
}
var organization = new EventTableOrganization(null, false, false, streamNum, null, EventTableOrganizationType.MULTIINDEX);
return(new EventTable[]
{
new MultiIndexEventTable(tables, organization)
});
},
ProcToQueryPlan = () =>
{
var writer = new StringWriter();
var delimiter = "";
foreach (var strategy in indexingStrategies)
{
writer.Write(delimiter);
writer.Write(strategy.ToQueryPlan());
delimiter = ", ";
}
return(GetType().FullName + " " + writer);
}
};
}
}
// there is one type of index
if (_indexesUsedByStreams.Count == 1)
{
return(JoinSetComposerPrototypeFactory.DetermineIndexing(
_queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[streamViewStreamNum], _historicalStreamNum, streamViewStreamNum));
}
// determine which index number the polling stream must use
var indexUsed = 0;
var found = false;
foreach (var desc in _indexesUsedByStreams.Values)
{
if (desc.Contains(streamViewStreamNum))
{
found = true;
break;
}
indexUsed++;
}
if (!found)
{
throw new IllegalStateException("MapIndex not found for use by stream " + streamViewStreamNum);
}
// Use one of the indexes built by the master index and a lookup strategy
var indexNumber = indexUsed;
HistoricalIndexLookupStrategy innerLookupStrategy = JoinSetComposerPrototypeFactory.DetermineIndexing(_queryGraph, _typesPerStream[_historicalStreamNum], _typesPerStream[streamViewStreamNum], _historicalStreamNum, streamViewStreamNum).First;
var lookupStrategy = new ProxyHistoricalIndexLookupStrategy
{
ProcLookup = (lookupEvent, index, context) =>
{
var multiIndex = (MultiIndexEventTable)index[0];
var indexToUse = multiIndex.Tables[indexNumber];
return(innerLookupStrategy.Lookup(lookupEvent, new EventTable[] { indexToUse }, context));
},
ProcToQueryPlan = () => GetType().FullName + " inner: " + innerLookupStrategy.ToQueryPlan()
};
return(new Pair <HistoricalIndexLookupStrategy, PollResultIndexingStrategy> (lookupStrategy, _masterIndexingStrategy));
}
/// <summary>
/// Poll for stored historical or reference data using events per stream and
/// returing for each event-per-stream row a separate list with events
/// representing the poll result.
/// </summary>
/// <param name="lookupEventsPerStream">is the events per stream where the
/// first dimension is a number of rows (often 1 depending on windows used) and
/// the second dimension is the number of streams participating in a join.</param>
/// <param name="indexingStrategy">the strategy to use for converting poll results into a indexed table for fast lookup</param>
/// <param name="exprEvaluatorContext">The expression evaluator context.</param>
/// <returns>
/// array of lists with one list for each event-per-stream row
/// </returns>
public EventTable[][] Poll(EventBean[][] lookupEventsPerStream, PollResultIndexingStrategy indexingStrategy, ExprEvaluatorContext exprEvaluatorContext)
{
DataCache localDataCache = _dataCacheThreadLocal.GetOrCreate();
bool strategyStarted = false;
EventTable[][] resultPerInputRow = new EventTable[lookupEventsPerStream.Length][];
// Get input parameters for each row
for (int row = 0; row < lookupEventsPerStream.Length; row++)
{
Object[] lookupValues = new Object[_inputParameters.Count];
// Build lookup keys
for (int valueNum = 0; valueNum < _inputParameters.Count; valueNum++)
{
EventBean[] eventsPerStream = lookupEventsPerStream[row];
Object lookupValue = _evaluators[valueNum].Evaluate(new EvaluateParams(eventsPerStream, true, exprEvaluatorContext));
lookupValues[valueNum] = lookupValue;
}
EventTable[] result = null;
// try the threadlocal iteration cache, if set
if (localDataCache != null)
{
result = localDataCache.GetCached(lookupValues);
}
// try the connection cache
if (result == null)
{
EventTable[] multi = _dataCache.GetCached(lookupValues);
if (multi != null)
{
result = multi;
if (localDataCache != null)
{
localDataCache.PutCached(lookupValues, result);
}
}
}
// use the result from cache
if (result != null)
// found in cache
{
resultPerInputRow[row] = result;
}
// not found in cache, get from actual polling (db query)
else
{
try
{
if (!strategyStarted)
{
_pollExecStrategy.Start();
strategyStarted = true;
}
// Poll using the polling execution strategy and lookup values
IList <EventBean> pollResult = _pollExecStrategy.Poll(lookupValues, exprEvaluatorContext);
// index the result, if required, using an indexing strategy
EventTable[] indexTable = indexingStrategy.Index(pollResult, _dataCache.IsActive);
// assign to row
resultPerInputRow[row] = indexTable;
// save in cache
_dataCache.PutCached(lookupValues, indexTable);
if (localDataCache != null)
{
localDataCache.PutCached(lookupValues, indexTable);
}
}
catch (EPException)
{
if (strategyStarted)
{
_pollExecStrategy.Done();
}
throw;
}
}
}
if (strategyStarted)
{
_pollExecStrategy.Done();
}
return(resultPerInputRow);
}
