/// <summary>
/// Given a child URI and a map of factory URIs, inspect the child URI against the factory
/// URIs and return a collection of entries for which the child URI falls within or is equals
/// to the factory URI.
/// </summary>
/// <param name="child">is the child URI to match against factory URIs</param>
/// <param name="uris">is a map of factory URI and an object</param>
/// <returns>matching factory URIs, if any</returns>
public static ICollection <KeyValuePair <Uri, V> > FilterSort <V>(
Uri child,
IDictionary <Uri, V> uris)
{
var childPathIsOpaque = IsOpaque(child);
var childPathIsRelative = !child.IsAbsoluteUri;
var childPathElements = ParsePathElements(child);
var result = new OrderedListDictionary <int, KeyValuePair <Uri, V> >();
foreach (var entry in uris)
{
var factoryUri = entry.Key;
// handle opaque (mailto:) and relative (a/b) using equals
if (childPathIsOpaque || childPathIsRelative || !factoryUri.IsAbsoluteUri || IsOpaque(factoryUri))
{
if (factoryUri.Equals(child))
{
result.Put(int.MinValue, entry); // Equals is a perfect match
}
continue;
}
// handle absolute URIs, compare scheme and authority if present
if (((child.Scheme != null) && (factoryUri.Scheme == null)) ||
((child.Scheme == null) && (factoryUri.Scheme != null)))
{
continue;
}
if ((child.Scheme != null) && (!child.Scheme.Equals(factoryUri.Scheme)))
{
continue;
}
if (((child.Authority != null) && (factoryUri.Authority == null)) ||
((child.Authority == null) && (factoryUri.Authority != null)))
{
continue;
}
if ((child.Authority != null) && (child.Authority != factoryUri.Authority))
{
continue;
}
// Match the child
string[] factoryPathElements = ParsePathElements(factoryUri);
int score = ComputeScore(childPathElements, factoryPathElements);
if (score > 0)
{
result.Put(score, entry); // Partial match if score is positive
}
}
return(result.Values);
}
private static void ReadState(
DataInput input,
CountMinSketchState state)
{
var depth = input.ReadInt();
var width = input.ReadInt();
var rowsTable = input.ReadInt();
var table = new long[rowsTable][];
for (var i = 0; i < rowsTable; i++) {
var colsRows = input.ReadInt();
table[i] = new long[colsRows];
for (var j = 0; j < colsRows; j++) {
table[i][j] = input.ReadLong();
}
}
var rowsHash = input.ReadInt();
var hash = new long[rowsHash];
for (var i = 0; i < rowsTable; i++) {
hash[i] = input.ReadLong();
}
var total = input.ReadLong();
state.Hashes = new CountMinSketchStateHashes(depth, width, table, hash, total);
var hasTopk = input.ReadBoolean();
state.Topk = null;
if (hasTopk) {
var topkMax = input.ReadInt();
var topMap = new OrderedListDictionary<long, object>(
Comparers.Default<long>().Inverse());
var refMap = new Dictionary<ByteBuffer, long>();
var numRows = input.ReadInt();
for (var i = 0; i < numRows; i++) {
var freq = input.ReadLong();
var numEntries = input.ReadInt();
if (numEntries == 1) {
var buf = ReadBytes(input);
topMap.Put(freq, buf);
refMap.Put(buf, freq);
}
else {
Deque<ByteBuffer> q = new ArrayDeque<ByteBuffer>(numEntries);
for (var j = 0; j < numEntries; j++) {
var buf = ReadBytes(input);
q.AddLast(buf);
refMap.Put(buf, freq);
}
topMap.Put(freq, q);
}
}
state.Topk = new CountMinSketchStateTopk(topkMax, topMap, refMap);
}
}
/// <summary>
/// NOTE: Code-generation-invoked method, method name and parameter order matters
/// </summary>
/// <param name="outgoingEvents">events</param>
/// <param name="orderKeys">keys</param>
/// <param name="comparator">comparator</param>
/// <returns>sorted</returns>
public static EventBean[] SortWOrderKeys(
EventBean[] outgoingEvents,
object[] orderKeys,
IComparer<object> comparator)
{
var sort = new OrderedListDictionary<object, object>(comparator);
if (outgoingEvents == null || outgoingEvents.Length < 2) {
return outgoingEvents;
}
for (var i = 0; i < outgoingEvents.Length; i++) {
var entry = sort.Get(orderKeys[i]);
if (entry == null) {
sort.Put(orderKeys[i], outgoingEvents[i]);
}
else if (entry is EventBean) {
IList<EventBean> list = new List<EventBean>();
list.Add((EventBean) entry);
list.Add(outgoingEvents[i]);
sort.Put(orderKeys[i], list);
}
else {
var list = (IList<EventBean>) entry;
list.Add(outgoingEvents[i]);
}
}
var result = new EventBean[outgoingEvents.Length];
var count = 0;
foreach (object entry in sort.Values) {
if (entry is IList<EventBean> output) {
foreach (var theEvent in output) {
result[count++] = theEvent;
}
}
else {
result[count++] = (EventBean) entry;
}
}
return result;
}
public IOrderedDictionary <string, V> PrefixMap(string key)
{
var result = new OrderedListDictionary <string, V>();
foreach (var entry in simple)
{
if (entry.Key.StartsWith(key))
{
result.Put(entry.Key, entry.Value);
}
}
return(result);
}
public object GetValue(
int aggColNum,
AggregationRow row,
EventBean[] eventsPerStream,
bool isNewData,
ExprEvaluatorContext exprEvaluatorContext)
{
AggregationStateSorted sorted = (AggregationStateSorted)row.GetAccessState(aggColNum);
var fromKey = _fromKeyEval.Evaluate(eventsPerStream, isNewData, exprEvaluatorContext);
if (fromKey == null)
{
return(null);
}
var fromInclusive = _fromInclusiveEval.Evaluate(eventsPerStream, isNewData, exprEvaluatorContext).AsBoxedBoolean();
if (fromInclusive == null)
{
return(null);
}
var toKey = _toKeyEval.Evaluate(eventsPerStream, isNewData, exprEvaluatorContext);
if (toKey == null)
{
return(null);
}
var toInclusive = _toInclusiveEval.Evaluate(eventsPerStream, isNewData, exprEvaluatorContext).AsBoxedBoolean();
if (toInclusive == null)
{
return(null);
}
var mapOfArrays = new OrderedListDictionary <object, object>(sorted.Sorted.KeyComparer);
var submap = sorted.Sorted.Between(fromKey, fromInclusive.Value, toKey, toInclusive.Value);
foreach (KeyValuePair <object, object> entry in submap)
{
mapOfArrays.Put(entry.Key, AggregatorAccessSortedImpl.CheckedPayloadGetUnderlyingArray(entry.Value, _underlyingClass));
}
return(mapOfArrays);
}
public static RowRecogPlan ValidateAndPlan(
IContainer container,
EventType parentEventType,
bool unbound,
StatementBaseInfo @base,
StatementCompileTimeServices services)
{
var statementRawInfo = @base.StatementRawInfo;
var matchRecognizeSpec = @base.StatementSpec.Raw.MatchRecognizeSpec;
var annotations = statementRawInfo.Annotations;
var iterateOnly = HintEnum.ITERATE_ONLY.GetHint(annotations) != null;
var additionalForgeables = new List<StmtClassForgeableFactory>();
// Expanded pattern already there
RowRecogExprNode expandedPatternNode = matchRecognizeSpec.Pattern;
// Determine single-row and multiple-row variables
var variablesSingle = new LinkedHashSet<string>();
var variablesMultiple = new LinkedHashSet<string>();
RowRecogHelper.RecursiveInspectVariables(expandedPatternNode, false, variablesSingle, variablesMultiple);
// each variable gets associated with a stream number (multiple-row variables as well to hold the current event for the expression).
var streamNum = 0;
var variableStreams = new LinkedHashMap<string, Pair<int, bool>>();
foreach (var variableSingle in variablesSingle) {
variableStreams.Put(variableSingle, new Pair<int, bool>(streamNum, false));
streamNum++;
}
foreach (var variableMultiple in variablesMultiple) {
variableStreams.Put(variableMultiple, new Pair<int, bool>(streamNum, true));
streamNum++;
}
// mapping of stream to variable
var streamVariables = new OrderedListDictionary<int, string>();
foreach (var entry in variableStreams) {
streamVariables.Put(entry.Value.First, entry.Key);
}
// determine visibility rules
var visibility = RowRecogHelper.DetermineVisibility(expandedPatternNode);
// assemble all single-row variables for expression validation
var allStreamNames = new string[variableStreams.Count];
var allTypes = new EventType[variableStreams.Count];
streamNum = 0;
foreach (var variableSingle in variablesSingle) {
allStreamNames[streamNum] = variableSingle;
allTypes[streamNum] = parentEventType;
streamNum++;
}
foreach (var variableMultiple in variablesMultiple) {
allStreamNames[streamNum] = variableMultiple;
allTypes[streamNum] = parentEventType;
streamNum++;
}
// determine type service for use with DEFINE
// validate each DEFINE clause expression
ISet<string> definedVariables = new HashSet<string>();
IList<ExprAggregateNode> aggregateNodes = new List<ExprAggregateNode>();
var isExprRequiresMultimatchState = new bool[variableStreams.Count];
var previousNodes = new OrderedListDictionary<int, IList<ExprPreviousMatchRecognizeNode>>();
for (var defineIndex = 0; defineIndex < matchRecognizeSpec.Defines.Count; defineIndex++) {
MatchRecognizeDefineItem defineItem = matchRecognizeSpec.Defines[defineIndex];
if (definedVariables.Contains(defineItem.Identifier)) {
throw new ExprValidationException(
"Variable '" + defineItem.Identifier + "' has already been defined");
}
definedVariables.Add(defineItem.Identifier);
// stream-type visibilities handled here
var typeServiceDefines = BuildDefineStreamTypeServiceDefine(
defineIndex,
variableStreams,
defineItem,
visibility,
parentEventType,
statementRawInfo,
services);
var exprNodeResult = HandlePreviousFunctions(defineItem.Expression, previousNodes);
var validationContext = new ExprValidationContextBuilder(typeServiceDefines, statementRawInfo, services)
.WithAllowBindingConsumption(true)
.WithDisablePropertyExpressionEventCollCache(true)
.Build();
ExprNode validated;
try {
// validate
validated = ExprNodeUtilityValidate.GetValidatedSubtree(
ExprNodeOrigin.MATCHRECOGDEFINE,
exprNodeResult,
validationContext);
// check aggregates
defineItem.Expression = validated;
ExprAggregateNodeUtil.GetAggregatesBottomUp(validated, aggregateNodes);
if (!aggregateNodes.IsEmpty()) {
throw new ExprValidationException("An aggregate function may not appear in a DEFINE clause");
}
}
catch (ExprValidationException ex) {
throw new ExprValidationException(
"Failed to validate condition expression for variable '" +
defineItem.Identifier +
"': " +
ex.Message,
ex);
}
// determine access to event properties from multi-matches
var visitor = new ExprNodeStreamRequiredVisitor();
validated.Accept(visitor);
var streamsRequired = visitor.StreamsRequired;
foreach (var streamRequired in streamsRequired) {
if (streamRequired >= variableStreams.Count) {
var streamNumIdent = variableStreams.Get(defineItem.Identifier).First;
isExprRequiresMultimatchState[streamNumIdent] = true;
break;
}
}
}
var defineAsksMultimatches = CollectionUtil.IsAnySet(isExprRequiresMultimatchState);
// determine type service for use with MEASURE
IDictionary<string, object> measureTypeDef = new LinkedHashMap<string, object>();
foreach (var variableSingle in variablesSingle) {
measureTypeDef.Put(variableSingle, parentEventType);
}
foreach (var variableMultiple in variablesMultiple) {
measureTypeDef.Put(variableMultiple, new[] {parentEventType});
}
var compositeTypeName = services.EventTypeNameGeneratorStatement.AnonymousRowrecogCompositeName;
var compositeTypeMetadata = new EventTypeMetadata(
compositeTypeName,
@base.ModuleName,
EventTypeTypeClass.MATCHRECOGDERIVED,
EventTypeApplicationType.OBJECTARR,
NameAccessModifier.TRANSIENT,
EventTypeBusModifier.NONBUS,
false,
EventTypeIdPair.Unassigned());
var compositeEventType = BaseNestableEventUtil.MakeOATypeCompileTime(
compositeTypeMetadata,
measureTypeDef,
null,
null,
null,
null,
services.BeanEventTypeFactoryPrivate,
services.EventTypeCompileTimeResolver);
services.EventTypeCompileTimeRegistry.NewType(compositeEventType);
StreamTypeService compositeTypeServiceMeasure =
new StreamTypeServiceImpl(compositeEventType, "MATCH_RECOGNIZE", true);
// find MEASURE clause aggregations
var measureReferencesMultivar = false;
IList<ExprAggregateNode> measureAggregateExprNodes = new List<ExprAggregateNode>();
foreach (var measureItem in matchRecognizeSpec.Measures) {
ExprAggregateNodeUtil.GetAggregatesBottomUp(measureItem.Expr, measureAggregateExprNodes);
}
AggregationServiceForgeDesc[] aggregationServices = null;
if (!measureAggregateExprNodes.IsEmpty()) {
aggregationServices = PlanAggregations(
measureAggregateExprNodes,
compositeTypeServiceMeasure,
allStreamNames,
allTypes,
streamVariables,
variablesMultiple,
@base,
services);
foreach (AggregationServiceForgeDesc svc in aggregationServices) {
if (svc != null) {
additionalForgeables.AddAll(svc.AdditionalForgeables);
}
}
}
// validate each MEASURE clause expression
IDictionary<string, object> rowTypeDef = new LinkedHashMap<string, object>();
var streamRefVisitor = new ExprNodeStreamUseCollectVisitor();
foreach (var measureItem in matchRecognizeSpec.Measures) {
if (measureItem.Name == null) {
throw new ExprValidationException(
"The measures clause requires that each expression utilizes the AS keyword to assign a column name");
}
var validated = ValidateMeasureClause(
measureItem.Expr,
compositeTypeServiceMeasure,
variablesMultiple,
variablesSingle,
statementRawInfo,
services);
measureItem.Expr = validated;
rowTypeDef.Put(measureItem.Name, validated.Forge.EvaluationType);
validated.Accept(streamRefVisitor);
}
// Determine if any of the multi-var streams are referenced in the measures (non-aggregated only)
foreach (var @ref in streamRefVisitor.Referenced) {
var rootPropName = @ref.RootPropertyNameIfAny;
if (rootPropName != null) {
if (variablesMultiple.Contains(rootPropName)) {
measureReferencesMultivar = true;
break;
}
}
var streamRequired = @ref.StreamReferencedIfAny;
if (streamRequired != null) {
var streamVariable = streamVariables.Get(streamRequired.Value);
if (streamVariable != null) {
var def = variableStreams.Get(streamVariable);
if (def != null && def.Second) {
measureReferencesMultivar = true;
break;
}
}
}
}
var collectMultimatches = measureReferencesMultivar || defineAsksMultimatches;
// create rowevent type
var rowTypeName = services.EventTypeNameGeneratorStatement.AnonymousRowrecogRowName;
var rowTypeMetadata = new EventTypeMetadata(
rowTypeName,
@base.ModuleName,
EventTypeTypeClass.MATCHRECOGDERIVED,
EventTypeApplicationType.MAP,
NameAccessModifier.TRANSIENT,
EventTypeBusModifier.NONBUS,
false,
EventTypeIdPair.Unassigned());
var rowEventType = BaseNestableEventUtil.MakeMapTypeCompileTime(
rowTypeMetadata,
rowTypeDef,
null,
null,
null,
null,
services.BeanEventTypeFactoryPrivate,
services.EventTypeCompileTimeResolver);
services.EventTypeCompileTimeRegistry.NewType(rowEventType);
// validate partition-by expressions, if any
ExprNode[] partitionBy;
MultiKeyClassRef partitionMultiKey;
if (!matchRecognizeSpec.PartitionByExpressions.IsEmpty()) {
StreamTypeService typeServicePartition = new StreamTypeServiceImpl(
parentEventType,
"MATCH_RECOGNIZE_PARTITION",
true);
IList<ExprNode> validated = new List<ExprNode>();
var validationContext =
new ExprValidationContextBuilder(typeServicePartition, statementRawInfo, services)
.WithAllowBindingConsumption(true)
.Build();
foreach (var partitionExpr in matchRecognizeSpec.PartitionByExpressions) {
validated.Add(
ExprNodeUtilityValidate.GetValidatedSubtree(
ExprNodeOrigin.MATCHRECOGPARTITION,
partitionExpr,
validationContext));
}
matchRecognizeSpec.PartitionByExpressions = validated;
partitionBy = ExprNodeUtilityQuery.ToArray(validated);
MultiKeyPlan multiKeyPlan = MultiKeyPlanner.PlanMultiKey(partitionBy, false, @base.StatementRawInfo, services.SerdeResolver);
partitionMultiKey = multiKeyPlan.ClassRef;
additionalForgeables.AddAll(multiKeyPlan.MultiKeyForgeables);
}
else {
partitionBy = null;
partitionMultiKey = null;
}
// validate interval if present
if (matchRecognizeSpec.Interval != null) {
var validationContext =
new ExprValidationContextBuilder(new StreamTypeServiceImpl(false), statementRawInfo, services)
.WithAllowBindingConsumption(true)
.Build();
var validated = (ExprTimePeriod) ExprNodeUtilityValidate.GetValidatedSubtree(
ExprNodeOrigin.MATCHRECOGINTERVAL,
matchRecognizeSpec.Interval.TimePeriodExpr,
validationContext);
matchRecognizeSpec.Interval.TimePeriodExpr = validated;
}
// compile variable definition expressions
IDictionary<string, ExprNode> variableDefinitions = new Dictionary<string, ExprNode>();
foreach (var defineItem in matchRecognizeSpec.Defines) {
variableDefinitions.Put(defineItem.Identifier, defineItem.Expression);
}
// create evaluators
var columnNames = new string[matchRecognizeSpec.Measures.Count];
var columnForges = new ExprNode[matchRecognizeSpec.Measures.Count];
var count = 0;
foreach (var measureItem in matchRecognizeSpec.Measures) {
columnNames[count] = measureItem.Name;
columnForges[count] = measureItem.Expr;
count++;
}
// build states
var strand = RowRecogHelper.BuildStartStates(
expandedPatternNode,
variableDefinitions,
variableStreams,
isExprRequiresMultimatchState);
var startStates = strand.StartStates.ToArray();
RowRecogNFAStateForge[] allStates = strand.AllStates.ToArray();
if (Log.IsInfoEnabled) {
Log.Info("NFA tree:\n" + RowRecogNFAViewUtil.Print(startStates));
}
// determine names of multimatching variables
string[] multimatchVariablesArray;
int[] multimatchStreamNumToVariable;
int[] multimatchVariableToStreamNum;
if (variablesSingle.Count == variableStreams.Count) {
multimatchVariablesArray = new string[0];
multimatchStreamNumToVariable = new int[0];
multimatchVariableToStreamNum = new int[0];
}
else {
multimatchVariablesArray = new string[variableStreams.Count - variablesSingle.Count];
multimatchVariableToStreamNum = new int[multimatchVariablesArray.Length];
multimatchStreamNumToVariable = new int[variableStreams.Count];
CompatExtensions.Fill(multimatchStreamNumToVariable, -1);
count = 0;
foreach (var entry in variableStreams) {
if (entry.Value.Second) {
var index = count;
multimatchVariablesArray[index] = entry.Key;
multimatchVariableToStreamNum[index] = entry.Value.First;
multimatchStreamNumToVariable[entry.Value.First] = index;
count++;
}
}
}
var numEventsEventsPerStreamDefine =
defineAsksMultimatches ? variableStreams.Count + 1 : variableStreams.Count;
// determine interval-or-terminated
var orTerminated = matchRecognizeSpec.Interval != null && matchRecognizeSpec.Interval.IsOrTerminated;
TimePeriodComputeForge intervalCompute = null;
if (matchRecognizeSpec.Interval != null) {
intervalCompute = matchRecognizeSpec.Interval.TimePeriodExpr.TimePeriodComputeForge;
}
EventType multimatchEventType = null;
if (defineAsksMultimatches) {
multimatchEventType = GetDefineMultimatchEventType(variableStreams, parentEventType, @base, services);
}
// determine previous-access indexes and assign "prev" node indexes
// Since an expression such as "prior(2, price), prior(8, price)" translates into {2, 8} the relative index is {0, 1}.
// Map the expression-supplied index to a relative index
int[] previousRandomAccessIndexes = null;
if (!previousNodes.IsEmpty()) {
previousRandomAccessIndexes = new int[previousNodes.Count];
var countPrev = 0;
foreach (var entry in previousNodes) {
previousRandomAccessIndexes[countPrev] = entry.Key;
foreach (var callback in entry.Value) {
callback.AssignedIndex = countPrev;
}
countPrev++;
}
}
RowRecogDescForge forge = new RowRecogDescForge(
parentEventType,
rowEventType,
compositeEventType,
multimatchEventType,
multimatchStreamNumToVariable,
multimatchVariableToStreamNum,
partitionBy,
partitionMultiKey,
variableStreams,
matchRecognizeSpec.Interval != null,
iterateOnly,
unbound,
orTerminated,
collectMultimatches,
defineAsksMultimatches,
numEventsEventsPerStreamDefine,
multimatchVariablesArray,
startStates,
allStates,
matchRecognizeSpec.IsAllMatches,
matchRecognizeSpec.Skip.Skip,
columnForges,
columnNames,
intervalCompute,
previousRandomAccessIndexes,
aggregationServices);
return new RowRecogPlan(forge, additionalForgeables);
}
