internal MapReduceStreamable(
IStreamable <TMapKey, TMapInputLeft> sourceLeft,
IStreamable <TMapKey, TMapInputRight> sourceRight,
Func <IStreamable <TMapKey, TMapInputLeft>, IStreamable <TMapKey, TMapInputRight>, IStreamable <TMapKey, TReduceInput> > mapper,
Expression <Func <TReduceInput, TReduceKey> > keySelector,
Func <IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>, IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TBind> > reducer,
Expression <Func <TReduceKey, TBind, TOutput> > resultSelector,
bool leftAsymmetric = false)
: base(
sourceLeft.Properties.MapReduce
(sourceRight?.Properties, mapper, keySelector, reducer, resultSelector))
{
Contract.Requires(sourceLeft != null);
this.sourceLeft = sourceLeft;
this.sourceRight = sourceRight;
this.mapper = mapper;
this.keySelector = keySelector;
this.reducer = reducer;
this.resultSelector = resultSelector;
this.leftAsymmetric = leftAsymmetric;
this.reduceInMap = this.sourceLeft.Properties.CanSpray(this.keySelector) && this.sourceLeft.Properties.Derive(a => this.mapper(a, null)).CanSpray(this.keySelector);
if (this.reduceInMap)
{
this.sprayComparer = this.sourceLeft.Properties.GetSprayComparerExpression(this.keySelector);
}
this.isMulticore = this.sourceLeft.Properties.IsMulticore;
}
public PercentileContinuousDoubleAggregate(double percentile, IComparerExpression <double> rankComparer, QueryContainer container)
: base(rankComparer, container)
{
Contract.Requires(rankComparer != null);
Contract.Requires(percentile >= 0.0 && percentile <= 1.0);
this.percentile = percentile;
}
public TumblingMaxAggregate(IComparerExpression <T> comparer)
{
Contract.Requires(comparer != null);
var stateExpression = Expression.Parameter(typeof(MinMaxState <T>), "state");
var timestampExpression = Expression.Parameter(typeof(long), "timestamp");
var inputExpression = Expression.Parameter(typeof(T), "input");
Expression <Func <MinMaxState <T> > > constructor = () => new MinMaxState <T>();
Expression <Func <MinMaxState <T>, long> > currentTimestamp = (state) => state.currentTimestamp;
Expression <Func <MinMaxState <T>, T> > currentValue = (state) => state.currentValue;
var currentTimestampExpression = currentTimestamp.ReplaceParametersInBody(stateExpression);
var comparerExpression = comparer.GetCompareExpr().ReplaceParametersInBody(
inputExpression, currentValue.ReplaceParametersInBody(stateExpression));
var typeInfo = typeof(MinMaxState <T>).GetTypeInfo();
this.accumulate = Expression.Lambda <Func <MinMaxState <T>, long, T, MinMaxState <T> > >(
Expression.Condition(
Expression.OrElse(
Expression.Equal(currentTimestampExpression, Expression.Constant(InvalidSyncTime)),
Expression.GreaterThan(comparerExpression, Expression.Constant(0))),
Expression.MemberInit(
(NewExpression)constructor.Body,
Expression.Bind(typeInfo.GetField("currentTimestamp"), timestampExpression),
Expression.Bind(typeInfo.GetField("currentValue"), inputExpression)),
stateExpression),
stateExpression,
timestampExpression,
inputExpression);
}
/// <summary>
/// Ungroup with group-selector
/// </summary>
internal static StreamProperties <TOuterKey, TResult> Ungroup <TOuterKey, TInnerKey, TPayload, TResult>(
this StreamProperties <CompoundGroupKey <TOuterKey, TInnerKey>, TPayload> source,
Expression <Func <TInnerKey, TPayload, TResult> > groupSelector)
{
IEqualityComparerExpression <TOuterKey> newKeyEqualityComparer = null;
if (source.KeyEqualityComparer != null)
{
newKeyEqualityComparer = ((CompoundGroupKeyEqualityComparer <TOuterKey, TInnerKey>)source.KeyEqualityComparer).outerComparer;
}
IComparerExpression <TOuterKey> newKeyComparer = null;
if (source.KeyComparer != null)
{
newKeyComparer = ((CompoundGroupKeyComparer <TOuterKey, TInnerKey>)source.KeyComparer).outerComparer;
}
return(new StreamProperties <TOuterKey, TResult>(
source.IsColumnar, source.IsConstantDuration, source.ConstantDurationLength,
source.IsConstantHop, source.ConstantHopLength, source.ConstantHopOffset,
source.IsIntervalFree, false,
false, false,
newKeyEqualityComparer,
EqualityComparerExpression <TResult> .Default,
newKeyComparer,
null,
new Dictionary <Expression, object>(),
new Dictionary <Expression, Guid?>(),
source.QueryContainer));
}
internal Map2ReduceStreamable(
IStreamable <TMapKey, TMapInputLeft1> sourceLeft1,
IStreamable <TMapKey, TMapInputRight1> sourceRight1,
Func <IStreamable <TMapKey, TMapInputLeft1>, IStreamable <TMapKey, TMapInputRight1>, IStreamable <TMapKey, TReduceInput1> > mapper1,
Expression <Func <TReduceInput1, TReduceKey> > keySelector1,
IStreamable <TMapKey, TMapInputLeft2> sourceLeft2,
IStreamable <TMapKey, TMapInputRight2> sourceRight2,
Func <IStreamable <TMapKey, TMapInputLeft2>, IStreamable <TMapKey, TMapInputRight2>, IStreamable <TMapKey, TReduceInput2> > mapper2,
Expression <Func <TReduceInput2, TReduceKey> > keySelector2,
Func <IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>, IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>, IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TBind> > reducer,
Expression <Func <TReduceKey, TBind, TOutput> > resultSelector,
bool leftAsymmetric1,
bool leftAsymmetric2,
OperationalHint reduceOptions)
: base(
sourceLeft1.Properties.Map2Reduce(
sourceRight1?.Properties,
sourceLeft2?.Properties,
sourceRight2?.Properties,
mapper1,
mapper2,
reducer,
keySelector1,
keySelector2,
resultSelector,
reduceOptions))
{
Contract.Requires(sourceLeft1 != null);
Contract.Requires(sourceLeft2 != null);
Contract.Requires(reducer != null);
this.sourceLeft1 = sourceLeft1;
this.sourceLeft2 = sourceLeft2;
this.sourceRight1 = sourceRight1;
this.sourceRight2 = sourceRight2;
this.mapper1 = mapper1;
this.mapper2 = mapper2;
this.keySelector1 = keySelector1;
this.keySelector2 = keySelector2;
this.reducer = reducer;
this.resultSelector = resultSelector;
this.leftAsymmetric1 = leftAsymmetric1;
this.leftAsymmetric2 = leftAsymmetric2;
this.reduceOptions = reduceOptions;
if (this.sourceLeft1.Properties.CanSpray(this.sourceLeft2.Properties, this.keySelector1, this.keySelector2) &&
this.sourceLeft1.Properties.Derive(a => this.mapper1(a, null)).CanSpray(
this.sourceLeft2.Properties.Derive(a => this.mapper2(a, null)),
this.keySelector1, this.keySelector2))
{
this.reduceInMap = true;
this.sprayComparer1 = this.sourceLeft1.Properties.GetSprayComparerExpression(this.keySelector1);
}
}
private static IComparerExpression <T> Reverse(IComparerExpression <T> comparer)
{
Contract.Requires(comparer != null);
Expression <Comparison <T> > expression = comparer.GetCompareExpr();
Expression <Comparison <T> > template = (left, right) => CallInliner.Call(expression, right, left);
Expression <Comparison <T> > reversedExpression = template.InlineCalls();
return(new ComparerExpression <T>(reversedExpression));
}
public SlidingMinAggregate(IComparerExpression <T> comparer, QueryContainer container)
{
Contract.Requires(comparer != null);
this.comparer = comparer.GetCompareExpr().Compile();
var generator = comparer.CreateSortedDictionaryGenerator <T, long>(container);
Expression <Func <Func <SortedDictionary <T, long> >, MinMaxState <T> > > template
= (g) => new MinMaxState <T> {
savedValues = new SortedMultiSet <T>(g), currentValue = default, currentTimestamp = InvalidSyncTime
internal void ProcessProperties()
{
if (this.sourceLeft1.Properties.CanSpray(this.sourceLeft2.Properties, this.keySelector1, this.keySelector2) && this.sourceLeft1.Properties.Derive(a => this.mapper1(a, null)).CanSpray(this.sourceLeft2.Properties.Derive(a => this.mapper2(a, null)), this.keySelector1, this.keySelector2))
{
this.reduceInMap = true;
this.sprayComparer1 = this.sourceLeft1.Properties.GetSprayComparerExpression(this.keySelector1);
this.sprayComparer2 = this.sourceLeft2.Properties.GetSprayComparerExpression(this.keySelector2);
}
}
protected SortedMultisetAggregateBase(IComparerExpression <T> comparer, QueryContainer container)
{
var generator = comparer.CreateSortedDictionaryGenerator <T, long>(container);
Expression <Func <Func <SortedDictionary <T, long> >, SortedMultiSet <T> > > template
= (g) => new SortedMultiSet <T>(g);
var replaced = template.ReplaceParametersInBody(generator);
initialState = Expression.Lambda <Func <SortedMultiSet <T> > >(replaced);
}
public TopKAggregate(int k, IComparerExpression <T> rankComparer, IComparerExpression <T> overallComparer, QueryContainer container)
: base(ThenOrderBy(Reverse(rankComparer), overallComparer), container)
{
Contract.Requires(rankComparer != null);
Contract.Requires(overallComparer != null);
Contract.Requires(k > 0);
this.compiledRankComparer = Reverse(rankComparer).GetCompareExpr().Compile();
this.k = k;
}
internal void ProcessProperties()
{
this.reduceInMap = this.sourceLeft.Properties.CanSpray(this.keySelector) && this.sourceLeft.Properties.Derive(a => this.mapper(a, null)).CanSpray(this.keySelector);
if (this.reduceInMap)
{
this.sprayComparer = this.sourceLeft.Properties.GetSprayComparerExpression(this.keySelector);
}
this.isMulticore = this.sourceLeft.Properties.IsMulticore;
}
public static bool TryGetCachedComparer <T>(out IComparerExpression <T> comparer)
{
Type t = typeof(T);
comparer = null;
if (typeComparerCache.TryGetValue(t, out object temp))
{
comparer = (IComparerExpression <T>)temp;
return(true);
}
return(false);
}
protected MinMaxAggregateBase(IComparerExpression <T> comparer, QueryContainer container)
{
Contract.Requires(comparer != null);
var generator = comparer.CreateSortedDictionaryGenerator <T, long>(container);
Expression <Func <Func <SortedDictionary <T, long> >, MinMaxState <T> > > template
= (g) => new MinMaxState <T> {
savedValues = new SortedMultiSet <T>(g)
};
var replaced = template.ReplaceParametersInBody(generator);
this.initialState = Expression.Lambda <Func <MinMaxState <T> > >(replaced);
}
/// <summary>
/// Computes a time-sensitive minimum aggregate using snapshot semantics with the provided ordering comparer.
/// </summary>
public IAggregate <TSource, MinMaxState <TValue>, TValue> Min <TValue>(
Expression <Func <TSource, TValue> > selector, IComparerExpression <TValue> comparer)
{
Invariant.IsNotNull(selector, nameof(selector));
Invariant.IsNotNull(comparer, nameof(comparer));
var aggregate = this.Properties.IsTumbling
? new TumblingMinAggregate <TValue>(comparer)
: this.Properties.IsConstantDuration
? new SlidingMinAggregate <TValue>(comparer, this.Properties.QueryContainer)
: (IAggregate <TValue, MinMaxState <TValue>, TValue>) new MinAggregate <TValue>(comparer, this.Properties.QueryContainer);
return(aggregate.SkipNulls().Wrap(selector).ApplyFilter(this.Filter));
}
private static IComparerExpression <T> ThenOrderBy(IComparerExpression <T> comparer1, IComparerExpression <T> comparer2)
{
Contract.Requires(comparer1 != null);
Contract.Requires(comparer2 != null);
Expression <Comparison <T> > primary = comparer1.GetCompareExpr();
Expression <Comparison <T> > secondary = comparer2.GetCompareExpr();
Expression <Comparison <T> > template =
(left, right) =>
CallInliner.Call(primary, left, right) == 0
? CallInliner.Call(secondary, left, right)
: CallInliner.Call(primary, left, right);
Expression <Comparison <T> > newExpression = template.InlineCalls();
return(new ComparerExpression <T>(newExpression));
}
public SprayGroupImportStreamable(
IStreamable <TKey, TSpray> source,
int totalBranches,
bool multicast = false,
IComparerExpression <TSpray> sprayComparer = null)
: base(source.Properties.ToMulticore(true))
{
Contract.Requires(source != null);
this.totalBranches = totalBranches;
this.Source = source;
this.spraySortOrderComparer = sprayComparer; // source.Properties.PayloadComparer;
this.IsColumnar = source.Properties.IsColumnar;
this.multicast = multicast;
this.observerClassId = Guid.NewGuid();
}
public SynchronousGAPipe(
SprayGroupImportStreamable <TKey, TSpray> stream,
IStreamObserver <TKey, TSpray> observer,
IComparerExpression <TSpray> spraySortOrderComparer,
int totalBranches,
bool multicast)
: base(stream, observer)
{
this.spraySortOrderComparer = spraySortOrderComparer?.GetCompareExpr();
this.spraySortOrderComparerFunc = this.spraySortOrderComparer?.Compile();
this.totalBranches = totalBranches;
this.multicast = multicast;
this.l1_spray = 0;
this.Observers = new List <IStreamObserver <TKey, TSpray> >();
this.pool = MemoryManager.GetMemoryPool <TKey, TSpray>(stream.Properties.IsColumnar);
}
protected MinMaxAggregateBase(IComparerExpression <T> comparer, QueryContainer container)
{
Contract.Requires(comparer != null);
this.generator = comparer.CreateSortedDictionaryGenerator <T, long>(container);
}
public MinAggregate(IComparerExpression <T> comparer, QueryContainer container)
: base(comparer, container)
{
}
public static void Add <T>(IComparerExpression <T> comparer) => typeComparerCache.Add(typeof(T), comparer);
/// <summary>
/// Computes a time-sensitive maximum aggregate using "snapshot windows" (SI terminology) with the provided ordering comparer.
/// </summary>
public IAggregate <TSource, MinMaxState <TValue>, TValue> Max <TValue>(Expression <Func <TSource, TValue> > selector, IComparerExpression <TValue> comparer)
{
Invariant.IsNotNull(selector, nameof(selector));
Invariant.IsNotNull(comparer, nameof(comparer));
if (this.Properties.IsConstantDuration)
{
var aggregate = new FixedIntervalMaxAggregate <TValue>(comparer, this.Properties.QueryContainer);
return(aggregate.SkipNulls().Wrap(selector).ApplyFilter(this.Filter));
}
else
{
var aggregate = new MaxAggregate <TValue>(comparer, this.Properties.QueryContainer);
return(aggregate.SkipNulls().Wrap(selector).ApplyFilter(this.Filter));
}
}
public CompoundGroupKeyComparer(IComparerExpression <TOuterKey> outerComparer, IComparerExpression <TInnerKey> innerComparer)
{
this.outerComparer = outerComparer ?? ComparerExpression <TOuterKey> .Default;
this.innerComparer = innerComparer ?? ComparerExpression <TInnerKey> .Default;
}
public static Expression <Func <SortedDictionary <TKey, TValue> > > CreateSortedDictionaryGenerator <TKey, TValue>(this IComparerExpression <TKey> comparerExp, QueryContainer container)
{
if (ComparerExpression <TKey> .IsSimpleDefault(comparerExp))
{
return(() => new SortedDictionary <TKey, TValue>());
}
var expr = comparerExp.GetCompareExpr();
if (container == null)
{
Expression <Func <Comparison <TKey>, SortedDictionary <TKey, TValue> > > template
= (c) => new SortedDictionary <TKey, TValue>(Comparer <TKey> .Create(c));
var replaced = template.ReplaceParametersInBody(expr);
return(Expression.Lambda <Func <SortedDictionary <TKey, TValue> > >(replaced));
}
var expression = expr.ToString();
var vars = VariableFinder.Find(expr).Select(o => o.GetHashCode());
var captures = vars.Aggregate(expression.GetHashCode(), (a, i) => a ^ i);
var key = Tuple.Create(expression + string.Concat(vars.Select(o => o.ToString(CultureInfo.InvariantCulture))), typeof(TKey), typeof(TValue));
Type temp;
lock (sentinel)
{
if (!DictionaryTypes.TryGetValue(key, out temp))
{
string typeName = Prefix
+ captures.ToString(CultureInfo.InvariantCulture)
+ typeof(TKey).ToString().GetHashCode().ToString(CultureInfo.InvariantCulture)
+ typeof(TValue).ToString().GetHashCode().ToString(CultureInfo.InvariantCulture);
typeName = typeName.Replace("-", "_");
var builderCode = new GeneratedSortedDictionary(typeName).TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(SortedDictionary <,>));
var a = Transformer.CompileSourceCode(builderCode, assemblyReferences, out string errorMessages);
temp = a.GetType(typeName + "`2");
temp = temp.MakeGenericType(typeof(TKey), typeof(TValue));
var init = temp.GetTypeInfo().GetMethod("Initialize", BindingFlags.Static | BindingFlags.Public);
init.Invoke(null, new object[] { Comparer <TKey> .Create(expr.Compile()) });
DictionaryTypes.Add(key, temp);
}
if (!container.TryGetSortedDictionaryType(key, out Type other))
{
container.RegisterSortedDictionaryType(key, temp);
}
}
return(Expression.Lambda <Func <SortedDictionary <TKey, TValue> > >(Expression.New(temp)));
}
public static IComparerExpression <TNewInput> TransformInput <TOldInput, TNewInput>(this IComparerExpression <TOldInput> comparer, Expression <Func <TNewInput, TOldInput> > transform)
{
Contract.Requires(comparer != null);
Contract.Requires(transform != null);
var expression = comparer.GetCompareExpr();
Expression <Comparison <TNewInput> > template =
(left, right) => CallInliner.Call(expression, CallInliner.Call(transform, left), CallInliner.Call(transform, right));
var transformedExpression = template.InlineCalls();
return(new ComparerExpression <TNewInput>(transformedExpression));
}
protected SortedMultisetAggregateBase(IComparerExpression <T> comparer, QueryContainer container) => this.generator = comparer.CreateSortedDictionaryGenerator <T, long>(container);
public FixedIntervalMinAggregate(IComparerExpression <T> comparer, QueryContainer container)
{
Contract.Requires(comparer != null);
this.comparer = comparer.GetCompareExpr().Compile();
this.generator = comparer.CreateSortedDictionaryGenerator <T, long>(container);
}
public static bool ExpressionEquals <T>(this IComparerExpression <T> source, IComparerExpression <T> other) => EqualityComparer.IsEqual(source.GetCompareExpr(), other.GetCompareExpr());
/// <summary>
/// Computes a time-sensitive top-k aggregate using snapshot semantics based on a key selector with the provided ordering comparer.
/// </summary>
public IAggregate <TSource, SortedMultiSet <TSource>, List <RankedEvent <TSource> > > TopK <TOrderValue>(Expression <Func <TSource, TOrderValue> > orderer, IComparerExpression <TOrderValue> comparer, int k)
{
Invariant.IsNotNull(orderer, nameof(orderer));
Invariant.IsNotNull(comparer, nameof(comparer));
Invariant.IsPositive(k, nameof(k));
var orderComparer = comparer.TransformInput(orderer);
var aggregate = new TopKAggregate <TSource>(k, orderComparer, this.Properties.QueryContainer);
return(aggregate.SkipNulls().ApplyFilter(this.Filter));
}
public TopKAggregate(int k, IComparerExpression <T> rankComparer, QueryContainer container)
: this(k, rankComparer, ComparerExpression <T> .Default, container)
{
}
