internal MapReduceStreamable( IStreamable <TMapKey, TMapInputLeft> sourceLeft, IStreamable <TMapKey, TMapInputRight> sourceRight, Func <IStreamable <TMapKey, TMapInputLeft>, IStreamable <TMapKey, TMapInputRight>, IStreamable <TMapKey, TReduceInput> > mapper, Expression <Func <TReduceInput, TReduceKey> > keySelector, IEqualityComparerExpression <TReduceKey> keyComparer, Func <IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>, IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TBind> > reducer, Expression <Func <TReduceKey, TBind, TOutput> > resultSelector, bool leftAsymmetric = false) : base( sourceLeft.Properties.MapReduce (sourceRight?.Properties, mapper, keySelector, reducer, resultSelector)) { Contract.Requires(sourceLeft != null); this.sourceLeft = sourceLeft; this.sourceRight = sourceRight; this.mapper = mapper; this.keySelector = keySelector; this.keyComparer = keyComparer; this.reducer = reducer; this.resultSelector = resultSelector; this.leftAsymmetric = leftAsymmetric; ProcessProperties(); }
/// <summary> /// Ungroup with group-selector /// </summary> internal static StreamProperties <TOuterKey, TResult> Ungroup <TOuterKey, TInnerKey, TPayload, TResult>( this StreamProperties <CompoundGroupKey <TOuterKey, TInnerKey>, TPayload> source, Expression <Func <TInnerKey, TPayload, TResult> > groupSelector) { IEqualityComparerExpression <TOuterKey> newKeyEqualityComparer = null; if (source.KeyEqualityComparer != null) { newKeyEqualityComparer = ((CompoundGroupKeyEqualityComparer <TOuterKey, TInnerKey>)source.KeyEqualityComparer).outerComparer; } IComparerExpression <TOuterKey> newKeyComparer = null; if (source.KeyComparer != null) { newKeyComparer = ((CompoundGroupKeyComparer <TOuterKey, TInnerKey>)source.KeyComparer).outerComparer; } return(new StreamProperties <TOuterKey, TResult>( source.IsColumnar, source.IsConstantDuration, source.ConstantDurationLength, source.IsConstantHop, source.ConstantHopLength, source.ConstantHopOffset, source.IsIntervalFree, false, false, false, newKeyEqualityComparer, EqualityComparerExpression <TResult> .Default, newKeyComparer, null, new Dictionary <Expression, object>(), new Dictionary <Expression, Guid?>(), source.QueryContainer)); }
/// <summary> /// Returns true iff the equality and hashcode functions defined in /// <paramref name="payloadEqualityComparer"/> can be used in columnar codegen. /// </summary> /// <typeparam name="T">The type of the payload</typeparam> /// <param name="payloadEqualityComparer">The equality and hashcode functions /// represented as lambda expressions.</param> /// <returns>True iff both the equality and hashcode functions can be used in columnar /// codegen operators.</returns> public static bool CanUsePayloadEquality <T>(this IEqualityComparerExpression <T> payloadEqualityComparer) { var typeofT = typeof(T); // If T is a struct, then even if the user defined equality function calls a method on the struct, // it can still be used because the active events will have a field of type T. if (typeofT.GetTypeInfo().IsValueType) { return(true); } // If T is a type for which the columnar representation is just a pseudo-field, e.g., string, // then the payload is not really represented as a set of its fields, but is an instance of T if (new ColumnarRepresentation(typeofT).noFields) { return(true); } // If T is a reference type, then codegen is going to define a structurally-equivalent type T' // and the active events will have a field of type T'. So if the equality or hash functions ever // use a value of type T (other than to dereference it for a field/property) then it cannot be used. var equalsIsBad = ParameterInstanceFinder.FoundInstance(payloadEqualityComparer.GetEqualsExpr()); var hashIsBad = ParameterInstanceFinder.FoundInstance(payloadEqualityComparer.GetGetHashCodeExpr()); return(!equalsIsBad && !hashIsBad); }
public LeftAntiSemiJoinStreamable(IStreamable <TKey, TLeft> left, IStreamable <TKey, TRight> right) : base(left.Properties.LASJ(right.Properties), left, right) { Contract.Requires(left != null); Contract.Requires(right != null); this.LeftComparer = left.Properties.PayloadEqualityComparer; Initialize(); }
internal MapReduceStreamable( IStreamable <TMapKey, TMapInputLeft> sourceLeft, IStreamable <TMapKey, TMapInputRight> sourceRight, Func <IStreamable <TMapKey, TMapInputLeft>, IStreamable <TMapKey, TMapInputRight>, IStreamable <TMapKey, TReduceInput> > mapper, Expression <Func <TReduceInput, TReduceKey> > keySelector, IEqualityComparerExpression <TReduceKey> keyComparer, bool leftAsymmetric = false) : this(sourceLeft, sourceRight, mapper, keySelector, keyComparer, null, null, leftAsymmetric) { }
internal Map2ReduceStreamable( IStreamable <TMapKey, TMapInputLeft1> sourceLeft1, IStreamable <TMapKey, TMapInputRight1> sourceRight1, Func <IStreamable <TMapKey, TMapInputLeft1>, IStreamable <TMapKey, TMapInputRight1>, IStreamable <TMapKey, TReduceInput1> > mapper1, Expression <Func <TReduceInput1, TReduceKey> > keySelector1, IStreamable <TMapKey, TMapInputLeft2> sourceLeft2, IStreamable <TMapKey, TMapInputRight2> sourceRight2, Func <IStreamable <TMapKey, TMapInputLeft2>, IStreamable <TMapKey, TMapInputRight2>, IStreamable <TMapKey, TReduceInput2> > mapper2, Expression <Func <TReduceInput2, TReduceKey> > keySelector2, IEqualityComparerExpression <TReduceKey> keyComparer, Func <IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>, IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>, IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TBind> > reducer, Expression <Func <TReduceKey, TBind, TOutput> > resultSelector, bool leftAsymmetric1, bool leftAsymmetric2, OperationalHint reduceOptions) : base( sourceLeft1.Properties.Map2Reduce( sourceRight1?.Properties, sourceLeft2?.Properties, sourceRight2?.Properties, mapper1, mapper2, reducer, keySelector1, keySelector2, resultSelector, reduceOptions)) { Contract.Requires(sourceLeft1 != null); Contract.Requires(sourceLeft2 != null); Contract.Requires(reducer != null); this.sourceLeft1 = sourceLeft1; this.sourceLeft2 = sourceLeft2; this.sourceRight1 = sourceRight1; this.sourceRight2 = sourceRight2; this.mapper1 = mapper1; this.mapper2 = mapper2; this.keySelector1 = keySelector1; this.keySelector2 = keySelector2; this.keyComparer = keyComparer; this.reducer = reducer; this.resultSelector = resultSelector; this.leftAsymmetric1 = leftAsymmetric1; this.leftAsymmetric2 = leftAsymmetric2; this.reduceOptions = reduceOptions; ProcessProperties(); }
public static Func <FastDictionary2 <TKey, TValue> > CreateFastDictionary2Generator <TKey, TValue>( this IEqualityComparerExpression <TKey> comparerExp, int capacity, Func <TKey, TKey, bool> equalsFunc, Func <TKey, int> getHashCodeFunc, QueryContainer container) { if (EqualityComparerExpression <TKey> .IsSimpleDefault(comparerExp)) { return(() => new FastDictionary2 <TKey, TValue>()); } if (container == null) { return(() => new FastDictionary2 <TKey, TValue>(capacity, equalsFunc, getHashCodeFunc)); } var equalsExp = comparerExp.GetEqualsExpr(); var getHashCodeExp = comparerExp.GetGetHashCodeExpr(); var vars = VariableFinder.Find(equalsExp).Select(o => o.GetHashCode()).ToList(); if (!vars.Any()) { vars.Add(string.Empty.StableHash()); } var hashvars = VariableFinder.Find(getHashCodeExp).Select(o => o.GetHashCode()).ToList(); if (!hashvars.Any()) { hashvars.Add(string.Empty.StableHash()); } var key = Tuple.Create( equalsExp.ToString() + getHashCodeExp.ToString() + string.Concat(vars.Aggregate((a, i) => a ^ i)) + string.Concat(hashvars.Aggregate((a, i) => a ^ i)), typeof(TKey), typeof(TValue)); Type temp; lock (sentinel) { if (!generatorCache.TryGetValue(key, out temp)) { string typeName = Prefix + classCounter++; var builderCode = new GeneratedFastDictionary(typeName, "2").TransformText(); var a = Transformer.CompileSourceCode(builderCode, Array.Empty <Assembly>(), out string errorMessages); temp = a.GetType(typeName + "`2"); temp = temp.MakeGenericType(typeof(TKey), typeof(TValue)); MethodInfo init = temp.GetTypeInfo().GetMethod("Initialize", BindingFlags.Static | BindingFlags.Public); init.Invoke(null, new object[] { equalsFunc, getHashCodeFunc, capacity }); generatorCache.Add(key, temp); } if (!container.TryGetFastDictionary2Type(key, out Type other)) { container.RegisterFastDictionary2Type(key, temp); } } return(() => (FastDictionary2 <TKey, TValue>)Activator.CreateInstance(temp)); }
public UngroupStreamable( IEqualityComparerExpression <TOuterKey> comparer, IStreamable <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult> source, Expression <Func <TInnerKey, TInnerResult, TResult> > resultSelector) : base(source.Properties.Ungroup(resultSelector)) { Contract.Requires(source != null); Contract.Requires(resultSelector != null); Source = source; ResultSelector = resultSelector; }
internal static bool IsSimpleDefault(IEqualityComparerExpression <T> input) { if (input != Default) { return(false); } if (input is GenericEqualityComparerExpression <T> ) { return(true); } return(primitiveTypes.Contains(typeof(T))); }
public static bool TryGetCachedComparer <T>(out IEqualityComparerExpression <T> comparer) { Type t = typeof(T); comparer = null; if (typeComparerCache.TryGetValue(t, out object temp)) { comparer = (IEqualityComparerExpression <T>)temp; return(true); } return(false); }
public MapDefinition( IStreamable <TMapKey, TMapInputLeft> sourceLeft, IStreamable <TMapKey, TMapInputRight> sourceRight, Func <IStreamable <TMapKey, TMapInputLeft>, IStreamable <TMapKey, TMapInputRight>, IStreamable <TMapKey, TReduceInput> > mapper, Expression <Func <TReduceInput, TReduceKey> > keySelector, IEqualityComparerExpression <TReduceKey> keyComparer, bool leftAsymmetric = false) { this.sourceLeft = sourceLeft; this.sourceRight = sourceRight; this.mapper = mapper; this.keySelector = keySelector; this.keyComparer = keyComparer; this.leftAsymmetric = leftAsymmetric; }
public ClipJoinStreamable(IStreamable <TKey, TLeft> left, IStreamable <TKey, TRight> right) : base(left.Properties.Clip(right.Properties), left, right) { Contract.Requires(left != null); Contract.Requires(right != null); this.LeftComparer = left.Properties.PayloadEqualityComparer; // This operator uses the equality method on payloads from the left side if (left.Properties.IsColumnar && !this.LeftComparer.CanUsePayloadEquality()) { throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "Type of payload, '{0}', to Clip does not have a valid equality operator for columnar mode.", typeof(TLeft).FullName)); } Initialize(); }
public LeftAntiSemiJoinStreamable(IStreamable <TKey, TLeft> left, IStreamable <TKey, TRight> right) : base(left.Properties.LASJ(right.Properties), left, right) { Contract.Requires(left != null); Contract.Requires(right != null); this.LeftComparer = left.Properties.PayloadEqualityComparer; Initialize(); // This operator uses the equality method on payloads if (this.Properties.IsColumnar && !this.Properties.PayloadEqualityComparer.CanUsePayloadEquality()) { throw new InvalidOperationException($"Type of payload, '{typeof(TLeft).FullName}', to LASJ does not have a valid equality operator for columnar mode."); } }
public static IEqualityComparerExpression <T> GetEqualityComparerExpression <T, T1>( Expression <Func <T, T1> > lambda1, IEqualityComparerExpression <T1> iece1) { var iece1EqualsExpr = iece1.GetEqualsExpr(); Expression <Func <T, T, bool> > equalsTemplate = (left, right) => CallInliner.Call(iece1EqualsExpr, CallInliner.Call(lambda1, left), CallInliner.Call(lambda1, right)); var iece1HashExpr = iece1.GetGetHashCodeExpr(); Expression <Func <T, int> > hashTemplate = value => CallInliner.Call(iece1HashExpr, CallInliner.Call(lambda1, value)); return(new EqualityComparerExpression <T>(equalsTemplate.InlineCalls(), hashTemplate.InlineCalls())); }
public GroupNestedStreamable( IEqualityComparerExpression <TInnerKey> comparer, IStreamable <TOuterKey, TSource> source, Expression <Func <TSource, TInnerKey> > keySelector) : base(source.Properties.GroupNested(keySelector)) { Contract.Requires(source != null); Contract.Requires(keySelector != null); Source = source; KeySelector = keySelector; if (Source.Properties.IsColumnar && !CanGenerateColumnar()) { properties = properties.ToRowBased(); Source = Source.ColumnToRow(); } }
public ShufflecastStreamable( IEqualityComparerExpression <TInnerKey> comparer, IStreamable <TInnerKey, TSource> source, int totalBranchesL2, Expression <Func <TInnerKey, int, TSource, int[]> > destinationSelector = null) : base(source.Properties) { Contract.Requires(source != null); Contract.Requires(totalBranchesL2 > 0); this.Source = source; this.totalBranchesL2 = totalBranchesL2; this.powerOf2 = ((totalBranchesL2 & (totalBranchesL2 - 1)) == 0); if (destinationSelector != null) { this.destinationSelectorCompiled = destinationSelector.Compile(); } }
public ShuffleNestedStreamable( IEqualityComparerExpression <TInnerKey> comparer, IStreamable <TOuterKey, TSource> source, Expression <Func <TSource, TInnerKey> > keySelector, int totalBranchesL2, int shuffleId) : base(source.Properties.GroupNested(keySelector)) { Contract.Requires(source != null); Contract.Requires(keySelector != null); Contract.Requires(totalBranchesL2 > 0); Source = source; KeySelector = keySelector; this.totalBranchesL2 = totalBranchesL2; this.shuffleId = shuffleId; powerOf2 = ((totalBranchesL2 & (totalBranchesL2 - 1)) == 0); if (totalBranchesL2 <= 1) { singleThreadedShuffler = new GroupNestedStreamable <TOuterKey, TSource, TInnerKey>(comparer, source, keySelector); this.properties = singleThreadedShuffler.Properties; } }
public static void Add <T>(IEqualityComparerExpression <T> comparer) => typeComparerCache.Add(typeof(T), comparer);
public BeatStreamable(IStreamable <TKey, TPayload> source, long offset, long period, IEqualityComparerExpression <TPayload> payloadComparer) : base(source, source.Properties, payloadComparer) { Contract.Requires(source != null); Contract.Requires(period > 0); Contract.Requires(payloadComparer != null); this.Offset = offset; this.Period = period; Initialize(); }
public ComparerExpressionForPartitionKey(IEqualityComparerExpression <T> comparer) => this.baseComparer = comparer;
public CompoundGroupKeyEqualityComparer(IEqualityComparerExpression <TOuterKey> outerComparer, IEqualityComparerExpression <TInnerKey> innerComparer) { this.outerComparer = outerComparer ?? EqualityComparerExpression <TOuterKey> .Default; this.innerComparer = innerComparer ?? EqualityComparerExpression <TInnerKey> .Default; }
private static bool IsEqualIECE <T>(IEqualityComparerExpression <T> o1, IEqualityComparerExpression <T> o2) { return(EqualityComparer.IsEqual(o1.GetEqualsExpr(), o2.GetEqualsExpr()) && EqualityComparer.IsEqual(o1.GetGetHashCodeExpr(), o2.GetGetHashCodeExpr())); }
protected UnaryStreamable(IStreamable <TKey, TSource> source, StreamProperties <TKey, TResult> properties, IEqualityComparerExpression <TResult> pcomparer) : base(properties) { Contract.Requires(source != null); this.Source = source; }
public static IEqualityComparerExpression <T> GetCompoundEqualityComparerExpression <T, T1, T2, T3>(Expression <Func <T, T1> > lambda1, IEqualityComparerExpression <T1> iece1, Expression <Func <T, T2> > lambda2, IEqualityComparerExpression <T2> iece2, Expression <Func <T, T3> > lambda3, IEqualityComparerExpression <T3> iece3) { var iece1EqualsExpr = iece1.GetEqualsExpr(); var iece2EqualsExpr = iece2.GetEqualsExpr(); var iece3EqualsExpr = iece3.GetEqualsExpr(); Expression <Func <T, T, bool> > equalsTemplate = (left, right) => CallInliner.Call(iece1EqualsExpr, CallInliner.Call(lambda1, left), CallInliner.Call(lambda1, right)) && CallInliner.Call(iece2EqualsExpr, CallInliner.Call(lambda2, left), CallInliner.Call(lambda2, right)) && CallInliner.Call(iece3EqualsExpr, CallInliner.Call(lambda3, left), CallInliner.Call(lambda3, right)) ; var iece1HashExpr = iece1.GetGetHashCodeExpr(); var iece2HashExpr = iece2.GetGetHashCodeExpr(); var iece3HashExpr = iece3.GetGetHashCodeExpr(); Expression <Func <T, int> > hashTemplate = value => CallInliner.Call(iece1HashExpr, CallInliner.Call(lambda1, value)) ^ CallInliner.Call(iece2HashExpr, CallInliner.Call(lambda2, value)) ^ CallInliner.Call(iece3HashExpr, CallInliner.Call(lambda3, value)) ; return(new EqualityComparerExpression <T>(equalsTemplate.InlineCalls(), hashTemplate.InlineCalls())); }
internal static bool IsSimpleDefault(IEqualityComparerExpression <T> input) => input == Default && input is PrimitiveEqualityComparerExpression <T>;
public static bool ExpressionEquals <T>(this IEqualityComparerExpression <T> source, IEqualityComparerExpression <T> other) { return(EqualityComparer.IsEqual(source.GetEqualsExpr(), other.GetEqualsExpr()) && EqualityComparer.IsEqual(source.GetGetHashCodeExpr(), other.GetGetHashCodeExpr())); }