internal MapReduceStreamable(
IStreamable <TMapKey, TMapInputLeft> sourceLeft,
IStreamable <TMapKey, TMapInputRight> sourceRight,
Func <IStreamable <TMapKey, TMapInputLeft>, IStreamable <TMapKey, TMapInputRight>, IStreamable <TMapKey, TReduceInput> > mapper,
Expression <Func <TReduceInput, TReduceKey> > keySelector,
IEqualityComparerExpression <TReduceKey> keyComparer,
Func <IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>, IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TBind> > reducer,
Expression <Func <TReduceKey, TBind, TOutput> > resultSelector,
bool leftAsymmetric = false)
: base(
sourceLeft.Properties.MapReduce
(sourceRight?.Properties, mapper, keySelector, reducer, resultSelector))
{
Contract.Requires(sourceLeft != null);
this.sourceLeft = sourceLeft;
this.sourceRight = sourceRight;
this.mapper = mapper;
this.keySelector = keySelector;
this.keyComparer = keyComparer;
this.reducer = reducer;
this.resultSelector = resultSelector;
this.leftAsymmetric = leftAsymmetric;
ProcessProperties();
}
/// <summary>
/// Ungroup with group-selector
/// </summary>
internal static StreamProperties <TOuterKey, TResult> Ungroup <TOuterKey, TInnerKey, TPayload, TResult>(
this StreamProperties <CompoundGroupKey <TOuterKey, TInnerKey>, TPayload> source,
Expression <Func <TInnerKey, TPayload, TResult> > groupSelector)
{
IEqualityComparerExpression <TOuterKey> newKeyEqualityComparer = null;
if (source.KeyEqualityComparer != null)
{
newKeyEqualityComparer = ((CompoundGroupKeyEqualityComparer <TOuterKey, TInnerKey>)source.KeyEqualityComparer).outerComparer;
}
IComparerExpression <TOuterKey> newKeyComparer = null;
if (source.KeyComparer != null)
{
newKeyComparer = ((CompoundGroupKeyComparer <TOuterKey, TInnerKey>)source.KeyComparer).outerComparer;
}
return(new StreamProperties <TOuterKey, TResult>(
source.IsColumnar, source.IsConstantDuration, source.ConstantDurationLength,
source.IsConstantHop, source.ConstantHopLength, source.ConstantHopOffset,
source.IsIntervalFree, false,
false, false,
newKeyEqualityComparer,
EqualityComparerExpression <TResult> .Default,
newKeyComparer,
null,
new Dictionary <Expression, object>(),
new Dictionary <Expression, Guid?>(),
source.QueryContainer));
}
/// <summary>
/// Returns true iff the equality and hashcode functions defined in
/// <paramref name="payloadEqualityComparer"/> can be used in columnar codegen.
/// </summary>
/// <typeparam name="T">The type of the payload</typeparam>
/// <param name="payloadEqualityComparer">The equality and hashcode functions
/// represented as lambda expressions.</param>
/// <returns>True iff both the equality and hashcode functions can be used in columnar
/// codegen operators.</returns>
public static bool CanUsePayloadEquality <T>(this IEqualityComparerExpression <T> payloadEqualityComparer)
{
var typeofT = typeof(T);
// If T is a struct, then even if the user defined equality function calls a method on the struct,
// it can still be used because the active events will have a field of type T.
if (typeofT.GetTypeInfo().IsValueType)
{
return(true);
}
// If T is a type for which the columnar representation is just a pseudo-field, e.g., string,
// then the payload is not really represented as a set of its fields, but is an instance of T
if (new ColumnarRepresentation(typeofT).noFields)
{
return(true);
}
// If T is a reference type, then codegen is going to define a structurally-equivalent type T'
// and the active events will have a field of type T'. So if the equality or hash functions ever
// use a value of type T (other than to dereference it for a field/property) then it cannot be used.
var equalsIsBad = ParameterInstanceFinder.FoundInstance(payloadEqualityComparer.GetEqualsExpr());
var hashIsBad = ParameterInstanceFinder.FoundInstance(payloadEqualityComparer.GetGetHashCodeExpr());
return(!equalsIsBad && !hashIsBad);
}
public LeftAntiSemiJoinStreamable(IStreamable <TKey, TLeft> left, IStreamable <TKey, TRight> right)
: base(left.Properties.LASJ(right.Properties), left, right)
{
Contract.Requires(left != null);
Contract.Requires(right != null);
this.LeftComparer = left.Properties.PayloadEqualityComparer;
Initialize();
}
internal MapReduceStreamable(
IStreamable <TMapKey, TMapInputLeft> sourceLeft,
IStreamable <TMapKey, TMapInputRight> sourceRight,
Func <IStreamable <TMapKey, TMapInputLeft>, IStreamable <TMapKey, TMapInputRight>, IStreamable <TMapKey, TReduceInput> > mapper,
Expression <Func <TReduceInput, TReduceKey> > keySelector,
IEqualityComparerExpression <TReduceKey> keyComparer,
bool leftAsymmetric = false)
: this(sourceLeft, sourceRight, mapper, keySelector, keyComparer, null, null, leftAsymmetric)
{
}
internal Map2ReduceStreamable(
IStreamable <TMapKey, TMapInputLeft1> sourceLeft1,
IStreamable <TMapKey, TMapInputRight1> sourceRight1,
Func <IStreamable <TMapKey, TMapInputLeft1>, IStreamable <TMapKey, TMapInputRight1>, IStreamable <TMapKey, TReduceInput1> > mapper1,
Expression <Func <TReduceInput1, TReduceKey> > keySelector1,
IStreamable <TMapKey, TMapInputLeft2> sourceLeft2,
IStreamable <TMapKey, TMapInputRight2> sourceRight2,
Func <IStreamable <TMapKey, TMapInputLeft2>, IStreamable <TMapKey, TMapInputRight2>, IStreamable <TMapKey, TReduceInput2> > mapper2,
Expression <Func <TReduceInput2, TReduceKey> > keySelector2,
IEqualityComparerExpression <TReduceKey> keyComparer,
Func <IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>, IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>, IStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TBind> > reducer,
Expression <Func <TReduceKey, TBind, TOutput> > resultSelector,
bool leftAsymmetric1,
bool leftAsymmetric2,
OperationalHint reduceOptions)
: base(
sourceLeft1.Properties.Map2Reduce(
sourceRight1?.Properties,
sourceLeft2?.Properties,
sourceRight2?.Properties,
mapper1,
mapper2,
reducer,
keySelector1,
keySelector2,
resultSelector,
reduceOptions))
{
Contract.Requires(sourceLeft1 != null);
Contract.Requires(sourceLeft2 != null);
Contract.Requires(reducer != null);
this.sourceLeft1 = sourceLeft1;
this.sourceLeft2 = sourceLeft2;
this.sourceRight1 = sourceRight1;
this.sourceRight2 = sourceRight2;
this.mapper1 = mapper1;
this.mapper2 = mapper2;
this.keySelector1 = keySelector1;
this.keySelector2 = keySelector2;
this.keyComparer = keyComparer;
this.reducer = reducer;
this.resultSelector = resultSelector;
this.leftAsymmetric1 = leftAsymmetric1;
this.leftAsymmetric2 = leftAsymmetric2;
this.reduceOptions = reduceOptions;
ProcessProperties();
}
public static Func <FastDictionary2 <TKey, TValue> > CreateFastDictionary2Generator <TKey, TValue>(
this IEqualityComparerExpression <TKey> comparerExp, int capacity, Func <TKey, TKey, bool> equalsFunc, Func <TKey, int> getHashCodeFunc, QueryContainer container)
{
if (EqualityComparerExpression <TKey> .IsSimpleDefault(comparerExp))
{
return(() => new FastDictionary2 <TKey, TValue>());
}
if (container == null)
{
return(() => new FastDictionary2 <TKey, TValue>(capacity, equalsFunc, getHashCodeFunc));
}
var equalsExp = comparerExp.GetEqualsExpr();
var getHashCodeExp = comparerExp.GetGetHashCodeExpr();
var vars = VariableFinder.Find(equalsExp).Select(o => o.GetHashCode()).ToList();
if (!vars.Any())
{
vars.Add(string.Empty.StableHash());
}
var hashvars = VariableFinder.Find(getHashCodeExp).Select(o => o.GetHashCode()).ToList();
if (!hashvars.Any())
{
hashvars.Add(string.Empty.StableHash());
}
var key =
Tuple.Create(
equalsExp.ToString() + getHashCodeExp.ToString() + string.Concat(vars.Aggregate((a, i) => a ^ i)) + string.Concat(hashvars.Aggregate((a, i) => a ^ i)),
typeof(TKey), typeof(TValue));
Type temp;
lock (sentinel)
{
if (!generatorCache.TryGetValue(key, out temp))
{
string typeName = Prefix + classCounter++;
var builderCode = new GeneratedFastDictionary(typeName, "2").TransformText();
var a = Transformer.CompileSourceCode(builderCode, Array.Empty <Assembly>(), out string errorMessages);
temp = a.GetType(typeName + "`2");
temp = temp.MakeGenericType(typeof(TKey), typeof(TValue));
MethodInfo init = temp.GetTypeInfo().GetMethod("Initialize", BindingFlags.Static | BindingFlags.Public);
init.Invoke(null, new object[] { equalsFunc, getHashCodeFunc, capacity });
generatorCache.Add(key, temp);
}
if (!container.TryGetFastDictionary2Type(key, out Type other))
{
container.RegisterFastDictionary2Type(key, temp);
}
}
return(() => (FastDictionary2 <TKey, TValue>)Activator.CreateInstance(temp));
}
public UngroupStreamable(
IEqualityComparerExpression <TOuterKey> comparer,
IStreamable <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult> source,
Expression <Func <TInnerKey, TInnerResult, TResult> > resultSelector)
: base(source.Properties.Ungroup(resultSelector))
{
Contract.Requires(source != null);
Contract.Requires(resultSelector != null);
Source = source;
ResultSelector = resultSelector;
}
internal static bool IsSimpleDefault(IEqualityComparerExpression <T> input)
{
if (input != Default)
{
return(false);
}
if (input is GenericEqualityComparerExpression <T> )
{
return(true);
}
return(primitiveTypes.Contains(typeof(T)));
}
public static bool TryGetCachedComparer <T>(out IEqualityComparerExpression <T> comparer)
{
Type t = typeof(T);
comparer = null;
if (typeComparerCache.TryGetValue(t, out object temp))
{
comparer = (IEqualityComparerExpression <T>)temp;
return(true);
}
return(false);
}
public MapDefinition(
IStreamable <TMapKey, TMapInputLeft> sourceLeft,
IStreamable <TMapKey, TMapInputRight> sourceRight,
Func <IStreamable <TMapKey, TMapInputLeft>, IStreamable <TMapKey, TMapInputRight>, IStreamable <TMapKey, TReduceInput> > mapper,
Expression <Func <TReduceInput, TReduceKey> > keySelector,
IEqualityComparerExpression <TReduceKey> keyComparer,
bool leftAsymmetric = false)
{
this.sourceLeft = sourceLeft;
this.sourceRight = sourceRight;
this.mapper = mapper;
this.keySelector = keySelector;
this.keyComparer = keyComparer;
this.leftAsymmetric = leftAsymmetric;
}
public ClipJoinStreamable(IStreamable <TKey, TLeft> left, IStreamable <TKey, TRight> right)
: base(left.Properties.Clip(right.Properties), left, right)
{
Contract.Requires(left != null);
Contract.Requires(right != null);
this.LeftComparer = left.Properties.PayloadEqualityComparer;
// This operator uses the equality method on payloads from the left side
if (left.Properties.IsColumnar && !this.LeftComparer.CanUsePayloadEquality())
{
throw new InvalidOperationException(string.Format(CultureInfo.InvariantCulture, "Type of payload, '{0}', to Clip does not have a valid equality operator for columnar mode.", typeof(TLeft).FullName));
}
Initialize();
}
public LeftAntiSemiJoinStreamable(IStreamable <TKey, TLeft> left, IStreamable <TKey, TRight> right)
: base(left.Properties.LASJ(right.Properties), left, right)
{
Contract.Requires(left != null);
Contract.Requires(right != null);
this.LeftComparer = left.Properties.PayloadEqualityComparer;
Initialize();
// This operator uses the equality method on payloads
if (this.Properties.IsColumnar && !this.Properties.PayloadEqualityComparer.CanUsePayloadEquality())
{
throw new InvalidOperationException($"Type of payload, '{typeof(TLeft).FullName}', to LASJ does not have a valid equality operator for columnar mode.");
}
}
public static IEqualityComparerExpression <T> GetEqualityComparerExpression <T, T1>(
Expression <Func <T, T1> > lambda1, IEqualityComparerExpression <T1> iece1)
{
var iece1EqualsExpr = iece1.GetEqualsExpr();
Expression <Func <T, T, bool> > equalsTemplate =
(left, right) =>
CallInliner.Call(iece1EqualsExpr, CallInliner.Call(lambda1, left),
CallInliner.Call(lambda1, right));
var iece1HashExpr = iece1.GetGetHashCodeExpr();
Expression <Func <T, int> > hashTemplate =
value =>
CallInliner.Call(iece1HashExpr, CallInliner.Call(lambda1, value));
return(new EqualityComparerExpression <T>(equalsTemplate.InlineCalls(), hashTemplate.InlineCalls()));
}
public GroupNestedStreamable(
IEqualityComparerExpression <TInnerKey> comparer,
IStreamable <TOuterKey, TSource> source, Expression <Func <TSource, TInnerKey> > keySelector)
: base(source.Properties.GroupNested(keySelector))
{
Contract.Requires(source != null);
Contract.Requires(keySelector != null);
Source = source;
KeySelector = keySelector;
if (Source.Properties.IsColumnar && !CanGenerateColumnar())
{
properties = properties.ToRowBased();
Source = Source.ColumnToRow();
}
}
public ShufflecastStreamable(
IEqualityComparerExpression <TInnerKey> comparer,
IStreamable <TInnerKey, TSource> source,
int totalBranchesL2,
Expression <Func <TInnerKey, int, TSource, int[]> > destinationSelector = null)
: base(source.Properties)
{
Contract.Requires(source != null);
Contract.Requires(totalBranchesL2 > 0);
this.Source = source;
this.totalBranchesL2 = totalBranchesL2;
this.powerOf2 = ((totalBranchesL2 & (totalBranchesL2 - 1)) == 0);
if (destinationSelector != null)
{
this.destinationSelectorCompiled = destinationSelector.Compile();
}
}
public ShuffleNestedStreamable(
IEqualityComparerExpression <TInnerKey> comparer, IStreamable <TOuterKey, TSource> source,
Expression <Func <TSource, TInnerKey> > keySelector, int totalBranchesL2,
int shuffleId)
: base(source.Properties.GroupNested(keySelector))
{
Contract.Requires(source != null);
Contract.Requires(keySelector != null);
Contract.Requires(totalBranchesL2 > 0);
Source = source;
KeySelector = keySelector;
this.totalBranchesL2 = totalBranchesL2;
this.shuffleId = shuffleId;
powerOf2 = ((totalBranchesL2 & (totalBranchesL2 - 1)) == 0);
if (totalBranchesL2 <= 1)
{
singleThreadedShuffler = new GroupNestedStreamable <TOuterKey, TSource, TInnerKey>(comparer, source, keySelector);
this.properties = singleThreadedShuffler.Properties;
}
}
public static void Add <T>(IEqualityComparerExpression <T> comparer) => typeComparerCache.Add(typeof(T), comparer);
public BeatStreamable(IStreamable <TKey, TPayload> source, long offset, long period, IEqualityComparerExpression <TPayload> payloadComparer)
: base(source, source.Properties, payloadComparer)
{
Contract.Requires(source != null);
Contract.Requires(period > 0);
Contract.Requires(payloadComparer != null);
this.Offset = offset;
this.Period = period;
Initialize();
}
public ComparerExpressionForPartitionKey(IEqualityComparerExpression <T> comparer) => this.baseComparer = comparer;
public CompoundGroupKeyEqualityComparer(IEqualityComparerExpression <TOuterKey> outerComparer, IEqualityComparerExpression <TInnerKey> innerComparer)
{
this.outerComparer = outerComparer ?? EqualityComparerExpression <TOuterKey> .Default;
this.innerComparer = innerComparer ?? EqualityComparerExpression <TInnerKey> .Default;
}
private static bool IsEqualIECE <T>(IEqualityComparerExpression <T> o1, IEqualityComparerExpression <T> o2)
{
return(EqualityComparer.IsEqual(o1.GetEqualsExpr(), o2.GetEqualsExpr()) &&
EqualityComparer.IsEqual(o1.GetGetHashCodeExpr(), o2.GetGetHashCodeExpr()));
}
protected UnaryStreamable(IStreamable <TKey, TSource> source, StreamProperties <TKey, TResult> properties, IEqualityComparerExpression <TResult> pcomparer)
: base(properties)
{
Contract.Requires(source != null);
this.Source = source;
}
public static IEqualityComparerExpression <T> GetCompoundEqualityComparerExpression <T, T1, T2, T3>(Expression <Func <T, T1> > lambda1, IEqualityComparerExpression <T1> iece1, Expression <Func <T, T2> > lambda2, IEqualityComparerExpression <T2> iece2, Expression <Func <T, T3> > lambda3, IEqualityComparerExpression <T3> iece3)
{
var iece1EqualsExpr = iece1.GetEqualsExpr();
var iece2EqualsExpr = iece2.GetEqualsExpr();
var iece3EqualsExpr = iece3.GetEqualsExpr();
Expression <Func <T, T, bool> > equalsTemplate =
(left, right) =>
CallInliner.Call(iece1EqualsExpr, CallInliner.Call(lambda1, left), CallInliner.Call(lambda1, right)) &&
CallInliner.Call(iece2EqualsExpr, CallInliner.Call(lambda2, left), CallInliner.Call(lambda2, right)) &&
CallInliner.Call(iece3EqualsExpr, CallInliner.Call(lambda3, left), CallInliner.Call(lambda3, right))
;
var iece1HashExpr = iece1.GetGetHashCodeExpr();
var iece2HashExpr = iece2.GetGetHashCodeExpr();
var iece3HashExpr = iece3.GetGetHashCodeExpr();
Expression <Func <T, int> > hashTemplate =
value =>
CallInliner.Call(iece1HashExpr, CallInliner.Call(lambda1, value)) ^
CallInliner.Call(iece2HashExpr, CallInliner.Call(lambda2, value)) ^
CallInliner.Call(iece3HashExpr, CallInliner.Call(lambda3, value))
;
return(new EqualityComparerExpression <T>(equalsTemplate.InlineCalls(), hashTemplate.InlineCalls()));
}
internal static bool IsSimpleDefault(IEqualityComparerExpression <T> input) => input == Default && input is PrimitiveEqualityComparerExpression <T>;
public static bool ExpressionEquals <T>(this IEqualityComparerExpression <T> source, IEqualityComparerExpression <T> other)
{
return(EqualityComparer.IsEqual(source.GetEqualsExpr(), other.GetEqualsExpr()) &&
EqualityComparer.IsEqual(source.GetGetHashCodeExpr(), other.GetGetHashCodeExpr()));
}
