private bool CanGenerateColumnar()
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTSource = typeof(TSource);
var typeOfTInnerKey = typeof(TInnerKey);
if (!typeOfTSource.CanRepresentAsColumnar())
{
return(false);
}
if (typeOfTOuterKey.GetPartitionType() != null)
{
return(false);
}
if (typeOfTInnerKey.GetPartitionType() != null)
{
return(false);
}
var keyEqComparer = Properties.KeyEqualityComparer;
string inlinedHashCodeComputation;
if (keyEqComparer is CompoundGroupKeyEqualityComparer <TOuterKey, TInnerKey> comparer)
{
var y = comparer.innerComparer.GetGetHashCodeExpr();
inlinedHashCodeComputation = y.Inline("key");
}
else
{
inlinedHashCodeComputation = keyEqComparer.GetGetHashCodeExpr().Inline("key");
}
var lookupKey = CacheKey.Create(inlinedHashCodeComputation, powerOf2);
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => ShuffleTemplate.Generate <TOuterKey, TSource, TInnerKey>(null, inlinedHashCodeComputation, false, this.powerOf2));
errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
private IStreamObserverAndSameKeyGroupedStreamObservable <TOuterKey, TSource, TOuterKey> GetPipe(IStreamObserver <TOuterKey, TSource> observer, int totalBranchesL2, int shuffleId)
{
var keyEqComparer = Properties.KeyEqualityComparer;
string inlinedHashCodeComputation;
if (keyEqComparer is CompoundGroupKeyEqualityComparer <TOuterKey, TInnerKey> comparer)
{
var y = comparer.innerComparer.GetGetHashCodeExpr();
inlinedHashCodeComputation = y.Inline("key");
}
else
{
inlinedHashCodeComputation = keyEqComparer.GetGetHashCodeExpr().Inline("key");
}
var lookupKey = CacheKey.Create(inlinedHashCodeComputation, powerOf2);
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => ShuffleTemplate.Generate <TOuterKey, TSource, TInnerKey>(null, inlinedHashCodeComputation, false, this.powerOf2));
Func <PlanNode, IQueryObject, PlanNode> planNode = ((PlanNode p, IQueryObject o) => new GroupPlanNode(
p,
o,
typeof(TOuterKey),
typeof(CompoundGroupKey <TOuterKey, TInnerKey>),
typeof(TSource),
null,
this.shuffleId,
this.totalBranchesL2,
true,
true,
generatedPipeType.Item2));
var instance = Activator.CreateInstance(generatedPipeType.Item1, this, observer, totalBranchesL2, shuffleId, planNode);
var returnValue = (IStreamObserverAndSameKeyGroupedStreamObservable <TOuterKey, TSource, TOuterKey>)instance;
return(returnValue);
}
public static Tuple <Type, string> Generate <TOuterKey, TSource, TInnerKey>(
Expression <Func <TSource, TInnerKey> > keySelector,
string inlinedHashCodeComputation,
bool nested,
bool powerOf2)
{
string errorMessages = null;
try
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTSource = typeof(TSource);
var typeOfTInnerKey = typeof(TInnerKey);
var generatedClassName = string.Format(CultureInfo.InvariantCulture, "ShuffleStreamablePipeGeneratedFrom_{0}_{1}_{2}_{3}", typeOfTOuterKey.GetValidIdentifier(), typeOfTSource.GetValidIdentifier(), typeOfTInnerKey.GetValidIdentifier(), shuffleCounter++);
var inputMessageRepresentation = new ColumnarRepresentation(typeOfTSource);
var template = new ShuffleTemplate(
generatedClassName,
typeOfTOuterKey,
typeOfTSource,
typeOfTInnerKey,
inlinedHashCodeComputation,
nested, powerOf2)
{
fields = inputMessageRepresentation.AllFields
};
var innerKeyIsAnonymous = typeOfTInnerKey.IsAnonymousTypeName();
if (keySelector != null)
{
var transformedKeySelector = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector);
var keySelectorAsNewExpression = keySelector.Body as NewExpression;
if (innerKeyIsAnonymous && keySelectorAsNewExpression != null)
{
var newPrime = (NewExpression)transformedKeySelector.Body;
template.transformedKeySelectorAsString = string.Format(CultureInfo.InvariantCulture, "({0})Activator.CreateInstance(typeof({0}), {1})",
template.TInnerKey,
string.Join(",", newPrime.Arguments.Select(m => m.ExpressionToCSharp())));
}
else
{
template.transformedKeySelectorAsString = transformedKeySelector.Body.ExpressionToCSharp();
if (Config.UseMultiString &&
typeOfTInnerKey.Equals(typeof(string)) &&
keySelector.IsSimpleFieldOrPropertyAccess())
{
template.inlinedHashCodeComputation = "hashCodeVector.col[i]";
var fieldName = ((MemberExpression)(keySelector.Body)).Member.Name;
template.vectorHashCodeInitialization = string.Format(CultureInfo.InvariantCulture, "var hashCodeVector = {0}{1}_col.GetHashCode(batch.bitvector);", Transformer.ColumnFieldPrefix, fieldName);
}
}
}
else
{
template.transformedKeySelectorAsString = string.Empty;
}
template.innerKeyIsAnonymous = innerKeyIsAnonymous;
template.staticCtor = Transformer.StaticCtor(template.CLASSNAME);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TSource>());
if (nested)
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
}
else
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TInnerKey, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TInnerKey, TSource>());
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (typeOfTInnerKey.IsAnonymousTypeName())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The inner key type for Shuffle is anonymous, causing the use of Activator.CreateInstance in an inner loop. This will lead to poor performance.\n";
}
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
var t = a.GetType(generatedClassName);
return(Tuple.Create(t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey), errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
