private bool CanGenerateColumnar() { var typeOfTOuterKey = typeof(TOuterKey); var typeOfTSource = typeof(TSource); var typeOfTInnerKey = typeof(TInnerKey); if (!typeOfTSource.CanRepresentAsColumnar()) { return(false); } if (typeOfTOuterKey.GetPartitionType() != null) { return(false); } if (typeOfTInnerKey.GetPartitionType() != null) { return(false); } var keyEqComparer = Properties.KeyEqualityComparer; string inlinedHashCodeComputation; if (keyEqComparer is CompoundGroupKeyEqualityComparer <TOuterKey, TInnerKey> comparer) { var y = comparer.innerComparer.GetGetHashCodeExpr(); inlinedHashCodeComputation = y.Inline("key"); } else { inlinedHashCodeComputation = keyEqComparer.GetGetHashCodeExpr().Inline("key"); } var lookupKey = CacheKey.Create(inlinedHashCodeComputation, powerOf2); var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => ShuffleTemplate.Generate <TOuterKey, TSource, TInnerKey>(null, inlinedHashCodeComputation, false, this.powerOf2)); errorMessages = generatedPipeType.Item2; return(generatedPipeType.Item1 != null); }
private IStreamObserverAndSameKeyGroupedStreamObservable <TOuterKey, TSource, TOuterKey> GetPipe(IStreamObserver <TOuterKey, TSource> observer, int totalBranchesL2, int shuffleId) { var keyEqComparer = Properties.KeyEqualityComparer; string inlinedHashCodeComputation; if (keyEqComparer is CompoundGroupKeyEqualityComparer <TOuterKey, TInnerKey> comparer) { var y = comparer.innerComparer.GetGetHashCodeExpr(); inlinedHashCodeComputation = y.Inline("key"); } else { inlinedHashCodeComputation = keyEqComparer.GetGetHashCodeExpr().Inline("key"); } var lookupKey = CacheKey.Create(inlinedHashCodeComputation, powerOf2); var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => ShuffleTemplate.Generate <TOuterKey, TSource, TInnerKey>(null, inlinedHashCodeComputation, false, this.powerOf2)); Func <PlanNode, IQueryObject, PlanNode> planNode = ((PlanNode p, IQueryObject o) => new GroupPlanNode( p, o, typeof(TOuterKey), typeof(CompoundGroupKey <TOuterKey, TInnerKey>), typeof(TSource), null, this.shuffleId, this.totalBranchesL2, true, true, generatedPipeType.Item2)); var instance = Activator.CreateInstance(generatedPipeType.Item1, this, observer, totalBranchesL2, shuffleId, planNode); var returnValue = (IStreamObserverAndSameKeyGroupedStreamObservable <TOuterKey, TSource, TOuterKey>)instance; return(returnValue); }
public static Tuple <Type, string> Generate <TOuterKey, TSource, TInnerKey>( Expression <Func <TSource, TInnerKey> > keySelector, string inlinedHashCodeComputation, bool nested, bool powerOf2) { string errorMessages = null; try { var typeOfTOuterKey = typeof(TOuterKey); var typeOfTSource = typeof(TSource); var typeOfTInnerKey = typeof(TInnerKey); var generatedClassName = string.Format(CultureInfo.InvariantCulture, "ShuffleStreamablePipeGeneratedFrom_{0}_{1}_{2}_{3}", typeOfTOuterKey.GetValidIdentifier(), typeOfTSource.GetValidIdentifier(), typeOfTInnerKey.GetValidIdentifier(), shuffleCounter++); var inputMessageRepresentation = new ColumnarRepresentation(typeOfTSource); var template = new ShuffleTemplate( generatedClassName, typeOfTOuterKey, typeOfTSource, typeOfTInnerKey, inlinedHashCodeComputation, nested, powerOf2) { fields = inputMessageRepresentation.AllFields }; var innerKeyIsAnonymous = typeOfTInnerKey.IsAnonymousTypeName(); if (keySelector != null) { var transformedKeySelector = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector); var keySelectorAsNewExpression = keySelector.Body as NewExpression; if (innerKeyIsAnonymous && keySelectorAsNewExpression != null) { var newPrime = (NewExpression)transformedKeySelector.Body; template.transformedKeySelectorAsString = string.Format(CultureInfo.InvariantCulture, "({0})Activator.CreateInstance(typeof({0}), {1})", template.TInnerKey, string.Join(",", newPrime.Arguments.Select(m => m.ExpressionToCSharp()))); } else { template.transformedKeySelectorAsString = transformedKeySelector.Body.ExpressionToCSharp(); if (Config.UseMultiString && typeOfTInnerKey.Equals(typeof(string)) && keySelector.IsSimpleFieldOrPropertyAccess()) { template.inlinedHashCodeComputation = "hashCodeVector.col[i]"; var fieldName = ((MemberExpression)(keySelector.Body)).Member.Name; template.vectorHashCodeInitialization = string.Format(CultureInfo.InvariantCulture, "var hashCodeVector = {0}{1}_col.GetHashCode(batch.bitvector);", Transformer.ColumnFieldPrefix, fieldName); } } } else { template.transformedKeySelectorAsString = string.Empty; } template.innerKeyIsAnonymous = innerKeyIsAnonymous; template.staticCtor = Transformer.StaticCtor(template.CLASSNAME); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TSource>()); if (nested) { assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>()); } else { assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TInnerKey, TSource>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TInnerKey, TSource>()); } assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector)); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); if (typeOfTInnerKey.IsAnonymousTypeName()) { if (errorMessages == null) { errorMessages = string.Empty; } errorMessages += "\nCodegen Warning: The inner key type for Shuffle is anonymous, causing the use of Activator.CreateInstance in an inner loop. This will lead to poor performance.\n"; } generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey); var t = a.GetType(generatedClassName); return(Tuple.Create(t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey), errorMessages)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }