protected CommonUnaryTemplate(string className, Type keyType, Type payloadType, Type resultType, bool suppressPayloadBatch = false) : base(className) { this.keyType = keyType; this.payloadType = payloadType; this.resultType = resultType; var tm = new TypeMapper(keyType, payloadType, resultType); this.TKey = tm.CSharpNameFor(keyType); this.TPayload = tm.CSharpNameFor(payloadType); this.TResult = tm.CSharpNameFor(resultType); if (!suppressPayloadBatch) { this.TKeyTPayloadGenericParameters = tm.GenericTypeVariables(keyType, payloadType).BracketedCommaSeparatedString(); this.BatchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName(keyType, payloadType); } this.payloadRepresentation = new ColumnarRepresentation(payloadType); this.fields = this.payloadRepresentation.AllFields; this.noFields = this.payloadRepresentation.noFields; this.resultRepresentation = new ColumnarRepresentation(resultType); }
private TemporalEgressTemplate(Type tKey, Type tPayload, Type tResult, string partitionString, string ingressType, bool isColumnar) : base($"GeneratedTemporalEgress_{TemporalEgressSequenceNumber++}") { var tm = new TypeMapper(tKey, tPayload, tResult); this.payloadRepresentation = new ColumnarRepresentation(tPayload); this.BatchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName( tKey == typeof(Empty) ? tKey : typeof(PartitionKey <>).MakeGenericType(tKey), tPayload); this.TKeyTPayloadGenericParameters = tm.GenericTypeVariables(tKey, tPayload).BracketedCommaSeparatedString(); this.fields = this.payloadRepresentation.AllFields; this.TKey = tm.CSharpNameFor(tKey); this.TPayload = tm.CSharpNameFor(tPayload); this.TResult = tm.CSharpNameFor(tResult); this.partitionString = partitionString; this.ingressType = ingressType; this.partitionKeyArgument = !string.IsNullOrEmpty(partitionString) ? "colkey[i].Key, " : string.Empty; this.genericArguments = string.IsNullOrEmpty(partitionString) ? this.TPayload : this.TKey + ", " + this.TPayload; this.egress = (ingressType != "StreamEvent") ? this.TResult : partitionString + "StreamEvent<" + this.genericArguments + ">"; this.inputKey = string.IsNullOrEmpty(partitionString) ? this.TKey : "PartitionKey<" + this.TKey + ">"; this.isColumnar = isColumnar; }
internal MemoryPoolTemplate(ColumnarRepresentation keyRepresentation, ColumnarRepresentation payloadRepresentation) { this.assemblyReferences = new List <Assembly>(); var keyType = keyRepresentation == null ? typeof(Empty) : keyRepresentation.RepresentationFor; Contract.Assume(Transformer.IsValidKeyType(keyType)); this.assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyType)); this.keyType = keyType; #region Decompose TPayload into columns var payloadType = payloadRepresentation.RepresentationFor; this.assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(payloadType)); this.payloadType = payloadType; var payloadTypes = payloadRepresentation.AllFields.Select(f => f.Type).Where(t => !t.MemoryPoolHasGetMethodFor()); this.types = new HashSet <Type>(payloadTypes.Distinct()); this.assemblyReferences.AddRange(this.types.SelectMany(t => Transformer.AssemblyReferencesNeededFor(t))); #endregion this.generatedClassName = Transformer.GetMemoryPoolClassName(keyType, payloadType); this.className = this.generatedClassName.CleanUpIdentifierName(); this.generatedClassName += "`2"; this.expandedCode = TransformText(); }
/// <summary> /// Generate a batch class definition to be used as a Union pipe. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// union pipe class. /// </summary> /// <typeparam name="TKey">The key type for both sides.</typeparam> /// <typeparam name="TPayload">The payload type.</typeparam> /// <returns> /// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TPayload"/>, <typeparamref name="TPayload"/>, <typeparamref name="TKey"/>, <typeparamref name="TPayload"/>>. /// </returns> internal static Tuple <Type, string> GenerateUnionPipeClass <TKey, TPayload>( UnionStreamable <TKey, TPayload> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TPayload, TPayload, TPayload>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); #if CODEGEN_TIMING Stopwatch sw = new Stopwatch(); sw.Start(); #endif var template = new UnionTemplate($"GeneratedUnion_{UnionSequenceNumber++}", typeof(TKey), typeof(TPayload)); var gps = template.tm.GenericTypeVariables(template.keyType, template.resultType); template.genericParameters = gps.BracketedCommaSeparatedString(); var resultRepresentation = new ColumnarRepresentation(template.resultType); var batchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName(template.keyType, template.resultType); var keyAndPayloadGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString(); template.GeneratedBatchName = batchGeneratedFrom_TKey_TPayload + keyAndPayloadGenericParameters; template.fields = resultRepresentation.AllFields; return(template.Generate <TKey, TPayload>()); }
public static void GetGeneratedCode(Type keyType, Type payloadType, out string generatedClassName, out string expandedCode, out List <Assembly> assemblyReferences) { var template = new SafeBatchTemplate(); assemblyReferences = new List <Assembly> { Transformer.SystemRuntimeSerializationDll // needed for [DataContract] and [DataMember] in generated code }; assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyType)); template.keyType = keyType; assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(payloadType)); template.payloadType = payloadType; template.needsPolymorphismCheck = !payloadType.GetTypeInfo().IsValueType&& !payloadType.IsAnonymousTypeName() && !payloadType.GetTypeInfo().IsSealed; template.payloadMightBeNull = payloadType.CanContainNull(); generatedClassName = Transformer.GetBatchClassName(keyType, payloadType); template.CLASSNAME = generatedClassName.CleanUpIdentifierName(); generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(keyType, payloadType); var payloadRepresentation = new ColumnarRepresentation(payloadType); template.fields = payloadRepresentation.AllFields; template.noPublicFields = payloadRepresentation.noFields; expandedCode = template.TransformText(); }
internal static SelectTransformationResult Transform( LambdaExpression function, IEnumerable <Tuple <ParameterExpression, SelectParameterInformation> > substitutionInformation, ColumnarRepresentation resultTypeInformation, bool noSwingingFields = false, bool hasStartEdge = false) { Contract.Requires(function != null); Contract.Requires(substitutionInformation != null); Contract.Requires(resultTypeInformation != null); var me = new SelectTransformer(function, substitutionInformation, resultTypeInformation, noSwingingFields, Config.UseMultiString && ((Config.MultiStringTransforms & Config.CodegenOptions.MultiStringFlags.VectorOperations) != 0), hasStartEdge); // Need to find any unmentioned fields from the result type var unmentionedFields = new List <MyFieldInfo>(); if (me.ProjectionReturningResultInstance == null) { foreach (var kv in resultTypeInformation.Fields) { var fieldInfo = kv.Value; if (me.swingingFields.Any(t => t.Item1.Equals(fieldInfo))) { continue; } if (me.computedFields.Keys.Any(fi => fi.Equals(fieldInfo))) { continue; } if (me.multiStringResultFields.Any(f => f.Equals(fieldInfo))) { continue; } unmentionedFields.Add(fieldInfo); } } return(new SelectTransformationResult() { Error = me.error, ComputedFields = me.computedFields, SwingingFields = me.swingingFields, UnmentionedFields = unmentionedFields, ProjectionReturningResultInstance = me.ProjectionReturningResultInstance, MultiStringOperations = me.multiStringOperations, NeedsSourceInstance = me.needsSourceInstance, }); }
private static LambdaExpression /*?*/ MakeOneParameterColumnar <TKey, TPayload>(LambdaExpression f, int index, string indexVariableName) { Contract.Requires(index >= 0 && index < f.Parameters.Count); var batchType = StreamMessageManager.GetStreamMessageType <TKey, TPayload>(); var d = new Dictionary <ParameterExpression, ColumnOriented.SubstitutionInformation>(); var payloadRepresentation = new ColumnarRepresentation(typeof(TPayload)); d.Add( f.Parameters[index], new ColumnOriented.SubstitutionInformation { columnarRepresentation = payloadRepresentation, nameForIndexVariable = indexVariableName, typeOfBatchVariable = batchType, }); return(ColumnOriented.Transform(f, d)); }
/// <summary> /// Generate a batch class definition to be used as a Clip pipe. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// aggregate class. /// </summary> /// <typeparam name="TKey">The key type for both sides.</typeparam> /// <typeparam name="TLeft">The payload type for the left side.</typeparam> /// <typeparam name="TRight">The payload type for the right side.</typeparam> /// <returns> /// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TKey"/>, <typeparamref name="TRight"/>>. /// </returns> internal static Tuple <Type, string> Generate <TKey, TLeft, TRight>(ClipJoinStreamable <TKey, TLeft, TRight> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TLeft>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); var template = new ClipJoinTemplate($"GeneratedClip_{ClipSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight)); var keyType = typeof(TKey); var leftType = typeof(TLeft); var rightType = typeof(TRight); var gps = template.tm.GenericTypeVariables(keyType, leftType, rightType); template.genericParameters = gps.BracketedCommaSeparatedString(); var resultRepresentation = new ColumnarRepresentation(leftType); template.ActiveEventType = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event"; #region Key Comparer var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparer = (left, right) => keyComparer.Inline(left, right); #endregion #region Left Comparer var leftComparer = stream.LeftComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftComparer, "index_ProcessLeftEvent", 0); template.leftComparer = (left, right) => newLambda.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType); template.TKeyTLeftGenericParameters = template.tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType); template.TKeyTRightGenericParameters = template.tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString(); template.leftFields = resultRepresentation.AllFields; template.noFields = resultRepresentation.noFields; return(template.Generate <TKey, TLeft, TRight>(leftComparer)); }
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new StartEdgeEquiJoinTemplate($"GeneratedStartEdgeEquiJoin_{StartEdgeEquiJoinSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult)); var gps = template.tm.GenericTypeVariables(template.keyType, template.leftType, template.rightType, template.resultType); template.genericParameters = gps.BracketedCommaSeparatedString(); template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType); template.leftFields = template.leftMessageRepresentation.AllFields; template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType); template.rightFields = template.rightMessageRepresentation.AllFields; var resultRepresentation = new ColumnarRepresentation(template.resultType); #region Key Comparer var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparer = (left, right) => keyComparer.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType); template.TKeyTLeftGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType); template.TKeyTRightGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.rightType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType); template.TKeyTResultGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString(); template.outputFields = resultRepresentation.AllFields; var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); #region LeftBatchSelector { var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i"); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var parameterMap = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex }, { selector.Parameters[1], rightEvent } }; if (projectionResult.ProjectionReturningResultInstance != null) { return($"this.output[index] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};"); } else { var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendLine($"this.output.{f.Name}.AddString({e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)});"); } else { sb.AppendLine($"this.output.{f.Name}.col[index] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};"); } } sb.AppendLine("}"); return(sb.ToString()); } }; } #endregion #region RightBatchSelector { var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j"); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var parameterMap = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex } }; if (projectionResult.ProjectionReturningResultInstance != null) { return($"this.output[index] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};"); } else { var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendLine($"this.output.{f.Name}.AddString({e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)});"); } else { sb.AppendLine($"this.output.{f.Name}.col[index] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};"); } } sb.AppendLine("}"); return(sb.ToString()); } }; } #endregion return(template.Generate <TKey, TLeft, TRight, TResult>(selector)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
public static Tuple <Type, string> Generate <TOuterKey, TSource, TInnerKey>( Expression <Func <TSource, TInnerKey> > keySelector, string inlinedHashCodeComputation, bool nested, bool powerOf2) { string errorMessages = null; try { var typeOfTOuterKey = typeof(TOuterKey); var typeOfTSource = typeof(TSource); var typeOfTInnerKey = typeof(TInnerKey); var generatedClassName = string.Format(CultureInfo.InvariantCulture, "ShuffleStreamablePipeGeneratedFrom_{0}_{1}_{2}_{3}", typeOfTOuterKey.GetValidIdentifier(), typeOfTSource.GetValidIdentifier(), typeOfTInnerKey.GetValidIdentifier(), shuffleCounter++); var inputMessageRepresentation = new ColumnarRepresentation(typeOfTSource); var template = new ShuffleTemplate( generatedClassName, typeOfTOuterKey, typeOfTSource, typeOfTInnerKey, inlinedHashCodeComputation, nested, powerOf2) { fields = inputMessageRepresentation.AllFields }; var innerKeyIsAnonymous = typeOfTInnerKey.IsAnonymousTypeName(); if (keySelector != null) { var transformedKeySelector = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector); var keySelectorAsNewExpression = keySelector.Body as NewExpression; if (innerKeyIsAnonymous && keySelectorAsNewExpression != null) { var newPrime = (NewExpression)transformedKeySelector.Body; template.transformedKeySelectorAsString = string.Format(CultureInfo.InvariantCulture, "({0})Activator.CreateInstance(typeof({0}), {1})", template.TInnerKey, string.Join(",", newPrime.Arguments.Select(m => m.ExpressionToCSharp()))); } else { template.transformedKeySelectorAsString = transformedKeySelector.Body.ExpressionToCSharp(); if (Config.UseMultiString && typeOfTInnerKey.Equals(typeof(string)) && keySelector.IsSimpleFieldOrPropertyAccess()) { template.inlinedHashCodeComputation = "hashCodeVector.col[i]"; var fieldName = ((MemberExpression)(keySelector.Body)).Member.Name; template.vectorHashCodeInitialization = string.Format(CultureInfo.InvariantCulture, "var hashCodeVector = {0}{1}_col.GetHashCode(batch.bitvector);", Transformer.ColumnFieldPrefix, fieldName); } } } else { template.transformedKeySelectorAsString = string.Empty; } template.innerKeyIsAnonymous = innerKeyIsAnonymous; template.staticCtor = Transformer.StaticCtor(template.CLASSNAME); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TSource>()); if (nested) { assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>()); } else { assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TInnerKey, TSource>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TInnerKey, TSource>()); } assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector)); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); if (typeOfTInnerKey.IsAnonymousTypeName()) { if (errorMessages == null) { errorMessages = string.Empty; } errorMessages += "\nCodegen Warning: The inner key type for Shuffle is anonymous, causing the use of Activator.CreateInstance in an inner loop. This will lead to poor performance.\n"; } generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey); var t = a.GetType(generatedClassName); return(Tuple.Create(t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey), errorMessages)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
/// <summary> /// Fifth component of tuple is needed when the projection body must be evaluated into a local to hold the result value. /// That is the case when the body looks like "new T(...)" or is the result of a method call. /// </summary> private SelectTransformer( LambdaExpression function, IEnumerable <Tuple <ParameterExpression, SelectParameterInformation> > substitutionInformation, ColumnarRepresentation resultTypeInformation, bool noSwingingFields, bool doMultiStringTransform, bool hasStartEdge) { this.parameterInformation = new Dictionary <ParameterExpression, SelectParameterInformation>(); foreach (var tup in substitutionInformation) { this.parameterInformation.Add(tup.Item1, tup.Item2); } this.resultTypeInformation = resultTypeInformation; this.noSwingingFields = noSwingingFields; this.doMultiStringTransform = doMultiStringTransform; this.swingingFields = new List <Tuple <MyFieldInfo, MyFieldInfo> >(); this.computedFields = new Dictionary <MyFieldInfo, Expression>(); this.multiStringOperations = new List <string>(); this.multiStringResultFields = new List <MyFieldInfo>(); var body = function.Body; // The projection might just be (e_1, e_2, ..., e_n) => e_i, i.e., projecting just the single parameter e_i. if (body is ParameterExpression parameter) { TransformSingleParameterSelect(parameter, hasStartEdge); return; } // The projection might just be (e_1, e_2, ..., e_n) => e_i.f, i.e., projecting just the single field f from one of the parameters e_i. if (body is MemberExpression memberExpression) { TransformSingleFieldSelect(memberExpression); return; } // Case: projection is (e_1, e_2, ..., e_n) => new { f1 = ..., f2 = ..., ...}), i.e., projecting into an anonymous type if (body is NewExpression newExpression && newExpression.Type.IsAnonymousType()) { Contract.Assume(newExpression.Type == resultTypeInformation.RepresentationFor); // REVIEW: Should these be turned into part of the if-test? Contract.Assume(newExpression.Arguments != null); Contract.Assume(newExpression.Members != null); Contract.Assume(newExpression.Arguments.Count == newExpression.Members.Count); TransformAnonymousTypeSelect(newExpression); return; } // Case: projection is (e_1, e_2, ..., e_n) => new T{ f1 = ..., f2 = ..., ... }), i.e., T is *not* an anonymous type // TODO: See if this can be unified with the code above for anonymous types. if (body is MemberInitExpression && !this.resultTypeInformation.noFields) { Visit(body); return; } // Case: projection is (e_1, e_2, ..., e_n) => f(e_1, e_2, ..., e_n)) *AND* the result type is a non-decomposable type, e.g., int. // In that case, the expression does not have to be evaluated into a local, but can just be assigned to the result // column's pseudo-field, "payload". // Note that f is either a real method call or else just an expression // that computes a result value (e.g., a type cast which shows up as a unary expression). if (this.resultTypeInformation.noFields) { if (this.doMultiStringTransform && IsMultiStringCall(body, out string s)) { this.multiStringOperations.Add($"resultBatch.{this.resultTypeInformation.PseudoField.Name} = {s};"); } else { var transformedBody = Visit(body); this.computedFields.Add(this.resultTypeInformation.PseudoField, transformedBody); } return; } // Otherwise, degenerate case: need to just evaluate the expression (with source field access transformed to columnar). // That expression will be passed to the setter for the indexer on the generated batch. // REVIEW: this is where something should be signalled so the user knows it isn't as fast as it could be. var transformedBody2 = Visit(body); this.ProjectionReturningResultInstance = transformedBody2.ExpressionToCSharp(); return; }
protected static EdgeInfo CreateListEdgeInfo <TKey, TPayload, TRegister, TAccumulator>(AfaStreamable <TKey, TPayload, TRegister, TAccumulator> stream, ColumnarRepresentation payloadRepresentation, int targetNodeNumber, ListElementArc <TPayload, TRegister> edge, string indexVariableName) { var edgeInfo = new EdgeInfo() { Type = EdgeInfo.EdgeType.List, EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNodeNumber), SourceNode = targetNodeNumber, Fence = (ts, evs, reg) => edge.Fence.Inline(ts, evs, reg), Transfer = edge.Transfer == null ? ((Func <string, string, string, string>)null) : (ts, evs, reg) => edge.Transfer.Inline(ts, evs, reg), }; return(edgeInfo); }
public static Tuple <Type, string> Generate <TKey, TPayload, TResult>(SelectStreamable <TKey, TPayload, TResult> stream) { Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string generatedClassName; string errorMessages = null; try { var keyType = typeof(TKey); var payloadType = typeof(TPayload); var resultType = typeof(TResult); generatedClassName = string.Format("Select_{0}_{1}_{2}_{3}", keyType.GetValidIdentifier(), payloadType.GetValidIdentifier(), resultType.GetValidIdentifier(), sequenceNumber++); var template = new SelectTemplate(generatedClassName, keyType, payloadType, resultType); var tm = new TypeMapper(keyType, payloadType, resultType); var gps = tm.GenericTypeVariables(keyType, payloadType, resultType); template.genericParameters = gps.BracketedCommaSeparatedString(); template.numberOfGenericParameters = gps.Count(); template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString(); template.MemoryPoolGenericParameters = $"<{template.TKey}, {template.TResult}>"; if (resultType == typeof(int) || resultType == typeof(long) || resultType == typeof(string)) { template.MemoryPoolGenericParameters = string.Empty; } var payloadParameterIndex = 0; if (stream.HasKey && stream.HasStartEdge) { payloadParameterIndex = 2; } else if (stream.HasKey || stream.HasStartEdge) { payloadParameterIndex = 1; } template.PARAMETER = stream.Selector.Parameters[payloadParameterIndex].Name; template.resultPayloadRepresentation = new ColumnarRepresentation(resultType); template.destinationFields = template.resultPayloadRepresentation.AllFields; if (template.numberOfGenericParameters > 0) { generatedClassName = generatedClassName + "`" + template.numberOfGenericParameters.ToString(CultureInfo.InvariantCulture); } var resultSelector = stream.Selector; var sourceMessageType = StreamMessageManager.GetStreamMessageType <TKey, TPayload>(); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); // Don't create a parameter substitution for the start edge parameter. That will just remain in the // body of the result selector and will be set as a local variable in the generated code. if (stream.HasKey) { var keyRepresentation = new ColumnarRepresentation(keyType, "key"); var keyParameterIndex = stream.HasStartEdge ? 1 : 0; parameterSubsitutions.Add(Tuple.Create( resultSelector.Parameters.ElementAt(keyParameterIndex), new SelectParameterInformation() { BatchName = "sourceBatch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = keyRepresentation, })); } parameterSubsitutions.Add(Tuple.Create( resultSelector.Parameters.ElementAt(payloadParameterIndex), new SelectParameterInformation() { BatchName = "sourceBatch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = new ColumnarRepresentation(payloadType) })); var projectionResult = SelectTransformer.Transform(resultSelector, parameterSubsitutions, template.resultPayloadRepresentation, false, stream.HasStartEdge); if (projectionResult.Error) { if (Config.CodegenOptions.SuperStrictColumnar) { throw new InvalidOperationException("Code Generation couldn't transform a selector!"); } return(Tuple.Create((Type)null, errorMessages)); } template.StartEdgeParameterName = stream.HasStartEdge ? resultSelector.Parameters.ElementAt(0).Name : null; template.computedFields = projectionResult.ComputedFields; template.swingingFields = projectionResult.SwingingFields; template.unassignedFields = projectionResult.UnmentionedFields; template.ProjectionReturningResultInstance = projectionResult.ProjectionReturningResultInstance; template.multiStringOperations = projectionResult.MultiStringOperations; template.needSourceInstance = projectionResult.NeedsSourceInstance; var d = new Dictionary <MyFieldInfo, int>(); foreach (var f in template.swingingFields) { var target = f.Item1; var source = f.Item2; if (!d.ContainsKey(source)) { d.Add(source, 0); } d[source] = d[source] + 1; } template.swungFieldsCount = d; template.nonSwingingFields = template.fields.Where(sf => !template.swingingFields.Any(swingingField => swingingField.Item2.Equals(sf))); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor( typeof(TKey), typeof(TPayload), typeof(TResult), typeof(IStreamable <,>)); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>()); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(stream.Selector)); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var t = a.GetType(generatedClassName); t = t.InstantiateAsNecessary(typeof(TKey), typeof(TPayload), typeof(TResult)); return(Tuple.Create(t, errorMessages)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
/// <summary> /// Generate a batch class definition to be used as a Clip pipe. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// aggregate class. /// </summary> /// <typeparam name="TKey">The key type for both sides.</typeparam> /// <typeparam name="TLeft">The payload type for the left side.</typeparam> /// <typeparam name="TRight">The payload type for the right side.</typeparam> /// <returns> /// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TKey"/>, <typeparamref name="TRight"/>>. /// </returns> internal static Tuple <Type, string> Generate <TKey, TLeft, TRight>(ClipJoinStreamable <TKey, TLeft, TRight> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TLeft>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new ClipJoinTemplate(); var keyType = template.keyType = typeof(TKey); var leftType = template.leftType = typeof(TLeft); var rightType = template.rightType = typeof(TRight); var tm = new TypeMapper(keyType, leftType, rightType); template.TKey = tm.CSharpNameFor(keyType); template.TLeft = tm.CSharpNameFor(leftType); template.TRight = tm.CSharpNameFor(rightType); var gps = tm.GenericTypeVariables(keyType, leftType, rightType); template.genericParameters = gps.BracketedCommaSeparatedString(); template.className = string.Format("GeneratedClip_{0}", ClipSequenceNumber++); var resultRepresentation = new ColumnarRepresentation(leftType); template.ActiveEventType = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event"; #region Key Comparer var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparer = (left, right) => keyComparer.Inline(left, right); #endregion #region Left Comparer var leftComparer = stream.LeftComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftComparer, "index_ProcessLeftEvent", 0); template.leftComparer = (left, right) => newLambda.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType); template.TKeyTLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType); template.TKeyTRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString(); template.leftFields = resultRepresentation.AllFields; template.noFields = resultRepresentation.noFields; template.staticCtor = Transformer.StaticCtor(template.className); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight)); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>()); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(leftComparer)); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType); var t = a.GetType(realClassName); if (t.GetTypeInfo().IsGenericType) { var list = keyType.GetAnonymousTypes(); list.AddRange(leftType.GetAnonymousTypes()); list.AddRange(rightType.GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> GenerateAFA <TKey, TPayload, TRegister, TAccumulator>( AfaStreamable <TKey, TPayload, TRegister, TAccumulator> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); var className = string.Format("Generated{1}AfaMultiEventList_{0}", AFASequenceNumber++, typeof(TKey).Equals(typeof(Empty)) ? string.Empty : "Grouped"); var template = new AfaMultiEventListTemplate(className, typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(TAccumulator)) { TKey = typeof(TKey).GetCSharpSourceSyntax() }; var payloadRepresentation = new ColumnarRepresentation(typeof(TPayload)); template.isFinal = stream.afa.isFinal; template.hasOutgoingArcs = stream.afa.hasOutgoingArcs; template.startStates = stream.afa.startStates; template.AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances; template.payloadIsAnon = typeof(TPayload).IsAnonymousTypeName(); template.payloadHasNoFields = payloadRepresentation.noFields; var d1 = stream.afa.uncompiledAfa.transitionInfo; var orderedKeys = d1.Keys.OrderBy(e => e).ToArray(); for (int i = 0; i < orderedKeys.Length; i++) { var sourceNodeNumber = orderedKeys[i]; var outgoingEdgesDictionary = d1[sourceNodeNumber]; var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray(); var edgeList = new List <EdgeInfo>(); for (int j = 0; j < orderedTargetNodes.Length; j++) { var targetNodeNumber = orderedTargetNodes[j]; var edge = outgoingEdgesDictionary[targetNodeNumber]; if (edge is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc) { var multiEdgeInfo = new MultiEdgeInfo() { SourceNode = sourceNodeNumber, TargetNode = targetNodeNumber, fromStartState = stream.afa.startStates.Any(n => n == sourceNodeNumber), EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNodeNumber), Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg), Accumulate = (ts, ev, reg, acc) => multiArc.Accumulate.Inline(ts, ev, reg, acc), Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg), }; multiEdgeInfo.Transfer = multiArc.Transfer == null ? (Func <string, string, string, string>)null : ((ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg)); if (multiArc.Dispose == null) { multiEdgeInfo.Dispose = (acc) => "// no dispose function"; } else { multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc); } multiEdgeInfo.SkipToEnd = multiArc.SkipToEnd == null ? (Func <string, string, string, string>)null : ((ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc)); edgeList.Add(multiEdgeInfo); continue; } if (edge is SingleElementArc <TPayload, TRegister> singleArc) { var edgeInfo = CreateSingleEdgeInfo(stream, targetNodeNumber, singleArc, "payloadoffset"); edgeList.Add(edgeInfo); continue; } if (edge is ListElementArc <TPayload, TRegister> listArc) { var edgeInfo = CreateListEdgeInfo(stream, payloadRepresentation, targetNodeNumber, listArc, "payloadoffset"); edgeList.Add(edgeInfo); continue; } } template.edgeInfos.Add(Tuple.Create(sourceNodeNumber, edgeList)); } for (int i = 0; i < stream.afa.startStates.Length; i++) { var startState = stream.afa.startStates[i]; var edgeList2 = new List <EdgeInfo>(); var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState]; foreach (var edge in outgoingEdgeDictionary) { var targetNode = edge.Key; var arc = edge.Value; if (arc is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc) { var eps = EpsilonClosure(stream.afa, targetNode); var multiEdgeInfo = new MultiEdgeInfo() { SourceNode = startState, TargetNode = targetNode, fromStartState = true, EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNode), Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg), Accumulate = (ts, ev, reg, acc) => multiArc.Accumulate.Inline(ts, ev, reg, acc), Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg), }; multiEdgeInfo.Transfer = multiArc.Transfer == null ? (Func <string, string, string, string>)null : ((ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg)); if (multiArc.Dispose == null) { multiEdgeInfo.Dispose = (acc) => "// no dispose function"; } else { multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc); } multiEdgeInfo.SkipToEnd = multiArc.SkipToEnd == null ? (Func <string, string, string, string>)null : ((ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc)); edgeList2.Add(multiEdgeInfo); continue; } if (arc is SingleElementArc <TPayload, TRegister> singleArc) { var edgeInfo = CreateSingleEdgeInfo(stream, targetNode, singleArc, "payloadoffset"); edgeList2.Add(edgeInfo); continue; } if (arc is ListElementArc <TPayload, TRegister> listArc) { var edgeInfo = CreateListEdgeInfo(stream, payloadRepresentation, targetNode, listArc, "payloadoffset"); edgeList2.Add(edgeInfo); continue; } } template.startEdgeInfos.Add(Tuple.Create(startState, edgeList2)); } template.isSyncTimeSimultaneityFree = true; // The handwritten version doesn't make a distinction. template.keyEqualityComparer = (left, right) => stream.Properties.KeyEqualityComparer.GetEqualsExpr().Inline(left, right); return(template.Generate <TKey, TPayload, TRegister, TAccumulator>()); }
private static Tuple <Type, string> GenerateInternal <TOuterKey, TInnerKey, TInnerResult, TResult>(Expression <Func <TInnerKey, TInnerResult, TResult> > resultSelector, bool isFirstLevelGroup) { Contract.Ensures(Contract.Result <Tuple <Type, string> >() != null); Contract.Ensures(typeof(Pipe <TOuterKey, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var typeOfTOuterKey = typeof(TOuterKey); var typeOfTInnerKey = typeof(TInnerKey); var typeofTInnerResult = typeof(TInnerResult); var typeofTResult = typeof(TResult); string expandedCode; List <Assembly> assemblyReferences; int numberOfGenericParameters; // inline: var ok = UngroupTemplate.GetGeneratedCode<TOuterKey, TInnerKey, TInnerResult, TResult>(resultSelector, false, out generatedClassName, out expandedCode, out assemblyReferences, out numberOfGenericParameters); var generatedClassName = string.Format( "UngroupPipeGeneratedFrom_{0}_{1}_{2}_{3}_{4}", typeOfTOuterKey.GetValidIdentifier(), typeOfTInnerKey.GetValidIdentifier(), typeofTInnerResult.GetValidIdentifier(), typeofTResult.GetValidIdentifier(), UngroupSequenceNumber++); var inputMessageType = StreamMessageManager.GetStreamMessageType <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult>(); var innerResultRepresentation = new ColumnarRepresentation(typeofTInnerResult); var resultRepresentation = new ColumnarRepresentation(typeofTResult); var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> > { // Leave the key parameter to the selector unchanged, so no substitution for parameters[0] Tuple.Create(resultSelector.Parameters[1], new SelectParameterInformation() { BatchName = "inputBatch", BatchType = inputMessageType, IndexVariableName = "i", parameterRepresentation = innerResultRepresentation, }) }; var result = SelectTransformer.Transform(resultSelector, parameterSubstitutions, resultRepresentation); if (result.Error) { return(Tuple.Create((Type)null, errorMessages)); } var template = new UngroupTemplate(generatedClassName, !isFirstLevelGroup, typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult) { innerResultRepresentation = innerResultRepresentation, swingingFields = result.SwingingFields, computedFields = result.ComputedFields, unassignedFields = result.UnmentionedFields, keyParameter = resultSelector.Parameters.First() }; template.staticCtor = Transformer.StaticCtor(template.CLASSNAME); expandedCode = template.TransformText(); assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); // input messages assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult>()); // output messages assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TResult>()); // memory pool assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TOuterKey, TResult>()); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(resultSelector)); numberOfGenericParameters = template.numberOfGenericParameters; var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); if (numberOfGenericParameters > 0) { generatedClassName = generatedClassName + "`" + numberOfGenericParameters.ToString(CultureInfo.InvariantCulture); } var t = a.GetType(generatedClassName); t = t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult); return(Tuple.Create(t, errorMessages)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> GenerateAFA <TPayload, TRegister, TAccumulator>( AfaStreamable <Empty, TPayload, TRegister, TAccumulator> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <Empty, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var className = string.Format("GeneratedUngroupedAfa_{0}", AFASequenceNumber++); var template = new UngroupedAfaTemplate(className, typeof(Empty), typeof(TPayload), typeof(TRegister), typeof(TAccumulator)); var payloadRepresentation = new ColumnarRepresentation(typeof(TPayload)); var registerRepresentation = new ColumnarRepresentation(typeof(TRegister)); template.isFinal = stream.afa.isFinal; template.hasOutgoingArcs = stream.afa.hasOutgoingArcs; template.startStates = stream.afa.startStates; template.AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances; var d1 = stream.afa.uncompiledAfa.transitionInfo; var orderedKeys = d1.Keys.OrderBy(e => e).ToArray(); for (int i = 0; i < orderedKeys.Length; i++) { var sourceNodeNumber = orderedKeys[i]; var outgoingEdgesDictionary = d1[sourceNodeNumber]; var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray(); var edgeList1 = new List <EdgeInfo>(); for (int j = 0; j < orderedTargetNodes.Length; j++) { var targetNodeNumber = orderedTargetNodes[j]; var edge = outgoingEdgesDictionary[targetNodeNumber]; if (edge is SingleElementArc <TPayload, TRegister> searc) { var edgeInfo = CreateSingleEdgeInfo(stream, targetNodeNumber, searc, "i"); edgeList1.Add(edgeInfo); } } template.currentlyActiveInfo.Add(Tuple.Create(sourceNodeNumber, edgeList1)); } for (int i = 0; i < stream.afa.startStates.Length; i++) { var startState = stream.afa.startStates[i]; var edgeList2 = new List <EdgeInfo>(); var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState]; foreach (var edge in outgoingEdgeDictionary) { var targetNode = edge.Key; var arc = edge.Value; if (arc is SingleElementArc <TPayload, TRegister> searc) { var edgeInfo = CreateSingleEdgeInfo(stream, targetNode, searc, "i"); edgeList2.Add(edgeInfo); } } template.newActivationInfo.Add(Tuple.Create(startState, edgeList2)); } template.isSyncTimeSimultaneityFree = stream.Properties.IsSyncTimeSimultaneityFree; var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TPayload), typeof(TRegister), typeof(Stack <>)); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TPayload>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TRegister>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <Empty, TRegister>()); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var t = a.GetType(template.className); if (t.GetTypeInfo().IsGenericType) { var list = new List <Type>(); list.AddRange(template.payloadType.GetAnonymousTypes()); list.AddRange(template.registerType.GetAnonymousTypes()); t = t.MakeGenericType(list.ToArray()); } return(Tuple.Create(t, errorMessages)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new FixedIntervalEquiJoinTemplate($"GeneratedFixedIntervalEquiJoin_{EquiJoinSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult)) { leftDuration = stream.Left.Properties.ConstantDurationLength.Value, rightDuration = stream.Right.Properties.ConstantDurationLength.Value }; var keyAndLeftGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType).BracketedCommaSeparatedString(); var keyAndRightGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.rightType).BracketedCommaSeparatedString(); template.TKeyTResultGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString(); template.genericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType, template.rightType, template.resultType).BracketedCommaSeparatedString(); template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType); template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType); var resultMessageRepresentation = new ColumnarRepresentation(template.resultType); var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType); var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType); template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType); template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters; template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters; template.leftFields = template.leftMessageRepresentation.AllFields; template.rightFields = template.rightMessageRepresentation.AllFields; template.resultFields = resultMessageRepresentation.AllFields; template.ActiveEventTypeLeft = template.leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left"; template.ActiveEventTypeRight = template.rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right"; #region Key Equals var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparerEquals = (left, right) => keyComparer.Inline(left, right); if (template.keyType.IsAnonymousType()) { template.keyComparerEquals = (left, right) => $"keyComparerEquals({left}, {right})"; } #endregion #region Left Payload Equals { var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0); template.leftComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Right Payload Equals { var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0); template.rightComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Result Selector { var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); if (!ConstantExpressionFinder.IsClosedExpression(selector)) { errorMessages = "result selector is not a closed expression"; throw new InvalidOperationException(); } #region LeftBatchSelector { var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i"); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region RightBatchSelector { var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j"); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region ActiveSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.activeSelector = (leftEvent, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion } #endregion return(template.Generate <TKey, TLeft, TRight, TResult>()); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
/// <summary> /// Generate a batch class definition to be used as an aggreate definition. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// aggregate class. /// </summary> /// <typeparam name="TKey">The key type for the aggregate.</typeparam> /// <typeparam name="TInput">The input type for the aggregate.</typeparam> /// <typeparam name="TOutput">The output type for the aggregate.</typeparam> /// <typeparam name="TState">The type for the accumulated state held by the aggregate.</typeparam> /// <typeparam name="TResult">The type of the result.</typeparam> /// <returns> /// A type that is defined to be a subtype of UnaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TInput"/>>. /// </returns> internal static Tuple <Type, string> Generate <TKey, TInput, TState, TOutput, TResult>( GroupedWindowStreamable <TKey, TInput, TState, TOutput, TResult> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(IStreamObserver <Empty, TInput>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { string expandedCode; var template = new GroupedWindowTemplate(); var keyType = template.keyType = typeof(TKey); var inputType = template.inputType = typeof(TInput); var stateType = template.stateType = typeof(TState); var outputType = template.outputType = typeof(TOutput); var resultType = template.resultType = typeof(TResult); template.TResult = resultType.GetCSharpSourceSyntax(); // BUGBUG: need to get any generic parameters needed template.isUngrouped = (keyType == typeof(Empty)); template.className = string.Format("GeneratedGroupedAggregate_{0}", GroupedAggregateSequenceNumber++); var inputMessageRepresentation = new ColumnarRepresentation(inputType); var resultRepresentation = new ColumnarRepresentation(resultType); var assemblyReferences = new List <Assembly>(); #region Key Selector var keySelector = stream.KeySelector; string transformedKeySelectorAsString; if (keyType.IsAnonymousTypeName()) { Contract.Assume(keySelector.Body is NewExpression); var transformedFunction = Extensions.TransformUnaryFunction <TKey, TInput>(keySelector); var newBody = (NewExpression)transformedFunction.Body; transformedKeySelectorAsString = string.Join(",", newBody.Arguments); } else { var transformedFunction = Extensions.TransformUnaryFunction <Empty, TInput>(keySelector).Body; if (transformedFunction == null) { return(null); } transformedKeySelectorAsString = transformedFunction.ExpressionToCSharp(); } template.keySelector = transformedKeySelectorAsString; assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector)); #endregion #region Key Comparer and HashCode var keyComparer = EqualityComparerExpression <TKey> .Default; template.keyComparerEquals = (left, right) => keyComparer.GetEqualsExpr().Inline(left, right); template.keyComparerGetHashCode = (x) => keyComparer.GetGetHashCodeExpr().Inline(x); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyComparer.GetEqualsExpr())); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyComparer.GetGetHashCodeExpr())); #endregion #region Aggregate functions var initialStateLambda = stream.Aggregate.InitialState(); if (ConstantExpressionFinder.IsClosedExpression(initialStateLambda)) { template.initialState = initialStateLambda.Body.ExpressionToCSharp(); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(initialStateLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for GroupedWindow: couldn't inline the initial state lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledInitialState = true; template.initialState = "initialState()"; } } var accumulateLambda = stream.Aggregate.Accumulate(); if (ConstantExpressionFinder.IsClosedExpression(accumulateLambda)) { var accTransformedLambda = Extensions.TransformFunction <TKey, TInput>(accumulateLambda, 2); template.accumulate = (stateArg, longArg) => accTransformedLambda.Inline(stateArg, longArg); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(accumulateLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for GroupedWindow: couldn't inline the accumulate lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledAccumulate = true; template.accumulate = (s1, s2) => string.Format("accumulate({0}, {1}, batch[i]);", s1, s2); } } var deaccumulateLambda = stream.Aggregate.Deaccumulate(); if (ConstantExpressionFinder.IsClosedExpression(deaccumulateLambda)) { var deaccumulateTransformedLambda = Extensions.TransformFunction <TKey, TInput>(deaccumulateLambda, 2); template.deaccumulate = (stateArg, longArg) => deaccumulateTransformedLambda.Inline(stateArg, longArg); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(deaccumulateLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { throw new InvalidOperationException("Code Generation couldn't inline a lambda!"); } else { template.useCompiledDeaccumulate = true; template.deaccumulate = (s1, s2) => string.Format("deaccumulate({0}, {1}, batch[i]);", s1, s2); } } var differenceLambda = stream.Aggregate.Difference(); if (ConstantExpressionFinder.IsClosedExpression(differenceLambda)) { template.difference = (stateArg1, stateArg2) => differenceLambda.Inline(stateArg1, stateArg2); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(differenceLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for GroupedWindow: couldn't inline the deaccumulate lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledDifference = true; template.deaccumulate = (s1, s2) => string.Format("difference({0}, {1});", s1, s2); } } var computeResultLambda = stream.Aggregate.ComputeResult(); if (ConstantExpressionFinder.IsClosedExpression(computeResultLambda)) { if (outputType.IsAnonymousType()) { if (computeResultLambda.Body is NewExpression newExpression) { errorMessages = "Code Generation for GroupedWindow: result selector must be a new expression for anonymous types"; throw new NotImplementedException(errorMessages); } else { template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg); } } else { template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg); } assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(computeResultLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for GroupedWindow: couldn't inline the result selector lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledComputeResult = true; template.computeResult = (stateArg) => "computeResult(" + stateArg + ")"; } } #endregion template.BatchGeneratedFrom_Unit_TInput = Transformer.GetBatchClassName(typeof(Empty), inputType); template.UnitTInputGenericParameters = string.Empty; // BUGBUG template.UnitTResultGenericParameters = string.Empty; // BUGBUG template.inputFields = inputMessageRepresentation.AllFields; template.outputFields = resultRepresentation.AllFields; var resultSelector = stream.ResultSelector; var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); // dont want the parameters substituted for at all var projectionResult = SelectTransformer.Transform(resultSelector, parameterSubstitutions, resultRepresentation, true); if (projectionResult.Error) { return(null); } template.finalResultSelector = (key, aggregateResult) => { var parameters = new Dictionary <ParameterExpression, string> { { resultSelector.Parameters.ElementAt(0), key } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); sb.AppendLine(string.Format("var {0} = {1};\n", resultSelector.Parameters.ElementAt(1).Name, aggregateResult)); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; sb.AppendFormat("this.batch.{0}.col[_c] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(parameters)); } sb.AppendLine("}"); return(sb.ToString()); }; assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(resultSelector)); template.staticCtor = Transformer.StaticCtor(template.className); expandedCode = template.TransformText(); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(typeof(Empty), typeof(TKey), typeof(TInput), typeof(TState), typeof(TOutput), typeof(FastDictionaryGenerator3))); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TInput>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TResult>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <Empty, TResult>()); var assembly = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var t = assembly.GetType(template.className); if (t.GetTypeInfo().IsGenericType) { var list = typeof(TKey).GetAnonymousTypes(); list.AddRange(typeof(TInput).GetAnonymousTypes()); list.AddRange(typeof(TState).GetAnonymousTypes()); list.AddRange(typeof(TOutput).GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new EquiJoinTemplate(); var keyType = template.keyType = typeof(TKey); var leftType = template.leftType = typeof(TLeft); var rightType = template.rightType = typeof(TRight); var resultType = template.resultType = typeof(TResult); var tm = new TypeMapper(keyType, leftType, rightType, resultType); template.TKey = tm.CSharpNameFor(keyType); template.TLeft = tm.CSharpNameFor(leftType); template.TRight = tm.CSharpNameFor(rightType); template.TResult = tm.CSharpNameFor(resultType); var keyAndLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString(); var keyAndRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString(); template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString(); template.genericParameters = tm.GenericTypeVariables(keyType, leftType, rightType, resultType).BracketedCommaSeparatedString(); template.className = string.Format("GeneratedEquiJoin_{0}", EquiJoinSequenceNumber++); template.leftMessageRepresentation = new ColumnarRepresentation(leftType); template.rightMessageRepresentation = new ColumnarRepresentation(rightType); var resultMessageRepresentation = new ColumnarRepresentation(resultType); var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType); var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType); template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType); template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters; template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters; template.leftFields = template.leftMessageRepresentation.AllFields; template.rightFields = template.rightMessageRepresentation.AllFields; template.resultFields = resultMessageRepresentation.AllFields; template.ActiveEventTypeLeft = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left"; template.ActiveEventTypeRight = rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right"; #region Key Equals var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparerEquals = (left, right) => keyComparer.Inline(left, right); if (keyType.IsAnonymousType()) { template.keyComparerEquals = (left, right) => string.Format("keyComparerEquals({0}, {1})", left, right); } #endregion #region Left Payload Equals { var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0); template.leftComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Right Payload Equals { var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0); template.rightComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Result Selector { var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); if (!ConstantExpressionFinder.IsClosedExpression(selector)) { errorMessages = "result selector is not a closed expression"; throw new InvalidOperationException(); } #region LeftBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), "i"), leftIndex }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region RightBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), "j"), rightIndex } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region ActiveSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.activeSelector = (leftEvent, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion } #endregion if (stream.Left.Properties.IsIntervalFree && stream.Right.Properties.IsConstantDuration) { template.endPointHeap = "EndPointQueue"; } else if (stream.Right.Properties.IsIntervalFree && stream.Left.Properties.IsConstantDuration) { template.endPointHeap = "EndPointQueue"; } else if (stream.Left.Properties.IsConstantDuration && stream.Right.Properties.IsConstantDuration && stream.Left.Properties.ConstantDurationLength == stream.Right.Properties.ConstantDurationLength) { template.endPointHeap = "EndPointQueue"; } else { template.endPointHeap = "EndPointHeap"; } template.staticCtor = Transformer.StaticCtor(template.className); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(keyType, leftType, rightType, resultType); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>()); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); if (keyType.IsAnonymousType()) { if (errorMessages == null) { errorMessages = string.Empty; } errorMessages += "\nCodegen Warning: The key type for an equi-join is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n"; } var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType, resultType); var t = a.GetType(realClassName); if (t.GetTypeInfo().IsGenericType) { var list = keyType.GetAnonymousTypes(); list.AddRange(leftType.GetAnonymousTypes()); list.AddRange(rightType.GetAnonymousTypes()); list.AddRange(resultType.GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
/// <summary> /// Generate a batch class definition to be used as a Union pipe. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// union pipe class. /// </summary> /// <typeparam name="TKey">The key type for both sides.</typeparam> /// <typeparam name="TPayload">The payload type.</typeparam> /// <returns> /// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TPayload"/>, <typeparamref name="TPayload"/>, <typeparamref name="TKey"/>, <typeparamref name="TPayload"/>>. /// </returns> internal static Tuple <Type, string> GenerateUnionPipeClass <TKey, TPayload>( UnionStreamable <TKey, TPayload> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TPayload, TPayload, TPayload>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); #if CODEGEN_TIMING Stopwatch sw = new Stopwatch(); sw.Start(); #endif string errorMessages = null; try { var template = new UnionTemplate(); var keyType = template.keyType = typeof(TKey); var payloadType = template.payloadType = typeof(TPayload); var tm = new TypeMapper(keyType, payloadType); template.TKey = tm.CSharpNameFor(keyType); template.TPayload = tm.CSharpNameFor(payloadType); var gps = tm.GenericTypeVariables(keyType, payloadType); template.genericParameters = gps.BracketedCommaSeparatedString(); template.className = string.Format("GeneratedUnion_{0}", UnionSequenceNumber++); var resultRepresentation = new ColumnarRepresentation(payloadType); var batchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName(keyType, payloadType); var keyAndPayloadGenericParameters = tm.GenericTypeVariables(keyType, payloadType).BracketedCommaSeparatedString(); template.GeneratedBatchName = batchGeneratedFrom_TKey_TPayload + keyAndPayloadGenericParameters; template.fields = resultRepresentation.AllFields; template.staticCtor = Transformer.StaticCtor(template.className); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload)); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>()); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, payloadType); var t = a.GetType(realClassName); if (t.GetTypeInfo().IsGenericType) { var list = keyType.GetAnonymousTypes(); list.AddRange(payloadType.GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> GenerateAFA <TKey, TPayload, TRegister, TAccumulator>( AfaStreamable <TKey, TPayload, TRegister, TAccumulator> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var className = string.Format("GeneratedGroupedAfaMultiEvent_{0}", AFASequenceNumber++); var template = new GroupedAfaMultiEventTemplate(className, typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(TAccumulator)); var payloadRepresentation = new ColumnarRepresentation(typeof(TPayload)); template.isFinal = stream.afa.isFinal; template.hasOutgoingArcs = stream.afa.hasOutgoingArcs; template.startStates = stream.afa.startStates; template.AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances; var d1 = stream.afa.uncompiledAfa.transitionInfo; var orderedKeys = d1.Keys.OrderBy(e => e).ToArray(); for (int i = 0; i < orderedKeys.Length; i++) { var sourceNodeNumber = orderedKeys[i]; var outgoingEdgesDictionary = d1[sourceNodeNumber]; var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray(); var edgeList = new List <MultiEdgeInfo>(); for (int j = 0; j < orderedTargetNodes.Length; j++) { var targetNodeNumber = orderedTargetNodes[j]; var edge = outgoingEdgesDictionary[targetNodeNumber]; if (edge is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc) { var multiEdgeInfo = new MultiEdgeInfo() { SourceNode = sourceNodeNumber, TargetNode = targetNodeNumber, fromStartState = stream.afa.startStates.Any(n => n == sourceNodeNumber), EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNodeNumber), Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg), Accumulate = (ts, ev, reg, acc) => { var transformedAccumulate = Extensions.TransformFunction <TKey, TPayload>(multiArc.Accumulate, 1, batchVariableName: "sourceBatch"); return(transformedAccumulate.Inline(ts, reg, acc)); }, Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg), }; if (multiArc.Transfer == null) { multiEdgeInfo.Transfer = null; } else { multiEdgeInfo.Transfer = (ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg); } if (multiArc.Dispose == null) { multiEdgeInfo.Dispose = (acc) => "// no dispose function"; } else { multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc); } if (multiArc.SkipToEnd == null) { multiEdgeInfo.SkipToEnd = null; } else { multiEdgeInfo.SkipToEnd = (ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc); } edgeList.Add(multiEdgeInfo); } } template.edgeInfos.Add(Tuple.Create(sourceNodeNumber, edgeList)); } for (int i = 0; i < stream.afa.startStates.Length; i++) { var startState = stream.afa.startStates[i]; var edgeList2 = new List <MultiEdgeInfo>(); var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState]; foreach (var edge in outgoingEdgeDictionary) { var targetNode = edge.Key; var arc = edge.Value; if (arc is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc) { var eps = EpsilonClosure(stream.afa, targetNode); var multiEdgeInfo = new MultiEdgeInfo() { SourceNode = startState, TargetNode = targetNode, fromStartState = true, EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNode), Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg), Accumulate = (ts, ev, reg, acc) => { var transformedAccumulate = Extensions.TransformFunction <TKey, TPayload>(multiArc.Accumulate, 1, batchVariableName: "sourceBatch"); return(transformedAccumulate.Inline(ts, reg, acc)); }, Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg), }; if (multiArc.Transfer == null) { multiEdgeInfo.Transfer = null; } else { multiEdgeInfo.Transfer = (ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg); } if (multiArc.Dispose == null) { multiEdgeInfo.Dispose = (acc) => "// no dispose function"; } else { multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc); } if (multiArc.SkipToEnd == null) { multiEdgeInfo.SkipToEnd = null; } else { multiEdgeInfo.SkipToEnd = (ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc); } edgeList2.Add(multiEdgeInfo); } } template.startEdgeInfos.Add(Tuple.Create(startState, edgeList2)); } template.isSyncTimeSimultaneityFree = true; // The handwritten version doesn't make a distinction. template.keyEqualityComparer = (left, right) => stream.Properties.KeyEqualityComparer.GetEqualsExpr().Inline(left, right); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TRegister)); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRegister>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TRegister>()); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var t = a.GetType(template.className); if (t.GetTypeInfo().IsGenericType) { var list = typeof(TKey).GetAnonymousTypes(); list.AddRange(template.payloadType.GetAnonymousTypes()); list.AddRange(template.registerType.GetAnonymousTypes()); t = t.MakeGenericType(list.ToArray()); } return(Tuple.Create(t, errorMessages)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
/// <summary> /// Generate a batch class definition to be used as StartEdgeEquiJoin operator. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// aggregate class. /// </summary> /// <typeparam name="TKey">The key type for both sides.</typeparam> /// <typeparam name="TLeft">The payload type for the left side.</typeparam> /// <typeparam name="TRight">The payload type for the right side.</typeparam> /// <typeparam name="TResult">The payload type for the resulting stream.</typeparam> /// <returns> /// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TRight"/>, <typeparamref name="TKey"/>, <typeparamref name="TResult"/>>. /// </returns> internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new IncreasingOrderEquiJoinTemplate($"GeneratedIncreasingOrderEquiJoin_{IOOEJSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult)); template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType); template.leftFields = template.leftMessageRepresentation.AllFields; template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType); template.rightFields = template.rightMessageRepresentation.AllFields; var resultRepresentation = new ColumnarRepresentation(template.resultType); var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); #region Key Comparer var keyComparer = stream.Left.Properties.KeyComparer.GetCompareExpr(); if (!ConstantExpressionFinder.IsClosedExpression(keyComparer)) { return(null); } template.joinKeyOrderComparer = (left, right) => keyComparer.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType); template.TKeyTLeftGenericParameters = string.Empty; // BUGBUG template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType); template.TKeyTRightGenericParameters = string.Empty; // BUGBUG template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType); template.TKeyTResultGenericParameters = string.Empty; // BUGBUG template.outputFields = resultRepresentation.AllFields; if (!ConstantExpressionFinder.IsClosedExpression(selector)) { return(null); } #region LeftBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), "i"), leftIndex }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region RightBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), "j"), rightIndex } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion template.getOutputBatch = string.Format( "pool.Get(out genericOutputBatch); output = ({0}{1})genericOutputBatch;", Transformer.GetBatchClassName(template.keyType, template.resultType), template.TKeyTResultGenericParameters); return(template.Generate <TKey, TLeft, TRight, TResult>(selector)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
public static Tuple <Type, string> Generate <TOuterKey, TSource, TInnerKey>( Expression <Func <TInnerKey, int> > hashComparer, Expression <Func <TSource, TInnerKey> > keySelector, bool nested) { var typeOfTOuterKey = typeof(TOuterKey); var typeOfTSource = typeof(TSource); var typeOfTInnerKey = typeof(TInnerKey); string expandedCode; List <Assembly> assemblyReferences; string errorMessages = null; try { string generatedClassName = $"GeneratedGroupStreamable_{GroupStreamableSequenceNumber++}"; string transformedKeySelectorAsString; MyFieldInfo swingingField = default; if (typeOfTInnerKey.IsAnonymousTypeName()) { Contract.Assume(keySelector.Body is NewExpression); var transformedFunction = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector); var newBody = (NewExpression)transformedFunction.Body; transformedKeySelectorAsString = string.Join(",", newBody.Arguments.Select(arg => arg.ExpressionToCSharp())); } else { var body = keySelector.Body; if (!nested && body is MemberExpression singleFieldProjection) { if (singleFieldProjection.Expression is ParameterExpression dereferencedObject) { // then can just swing a pointer to set the key of the result message Contract.Assume(dereferencedObject == keySelector.Parameters.ElementAt(0)); var f = singleFieldProjection.Member; var sourceMessageRepresentation = new ColumnarRepresentation(typeOfTSource); var fieldToSwingFrom = sourceMessageRepresentation.Fields[f.Name]; swingingField = fieldToSwingFrom; transformedKeySelectorAsString = swingingField.Name + "_col[i]"; } else { var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body; if (transformedPredicate == null) { return(Tuple.Create((Type)null, errorMessages)); } transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp(); } } else { var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body; if (transformedPredicate == null) { return(Tuple.Create((Type)null, errorMessages)); } transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp(); } } var inlinedHashCodeComputation = hashComparer.Inline("key"); var template = new GroupTemplate( generatedClassName, typeOfTOuterKey, typeOfTSource, typeOfTInnerKey, transformedKeySelectorAsString, inlinedHashCodeComputation, nested) { swingingField = swingingField, payloadMightBeNull = typeOfTSource.CanContainNull() }; if (!nested && Config.UseMultiString && typeOfTInnerKey.Equals(typeof(string)) && keySelector.IsSimpleFieldOrPropertyAccess()) { var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body; template.transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp(); template.inlinedHashCodeComputation = "hashCodeVector.col[i]"; var fieldName = ((MemberExpression)keySelector.Body).Member.Name; template.vectorHashCodeInitialization = $"resultBatch.hash = {Transformer.ColumnFieldPrefix}{fieldName}_col.GetHashCode(batch.bitvector);"; template.swingingHashColumn = true; } template.fields = new ColumnarRepresentation(typeOfTSource).AllFields; expandedCode = template.TransformText(); assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TSource>()); if (nested) { assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>()); } else { assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TInnerKey, TSource>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TInnerKey, TSource>()); } assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector)); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); if (typeOfTInnerKey.IsAnonymousTypeName()) { if (errorMessages == null) { errorMessages = string.Empty; } errorMessages += "\nCodegen Warning: The inner key type for Group is anonymous, causing the use of Activator.CreateInstance in an inner loop. This will lead to poor performance.\n"; } generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey); var t = a.GetType(generatedClassName); t = t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey); return(Tuple.Create(t, errorMessages)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new StartEdgeEquiJoinTemplate(); var keyType = template.keyType = typeof(TKey); var leftType = template.leftType = typeof(TLeft); var rightType = template.rightType = typeof(TRight); var resultType = template.resultType = typeof(TResult); var tm = new TypeMapper(keyType, leftType, rightType, resultType); template.TKey = tm.CSharpNameFor(keyType); template.TLeft = tm.CSharpNameFor(leftType); template.TRight = tm.CSharpNameFor(rightType); template.TResult = tm.CSharpNameFor(resultType); var gps = tm.GenericTypeVariables(keyType, leftType, rightType, resultType); template.genericParameters = gps.BracketedCommaSeparatedString(); template.className = string.Format("GeneratedStartEdgeEquiJoin_{0}", StartEdgeEquiJoinSequenceNumber++); template.leftMessageRepresentation = new ColumnarRepresentation(leftType); template.leftFields = template.leftMessageRepresentation.AllFields; template.rightMessageRepresentation = new ColumnarRepresentation(rightType); template.rightFields = template.rightMessageRepresentation.AllFields; var outputMessageRepresentation = new ColumnarRepresentation(resultType); var resultRepresentation = outputMessageRepresentation; #region Key Comparer var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparer = (left, right) => keyComparer.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType); template.TKeyTLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType); template.TKeyTRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType); template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString(); template.outputFields = resultRepresentation.AllFields; var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); #region LeftBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), "i"), leftIndex }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region RightBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), "j"), rightIndex } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion template.staticCtor = Transformer.StaticCtor(template.className); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult)); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>()); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(selector)); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType, resultType); var t = a.GetType(realClassName); if (t.GetTypeInfo().IsGenericType) { var list = keyType.GetAnonymousTypes(); list.AddRange(leftType.GetAnonymousTypes()); list.AddRange(rightType.GetAnonymousTypes()); list.AddRange(resultType.GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }