/// <summary> /// Generate a batch class definition to be used as a Beat pipe. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// Beat pipe class. /// </summary> /// <typeparam name="TKey">The key type for both sides.</typeparam> /// <typeparam name="TPayload">The payload type.</typeparam> /// <returns> /// A type that is defined to be a subtype of UnaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TPayload"/>, <typeparamref name="TPayload"/>>. /// </returns> internal static Tuple <Type, string> Generate <TKey, TPayload>(BeatStreamable <TKey, TPayload> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TPayload>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); #if CODEGEN_TIMING Stopwatch sw = new Stopwatch(); sw.Start(); #endif var template = new BeatTemplate( string.Format("GeneratedBeat_{0}", BeatSequenceNumber++), typeof(TKey), typeof(TPayload)); template.ActiveEventType = typeof(TPayload).GetTypeInfo().IsValueType ? template.TPayload : "Active_Event"; #region Key Comparer var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparer = (left, right) => keyComparer.Inline(left, right); #endregion #region Payload Comparer var payloadComparer = stream.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TPayload>(payloadComparer, "index"); template.payloadComparer = (left, right) => newLambda.Inline(left, right); #endregion return(template.Generate <TKey, TPayload>(typeof(IStreamable <,>))); }
internal static Tuple <Type, string> Generate <TKey, TPayload>(ExtendLifetimeStreamable <TKey, TPayload> stream, long duration, bool useCompiledKeyComparer, bool useCompiledPayloadComparer) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TPayload>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); #if CODEGEN_TIMING Stopwatch sw = new Stopwatch(); sw.Start(); #endif ExtendLifetimeBaseTemplate template; var className = string.Format("GeneratedExtendLifetime_{0}", ExtendLifetimeSequenceNumber++); if (duration < 0) { template = new ExtendLifetimeNegativeTemplate(className, typeof(TKey), typeof(TPayload)); } else { template = new ExtendLifetimeTemplate(className, typeof(TKey), typeof(TPayload)); } template.ActiveEventType = typeof(TPayload).GetTypeInfo().IsValueType ? template.TPayload : "Active_Event"; #region Key Comparer template.useCompiledKeyComparer = useCompiledKeyComparer; if (useCompiledKeyComparer) { template.keyComparer = (left, right) => string.Format("keyComparer({0}, {1}", left, right); } else { var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparer = (left, right) => keyComparer.Inline(left, right); } #endregion #region Payload Comparer template.useCompiledPayloadComparer = useCompiledPayloadComparer; if (useCompiledPayloadComparer) { template.payloadComparer = (left, right) => string.Format("payloadComparer({0}.Payload, {1}.Payload", left, right); } else { var payloadComparer = stream.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TPayload>(payloadComparer, "index"); template.payloadComparer = (left, right) => newLambda.Inline(left, right); } #endregion return(template.Generate <TKey, TPayload>(typeof(IStreamable <,>))); }
/// <summary> /// Generate a batch class definition to be used as a Clip pipe. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// aggregate class. /// </summary> /// <typeparam name="TKey">The key type for both sides.</typeparam> /// <typeparam name="TLeft">The payload type for the left side.</typeparam> /// <typeparam name="TRight">The payload type for the right side.</typeparam> /// <returns> /// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TKey"/>, <typeparamref name="TRight"/>>. /// </returns> internal static Tuple <Type, string> Generate <TKey, TLeft, TRight>(ClipJoinStreamable <TKey, TLeft, TRight> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TLeft>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); var template = new ClipJoinTemplate($"GeneratedClip_{ClipSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight)); var keyType = typeof(TKey); var leftType = typeof(TLeft); var rightType = typeof(TRight); var gps = template.tm.GenericTypeVariables(keyType, leftType, rightType); template.genericParameters = gps.BracketedCommaSeparatedString(); var resultRepresentation = new ColumnarRepresentation(leftType); template.ActiveEventType = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event"; #region Key Comparer var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparer = (left, right) => keyComparer.Inline(left, right); #endregion #region Left Comparer var leftComparer = stream.LeftComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftComparer, "index_ProcessLeftEvent", 0); template.leftComparer = (left, right) => newLambda.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType); template.TKeyTLeftGenericParameters = template.tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType); template.TKeyTRightGenericParameters = template.tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString(); template.leftFields = resultRepresentation.AllFields; template.noFields = resultRepresentation.noFields; return(template.Generate <TKey, TLeft, TRight>(leftComparer)); }
public static Tuple <Type, string> Generate <TKey, TPayload, TResult>(SelectManyStreamable <TKey, TPayload, TResult> stream) { Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string generatedClassName; string expandedCode; string errorMessages = null; try { generatedClassName = $"SelectMany_{sequenceNumber++}"; var keyType = typeof(TKey); var payloadType = typeof(TPayload); var resultType = typeof(TResult); var template = new SelectManyTemplate(generatedClassName, keyType, payloadType, resultType); var tm = new TypeMapper(keyType, payloadType, resultType); var gps = tm.GenericTypeVariables(keyType, payloadType, resultType); template.genericParameters = gps.BracketedCommaSeparatedString(); template.numberOfGenericParameters = gps.Count(); template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString(); template.MemoryPoolGenericParameters = $"<{template.TKey}, {template.TResult}>"; if (resultType == typeof(int) || resultType == typeof(long) || resultType == typeof(string)) { template.MemoryPoolGenericParameters = string.Empty; } var payloadParameterIndex = 0; if (stream.HasKey && stream.HasStartEdge) { payloadParameterIndex = 2; } else if (stream.HasKey || stream.HasStartEdge) { payloadParameterIndex = 1; } var selector = stream.Selector; var payloadParameter = selector.Parameters.ElementAt(payloadParameterIndex); template.PARAMETER = payloadParameter.Name; template.resultPayloadRepresentation = new ColumnarRepresentation(resultType); template.resultFields = template.resultPayloadRepresentation.AllFields; if (template.numberOfGenericParameters > 0) { generatedClassName = generatedClassName + "`" + template.numberOfGenericParameters.ToString(CultureInfo.InvariantCulture); } expandedCode = string.Empty; Expression transformedSelector = selector; // No substitutions are made for the start edge parameter or key parameter. Both just remain in the // body of the result selector and are set as local variables in the generated code. var keyParameterIndex = stream.HasStartEdge ? 1 : 0; var tuple = OptimizeSelectMany(selector.Body); if (tuple != null) { template.enumerableRepeatSelector = true; var resultSelector = stream.Selector; var sourceMessageType = StreamMessageManager.GetStreamMessageType <TKey, TPayload>(); var pseudoLambdaParameters = new ParameterExpression[stream.HasKey ? 2 : 1]; var pseudoLambdaIndex = 0; if (stream.HasKey) { pseudoLambdaParameters[pseudoLambdaIndex++] = selector.Parameters[keyParameterIndex]; } pseudoLambdaParameters[pseudoLambdaIndex] = payloadParameter; var pseudoLambda = Expression.Lambda(tuple.Item1, pseudoLambdaParameters); var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> > { Tuple.Create(payloadParameter, new SelectParameterInformation() { BatchName = "batch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = new ColumnarRepresentation(payloadType) }) }; var projectionResult = SelectTransformer.Transform(pseudoLambda, parameterSubstitutions, template.resultPayloadRepresentation, true, stream.HasStartEdge); template.computedFields = projectionResult.ComputedFields; template.useEnumerator = false; var loopCounter = tuple.Item2; var newParameters = new ParameterExpression[stream.HasKey ? 2 : 1]; newParameters[0] = payloadParameter; if (stream.HasKey) { newParameters[1] = selector.Parameters[keyParameterIndex]; } var loopCounterLambda = Expression.Lambda(loopCounter, payloadParameter); var transformedLoopCounter = Extensions.TransformFunction <TKey, TPayload>(loopCounterLambda, 0); template.loopCounter = transformedLoopCounter.Body.ExpressionToCSharp(); // REVIEW: Alternative: use Inline to replace occurrences of the key parameter // with "batch.key.col[i]". if (stream.HasKey) { template.keyParameterName = selector.Parameters[keyParameterIndex].Name; } } else { transformedSelector = Extensions.TransformFunction <TKey, TPayload>(stream.Selector, payloadParameterIndex).Body; if (transformedSelector == null) { template.useEnumerator = true; template.transformedSelectorAsSource = stream.Selector.ExpressionToCSharp(); } else { var tuple2 = OptimizeSelectMany(transformedSelector); if (tuple2 != null) { template.useEnumerator = false; template.loopCounter = tuple2.Item2.ExpressionToCSharp(); template.transformedSelectorAsSource = tuple2.Item1.ExpressionToCSharp(); } else { template.useEnumerator = true; template.transformedSelectorAsSource = transformedSelector.ExpressionToCSharp(); } } } template.StartEdgeParameterName = stream.HasStartEdge ? selector.Parameters.ElementAt(0).Name : null; template.hasKey = stream.HasKey; expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TResult)); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>()); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(stream.Selector)); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var t = a.GetType(generatedClassName); t = t.InstantiateAsNecessary(typeof(TKey), typeof(TPayload), typeof(TResult)); return(Tuple.Create(t, errorMessages)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> GenerateAFA <TKey, TPayload, TRegister, TAccumulator>( AfaStreamable <TKey, TPayload, TRegister, TAccumulator> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var className = string.Format("GeneratedGroupedAfaMultiEvent_{0}", AFASequenceNumber++); var template = new GroupedAfaMultiEventTemplate(className, typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(TAccumulator)); var payloadRepresentation = new ColumnarRepresentation(typeof(TPayload)); template.isFinal = stream.afa.isFinal; template.hasOutgoingArcs = stream.afa.hasOutgoingArcs; template.startStates = stream.afa.startStates; template.AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances; var d1 = stream.afa.uncompiledAfa.transitionInfo; var orderedKeys = d1.Keys.OrderBy(e => e).ToArray(); for (int i = 0; i < orderedKeys.Length; i++) { var sourceNodeNumber = orderedKeys[i]; var outgoingEdgesDictionary = d1[sourceNodeNumber]; var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray(); var edgeList = new List <MultiEdgeInfo>(); for (int j = 0; j < orderedTargetNodes.Length; j++) { var targetNodeNumber = orderedTargetNodes[j]; var edge = outgoingEdgesDictionary[targetNodeNumber]; if (edge is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc) { var multiEdgeInfo = new MultiEdgeInfo() { SourceNode = sourceNodeNumber, TargetNode = targetNodeNumber, fromStartState = stream.afa.startStates.Any(n => n == sourceNodeNumber), EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNodeNumber), Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg), Accumulate = (ts, ev, reg, acc) => { var transformedAccumulate = Extensions.TransformFunction <TKey, TPayload>(multiArc.Accumulate, 1, batchVariableName: "sourceBatch"); return(transformedAccumulate.Inline(ts, reg, acc)); }, Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg), }; if (multiArc.Transfer == null) { multiEdgeInfo.Transfer = null; } else { multiEdgeInfo.Transfer = (ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg); } if (multiArc.Dispose == null) { multiEdgeInfo.Dispose = (acc) => "// no dispose function"; } else { multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc); } if (multiArc.SkipToEnd == null) { multiEdgeInfo.SkipToEnd = null; } else { multiEdgeInfo.SkipToEnd = (ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc); } edgeList.Add(multiEdgeInfo); } } template.edgeInfos.Add(Tuple.Create(sourceNodeNumber, edgeList)); } for (int i = 0; i < stream.afa.startStates.Length; i++) { var startState = stream.afa.startStates[i]; var edgeList2 = new List <MultiEdgeInfo>(); var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState]; foreach (var edge in outgoingEdgeDictionary) { var targetNode = edge.Key; var arc = edge.Value; if (arc is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc) { var eps = EpsilonClosure(stream.afa, targetNode); var multiEdgeInfo = new MultiEdgeInfo() { SourceNode = startState, TargetNode = targetNode, fromStartState = true, EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNode), Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg), Accumulate = (ts, ev, reg, acc) => { var transformedAccumulate = Extensions.TransformFunction <TKey, TPayload>(multiArc.Accumulate, 1, batchVariableName: "sourceBatch"); return(transformedAccumulate.Inline(ts, reg, acc)); }, Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg), }; if (multiArc.Transfer == null) { multiEdgeInfo.Transfer = null; } else { multiEdgeInfo.Transfer = (ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg); } if (multiArc.Dispose == null) { multiEdgeInfo.Dispose = (acc) => "// no dispose function"; } else { multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc); } if (multiArc.SkipToEnd == null) { multiEdgeInfo.SkipToEnd = null; } else { multiEdgeInfo.SkipToEnd = (ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc); } edgeList2.Add(multiEdgeInfo); } } template.startEdgeInfos.Add(Tuple.Create(startState, edgeList2)); } template.isSyncTimeSimultaneityFree = true; // The handwritten version doesn't make a distinction. template.keyEqualityComparer = (left, right) => stream.Properties.KeyEqualityComparer.GetEqualsExpr().Inline(left, right); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TRegister)); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRegister>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TRegister>()); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var t = a.GetType(template.className); if (t.GetTypeInfo().IsGenericType) { var list = typeof(TKey).GetAnonymousTypes(); list.AddRange(template.payloadType.GetAnonymousTypes()); list.AddRange(template.registerType.GetAnonymousTypes()); t = t.MakeGenericType(list.ToArray()); } return(Tuple.Create(t, errorMessages)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
/// <summary> /// Generate a batch class definition to be used as an aggreate definition. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// aggregate class. /// </summary> /// <typeparam name="TKey">The key type for the aggregate.</typeparam> /// <typeparam name="TInput">The input type for the aggregate.</typeparam> /// <typeparam name="TOutput">The output type for the aggregate.</typeparam> /// <typeparam name="TState">The type for the accumulated state held by the aggregate.</typeparam> /// <typeparam name="TResult">The type of the result.</typeparam> /// <returns> /// A type that is defined to be a subtype of UnaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TInput"/>>. /// </returns> internal static Tuple <Type, string> Generate <TKey, TInput, TState, TOutput, TResult>( GroupedWindowStreamable <TKey, TInput, TState, TOutput, TResult> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(IStreamObserver <Empty, TInput>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { string expandedCode; var template = new GroupedWindowTemplate(); var keyType = template.keyType = typeof(TKey); var inputType = template.inputType = typeof(TInput); var stateType = template.stateType = typeof(TState); var outputType = template.outputType = typeof(TOutput); var resultType = template.resultType = typeof(TResult); template.TResult = resultType.GetCSharpSourceSyntax(); // BUGBUG: need to get any generic parameters needed template.isUngrouped = (keyType == typeof(Empty)); template.className = string.Format("GeneratedGroupedAggregate_{0}", GroupedAggregateSequenceNumber++); var inputMessageRepresentation = new ColumnarRepresentation(inputType); var resultRepresentation = new ColumnarRepresentation(resultType); var assemblyReferences = new List <Assembly>(); #region Key Selector var keySelector = stream.KeySelector; string transformedKeySelectorAsString; if (keyType.IsAnonymousTypeName()) { Contract.Assume(keySelector.Body is NewExpression); var transformedFunction = Extensions.TransformUnaryFunction <TKey, TInput>(keySelector); var newBody = (NewExpression)transformedFunction.Body; transformedKeySelectorAsString = string.Join(",", newBody.Arguments); } else { var transformedFunction = Extensions.TransformUnaryFunction <Empty, TInput>(keySelector).Body; if (transformedFunction == null) { return(null); } transformedKeySelectorAsString = transformedFunction.ExpressionToCSharp(); } template.keySelector = transformedKeySelectorAsString; assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector)); #endregion #region Key Comparer and HashCode var keyComparer = EqualityComparerExpression <TKey> .Default; template.keyComparerEquals = (left, right) => keyComparer.GetEqualsExpr().Inline(left, right); template.keyComparerGetHashCode = (x) => keyComparer.GetGetHashCodeExpr().Inline(x); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyComparer.GetEqualsExpr())); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyComparer.GetGetHashCodeExpr())); #endregion #region Aggregate functions var initialStateLambda = stream.Aggregate.InitialState(); if (ConstantExpressionFinder.IsClosedExpression(initialStateLambda)) { template.initialState = initialStateLambda.Body.ExpressionToCSharp(); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(initialStateLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for GroupedWindow: couldn't inline the initial state lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledInitialState = true; template.initialState = "initialState()"; } } var accumulateLambda = stream.Aggregate.Accumulate(); if (ConstantExpressionFinder.IsClosedExpression(accumulateLambda)) { var accTransformedLambda = Extensions.TransformFunction <TKey, TInput>(accumulateLambda, 2); template.accumulate = (stateArg, longArg) => accTransformedLambda.Inline(stateArg, longArg); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(accumulateLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for GroupedWindow: couldn't inline the accumulate lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledAccumulate = true; template.accumulate = (s1, s2) => string.Format("accumulate({0}, {1}, batch[i]);", s1, s2); } } var deaccumulateLambda = stream.Aggregate.Deaccumulate(); if (ConstantExpressionFinder.IsClosedExpression(deaccumulateLambda)) { var deaccumulateTransformedLambda = Extensions.TransformFunction <TKey, TInput>(deaccumulateLambda, 2); template.deaccumulate = (stateArg, longArg) => deaccumulateTransformedLambda.Inline(stateArg, longArg); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(deaccumulateLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { throw new InvalidOperationException("Code Generation couldn't inline a lambda!"); } else { template.useCompiledDeaccumulate = true; template.deaccumulate = (s1, s2) => string.Format("deaccumulate({0}, {1}, batch[i]);", s1, s2); } } var differenceLambda = stream.Aggregate.Difference(); if (ConstantExpressionFinder.IsClosedExpression(differenceLambda)) { template.difference = (stateArg1, stateArg2) => differenceLambda.Inline(stateArg1, stateArg2); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(differenceLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for GroupedWindow: couldn't inline the deaccumulate lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledDifference = true; template.deaccumulate = (s1, s2) => string.Format("difference({0}, {1});", s1, s2); } } var computeResultLambda = stream.Aggregate.ComputeResult(); if (ConstantExpressionFinder.IsClosedExpression(computeResultLambda)) { if (outputType.IsAnonymousType()) { if (computeResultLambda.Body is NewExpression newExpression) { errorMessages = "Code Generation for GroupedWindow: result selector must be a new expression for anonymous types"; throw new NotImplementedException(errorMessages); } else { template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg); } } else { template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg); } assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(computeResultLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for GroupedWindow: couldn't inline the result selector lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledComputeResult = true; template.computeResult = (stateArg) => "computeResult(" + stateArg + ")"; } } #endregion template.BatchGeneratedFrom_Unit_TInput = Transformer.GetBatchClassName(typeof(Empty), inputType); template.UnitTInputGenericParameters = string.Empty; // BUGBUG template.UnitTResultGenericParameters = string.Empty; // BUGBUG template.inputFields = inputMessageRepresentation.AllFields; template.outputFields = resultRepresentation.AllFields; var resultSelector = stream.ResultSelector; var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); // dont want the parameters substituted for at all var projectionResult = SelectTransformer.Transform(resultSelector, parameterSubstitutions, resultRepresentation, true); if (projectionResult.Error) { return(null); } template.finalResultSelector = (key, aggregateResult) => { var parameters = new Dictionary <ParameterExpression, string> { { resultSelector.Parameters.ElementAt(0), key } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); sb.AppendLine(string.Format("var {0} = {1};\n", resultSelector.Parameters.ElementAt(1).Name, aggregateResult)); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; sb.AppendFormat("this.batch.{0}.col[_c] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(parameters)); } sb.AppendLine("}"); return(sb.ToString()); }; assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(resultSelector)); template.staticCtor = Transformer.StaticCtor(template.className); expandedCode = template.TransformText(); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(typeof(Empty), typeof(TKey), typeof(TInput), typeof(TState), typeof(TOutput), typeof(FastDictionaryGenerator3))); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TInput>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TResult>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <Empty, TResult>()); var assembly = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var t = assembly.GetType(template.className); if (t.GetTypeInfo().IsGenericType) { var list = typeof(TKey).GetAnonymousTypes(); list.AddRange(typeof(TInput).GetAnonymousTypes()); list.AddRange(typeof(TState).GetAnonymousTypes()); list.AddRange(typeof(TOutput).GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new FixedIntervalEquiJoinTemplate($"GeneratedFixedIntervalEquiJoin_{EquiJoinSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult)) { leftDuration = stream.Left.Properties.ConstantDurationLength.Value, rightDuration = stream.Right.Properties.ConstantDurationLength.Value }; var keyAndLeftGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType).BracketedCommaSeparatedString(); var keyAndRightGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.rightType).BracketedCommaSeparatedString(); template.TKeyTResultGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString(); template.genericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType, template.rightType, template.resultType).BracketedCommaSeparatedString(); template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType); template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType); var resultMessageRepresentation = new ColumnarRepresentation(template.resultType); var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType); var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType); template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType); template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters; template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters; template.leftFields = template.leftMessageRepresentation.AllFields; template.rightFields = template.rightMessageRepresentation.AllFields; template.resultFields = resultMessageRepresentation.AllFields; template.ActiveEventTypeLeft = template.leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left"; template.ActiveEventTypeRight = template.rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right"; #region Key Equals var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparerEquals = (left, right) => keyComparer.Inline(left, right); if (template.keyType.IsAnonymousType()) { template.keyComparerEquals = (left, right) => $"keyComparerEquals({left}, {right})"; } #endregion #region Left Payload Equals { var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0); template.leftComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Right Payload Equals { var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0); template.rightComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Result Selector { var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); if (!ConstantExpressionFinder.IsClosedExpression(selector)) { errorMessages = "result selector is not a closed expression"; throw new InvalidOperationException(); } #region LeftBatchSelector { var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i"); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region RightBatchSelector { var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j"); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region ActiveSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.activeSelector = (leftEvent, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion } #endregion return(template.Generate <TKey, TLeft, TRight, TResult>()); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new EquiJoinTemplate(); var keyType = template.keyType = typeof(TKey); var leftType = template.leftType = typeof(TLeft); var rightType = template.rightType = typeof(TRight); var resultType = template.resultType = typeof(TResult); var tm = new TypeMapper(keyType, leftType, rightType, resultType); template.TKey = tm.CSharpNameFor(keyType); template.TLeft = tm.CSharpNameFor(leftType); template.TRight = tm.CSharpNameFor(rightType); template.TResult = tm.CSharpNameFor(resultType); var keyAndLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString(); var keyAndRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString(); template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString(); template.genericParameters = tm.GenericTypeVariables(keyType, leftType, rightType, resultType).BracketedCommaSeparatedString(); template.className = string.Format("GeneratedEquiJoin_{0}", EquiJoinSequenceNumber++); template.leftMessageRepresentation = new ColumnarRepresentation(leftType); template.rightMessageRepresentation = new ColumnarRepresentation(rightType); var resultMessageRepresentation = new ColumnarRepresentation(resultType); var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType); var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType); template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType); template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters; template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters; template.leftFields = template.leftMessageRepresentation.AllFields; template.rightFields = template.rightMessageRepresentation.AllFields; template.resultFields = resultMessageRepresentation.AllFields; template.ActiveEventTypeLeft = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left"; template.ActiveEventTypeRight = rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right"; #region Key Equals var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparerEquals = (left, right) => keyComparer.Inline(left, right); if (keyType.IsAnonymousType()) { template.keyComparerEquals = (left, right) => string.Format("keyComparerEquals({0}, {1})", left, right); } #endregion #region Left Payload Equals { var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0); template.leftComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Right Payload Equals { var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0); template.rightComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Result Selector { var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); if (!ConstantExpressionFinder.IsClosedExpression(selector)) { errorMessages = "result selector is not a closed expression"; throw new InvalidOperationException(); } #region LeftBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), "i"), leftIndex }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region RightBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), "j"), rightIndex } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region ActiveSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.activeSelector = (leftEvent, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion } #endregion if (stream.Left.Properties.IsIntervalFree && stream.Right.Properties.IsConstantDuration) { template.endPointHeap = "EndPointQueue"; } else if (stream.Right.Properties.IsIntervalFree && stream.Left.Properties.IsConstantDuration) { template.endPointHeap = "EndPointQueue"; } else if (stream.Left.Properties.IsConstantDuration && stream.Right.Properties.IsConstantDuration && stream.Left.Properties.ConstantDurationLength == stream.Right.Properties.ConstantDurationLength) { template.endPointHeap = "EndPointQueue"; } else { template.endPointHeap = "EndPointHeap"; } template.staticCtor = Transformer.StaticCtor(template.className); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(keyType, leftType, rightType, resultType); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>()); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); if (keyType.IsAnonymousType()) { if (errorMessages == null) { errorMessages = string.Empty; } errorMessages += "\nCodegen Warning: The key type for an equi-join is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n"; } var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType, resultType); var t = a.GetType(realClassName); if (t.GetTypeInfo().IsGenericType) { var list = keyType.GetAnonymousTypes(); list.AddRange(leftType.GetAnonymousTypes()); list.AddRange(rightType.GetAnonymousTypes()); list.AddRange(resultType.GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
/// <summary> /// Generate a batch class definition to be used as a Clip pipe. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// aggregate class. /// </summary> /// <typeparam name="TKey">The key type for both sides.</typeparam> /// <typeparam name="TLeft">The payload type for the left side.</typeparam> /// <typeparam name="TRight">The payload type for the right side.</typeparam> /// <returns> /// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TKey"/>, <typeparamref name="TRight"/>>. /// </returns> internal static Tuple <Type, string> Generate <TKey, TLeft, TRight>(ClipJoinStreamable <TKey, TLeft, TRight> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TLeft>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new ClipJoinTemplate(); var keyType = template.keyType = typeof(TKey); var leftType = template.leftType = typeof(TLeft); var rightType = template.rightType = typeof(TRight); var tm = new TypeMapper(keyType, leftType, rightType); template.TKey = tm.CSharpNameFor(keyType); template.TLeft = tm.CSharpNameFor(leftType); template.TRight = tm.CSharpNameFor(rightType); var gps = tm.GenericTypeVariables(keyType, leftType, rightType); template.genericParameters = gps.BracketedCommaSeparatedString(); template.className = string.Format("GeneratedClip_{0}", ClipSequenceNumber++); var resultRepresentation = new ColumnarRepresentation(leftType); template.ActiveEventType = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event"; #region Key Comparer var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparer = (left, right) => keyComparer.Inline(left, right); #endregion #region Left Comparer var leftComparer = stream.LeftComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftComparer, "index_ProcessLeftEvent", 0); template.leftComparer = (left, right) => newLambda.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType); template.TKeyTLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType); template.TKeyTRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString(); template.leftFields = resultRepresentation.AllFields; template.noFields = resultRepresentation.noFields; template.staticCtor = Transformer.StaticCtor(template.className); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight)); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>()); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(leftComparer)); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType); var t = a.GetType(realClassName); if (t.GetTypeInfo().IsGenericType) { var list = keyType.GetAnonymousTypes(); list.AddRange(leftType.GetAnonymousTypes()); list.AddRange(rightType.GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
public static Tuple <Type, string> Generate <TKey, TInput, TState, TOutput>(SnapshotWindowStreamable <TKey, TInput, TState, TOutput> stream, AggregatePipeType pipeType) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(IStreamObserver <TKey, TInput>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); var container = stream.Properties.QueryContainer; string generatedClassName; string expandedCode; List <Assembly> assemblyReferences; string errorMessages = null; try { AggregateTemplate template; switch (pipeType) { case AggregatePipeType.StartEdge: template = new SnapshotWindowStartEdgeTemplate(); break; case AggregatePipeType.PriorityQueue: template = new SnapshotWindowPriorityQueueTemplate(); break; case AggregatePipeType.Tumbling: template = new SnapshotWindowTumblingTemplate(); break; case AggregatePipeType.Sliding: template = new SnapshotWindowSlidingTemplate(); break; case AggregatePipeType.Hopping: template = new SnapshotWindowHoppingTemplate { hopsPerDuration = ((int)(stream.Source.Properties.ConstantDurationLength.Value / stream.Source.Properties.ConstantHopLength) + 1).ToString() }; break; default: Contract.Assert(false, "case meant to be exhaustive"); throw new InvalidOperationException("case meant to be exhaustive"); } template.isUngrouped = (typeof(TKey) == typeof(Empty)); var keyType = template.keyType = typeof(TKey); var inputType = template.inputType = typeof(TInput); var stateType = template.stateType = typeof(TState); var outputType = template.outputType = typeof(TOutput); template.inputFields = new ColumnarRepresentation(inputType).AllFields; template.outputFields = new ColumnarRepresentation(outputType).AllFields; assemblyReferences = new List <Assembly>(); #region Key Comparer IEqualityComparerExpression <TKey> keyComparer; keyComparer = stream.Properties.KeyEqualityComparer; var equalsExpression = keyComparer.GetEqualsExpr(); var getHashcodeExpression = keyComparer.GetGetHashCodeExpr(); template.inlinedKeyComparerEquals = (left, right) => string.Format("({0})", equalsExpression.Inline(left, right)); template.inlinedKeyComparerGetHashCode = (x) => string.Format("({0}/* inlined GetHashCode */)", getHashcodeExpression.Inline(x)); if (keyType.IsAnonymousType()) { template.inlinedKeyComparerEquals = (left, right) => string.Format("keyComparerEquals({0}, {1})", left, right); template.inlinedKeyComparerGetHashCode = (x) => string.Format("keyComparerGetHashCode({0})", x); } assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(equalsExpression)); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(getHashcodeExpression)); #endregion #region Aggregate Functions var initialStateLambda = stream.Aggregate.InitialState(); if (ConstantExpressionFinder.IsClosedExpression(initialStateLambda)) { template.initialState = initialStateLambda.Body.ExpressionToCSharp(); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(initialStateLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for Aggregate: couldn't inline the initial state lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledInitialState = true; template.initialState = "initialState()"; } } var accumulateLambda = stream.Aggregate.Accumulate(); if (ConstantExpressionFinder.IsClosedExpression(accumulateLambda)) { var accTransformedLambda = Extensions.TransformFunction <TKey, TInput>(accumulateLambda, 2); template.accumulate = (stateArg, longArg) => accTransformedLambda.Inline(stateArg, longArg); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(accumulateLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for Aggregate: couldn't inline the accumulate lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledAccumulate = true; template.accumulate = (s1, s2) => string.Format("accumulate({0}, {1}, batch[i]);", s1, s2); } } var deaccumulateLambda = stream.Aggregate.Deaccumulate(); if (ConstantExpressionFinder.IsClosedExpression(deaccumulateLambda)) { var deaccumulateTransformedLambda = Extensions.TransformFunction <TKey, TInput>(deaccumulateLambda, 2); template.deaccumulate = (stateArg, longArg) => deaccumulateTransformedLambda.Inline(stateArg, longArg); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(deaccumulateLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for Aggregate: couldn't inline the deaccumulate lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledDeaccumulate = true; template.deaccumulate = (s1, s2) => string.Format("deaccumulate({0}, {1}, batch[i]);", s1, s2); } } var differenceLambda = stream.Aggregate.Difference(); if (ConstantExpressionFinder.IsClosedExpression(differenceLambda)) { template.difference = (stateArg1, stateArg2) => differenceLambda.Inline(stateArg1, stateArg2); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(differenceLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for Aggregate: couldn't inline the difference lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledDifference = true; template.difference = (s1, s2) => string.Format("difference({0}, {1});", s1, s2); } } var computeResultLambda = stream.Aggregate.ComputeResult(); if (ConstantExpressionFinder.IsClosedExpression(computeResultLambda)) { if (outputType.IsAnonymousType()) { if (computeResultLambda.Body is NewExpression newExpression) { var outputBatchType = StreamMessageManager.GetStreamMessageType <TKey, TOutput>(); var foo = Transform(newExpression, outputBatchType); template.computeResult = (stateArg) => Expression.Lambda(foo, computeResultLambda.Parameters.ToArray()).Inline(stateArg); } else { template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg); } } else { template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg); } assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(computeResultLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar || outputType.IsAnonymousType()) { // second disjunct is because if we aren't inlining the computeResult function and // the output type is anonymous, calling the compiled computeResult function returns // a value of the anonymous type and since the generated operator represents the anonymous // type as a generic parameter, it can't use the "field" (i.e., property) names to // get the individual pieces to assign to each column of the output message. errorMessages = "Code Generation for Aggregate: couldn't inline the compute result lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledComputeResult = true; template.computeResult = (stateArg) => "computeResult(" + stateArg + ");"; } } #endregion generatedClassName = template.className = string.Format("Aggregate_{0}", sequenceNumber++); generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(keyType, inputType, stateType, outputType); template.staticCtor = Transformer.StaticCtor(template.className); expandedCode = template.TransformText(); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TInput), typeof(TState), typeof(TOutput), typeof(FastDictionaryGenerator), typeof(SortedDictionary <,>))); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TInput>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TOutput>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TOutput>()); if (container != null) { assemblyReferences.AddRange(container.CollectedGeneratedTypes.Select(o => o.GetTypeInfo().Assembly)); } var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); if (keyType.IsAnonymousType()) { if (errorMessages == null) { errorMessages = string.Empty; } errorMessages += "\nCodegen Warning: The key type for an aggregate is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n"; } var t = a.GetType(generatedClassName); if (t.GetTypeInfo().IsGenericType) { var list = typeof(TKey).GetAnonymousTypes(); list.AddRange(typeof(TInput).GetAnonymousTypes()); list.AddRange(typeof(TState).GetAnonymousTypes()); list.AddRange(typeof(TOutput).GetAnonymousTypes()); t = t.MakeGenericType(list.ToArray()); } return(Tuple.Create(t, errorMessages)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> GenerateAFA <TKey, TPayload, TRegister, TAccumulator>( AfaStreamable <TKey, TPayload, TRegister, TAccumulator> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); var className = $"GeneratedGroupedAfaMultiEvent_{AFASequenceNumber++}"; var template = new GroupedAfaMultiEventTemplate(className, typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(TAccumulator)) { isFinal = stream.afa.isFinal, hasOutgoingArcs = stream.afa.hasOutgoingArcs, startStates = stream.afa.startStates, AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances, isSyncTimeSimultaneityFree = true, // The handwritten version doesn't make a distinction. keyEqualityComparer = (left, right) => stream.Properties.KeyEqualityComparer.GetEqualsExpr().Inline(left, right), }; var d1 = stream.afa.uncompiledAfa.transitionInfo; var orderedKeys = d1.Keys.OrderBy(e => e).ToArray(); for (int i = 0; i < orderedKeys.Length; i++) { var sourceNodeNumber = orderedKeys[i]; var outgoingEdgesDictionary = d1[sourceNodeNumber]; var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray(); var edgeList = new List <MultiEdgeInfo>(); for (int j = 0; j < orderedTargetNodes.Length; j++) { var targetNodeNumber = orderedTargetNodes[j]; var edge = outgoingEdgesDictionary[targetNodeNumber]; if (edge is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc) { var multiEdgeInfo = new MultiEdgeInfo() { SourceNode = sourceNodeNumber, TargetNode = targetNodeNumber, fromStartState = stream.afa.startStates.Any(n => n == sourceNodeNumber), EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNodeNumber), Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg), Accumulate = (ts, ev, reg, acc) => { var transformedAccumulate = Extensions.TransformFunction <TKey, TPayload>(multiArc.Accumulate, 1, batchVariableName: "sourceBatch"); return(transformedAccumulate.Inline(ts, reg, acc)); }, Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg), }; multiEdgeInfo.Transfer = multiArc.Transfer == null ? (Func <string, string, string, string>)null : ((ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg)); if (multiArc.Dispose == null) { multiEdgeInfo.Dispose = (acc) => "// no dispose function"; } else { multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc); } multiEdgeInfo.SkipToEnd = multiArc.SkipToEnd == null ? (Func <string, string, string, string>)null : ((ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc)); edgeList.Add(multiEdgeInfo); } } template.edgeInfos.Add(Tuple.Create(sourceNodeNumber, edgeList)); } for (int i = 0; i < stream.afa.startStates.Length; i++) { var startState = stream.afa.startStates[i]; var edgeList2 = new List <MultiEdgeInfo>(); var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState]; foreach (var edge in outgoingEdgeDictionary) { var targetNode = edge.Key; var arc = edge.Value; if (arc is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc) { var eps = EpsilonClosure(stream.afa, targetNode); var multiEdgeInfo = new MultiEdgeInfo() { SourceNode = startState, TargetNode = targetNode, fromStartState = true, EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNode), Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg), Accumulate = (ts, ev, reg, acc) => { var transformedAccumulate = Extensions.TransformFunction <TKey, TPayload>(multiArc.Accumulate, 1, batchVariableName: "sourceBatch"); return(transformedAccumulate.Inline(ts, reg, acc)); }, Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg), }; multiEdgeInfo.Transfer = multiArc.Transfer == null ? (Func <string, string, string, string>)null : ((ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg)); if (multiArc.Dispose == null) { multiEdgeInfo.Dispose = (acc) => "// no dispose function"; } else { multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc); } multiEdgeInfo.SkipToEnd = multiArc.SkipToEnd == null ? (Func <string, string, string, string>)null : ((ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc)); edgeList2.Add(multiEdgeInfo); } } template.startEdgeInfos.Add(Tuple.Create(startState, edgeList2)); } return(template.Generate <TKey, TPayload, TRegister, TAccumulator>()); }