public FixedIntervalEquiJoinPipe( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector, IStreamObserver <TKey, TResult> observer) : base(stream, observer) { this.selector = selector.Compile(); this.leftDuration = stream.Left.Properties.ConstantDurationLength.Value; this.rightDuration = stream.Right.Properties.ConstantDurationLength.Value; this.keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); this.keyComparerEquals = this.keyComparer.Compile(); if (this.leftDuration == this.rightDuration) { this.endPointHeap = new EndPointQueue(); } else { this.endPointHeap = new EndPointHeap(); } this.pool = MemoryManager.GetMemoryPool <TKey, TResult>(stream.Properties.IsColumnar); this.pool.Get(out this.output); this.output.Allocate(); }
public StartEdgeEquiJoinPipe( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector, IStreamObserver <TKey, TResult> observer) : base(stream, observer) { this.selectorExpr = selector; this.selector = this.selectorExpr.Compile(); this.keyComparerExpr = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); this.keyComparer = this.keyComparerExpr.Compile(); this.errorMessages = stream.ErrorMessages; this.pool = MemoryManager.GetMemoryPool <TKey, TResult>(stream.Properties.IsColumnar); this.pool.Get(out this.output); this.output.Allocate(); }
public IncreasingOrderEquiJoinPipe( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector, IStreamObserver <TKey, TResult> observer) : base(stream, observer) { this.selectorExpr = selector; this.selector = this.selectorExpr.Compile(); this.joinKeyOrderComparerExpression = stream.Left.Properties.KeyComparer.GetCompareExpr(); this.joinKeyOrderComparer = this.joinKeyOrderComparerExpression.Compile(); this.currentLeftList = new List <ActiveEvent <TLeft> >(); this.currentRightList = new List <ActiveEvent <TRight> >(); this.errorMessages = stream.ErrorMessages; this.pool = MemoryManager.GetMemoryPool <TKey, TResult>(stream.Properties.IsColumnar); this.pool.Get(out this.output); this.output.Allocate(); }
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new FixedIntervalEquiJoinTemplate($"GeneratedFixedIntervalEquiJoin_{EquiJoinSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult)) { leftDuration = stream.Left.Properties.ConstantDurationLength.Value, rightDuration = stream.Right.Properties.ConstantDurationLength.Value }; var keyAndLeftGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType).BracketedCommaSeparatedString(); var keyAndRightGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.rightType).BracketedCommaSeparatedString(); template.TKeyTResultGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString(); template.genericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType, template.rightType, template.resultType).BracketedCommaSeparatedString(); template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType); template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType); var resultMessageRepresentation = new ColumnarRepresentation(template.resultType); var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType); var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType); template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType); template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters; template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters; template.leftFields = template.leftMessageRepresentation.AllFields; template.rightFields = template.rightMessageRepresentation.AllFields; template.resultFields = resultMessageRepresentation.AllFields; template.ActiveEventTypeLeft = template.leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left"; template.ActiveEventTypeRight = template.rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right"; #region Key Equals var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparerEquals = (left, right) => keyComparer.Inline(left, right); if (template.keyType.IsAnonymousType()) { template.keyComparerEquals = (left, right) => $"keyComparerEquals({left}, {right})"; } #endregion #region Left Payload Equals { var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0); template.leftComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Right Payload Equals { var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0); template.rightComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Result Selector { var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); if (!ConstantExpressionFinder.IsClosedExpression(selector)) { errorMessages = "result selector is not a closed expression"; throw new InvalidOperationException(); } #region LeftBatchSelector { var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i"); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region RightBatchSelector { var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j"); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region ActiveSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.activeSelector = (leftEvent, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion } #endregion return(template.Generate <TKey, TLeft, TRight, TResult>()); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new StartEdgeEquiJoinTemplate($"GeneratedStartEdgeEquiJoin_{StartEdgeEquiJoinSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult)); var gps = template.tm.GenericTypeVariables(template.keyType, template.leftType, template.rightType, template.resultType); template.genericParameters = gps.BracketedCommaSeparatedString(); template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType); template.leftFields = template.leftMessageRepresentation.AllFields; template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType); template.rightFields = template.rightMessageRepresentation.AllFields; var resultRepresentation = new ColumnarRepresentation(template.resultType); #region Key Comparer var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparer = (left, right) => keyComparer.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType); template.TKeyTLeftGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType); template.TKeyTRightGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.rightType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType); template.TKeyTResultGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString(); template.outputFields = resultRepresentation.AllFields; var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); #region LeftBatchSelector { var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i"); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var parameterMap = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex }, { selector.Parameters[1], rightEvent } }; if (projectionResult.ProjectionReturningResultInstance != null) { return($"this.output[index] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};"); } else { var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendLine($"this.output.{f.Name}.AddString({e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)});"); } else { sb.AppendLine($"this.output.{f.Name}.col[index] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};"); } } sb.AppendLine("}"); return(sb.ToString()); } }; } #endregion #region RightBatchSelector { var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j"); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var parameterMap = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex } }; if (projectionResult.ProjectionReturningResultInstance != null) { return($"this.output[index] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};"); } else { var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendLine($"this.output.{f.Name}.AddString({e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)});"); } else { sb.AppendLine($"this.output.{f.Name}.col[index] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};"); } } sb.AppendLine("}"); return(sb.ToString()); } }; } #endregion return(template.Generate <TKey, TLeft, TRight, TResult>(selector)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
/// <summary> /// Generate a batch class definition to be used as StartEdgeEquiJoin operator. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// aggregate class. /// </summary> /// <typeparam name="TKey">The key type for both sides.</typeparam> /// <typeparam name="TLeft">The payload type for the left side.</typeparam> /// <typeparam name="TRight">The payload type for the right side.</typeparam> /// <typeparam name="TResult">The payload type for the resulting stream.</typeparam> /// <returns> /// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TRight"/>, <typeparamref name="TKey"/>, <typeparamref name="TResult"/>>. /// </returns> internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new IncreasingOrderEquiJoinTemplate($"GeneratedIncreasingOrderEquiJoin_{IOOEJSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult)); template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType); template.leftFields = template.leftMessageRepresentation.AllFields; template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType); template.rightFields = template.rightMessageRepresentation.AllFields; var resultRepresentation = new ColumnarRepresentation(template.resultType); var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); #region Key Comparer var keyComparer = stream.Left.Properties.KeyComparer.GetCompareExpr(); if (!ConstantExpressionFinder.IsClosedExpression(keyComparer)) { return(null); } template.joinKeyOrderComparer = (left, right) => keyComparer.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType); template.TKeyTLeftGenericParameters = string.Empty; // BUGBUG template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType); template.TKeyTRightGenericParameters = string.Empty; // BUGBUG template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType); template.TKeyTResultGenericParameters = string.Empty; // BUGBUG template.outputFields = resultRepresentation.AllFields; if (!ConstantExpressionFinder.IsClosedExpression(selector)) { return(null); } #region LeftBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), "i"), leftIndex }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region RightBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), "j"), rightIndex } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion template.getOutputBatch = string.Format( "pool.Get(out genericOutputBatch); output = ({0}{1})genericOutputBatch;", Transformer.GetBatchClassName(template.keyType, template.resultType), template.TKeyTResultGenericParameters); return(template.Generate <TKey, TLeft, TRight, TResult>(selector)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new EquiJoinTemplate(); var keyType = template.keyType = typeof(TKey); var leftType = template.leftType = typeof(TLeft); var rightType = template.rightType = typeof(TRight); var resultType = template.resultType = typeof(TResult); var tm = new TypeMapper(keyType, leftType, rightType, resultType); template.TKey = tm.CSharpNameFor(keyType); template.TLeft = tm.CSharpNameFor(leftType); template.TRight = tm.CSharpNameFor(rightType); template.TResult = tm.CSharpNameFor(resultType); var keyAndLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString(); var keyAndRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString(); template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString(); template.genericParameters = tm.GenericTypeVariables(keyType, leftType, rightType, resultType).BracketedCommaSeparatedString(); template.className = string.Format("GeneratedEquiJoin_{0}", EquiJoinSequenceNumber++); template.leftMessageRepresentation = new ColumnarRepresentation(leftType); template.rightMessageRepresentation = new ColumnarRepresentation(rightType); var resultMessageRepresentation = new ColumnarRepresentation(resultType); var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType); var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType); template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType); template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters; template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters; template.leftFields = template.leftMessageRepresentation.AllFields; template.rightFields = template.rightMessageRepresentation.AllFields; template.resultFields = resultMessageRepresentation.AllFields; template.ActiveEventTypeLeft = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left"; template.ActiveEventTypeRight = rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right"; #region Key Equals var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparerEquals = (left, right) => keyComparer.Inline(left, right); if (keyType.IsAnonymousType()) { template.keyComparerEquals = (left, right) => string.Format("keyComparerEquals({0}, {1})", left, right); } #endregion #region Left Payload Equals { var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0); template.leftComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Right Payload Equals { var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0); template.rightComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Result Selector { var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); if (!ConstantExpressionFinder.IsClosedExpression(selector)) { errorMessages = "result selector is not a closed expression"; throw new InvalidOperationException(); } #region LeftBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), "i"), leftIndex }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region RightBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), "j"), rightIndex } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region ActiveSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.activeSelector = (leftEvent, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion } #endregion if (stream.Left.Properties.IsIntervalFree && stream.Right.Properties.IsConstantDuration) { template.endPointHeap = "EndPointQueue"; } else if (stream.Right.Properties.IsIntervalFree && stream.Left.Properties.IsConstantDuration) { template.endPointHeap = "EndPointQueue"; } else if (stream.Left.Properties.IsConstantDuration && stream.Right.Properties.IsConstantDuration && stream.Left.Properties.ConstantDurationLength == stream.Right.Properties.ConstantDurationLength) { template.endPointHeap = "EndPointQueue"; } else { template.endPointHeap = "EndPointHeap"; } template.staticCtor = Transformer.StaticCtor(template.className); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(keyType, leftType, rightType, resultType); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>()); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); if (keyType.IsAnonymousType()) { if (errorMessages == null) { errorMessages = string.Empty; } errorMessages += "\nCodegen Warning: The key type for an equi-join is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n"; } var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType, resultType); var t = a.GetType(realClassName); if (t.GetTypeInfo().IsGenericType) { var list = keyType.GetAnonymousTypes(); list.AddRange(leftType.GetAnonymousTypes()); list.AddRange(rightType.GetAnonymousTypes()); list.AddRange(resultType.GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new StartEdgeEquiJoinTemplate(); var keyType = template.keyType = typeof(TKey); var leftType = template.leftType = typeof(TLeft); var rightType = template.rightType = typeof(TRight); var resultType = template.resultType = typeof(TResult); var tm = new TypeMapper(keyType, leftType, rightType, resultType); template.TKey = tm.CSharpNameFor(keyType); template.TLeft = tm.CSharpNameFor(leftType); template.TRight = tm.CSharpNameFor(rightType); template.TResult = tm.CSharpNameFor(resultType); var gps = tm.GenericTypeVariables(keyType, leftType, rightType, resultType); template.genericParameters = gps.BracketedCommaSeparatedString(); template.className = string.Format("GeneratedStartEdgeEquiJoin_{0}", StartEdgeEquiJoinSequenceNumber++); template.leftMessageRepresentation = new ColumnarRepresentation(leftType); template.leftFields = template.leftMessageRepresentation.AllFields; template.rightMessageRepresentation = new ColumnarRepresentation(rightType); template.rightFields = template.rightMessageRepresentation.AllFields; var outputMessageRepresentation = new ColumnarRepresentation(resultType); var resultRepresentation = outputMessageRepresentation; #region Key Comparer var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparer = (left, right) => keyComparer.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType); template.TKeyTLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType); template.TKeyTRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType); template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString(); template.outputFields = resultRepresentation.AllFields; var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); #region LeftBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), "i"), leftIndex }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region RightBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), "j"), rightIndex } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion template.staticCtor = Transformer.StaticCtor(template.className); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult)); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>()); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(selector)); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType, resultType); var t = a.GetType(realClassName); if (t.GetTypeInfo().IsGenericType) { var list = keyType.GetAnonymousTypes(); list.AddRange(leftType.GetAnonymousTypes()); list.AddRange(rightType.GetAnonymousTypes()); list.AddRange(resultType.GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }