private TemporalEgressTemplate(Type tKey, Type tPayload, Type tResult, string partitionString, string ingressType, bool isColumnar) : base($"GeneratedTemporalEgress_{TemporalEgressSequenceNumber++}") { var tm = new TypeMapper(tKey, tPayload, tResult); this.payloadRepresentation = new ColumnarRepresentation(tPayload); this.BatchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName( tKey == typeof(Empty) ? tKey : typeof(PartitionKey <>).MakeGenericType(tKey), tPayload); this.TKeyTPayloadGenericParameters = tm.GenericTypeVariables(tKey, tPayload).BracketedCommaSeparatedString(); this.fields = this.payloadRepresentation.AllFields; this.TKey = tm.CSharpNameFor(tKey); this.TPayload = tm.CSharpNameFor(tPayload); this.TResult = tm.CSharpNameFor(tResult); this.partitionString = partitionString; this.ingressType = ingressType; this.partitionKeyArgument = !string.IsNullOrEmpty(partitionString) ? "colkey[i].Key, " : string.Empty; this.genericArguments = string.IsNullOrEmpty(partitionString) ? this.TPayload : this.TKey + ", " + this.TPayload; this.egress = (ingressType != "StreamEvent") ? this.TResult : partitionString + "StreamEvent<" + this.genericArguments + ">"; this.inputKey = string.IsNullOrEmpty(partitionString) ? this.TKey : "PartitionKey<" + this.TKey + ">"; this.isColumnar = isColumnar; }
protected CommonUnaryTemplate(string className, Type keyType, Type payloadType, Type resultType, bool suppressPayloadBatch = false) : base(className) { this.keyType = keyType; this.payloadType = payloadType; this.resultType = resultType; var tm = new TypeMapper(keyType, payloadType, resultType); this.TKey = tm.CSharpNameFor(keyType); this.TPayload = tm.CSharpNameFor(payloadType); this.TResult = tm.CSharpNameFor(resultType); if (!suppressPayloadBatch) { this.TKeyTPayloadGenericParameters = tm.GenericTypeVariables(keyType, payloadType).BracketedCommaSeparatedString(); this.BatchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName(keyType, payloadType); } this.payloadRepresentation = new ColumnarRepresentation(payloadType); this.fields = this.payloadRepresentation.AllFields; this.noFields = this.payloadRepresentation.noFields; this.resultRepresentation = new ColumnarRepresentation(resultType); }
public ShuffleTemplate( string className, Type outerKeyType, Type sourceType, Type innerKeyType, string inlinedHashCodeComputation, bool nested, bool powerOf2) : base(className) { Contract.Requires(className != null); Contract.Requires(outerKeyType != null); Contract.Requires(sourceType != null); Contract.Requires(innerKeyType != null); this.inlinedHashCodeComputation = inlinedHashCodeComputation; this.isFirstLevelGroup = !nested; this.powerOf2 = powerOf2; var tm = new TypeMapper(outerKeyType, sourceType, innerKeyType); this.TOuterKey = tm.CSharpNameFor(outerKeyType); this.TSource = tm.CSharpNameFor(sourceType); this.TInnerKey = tm.CSharpNameFor(innerKeyType); this.genericParameters = tm.GenericTypeVariables(outerKeyType, sourceType, innerKeyType).BracketedCommaSeparatedString(); this.resultBatchClassType = this.isFirstLevelGroup ? Transformer.GetBatchClassName(innerKeyType, sourceType) : Transformer.GetBatchClassName(typeof(CompoundGroupKey <,>).MakeGenericType(outerKeyType, innerKeyType), sourceType); this.resultBatchGenericParameters = this.genericParameters; this.sourceBatchClassType = Transformer.GetBatchClassName(outerKeyType, sourceType); this.TOuterKeyTSourceGenericParameters = tm.GenericTypeVariables(outerKeyType, sourceType).BracketedCommaSeparatedString(); }
/// <summary> /// Generate a batch class definition to be used as a Union pipe. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// union pipe class. /// </summary> /// <typeparam name="TKey">The key type for both sides.</typeparam> /// <typeparam name="TPayload">The payload type.</typeparam> /// <returns> /// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TPayload"/>, <typeparamref name="TPayload"/>, <typeparamref name="TKey"/>, <typeparamref name="TPayload"/>>. /// </returns> internal static Tuple <Type, string> GenerateUnionPipeClass <TKey, TPayload>( UnionStreamable <TKey, TPayload> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TPayload, TPayload, TPayload>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); #if CODEGEN_TIMING Stopwatch sw = new Stopwatch(); sw.Start(); #endif var template = new UnionTemplate($"GeneratedUnion_{UnionSequenceNumber++}", typeof(TKey), typeof(TPayload)); var gps = template.tm.GenericTypeVariables(template.keyType, template.resultType); template.genericParameters = gps.BracketedCommaSeparatedString(); var resultRepresentation = new ColumnarRepresentation(template.resultType); var batchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName(template.keyType, template.resultType); var keyAndPayloadGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString(); template.GeneratedBatchName = batchGeneratedFrom_TKey_TPayload + keyAndPayloadGenericParameters; template.fields = resultRepresentation.AllFields; return(template.Generate <TKey, TPayload>()); }
public static void GetGeneratedCode(Type keyType, Type payloadType, out string generatedClassName, out string expandedCode, out List <Assembly> assemblyReferences) { var template = new SafeBatchTemplate(); assemblyReferences = new List <Assembly> { Transformer.SystemRuntimeSerializationDll // needed for [DataContract] and [DataMember] in generated code }; assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyType)); template.keyType = keyType; assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(payloadType)); template.payloadType = payloadType; template.needsPolymorphismCheck = !payloadType.GetTypeInfo().IsValueType&& !payloadType.IsAnonymousTypeName() && !payloadType.GetTypeInfo().IsSealed; template.payloadMightBeNull = payloadType.CanContainNull(); generatedClassName = Transformer.GetBatchClassName(keyType, payloadType); template.CLASSNAME = generatedClassName.CleanUpIdentifierName(); generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(keyType, payloadType); var payloadRepresentation = new ColumnarRepresentation(payloadType); template.fields = payloadRepresentation.AllFields; template.noPublicFields = payloadRepresentation.noFields; expandedCode = template.TransformText(); }
private UngroupedDAfaTemplate(string className, Type payloadType, Type registerType, Type accumulatorType) : base(className, typeof(Empty), payloadType, registerType, accumulatorType) { if (Config.ForceRowBasedExecution) { this.sourceBatchTypeName = $"Microsoft.StreamProcessing.StreamMessage<Microsoft.StreamProcessing.Empty, {this.TPayload}>"; this.resultBatchTypeName = $"Microsoft.StreamProcessing.StreamMessage<Microsoft.StreamProcessing.Empty, {this.TRegister}>"; } else { this.sourceBatchTypeName = Transformer.GetBatchClassName(typeof(Empty), payloadType); this.resultBatchTypeName = Transformer.GetBatchClassName(typeof(Empty), registerType); } }
private UngroupedDAfaTemplate(string className, Type keyType, Type payloadType, Type registerType, Type accumulatorType) : base(className, keyType, payloadType, registerType, accumulatorType) { if (Config.ForceRowBasedExecution) { this.sourceBatchTypeName = string.Format("Microsoft.StreamProcessing.StreamMessage<{0}, {1}>", this.TKey, this.TPayload); this.resultBatchTypeName = string.Format("Microsoft.StreamProcessing.StreamMessage<{0}, {1}>", this.TKey, this.TRegister); } else { this.sourceBatchTypeName = Transformer.GetBatchClassName(keyType, payloadType); this.resultBatchTypeName = Transformer.GetBatchClassName(keyType, registerType); } }
private GroupedAfaEventListTemplate(string className, Type keyType, Type payloadType, Type registerType, Type accumulatorType) : base(className, keyType, payloadType, registerType, accumulatorType) { if (Config.ForceRowBasedExecution) { this.sourceBatchTypeName = $"Microsoft.StreamProcessing.StreamMessage<{this.TKey}, {this.TPayload}>"; this.resultBatchTypeName = $"Microsoft.StreamProcessing.StreamMessage<{this.TKey}, {this.TRegister}>"; } else { this.sourceBatchTypeName = Transformer.GetBatchClassName(keyType, payloadType); this.resultBatchTypeName = Transformer.GetBatchClassName(keyType, registerType); } }
private UngroupTemplate( string className, bool ungroupingFromCompound, Type outerKeyType, Type innerKeyType, Type innerResultType, Type resultType) { Contract.Requires(className != null); Contract.Requires(outerKeyType != null); Contract.Requires(innerKeyType != null); Contract.Requires(innerResultType != null); Contract.Requires(resultType != null); this.CLASSNAME = className; this.outerKeyType = outerKeyType; this.innerKeyType = innerKeyType; this.innerResultType = innerResultType; this.resultType = resultType; this.ungroupingToUnit = outerKeyType == typeof(Empty); this.ungroupingFromCompound = ungroupingFromCompound; var tm = new TypeMapper(outerKeyType, innerKeyType, innerResultType, resultType); this.TOuterKey = tm.CSharpNameFor(outerKeyType); this.TInnerKey = tm.CSharpNameFor(innerKeyType); this.TInnerResult = tm.CSharpNameFor(innerResultType); this.TResult = tm.CSharpNameFor(resultType); var gps = tm.GenericTypeVariables(outerKeyType, innerKeyType, innerResultType, resultType); this.genericParameters = gps.BracketedCommaSeparatedString(); this.numberOfGenericParameters = gps.Count(); this.inputBatchClassType = ungroupingFromCompound ? Transformer.GetBatchClassName(typeof(CompoundGroupKey <,>).MakeGenericType(outerKeyType, innerKeyType), innerResultType) : Transformer.GetBatchClassName(innerKeyType, innerResultType); this.inputBatchGenericParameters = ungroupingFromCompound ? tm.GenericTypeVariables(outerKeyType, innerKeyType, innerResultType).BracketedCommaSeparatedString() : tm.GenericTypeVariables(innerKeyType, innerResultType).BracketedCommaSeparatedString(); this.resultBatchClassType = Transformer.GetBatchClassName(outerKeyType, resultType); this.resultBatchGenericParameters = tm.GenericTypeVariables(outerKeyType, resultType).BracketedCommaSeparatedString(); }
/// <summary> /// Generate a batch class definition to be used as a Clip pipe. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// aggregate class. /// </summary> /// <typeparam name="TKey">The key type for both sides.</typeparam> /// <typeparam name="TLeft">The payload type for the left side.</typeparam> /// <typeparam name="TRight">The payload type for the right side.</typeparam> /// <returns> /// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TKey"/>, <typeparamref name="TRight"/>>. /// </returns> internal static Tuple <Type, string> Generate <TKey, TLeft, TRight>(ClipJoinStreamable <TKey, TLeft, TRight> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TLeft>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); var template = new ClipJoinTemplate($"GeneratedClip_{ClipSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight)); var keyType = typeof(TKey); var leftType = typeof(TLeft); var rightType = typeof(TRight); var gps = template.tm.GenericTypeVariables(keyType, leftType, rightType); template.genericParameters = gps.BracketedCommaSeparatedString(); var resultRepresentation = new ColumnarRepresentation(leftType); template.ActiveEventType = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event"; #region Key Comparer var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparer = (left, right) => keyComparer.Inline(left, right); #endregion #region Left Comparer var leftComparer = stream.LeftComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftComparer, "index_ProcessLeftEvent", 0); template.leftComparer = (left, right) => newLambda.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType); template.TKeyTLeftGenericParameters = template.tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType); template.TKeyTRightGenericParameters = template.tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString(); template.leftFields = resultRepresentation.AllFields; template.noFields = resultRepresentation.noFields; return(template.Generate <TKey, TLeft, TRight>(leftComparer)); }
private void GetOutputBatch() { this.Write(" pool.Get(out genericOutputBatch);\r\n genericOutputBatch.Allocate();\r\n output =" + " ("); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(this.keyType, this.leftType))); this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(this.TKeyTLeftGenericParameters)); this.Write(")genericOutputBatch;\r\n"); foreach (var f in this.leftFields.Where(fld => fld.OptimizeString())) { this.Write(" output."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".Initialize();\r\n"); } }
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new EquiJoinTemplate(); var keyType = template.keyType = typeof(TKey); var leftType = template.leftType = typeof(TLeft); var rightType = template.rightType = typeof(TRight); var resultType = template.resultType = typeof(TResult); var tm = new TypeMapper(keyType, leftType, rightType, resultType); template.TKey = tm.CSharpNameFor(keyType); template.TLeft = tm.CSharpNameFor(leftType); template.TRight = tm.CSharpNameFor(rightType); template.TResult = tm.CSharpNameFor(resultType); var keyAndLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString(); var keyAndRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString(); template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString(); template.genericParameters = tm.GenericTypeVariables(keyType, leftType, rightType, resultType).BracketedCommaSeparatedString(); template.className = string.Format("GeneratedEquiJoin_{0}", EquiJoinSequenceNumber++); template.leftMessageRepresentation = new ColumnarRepresentation(leftType); template.rightMessageRepresentation = new ColumnarRepresentation(rightType); var resultMessageRepresentation = new ColumnarRepresentation(resultType); var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType); var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType); template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType); template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters; template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters; template.leftFields = template.leftMessageRepresentation.AllFields; template.rightFields = template.rightMessageRepresentation.AllFields; template.resultFields = resultMessageRepresentation.AllFields; template.ActiveEventTypeLeft = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left"; template.ActiveEventTypeRight = rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right"; #region Key Equals var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparerEquals = (left, right) => keyComparer.Inline(left, right); if (keyType.IsAnonymousType()) { template.keyComparerEquals = (left, right) => string.Format("keyComparerEquals({0}, {1})", left, right); } #endregion #region Left Payload Equals { var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0); template.leftComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Right Payload Equals { var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0); template.rightComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Result Selector { var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); if (!ConstantExpressionFinder.IsClosedExpression(selector)) { errorMessages = "result selector is not a closed expression"; throw new InvalidOperationException(); } #region LeftBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), "i"), leftIndex }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region RightBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), "j"), rightIndex } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region ActiveSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.activeSelector = (leftEvent, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion } #endregion if (stream.Left.Properties.IsIntervalFree && stream.Right.Properties.IsConstantDuration) { template.endPointHeap = "EndPointQueue"; } else if (stream.Right.Properties.IsIntervalFree && stream.Left.Properties.IsConstantDuration) { template.endPointHeap = "EndPointQueue"; } else if (stream.Left.Properties.IsConstantDuration && stream.Right.Properties.IsConstantDuration && stream.Left.Properties.ConstantDurationLength == stream.Right.Properties.ConstantDurationLength) { template.endPointHeap = "EndPointQueue"; } else { template.endPointHeap = "EndPointHeap"; } template.staticCtor = Transformer.StaticCtor(template.className); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(keyType, leftType, rightType, resultType); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>()); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); if (keyType.IsAnonymousType()) { if (errorMessages == null) { errorMessages = string.Empty; } errorMessages += "\nCodegen Warning: The key type for an equi-join is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n"; } var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType, resultType); var t = a.GetType(realClassName); if (t.GetTypeInfo().IsGenericType) { var list = keyType.GetAnonymousTypes(); list.AddRange(leftType.GetAnonymousTypes()); list.AddRange(rightType.GetAnonymousTypes()); list.AddRange(resultType.GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
/// <summary> /// Create the template output /// </summary> public override string TransformText() { this.Write(@"// ********************************************************************* // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License // ********************************************************************* using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Linq.Expressions; using System.Text; using System.Threading.Tasks; using System.Reflection; using System.Runtime.Serialization; using Microsoft.StreamProcessing; using Microsoft.StreamProcessing.Internal; using Microsoft.StreamProcessing.Internal.Collections; "); if (this.keyType.Namespace != null) { this.Write("using "); this.Write(this.ToStringHelper.ToStringWithCulture(this.keyType.Namespace)); this.Write(";\r\n"); } if (this.payloadType.Namespace != null) { this.Write("using "); this.Write(this.ToStringHelper.ToStringWithCulture(this.payloadType.Namespace)); this.Write(";\r\n"); } if (this.resultType.Namespace != null) { this.Write("using "); this.Write(this.ToStringHelper.ToStringWithCulture(this.resultType.Namespace)); this.Write(";\r\n"); } this.Write("\r\n// TKey: "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write("\r\n// TPayload: "); this.Write(this.ToStringHelper.ToStringWithCulture(TPayload)); this.Write("\r\n// TResult: "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("\r\n// Source Fields: "); this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.fields.Select(f => f.OriginalName)))); this.Write("\r\n// Destination Fields: "); this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.resultFields.Select(f => f.OriginalName)))); this.Write("\r\n// Computed Fields: "); this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.computedFields.Keys.Select(f => f.OriginalName)))); this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters)); this.Write(" : UnaryPipe<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TPayload)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write(">\r\n{\r\n private "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.resultType))); this.Write(this.ToStringHelper.ToStringWithCulture(MemoryPoolGenericParameters)); this.Write(" pool;\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGene" + "rator;\r\n\r\n [DataMember]\r\n private int iter;\r\n\r\n private StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("> genericBatch;\r\n [DataMember]\r\n private "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters)); this.Write(" batch;\r\n\r\n // Fields used to point directly to the arrays within the result b" + "atch\r\n private long[] dest_vsync;\r\n private long[] dest_vother;\r\n priva" + "te "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write("[] destkey;\r\n private int[] dest_hash;\r\n\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor)); this.Write("\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("() { }\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("(\r\n IStreamable<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("> stream,\r\n IStreamObserver<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator)\r\n" + " : base(stream, observer)\r\n {\r\n pool = MemoryManager.GetMemoryP" + "ool<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write(">() as "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.resultType))); this.Write(this.ToStringHelper.ToStringWithCulture(MemoryPoolGenericParameters)); this.Write(@"; this.queryPlanGenerator = queryPlanGenerator; MyAllocate(); } public override void ProduceQueryPlan(PlanNode previous) { Observer.ProduceQueryPlan(queryPlanGenerator(previous, this)); } private void MyAllocate() { pool.Get(out genericBatch); genericBatch.Allocate(); this.batch = genericBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters)); this.Write(@"; this.UpdatePointers(); } protected override void UpdatePointers() { // Assign pointers to bookkeeping arrays dest_vsync = this.batch.vsync.col; dest_vother = this.batch.vother.col; destkey = this.batch.key.col; dest_hash = this.batch.hash.col; } protected override void DisposeState() => this.batch.Free(); public override unsafe void OnNext(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TPayload)); this.Write("> _inBatch)\r\n {\r\n var batch = _inBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TPayload)); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTPayloadGenericParameters)); this.Write(";\r\n\r\n var count = batch.Count;\r\n this.batch.iter = batch.iter;\r\n\r\n " + " // Create locals that point directly to the arrays within the columns in " + "the destination batch.\r\n\r\n "); foreach (var f in this.computedFields.Keys) { this.Write(" "); if (f.OptimizeString()) { this.Write("\r\n var dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = this.batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(";\r\n "); } else { this.Write("\r\n var dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = this.batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col;\r\n "); } this.Write(" "); } this.Write("\r\n // Create locals that point directly to the arrays within the columns i" + "n the source batch.\r\n "); foreach (var f in this.fields) { this.Write("\r\n "); if (f.canBeFixed) { this.Write("\r\n fixed ("); this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName)); this.Write("* "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write("_col = batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col)\r\n {\r\n "); } else { this.Write("\r\n var "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write("_col = batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col;\r\n "); } this.Write(" "); } this.Write("\r\n var srckey = batch.key.col;\r\n "); if (this.hasKey) { this.Write("\r\n var key_col = srckey; // hack until MakeColumnOriented is fixed\r\n " + " "); } this.Write(@" fixed (long* src_bv = batch.bitvector.col, src_vsync = batch.vsync.col, src_vother = batch.vother.col) fixed (int* src_hash = batch.hash.col) { for (int i = 0; i < count; i++) { if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0) { "); if (this.StartEdgeParameterName != null) { this.Write("\r\n var "); this.Write(this.ToStringHelper.ToStringWithCulture(this.StartEdgeParameterName)); this.Write(" = src_vsync[i] < src_vother[i] ? src_vsync[i] : src_vother[i];\r\n " + " "); } this.Write("\r\n "); if (this.useEnumerator) { this.Write("\r\n var enumerator = "); this.Write(this.ToStringHelper.ToStringWithCulture(transformedSelectorAsSource)); this.Write(".GetEnumerator();\r\n while (enumerator.MoveNext())\r\n " + " "); } else { this.Write(" "); if (!this.enumerableRepeatSelector) { this.Write("\r\n var e_prime = "); this.Write(this.ToStringHelper.ToStringWithCulture(transformedSelectorAsSource)); this.Write(";\r\n "); } this.Write("\r\n "); if (this.keyParameterName != null) { this.Write(" var "); this.Write(this.ToStringHelper.ToStringWithCulture(this.keyParameterName)); this.Write(" = srckey[i];\r\n "); } this.Write(" for (int _x = 0; _x < "); this.Write(this.ToStringHelper.ToStringWithCulture(this.loopCounter)); this.Write("; _x++)\r\n "); } this.Write("\r\n {\r\n dest_vsync[iter] = src_vsync[i];" + "\r\n dest_vother[iter] = src_vother[i];\r\n " + " "); if (this.useEnumerator) { this.Write("\r\n this.batch[iter] = enumerator.Current;\r\n " + " "); } else if (this.enumerableRepeatSelector) { this.Write("\r\n "); foreach (var kv in this.computedFields) { var f = kv.Key; var v = kv.Value.ExpressionToCSharp(); this.Write(" "); if (f.OptimizeString()) { this.Write("\r\n dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".AddString("); this.Write(this.ToStringHelper.ToStringWithCulture(v)); this.Write(");\r\n "); } else { this.Write("\r\n dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write("[iter] = "); this.Write(this.ToStringHelper.ToStringWithCulture(v)); this.Write(";\r\n "); } this.Write(" "); } this.Write(" "); } else { this.Write("\r\n this.batch[iter] = e_prime;\r\n "); } this.Write(@" destkey[iter] = srckey[i]; dest_hash[iter] = src_hash[i]; iter++; if (iter == Config.DataBatchSize) { FlushContents(); // Create locals that point directly to the arrays within the columns in the destination batch. "); foreach (var f in this.computedFields.Keys) { this.Write(" "); if (f.OptimizeString()) { this.Write("\r\n dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = this.batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(";\r\n "); } else { this.Write("\r\n dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = this.batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col;\r\n "); } this.Write(" "); } this.Write("\r\n this.batch.iter = batch.iter;\r\n " + " }\r\n }\r\n "); if (this.useEnumerator) { this.Write("\r\n enumerator.Dispose();\r\n "); } this.Write(@" } else if (src_vother[i] < 0) { dest_vsync[iter] = src_vsync[i]; dest_vother[iter] = src_vother[i]; destkey[iter] = default; this.batch[iter] = default; dest_hash[iter] = src_hash[i]; this.batch.bitvector.col[(iter) >> 6] |= (1L << ((iter) & 0x3f)); iter++; if (iter == Config.DataBatchSize) { FlushContents(); // Create locals that point directly to the arrays within the columns in the destination batch. "); foreach (var f in this.computedFields.Keys) { this.Write(" "); if (f.OptimizeString()) { this.Write("\r\n dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = this.batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(";\r\n "); } else { this.Write("\r\n dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = this.batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col;\r\n "); } this.Write(" "); } this.Write("\r\n this.batch.iter = batch.iter;\r\n }\r\n " + " }\r\n }\r\n\r\n } // end src_hash, src_bv, src_vsync," + " src_vother\r\n\r\n "); foreach (var f in this.fields.Where(fld => fld.canBeFixed)) { this.Write("\r\n }\r\n "); } this.Write(@" batch.Free(); } protected override void FlushContents() { if (iter == 0) return; this.batch.Count = iter; this.batch.Seal(); this.Observer.OnNext(this.batch); iter = 0; MyAllocate(); } public override int CurrentlyBufferedOutputCount => iter; public override int CurrentlyBufferedInputCount => 0; } "); return(this.GenerationEnvironment.ToString()); }
/// <summary> /// Create the template output /// </summary> public override string TransformText() { this.Write(@"// ********************************************************************* // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License // ********************************************************************* using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.Contracts; using System.Linq.Expressions; using System.Runtime.CompilerServices; using System.Runtime.Serialization; using System.Threading; using Microsoft.StreamProcessing; using Microsoft.StreamProcessing.Aggregates; using Microsoft.StreamProcessing.Internal; using Microsoft.StreamProcessing.Internal.Collections; // TKey: "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write("\r\n// TLeft: "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("\r\n// TRight: "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters)); this.Write(" : BinaryPipe<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(">\r\n{\r\n private readonly MemoryPool<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> pool;\r\n\r\n [SchemaSerialization]\r\n private readonly Expression<Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", bool>> keyComparer;\r\n [SchemaSerialization]\r\n private readonly Expression" + "<Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(", bool>> leftComparer;\r\n\r\n private StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> genericOutputBatch;\r\n [DataMember]\r\n private "); this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TLeft)); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters)); this.Write(@" output; [DataMember] private FastMap<LeftInterval> leftIntervalMap = new FastMap<LeftInterval>(); [DataMember] private FastMap<LeftEdge> leftEdgeMap = new FastMap<LeftEdge>(); [DataMember] private RemovableEndPointHeap leftEndPointHeap; [DataMember] private long nextLeftTime = long.MinValue; [DataMember] private long nextRightTime = long.MinValue; [DataMember] private long currTime = long.MinValue; private readonly Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator; "); this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor)); this.Write("\r\n\r\n [Obsolete(\"Used only by serialization. Do not call directly.\")]\r\n publ" + "ic "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("() { }\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("(\r\n IStreamable<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> stream,\r\n IStreamObserver<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> observer,\r\n IEqualityComparerExpression<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write("> keyEqualityComparer,\r\n IEqualityComparerExpression<"); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(@"> leftComparer, Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator) : base(stream, observer) { this.queryPlanGenerator = queryPlanGenerator; this.keyComparer = keyEqualityComparer.GetEqualsExpr(); this.leftComparer = leftComparer.GetEqualsExpr(); this.leftEndPointHeap = new RemovableEndPointHeap(); this.pool = MemoryManager.GetMemoryPool<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(">(true /*stream.Properties.IsColumnar*/);\r\n "); GetOutputBatch(); this.Write(@" } public override int CurrentlyBufferedOutputCount => output.Count; public override int CurrentlyBufferedLeftInputCount => base.CurrentlyBufferedLeftInputCount + leftEdgeMap.Count + leftIntervalMap.Count; public override int CurrentlyBufferedRightInputCount => base.CurrentlyBufferedRightInputCount; protected override void ProduceBinaryQueryPlan(PlanNode left, PlanNode right) { this.Observer.ProduceQueryPlan(queryPlanGenerator(left, right, this)); } protected override void DisposeState() => this.output.Free(); [MethodImpl(MethodImplOptions.AggressiveInlining)] protected override void ProcessBothBatches(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> genericLeftBatch, StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write("> genericRightBatch, out bool leftBatchDone, out bool rightBatchDone, out bool le" + "ftBatchFree, out bool rightBatchFree)\r\n {\r\n var leftBatch = genericLef" + "tBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, leftType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters)); this.Write(";\r\n var rightBatch = genericRightBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, rightType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTRightGenericParameters)); this.Write(@"; leftBatchFree = rightBatchFree = true; if (!GoToVisibleRow(leftBatch)) { leftBatchDone = true; rightBatchDone = false; return; } nextLeftTime = leftBatch.vsync.col[leftBatch.iter]; if (!GoToVisibleRow(rightBatch)) { leftBatchDone = false; rightBatchDone = true; return; } nextRightTime = rightBatch.vsync.col[rightBatch.iter]; while (true) { if (nextLeftTime <= nextRightTime) { UpdateTime(nextLeftTime); /* ProcessLeftEvent( nextLeftTime, leftBatch.vother.col[leftBatch.iter], ref leftBatch.key.col[leftBatch.iter], leftBatch[leftBatch.iter], leftBatch.hash.col[leftBatch.iter]); */ "); ProcessLeftEvent("nextLeftTime", "leftBatch.vother.col[leftBatch.iter]", "leftBatch.key.col[leftBatch.iter]", "leftBatch", "leftBatch.iter", "leftBatch.hash.col[leftBatch.iter]"); this.Write(@" leftBatch.iter++; if (!GoToVisibleRow(leftBatch)) { leftBatchDone = true; rightBatchDone = false; return; } nextLeftTime = leftBatch.vsync.col[leftBatch.iter]; } else { UpdateTime(nextRightTime); ProcessRightEvent( nextRightTime, rightBatch.vother.col[rightBatch.iter], ref rightBatch.key.col[rightBatch.iter], rightBatch.hash.col[rightBatch.iter]); rightBatch.iter++; if (!GoToVisibleRow(rightBatch)) { leftBatchDone = false; rightBatchDone = true; return; } nextRightTime = rightBatch.vsync.col[rightBatch.iter]; } } } [MethodImpl(MethodImplOptions.AggressiveInlining)] protected override void ProcessLeftBatch(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> genericBatch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var b" + "atch = genericBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, leftType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters)); this.Write(@"; isBatchFree = true; while (true) { if (!GoToVisibleRow(batch)) { isBatchDone = true; return; } nextLeftTime = batch.vsync.col[batch.iter]; if (nextLeftTime > nextRightTime) { isBatchDone = false; return; } UpdateTime(nextLeftTime); /* ProcessLeftEvent( nextLeftTime, batch.vother.col[batch.iter], ref batch.key.col[batch.iter], batch[batch.iter], batch.hash.col[batch.iter]); */ "); ProcessLeftEvent("nextLeftTime", "batch.vother.col[batch.iter]", "batch.key.col[batch.iter]", "batch", "batch.iter", "batch.hash.col[batch.iter]"); this.Write("\r\n batch.iter++;\r\n }\r\n }\r\n\r\n [MethodImpl(MethodImplOption" + "s.AggressiveInlining)]\r\n protected override void ProcessRightBatch(StreamMess" + "age<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write("> genericBatch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var b" + "atch = genericBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, rightType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTRightGenericParameters)); this.Write(@"; isBatchFree = true; while (true) { if (!GoToVisibleRow(batch)) { isBatchDone = true; return; } nextRightTime = batch.vsync.col[batch.iter]; if (nextRightTime > nextLeftTime) { isBatchDone = false; return; } UpdateTime(nextRightTime); ProcessRightEvent( nextRightTime, batch.vother.col[batch.iter], ref batch.key.col[batch.iter], batch.hash.col[batch.iter]); batch.iter++; } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool GoToVisibleRow<TPayload>(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(@", TPayload> batch) { while (batch.iter < batch.Count && (batch.bitvector.col[batch.iter >> 6] & (1L << (batch.iter & 0x3f))) != 0 && batch.vother.col[batch.iter] >= 0) { batch.iter++; } return (batch.iter != batch.Count); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void UpdateTime(long time) { if (time != currTime) { currTime = time; ReachTime(); } } /* [MethodImpl(MethodImplOptions.AggressiveInlining)] private void ProcessLeftEvent(long start, long end, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(" payload, int hash)\r\n {\r\n if (start < end)\r\n {\r\n // R" + "ow is a start edge or interval.\r\n bool isInterval = end < StreamEvent" + ".InfinitySyncTime;\r\n if (isInterval)\r\n {\r\n " + "bool isFullyOutputtable = nextRightTime >= end;\r\n if (isFullyOutp" + "uttable)\r\n {\r\n // Output full interval.\r\n " + " AddToBatch(start, end, ref key, ref payload, hash);\r\n " + " }\r\n else\r\n {\r\n // Insert in" + "to map to remember interval.\r\n int mapIndex = leftIntervalMap" + ".Insert(hash);\r\n\r\n // Insert into heap to schedule removal at" + " endpoint.\r\n int heapIndex = leftEndPointHeap.Insert(end, map" + "Index);\r\n\r\n // Set value in map, also remembering heap\'s inde" + "x.\r\n leftIntervalMap.Values[mapIndex].Initialize(start, ref k" + "ey, ref payload, heapIndex);\r\n\r\n // Output start edge.\r\n " + " AddToBatch(start, StreamEvent.InfinitySyncTime, ref key, ref payl" + "oad, hash);\r\n }\r\n }\r\n else\r\n {\r\n" + " int index = leftEdgeMap.Insert(hash);\r\n leftEdgeM" + "ap.Values[index].Populate(start, ref key, ref payload);\r\n\r\n // Ou" + "tput start edge.\r\n AddToBatch(start, StreamEvent.InfinitySyncTime" + ", ref key, ref payload, hash);\r\n }\r\n }\r\n else if (end =" + "= StreamEvent.PunctuationOtherTime)\r\n {\r\n AddPunctuationToBatc" + "h(start);\r\n }\r\n else\r\n {\r\n // Row is an end edge" + ".\r\n var leftEvents = leftEdgeMap.Find(hash);\r\n int index;\r" + "\n while (leftEvents.Next(out index))\r\n {\r\n " + "var leftEdge = leftEdgeMap.Values[index];\r\n var leftKey = leftEdg" + "e.Batch.key.col[leftEdge.Index];\r\n if (AreSame(end, ref key, ref " + "payload, ref leftEdgeMap.Values[index]))\r\n {\r\n " + " // Output end edge.\r\n AddToBatch(start, end, ref key, ref pa" + "yload, hash);\r\n\r\n // Remove from leftMap.\r\n " + " leftEvents.Remove();\r\n break;\r\n }\r\n " + " }\r\n }\r\n }\r\n*/\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlini" + "ng)]\r\n private void ProcessRightEvent(long start, long end, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(@" key, int hash) { if (end == StreamEvent.PunctuationOtherTime) { AddPunctuationToBatch(start); return; } else if (start >= end) { // Row is an end edge, which we don't care about because the start edge would have already // removed all joining left events. return; } // Mark any matching left intervals as no longer active. int index; var leftIntervals = leftIntervalMap.Find(hash); while (leftIntervals.Next(out index)) { long leftStart = leftIntervalMap.Values[index].Start; var leftInterval = leftIntervalMap.Values[index]; var leftIntervalKey = leftInterval.Key; if (leftStart < start && "); this.Write(this.ToStringHelper.ToStringWithCulture(this.keyComparer("key", "leftIntervalKey"))); this.Write(@") { // Output end edge. /* AddToBatch( start, leftStart, ref leftIntervalMap.Values[index].Key, ref leftIntervalMap.Values[index].Payload, hash); */ "); AddToBatch("start", "leftStart", "leftIntervalKey", "leftIntervalMap.Values[index]", "hash"); this.Write(@" // Remove from heap and map. leftEndPointHeap.Remove(leftIntervalMap.Values[index].HeapIndex); leftIntervals.Remove(); } } // Remove any matching left edges. var leftEdges = leftEdgeMap.Find(hash); while (leftEdges.Next(out index)) { var leftEdge = leftEdgeMap.Values[index]; long leftStart = leftEdge.Start; var leftEdgeKey = leftEdge.Key; if (leftStart < start && "); this.Write(this.ToStringHelper.ToStringWithCulture(this.keyComparer("key", "leftEdgeKey"))); this.Write(@") { // Output end edge. /* AddToBatch( start, leftStart, ref leftEdgeMap.Values[index].Key, ref leftEdgeMap.Values[index].Payload, hash); */ "); AddToBatch("start", "leftStart", "leftEdgeKey", "leftEdge", "hash"); this.Write(@" // Remove left event. leftEdges.Remove(); } } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void ReachTime() { // Carry-out all interval endpoints for left intervals that end prior or at new current time. long endPointTime; int index; while (leftEndPointHeap.TryGetNextInclusive(currTime, out endPointTime, out index)) { // Output end edge. /* AddToBatch( endPointTime, leftIntervalMap.Values[index].Start, ref leftIntervalMap.Values[index].Key, ref leftIntervalMap.Values[index].Payload, leftIntervalMap.GetHash(index)); */ var leftInterval = leftIntervalMap.Values[index]; var leftIntervalKey = leftInterval.Key; var leftIntervalStart = leftInterval.Start; var leftIntervalHash = leftIntervalMap.GetHash(index); "); AddToBatch("endPointTime", "leftIntervalStart", "leftIntervalKey", "leftInterval", "leftIntervalHash"); this.Write(@" // Remove from leftMap. leftIntervalMap.Remove(index); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void AddPunctuationToBatch(long start) { if (start > lastCTI) { lastCTI = start; int index = output.Count++; output.vsync.col[index] = start; output.vother.col[index] = StreamEvent.PunctuationOtherTime; output.key.col[index] = default; "); foreach (var f in this.leftFields) { if (f.OptimizeString()) { this.Write(" output."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".AddString(string.Empty);\r\n"); } else { this.Write(" output."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col[index] = default;\r\n"); } } this.Write(@" output.hash.col[index] = 0; output.bitvector.col[index >> 6] |= (1L << (index & 0x3f)); if (output.Count == Config.DataBatchSize) FlushContents(); } } /* [MethodImpl(MethodImplOptions.AggressiveInlining)] private void AddToBatch(long start, long end, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(@" payload, int hash) { int index = output.Count++; output.vsync.col[index] = start; output.vother.col[index] = end; output.key.col[index] = key; output[index] = payload; output.hash.col[index] = hash; if (output.Count == Config.DataBatchSize) { this.Observer.OnNext(output); pool.Get(out output); output.Allocate(); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool AreSame(long start, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(@" payload, ref LeftEdge active) { return start == active.Start && keyComparerEquals(key, active.Key) && leftComparerEquals(payload, active.Payload); } */ protected override void FlushContents() { if (output.Count == 0) return; this.Observer.OnNext(output); "); GetOutputBatch(); this.Write(" }\r\n"); if (!noFields && !this.leftType.GetTypeInfo().IsValueType) { this.Write(" [DataContract]\r\n private struct "); this.Write(this.ToStringHelper.ToStringWithCulture(ActiveEventType)); this.Write("\r\n {\r\n"); foreach (var f in this.leftFields) { this.Write(" [DataMember]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax())); this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(";\r\n"); } this.Write(" }\r\n"); } this.Write("/*\r\n [DataContract]\r\n private struct LeftInterval\r\n {\r\n [DataMemb" + "er]\r\n public long Start;\r\n [DataMember]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" Key;\r\n [DataMember]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(" Payload;\r\n [DataMember]\r\n public int HeapIndex;\r\n\r\n [Method" + "Impl(MethodImplOptions.AggressiveInlining)]\r\n public void Initialize(long" + " start, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(@" payload, int heapIndex) { Start = start; Key = key; Payload = payload; HeapIndex = heapIndex; } public override string ToString() { return ""[Start="" + Start + "", Key='"" + Key + ""', Payload='"" + Payload + ""', HeapIndex="" + HeapIndex + ""]""; } } */ [DataContract] private struct LeftInterval { [DataMember] public long Start; [DataMember] public "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" Key;\r\n [DataMember]\r\n public int HeapIndex;\r\n [DataMember]\r" + "\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(ActiveEventType)); this.Write(" Payload;\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " + "public void Initialize(long start, "); this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TLeft)); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters)); this.Write(" batch, int index, int heapIndex)\r\n {\r\n Start = start;\r\n " + " Key = batch.key.col[index];\r\n"); foreach (var f in this.leftFields) { this.Write(" this.Payload."); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(" = "); this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index"))); this.Write(";\r\n"); } this.Write(@" HeapIndex = heapIndex; } public override string ToString() { //return string.Format(""[Start={0}, Key='{1}', Payload='{2}', HeapIndex={3}"", Start, Key, Batch[Index], HeapIndex); System.Text.StringBuilder sb = new System.Text.StringBuilder(); sb.AppendFormat(""[Start={0}"", Start); sb.AppendFormat("", Key='{0}'"", Key); "); foreach (var f in this.leftFields) { this.Write(" sb.AppendFormat(\", "); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write("={0}, \", Payload."); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(");\r\n "); } this.Write(@" sb.AppendFormat("", Key='{0}'"", HeapIndex); sb.Append(""]""); return sb.ToString(); } } /* [DataContract] private struct LeftEdge { [DataMember] public long Start; [DataMember] public "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" Key;\r\n [DataMember]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(" Payload;\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " + "public void Populate(long start, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(@" payload) { Start = start; Key = key; Payload = payload; } public override string ToString() { return ""[Start="" + Start + "", Key='"" + Key + ""', Payload='"" + Payload + ""]""; } } */ [DataContract] private struct LeftEdge { [DataMember] public long Start; [DataMember] public "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" Key;\r\n [DataMember]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(ActiveEventType)); this.Write(" Payload;\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " + "public void Populate(long start, "); this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TLeft)); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters)); this.Write(" batch, int index)\r\n {\r\n Start = start;\r\n Key = batc" + "h.key.col[index];\r\n"); foreach (var f in this.leftFields) { this.Write(" this.Payload."); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(" = "); this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index"))); this.Write(";\r\n"); } this.Write(@" } public override string ToString() { System.Text.StringBuilder sb = new System.Text.StringBuilder(); sb.AppendFormat(""[Start={0}"", Start); sb.AppendFormat("", Key='{0}'"", Key); "); foreach (var f in this.leftFields) { this.Write(" sb.AppendFormat(\", "); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write("={0}, \", Payload."); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(");\r\n "); } this.Write(" sb.Append(\"]\");\r\n return sb.ToString();\r\n //ret" + "urn string.Format(\"[Start={0}, Key=\'{1}\', Payload=\'{2}\'\", Start, KeyColumn.col[R" + "owIndex], Batch[RowIndex]);\r\n }\r\n }\r\n}\r\n\r\n"); return(this.GenerationEnvironment.ToString()); }
/// <summary> /// Create the template output /// </summary> public override string TransformText() { this.Write(@"// ********************************************************************* // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License // ********************************************************************* using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.Contracts; using System.Linq.Expressions; using System.Runtime.CompilerServices; using System.Runtime.Serialization; using System.Threading; using Microsoft.StreamProcessing; using Microsoft.StreamProcessing.Aggregates; using Microsoft.StreamProcessing.Internal; using Microsoft.StreamProcessing.Internal.Collections; [assembly: IgnoresAccessChecksTo(""Microsoft.StreamProcessing"")] // TKey: "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write("\r\n// TLeft: "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("\r\n// TRight: "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write("\r\n\r\n"); var leftBatchTypeName = Transformer.GetBatchClassName(this.keyType, this.leftType) + this.TKeyTLeftGenericParameters; var rightBatchTypeName = Transformer.GetBatchClassName(this.keyType, this.rightType) + this.TKeyTRightGenericParameters; this.Write("\r\n[DataContract]\r\n[KnownType(typeof(EndPointHeap))]\r\n[KnownType(typeof(EndPointQu" + "eue))]\r\ninternal sealed class "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters)); this.Write(" : BinaryPipe<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(">, IBinaryObserver\r\n{\r\n private const long NotActive = long.MaxValue;\r\n pri" + "vate readonly MemoryPool<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> pool;\r\n\r\n [SchemaSerialization]\r\n private readonly Expression<Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", bool>> keyComparer;\r\n [SchemaSerialization]\r\n private readonly Expression" + "<Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(", bool>> leftComparer;\r\n\r\n private StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> genericOutputBatch;\r\n [DataMember]\r\n private "); this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TLeft)); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters)); this.Write(@" output; [DataMember] private FastMap<LeftEvent> leftIntervalMap = new FastMap<LeftEvent>(); [DataMember] private FastMap<LeftEvent> leftEdgeMap = new FastMap<LeftEvent>(); [DataMember] private IEndPointOrderer leftEndPointHeap; [DataMember] private FastMap<RightEvent> rightMap = new FastMap<RightEvent>(); [DataMember] private FastStack<QueuedEndEdge> rightEndEdges = new FastStack<QueuedEndEdge>(); [DataMember] private IEndPointOrderer rightEndPointHeap; [DataMember] private long nextLeftTime = long.MinValue; [DataMember] private long nextRightTime = long.MinValue; [DataMember] private long currTime = long.MinValue; private readonly Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator; "); this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor)); this.Write("\r\n\r\n [Obsolete(\"Used only by serialization. Do not call directly.\")]\r\n publ" + "ic "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("() { }\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("(\r\n IStreamable<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> stream,\r\n IStreamObserver<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> observer,\r\n IEqualityComparerExpression<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write("> keyEqualityComparer,\r\n IEqualityComparerExpression<"); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(@"> leftComparer, Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator, bool leftIsConstantDuration, bool rightIsConstantDuration) : base(stream, observer) { this.queryPlanGenerator = queryPlanGenerator; this.keyComparer = keyEqualityComparer.GetEqualsExpr(); this.leftComparer = leftComparer.GetEqualsExpr(); if (leftIsConstantDuration) this.leftEndPointHeap = new EndPointQueue(); else this.leftEndPointHeap = new EndPointHeap(); if (rightIsConstantDuration) this.rightEndPointHeap = new EndPointQueue(); else this.rightEndPointHeap = new EndPointHeap(); this.pool = MemoryManager.GetMemoryPool<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(">(true);\r\n GetOutputBatch();\r\n }\r\n\r\n public override int CurrentlyBu" + "fferedOutputCount => this.output.Count;\r\n public override int CurrentlyBuffer" + "edLeftInputCount => base.CurrentlyBufferedLeftInputCount + this.leftEdgeMap.Coun" + "t + this.leftIntervalMap.Count;\r\n public override int CurrentlyBufferedRightI" + "nputCount => base.CurrentlyBufferedRightInputCount + this.rightEndEdges.Count + " + "this.rightMap.Count;\r\n\r\n protected override void ProduceBinaryQueryPlan(PlanN" + "ode left, PlanNode right)\r\n {\r\n Observer.ProduceQueryPlan(queryPlanGen" + "erator(left, right, this));\r\n }\r\n\r\n protected override void DisposeState()" + "\r\n {\r\n this.output.Free();\r\n\r\n var allLeftEdgeTraverser = this." + "leftEdgeMap.Traverse();\r\n allLeftEdgeTraverser.currIndex = 0;\r\n in" + "t x, y;\r\n while (allLeftEdgeTraverser.Next(out x, out y))\r\n {\r\n " + " var leftInterval = this.leftEdgeMap.Values[x];\r\n }\r\n var " + "allLeftEdgeInvisibleTraverser = this.leftEdgeMap.TraverseInvisible();\r\n w" + "hile (allLeftEdgeInvisibleTraverser.Next(out x, out y))\r\n {\r\n " + "var leftEdge = this.leftEdgeMap.Values[x];\r\n }\r\n\r\n var allLeftInte" + "rvalTraverser = this.leftIntervalMap.Traverse();\r\n allLeftIntervalTravers" + "er.currIndex = 0;\r\n while (allLeftIntervalTraverser.Next(out x, out y))\r\n" + " {\r\n var leftInterval = this.leftIntervalMap.Values[x];\r\n " + " }\r\n var allLeftIntervalInvisibleTraverser = this.leftIntervalMap.Trave" + "rseInvisible();\r\n while (allLeftIntervalInvisibleTraverser.Next(out x, ou" + "t y))\r\n {\r\n var leftInterval = this.leftIntervalMap.Values[x];" + "\r\n }\r\n }\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " + " protected override void ProcessBothBatches(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> genericLeftBatch, StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write("> genericRightBatch, out bool leftBatchDone, out bool rightBatchDone, out bool le" + "ftBatchFree, out bool rightBatchFree)\r\n {\r\n var leftBatch = genericLef" + "tBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName)); this.Write(";\r\n var rightBatch = genericRightBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName)); this.Write(";\r\n\r\n leftBatchFree = rightBatchFree = true;\r\n if (!GoToVisibleRow(" + "leftBatch))\r\n {\r\n leftBatchDone = true;\r\n rightBatc" + "hDone = false;\r\n return;\r\n }\r\n\r\n this.nextLeftTime = le" + "ftBatch.vsync.col[leftBatch.iter];\r\n\r\n if (!GoToVisibleRow(rightBatch))\r\n" + " {\r\n leftBatchDone = false;\r\n rightBatchDone = true" + ";\r\n return;\r\n }\r\n\r\n this.nextRightTime = rightBatch.vsy" + "nc.col[rightBatch.iter];\r\n\r\n while (true)\r\n {\r\n if (thi" + "s.nextLeftTime <= this.nextRightTime)\r\n {\r\n UpdateTime" + "(this.nextLeftTime);\r\n\r\n ProcessLeftEvent(\r\n t" + "his.nextLeftTime,\r\n leftBatch.vother.col[leftBatch.iter],\r\n " + " ref leftBatch.key.col[leftBatch.iter],\r\n le" + "ftBatch,\r\n leftBatch.iter,\r\n leftBatch.has" + "h.col[leftBatch.iter]);\r\n\r\n leftBatch.iter++;\r\n\r\n " + "if (!GoToVisibleRow(leftBatch))\r\n {\r\n leftBatc" + "hDone = true;\r\n rightBatchDone = false;\r\n " + "return;\r\n }\r\n\r\n this.nextLeftTime = leftBatch.vsyn" + "c.col[leftBatch.iter];\r\n }\r\n else\r\n {\r\n " + " UpdateTime(this.nextRightTime);\r\n\r\n ProcessRightEvent(\r\n " + " this.nextRightTime,\r\n rightBatch.vother.co" + "l[rightBatch.iter],\r\n ref rightBatch.key.col[rightBatch.iter]" + ",\r\n rightBatch.hash.col[rightBatch.iter]);\r\n\r\n " + " rightBatch.iter++;\r\n\r\n if (!GoToVisibleRow(rightBatch))\r\n " + " {\r\n leftBatchDone = false;\r\n righ" + "tBatchDone = true;\r\n return;\r\n }\r\n\r\n " + " this.nextRightTime = rightBatch.vsync.col[rightBatch.iter];\r\n }" + "\r\n }\r\n }\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " + " protected override void ProcessLeftBatch(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> genericBatch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var b" + "atch = genericBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName)); this.Write(@"; isBatchFree = true; while (true) { if (!GoToVisibleRow(batch)) { isBatchDone = true; return; } this.nextLeftTime = batch.vsync.col[batch.iter]; if (this.nextLeftTime > this.nextRightTime) { isBatchDone = false; return; } UpdateTime(this.nextLeftTime); ProcessLeftEvent( this.nextLeftTime, batch.vother.col[batch.iter], ref batch.key.col[batch.iter], batch, batch.iter, batch.hash.col[batch.iter]); batch.iter++; } } [MethodImpl(MethodImplOptions.AggressiveInlining)] protected override void ProcessRightBatch(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write("> genericBatch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var b" + "atch = genericBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName)); this.Write(@"; isBatchFree = true; while (true) { if (!GoToVisibleRow(batch)) { isBatchDone = true; return; } this.nextRightTime = batch.vsync.col[batch.iter]; if (this.nextRightTime > this.nextLeftTime) { isBatchDone = false; return; } UpdateTime(this.nextRightTime); ProcessRightEvent( this.nextRightTime, batch.vother.col[batch.iter], ref batch.key.col[batch.iter], batch.hash.col[batch.iter]); batch.iter++; } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool GoToVisibleRow<TPayload>(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(@", TPayload> batch) { while (batch.iter < batch.Count && (batch.bitvector.col[batch.iter >> 6] & (1L << (batch.iter & 0x3f))) != 00 && batch.vother.col[batch.iter] >= 0) { batch.iter++; } if (batch.iter == batch.Count) { return false; } return true; } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void UpdateTime(long time) { if (time != this.currTime) { LeaveTime(); this.currTime = time; ReachTime(); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void ProcessLeftEvent(long start, long end, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, "); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName)); this.Write(" batch, int batchIndex, int hash)\r\n {\r\n if (start < end)\r\n {\r\n " + " // Row is a start edge or interval.\r\n bool isProcessable = " + "this.nextRightTime > start;\r\n bool isInterval = end < StreamEvent.Inf" + "initySyncTime;\r\n var map = isInterval ? this.leftIntervalMap : this.l" + "eftEdgeMap;\r\n if (isProcessable)\r\n {\r\n int " + "matchingRight;\r\n if (FindOnRight(ref key, hash, out matchingRight" + "))\r\n {\r\n // Row joins with something on right," + " so not currently visible.\r\n int index = map.Insert(hash);\r\n " + " map.Values[index].Populate(start, NotActive, end, ref key, ba" + "tch, batchIndex);\r\n if (isInterval)\r\n {\r\n " + " this.leftEndPointHeap.Insert(end, index);\r\n " + " }\r\n }\r\n else\r\n {\r\n " + " // Row does not join (and it is processable).\r\n bool" + " isFullyOutputtable = isInterval && this.nextRightTime >= end;\r\n " + " if (isFullyOutputtable)\r\n {\r\n // W" + "ill never join because right has advanced beyond endtime, so output interval.\r\n " + " AddToBatch(start, end, ref key, batch, batchIndex, hash);" + "\r\n }\r\n else\r\n {\r\n " + " // Output start edge.\r\n int index = map" + ".Insert(hash);\r\n map.Values[index].Populate(start, start," + " end, ref key, batch, batchIndex);\r\n AddToBatch(start, St" + "reamEvent.InfinitySyncTime, ref key, batch, batchIndex, hash);\r\n " + " if (isInterval)\r\n {\r\n " + "this.leftEndPointHeap.Insert(end, index);\r\n }\r\n " + " }\r\n }\r\n }\r\n else\r\n {\r\n" + " // Row is not yet processable, so insert as invisible.\r\n " + " int index = map.InsertInvisible(hash);\r\n map.Values[index" + "].Populate(start, NotActive, end, ref key, batch, batchIndex);\r\n }\r\n " + " }\r\n else if (end == StreamEvent.PunctuationOtherTime)\r\n {\r\n" + " AddPunctuationToBatch(start);\r\n }\r\n else\r\n {\r\n " + " // Row is an end edge.\r\n\r\n // Remove from this.leftEdgeMap" + ".\r\n var leftEvents = this.leftEdgeMap.Find(hash);\r\n int in" + "dex;\r\n while (leftEvents.Next(out index))\r\n {\r\n " + " var leftEdge = this.leftEdgeMap.Values[index];\r\n if (AreSame" + "(end, StreamEvent.InfinitySyncTime, ref key, batch, batchIndex, ref leftEdge))\r\n" + " {\r\n long currentStart = leftEdge.CurrentStart" + ";\r\n if (currentStart != NotActive)\r\n {\r\n " + " // Matching left start edge is currently visible, so outpu" + "t end edge.\r\n AddToBatch(start, currentStart, ref key, ba" + "tch, batchIndex, hash);\r\n }\r\n\r\n leftEvents" + ".Remove();\r\n break;\r\n }\r\n }\r\n " + " }\r\n }\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n privat" + "e void ProcessRightEvent(long start, long end, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, int hash)\r\n {\r\n if (start < end)\r\n {\r\n // Row i" + "s a start edge or interval.\r\n int index;\r\n if (FindOnRight" + "(ref key, hash, out index))\r\n {\r\n // Corresponding key" + " already exists in map, so any joining on left and already not active.\r\n " + " this.rightMap.Values[index].Count++;\r\n }\r\n else\r\n " + " {\r\n // First instance of this key, so insert and make " + "any joining left entries not active.\r\n index = this.rightMap.Inse" + "rt(hash);\r\n this.rightMap.Values[index].Initialize(ref key);\r\n " + " MakeMatchingLeftInvisible(start, ref key, hash);\r\n }\r\n\r\n" + " if (end != StreamEvent.InfinitySyncTime)\r\n {\r\n " + " // Row is an interval, so schedule removal of interval.\r\n th" + "is.rightEndPointHeap.Insert(end, index);\r\n }\r\n }\r\n else" + " if (end == StreamEvent.PunctuationOtherTime)\r\n {\r\n AddPunctua" + "tionToBatch(start);\r\n }\r\n else\r\n {\r\n // Row is a" + "n end edge.\r\n\r\n // Queue for removal when time advances.\r\n " + " int index = this.rightEndEdges.Push();\r\n this.rightEndEdges.Values[i" + "ndex].Populate(ref key, hash);\r\n }\r\n }\r\n\r\n [MethodImpl(MethodImplOp" + "tions.AggressiveInlining)]\r\n private void LeaveTime()\r\n {\r\n // Carr" + "y-out all queued end edges for right events.\r\n int index;\r\n int ha" + "sh;\r\n for (int i = 0; i < this.rightEndEdges.Count; i++)\r\n {\r\n " + " hash = this.rightEndEdges.Values[i].Hash;\r\n if (FindOnRight(r" + "ef this.rightEndEdges.Values[i].Key, hash, out index))\r\n {\r\n " + " int count = this.rightMap.Values[index].Count - 1;\r\n if (c" + "ount > 0)\r\n {\r\n this.rightMap.Values[index].Co" + "unt = count;\r\n }\r\n else\r\n {\r\n " + " MakeMatchingLeftVisible(this.currTime, ref this.rightMap.Values[in" + "dex].Key, hash);\r\n this.rightMap.Remove(index);\r\n " + " }\r\n }\r\n }\r\n\r\n this.rightEndEdges.Clear();\r\n\r\n " + " // Actually insert all pending left start intervals.\r\n var leftEvents =" + " this.leftIntervalMap.TraverseInvisible();\r\n while (leftEvents.Next(out i" + "ndex, out hash))\r\n {\r\n var leftInterval = this.leftIntervalMap" + ".Values[index];\r\n long end = leftInterval.End;\r\n int match" + "ingRight;\r\n if (FindOnRight(ref leftInterval.Key, hash, out matchingR" + "ight))\r\n {\r\n leftEvents.MakeVisible();\r\n " + " this.leftEndPointHeap.Insert(end, index);\r\n }\r\n else\r\n " + " {\r\n // Row does not join.\r\n bool isFully" + "Outputtable = this.nextRightTime >= end;\r\n if (isFullyOutputtable" + ")\r\n {\r\n AddToBatch(\r\n t" + "his.currTime,\r\n end,\r\n ref leftInt" + "erval.Key,\r\n ref leftInterval,\r\n h" + "ash);\r\n leftEvents.Remove();\r\n }\r\n " + " else\r\n {\r\n leftEvents.MakeVisible();\r\n " + " this.leftIntervalMap.Values[index].CurrentStart = this.currTime;" + "\r\n AddToBatch(\r\n this.currTime,\r\n " + " StreamEvent.InfinitySyncTime,\r\n ref th" + "is.leftIntervalMap.Values[index].Key,\r\n ref this.leftInte" + "rvalMap.Values[index],\r\n hash);\r\n\r\n th" + "is.leftEndPointHeap.Insert(end, index);\r\n }\r\n }\r\n " + " }\r\n\r\n // Actually insert all pending left start edges.\r\n leftEv" + "ents = this.leftEdgeMap.TraverseInvisible();\r\n while (leftEvents.Next(out" + " index, out hash))\r\n {\r\n int matchingRight;\r\n if (!" + "FindOnRight(ref this.leftEdgeMap.Values[index].Key, hash, out matchingRight))\r\n " + " {\r\n // Row does not join, so output start edge.\r\n " + " this.leftEdgeMap.Values[index].CurrentStart = this.currTime;\r\n " + " AddToBatch(\r\n this.currTime,\r\n St" + "reamEvent.InfinitySyncTime,\r\n ref this.leftEdgeMap.Values[ind" + "ex].Key,\r\n ref this.leftEdgeMap.Values[index],\r\n " + " hash);\r\n }\r\n\r\n leftEvents.MakeVisible();\r\n }" + "\r\n }\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n private v" + "oid ReachTime()\r\n {\r\n // Carry-out all interval endpoints for left int" + "ervals that end prior to (or at) new current time.\r\n long leftTime;\r\n " + " int leftIndex;\r\n if (!this.leftEndPointHeap.TryPeekNext(out leftTime," + " out leftIndex))\r\n {\r\n leftTime = long.MaxValue;\r\n }\r\n\r" + "\n long rightTime;\r\n int rightIndex;\r\n if (!this.rightEndPoi" + "ntHeap.TryPeekNext(out rightTime, out rightIndex))\r\n {\r\n right" + "Time = long.MaxValue;\r\n }\r\n\r\n while (leftTime <= this.currTime || " + "rightTime < this.currTime)\r\n {\r\n if (leftTime <= rightTime)\r\n " + " {\r\n // Always process left end-points first as they ma" + "y end intervals that right end-points may\r\n // try to make visibl" + "e.\r\n var leftInterval = this.leftIntervalMap.Values[leftIndex];\r\n" + " long currentStart = leftInterval.CurrentStart;\r\n " + "if (currentStart != NotActive)\r\n {\r\n // Matchi" + "ng left start edge is currently visible, so output end edge.\r\n " + " AddToBatch(\r\n leftTime,\r\n curren" + "tStart,\r\n ref leftInterval.Key,\r\n " + "ref leftInterval,\r\n this.leftIntervalMap.GetHash(leftInde" + "x));\r\n }\r\n\r\n this.leftIntervalMap.Remove(leftIndex" + ");\r\n\r\n this.leftEndPointHeap.RemoveTop();\r\n if (!t" + "his.leftEndPointHeap.TryPeekNext(out leftTime, out leftIndex))\r\n " + "{\r\n leftTime = long.MaxValue;\r\n }\r\n " + " }\r\n else\r\n {\r\n // Process right end-point " + "up to but not including the current time.\r\n int count = this.righ" + "tMap.Values[rightIndex].Count - 1;\r\n if (count > 0)\r\n " + " {\r\n this.rightMap.Values[rightIndex].Count = count;\r\n " + " }\r\n else\r\n {\r\n Make" + "MatchingLeftVisible(rightTime, ref this.rightMap.Values[rightIndex].Key, this.ri" + "ghtMap.GetHash(rightIndex));\r\n this.rightMap.Remove(rightInde" + "x);\r\n }\r\n\r\n this.rightEndPointHeap.RemoveTop();\r\n " + " if (!this.rightEndPointHeap.TryPeekNext(out rightTime, out rightI" + "ndex))\r\n {\r\n rightTime = long.MaxValue;\r\n " + " }\r\n }\r\n }\r\n }\r\n\r\n [MethodImpl(MethodImplOptio" + "ns.AggressiveInlining)]\r\n private bool FindOnRight(ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, int hash, out int index)\r\n {\r\n int rightIndex;\r\n var right" + "Events = this.rightMap.Find(hash);\r\n while (rightEvents.Next(out rightInd" + "ex))\r\n {\r\n if ("); this.Write(this.ToStringHelper.ToStringWithCulture(keyComparer("key", "this.rightMap.Values[rightIndex].Key"))); this.Write(@") { index = rightIndex; return true; } } index = 0; return false; } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void MakeMatchingLeftInvisible(long time, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, int hash)\r\n {\r\n // Make matching left intervals invisible.\r\n " + " int index;\r\n var leftEvents = this.leftIntervalMap.Find(hash);\r\n " + " while (leftEvents.Next(out index))\r\n {\r\n if ("); this.Write(this.ToStringHelper.ToStringWithCulture(keyComparer("key", "this.leftIntervalMap.Values[index].Key"))); this.Write(@") { // Output end edge. AddToBatch( time, this.leftIntervalMap.Values[index].CurrentStart, ref this.leftIntervalMap.Values[index].Key, ref this.leftIntervalMap.Values[index], hash); // Mark left event as not visible. this.leftIntervalMap.Values[index].CurrentStart = NotActive; } } // Make matching left edges invisible. leftEvents = this.leftEdgeMap.Find(hash); while (leftEvents.Next(out index)) { if ("); this.Write(this.ToStringHelper.ToStringWithCulture(keyComparer("key", "this.leftEdgeMap.Values[index].Key"))); this.Write(@") { // Output end edge. AddToBatch( time, this.leftEdgeMap.Values[index].CurrentStart, ref this.leftEdgeMap.Values[index].Key, ref this.leftEdgeMap.Values[index], hash); // Mark left event as not visible. this.leftEdgeMap.Values[index].CurrentStart = NotActive; } } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void MakeMatchingLeftVisible(long time, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, int hash)\r\n {\r\n // Make matching left intervals visible.\r\n " + " int index;\r\n var leftEvents = this.leftIntervalMap.Find(hash);\r\n " + "while (leftEvents.Next(out index))\r\n {\r\n if ("); this.Write(this.ToStringHelper.ToStringWithCulture(keyComparer("key", "this.leftIntervalMap.Values[index].Key"))); this.Write(@") { long end = this.leftIntervalMap.Values[index].End; bool isFullyOutputtable = this.nextRightTime >= end; if (isFullyOutputtable) { // Output interval. AddToBatch( time, end, ref this.leftIntervalMap.Values[index].Key, ref this.leftIntervalMap.Values[index], hash); // Mark left event as not visible so that an end-edge is not outputted when the interval actually endss. this.leftIntervalMap.Values[index].CurrentStart = NotActive; } else { // Output start edge. AddToBatch( time, StreamEvent.InfinitySyncTime, ref this.leftIntervalMap.Values[index].Key, ref this.leftIntervalMap.Values[index], hash); // Mark left event as visible. this.leftIntervalMap.Values[index].CurrentStart = time; } } } // Make matching left edges visible. leftEvents = this.leftEdgeMap.Find(hash); while (leftEvents.Next(out index)) { if ("); this.Write(this.ToStringHelper.ToStringWithCulture(keyComparer("key", "this.leftEdgeMap.Values[index].Key"))); this.Write(@") { // Output start edge. AddToBatch( time, StreamEvent.InfinitySyncTime, ref this.leftEdgeMap.Values[index].Key, ref this.leftEdgeMap.Values[index], hash); // Mark left event as visible. this.leftEdgeMap.Values[index].CurrentStart = time; } } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void AddPunctuationToBatch(long start) { if (start > lastCTI) { lastCTI = start; int index = this.output.Count++; this.output.vsync.col[index] = start; this.output.vother.col[index] = StreamEvent.PunctuationOtherTime; this.output.key.col[index] = default; "); foreach (var f in this.leftFields) { if (f.OptimizeString()) { this.Write(" this.output."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".AddString(string.Empty);\r\n"); } else { this.Write(" this.output."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col[index] = default;\r\n"); } } this.Write(@" this.output.hash.col[index] = 0; this.output.bitvector.col[index >> 6] |= (1L << (index & 0x3f)); if (this.output.Count == Config.DataBatchSize) FlushContents(); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void AddToBatch(long start, long end, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, "); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName)); this.Write(" batch, int batchIndex, int hash)\r\n {\r\n int index = this.output.Count++" + ";\r\n this.output.vsync.col[index] = start;\r\n this.output.vother.col" + "[index] = end;\r\n this.output.key.col[index] = key;\r\n"); foreach (var f in this.leftFields) { if (f.OptimizeString()) { this.Write(" this.output."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".AddString(batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write("[batchIndex]);\r\n"); } else { this.Write(" this.output."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col[index] = batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col[batchIndex];\r\n"); } } this.Write(@" this.output.hash.col[index] = hash; if (this.output.Count == Config.DataBatchSize) { this.output.Seal(); this.Observer.OnNext(this.output); GetOutputBatch(); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void AddToBatch(long start, long end, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, ref LeftEvent leftEvent, int hash)\r\n {\r\n int index = this.output." + "Count++;\r\n this.output.vsync.col[index] = start;\r\n this.output.vot" + "her.col[index] = end;\r\n this.output.key.col[index] = key;\r\n"); foreach (var f in this.leftFields) { string source; if (noLeftFields) { source = "leftEvent.Payload"; } else { source = "leftEvent.Payload." + f.OriginalName; } if (f.OptimizeString()) { this.Write(" this.output."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".AddString("); this.Write(this.ToStringHelper.ToStringWithCulture(source)); this.Write(");\r\n"); } else { this.Write(" this.output."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col[index] = "); this.Write(this.ToStringHelper.ToStringWithCulture(source)); this.Write(";\r\n"); } } this.Write(" this.output.hash.col[index] = hash;\r\n\r\n if (this.output.Count == C" + "onfig.DataBatchSize) FlushContents();\r\n }\r\n\r\n [MethodImpl(MethodImplOption" + "s.AggressiveInlining)]\r\n private bool AreSame(long start, long end, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, "); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName)); this.Write(" batch, int index, ref LeftEvent active)\r\n {\r\n return start == active.S" + "tart &&\r\n end == active.End &&\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(keyComparer("key", "active.Key"))); this.Write(" &&\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(leftComparer("batch", "active.Payload"))); this.Write(@"; } protected override void FlushContents() { if (this.output.Count == 0) return; this.output.Seal(); this.Observer.OnNext(this.output); GetOutputBatch(); } private void GetOutputBatch() { this.pool.Get(out this.genericOutputBatch); this.genericOutputBatch.Allocate(); this.output = ("); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName)); this.Write(")this.genericOutputBatch;\r\n"); foreach (var f in this.leftFields.Where(fld => fld.OptimizeString())) { this.Write(" this.output."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".Initialize();\r\n"); } this.Write(" }\r\n\r\n"); if (!noLeftFields && !this.leftType.GetTypeInfo().IsValueType) { this.Write(" [DataContract]\r\n private struct "); this.Write(this.ToStringHelper.ToStringWithCulture(ActiveEventType)); this.Write("\r\n {\r\n "); foreach (var f in this.leftFields) { this.Write(" [DataMember]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax())); this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(";\r\n "); } this.Write(" }\r\n"); } this.Write("\r\n [DataContract]\r\n private struct LeftEvent\r\n {\r\n [DataMember]\r\n" + " public long Start;\r\n [DataMember]\r\n public long CurrentSta" + "rt;\r\n [DataMember]\r\n public long End;\r\n [DataMember]\r\n " + " public "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" Key;\r\n [DataMember]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(ActiveEventType)); this.Write(" Payload;\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " + "public void Populate(long start, long currentStart, long end, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, "); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName)); this.Write(" batch, int index)\r\n {\r\n Start = start;\r\n CurrentSta" + "rt = currentStart;\r\n End = end;\r\n Key = key;\r\n"); if (noLeftFields) { this.Write(" this.Payload = batch.payload.col[index];\r\n"); } else { foreach (var f in leftFields) { this.Write(" this.Payload."); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(" = "); this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index"))); this.Write(";\r\n"); } } this.Write(" }\r\n\r\n public override string ToString()\r\n {\r\n Sy" + "stem.Text.StringBuilder sb = new System.Text.StringBuilder();\r\n sb.Ap" + "pendFormat(\"[Start={0}\", Start);\r\n sb.AppendFormat(\", Key=\'{0}\'\", Key" + ");\r\n"); if (noLeftFields) { this.Write(" sb.AppendFormat(\", payload={0}, \", this.Payload);\r\n"); } else { foreach (var f in leftFields) { this.Write(" sb.AppendFormat(\", "); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write("={0}, \", this.Payload."); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(");\r\n"); } } this.Write(" sb.Append(\"]\");\r\n return sb.ToString();\r\n }\r\n }\r" + "\n\r\n [DataContract]\r\n private struct RightEvent\r\n {\r\n [DataMember" + "]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" Key;\r\n [DataMember]\r\n public int Count;\r\n\r\n [MethodImpl(Met" + "hodImplOptions.AggressiveInlining)]\r\n public void Initialize(ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(@" key) { Key = key; Count = 1; } public override string ToString() { return ""[Key='"" + Key + ""', Count="" + Count + ""]""; } } [DataContract] private struct QueuedEndEdge { [DataMember] public "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" Key;\r\n [DataMember]\r\n public int Hash;\r\n\r\n [MethodImpl(Meth" + "odImplOptions.AggressiveInlining)]\r\n public void Populate(ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, int hash)\r\n {\r\n Key = key;\r\n Hash = hash;\r\n " + " }\r\n\r\n public override string ToString()\r\n {\r\n retu" + "rn \"[Key=\'\" + Key + \"\', Hash=\" + Hash + \"]\";\r\n }\r\n }\r\n}\r\n"); return(this.GenerationEnvironment.ToString()); }
/// <summary> /// Create the template output /// </summary> public override string TransformText() { this.Write("\r\n"); this.Write(@"// ********************************************************************* // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License // ********************************************************************* using System; using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; using System.Runtime.CompilerServices; using System.Runtime.Serialization; using Microsoft.StreamProcessing; using Microsoft.StreamProcessing.Internal; using Microsoft.StreamProcessing.Internal.Collections; using Microsoft.StreamProcessing.Aggregates; "); List <string> genericParamList = new List <string>(); int oldCount = 0; var TKey = keyType.GetCSharpSourceSyntax(ref genericParamList); var keyGenericParameters = new List <string>(genericParamList.Skip(oldCount)); oldCount = genericParamList.Count; var TInput = inputType.GetCSharpSourceSyntax(ref genericParamList); var inputGenericParameters = new List <string>(genericParamList.Skip(oldCount)); oldCount = genericParamList.Count; var TState = stateType.GetCSharpSourceSyntax(ref genericParamList); var stateGenericParameters = new List <string>(genericParamList.Skip(oldCount)); oldCount = genericParamList.Count; var TOutput = outputType.GetCSharpSourceSyntax(ref genericParamList); var outputGenericParameters = new List <string>(genericParamList.Skip(oldCount)); var genericParameters = genericParamList.BracketedCommaSeparatedString(); var TKeyTInputGenericParameters = keyGenericParameters.Concat(inputGenericParameters).BracketedCommaSeparatedString(); var TKeyTOutputGenericParameters = keyGenericParameters.Concat(outputGenericParameters).BracketedCommaSeparatedString(); var BatchGeneratedFrom_TKey_TInput = Transformer.GetBatchClassName(keyType, inputType); var genericParameters2 = string.Format("<{0}, {1}>", TKey, TOutput); if (!keyType.KeyTypeNeedsGeneratedMemoryPool() && outputType.MemoryPoolHasGetMethodFor()) { genericParameters2 = string.Empty; } else if (!outputType.CanRepresentAsColumnar()) { genericParameters2 = string.Empty; } Func <string, string> assignToOutput = rhs => this.outputType.IsAnonymousType() ? rhs : ( this.outputFields.Count() == 1 ? string.Format("this.batch.{0}.col[c] = {1};", this.outputFields.First().Name, rhs) : "temporaryOutput = " + rhs + ";\r\n" + String.Join("\r\n", this.outputFields.Select(f => "dest_" + f.Name + "[c] = temporaryOutput." + f.OriginalName + ";"))) ; var getOutputBatch = string.Format("this.pool.Get(out genericOutputbatch); this.batch = ({0}{1})genericOutputbatch;", Transformer.GetBatchClassName(keyType, outputType), TKeyTOutputGenericParameters); this.Write("[assembly: IgnoresAccessChecksTo(\"Microsoft.StreamProcessing\")]\r\n\r\n// genericPara" + "ms2 = \""); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2)); this.Write("\"\r\n\r\n[DataContract]\r\nstruct StateAndActive\r\n{\r\n [DataMember]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(" state;\r\n [DataMember]\r\n public ulong active;\r\n}\r\n\r\n[DataContract]\r\nstruct " + "HeldStateStruct\r\n{\r\n [DataMember]\r\n public long timestamp;\r\n [DataMembe" + "r]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(" state;\r\n}\r\n\r\n[DataContract]\r\nsealed class HeldState\r\n{\r\n [DataMember]\r\n pu" + "blic long timestamp;\r\n [DataMember]\r\n public StateAndActive state;\r\n}\r\n\r\n/" + "/ TKey: "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write("\r\n// TInput: "); this.Write(this.ToStringHelper.ToStringWithCulture(TInput)); this.Write("\r\n// TState: "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write("\r\n// TOutput: "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write("\r\n/// <summary>\r\n/// Operator that uses a full-fledged priority queue as ECQ\r\n///" + " </summary>\r\n[DataContract]\r\ninternal sealed class "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters)); this.Write(" : UnaryPipe<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TInput)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write(">\r\n{\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGenera" + "tor;\r\n private readonly IAggregate<"); this.Write(this.ToStringHelper.ToStringWithCulture(TInput)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write("> aggregate;\r\n\r\n [DataContract]\r\n class EcqState\r\n {\r\n [DataMembe" + "r]\r\n public long timestamp;\r\n [DataMember]\r\n public FastDic" + "tionary<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", StateAndActive> states;\r\n }\r\n\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType))); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2)); this.Write(" pool;\r\n\r\n private StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write("> genericOutputbatch;\r\n [DataMember]\r\n private "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, outputType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTOutputGenericParameters)); this.Write(" batch;\r\n\r\n"); if (this.useCompiledInitialState) { this.Write(" private readonly Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write("> initialState;\r\n"); } if (this.useCompiledAccumulate) { this.Write(" private readonly Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(", long, "); this.Write(this.ToStringHelper.ToStringWithCulture(TInput)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write("> accumulate;\r\n"); } if (this.useCompiledDeaccumulate) { this.Write(" private readonly Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(", long, "); this.Write(this.ToStringHelper.ToStringWithCulture(TInput)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write("> deaccumulate;\r\n"); } if (this.useCompiledDifference) { this.Write(" private readonly Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write("> difference;\r\n"); } if (this.useCompiledComputeResult) { this.Write(" private readonly Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write("> computeResult;\r\n"); } this.Write("\r\n private readonly IEqualityComparerExpression<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write("> keyComparer;\r\n private readonly Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", bool> keyComparerEquals;\r\n private readonly Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", int> keyComparerGetHashCode;\r\n\r\n [DataMember]\r\n private FastDictionary2<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", HeldState> aggregateByKey;\r\n [DataMember]\r\n private FastDictionary<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", HeldState> heldAggregates;\r\n [DataMember]\r\n private SortedDictionary<long" + ", EcqState> ecq;\r\n [DataMember]\r\n private long lastSyncTime = long.MinValu" + "e;\r\n\r\n private HeldState currentState;\r\n private "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" currentKey;\r\n private int currentHash;\r\n\r\n private Func<FastDictionary<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", StateAndActive>> stateDictGenerator;\r\n\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor)); this.Write("\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("() { }\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("(\r\n Streamable<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write("> stream,\r\n IStreamObserver<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator,\r\n" + " IAggregate<"); this.Write(this.ToStringHelper.ToStringWithCulture(TInput)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write("> aggregate)\r\n : base(stream, observer)\r\n {\r\n this.aggregate = a" + "ggregate;\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n"); if (this.useCompiledInitialState) { this.Write(" initialState = aggregate.InitialState().Compile();\r\n"); } if (this.useCompiledAccumulate) { this.Write(" accumulate = aggregate.Accumulate().Compile();\r\n"); } if (this.useCompiledDeaccumulate) { this.Write(" deaccumulate = aggregate.Deaccumulate().Compile();\r\n"); } if (this.useCompiledDifference) { this.Write(" difference = aggregate.Difference().Compile();\r\n"); } if (this.useCompiledComputeResult) { this.Write(" computeResult = aggregate.ComputeResult().Compile();\r\n"); } this.Write(@" this.keyComparer = stream.Properties.KeyEqualityComparer; this.keyComparerEquals = this.keyComparer.GetEqualsExpr().Compile(); this.keyComparerGetHashCode = this.keyComparer.GetGetHashCodeExpr().Compile(); this.stateDictGenerator = this.keyComparer.CreateFastDictionaryGenerator<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", StateAndActive>(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream." + "Properties.QueryContainer);\r\n\r\n this.pool = MemoryManager.GetMemoryPool<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write(">() as "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType))); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2)); this.Write(";\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch)); this.Write("\r\n this.batch.Allocate();\r\n\r\n this.aggregateByKey = this.keyCompare" + "r.CreateFastDictionary2Generator<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", HeldState>(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Prope" + "rties.QueryContainer).Invoke();\r\n this.heldAggregates = this.keyComparer." + "CreateFastDictionaryGenerator<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(@", HeldState>(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Properties.QueryContainer).Invoke(); this.ecq = new SortedDictionary<long, EcqState>(); } public override void ProduceQueryPlan(PlanNode previous) => Observer.ProduceQueryPlan(queryPlanGenerator(previous, this)); protected override void FlushContents() { if (this.batch == null || this.batch.Count == 0) return; this.batch.Seal(); this.Observer.OnNext(this.batch); "); this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch)); this.Write(@" this.batch.Allocate(); } protected override void DisposeState() => this.batch.Free(); public override int CurrentlyBufferedOutputCount => this.batch.Count; public override int CurrentlyBufferedInputCount => this.aggregateByKey.Count + this.ecq.Values.Select(o => o.states.Count).Sum(); public override unsafe void OnNext(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TInput)); this.Write("> inputBatch)\r\n {\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput)); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters)); this.Write(" generatedBatch = inputBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput)); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters)); this.Write(";\r\n\r\n this.batch.iter = generatedBatch.iter;\r\n\r\n var count = genera" + "tedBatch.Count;\r\n\r\n"); if (this.outputFields.Count() > 1) { this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write(" temporaryOutput;\r\n"); } this.Write("\r\n // Create locals that point directly to the arrays within the columns i" + "n the input batch.\r\n"); foreach (var f in this.inputFields) { if (f.canBeFixed) { this.Write(" fixed ("); this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName)); this.Write("* "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write("_col = generatedBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col)\r\n {\r\n"); } else { this.Write(" var "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write("_col = generatedBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col;\r\n"); } } this.Write("\r\n // Create locals that point directly to the arrays within the columns i" + "n the output batch.\r\n"); foreach (var f in this.outputFields) { if (f.canBeFixed) { this.Write(" fixed ("); this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName)); this.Write("* dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = this.batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col)\r\n {\r\n"); } else { this.Write(" var dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = this.batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col;\r\n"); } } this.Write("\r\n fixed (long* col_vsync = generatedBatch.vsync.col)\r\n fixed (long" + "* col_vother = generatedBatch.vother.col)\r\n fixed (int* col_hash = genera" + "tedBatch.hash.col)\r\n fixed (long* col_bv = generatedBatch.bitvector.col)\r" + "\n for (int i = 0; i < count; i++)\r\n {\r\n if ((col_bv[i >" + "> 6] & (1L << (i & 0x3f))) != 0)\r\n {\r\n if (col_vother[" + "i] == long.MinValue)\r\n {\r\n // We have found a " + "row that corresponds to punctuation\r\n OnPunctuation(col_vsync" + "[i]);\r\n\r\n int c = this.batch.Count;\r\n this" + ".batch.vsync.col[c] = col_vsync[i];\r\n this.batch.vother.col[c" + "] = long.MinValue;\r\n this.batch.key.col[c] = default;\r\n " + " this.batch[c] = default;\r\n this.batch.hash.col[" + "c] = 0;\r\n this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f" + ");\r\n this.batch.Count++;\r\n if (this.batch." + "Count == Config.DataBatchSize) FlushContents();\r\n }\r\n " + " continue;\r\n }\r\n\r\n var syncTime = col_vsync[i];\r\n " + " var key_i = generatedBatch.key.col[i];\r\n\r\n HeldState heldState;" + "\r\n bool cachedState = false;\r\n\r\n // Handle time moving for" + "ward\r\n if (syncTime > this.lastSyncTime)\r\n {\r\n " + " /* Issue start edges for held aggregates */\r\n if (currentStat" + "e != null && this.heldAggregates.Count == 1)\r\n {\r\n " + " // there is just one held aggregate, and currentState is set\r\n " + " // so currentState has to be the held aggregate\r\n cach" + "edState = true;\r\n\r\n if (currentState.state.active > 0)\r\n " + " {\r\n int c = this.batch.Count;\r\n " + " this.batch.vsync.col[c] = currentState.timestamp;\r\n " + " this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;\r\n " + " "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state")))); this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n " + " this.batch.hash.col[c] = "); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey"))); this.Write(@"; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) { this.batch.iter = generatedBatch.iter; FlushContents(); this.batch.iter = generatedBatch.iter; } } else { this.aggregateByKey.Remove(currentKey, currentHash); currentState = null; } } else { int iter1 = FastDictionary<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(@", HeldState>.IteratorStart; while (this.heldAggregates.Iterate(ref iter1)) { var iter1entry = this.heldAggregates.entries[iter1]; if (iter1entry.value.state.active > 0) { int c = this.batch.Count; this.batch.vsync.col[c] = iter1entry.value.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value.state.state")))); this.Write("\r\n this.batch.key.col[c] = iter1entry.key;\r\n " + " this.batch.hash.col[c] = "); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("iter1entry.key"))); this.Write(@"; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) FlushContents(); } else { this.aggregateByKey.Remove(iter1entry.key); // , (currentKey, currentHash); currentState = null; } } // Time has moved forward, clear the held aggregates this.heldAggregates.Clear(); currentState = null; } /* Process the ECQ up until the new sync time */ long ve; EcqState ecqState; while (this.ecq.Count > 0 && this.ecq.TryGetFirst(out ve, out ecqState) && ve <= syncTime) { this.ecq.Remove(ve); int iter = FastDictionary<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(">.IteratorStart;\r\n\r\n while (ecqState.states.Iterate(ref iter))" + "\r\n {\r\n if (currentState == null || !"); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "ecqState.states.entries[iter].key"))); this.Write(@") { int index; this.aggregateByKey.Lookup(ecqState.states.entries[iter].key, out index); heldState = this.aggregateByKey.entries[index].value; } else heldState = currentState; if (heldState.state.active > 0) { // Issue end edge int c = this.batch.Count; this.batch.vsync.col[c] = ecqState.timestamp; this.batch.vother.col[c] = heldState.timestamp; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state")))); this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries[ite" + "r].key;\r\n this.batch.hash.col[c] = "); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key"))); this.Write(@"; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) FlushContents(); } // Update aggregate heldState.state.state = "); this.Write(this.ToStringHelper.ToStringWithCulture(difference("heldState.state.state", "ecqState.states.entries[iter].value.state"))); this.Write(@"; heldState.state.active -= ecqState.states.entries[iter].value.active; if (ecqState.timestamp < syncTime) { if (heldState.state.active > 0) { // Issue start edge int c = this.batch.Count; this.batch.vsync.col[c] = ecqState.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state")))); this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries" + "[iter].key;\r\n this.batch.hash.col[c] = "); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key"))); this.Write(";\r\n this.batch.Count++;\r\n " + " if (this.batch.Count == Config.DataBatchSize) FlushContents();\r\n " + " }\r\n else\r\n " + " {\r\n // remove from aggregateByKey\r\n " + " this.aggregateByKey.Remove(ecqState.states.entries[iter].k" + "ey);\r\n currentState = null;\r\n " + " }\r\n }\r\n else\r\n " + " {\r\n if (cachedState)\r\n " + " {\r\n cachedState = false;\r\n " + " if (heldState != currentState)\r\n " + " {\r\n this.heldAggregates.Clear();\r\n " + " currentState = null;\r\n " + " int index;\r\n this.heldAggregates.Lookup(" + "ecqState.states.entries[iter].key, out index);\r\n " + " this.heldAggregates.Insert(ref index, ecqState.states.entries[iter].key, hel" + "dState);\r\n }\r\n }\r\n " + " else\r\n {\r\n " + " int index;\r\n this.heldAggregates.Loo" + "kup(ecqState.states.entries[iter].key, out index);\r\n " + " this.heldAggregates.Insert(ref index, ecqState.states.entries[iter].key, hel" + "dState);\r\n }\r\n }\r\n\r\n " + " // Update timestamp\r\n heldState.timestamp =" + " ecqState.timestamp;\r\n }\r\n }\r\n\r\n " + " // Since sync time changed, set this.lastSyncTime\r\n this.lastSy" + "ncTime = syncTime;\r\n }\r\n\r\n if (currentState == null || cur" + "rentHash != col_hash[i] || !"); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "key_i"))); this.Write(@") { if (cachedState) { cachedState = false; this.heldAggregates.Clear(); } // Need to retrieve the key from the dictionary currentKey = key_i; currentHash = col_hash[i]; int index; if (!this.heldAggregates.Lookup(currentKey, currentHash, out index)) { // First time group is active for this time int aggindex; if (!this.aggregateByKey.Lookup(currentKey, currentHash, out aggindex)) { // New group. Create new state currentState = new HeldState(); currentState.state.state = "); this.Write(this.ToStringHelper.ToStringWithCulture(initialState)); this.Write(@"; currentState.timestamp = syncTime; this.aggregateByKey.Insert(currentKey, currentState, currentHash); // No output because initial state is empty } else { currentState = this.aggregateByKey.entries[aggindex].value; if (syncTime > currentState.timestamp) { if (currentState.state.active > 0) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = currentState.timestamp; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state")))); this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n " + " this.batch.hash.col[c] = "); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey"))); this.Write(@"; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) FlushContents(); } currentState.timestamp = syncTime; } } this.heldAggregates.Insert(ref index, currentKey, currentState); } else { // read new currentState from _heldAgg index currentState = this.heldAggregates.entries[index].value; } } else { if (syncTime > currentState.timestamp) { if (currentState.state.active > 0) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = currentState.timestamp; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state")))); this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n " + " this.batch.hash.col[c] = "); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey"))); this.Write(@"; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) FlushContents(); } currentState.timestamp = syncTime; } } if (col_vsync[i] < col_vother[i]) // insert event { currentState.state.state = "); this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("currentState.state.state", "col_vsync[i]" /*, "col_payload[i]"*/))); this.Write(@"; currentState.state.active++; // Update ECQ if (col_vother[i] < StreamEvent.InfinitySyncTime) { EcqState eState; int index; if (this.ecq.Count > 0) { if (!this.ecq.TryGetValue(col_vother[i], out eState)) { eState = new EcqState(); eState.timestamp = col_vother[i]; eState.states = this.stateDictGenerator.Invoke(); eState.states.Lookup(key_i, col_hash[i], out index); eState.states.Insert(ref index, key_i, new StateAndActive { state = "); this.Write(this.ToStringHelper.ToStringWithCulture(initialState)); this.Write(@" }); this.ecq.Add(col_vother[i], eState); } else { if (!eState.states.Lookup(key_i, col_hash[i], out index)) { eState.states.Insert(ref index, key_i, new StateAndActive { state = "); this.Write(this.ToStringHelper.ToStringWithCulture(initialState)); this.Write(@" }); } } } else { eState = new EcqState(); eState.timestamp = col_vother[i]; eState.states = this.stateDictGenerator.Invoke(); eState.states.Lookup(key_i, col_hash[i], out index); eState.states.Insert(ref index, key_i, new StateAndActive { state = "); this.Write(this.ToStringHelper.ToStringWithCulture(initialState)); this.Write(" });\r\n this.ecq.Add(col_vother[i], eState);\r\n " + " }\r\n\r\n eState.states.entries[index].value.state = "); this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("eState.states.entries[index].value.state", "col_vsync[i]"))); this.Write(";\r\n eState.states.entries[index].value.active++;\r\n " + " }\r\n }\r\n else // is a retraction\r\n {\r\n " + " currentState.state.state = "); this.Write(this.ToStringHelper.ToStringWithCulture(deaccumulate("currentState.state.state", "col_vsync[i]" /*, "col_payload[i]"*/))); this.Write(";\r\n currentState.state.active--;\r\n }\r\n }\r\n\r\n"); foreach (var f in this.inputFields.Where(fld => fld.canBeFixed)) { this.Write(" }\r\n"); } foreach (var f in this.outputFields.Where(fld => fld.canBeFixed)) { this.Write(" }\r\n"); } this.Write("\r\n generatedBatch.Release();\r\n generatedBatch.Return();\r\n }\r\n\r\n\r" + "\n public void OnPunctuation(long syncTime)\r\n {\r\n HeldState heldStat" + "e;\r\n\r\n"); if (this.outputFields.Count() > 1) { this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write(" temporaryOutput;\r\n"); foreach (var f in this.outputFields) { this.Write(" var dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = this.batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col;\r\n"); } } this.Write(@" // Handle time moving forward if (syncTime > this.lastSyncTime) { /* Issue start edges for held aggregates */ if (currentState != null && this.heldAggregates.Count == 1) { // there is just one held aggregate, and currentState is set // so currentState has to be the held aggregate if (currentState.state.active > 0) { int c = this.batch.Count; this.batch.vsync.col[c] = currentState.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state")))); this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n th" + "is.batch.hash.col[c] = "); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey"))); this.Write(@"; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) FlushContents(); } else { this.aggregateByKey.Remove(currentKey, currentHash); currentState = null; } } else { int iter1 = FastDictionary<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(@", HeldState>.IteratorStart; while (this.heldAggregates.Iterate(ref iter1)) { var iter1entry = this.heldAggregates.entries[iter1]; if (iter1entry.value.state.active > 0) { int c = this.batch.Count; this.batch.vsync.col[c] = iter1entry.value.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value.state.state")))); this.Write("\r\n this.batch.key.col[c] = iter1entry.key;\r\n " + " this.batch.hash.col[c] = "); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("iter1entry.key"))); this.Write(@"; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) FlushContents(); } else { this.aggregateByKey.Remove(iter1entry.key); currentState = null; } } } // Time has moved forward, clear the held aggregates this.heldAggregates.Clear(); currentState = null; /* Process the ECQ up until the new sync time */ long ve; EcqState ecqState; while (this.ecq.Count > 0 && this.ecq.TryGetFirst(out ve, out ecqState) && ve <= syncTime) { this.ecq.Remove(ve); int iter = FastDictionary<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(">.IteratorStart;\r\n\r\n while (ecqState.states.Iterate(ref iter))\r\n " + " {\r\n if (currentState == null || !"); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "ecqState.states.entries[iter].key"))); this.Write(@") { int index; this.aggregateByKey.Lookup(ecqState.states.entries[iter].key, out index); heldState = this.aggregateByKey.entries[index].value; } else heldState = currentState; if (heldState.state.active > 0) { // Issue end edge int c = this.batch.Count; this.batch.vsync.col[c] = ecqState.timestamp; this.batch.vother.col[c] = heldState.timestamp; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state")))); this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries[iter].k" + "ey;\r\n this.batch.hash.col[c] = "); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key"))); this.Write(";\r\n this.batch.Count++;\r\n if (this." + "batch.Count == Config.DataBatchSize) FlushContents();\r\n }\r\n\r\n" + " // Update aggregate\r\n heldState.state.sta" + "te = "); this.Write(this.ToStringHelper.ToStringWithCulture(difference("heldState.state.state", "ecqState.states.entries[iter].value.state"))); this.Write(@"; heldState.state.active -= ecqState.states.entries[iter].value.active; if (ecqState.timestamp < syncTime) { if (heldState.state.active > 0) { // Issue start edge int c = this.batch.Count; this.batch.vsync.col[c] = ecqState.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state")))); this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries[ite" + "r].key;\r\n this.batch.hash.col[c] = "); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key"))); this.Write(@"; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) FlushContents(); } else { // remove from aggregateByKey this.aggregateByKey.Remove(ecqState.states.entries[iter].key); currentState = null; } } else { int index; this.heldAggregates.Lookup(ecqState.states.entries[iter].key, out index); this.heldAggregates.Insert(ref index, ecqState.states.entries[iter].key, heldState); } // Update timestamp heldState.timestamp = ecqState.timestamp; } } // Since sync time changed, set this.lastSyncTime this.lastSyncTime = syncTime; } } protected override void UpdatePointers() { int iter1 = FastDictionary<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", HeldState>.IteratorStart;\r\n while (this.heldAggregates.Iterate(ref iter1" + "))\r\n {\r\n int iter2;\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(@" key = this.heldAggregates.entries[iter1].key; bool found = this.aggregateByKey.Lookup(key, out iter2); if (!found) throw new InvalidOperationException(); this.heldAggregates.entries[iter1].value = this.aggregateByKey.entries[iter2].value; } } } "); return(this.GenerationEnvironment.ToString()); }
/// <summary> /// Create the template output /// </summary> public override string TransformText() { this.Write(@"// ********************************************************************* // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License // ********************************************************************* using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.Contracts; using System.Linq.Expressions; using System.Runtime.CompilerServices; using System.Runtime.Serialization; using System.Threading; using Microsoft.StreamProcessing; using Microsoft.StreamProcessing.Aggregates; using Microsoft.StreamProcessing.Internal; using Microsoft.StreamProcessing.Internal.Collections; // TKey: "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write("\r\n// TLeft: "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("\r\n// TRight: "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write("\r\n// TResult: "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters)); this.Write(" : BinaryPipe<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write(">\r\n{\r\n private const int DefaultCapacity = 64;\r\n private readonly MemoryPoo" + "l<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write(@"> pool; private readonly Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator; [DataMember] private FastMap<ActiveEventLeft> leftEdgeMap = new FastMap<ActiveEventLeft>(DefaultCapacity); [DataMember] private FastMap<ActiveEventRight> rightEdgeMap = new FastMap<ActiveEventRight>(DefaultCapacity); private StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("> genericOutputBatch;\r\n [DataMember]\r\n private "); this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TResult)); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters)); this.Write(@" output; [DataMember] private long nextLeftTime = long.MinValue; [DataMember] private bool isLeftComplete; [DataMember] private long nextRightTime = long.MinValue; [DataMember] private bool isRightComplete; "); this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor)); this.Write("\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("() { }\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("(\r\n IStreamable<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("> stream,\r\n IStreamObserver<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("> observer,\r\n Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> qu" + "eryPlanGenerator)\r\n : base(stream, observer)\r\n {\r\n pool = Memor" + "yManager.GetMemoryPool<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write(">(true /*stream.Properties.IsColumnar*/);\r\n "); GetOutputBatch(); this.Write(" this.queryPlanGenerator = queryPlanGenerator;\r\n }\r\n\r\n protected ove" + "rride void FlushContents()\r\n {\r\n if (output.Count == 0) return;\r\n " + " output.Seal();\r\n this.Observer.OnNext(output);\r\n "); GetOutputBatch(); this.Write(@" } public override int CurrentlyBufferedOutputCount => output.Count; public override int CurrentlyBufferedLeftInputCount => base.CurrentlyBufferedLeftInputCount + leftEdgeMap.Count; public override int CurrentlyBufferedRightInputCount => base.CurrentlyBufferedRightInputCount + rightEdgeMap.Count; protected override void ProduceBinaryQueryPlan(PlanNode left, PlanNode right) { Observer.ProduceQueryPlan(queryPlanGenerator(left, right, this)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] protected override void DisposeState() => this.output.Free(); [MethodImpl(MethodImplOptions.AggressiveInlining)] protected override void ProcessBothBatches(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> leftBatch, StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write("> rightBatch, out bool leftBatchDone, out bool rightBatchDone, out bool leftBatch" + "Free, out bool rightBatchFree)\r\n {\r\n var generatedLeftBatch = leftBatc" + "h as "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, leftType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters)); this.Write(";\r\n var generatedRightBatch = rightBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, rightType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTRightGenericParameters)); this.Write(@"; leftBatchFree = rightBatchFree = true; if (!GoToVisibleRow(leftBatch)) { leftBatchDone = true; rightBatchDone = false; return; } UpdateNextLeftTime(leftBatch.vsync.col[leftBatch.iter]); if (!GoToVisibleRow(rightBatch)) { leftBatchDone = false; rightBatchDone = true; return; } UpdateNextRightTime(rightBatch.vsync.col[rightBatch.iter]); FastMap<ActiveEventRight>.FindTraverser rightEdges = default; while (true) { bool leftPunctuation = leftBatch.vother.col[leftBatch.iter] == StreamEvent.PunctuationOtherTime; bool rightPunctuation = rightBatch.vother.col[rightBatch.iter] == StreamEvent.PunctuationOtherTime; if (nextLeftTime <= nextRightTime) { if (leftPunctuation) { AddPunctuationToBatch(nextLeftTime); } else { // ProcessLeftStartEdge if (true) { bool first = true; "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(@" key = default; var hash = leftBatch.hash.col[leftBatch.iter]; if (rightEdgeMap.Find(hash, ref rightEdges)) { int rightIndex; while (rightEdges.Next(out rightIndex)) { if (first) { key = leftBatch.key.col[leftBatch.iter]; first = false; } //if (keyComparer(key, rightEdgeMap.Values[rightIndex].Key)) var activeEventRight = rightEdgeMap.Values[rightIndex]; var rightKey = activeEventRight.Key; if ("); this.Write(this.ToStringHelper.ToStringWithCulture(this.keyComparer("key", "rightKey"))); this.Write(@") { //payload = leftBatch[leftBatch.iter]; //OutputStartEdge(nextLeftTime, ref key, ref payload, ref rightEdgeMap.Values[rightIndex].Payload, hash); "); OutputStartEdgeWithActiveEventRight("nextLeftTime", "key", "generatedLeftBatch", "leftBatch.iter", "activeEventRight", "hash"); this.Write(@" } } } if (!isRightComplete) { int newIndex = leftEdgeMap.Insert(hash); if (first) key = leftBatch.key.col[leftBatch.iter]; leftEdgeMap.Values[newIndex].Populate(generatedLeftBatch, leftBatch.iter); } } } leftBatch.iter++; if (!GoToVisibleRow(leftBatch)) { leftBatchDone = true; rightBatchDone = false; return; } UpdateNextLeftTime(leftBatch.vsync.col[leftBatch.iter]); } else { if (rightPunctuation) { AddPunctuationToBatch(nextRightTime); } else { // ProcessRightStartEdge "); ProcessRightStartEdge2( "nextRightTime", "generatedRightBatch", "rightBatch.iter", "rightBatch.hash.col[rightBatch.iter]"); this.Write(@" } rightBatch.iter++; if (!GoToVisibleRow(rightBatch)) { leftBatchDone = false; rightBatchDone = true; return; } UpdateNextRightTime(rightBatch.vsync.col[rightBatch.iter]); } } } [MethodImpl(MethodImplOptions.AggressiveInlining)] protected override void ProcessLeftBatch(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> batch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var generate" + "dLeftBatch = batch as "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, leftType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters)); this.Write(@"; isBatchFree = true; while (true) { if (!GoToVisibleRow(batch)) { isBatchDone = true; return; } UpdateNextLeftTime(batch.vsync.col[batch.iter]); if (nextLeftTime > nextRightTime) { isBatchDone = false; return; } if (batch.vother.col[batch.iter] == StreamEvent.PunctuationOtherTime) { AddPunctuationToBatch(batch.vsync.col[batch.iter]); batch.iter++; continue; } // ProcessLeftStartEdge FastMap<ActiveEventRight>.FindTraverser rightEdges = default; if (true) { bool first = true; "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(@" key = default; var hash = batch.hash.col[batch.iter]; if (rightEdgeMap.Find(hash, ref rightEdges)) { int rightIndex; while (rightEdges.Next(out rightIndex)) { if (first) { key = batch.key.col[batch.iter]; first = false; } //if (keyComparer(key, rightEdgeMap.Values[rightIndex].Key)) var activeEventRight = rightEdgeMap.Values[rightIndex]; var rightKey = activeEventRight.Key; if ("); this.Write(this.ToStringHelper.ToStringWithCulture(this.keyComparer("key", "rightKey"))); this.Write(")\r\n {\r\n //payload = batch[batch" + ".iter];\r\n //OutputStartEdge(nextLeftTime, ref key, re" + "f payload, ref rightEdgeMap.Values[rightIndex].Payload, hash);\r\n " + " "); OutputStartEdgeWithActiveEventRight("nextLeftTime", "key", "generatedLeftBatch", "batch.iter", "activeEventRight", "hash"); this.Write(@" } } } if (!isRightComplete) { int newIndex = leftEdgeMap.Insert(hash); if (first) key = batch.key.col[batch.iter]; //payload = batch[batch.iter]; // potential rare recomputation //leftEdgeMap.Values[newIndex].Populate(ref key, ref payload); leftEdgeMap.Values[newIndex].Populate(generatedLeftBatch, batch.iter); } } batch.iter++; } } [MethodImpl(MethodImplOptions.AggressiveInlining)] protected override void ProcessRightBatch(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write("> batch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var generate" + "dBatch = batch as "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, rightType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTRightGenericParameters)); this.Write(@"; isBatchFree = true; while (true) { if (!GoToVisibleRow(batch)) { isBatchDone = true; return; } UpdateNextRightTime(batch.vsync.col[batch.iter]); if (nextRightTime > nextLeftTime) { isBatchDone = false; return; } if (batch.vother.col[batch.iter] == StreamEvent.PunctuationOtherTime) { AddPunctuationToBatch(batch.vsync.col[batch.iter]); batch.iter++; continue; } // ProcessRightStartEdge "); ProcessRightStartEdge2( "nextRightTime", "generatedBatch", "batch.iter", "batch.hash.col[batch.iter]"); this.Write("\r\n batch.iter++;\r\n }\r\n }\r\n\r\n\r\n [MethodImpl(MethodImplOpti" + "ons.AggressiveInlining)]\r\n private bool GoToVisibleRow<TPayload>(StreamMessag" + "e<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", TPayload> batch)\r\n {\r\n while (batch.iter < batch.Count && (batch.bitv" + "ector.col[batch.iter >> 6] & (1L << (batch.iter & 0x3f))) != 0 && batch.vother.c" + "ol[batch.iter] >= 0)\r\n {\r\n batch.iter++;\r\n }\r\n\r\n " + " return (batch.iter != batch.Count);\r\n }\r\n\r\n [MethodImpl(MethodImplOptions" + ".AggressiveInlining)]\r\n private void UpdateNextLeftTime(long time)\r\n {\r\n " + " nextLeftTime = time;\r\n if (nextLeftTime == StreamEvent.InfinitySync" + "Time)\r\n {\r\n isLeftComplete = true;\r\n }\r\n }\r\n\r\n [M" + "ethodImpl(MethodImplOptions.AggressiveInlining)]\r\n private void UpdateNextRig" + "htTime(long time)\r\n {\r\n nextRightTime = time;\r\n if (nextRightTi" + "me == StreamEvent.InfinitySyncTime)\r\n {\r\n isRightComplete = tr" + "ue;\r\n }\r\n }\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r" + "\n private void AddPunctuationToBatch(long start)\r\n {\r\n if (start > " + "lastCTI)\r\n {\r\n lastCTI = start;\r\n\r\n int index = out" + "put.Count++;\r\n output.vsync.col[index] = start;\r\n output.v" + "other.col[index] = StreamEvent.PunctuationOtherTime;\r\n output.key.col" + "[index] = default;\r\n output[index] = default;\r\n output.has" + "h.col[index] = 0;\r\n output.bitvector.col[index >> 6] |= (1L << (index" + " & 0x3f));\r\n\r\n if (output.Count == Config.DataBatchSize) FlushContent" + "s();\r\n }\r\n }\r\n\r\n [DataContract]\r\n private struct ActiveEventLeft" + "\r\n {\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" Key;\r\n "); foreach (var f in this.leftFields) { this.Write("\r\n [DataMember]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax())); this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(";\r\n "); } this.Write("\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n public vo" + "id Populate("); this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TLeft)); this.Write(" batch, int index)\r\n {\r\n Key = batch.key.col[index];\r\n " + " "); foreach (var f in this.leftFields) { this.Write("\r\n this."); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(" = "); this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index"))); this.Write(";\r\n "); } this.Write("\r\n }\r\n\r\n public override string ToString()\r\n {\r\n " + "return \"[Payload=\'\" + \"\']\";\r\n }\r\n }\r\n\r\n [DataContract]\r\n private" + " struct ActiveEventRight\r\n {\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" Key;\r\n "); foreach (var f in this.rightFields) { this.Write("\r\n [DataMember]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax())); this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(";\r\n "); } this.Write("\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n public void" + " Populate("); this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TRight)); this.Write(" batch, int index)\r\n {\r\n Key = batch.key.col[index];\r\n " + " "); foreach (var f in this.rightFields) { this.Write("\r\n this."); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(" = "); this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index"))); this.Write(";\r\n "); } this.Write("\r\n }\r\n\r\n public override string ToString()\r\n {\r\n " + "return \"[Payload=\'\" + \"\']\";\r\n }\r\n }\r\n}\r\n\r\n"); return(this.GenerationEnvironment.ToString()); }
/// <summary> /// Create the template output /// </summary> public override string TransformText() { this.Write(@"// ********************************************************************* // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License // ********************************************************************* using System; using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; using System.Reflection; using System.Runtime.CompilerServices; using System.Runtime.Serialization; using Microsoft.StreamProcessing; using Microsoft.StreamProcessing.Internal; using Microsoft.StreamProcessing.Internal.Collections; "); if (this.keyType.Namespace != null) { this.Write("using "); this.Write(this.ToStringHelper.ToStringWithCulture(this.keyType.Namespace)); this.Write(";\r\n"); } if (this.payloadType.Namespace != null && this.payloadType.Namespace != this.keyType.Namespace) { this.Write("using "); this.Write(this.ToStringHelper.ToStringWithCulture(this.payloadType.Namespace)); this.Write(";\r\n"); } if (this.resultType.Namespace != null && this.resultType.Namespace != this.keyType.Namespace && this.resultType.Namespace != this.payloadType.Namespace) { this.Write("using "); this.Write(this.ToStringHelper.ToStringWithCulture(this.resultType.Namespace)); this.Write(";\r\n"); } this.Write("[assembly: IgnoresAccessChecksTo(\"Microsoft.StreamProcessing\")]\r\n\r\n// Source Fiel" + "ds: "); this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.fields.Select(f => f.OriginalName)))); this.Write("\r\n// Destination Fields: "); this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.destinationFields.Select(f => f.OriginalName)))); this.Write("\r\n// Computed Fields: "); this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.computedFields.Keys.Select(f => f.OriginalName)))); this.Write("\r\n// Swinging Fields: "); this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.swingingFields.Select(tup => string.Format("<{0},{1}>", tup.Item1.OriginalName, tup.Item2.Name))))); this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters)); this.Write(" : UnaryPipe<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TPayload)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write(">\r\n{\r\n private "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.resultType))); this.Write(this.ToStringHelper.ToStringWithCulture(MemoryPoolGenericParameters)); this.Write(" pool;\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGene" + "rator;\r\n"); foreach (var f in this.unassignedFields) { if (!f.OptimizeString()) { this.Write("\r\n private readonly ColumnBatch<"); this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName)); this.Write("> sharedDefaultColumnFor_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(";\r\n"); } } this.Write("\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor)); this.Write("\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("() { }\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("(\r\n IStreamable<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("> stream,\r\n IStreamObserver<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator)\r\n" + " : base(stream, observer)\r\n {\r\n pool = MemoryManager.GetMemoryP" + "ool<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write(">() as "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.resultType))); this.Write(this.ToStringHelper.ToStringWithCulture(MemoryPoolGenericParameters)); this.Write(";\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n"); foreach (var f in this.unassignedFields.Where(fld => !fld.OptimizeString())) { this.Write("\r\n pool.Get(out sharedDefaultColumnFor_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(");\r\n Array.Clear(sharedDefaultColumnFor_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col, 0, sharedDefaultColumnFor_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col.Length);\r\n"); } this.Write("\r\n }\r\n\r\n public override void ProduceQueryPlan(PlanNode previous)\r\n {\r\n " + " Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));\r\n }\r\n\r\n" + " protected override void DisposeState()\r\n {\r\n"); foreach (var f in this.unassignedFields.Where(fld => !fld.OptimizeString())) { this.Write(" this.sharedDefaultColumnFor_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".Return();\r\n"); } this.Write(" }\r\n\r\n public override unsafe void OnNext(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TPayload)); this.Write("> _inBatch)\r\n {\r\n var sourceBatch = _inBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TPayload)); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTPayloadGenericParameters)); this.Write(";\r\n\r\n StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("> _genBatch; // Need this type to call Get with so the right subtype will be retu" + "rned\r\n pool.Get(out _genBatch);\r\n\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters)); this.Write(" resultBatch = _genBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters)); this.Write(@"; var count = sourceBatch.Count; resultBatch.vsync = sourceBatch.vsync; resultBatch.vother = sourceBatch.vother; resultBatch.key = sourceBatch.key; resultBatch.hash = sourceBatch.hash; resultBatch.bitvector = sourceBatch.bitvector; "); if (resultType.CanContainNull()) { this.Write(" pool.GetBV(out resultBatch._nullnessvector);\r\n"); } this.Write(@" // Get memory pools for the result columns. // When no transformation was done to the query, then needed for all fields in the result type. // When the query was transformed, then this is needed only for computed fields, since // any field that is just assigned a field from the source type will get its value by // swinging a pointer for the corresponding column. "); foreach (var f in (this.ProjectionReturningResultInstance != null ? this.destinationFields : this.computedFields.Keys)) { this.Write(" pool.Get(out resultBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(");\r\n"); } this.Write("\r\n"); foreach (var f in this.unassignedFields) { if (!f.OptimizeString()) { this.Write(" this.sharedDefaultColumnFor_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".IncrementRefCount(1);\r\n"); } this.Write(" resultBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = this.sharedDefaultColumnFor_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(";\r\n"); } this.Write("\r\n // Create locals that point directly to the arrays within the columns i" + "n the destination batch.\r\n\r\n"); foreach (var f in this.computedFields.Keys) { if (f.OptimizeString()) { this.Write(" var dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = resultBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(";\r\n"); } else { this.Write(" var dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = resultBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col;\r\n"); } } this.Write("\r\n // Create locals that point directly to the arrays within the columns i" + "n the source batch.\r\n"); if (this.ProjectionReturningResultInstance != null || 0 < this.computedFields.Count()) { foreach (var f in this.fields) { if (f.canBeFixed) { this.Write(" fixed ("); this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName)); this.Write("* "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write("_col = sourceBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col)\r\n {\r\n"); } else { this.Write(" var "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write("_col = sourceBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col;\r\n"); } } this.Write("\r\n fixed (long* bv = sourceBatch.bitvector.col)\r\n {\r\n fo" + "r (int i = 0; i < count; i++)\r\n {\r\n if ((bv[i >> 6] & " + "(1L << (i & 0x3f))) == 0)\r\n {\r\n"); if (this.StartEdgeParameterName != null) { this.Write(" var "); this.Write(this.ToStringHelper.ToStringWithCulture(this.StartEdgeParameterName)); this.Write(" = sourceBatch.vsync.col[i] < sourceBatch.vother.col[i] ? sourceBatch.vsync.col[i" + "] : sourceBatch.vother.col[i];\r\n"); } if (this.needSourceInstance) { this.Write(" var "); this.Write(this.ToStringHelper.ToStringWithCulture(PARAMETER)); this.Write(" = sourceBatch[i];\r\n"); } if (this.ProjectionReturningResultInstance != null) { this.Write(" resultBatch[i] = "); this.Write(this.ToStringHelper.ToStringWithCulture(this.ProjectionReturningResultInstance)); this.Write(";\r\n"); } else { foreach (var kv in this.computedFields) { var f = kv.Key; var v = kv.Value.ExpressionToCSharp(); if (f.OptimizeString()) { this.Write(" dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".AddString("); this.Write(this.ToStringHelper.ToStringWithCulture(v)); this.Write(");\r\n"); } else { this.Write(" dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write("[i] = "); this.Write(this.ToStringHelper.ToStringWithCulture(v)); this.Write(";\r\n"); } } } this.Write(" }\r\n }\r\n }\r\n\r\n"); foreach (var f in this.fields.Where(fld => fld.canBeFixed)) { this.Write(" }\r\n"); } } this.Write("\r\n #region MultiString Vector Operations (if any)\r\n"); foreach (var mso in this.multiStringOperations) { this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(mso)); this.Write("\r\n"); } this.Write(@" #endregion #region Swinging Fields // When the query is transformed, then any fields in the result batch that // are just assigned from a field in the source batch are computed by just // swinging the pointer for the corresponding column. "); foreach (var tuple in this.swingingFields) { var destField = tuple.Item1.Name; var sourceField = tuple.Item2.Name; this.Write("\r\n // Special case for a field in the output being assigned the key field\r" + "\n // when both are strings and the output field is a MultiString. The key" + "\r\n // field is a CB<string> so we need to call MultiString.FromColumnBatc" + "h\r\n"); if (tuple.Item1.OptimizeString() && sourceField.Equals("key")) { this.Write(" resultBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(destField)); this.Write(" = MultiString.FromColumnBatch(sourceBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(sourceField)); this.Write(", sourceBatch.bitvector, pool.charArrayPool, pool.intPool, pool.shortPool, pool.b" + "itvectorPool);\r\n"); } else { this.Write(" resultBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(destField)); this.Write(" = sourceBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(sourceField)); this.Write(";\r\n"); } } this.Write(@" // We avoid incrementing the ref counts on the ouput batch because for the // most part we are throwing away our reference to the columns from the source batch // so the reference count doesn't change. But the same field might get swung to more // than one column in the output batch. In that case, we need to increment the ref count. "); foreach (var kv in this.swungFieldsCount) { var sourceField = kv.Key; var count = kv.Value; if (sourceField.Name.Equals("key") && !sourceField.OptimizeString()) { // special case: we "pointer swing" the key field once just because the key // of the output batch is the key of the source batch // (Note that this really applies only to SelectKey since Select is not able // to reference the key field in the selector.) count++; } if (count == 1) { continue; } this.Write(" sourceBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(sourceField.Name)); this.Write(".IncrementRefCount("); this.Write(this.ToStringHelper.ToStringWithCulture(count - 1)); this.Write(");\r\n"); } this.Write(@" #endregion resultBatch.Count = count; resultBatch.Seal(); // Return source columns as necessary. // When the query is transformed, this is the ""non-swinging"" fields. // Otherwise it is all source fields "); foreach (var sourceField in this.nonSwingingFields) { if (sourceField.OptimizeString()) { this.Write(" sourceBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(sourceField.Name)); this.Write(".Dispose();\r\n"); } else { this.Write(" sourceBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(sourceField.Name)); this.Write(".Return();\r\n"); } } if (payloadType.CanContainNull()) { this.Write(" sourceBatch._nullnessvector.ReturnClear();\r\n"); } this.Write("\r\n sourceBatch.Return();\r\n this.Observer.OnNext(resultBatch);\r\n " + "}\r\n\r\n public override int CurrentlyBufferedOutputCount => 0;\r\n\r\n public ov" + "erride int CurrentlyBufferedInputCount => 0;\r\n}\r\n"); return(this.GenerationEnvironment.ToString()); }
/// <summary> /// Create the template output /// </summary> public override string TransformText() { this.Write("// *********************************************************************\r\n// Copy" + "right (c) Microsoft Corporation. All rights reserved.\r\n// Licensed under the MI" + "T License\r\n// ******************************************************************" + "***\r\n"); var leftBatchTypeName = string.Format("{0}{1}", Transformer.GetBatchClassName(keyType, leftType), !String.IsNullOrWhiteSpace(TKeyTLeftGenericParameters) ? "<" + TKeyTLeftGenericParameters + ">" : string.Empty); var rightBatchTypeName = string.Format("{0}{1}", Transformer.GetBatchClassName(keyType, rightType), !String.IsNullOrWhiteSpace(TKeyTRightGenericParameters) ? "<" + TKeyTRightGenericParameters + ">" : string.Empty); this.Write(@" using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics.Contracts; using System.Linq.Expressions; using System.Runtime.CompilerServices; using System.Runtime.Serialization; using System.Threading; using Microsoft.StreamProcessing; using Microsoft.StreamProcessing.Aggregates; using Microsoft.StreamProcessing.Internal; using Microsoft.StreamProcessing.Internal.Collections; // TKey: "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write("\r\n// TLeft: "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("\r\n// TRight: "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write("\r\n// TResult: "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters)); this.Write(" : BinaryPipe<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write(">\r\n{\r\n private const int DefaultCapacity = 64;\r\n private readonly MemoryPoo" + "l<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("> pool;\r\n private readonly Func<PlanNode, PlanNode, IBinaryObserver, BinaryPla" + "nNode> queryPlanGenerator;\r\n\r\n /* Comparer for ordering of join key - should " + "eventually be a stream property */\r\n\r\n private StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("> genericOutputBatch;\r\n [DataMember]\r\n private "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters)); this.Write(" output;\r\n\r\n [DataMember]\r\n private long nextLeftTime = long.MinValue;\r\n " + " [DataMember]\r\n private long nextRightTime = long.MinValue;\r\n [DataMember]" + "\r\n private "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" nextLeftKey;\r\n [DataMember]\r\n private "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" nextRightKey;\r\n [DataMember]\r\n private "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" currentRightKey;\r\n [DataMember]\r\n private List<ActiveEventRight> currentRi" + "ghtList;\r\n [DataMember]\r\n private "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" currentLeftKey;\r\n [DataMember]\r\n private List<ActiveEventLeft> currentLeft" + "List;\r\n\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor)); this.Write("\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("() { }\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("(\r\n IStreamable<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write("> stream,\r\n IStreamObserver<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write(@"> observer, Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator) : base(stream, observer) { currentLeftList = new List<ActiveEventLeft>(); currentRightList = new List<ActiveEventRight>(); pool = MemoryManager.GetMemoryPool<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TResult)); this.Write(">(true /*stream.Properties.IsColumnar*/);\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch)); this.Write(@" output.Allocate(); this.queryPlanGenerator = queryPlanGenerator; } protected override void FlushContents() { if (output.Count == 0) return; output.Seal(); this.Observer.OnNext(output); "); this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch)); this.Write(@" output.Allocate(); } public override int CurrentlyBufferedOutputCount => output.Count; public override int CurrentlyBufferedLeftInputCount => base.CurrentlyBufferedLeftInputCount + currentLeftList.Count; public override int CurrentlyBufferedRightInputCount => base.CurrentlyBufferedRightInputCount + currentRightList.Count; protected override void ProduceBinaryQueryPlan(PlanNode left, PlanNode right) { Observer.ProduceQueryPlan(queryPlanGenerator(left, right, this)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] protected override void DisposeState() => output.Free(); [MethodImpl(MethodImplOptions.AggressiveInlining)] protected override void ProcessBothBatches(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write("> leftBatch, StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write("> rightBatch, out bool leftBatchDone, out bool rightBatchDone, out bool leftBatch" + "Free, out bool rightBatchFree)\r\n {\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName)); this.Write(" generatedLeftBatch = null;\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName)); this.Write(" generatedRightBatch = null;\r\n\r\n leftBatchFree = rightBatchFree = true;\r\n " + " generatedLeftBatch = ("); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName)); this.Write(@") leftBatch; if (!GoToVisibleRow(leftBatch)) { leftBatchDone = true; rightBatchDone = false; return; } UpdateNextLeftTime(leftBatch.vsync.col[leftBatch.iter]); nextLeftKey = leftBatch.key.col[leftBatch.iter]; generatedRightBatch = ("); this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName)); this.Write(@") rightBatch; if (!GoToVisibleRow(rightBatch)) { leftBatchDone = false; rightBatchDone = true; return; } UpdateNextRightTime(rightBatch.vsync.col[rightBatch.iter]); nextRightKey = rightBatch.key.col[rightBatch.iter]; while (true) { bool leftPunctuation = leftBatch.vother.col[leftBatch.iter] == StreamEvent.PunctuationOtherTime; bool rightPunctuation = rightBatch.vother.col[rightBatch.iter] == StreamEvent.PunctuationOtherTime; int compare = (leftPunctuation || rightPunctuation) ? 0 : "); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "nextRightKey"))); this.Write(@"; if (compare == 0) { if (nextLeftTime <= nextRightTime) { // process left if (leftPunctuation) { AddPunctuationToBatch(nextLeftTime); } else { #region ProcessLeftStartEdge /* ProcessLeftStartEdge( nextLeftTime, ref leftBatch.key.col[leftBatch.iter], leftBatch[leftBatch.iter], leftBatch.hash.col[leftBatch.iter], compare); */ { if (currentRightList.Count > 0) { int compare2 = "); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "currentRightKey"))); this.Write(@"; Contract.Assert(compare2 >= 0, ""Unexpected comparison in sort-ordered join""); if (compare2 == 0) { // perform the join for (int i = 0; i < currentRightList.Count; i++) { var t = currentRightList[i]; OutputStartEdge(nextLeftTime > t.Timestamp ? nextLeftTime : t.Timestamp, ref nextLeftKey, generatedLeftBatch, leftBatch.iter, t, leftBatch.hash.col[leftBatch.iter]); } } else { // clear the right array currentRightList.Clear(); } } if (compare >= 0) { // update the left array if ((currentLeftList.Count != 0) && ("); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "currentLeftKey"))); this.Write(" != 0))\r\n {\r\n C" + "ontract.Assert("); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "currentLeftKey"))); this.Write(" > 0);\r\n currentLeftList.Clear();\r\n " + " }\r\n currentLeftKey = nextLef" + "tKey;\r\n var leftAE = new ActiveEventLeft();\r\n " + " leftAE.Populate(ref nextLeftTime, generatedLeftBatch" + ", leftBatch.iter);\r\n currentLeftList.Add(leftAE);" + "\r\n }\r\n }\r\n " + " #endregion\r\n }\r\n\r\n leftBatch.iter++;\r\n" + "\r\n if (!GoToVisibleRow(leftBatch))\r\n {\r\n " + " leftBatchDone = true;\r\n rightBatchD" + "one = false;\r\n return;\r\n }\r\n\r\n " + " nextLeftTime = leftBatch.vsync.col[leftBatch.iter];\r\n " + " nextLeftKey = leftBatch.key.col[leftBatch.iter];\r\n }\r\n " + " else\r\n {\r\n // process right\r\n " + " if (rightPunctuation)\r\n {\r\n " + " AddPunctuationToBatch(nextRightTime);\r\n }\r\n " + " else\r\n {\r\n #region ProcessRightSt" + "artEdge\r\n /* Inlined version of:\r\n " + " ProcessRightStartEdge(\r\n nextRightTime,\r\n " + " ref rightBatch.key.col[rightBatch.iter],\r\n " + " rightBatch[rightBatch.iter],\r\n rightBatch." + "hash.col[rightBatch.iter], compare);\r\n */\r\n\r\n " + " if (currentLeftList.Count > 0)\r\n {\r\n " + " int compare2 = "); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextRightKey", "currentLeftKey"))); this.Write(@"; Contract.Assert(compare2 >= 0, ""Unexpected comparison in sort-ordered join""); if (compare2 == 0) { // perform the join for (int i = 0; i < currentLeftList.Count; i++) { var t = currentLeftList[i]; #region OutputStartEdge /* OutputStartEdge(nextRightTime > t.Timestamp ? nextRightTime : t.Timestamp, ref nextRightKey, ref t.Payload, ref rightP, rightBatch.hash.col[rightBatch.iter]); */ int index = output.Count++; output.vsync.col[index] = nextRightTime > t.Timestamp ? nextRightTime : t.Timestamp; output.vother.col[index] = StreamEvent.InfinitySyncTime; output.key.col[index] = nextRightKey; "); this.Write(this.ToStringHelper.ToStringWithCulture(this.leftMessageRepresentation.noFields ? this.rightBatchSelector("t.payload", "generatedRightBatch", "rightBatch.iter") : this.rightBatchSelector("t", "generatedRightBatch", "rightBatch.iter"))); this.Write(@" output.hash.col[index] = rightBatch.hash.col[rightBatch.iter]; if (output.Count == Config.DataBatchSize) { output.Seal(); this.Observer.OnNext(output); "); this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch)); this.Write(@" output.Allocate(); } #endregion } } else { // clear the left array currentLeftList.Clear(); } } if (compare <= 0) { // update the right array if ((currentRightList.Count != 0) && ("); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("rightBatch.key.col[rightBatch.iter]", "currentRightKey"))); this.Write(" != 0))\r\n {\r\n Contract." + "Assert("); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("rightBatch.key.col[rightBatch.iter]", "currentRightKey"))); this.Write(" > 0);\r\n currentRightList.Clear();\r\n " + " }\r\n currentRightKey = rightBatch.key.co" + "l[rightBatch.iter];\r\n var rightAE = new ActiveEventRi" + "ght();\r\n rightAE.Populate(ref nextRightTime, generate" + "dRightBatch, rightBatch.iter);\r\n currentRightList.Add" + "(rightAE);\r\n }\r\n #endregion\r\n " + " }\r\n\r\n rightBatch.iter++;\r\n\r\n " + " #region GoToVisibleRow\r\n /* Inlined version of:\r\n " + " if (!GoToVisibleRow(rightBatch))\r\n {\r\n " + " leftBatchDone = false;\r\n right" + "BatchDone = true;\r\n return;\r\n " + "}\r\n */\r\n while (rightBatch.iter < rightBat" + "ch.Count &&\r\n (rightBatch.bitvector.col[rightBatch.iter >" + "> 6] & (1L << (rightBatch.iter & 0x3f))) != 0 &&\r\n rightB" + "atch.vother.col[rightBatch.iter] >= 0)\r\n {\r\n " + " rightBatch.iter++;\r\n }\r\n if (rightB" + "atch.iter == rightBatch.Count)\r\n {\r\n l" + "eftBatchDone = false;\r\n rightBatchDone = true;\r\n " + " return;\r\n }\r\n #endregion\r\n\r" + "\n nextRightTime = rightBatch.vsync.col[rightBatch.iter];\r\n " + " nextRightKey = rightBatch.key.col[rightBatch.iter];\r\n " + " }\r\n }\r\n else if (compare < 0)\r\n {\r\n " + " // process left\r\n #region ProcessLeftStartEdge\r\n " + " /* Inlined version of:\r\n ProcessLeftStartEdge(\r\n " + " nextLeftTime,\r\n ref leftBatch.key.col" + "[leftBatch.iter],\r\n leftBatch[leftBatch.iter],\r\n " + " leftBatch.hash.col[leftBatch.iter], compare);\r\n */" + "\r\n {\r\n\r\n\r\n if (currentRightList.Count > 0)\r\n " + " {\r\n int compare2 = "); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "currentRightKey"))); this.Write(";\r\n\r\n Contract.Assert(compare2 >= 0, \"Unexpected compariso" + "n in sort-ordered join\");\r\n\r\n if (compare2 == 0)\r\n " + " {\r\n // perform the join\r\n " + " for (int i = 0; i < currentRightList.Count; i++)\r\n " + " {\r\n var t = currentRightList[i];\r\n" + " OutputStartEdge(nextLeftTime > t.Timestamp ? nex" + "tLeftTime : t.Timestamp, ref nextLeftKey, generatedLeftBatch, leftBatch.iter, t," + " leftBatch.hash.col[leftBatch.iter]);\r\n }\r\n " + " }\r\n else\r\n {\r\n " + " // clear the right array\r\n curr" + "entRightList.Clear();\r\n }\r\n }\r\n " + " }\r\n #endregion\r\n\r\n leftBatch.iter++;\r\n\r\n " + " #region GoToVisibleRow\r\n /* Inlined version of:\r\n " + " if (!GoToVisibleRow(leftBatch))\r\n {\r\n " + " leftBatchDone = true;\r\n rightBatchDone" + " = false;\r\n return;\r\n }\r\n " + " */\r\n while (leftBatch.iter < leftBatch.Count &&\r\n " + " (leftBatch.bitvector.col[leftBatch.iter >> 6] & (1L << (leftBatch.iter & " + "0x3f))) != 0 &&\r\n leftBatch.vother.col[leftBatch.iter] >= 0)\r" + "\n {\r\n leftBatch.iter++;\r\n }\r\n " + " if (leftBatch.iter == leftBatch.Count)\r\n {\r\n " + " leftBatchDone = true;\r\n rightBatchDone = false;\r" + "\n return;\r\n }\r\n #endregion\r\n\r\n\r" + "\n #region UpdateNextLeftTime\r\n nextLeftTime = left" + "Batch.vsync.col[leftBatch.iter];\r\n #endregion\r\n\r\n " + "nextLeftKey = leftBatch.key.col[leftBatch.iter];\r\n\r\n }\r\n e" + "lse // hot path if larger right side of join matches very few things on the left" + " side\r\n {\r\n // process right\r\n #region " + "ProcessRightStartEdge\r\n /* Inlined version of:\r\n P" + "rocessRightStartEdge(\r\n nextRightTime,\r\n r" + "ef rightBatch.key.col[rightBatch.iter],\r\n rightBatch.payload." + "col[rightBatch.iter],\r\n rightBatch.hash.col[rightBatch.iter]," + " compare);\r\n */\r\n if (currentLeftList.Count > 0)\r\n" + " {\r\n int compare2 = "); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextRightKey", "currentLeftKey"))); this.Write(@"; Contract.Assert(compare2 >= 0, ""Unexpected comparison in sort-ordered join""); if (compare2 == 0) { // perform the join for (int i = 0; i < currentLeftList.Count; i++) { var t = currentLeftList[i]; #region OutputStartEdge /* OutputStartEdge(nextRightTime > t.Timestamp ? nextRightTime : t.Timestamp, ref nextRightKey, ref t.Payload, ref rightP, rightBatch.hash.col[rightBatch.iter]); */ int index = output.Count++; output.vsync.col[index] = nextRightTime > t.Timestamp ? nextRightTime : t.Timestamp; output.vother.col[index] = StreamEvent.InfinitySyncTime; output.key.col[index] = nextRightKey; "); this.Write(this.ToStringHelper.ToStringWithCulture(this.leftMessageRepresentation.noFields ? this.rightBatchSelector("t.payload", "generatedRightBatch", "rightBatch.iter") : this.rightBatchSelector("t", "generatedRightBatch", "rightBatch.iter"))); this.Write(@" output.hash.col[index] = rightBatch.hash.col[rightBatch.iter]; if (output.Count == Config.DataBatchSize) { output.Seal(); this.Observer.OnNext(output); "); this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch)); this.Write(@" output.Allocate(); } #endregion } } else { // clear the left array currentLeftList.Clear(); } } /* if (compare <= 0) { // update the right array if ((currentRightList.Count != 0) && ("); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("rightBatch.key.col[rightBatch.iter]", "currentRightKey"))); this.Write(" != 0))\r\n {\r\n Contract.Assert("); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("rightBatch.key.col[rightBatch.iter]", "currentRightKey"))); this.Write(" > 0);\r\n currentRightList.Clear();\r\n " + " }\r\n currentRightKey = rightBatch.key.col[rightBatch" + ".iter];\r\n ActiveEvent<"); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write("> rightAE = new ActiveEvent<"); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write(">();\r\n rightAE.Populate(ref nextRightTime, ref rightBatch." + "payload.col[rightBatch.iter]);\r\n currentRightList.Add(rig" + "htAE);\r\n }\r\n */\r\n #endregion\r\n\r" + "\n rightBatch.iter++;\r\n\r\n #region GoToVisibleRow\r\n " + " /* Inlined version of:\r\n if (!GoToVisibleRow(r" + "ightBatch))\r\n {\r\n leftBatchDone = fals" + "e;\r\n rightBatchDone = true;\r\n retu" + "rn;\r\n }\r\n */\r\n while (rightBatc" + "h.iter < rightBatch.Count &&\r\n (rightBatch.bitvector.col[righ" + "tBatch.iter >> 6] & (1L << (rightBatch.iter & 0x3f))) != 0 &&\r\n " + " rightBatch.vother.col[rightBatch.iter] >= 0)\r\n {\r\n " + " rightBatch.iter++;\r\n }\r\n if (rightBatch.it" + "er == rightBatch.Count)\r\n {\r\n leftBatchDone = " + "false;\r\n rightBatchDone = true;\r\n return;\r" + "\n }\r\n #endregion\r\n\r\n #region Update" + "NextRightTime\r\n /* Inlined version of: UpdateNextRightTime(rightB" + "atch.vsync.col[rightBatch.iter]); */\r\n nextRightTime = rightBatch" + ".vsync.col[rightBatch.iter];\r\n #endregion\r\n\r\n next" + "RightKey = rightBatch.key.col[rightBatch.iter];\r\n }\r\n }\r\n }" + "\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n protected overri" + "de void ProcessLeftBatch(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TLeft)); this.Write(@"> batch, out bool isBatchDone, out bool isBatchFree) { isBatchFree = true; while (true) { if (!GoToVisibleRow(batch)) { isBatchDone = true; return; } UpdateNextLeftTime(batch.vsync.col[batch.iter]); if (batch.vother.col[batch.iter] == StreamEvent.PunctuationOtherTime) { if (nextLeftTime > nextRightTime) { isBatchDone = false; return; } AddPunctuationToBatch(batch.vsync.col[batch.iter]); batch.iter++; continue; } nextLeftKey = batch.key.col[batch.iter]; int compare = "); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "nextRightKey"))); this.Write(";\r\n if ((compare == 0) && (nextLeftTime <= nextRightTime))\r\n " + " {\r\n ProcessLeftStartEdge(\r\n nextLeftTime,\r\n " + " ref batch.key.col[batch.iter],\r\n ("); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName)); this.Write(@")batch, batch.iter, batch.hash.col[batch.iter], compare); batch.iter++; } else { isBatchDone = false; return; } } } [MethodImpl(MethodImplOptions.AggressiveInlining)] protected override void ProcessRightBatch(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TRight)); this.Write(@"> batch, out bool isBatchDone, out bool isBatchFree) { isBatchFree = true; while (true) { if (!GoToVisibleRow(batch)) { isBatchDone = true; return; } UpdateNextRightTime(batch.vsync.col[batch.iter]); if (batch.vother.col[batch.iter] == StreamEvent.PunctuationOtherTime) { if (nextRightTime > nextLeftTime) { isBatchDone = false; return; } AddPunctuationToBatch(batch.vsync.col[batch.iter]); batch.iter++; continue; } nextRightKey = batch.key.col[batch.iter]; int compare = "); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "nextRightKey"))); this.Write(";\r\n if ((compare == 0) && (nextRightTime <= nextLeftTime))\r\n " + " {\r\n ProcessRightStartEdge(\r\n nextRightTime,\r" + "\n ref batch.key.col[batch.iter],\r\n ("); this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName)); this.Write(@")batch,batch.iter, batch.hash.col[batch.iter], compare); batch.iter++; } else { isBatchDone = false; return; } } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool GoToVisibleRow<TPayload>(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(@", TPayload> batch) { while (batch.iter < batch.Count && (batch.bitvector.col[batch.iter >> 6] & (1L << (batch.iter & 0x3f))) != 0 && batch.vother.col[batch.iter] >= 0) { batch.iter++; } if (batch.iter == batch.Count) { return false; } return true; } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void UpdateNextLeftTime(long time) { nextLeftTime = time; } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void UpdateNextRightTime(long time) { nextRightTime = time; } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void ProcessLeftStartEdge(long start, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, "); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName)); this.Write(" leftBatch, int leftIndex, int hash, int nextLeftRightCompareResult)\r\n {\r\n " + " if (currentRightList.Count > 0)\r\n {\r\n int compare = "); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentRightKey"))); this.Write(@"; Contract.Assert(compare >= 0, ""Unexpected comparison in sort-ordered join""); if (compare == 0) { // perform the join for (int i = 0; i < currentRightList.Count; i++) { var t = currentRightList[i]; OutputStartEdge(start > t.Timestamp ? start : t.Timestamp, ref key, leftBatch, leftIndex, t, hash); } } else { // clear the right array currentRightList.Clear(); } } if (nextLeftRightCompareResult >= 0) { // update the left array if ((currentLeftList.Count != 0) && ("); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentLeftKey"))); this.Write(" != 0))\r\n {\r\n Contract.Assert("); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentLeftKey"))); this.Write(@" > 0); currentLeftList.Clear(); } currentLeftKey = key; var leftAE = new ActiveEventLeft(); leftAE.Populate(ref start, leftBatch, leftIndex); currentLeftList.Add(leftAE); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void ProcessRightStartEdge(long start, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, "); this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName)); this.Write(" rightBatch, int rightIndex, int hash, int nextLeftRightCompareResult)\r\n {\r\n " + " if (currentLeftList.Count > 0)\r\n {\r\n int compare = "); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentLeftKey"))); this.Write(@"; Contract.Assert(compare >= 0, ""Unexpected comparison in sort-ordered join""); if (compare == 0) { // perform the join for (int i = 0; i < currentLeftList.Count; i++) { var t = currentLeftList[i]; OutputStartEdge(start > t.Timestamp ? start : t.Timestamp, ref key, t, rightBatch, rightIndex, hash); } } else { // clear the left array currentLeftList.Clear(); } } if (nextLeftRightCompareResult <= 0) { // update the right array if ((currentRightList.Count != 0) && ("); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentRightKey"))); this.Write(" != 0))\r\n {\r\n Contract.Assert("); this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentRightKey"))); this.Write(@" > 0); currentRightList.Clear(); } currentRightKey = key; var rightAE = new ActiveEventRight(); rightAE.Populate(ref start, rightBatch, rightIndex); currentRightList.Add(rightAE); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void AddPunctuationToBatch(long start) { if (start > lastCTI) { lastCTI = start; int index = output.Count++; output.vsync.col[index] = start; output.vother.col[index] = StreamEvent.PunctuationOtherTime; output.key.col[index] = default; output[index] = default; output.hash.col[index] = 0; output.bitvector.col[index >> 6] |= (1L << (index & 0x3f)); if (output.Count == Config.DataBatchSize) FlushContents(); } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private void OutputStartEdge(long start, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, ActiveEventLeft leftEvent, "); this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName)); this.Write(" rightBatch, int rightIndex, int hash)\r\n {\r\n int index = output.Count++" + ";\r\n output.vsync.col[index] = start;\r\n output.vother.col[index] = " + "StreamEvent.InfinitySyncTime;\r\n output.key.col[index] = key;\r\n"); if (this.leftMessageRepresentation.noFields) { this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchSelector("leftEvent.payload", "rightBatch", "rightIndex"))); this.Write("\r\n"); } else { this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchSelector("leftEvent", "rightBatch", "rightIndex"))); this.Write("\r\n"); } this.Write(" output.hash.col[index] = hash;\r\n\r\n if (output.Count == Config.Data" + "BatchSize) FlushContents();\r\n }\r\n [MethodImpl(MethodImplOptions.Aggressive" + "Inlining)]\r\n private void OutputStartEdge(long start, ref "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(" key, "); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName)); this.Write(@" leftBatch, int leftIndex, ActiveEventRight rightEvent, int hash) { int index = output.Count++; output.vsync.col[index] = start; output.vother.col[index] = StreamEvent.InfinitySyncTime; output.key.col[index] = key; "); if (this.rightMessageRepresentation.noFields) { this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchSelector("leftBatch", "leftIndex", "rightEvent.payload"))); this.Write("\r\n"); } else { this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchSelector("leftBatch", "leftIndex", "rightEvent"))); this.Write("\r\n"); } this.Write(" output.hash.col[index] = hash;\r\n\r\n if (output.Count == Config.Data" + "BatchSize) FlushContents();\r\n }\r\n\r\n [DataContract]\r\n private struct Act" + "iveEventLeft\r\n {\r\n [DataMember]\r\n public long Timestamp;\r\n\r\n " + " "); foreach (var f in this.leftFields) { this.Write("\r\n [DataMember]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax())); this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(";\r\n "); } this.Write("\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n public void" + " Populate(ref long timestamp, "); this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName)); this.Write(" batch, int index)\r\n {\r\n Timestamp = timestamp;\r\n "); foreach (var f in this.leftFields) { this.Write("\r\n this."); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(" = "); this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index"))); this.Write(";\r\n "); } this.Write(@" } public override string ToString() { return ""[Timestamp='"" + Timestamp + ""', Payload='"" + ""']""; } } [DataContract] private struct ActiveEventRight { [DataMember] public long Timestamp; "); foreach (var f in this.rightFields) { this.Write("\r\n [DataMember]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax())); this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(";\r\n "); } this.Write("\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n public void" + " Populate(ref long timestamp, "); this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName)); this.Write(" batch, int index)\r\n {\r\n Timestamp = timestamp;\r\n "); foreach (var f in this.rightFields) { this.Write("\r\n this."); this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName)); this.Write(" = "); this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index"))); this.Write(";\r\n "); } this.Write("\r\n }\r\n\r\n public override string ToString()\r\n {\r\n " + "return \"[Timestamp=\'\" + Timestamp + \"\', Payload=\'\" + \"\']\";\r\n }\r\n }\r\n}"); return(this.GenerationEnvironment.ToString()); }
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new StartEdgeEquiJoinTemplate($"GeneratedStartEdgeEquiJoin_{StartEdgeEquiJoinSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult)); var gps = template.tm.GenericTypeVariables(template.keyType, template.leftType, template.rightType, template.resultType); template.genericParameters = gps.BracketedCommaSeparatedString(); template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType); template.leftFields = template.leftMessageRepresentation.AllFields; template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType); template.rightFields = template.rightMessageRepresentation.AllFields; var resultRepresentation = new ColumnarRepresentation(template.resultType); #region Key Comparer var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparer = (left, right) => keyComparer.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType); template.TKeyTLeftGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType); template.TKeyTRightGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.rightType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType); template.TKeyTResultGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString(); template.outputFields = resultRepresentation.AllFields; var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); #region LeftBatchSelector { var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i"); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var parameterMap = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex }, { selector.Parameters[1], rightEvent } }; if (projectionResult.ProjectionReturningResultInstance != null) { return($"this.output[index] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};"); } else { var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendLine($"this.output.{f.Name}.AddString({e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)});"); } else { sb.AppendLine($"this.output.{f.Name}.col[index] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};"); } } sb.AppendLine("}"); return(sb.ToString()); } }; } #endregion #region RightBatchSelector { var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j"); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var parameterMap = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex } }; if (projectionResult.ProjectionReturningResultInstance != null) { return($"this.output[index] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};"); } else { var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendLine($"this.output.{f.Name}.AddString({e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)});"); } else { sb.AppendLine($"this.output.{f.Name}.col[index] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};"); } } sb.AppendLine("}"); return(sb.ToString()); } }; } #endregion return(template.Generate <TKey, TLeft, TRight, TResult>(selector)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
/// <summary> /// Generate a batch class definition to be used as a Clip pipe. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// aggregate class. /// </summary> /// <typeparam name="TKey">The key type for both sides.</typeparam> /// <typeparam name="TLeft">The payload type for the left side.</typeparam> /// <typeparam name="TRight">The payload type for the right side.</typeparam> /// <returns> /// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TKey"/>, <typeparamref name="TRight"/>>. /// </returns> internal static Tuple <Type, string> Generate <TKey, TLeft, TRight>(ClipJoinStreamable <TKey, TLeft, TRight> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TLeft>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new ClipJoinTemplate(); var keyType = template.keyType = typeof(TKey); var leftType = template.leftType = typeof(TLeft); var rightType = template.rightType = typeof(TRight); var tm = new TypeMapper(keyType, leftType, rightType); template.TKey = tm.CSharpNameFor(keyType); template.TLeft = tm.CSharpNameFor(leftType); template.TRight = tm.CSharpNameFor(rightType); var gps = tm.GenericTypeVariables(keyType, leftType, rightType); template.genericParameters = gps.BracketedCommaSeparatedString(); template.className = string.Format("GeneratedClip_{0}", ClipSequenceNumber++); var resultRepresentation = new ColumnarRepresentation(leftType); template.ActiveEventType = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event"; #region Key Comparer var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparer = (left, right) => keyComparer.Inline(left, right); #endregion #region Left Comparer var leftComparer = stream.LeftComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftComparer, "index_ProcessLeftEvent", 0); template.leftComparer = (left, right) => newLambda.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType); template.TKeyTLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType); template.TKeyTRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString(); template.leftFields = resultRepresentation.AllFields; template.noFields = resultRepresentation.noFields; template.staticCtor = Transformer.StaticCtor(template.className); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight)); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>()); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(leftComparer)); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType); var t = a.GetType(realClassName); if (t.GetTypeInfo().IsGenericType) { var list = keyType.GetAnonymousTypes(); list.AddRange(leftType.GetAnonymousTypes()); list.AddRange(rightType.GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
/// <summary> /// Create the template output /// </summary> public override string TransformText() { this.Write("\r\n"); this.Write(@"// ********************************************************************* // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License // ********************************************************************* using System; using System.Collections.Generic; using System.Linq; using System.Linq.Expressions; using System.Runtime.CompilerServices; using System.Runtime.Serialization; using Microsoft.StreamProcessing; using Microsoft.StreamProcessing.Internal; using Microsoft.StreamProcessing.Internal.Collections; using Microsoft.StreamProcessing.Aggregates; "); List <string> genericParamList = new List <string>(); int oldCount = 0; var TKey = keyType.GetCSharpSourceSyntax(ref genericParamList); var keyGenericParameters = new List <string>(genericParamList.Skip(oldCount)); oldCount = genericParamList.Count; var TInput = inputType.GetCSharpSourceSyntax(ref genericParamList); var inputGenericParameters = new List <string>(genericParamList.Skip(oldCount)); oldCount = genericParamList.Count; var TState = stateType.GetCSharpSourceSyntax(ref genericParamList); var stateGenericParameters = new List <string>(genericParamList.Skip(oldCount)); oldCount = genericParamList.Count; var TOutput = outputType.GetCSharpSourceSyntax(ref genericParamList); var outputGenericParameters = new List <string>(genericParamList.Skip(oldCount)); var genericParameters = genericParamList.BracketedCommaSeparatedString(); var TKeyTInputGenericParameters = keyGenericParameters.Concat(inputGenericParameters).BracketedCommaSeparatedString(); var TKeyTOutputGenericParameters = keyGenericParameters.Concat(outputGenericParameters).BracketedCommaSeparatedString(); var BatchGeneratedFrom_TKey_TInput = Transformer.GetBatchClassName(keyType, inputType); var genericParameters2 = string.Format("<{0}, {1}>", TKey, TOutput); if (!keyType.KeyTypeNeedsGeneratedMemoryPool() && outputType.MemoryPoolHasGetMethodFor()) { genericParameters2 = string.Empty; } else if (!outputType.CanRepresentAsColumnar()) { genericParameters2 = string.Empty; } Func <string, string> assignToOutput = rhs => this.outputType.IsAnonymousType() ? rhs : ( this.outputFields.Count() == 1 ? string.Format("this.batch.{0}.col[c] = {1};", this.outputFields.First().Name, rhs) : "temporaryOutput = " + rhs + ";\r\n" + String.Join("\r\n", this.outputFields.Select(f => "dest_" + f.Name + "[c] = temporaryOutput." + f.OriginalName + ";"))) ; var getOutputBatch = string.Format("this.pool.Get(out genericOutputbatch); this.batch = ({0}{1})genericOutputbatch;", Transformer.GetBatchClassName(keyType, outputType), TKeyTOutputGenericParameters); this.Write("[assembly: IgnoresAccessChecksTo(\"Microsoft.StreamProcessing\")]\r\n\r\n// genericPara" + "ms2 = \""); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2)); this.Write("\"\r\n\r\n[DataContract]\r\nstruct StateAndActive\r\n{\r\n [DataMember]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(" state;\r\n [DataMember]\r\n public ulong active;\r\n}\r\n\r\n[DataContract]\r\nstruct " + "HeldStateStruct\r\n{\r\n [DataMember]\r\n public long timestamp;\r\n [DataMembe" + "r]\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(" state;\r\n}\r\n\r\n[DataContract]\r\nsealed class HeldState\r\n{\r\n [DataMember]\r\n pu" + "blic long timestamp;\r\n [DataMember]\r\n public StateAndActive state;\r\n}\r\n\r\n/" + "/ TKey: "); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write("\r\n// TInput: "); this.Write(this.ToStringHelper.ToStringWithCulture(TInput)); this.Write("\r\n// TState: "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write("\r\n// TOutput: "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write("\r\n/// <summary>\r\n/// Operator only has to deal with start edges\r\n/// </summary>\r\n" + "[DataContract]\r\ninternal sealed class "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters)); this.Write(" : UnaryPipe<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TInput)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write(">\r\n{\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGenera" + "tor;\r\n private readonly IAggregate<"); this.Write(this.ToStringHelper.ToStringWithCulture(TInput)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write("> aggregate;\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType))); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2)); this.Write(" pool;\r\n\r\n private StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write("> genericOutputbatch;\r\n [DataMember]\r\n private "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, outputType))); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTOutputGenericParameters)); this.Write(" batch;\r\n\r\n"); if (this.useCompiledInitialState) { this.Write(" private readonly Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write("> initialState;\r\n"); } if (this.useCompiledAccumulate) { this.Write(" private readonly Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(", long, "); this.Write(this.ToStringHelper.ToStringWithCulture(TInput)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write("> accumulate;\r\n"); } if (this.useCompiledComputeResult) { this.Write(" private readonly Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write("> computeResult;\r\n"); } this.Write("\r\n private readonly IEqualityComparerExpression<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write("> keyComparer;\r\n\r\n"); if (!this.isUngrouped) { this.Write(" [DataMember]\r\n private FastDictionary3<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", HeldStateStruct> heldAggregates;\r\n"); } else { this.Write(" [DataMember]\r\n private HeldStateStruct currentState;\r\n [DataMember]\r\n " + " private bool currentStateExists;\r\n [DataMember]\r\n private bool isDirty;\r" + "\n"); } this.Write("\r\n [DataMember]\r\n private long lastSyncTime = long.MinValue;\r\n\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor)); this.Write("\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("() { }\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("(\r\n Streamable<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write("> stream,\r\n IStreamObserver<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator,\r\n" + " IAggregate<"); this.Write(this.ToStringHelper.ToStringWithCulture(TInput)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TState)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write("> aggregate)\r\n : base(stream, observer)\r\n {\r\n this.aggregate = a" + "ggregate;\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n\r\n"); if (this.useCompiledInitialState) { this.Write(" initialState = aggregate.InitialState().Compile();\r\n"); } if (this.useCompiledAccumulate) { this.Write(" accumulate = aggregate.Accumulate().Compile();\r\n"); } if (this.useCompiledComputeResult) { this.Write(" computeResult = aggregate.ComputeResult().Compile();\r\n"); } this.Write("\r\n this.keyComparer = stream.Properties.KeyEqualityComparer;\r\n\r\n th" + "is.pool = MemoryManager.GetMemoryPool<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write(">() as "); this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType))); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2)); this.Write(";\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch)); this.Write("\r\n this.batch.Allocate();\r\n\r\n"); if (!this.isUngrouped) { this.Write(" var generator = this.keyComparer.CreateFastDictionary3Generator<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", HeldStateStruct>(1, this.keyComparer.GetEqualsExpr().Compile(), this.keyCompare" + "r.GetGetHashCodeExpr().Compile(), stream.Properties.QueryContainer);\r\n th" + "is.heldAggregates = generator.Invoke();\r\n"); } else { this.Write(" isDirty = false;\r\n"); } this.Write(@" } public override void ProduceQueryPlan(PlanNode previous) => Observer.ProduceQueryPlan(queryPlanGenerator(previous, this)); protected override void FlushContents() { if (this.batch == null || this.batch.Count == 0) return; this.batch.Seal(); this.Observer.OnNext(this.batch); "); this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch)); this.Write("\r\n this.batch.Allocate();\r\n }\r\n\r\n protected override void DisposeSta" + "te() => this.batch.Free();\r\n\r\n public override int CurrentlyBufferedOutputCou" + "nt => this.batch.Count;\r\n\r\n public override int CurrentlyBufferedInputCount =" + "> "); this.Write(this.ToStringHelper.ToStringWithCulture(this.isUngrouped ? "0" : "this.heldAggregates.Count")); this.Write(";\r\n\r\n public override unsafe void OnNext(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TInput)); this.Write("> inputBatch)\r\n {\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput)); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters)); this.Write(" generatedBatch = inputBatch as "); this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput)); this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters)); this.Write(";\r\n\r\n var count = generatedBatch.Count;\r\n\r\n"); if (this.outputFields.Count() > 1) { this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write(" temporaryOutput;\r\n"); } this.Write("\r\n // Create locals that point directly to the arrays within the columns i" + "n the input batch.\r\n"); foreach (var f in this.inputFields) { if (f.canBeFixed) { this.Write(" fixed ("); this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName)); this.Write("* "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write("_col = generatedBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col)\r\n {\r\n"); } else { this.Write(" var "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write("_col = generatedBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col;\r\n"); } } this.Write("\r\n // Create locals that point directly to the arrays within the columns i" + "n the output batch.\r\n"); foreach (var f in this.outputFields) { if (f.canBeFixed) { this.Write(" fixed ("); this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName)); this.Write("* dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = this.batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col)\r\n {\r\n"); } else { this.Write(" var dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = this.batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col;\r\n"); } } this.Write(@" var col_key = generatedBatch.key.col; fixed (long* col_vsync = generatedBatch.vsync.col) fixed (long* col_vother = generatedBatch.vother.col) fixed (int* col_hash = generatedBatch.hash.col) fixed (long* col_bv = generatedBatch.bitvector.col) for (int i = 0; i < count; i++) { if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0) { if (col_vother[i] == long.MinValue) { // We have found a row that corresponds to punctuation OnPunctuation(col_vsync[i]); int c = this.batch.Count; this.batch.vsync.col[c] = col_vsync[i]; this.batch.vother.col[c] = long.MinValue; this.batch.key.col[c] = default; this.batch[c] = default; this.batch.hash.col[c] = 0; this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f); this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) FlushContents(); } continue; } var syncTime = col_vsync[i]; // Handle time moving forward if (syncTime > this.lastSyncTime) { "); if (this.isUngrouped) { this.Write(@" if (currentStateExists && isDirty) // there exists earlier state { int c = this.batch.Count; this.batch.vsync.col[c] = currentState.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state")))); this.Write("\r\n this.batch.hash.col[c] = 0;\r\n this.batch" + ".Count++;\r\n if (this.batch.Count == Config.DataBatchSize) Flu" + "shContents();\r\n }\r\n isDirty = false;\r\n"); } else { this.Write(" int iter1 = FastDictionary3<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(@", HeldStateStruct>.IteratorStart; while (this.heldAggregates.IterateDirty(ref iter1)) { var iter1entry = this.heldAggregates.entries[iter1]; int c = this.batch.Count; this.batch.vsync.col[c] = iter1entry.value.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value.state")))); this.Write(@" this.batch.key.col[c] = iter1entry.key; this.batch.hash.col[c] = iter1entry.hash; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) FlushContents(); } // Time has moved forward, clean the held aggregates this.heldAggregates.Clean(); "); } this.Write("\r\n // Since sync time changed, set this.lastSyncTime\r\n " + " this.lastSyncTime = syncTime;\r\n }\r\n\r\n"); if (this.isUngrouped) { this.Write(" if (!currentStateExists)\r\n {\r\n currentState" + ".state = "); this.Write(this.ToStringHelper.ToStringWithCulture(initialState)); this.Write(@"; currentState.timestamp = syncTime; currentStateExists = true; isDirty = true; } else { if (!isDirty) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = currentState.timestamp; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state")))); this.Write(@" this.batch.hash.col[c] = 0; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) FlushContents(); currentState.timestamp = syncTime; isDirty = true; } } currentState.state = "); this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("currentState.state", "col_vsync[i]"))); this.Write(";\r\n"); } else { this.Write(@" // Retrieve the key from the dictionary var currentKey = col_key[i]; var currentHash = col_hash[i]; int index; bool heldAggsLookup = false; { int num = currentHash & 0x7fffffff; index = num % this.heldAggregates.Size; do { if ((this.heldAggregates.bitvector[index >> 3] & (0x1 << (index & 0x7))) == 0) { heldAggsLookup = false; break; } if ((currentHash == this.heldAggregates.entries[index].hash) && ("); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "this.heldAggregates.entries[index].key"))); this.Write(@")) { heldAggsLookup = true; break; } index++; if (index == this.heldAggregates.Size) index = 0; } while (true); } if (!heldAggsLookup) // if (!this.heldAggregates.Lookup(currentKey, currentHash, out index)) { // New group. Create new state HeldStateStruct currentState; currentState.state = "); this.Write(this.ToStringHelper.ToStringWithCulture(initialState)); this.Write(@"; currentState.timestamp = syncTime; // No output because initial state is empty this.heldAggregates.Insert(ref index, currentKey, currentState, currentHash); } else { // read new currentState from _heldAgg index var currentState = this.heldAggregates.entries[index].value; if (this.heldAggregates.IsClean(ref index)) { // Output end edge int c = this.batch.Count; this.batch.vsync.col[c] = syncTime; this.batch.vother.col[c] = currentState.timestamp; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state")))); this.Write(@" this.batch.key.col[c] = currentKey; this.batch.hash.col[c] = currentHash; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) FlushContents(); currentState.timestamp = syncTime; this.heldAggregates.SetDirty(ref index); } } this.heldAggregates.entries[index].value.state = "); this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("this.heldAggregates.entries[index].value.state", "col_vsync[i]"))); this.Write(";\r\n"); } this.Write(" }\r\n\r\n"); foreach (var f in this.inputFields.Where(fld => fld.canBeFixed)) { this.Write(" }\r\n"); } foreach (var f in this.outputFields.Where(fld => fld.canBeFixed)) { this.Write(" }\r\n"); } this.Write("\r\n generatedBatch.Release();\r\n generatedBatch.Return();\r\n }\r\n\r\n " + " public void OnPunctuation(long syncTime)\r\n {\r\n"); if (this.outputFields.Count() > 1) { this.Write(" "); this.Write(this.ToStringHelper.ToStringWithCulture(TOutput)); this.Write(" temporaryOutput;\r\n"); foreach (var f in this.outputFields) { this.Write(" var dest_"); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = this.batch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col;\r\n"); } } this.Write("\r\n // Handle time moving forward\r\n if (syncTime > this.lastSyncTime" + ")\r\n {\r\n"); if (this.isUngrouped) { this.Write(@" if (currentStateExists && isDirty) // need to send start edge if state is dirty { int c = this.batch.Count; this.batch.vsync.col[c] = currentState.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state")))); this.Write("\r\n\r\n this.batch.hash.col[c] = 0;\r\n this.batch.Count" + "++;\r\n if (this.batch.Count == Config.DataBatchSize) FlushContents" + "();\r\n }\r\n isDirty = false;\r\n"); } else { this.Write(" int iter1 = FastDictionary3<"); this.Write(this.ToStringHelper.ToStringWithCulture(TKey)); this.Write(@", HeldStateStruct>.IteratorStart; while (this.heldAggregates.IterateDirty(ref iter1)) { var iter1entry = this.heldAggregates.entries[iter1]; int c = this.batch.Count; this.batch.vsync.col[c] = iter1entry.value.timestamp; this.batch.vother.col[c] = StreamEvent.InfinitySyncTime; "); this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value.state")))); this.Write(@" this.batch.key.col[c] = iter1entry.key; this.batch.hash.col[c] = iter1entry.hash; this.batch.Count++; if (this.batch.Count == Config.DataBatchSize) FlushContents(); } // Time has moved forward, clean the held aggregates this.heldAggregates.Clean(); "); } this.Write("\r\n // Since sync time changed, set this.lastSyncTime\r\n this" + ".lastSyncTime = syncTime;\r\n }\r\n }\r\n}//\r\n"); return(this.GenerationEnvironment.ToString()); }
/// <summary> /// Generate a batch class definition to be used as an aggreate definition. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// aggregate class. /// </summary> /// <typeparam name="TKey">The key type for the aggregate.</typeparam> /// <typeparam name="TInput">The input type for the aggregate.</typeparam> /// <typeparam name="TOutput">The output type for the aggregate.</typeparam> /// <typeparam name="TState">The type for the accumulated state held by the aggregate.</typeparam> /// <typeparam name="TResult">The type of the result.</typeparam> /// <returns> /// A type that is defined to be a subtype of UnaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TInput"/>>. /// </returns> internal static Tuple <Type, string> Generate <TKey, TInput, TState, TOutput, TResult>( GroupedWindowStreamable <TKey, TInput, TState, TOutput, TResult> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(IStreamObserver <Empty, TInput>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { string expandedCode; var template = new GroupedWindowTemplate(); var keyType = template.keyType = typeof(TKey); var inputType = template.inputType = typeof(TInput); var stateType = template.stateType = typeof(TState); var outputType = template.outputType = typeof(TOutput); var resultType = template.resultType = typeof(TResult); template.TResult = resultType.GetCSharpSourceSyntax(); // BUGBUG: need to get any generic parameters needed template.isUngrouped = (keyType == typeof(Empty)); template.className = string.Format("GeneratedGroupedAggregate_{0}", GroupedAggregateSequenceNumber++); var inputMessageRepresentation = new ColumnarRepresentation(inputType); var resultRepresentation = new ColumnarRepresentation(resultType); var assemblyReferences = new List <Assembly>(); #region Key Selector var keySelector = stream.KeySelector; string transformedKeySelectorAsString; if (keyType.IsAnonymousTypeName()) { Contract.Assume(keySelector.Body is NewExpression); var transformedFunction = Extensions.TransformUnaryFunction <TKey, TInput>(keySelector); var newBody = (NewExpression)transformedFunction.Body; transformedKeySelectorAsString = string.Join(",", newBody.Arguments); } else { var transformedFunction = Extensions.TransformUnaryFunction <Empty, TInput>(keySelector).Body; if (transformedFunction == null) { return(null); } transformedKeySelectorAsString = transformedFunction.ExpressionToCSharp(); } template.keySelector = transformedKeySelectorAsString; assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector)); #endregion #region Key Comparer and HashCode var keyComparer = EqualityComparerExpression <TKey> .Default; template.keyComparerEquals = (left, right) => keyComparer.GetEqualsExpr().Inline(left, right); template.keyComparerGetHashCode = (x) => keyComparer.GetGetHashCodeExpr().Inline(x); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyComparer.GetEqualsExpr())); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyComparer.GetGetHashCodeExpr())); #endregion #region Aggregate functions var initialStateLambda = stream.Aggregate.InitialState(); if (ConstantExpressionFinder.IsClosedExpression(initialStateLambda)) { template.initialState = initialStateLambda.Body.ExpressionToCSharp(); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(initialStateLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for GroupedWindow: couldn't inline the initial state lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledInitialState = true; template.initialState = "initialState()"; } } var accumulateLambda = stream.Aggregate.Accumulate(); if (ConstantExpressionFinder.IsClosedExpression(accumulateLambda)) { var accTransformedLambda = Extensions.TransformFunction <TKey, TInput>(accumulateLambda, 2); template.accumulate = (stateArg, longArg) => accTransformedLambda.Inline(stateArg, longArg); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(accumulateLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for GroupedWindow: couldn't inline the accumulate lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledAccumulate = true; template.accumulate = (s1, s2) => string.Format("accumulate({0}, {1}, batch[i]);", s1, s2); } } var deaccumulateLambda = stream.Aggregate.Deaccumulate(); if (ConstantExpressionFinder.IsClosedExpression(deaccumulateLambda)) { var deaccumulateTransformedLambda = Extensions.TransformFunction <TKey, TInput>(deaccumulateLambda, 2); template.deaccumulate = (stateArg, longArg) => deaccumulateTransformedLambda.Inline(stateArg, longArg); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(deaccumulateLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { throw new InvalidOperationException("Code Generation couldn't inline a lambda!"); } else { template.useCompiledDeaccumulate = true; template.deaccumulate = (s1, s2) => string.Format("deaccumulate({0}, {1}, batch[i]);", s1, s2); } } var differenceLambda = stream.Aggregate.Difference(); if (ConstantExpressionFinder.IsClosedExpression(differenceLambda)) { template.difference = (stateArg1, stateArg2) => differenceLambda.Inline(stateArg1, stateArg2); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(differenceLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for GroupedWindow: couldn't inline the deaccumulate lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledDifference = true; template.deaccumulate = (s1, s2) => string.Format("difference({0}, {1});", s1, s2); } } var computeResultLambda = stream.Aggregate.ComputeResult(); if (ConstantExpressionFinder.IsClosedExpression(computeResultLambda)) { if (outputType.IsAnonymousType()) { if (computeResultLambda.Body is NewExpression newExpression) { errorMessages = "Code Generation for GroupedWindow: result selector must be a new expression for anonymous types"; throw new NotImplementedException(errorMessages); } else { template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg); } } else { template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg); } assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(computeResultLambda)); } else { if (Config.CodegenOptions.SuperStrictColumnar) { errorMessages = "Code Generation for GroupedWindow: couldn't inline the result selector lambda!"; throw new InvalidOperationException(errorMessages); } else { template.useCompiledComputeResult = true; template.computeResult = (stateArg) => "computeResult(" + stateArg + ")"; } } #endregion template.BatchGeneratedFrom_Unit_TInput = Transformer.GetBatchClassName(typeof(Empty), inputType); template.UnitTInputGenericParameters = string.Empty; // BUGBUG template.UnitTResultGenericParameters = string.Empty; // BUGBUG template.inputFields = inputMessageRepresentation.AllFields; template.outputFields = resultRepresentation.AllFields; var resultSelector = stream.ResultSelector; var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); // dont want the parameters substituted for at all var projectionResult = SelectTransformer.Transform(resultSelector, parameterSubstitutions, resultRepresentation, true); if (projectionResult.Error) { return(null); } template.finalResultSelector = (key, aggregateResult) => { var parameters = new Dictionary <ParameterExpression, string> { { resultSelector.Parameters.ElementAt(0), key } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); sb.AppendLine(string.Format("var {0} = {1};\n", resultSelector.Parameters.ElementAt(1).Name, aggregateResult)); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; sb.AppendFormat("this.batch.{0}.col[_c] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(parameters)); } sb.AppendLine("}"); return(sb.ToString()); }; assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(resultSelector)); template.staticCtor = Transformer.StaticCtor(template.className); expandedCode = template.TransformText(); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(typeof(Empty), typeof(TKey), typeof(TInput), typeof(TState), typeof(TOutput), typeof(FastDictionaryGenerator3))); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TInput>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TResult>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <Empty, TResult>()); var assembly = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var t = assembly.GetType(template.className); if (t.GetTypeInfo().IsGenericType) { var list = typeof(TKey).GetAnonymousTypes(); list.AddRange(typeof(TInput).GetAnonymousTypes()); list.AddRange(typeof(TState).GetAnonymousTypes()); list.AddRange(typeof(TOutput).GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new FixedIntervalEquiJoinTemplate($"GeneratedFixedIntervalEquiJoin_{EquiJoinSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult)) { leftDuration = stream.Left.Properties.ConstantDurationLength.Value, rightDuration = stream.Right.Properties.ConstantDurationLength.Value }; var keyAndLeftGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType).BracketedCommaSeparatedString(); var keyAndRightGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.rightType).BracketedCommaSeparatedString(); template.TKeyTResultGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString(); template.genericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType, template.rightType, template.resultType).BracketedCommaSeparatedString(); template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType); template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType); var resultMessageRepresentation = new ColumnarRepresentation(template.resultType); var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType); var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType); template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType); template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters; template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters; template.leftFields = template.leftMessageRepresentation.AllFields; template.rightFields = template.rightMessageRepresentation.AllFields; template.resultFields = resultMessageRepresentation.AllFields; template.ActiveEventTypeLeft = template.leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left"; template.ActiveEventTypeRight = template.rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right"; #region Key Equals var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparerEquals = (left, right) => keyComparer.Inline(left, right); if (template.keyType.IsAnonymousType()) { template.keyComparerEquals = (left, right) => $"keyComparerEquals({left}, {right})"; } #endregion #region Left Payload Equals { var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0); template.leftComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Right Payload Equals { var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr(); var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0); template.rightComparerEquals = (left, right) => newLambda.Inline(left, right); } #endregion #region Result Selector { var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); if (!ConstantExpressionFinder.IsClosedExpression(selector)) { errorMessages = "result selector is not a closed expression"; throw new InvalidOperationException(); } #region LeftBatchSelector { var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i"); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region RightBatchSelector { var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j"); var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region ActiveSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.activeSelector = (leftEvent, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion } #endregion return(template.Generate <TKey, TLeft, TRight, TResult>()); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
private static Tuple <Type, string> Generate <TKey, TSource, TResult>( string ingressType, string partitionString, Expression <Func <TSource, TKey> > partitionExpression, Expression <Func <TSource, long> > startEdgeExpression, Expression <Func <TSource, long> > endEdgeExpression, string latencyOption, string diagnosticOption, FuseModule fuseModule) { var template = new TemporalIngressTemplate( $"GeneratedTemporalIngress_{TemporalIngressSequenceNumber++}", typeof(TKey), typeof(TSource), fuseModule?.OutputType ?? typeof(TResult)); var tm = new TypeMapper(template.keyType, template.payloadType, template.resultType); var gps = tm.GenericTypeVariables(template.keyType, template.payloadType); template.genericParameters = gps.BracketedCommaSeparatedString(); template.payloadOrResult = tm.CSharpNameFor(template.resultRepresentation.RepresentationFor); var batchGeneratedFrom_TKey_TPayload = string.IsNullOrEmpty(partitionString) ? Transformer.GetBatchClassName(template.keyType, template.resultType) : Transformer.GetBatchClassName(typeof(PartitionKey <TKey>), template.resultType); var keyAndPayloadGenericParameters = tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString(); template.GeneratedBatchName = batchGeneratedFrom_TKey_TPayload + keyAndPayloadGenericParameters; template.keyOrNothing = string.IsNullOrEmpty(partitionString) ? string.Empty : template.TKey + ", "; template.diagnosticOption = diagnosticOption; template.needsStreamEvent = ingressType == "StreamEvent"; template.genericArguments = (string.IsNullOrEmpty(partitionString) ? string.Empty : template.TKey + ", ") + template.TPayload; template.adjustedGenericArgs = (string.IsNullOrEmpty(partitionString) ? string.Empty : template.TKey + ", ") + (!fuseModule.IsEmpty && Config.AllowFloatingReorderPolicy ? template.TResult : template.TPayload); template.emptyOrPartition = string.IsNullOrEmpty(partitionString) ? "Microsoft.StreamProcessing.Empty.Default" : "new PartitionKey<" + template.TKey + ">(value.PartitionKey)"; template.partitionString = partitionString; template.baseStructure = partitionString + "StreamEvent<" + template.genericArguments + ">"; template.inheritBase = (ingressType != "StreamEvent") ? template.TPayload : template.baseStructure; template.ingressType = ingressType; template.latencyOption = latencyOption; template.partitionFunction = x => partitionExpression == null ? string.Empty : partitionExpression.Body.ExpressionToCSharpStringWithParameterSubstitution( new Dictionary <ParameterExpression, string> { { partitionExpression.Parameters.Single(), x } }); template.startEdgeFunction = x => startEdgeExpression == null ? string.Empty : startEdgeExpression.Body.ExpressionToCSharpStringWithParameterSubstitution( new Dictionary <ParameterExpression, string> { { startEdgeExpression.Parameters.Single(), x } }); template.endEdgeFunction = x => endEdgeExpression == null ? "StreamEvent.InfinitySyncTime" : endEdgeExpression.Body.ExpressionToCSharpStringWithParameterSubstitution( new Dictionary <ParameterExpression, string> { { endEdgeExpression.Parameters.Single(), x } }); template.fusionOption = fuseModule.IsEmpty ? "Simple" : (Config.AllowFloatingReorderPolicy ? "Disordered" : "Fused"); template.resultMightBeNull = template.resultType.CanContainNull(); Expression[] expressions = null; if (!fuseModule.IsEmpty) { template.valueString = fuseModule.Coalesce <TSource, TResult, TKey>("value.SyncTime", "value.OtherTime", "value.Payload", template.emptyOrPartition, out template.leadingText, out template.trailingText); template.generatedEndTimeVariable = "generatedEndTimeVariable"; if (Config.AllowFloatingReorderPolicy) { template.valueString = "value.Payload"; template.generatedEndTimeVariable = "value.OtherTime"; } expressions = fuseModule.GetCodeGenExpressions(); } return(template.Generate <TKey, TSource, TResult>(new Type[] { typeof(IStreamable <,>) }, expressions)); }
/// <summary> /// Generate a batch class definition to be used as StartEdgeEquiJoin operator. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// aggregate class. /// </summary> /// <typeparam name="TKey">The key type for both sides.</typeparam> /// <typeparam name="TLeft">The payload type for the left side.</typeparam> /// <typeparam name="TRight">The payload type for the right side.</typeparam> /// <typeparam name="TResult">The payload type for the resulting stream.</typeparam> /// <returns> /// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TRight"/>, <typeparamref name="TKey"/>, <typeparamref name="TResult"/>>. /// </returns> internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new IncreasingOrderEquiJoinTemplate($"GeneratedIncreasingOrderEquiJoin_{IOOEJSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult)); template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType); template.leftFields = template.leftMessageRepresentation.AllFields; template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType); template.rightFields = template.rightMessageRepresentation.AllFields; var resultRepresentation = new ColumnarRepresentation(template.resultType); var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); #region Key Comparer var keyComparer = stream.Left.Properties.KeyComparer.GetCompareExpr(); if (!ConstantExpressionFinder.IsClosedExpression(keyComparer)) { return(null); } template.joinKeyOrderComparer = (left, right) => keyComparer.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType); template.TKeyTLeftGenericParameters = string.Empty; // BUGBUG template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType); template.TKeyTRightGenericParameters = string.Empty; // BUGBUG template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType); template.TKeyTResultGenericParameters = string.Empty; // BUGBUG template.outputFields = resultRepresentation.AllFields; if (!ConstantExpressionFinder.IsClosedExpression(selector)) { return(null); } #region LeftBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), "i"), leftIndex }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region RightBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), "j"), rightIndex } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion template.getOutputBatch = string.Format( "pool.Get(out genericOutputBatch); output = ({0}{1})genericOutputBatch;", Transformer.GetBatchClassName(template.keyType, template.resultType), template.TKeyTResultGenericParameters); return(template.Generate <TKey, TLeft, TRight, TResult>(selector)); } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
/// <summary> /// Create the template output /// </summary> public override string TransformText() { this.Write("\r\n"); this.Write(@"// ********************************************************************* // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License // ********************************************************************* using System; using System.Collections.Generic; using System.Linq.Expressions; using System.Runtime.CompilerServices; using System.Runtime.Serialization; using Microsoft.StreamProcessing; using Microsoft.StreamProcessing.Internal; using Microsoft.StreamProcessing.Internal.Collections; [assembly: IgnoresAccessChecksTo(""Microsoft.StreamProcessing"")] "); List <string> genericParamList = new List <string>(); var TOuterKey = outerKeyType.GetCSharpSourceSyntax(ref genericParamList); var outerKeyGenericParameterCount = genericParamList.Count; var TSource = sourceType.GetCSharpSourceSyntax(ref genericParamList); var sourceGenericParameterCount = genericParamList.Count; var sourceGenericParameters = 0 < genericParamList.Count ? "<" + String.Join(",", genericParamList) + ">" : string.Empty; var TInnerKey = innerKeyType.GetCSharpSourceSyntax(ref genericParamList); var genericParameters = 0 < genericParamList.Count ? "<" + String.Join(",", genericParamList) + ">" : string.Empty; var outKeyInnerKeyGenericParameterList = genericParamList.Take(outerKeyGenericParameterCount).Concat(genericParamList.Skip(sourceGenericParameterCount)); var TOuterKeyTInnerKeyGenericParameters = 0 < outKeyInnerKeyGenericParameterList.Count() ? "<" + String.Join(",", outKeyInnerKeyGenericParameterList) + ">" : string.Empty; var resultBatchGenericParameters = genericParameters; var resultBatchClassType = isFirstLevelGroup ? Transformer.GetBatchClassName(innerKeyType, sourceType) : Transformer.GetBatchClassName(typeof(CompoundGroupKey <,>).MakeGenericType(outerKeyType, innerKeyType), sourceType); var sourceBatchClassType = Transformer.GetBatchClassName(outerKeyType, sourceType); var innerKeyIsAnonymous = innerKeyType.IsAnonymousTypeName(); if (innerKeyIsAnonymous) { transformedKeySelectorAsString = string.Format("({0})Activator.CreateInstance(typeof({0}), {1} )", TInnerKey, transformedKeySelectorAsString); } var outputKey = !isFirstLevelGroup ? "CompoundGroupKey<" + TOuterKey + ", " + TInnerKey + ">" : TInnerKey; var nestedInfix = !isFirstLevelGroup ? "Nested" : string.Empty; var ungroupingToUnit = isFirstLevelGroup && innerKeyType == typeof(Empty); var swingingKeyFromField = isFirstLevelGroup && !this.swingingField.Equals(default(MyFieldInfo)); this.Write("\r\n// TOuterKey: "); this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey)); this.Write("\r\n// TInnerKey: "); this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey)); this.Write("\r\n// TSource: "); this.Write(this.ToStringHelper.ToStringWithCulture(TSource)); this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters)); this.Write(" :\r\n Pipe<"); this.Write(this.ToStringHelper.ToStringWithCulture(outputKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TSource)); this.Write(">, IStreamObserver<"); this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TSource)); this.Write(">\r\n{\r\n"); if (innerKeyIsAnonymous) { this.Write(" [SchemaSerialization]\r\n private readonly Expression<Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey)); this.Write(", int>> keyComparer;\r\n private readonly Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey)); this.Write(", int> innerHashCode;\r\n"); } this.Write(" private readonly MemoryPool<"); this.Write(this.ToStringHelper.ToStringWithCulture(outputKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TSource)); this.Write("> l1Pool;\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanG" + "enerator;\r\n\r\n"); if (ungroupingToUnit) { this.Write("\r\n private readonly ColumnBatch<Microsoft.StreamProcessing.Empty> unitColumn;\r" + "\n private readonly ColumnBatch<int> unitHashColumn;\r\n"); } this.Write("\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor)); this.Write("\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("() { }\r\n\r\n public "); this.Write(this.ToStringHelper.ToStringWithCulture(className)); this.Write("(\r\n IStreamable<"); this.Write(this.ToStringHelper.ToStringWithCulture(outputKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TSource)); this.Write("> stream,\r\n IStreamObserver<"); this.Write(this.ToStringHelper.ToStringWithCulture(outputKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TSource)); this.Write("> observer,\r\n Expression<Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey)); this.Write(", int>> keyComparer,\r\n Expression<Func<"); this.Write(this.ToStringHelper.ToStringWithCulture(TSource)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey)); this.Write(">> keySelector,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerato" + "r)\r\n : base(stream, observer)\r\n {\r\n"); if (innerKeyIsAnonymous) { if (isFirstLevelGroup) { this.Write(" var _keySelector = keySelector.Compile();\r\n"); } this.Write(" this.keyComparer = keyComparer;\r\n this.innerHashCode = keyComparer" + ".Compile();\r\n"); } this.Write(" this.l1Pool = MemoryManager.GetMemoryPool<"); this.Write(this.ToStringHelper.ToStringWithCulture(outputKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TSource)); this.Write(">(true);\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n"); if (ungroupingToUnit) { this.Write("\r\n this.l1Pool.GetKey(out this.unitColumn);\r\n this.l1Pool.Get(out t" + "his.unitHashColumn);\r\n Array.Clear(this.unitHashColumn.col, 0, this.unitH" + "ashColumn.col.Length);\r\n"); } this.Write(@" } public override void ProduceQueryPlan(PlanNode previous) { Observer.ProduceQueryPlan(queryPlanGenerator(previous, this)); } protected override void FlushContents() { } public override int CurrentlyBufferedOutputCount => 0; public override int CurrentlyBufferedInputCount => 0; protected override void DisposeState() { "); if (ungroupingToUnit) { this.Write(" this.unitColumn.Return();\r\n this.unitHashColumn.Return();\r\n"); } this.Write(" }\r\n\r\n public unsafe void OnNext(StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TSource)); this.Write("> batch)\r\n {\r\n StreamMessage<"); this.Write(this.ToStringHelper.ToStringWithCulture(outputKey)); this.Write(", "); this.Write(this.ToStringHelper.ToStringWithCulture(TSource)); this.Write("> resultBatchUncast; // Need this type to call Get with so the right subtype will" + " be returned\r\n this.l1Pool.Get(out resultBatchUncast);\r\n\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchClassType)); this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchGenericParameters)); this.Write(" resultBatch = resultBatchUncast as "); this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchClassType)); this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchGenericParameters)); this.Write(";\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(sourceBatchClassType)); this.Write(this.ToStringHelper.ToStringWithCulture(sourceGenericParameters)); this.Write(" inputBatch = batch as "); this.Write(this.ToStringHelper.ToStringWithCulture(sourceBatchClassType)); this.Write(this.ToStringHelper.ToStringWithCulture(sourceGenericParameters)); this.Write(";\r\n\r\n resultBatch.vsync = batch.vsync;\r\n resultBatch.vother = batch" + ".vother;\r\n"); foreach (var f in this.fields) { this.Write(" resultBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(" = inputBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(";\r\n"); } if (this.payloadMightBeNull) { this.Write(" resultBatch._nullnessvector = inputBatch._nullnessvector;\r\n"); } this.Write("\r\n"); if (!ungroupingToUnit && !swingingHashColumn) { this.Write("\r\n resultBatch.hash = batch.hash.MakeWritable(this.l1Pool.intPool);\r\n"); } this.Write(" resultBatch.bitvector = batch.bitvector;\r\n\r\n"); if (!ungroupingToUnit && (!isFirstLevelGroup || this.swingingField.Equals(default(MyFieldInfo)))) { this.Write(" this.l1Pool.GetKey(out resultBatchUncast.key);\r\n"); } this.Write("\r\n resultBatch.Count = batch.Count;\r\n\r\n var count = batch.Count;\r\n\r" + "\n"); if (ungroupingToUnit) { this.Write(" this.unitColumn.IncrementRefCount(1);\r\n resultBatch.key = unitColu" + "mn;\r\n this.unitHashColumn.IncrementRefCount(1);\r\n resultBatch.hash" + " = unitHashColumn;\r\n"); } else { this.Write("\r\n var src_key = batch.key.col;\r\n"); if (!isFirstLevelGroup || this.swingingField.Equals(default(MyFieldInfo))) { this.Write("\r\n var dest_key = resultBatch.key.col;\r\n"); } this.Write(" var dest_payload = resultBatch;\r\n\r\n"); if (swingingKeyFromField) { this.Write(" // The key selector was just e => e.f *and* isFirstLevelGroup so just swi" + "ng the key field\r\n"); if (this.swingingField.OptimizeString()) { this.Write("\r\n resultBatch.key = resultBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(this.swingingField.Name)); this.Write(".ToColumnBatch(this.l1Pool.stringPool, resultBatch.bitvector);\r\n"); } else { this.Write("\r\n resultBatch.key = resultBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(this.swingingField.Name)); this.Write(";\r\n resultBatch.key.IncrementRefCount(1);\r\n"); } } this.Write("\r\n\r\n"); foreach (var f in this.fields) { this.Write("\r\n"); if (f.canBeFixed) { this.Write(" fixed ("); this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName)); this.Write("* "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write("_col = resultBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col)\r\n {\r\n"); } else { this.Write("\r\n"); if (f.OptimizeString()) { this.Write(" var "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write("_col = resultBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(";\r\n"); } else { this.Write(" var "); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write("_col = resultBatch."); this.Write(this.ToStringHelper.ToStringWithCulture(f.Name)); this.Write(".col;\r\n"); } } } this.Write("\r\n "); this.Write(this.ToStringHelper.ToStringWithCulture(vectorHashCodeInitialization)); this.Write("\r\n\r\n"); if (!swingingKeyFromField || !swingingHashColumn) { this.Write("\r\n fixed (long* src_bv = batch.bitvector.col) {\r\n"); if (!ungroupingToUnit) { this.Write(" fixed (int* dest_hash = resultBatch.hash.col) {\r\n"); } this.Write(" for (int i = 0; i < count; i++)\r\n {\r\n if ((" + "src_bv[i >> 6] & (1L << (i & 0x3f))) != 0) continue;\r\n\r\n"); if (!swingingKeyFromField || !this.swingingField.OptimizeString()) { this.Write("\r\n var key = "); this.Write(this.ToStringHelper.ToStringWithCulture(this.transformedKeySelectorAsString)); this.Write(";\r\n"); } if (innerKeyIsAnonymous) { this.Write(" var innerHash = this.innerHashCode(key);\r\n"); } else if (swingingHashColumn) { this.Write("\r\n // don\'t compute hash because that was done by calling MultiStr" + "ing.GetHashCode\r\n"); } else { this.Write(" var innerHash = "); this.Write(this.ToStringHelper.ToStringWithCulture(inlinedHashCodeComputation)); this.Write(";\r\n"); } this.Write("\r\n"); if (isFirstLevelGroup) { if (!this.swingingField.Equals(default(MyFieldInfo))) { this.Write(" // no assignment to key, pointer was swung!\r\n"); } else { this.Write(" dest_key[i] = key;\r\n"); } if (!swingingHashColumn) { this.Write("\r\n dest_hash[i] = innerHash;\r\n"); } } else { this.Write(" var hash = dest_hash[i] ^ innerHash;\r\n dest_key[i]" + ".outerGroup = src_key[i];\r\n dest_key[i].innerGroup = key;\r\n " + " dest_key[i].hashCode = hash;\r\n dest_hash[i] = hash;\r\n"); } this.Write(" }\r\n"); if (!ungroupingToUnit) { this.Write(" } // end of fixed for dest_hash\r\n"); } this.Write(" } // end of fixed for src_bv\r\n"); } this.Write("\r\n "); if (!String.IsNullOrWhiteSpace(vectorHashCodeInitialization) && !swingingHashColumn) { this.Write("\r\n this.hashCodeVector.Return();\r\n "); } this.Write("\r\n"); foreach (var f in this.fields.Where(fld => fld.canBeFixed)) { this.Write("\r\n }\r\n"); } this.Write("\r\n"); } this.Write(" batch.ReleaseKey();\r\n"); if (ungroupingToUnit || swingingHashColumn) { this.Write("\r\n batch.hash.Return();\r\n"); } this.Write("\r\n batch.Return();\r\n\r\n this.Observer.OnNext(resultBatch);\r\n }\r\n\r" + "\n public void OnError(Exception error)\r\n {\r\n Observer.OnError(error" + ");\r\n }\r\n}\r\n"); return(this.GenerationEnvironment.ToString()); }
/// <summary> /// Generate a batch class definition to be used as a Union pipe. /// Compile the definition, dynamically load the assembly containing it, and return the Type representing the /// union pipe class. /// </summary> /// <typeparam name="TKey">The key type for both sides.</typeparam> /// <typeparam name="TPayload">The payload type.</typeparam> /// <returns> /// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TPayload"/>, <typeparamref name="TPayload"/>, <typeparamref name="TKey"/>, <typeparamref name="TPayload"/>>. /// </returns> internal static Tuple <Type, string> GenerateUnionPipeClass <TKey, TPayload>( UnionStreamable <TKey, TPayload> stream) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TPayload, TPayload, TPayload>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); #if CODEGEN_TIMING Stopwatch sw = new Stopwatch(); sw.Start(); #endif string errorMessages = null; try { var template = new UnionTemplate(); var keyType = template.keyType = typeof(TKey); var payloadType = template.payloadType = typeof(TPayload); var tm = new TypeMapper(keyType, payloadType); template.TKey = tm.CSharpNameFor(keyType); template.TPayload = tm.CSharpNameFor(payloadType); var gps = tm.GenericTypeVariables(keyType, payloadType); template.genericParameters = gps.BracketedCommaSeparatedString(); template.className = string.Format("GeneratedUnion_{0}", UnionSequenceNumber++); var resultRepresentation = new ColumnarRepresentation(payloadType); var batchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName(keyType, payloadType); var keyAndPayloadGenericParameters = tm.GenericTypeVariables(keyType, payloadType).BracketedCommaSeparatedString(); template.GeneratedBatchName = batchGeneratedFrom_TKey_TPayload + keyAndPayloadGenericParameters; template.fields = resultRepresentation.AllFields; template.staticCtor = Transformer.StaticCtor(template.className); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload)); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>()); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, payloadType); var t = a.GetType(realClassName); if (t.GetTypeInfo().IsGenericType) { var list = keyType.GetAnonymousTypes(); list.AddRange(payloadType.GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector) { Contract.Requires(stream != null); Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1)); string errorMessages = null; try { var template = new StartEdgeEquiJoinTemplate(); var keyType = template.keyType = typeof(TKey); var leftType = template.leftType = typeof(TLeft); var rightType = template.rightType = typeof(TRight); var resultType = template.resultType = typeof(TResult); var tm = new TypeMapper(keyType, leftType, rightType, resultType); template.TKey = tm.CSharpNameFor(keyType); template.TLeft = tm.CSharpNameFor(leftType); template.TRight = tm.CSharpNameFor(rightType); template.TResult = tm.CSharpNameFor(resultType); var gps = tm.GenericTypeVariables(keyType, leftType, rightType, resultType); template.genericParameters = gps.BracketedCommaSeparatedString(); template.className = string.Format("GeneratedStartEdgeEquiJoin_{0}", StartEdgeEquiJoinSequenceNumber++); template.leftMessageRepresentation = new ColumnarRepresentation(leftType); template.leftFields = template.leftMessageRepresentation.AllFields; template.rightMessageRepresentation = new ColumnarRepresentation(rightType); template.rightFields = template.rightMessageRepresentation.AllFields; var outputMessageRepresentation = new ColumnarRepresentation(resultType); var resultRepresentation = outputMessageRepresentation; #region Key Comparer var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); template.keyComparer = (left, right) => keyComparer.Inline(left, right); #endregion template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType); template.TKeyTLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType); template.TKeyTRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString(); template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType); template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString(); template.outputFields = resultRepresentation.AllFields; var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>(); var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>(); #region LeftBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[0], new SelectParameterInformation() { BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) => { var d = new Dictionary <ParameterExpression, string> { { Expression.Variable(leftMessageType, "leftBatch"), leftBatch }, { Expression.Variable(typeof(int), "i"), leftIndex }, { selector.Parameters[1], rightEvent } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion #region RightBatchSelector { var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >() { Tuple.Create(selector.Parameters[1], new SelectParameterInformation() { BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation, }), }; var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true); if (projectionResult.Error) { errorMessages = "error while transforming the result selector"; throw new InvalidOperationException(); } template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) => { var d = new Dictionary <ParameterExpression, string> { { selector.Parameters[0], leftEvent }, { Expression.Variable(rightMessageType, "rightBatch"), rightBatch }, { Expression.Variable(typeof(int), "j"), rightIndex } }; var sb = new System.Text.StringBuilder(); sb.AppendLine("{"); foreach (var kv in projectionResult.ComputedFields) { var f = kv.Key; var e = kv.Value; if (f.OptimizeString()) { sb.AppendFormat( "output.{0}.AddString({1});\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } else { sb.AppendFormat( "output.{0}.col[index] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(d)); } } sb.AppendLine("}"); return(sb.ToString()); }; } #endregion template.staticCtor = Transformer.StaticCtor(template.className); var expandedCode = template.TransformText(); var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult)); assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>()); assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>()); assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>()); assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(selector)); var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages); var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType, resultType); var t = a.GetType(realClassName); if (t.GetTypeInfo().IsGenericType) { var list = keyType.GetAnonymousTypes(); list.AddRange(leftType.GetAnonymousTypes()); list.AddRange(rightType.GetAnonymousTypes()); list.AddRange(resultType.GetAnonymousTypes()); return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages)); } else { return(Tuple.Create(t, errorMessages)); } } catch { if (Config.CodegenOptions.DontFallBackToRowBasedExecution) { throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!"); } return(Tuple.Create((Type)null, errorMessages)); } }