private TemporalEgressTemplate(Type tKey, Type tPayload, Type tResult, string partitionString, string ingressType, bool isColumnar)
: base($"GeneratedTemporalEgress_{TemporalEgressSequenceNumber++}")
{
var tm = new TypeMapper(tKey, tPayload, tResult);
this.payloadRepresentation = new ColumnarRepresentation(tPayload);
this.BatchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName(
tKey == typeof(Empty)
? tKey
: typeof(PartitionKey <>).MakeGenericType(tKey), tPayload);
this.TKeyTPayloadGenericParameters = tm.GenericTypeVariables(tKey, tPayload).BracketedCommaSeparatedString();
this.fields = this.payloadRepresentation.AllFields;
this.TKey = tm.CSharpNameFor(tKey);
this.TPayload = tm.CSharpNameFor(tPayload);
this.TResult = tm.CSharpNameFor(tResult);
this.partitionString = partitionString;
this.ingressType = ingressType;
this.partitionKeyArgument = !string.IsNullOrEmpty(partitionString) ? "colkey[i].Key, " : string.Empty;
this.genericArguments = string.IsNullOrEmpty(partitionString) ? this.TPayload : this.TKey + ", " + this.TPayload;
this.egress = (ingressType != "StreamEvent")
? this.TResult
: partitionString + "StreamEvent<" + this.genericArguments + ">";
this.inputKey = string.IsNullOrEmpty(partitionString) ? this.TKey : "PartitionKey<" + this.TKey + ">";
this.isColumnar = isColumnar;
}
protected CommonUnaryTemplate(string className, Type keyType, Type payloadType, Type resultType, bool suppressPayloadBatch = false)
: base(className)
{
this.keyType = keyType;
this.payloadType = payloadType;
this.resultType = resultType;
var tm = new TypeMapper(keyType, payloadType, resultType);
this.TKey = tm.CSharpNameFor(keyType);
this.TPayload = tm.CSharpNameFor(payloadType);
this.TResult = tm.CSharpNameFor(resultType);
if (!suppressPayloadBatch)
{
this.TKeyTPayloadGenericParameters = tm.GenericTypeVariables(keyType, payloadType).BracketedCommaSeparatedString();
this.BatchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName(keyType, payloadType);
}
this.payloadRepresentation = new ColumnarRepresentation(payloadType);
this.fields = this.payloadRepresentation.AllFields;
this.noFields = this.payloadRepresentation.noFields;
this.resultRepresentation = new ColumnarRepresentation(resultType);
}
public ShuffleTemplate(
string className,
Type outerKeyType,
Type sourceType,
Type innerKeyType,
string inlinedHashCodeComputation,
bool nested, bool powerOf2) : base(className)
{
Contract.Requires(className != null);
Contract.Requires(outerKeyType != null);
Contract.Requires(sourceType != null);
Contract.Requires(innerKeyType != null);
this.inlinedHashCodeComputation = inlinedHashCodeComputation;
this.isFirstLevelGroup = !nested;
this.powerOf2 = powerOf2;
var tm = new TypeMapper(outerKeyType, sourceType, innerKeyType);
this.TOuterKey = tm.CSharpNameFor(outerKeyType);
this.TSource = tm.CSharpNameFor(sourceType);
this.TInnerKey = tm.CSharpNameFor(innerKeyType);
this.genericParameters = tm.GenericTypeVariables(outerKeyType, sourceType, innerKeyType).BracketedCommaSeparatedString();
this.resultBatchClassType = this.isFirstLevelGroup
? Transformer.GetBatchClassName(innerKeyType, sourceType)
: Transformer.GetBatchClassName(typeof(CompoundGroupKey <,>).MakeGenericType(outerKeyType, innerKeyType), sourceType);
this.resultBatchGenericParameters = this.genericParameters;
this.sourceBatchClassType = Transformer.GetBatchClassName(outerKeyType, sourceType);
this.TOuterKeyTSourceGenericParameters = tm.GenericTypeVariables(outerKeyType, sourceType).BracketedCommaSeparatedString();
}
/// <summary>
/// Generate a batch class definition to be used as a Union pipe.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// union pipe class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TPayload">The payload type.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TPayload"/>, <typeparamref name="TPayload"/>, <typeparamref name="TKey"/>, <typeparamref name="TPayload"/>>.
/// </returns>
internal static Tuple <Type, string> GenerateUnionPipeClass <TKey, TPayload>(
UnionStreamable <TKey, TPayload> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TPayload, TPayload, TPayload>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
#if CODEGEN_TIMING
Stopwatch sw = new Stopwatch();
sw.Start();
#endif
var template = new UnionTemplate($"GeneratedUnion_{UnionSequenceNumber++}", typeof(TKey), typeof(TPayload));
var gps = template.tm.GenericTypeVariables(template.keyType, template.resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
var resultRepresentation = new ColumnarRepresentation(template.resultType);
var batchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName(template.keyType, template.resultType);
var keyAndPayloadGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString();
template.GeneratedBatchName = batchGeneratedFrom_TKey_TPayload + keyAndPayloadGenericParameters;
template.fields = resultRepresentation.AllFields;
return(template.Generate <TKey, TPayload>());
}
public static void GetGeneratedCode(Type keyType, Type payloadType, out string generatedClassName, out string expandedCode, out List <Assembly> assemblyReferences)
{
var template = new SafeBatchTemplate();
assemblyReferences = new List <Assembly>
{
Transformer.SystemRuntimeSerializationDll // needed for [DataContract] and [DataMember] in generated code
};
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyType));
template.keyType = keyType;
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(payloadType));
template.payloadType = payloadType;
template.needsPolymorphismCheck =
!payloadType.GetTypeInfo().IsValueType&&
!payloadType.IsAnonymousTypeName() &&
!payloadType.GetTypeInfo().IsSealed;
template.payloadMightBeNull = payloadType.CanContainNull();
generatedClassName = Transformer.GetBatchClassName(keyType, payloadType);
template.CLASSNAME = generatedClassName.CleanUpIdentifierName();
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(keyType, payloadType);
var payloadRepresentation = new ColumnarRepresentation(payloadType);
template.fields = payloadRepresentation.AllFields;
template.noPublicFields = payloadRepresentation.noFields;
expandedCode = template.TransformText();
}
private UngroupedDAfaTemplate(string className, Type payloadType, Type registerType, Type accumulatorType)
: base(className, typeof(Empty), payloadType, registerType, accumulatorType)
{
if (Config.ForceRowBasedExecution)
{
this.sourceBatchTypeName = $"Microsoft.StreamProcessing.StreamMessage<Microsoft.StreamProcessing.Empty, {this.TPayload}>";
this.resultBatchTypeName = $"Microsoft.StreamProcessing.StreamMessage<Microsoft.StreamProcessing.Empty, {this.TRegister}>";
}
else
{
this.sourceBatchTypeName = Transformer.GetBatchClassName(typeof(Empty), payloadType);
this.resultBatchTypeName = Transformer.GetBatchClassName(typeof(Empty), registerType);
}
}
private UngroupedDAfaTemplate(string className, Type keyType, Type payloadType, Type registerType, Type accumulatorType)
: base(className, keyType, payloadType, registerType, accumulatorType)
{
if (Config.ForceRowBasedExecution)
{
this.sourceBatchTypeName = string.Format("Microsoft.StreamProcessing.StreamMessage<{0}, {1}>", this.TKey, this.TPayload);
this.resultBatchTypeName = string.Format("Microsoft.StreamProcessing.StreamMessage<{0}, {1}>", this.TKey, this.TRegister);
}
else
{
this.sourceBatchTypeName = Transformer.GetBatchClassName(keyType, payloadType);
this.resultBatchTypeName = Transformer.GetBatchClassName(keyType, registerType);
}
}
private GroupedAfaEventListTemplate(string className, Type keyType, Type payloadType, Type registerType, Type accumulatorType)
: base(className, keyType, payloadType, registerType, accumulatorType)
{
if (Config.ForceRowBasedExecution)
{
this.sourceBatchTypeName = $"Microsoft.StreamProcessing.StreamMessage<{this.TKey}, {this.TPayload}>";
this.resultBatchTypeName = $"Microsoft.StreamProcessing.StreamMessage<{this.TKey}, {this.TRegister}>";
}
else
{
this.sourceBatchTypeName = Transformer.GetBatchClassName(keyType, payloadType);
this.resultBatchTypeName = Transformer.GetBatchClassName(keyType, registerType);
}
}
private UngroupTemplate(
string className,
bool ungroupingFromCompound,
Type outerKeyType,
Type innerKeyType,
Type innerResultType,
Type resultType)
{
Contract.Requires(className != null);
Contract.Requires(outerKeyType != null);
Contract.Requires(innerKeyType != null);
Contract.Requires(innerResultType != null);
Contract.Requires(resultType != null);
this.CLASSNAME = className;
this.outerKeyType = outerKeyType;
this.innerKeyType = innerKeyType;
this.innerResultType = innerResultType;
this.resultType = resultType;
this.ungroupingToUnit = outerKeyType == typeof(Empty);
this.ungroupingFromCompound = ungroupingFromCompound;
var tm = new TypeMapper(outerKeyType, innerKeyType, innerResultType, resultType);
this.TOuterKey = tm.CSharpNameFor(outerKeyType);
this.TInnerKey = tm.CSharpNameFor(innerKeyType);
this.TInnerResult = tm.CSharpNameFor(innerResultType);
this.TResult = tm.CSharpNameFor(resultType);
var gps = tm.GenericTypeVariables(outerKeyType, innerKeyType, innerResultType, resultType);
this.genericParameters = gps.BracketedCommaSeparatedString();
this.numberOfGenericParameters = gps.Count();
this.inputBatchClassType = ungroupingFromCompound
? Transformer.GetBatchClassName(typeof(CompoundGroupKey <,>).MakeGenericType(outerKeyType, innerKeyType), innerResultType)
: Transformer.GetBatchClassName(innerKeyType, innerResultType);
this.inputBatchGenericParameters = ungroupingFromCompound
? tm.GenericTypeVariables(outerKeyType, innerKeyType, innerResultType).BracketedCommaSeparatedString()
: tm.GenericTypeVariables(innerKeyType, innerResultType).BracketedCommaSeparatedString();
this.resultBatchClassType = Transformer.GetBatchClassName(outerKeyType, resultType);
this.resultBatchGenericParameters = tm.GenericTypeVariables(outerKeyType, resultType).BracketedCommaSeparatedString();
}
/// <summary>
/// Generate a batch class definition to be used as a Clip pipe.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TLeft">The payload type for the left side.</typeparam>
/// <typeparam name="TRight">The payload type for the right side.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TKey"/>, <typeparamref name="TRight"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight>(ClipJoinStreamable <TKey, TLeft, TRight> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TLeft>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
var template = new ClipJoinTemplate($"GeneratedClip_{ClipSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight));
var keyType = typeof(TKey);
var leftType = typeof(TLeft);
var rightType = typeof(TRight);
var gps = template.tm.GenericTypeVariables(keyType, leftType, rightType);
template.genericParameters = gps.BracketedCommaSeparatedString();
var resultRepresentation = new ColumnarRepresentation(leftType);
template.ActiveEventType = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event";
#region Key Comparer
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
#region Left Comparer
var leftComparer = stream.LeftComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftComparer, "index_ProcessLeftEvent", 0);
template.leftComparer = (left, right) => newLambda.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
template.TKeyTLeftGenericParameters = template.tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.TKeyTRightGenericParameters = template.tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString();
template.leftFields = resultRepresentation.AllFields;
template.noFields = resultRepresentation.noFields;
return(template.Generate <TKey, TLeft, TRight>(leftComparer));
}
private void GetOutputBatch()
{
this.Write(" pool.Get(out genericOutputBatch);\r\n genericOutputBatch.Allocate();\r\n output =" +
" (");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(this.keyType, this.leftType)));
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.TKeyTLeftGenericParameters));
this.Write(")genericOutputBatch;\r\n");
foreach (var f in this.leftFields.Where(fld => fld.OptimizeString()))
{
this.Write(" output.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".Initialize();\r\n");
}
}
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new EquiJoinTemplate();
var keyType = template.keyType = typeof(TKey);
var leftType = template.leftType = typeof(TLeft);
var rightType = template.rightType = typeof(TRight);
var resultType = template.resultType = typeof(TResult);
var tm = new TypeMapper(keyType, leftType, rightType, resultType);
template.TKey = tm.CSharpNameFor(keyType);
template.TLeft = tm.CSharpNameFor(leftType);
template.TRight = tm.CSharpNameFor(rightType);
template.TResult = tm.CSharpNameFor(resultType);
var keyAndLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString();
var keyAndRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString();
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.genericParameters = tm.GenericTypeVariables(keyType, leftType, rightType, resultType).BracketedCommaSeparatedString();
template.className = string.Format("GeneratedEquiJoin_{0}", EquiJoinSequenceNumber++);
template.leftMessageRepresentation = new ColumnarRepresentation(leftType);
template.rightMessageRepresentation = new ColumnarRepresentation(rightType);
var resultMessageRepresentation = new ColumnarRepresentation(resultType);
var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType);
template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters;
template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters;
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightFields = template.rightMessageRepresentation.AllFields;
template.resultFields = resultMessageRepresentation.AllFields;
template.ActiveEventTypeLeft = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left";
template.ActiveEventTypeRight = rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right";
#region Key Equals
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparerEquals =
(left, right) =>
keyComparer.Inline(left, right);
if (keyType.IsAnonymousType())
{
template.keyComparerEquals =
(left, right) => string.Format("keyComparerEquals({0}, {1})", left, right);
}
#endregion
#region Left Payload Equals
{
var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0);
template.leftComparerEquals = (left, right) => newLambda.Inline(left, right);
}
#endregion
#region Right Payload Equals
{
var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0);
template.rightComparerEquals = (left, right) => newLambda.Inline(left, right);
}
#endregion
#region Result Selector
{
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
errorMessages = "result selector is not a closed expression";
throw new InvalidOperationException();
}
#region LeftBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), "i"), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), "j"), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region ActiveSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >();
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.activeSelector = (leftEvent, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
}
#endregion
if (stream.Left.Properties.IsIntervalFree && stream.Right.Properties.IsConstantDuration)
{
template.endPointHeap = "EndPointQueue";
}
else if (stream.Right.Properties.IsIntervalFree && stream.Left.Properties.IsConstantDuration)
{
template.endPointHeap = "EndPointQueue";
}
else if (stream.Left.Properties.IsConstantDuration && stream.Right.Properties.IsConstantDuration &&
stream.Left.Properties.ConstantDurationLength == stream.Right.Properties.ConstantDurationLength)
{
template.endPointHeap = "EndPointQueue";
}
else
{
template.endPointHeap = "EndPointHeap";
}
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(keyType, leftType, rightType, resultType);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (keyType.IsAnonymousType())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The key type for an equi-join is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n";
}
var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType, resultType);
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(leftType.GetAnonymousTypes());
list.AddRange(rightType.GetAnonymousTypes());
list.AddRange(resultType.GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Text;
using System.Threading.Tasks;
using System.Reflection;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
");
if (this.keyType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyType.Namespace));
this.Write(";\r\n");
}
if (this.payloadType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.payloadType.Namespace));
this.Write(";\r\n");
}
if (this.resultType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.resultType.Namespace));
this.Write(";\r\n");
}
this.Write("\r\n// TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TPayload: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("\r\n// TResult: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("\r\n// Source Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.fields.Select(f => f.OriginalName))));
this.Write("\r\n// Destination Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.resultFields.Select(f => f.OriginalName))));
this.Write("\r\n// Computed Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.computedFields.Keys.Select(f => f.OriginalName))));
this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : UnaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">\r\n{\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(MemoryPoolGenericParameters));
this.Write(" pool;\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGene" +
"rator;\r\n\r\n [DataMember]\r\n private int iter;\r\n\r\n private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> genericBatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters));
this.Write(" batch;\r\n\r\n // Fields used to point directly to the arrays within the result b" +
"atch\r\n private long[] dest_vsync;\r\n private long[] dest_vother;\r\n priva" +
"te ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("[] destkey;\r\n private int[] dest_hash;\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator)\r\n" +
" : base(stream, observer)\r\n {\r\n pool = MemoryManager.GetMemoryP" +
"ool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">() as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(MemoryPoolGenericParameters));
this.Write(@";
this.queryPlanGenerator = queryPlanGenerator;
MyAllocate();
}
public override void ProduceQueryPlan(PlanNode previous)
{
Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));
}
private void MyAllocate()
{
pool.Get(out genericBatch);
genericBatch.Allocate();
this.batch = genericBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters));
this.Write(@";
this.UpdatePointers();
}
protected override void UpdatePointers()
{
// Assign pointers to bookkeeping arrays
dest_vsync = this.batch.vsync.col;
dest_vother = this.batch.vother.col;
destkey = this.batch.key.col;
dest_hash = this.batch.hash.col;
}
protected override void DisposeState() => this.batch.Free();
public override unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("> _inBatch)\r\n {\r\n var batch = _inBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TPayload));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTPayloadGenericParameters));
this.Write(";\r\n\r\n var count = batch.Count;\r\n this.batch.iter = batch.iter;\r\n\r\n " +
" // Create locals that point directly to the arrays within the columns in " +
"the destination batch.\r\n\r\n ");
foreach (var f in this.computedFields.Keys)
{
this.Write(" ");
if (f.OptimizeString())
{
this.Write("\r\n var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n ");
}
else
{
this.Write("\r\n var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n ");
}
this.Write(" ");
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the source batch.\r\n ");
foreach (var f in this.fields)
{
this.Write("\r\n ");
if (f.canBeFixed)
{
this.Write("\r\n fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n ");
}
else
{
this.Write("\r\n var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n ");
}
this.Write(" ");
}
this.Write("\r\n var srckey = batch.key.col;\r\n ");
if (this.hasKey)
{
this.Write("\r\n var key_col = srckey; // hack until MakeColumnOriented is fixed\r\n " +
" ");
}
this.Write(@"
fixed (long* src_bv = batch.bitvector.col, src_vsync = batch.vsync.col, src_vother = batch.vother.col)
fixed (int* src_hash = batch.hash.col)
{
for (int i = 0; i < count; i++)
{
if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0)
{
");
if (this.StartEdgeParameterName != null)
{
this.Write("\r\n var ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.StartEdgeParameterName));
this.Write(" = src_vsync[i] < src_vother[i] ? src_vsync[i] : src_vother[i];\r\n " +
" ");
}
this.Write("\r\n ");
if (this.useEnumerator)
{
this.Write("\r\n var enumerator = ");
this.Write(this.ToStringHelper.ToStringWithCulture(transformedSelectorAsSource));
this.Write(".GetEnumerator();\r\n while (enumerator.MoveNext())\r\n " +
" ");
}
else
{
this.Write(" ");
if (!this.enumerableRepeatSelector)
{
this.Write("\r\n var e_prime = ");
this.Write(this.ToStringHelper.ToStringWithCulture(transformedSelectorAsSource));
this.Write(";\r\n ");
}
this.Write("\r\n ");
if (this.keyParameterName != null)
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyParameterName));
this.Write(" = srckey[i];\r\n ");
}
this.Write(" for (int _x = 0; _x < ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.loopCounter));
this.Write("; _x++)\r\n ");
}
this.Write("\r\n {\r\n dest_vsync[iter] = src_vsync[i];" +
"\r\n dest_vother[iter] = src_vother[i];\r\n " +
" ");
if (this.useEnumerator)
{
this.Write("\r\n this.batch[iter] = enumerator.Current;\r\n " +
" ");
}
else if (this.enumerableRepeatSelector)
{
this.Write("\r\n ");
foreach (var kv in this.computedFields)
{
var f = kv.Key;
var v = kv.Value.ExpressionToCSharp();
this.Write(" ");
if (f.OptimizeString())
{
this.Write("\r\n dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".AddString(");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(");\r\n ");
}
else
{
this.Write("\r\n dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[iter] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(";\r\n ");
}
this.Write(" ");
}
this.Write(" ");
}
else
{
this.Write("\r\n this.batch[iter] = e_prime;\r\n ");
}
this.Write(@"
destkey[iter] = srckey[i];
dest_hash[iter] = src_hash[i];
iter++;
if (iter == Config.DataBatchSize)
{
FlushContents();
// Create locals that point directly to the arrays within the columns in the destination batch.
");
foreach (var f in this.computedFields.Keys)
{
this.Write(" ");
if (f.OptimizeString())
{
this.Write("\r\n dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n ");
}
else
{
this.Write("\r\n dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n ");
}
this.Write(" ");
}
this.Write("\r\n this.batch.iter = batch.iter;\r\n " +
" }\r\n }\r\n ");
if (this.useEnumerator)
{
this.Write("\r\n enumerator.Dispose();\r\n ");
}
this.Write(@" }
else if (src_vother[i] < 0)
{
dest_vsync[iter] = src_vsync[i];
dest_vother[iter] = src_vother[i];
destkey[iter] = default;
this.batch[iter] = default;
dest_hash[iter] = src_hash[i];
this.batch.bitvector.col[(iter) >> 6] |= (1L << ((iter) & 0x3f));
iter++;
if (iter == Config.DataBatchSize)
{
FlushContents();
// Create locals that point directly to the arrays within the columns in the destination batch.
");
foreach (var f in this.computedFields.Keys)
{
this.Write(" ");
if (f.OptimizeString())
{
this.Write("\r\n dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n ");
}
else
{
this.Write("\r\n dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n ");
}
this.Write(" ");
}
this.Write("\r\n this.batch.iter = batch.iter;\r\n }\r\n " +
" }\r\n }\r\n\r\n } // end src_hash, src_bv, src_vsync," +
" src_vother\r\n\r\n ");
foreach (var f in this.fields.Where(fld => fld.canBeFixed))
{
this.Write("\r\n }\r\n ");
}
this.Write(@"
batch.Free();
}
protected override void FlushContents()
{
if (iter == 0) return;
this.batch.Count = iter;
this.batch.Seal();
this.Observer.OnNext(this.batch);
iter = 0;
MyAllocate();
}
public override int CurrentlyBufferedOutputCount => iter;
public override int CurrentlyBufferedInputCount => 0;
}
");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using System.Threading;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Aggregates;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
// TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TLeft: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("\r\n// TRight: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : BinaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(">\r\n{\r\n private readonly MemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> pool;\r\n\r\n [SchemaSerialization]\r\n private readonly Expression<Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", bool>> keyComparer;\r\n [SchemaSerialization]\r\n private readonly Expression" +
"<Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(", bool>> leftComparer;\r\n\r\n private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> genericOutputBatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TLeft));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(@" output;
[DataMember]
private FastMap<LeftInterval> leftIntervalMap = new FastMap<LeftInterval>();
[DataMember]
private FastMap<LeftEdge> leftEdgeMap = new FastMap<LeftEdge>();
[DataMember]
private RemovableEndPointHeap leftEndPointHeap;
[DataMember]
private long nextLeftTime = long.MinValue;
[DataMember]
private long nextRightTime = long.MinValue;
[DataMember]
private long currTime = long.MinValue;
private readonly Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator;
");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n [Obsolete(\"Used only by serialization. Do not call directly.\")]\r\n publ" +
"ic ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> observer,\r\n IEqualityComparerExpression<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("> keyEqualityComparer,\r\n IEqualityComparerExpression<");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(@"> leftComparer,
Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator)
: base(stream, observer)
{
this.queryPlanGenerator = queryPlanGenerator;
this.keyComparer = keyEqualityComparer.GetEqualsExpr();
this.leftComparer = leftComparer.GetEqualsExpr();
this.leftEndPointHeap = new RemovableEndPointHeap();
this.pool = MemoryManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(">(true /*stream.Properties.IsColumnar*/);\r\n ");
GetOutputBatch();
this.Write(@" }
public override int CurrentlyBufferedOutputCount => output.Count;
public override int CurrentlyBufferedLeftInputCount => base.CurrentlyBufferedLeftInputCount + leftEdgeMap.Count + leftIntervalMap.Count;
public override int CurrentlyBufferedRightInputCount => base.CurrentlyBufferedRightInputCount;
protected override void ProduceBinaryQueryPlan(PlanNode left, PlanNode right)
{
this.Observer.ProduceQueryPlan(queryPlanGenerator(left, right, this));
}
protected override void DisposeState() => this.output.Free();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessBothBatches(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> genericLeftBatch, StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("> genericRightBatch, out bool leftBatchDone, out bool rightBatchDone, out bool le" +
"ftBatchFree, out bool rightBatchFree)\r\n {\r\n var leftBatch = genericLef" +
"tBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, leftType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(";\r\n var rightBatch = genericRightBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, rightType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTRightGenericParameters));
this.Write(@";
leftBatchFree = rightBatchFree = true;
if (!GoToVisibleRow(leftBatch))
{
leftBatchDone = true;
rightBatchDone = false;
return;
}
nextLeftTime = leftBatch.vsync.col[leftBatch.iter];
if (!GoToVisibleRow(rightBatch))
{
leftBatchDone = false;
rightBatchDone = true;
return;
}
nextRightTime = rightBatch.vsync.col[rightBatch.iter];
while (true)
{
if (nextLeftTime <= nextRightTime)
{
UpdateTime(nextLeftTime);
/*
ProcessLeftEvent(
nextLeftTime,
leftBatch.vother.col[leftBatch.iter],
ref leftBatch.key.col[leftBatch.iter],
leftBatch[leftBatch.iter],
leftBatch.hash.col[leftBatch.iter]);
*/
");
ProcessLeftEvent("nextLeftTime", "leftBatch.vother.col[leftBatch.iter]", "leftBatch.key.col[leftBatch.iter]", "leftBatch", "leftBatch.iter", "leftBatch.hash.col[leftBatch.iter]");
this.Write(@"
leftBatch.iter++;
if (!GoToVisibleRow(leftBatch))
{
leftBatchDone = true;
rightBatchDone = false;
return;
}
nextLeftTime = leftBatch.vsync.col[leftBatch.iter];
}
else
{
UpdateTime(nextRightTime);
ProcessRightEvent(
nextRightTime,
rightBatch.vother.col[rightBatch.iter],
ref rightBatch.key.col[rightBatch.iter],
rightBatch.hash.col[rightBatch.iter]);
rightBatch.iter++;
if (!GoToVisibleRow(rightBatch))
{
leftBatchDone = false;
rightBatchDone = true;
return;
}
nextRightTime = rightBatch.vsync.col[rightBatch.iter];
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessLeftBatch(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> genericBatch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var b" +
"atch = genericBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, leftType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(@";
isBatchFree = true;
while (true)
{
if (!GoToVisibleRow(batch))
{
isBatchDone = true;
return;
}
nextLeftTime = batch.vsync.col[batch.iter];
if (nextLeftTime > nextRightTime)
{
isBatchDone = false;
return;
}
UpdateTime(nextLeftTime);
/*
ProcessLeftEvent(
nextLeftTime,
batch.vother.col[batch.iter],
ref batch.key.col[batch.iter],
batch[batch.iter],
batch.hash.col[batch.iter]);
*/
");
ProcessLeftEvent("nextLeftTime", "batch.vother.col[batch.iter]", "batch.key.col[batch.iter]", "batch", "batch.iter", "batch.hash.col[batch.iter]");
this.Write("\r\n batch.iter++;\r\n }\r\n }\r\n\r\n [MethodImpl(MethodImplOption" +
"s.AggressiveInlining)]\r\n protected override void ProcessRightBatch(StreamMess" +
"age<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("> genericBatch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var b" +
"atch = genericBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, rightType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTRightGenericParameters));
this.Write(@";
isBatchFree = true;
while (true)
{
if (!GoToVisibleRow(batch))
{
isBatchDone = true;
return;
}
nextRightTime = batch.vsync.col[batch.iter];
if (nextRightTime > nextLeftTime)
{
isBatchDone = false;
return;
}
UpdateTime(nextRightTime);
ProcessRightEvent(
nextRightTime,
batch.vother.col[batch.iter],
ref batch.key.col[batch.iter],
batch.hash.col[batch.iter]);
batch.iter++;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool GoToVisibleRow<TPayload>(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", TPayload> batch)
{
while (batch.iter < batch.Count && (batch.bitvector.col[batch.iter >> 6] & (1L << (batch.iter & 0x3f))) != 0 && batch.vother.col[batch.iter] >= 0)
{
batch.iter++;
}
return (batch.iter != batch.Count);
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void UpdateTime(long time)
{
if (time != currTime)
{
currTime = time;
ReachTime();
}
}
/*
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ProcessLeftEvent(long start, long end, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(" payload, int hash)\r\n {\r\n if (start < end)\r\n {\r\n // R" +
"ow is a start edge or interval.\r\n bool isInterval = end < StreamEvent" +
".InfinitySyncTime;\r\n if (isInterval)\r\n {\r\n " +
"bool isFullyOutputtable = nextRightTime >= end;\r\n if (isFullyOutp" +
"uttable)\r\n {\r\n // Output full interval.\r\n " +
" AddToBatch(start, end, ref key, ref payload, hash);\r\n " +
" }\r\n else\r\n {\r\n // Insert in" +
"to map to remember interval.\r\n int mapIndex = leftIntervalMap" +
".Insert(hash);\r\n\r\n // Insert into heap to schedule removal at" +
" endpoint.\r\n int heapIndex = leftEndPointHeap.Insert(end, map" +
"Index);\r\n\r\n // Set value in map, also remembering heap\'s inde" +
"x.\r\n leftIntervalMap.Values[mapIndex].Initialize(start, ref k" +
"ey, ref payload, heapIndex);\r\n\r\n // Output start edge.\r\n " +
" AddToBatch(start, StreamEvent.InfinitySyncTime, ref key, ref payl" +
"oad, hash);\r\n }\r\n }\r\n else\r\n {\r\n" +
" int index = leftEdgeMap.Insert(hash);\r\n leftEdgeM" +
"ap.Values[index].Populate(start, ref key, ref payload);\r\n\r\n // Ou" +
"tput start edge.\r\n AddToBatch(start, StreamEvent.InfinitySyncTime" +
", ref key, ref payload, hash);\r\n }\r\n }\r\n else if (end =" +
"= StreamEvent.PunctuationOtherTime)\r\n {\r\n AddPunctuationToBatc" +
"h(start);\r\n }\r\n else\r\n {\r\n // Row is an end edge" +
".\r\n var leftEvents = leftEdgeMap.Find(hash);\r\n int index;\r" +
"\n while (leftEvents.Next(out index))\r\n {\r\n " +
"var leftEdge = leftEdgeMap.Values[index];\r\n var leftKey = leftEdg" +
"e.Batch.key.col[leftEdge.Index];\r\n if (AreSame(end, ref key, ref " +
"payload, ref leftEdgeMap.Values[index]))\r\n {\r\n " +
" // Output end edge.\r\n AddToBatch(start, end, ref key, ref pa" +
"yload, hash);\r\n\r\n // Remove from leftMap.\r\n " +
" leftEvents.Remove();\r\n break;\r\n }\r\n " +
" }\r\n }\r\n }\r\n*/\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlini" +
"ng)]\r\n private void ProcessRightEvent(long start, long end, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@" key, int hash)
{
if (end == StreamEvent.PunctuationOtherTime)
{
AddPunctuationToBatch(start);
return;
}
else if (start >= end)
{
// Row is an end edge, which we don't care about because the start edge would have already
// removed all joining left events.
return;
}
// Mark any matching left intervals as no longer active.
int index;
var leftIntervals = leftIntervalMap.Find(hash);
while (leftIntervals.Next(out index))
{
long leftStart = leftIntervalMap.Values[index].Start;
var leftInterval = leftIntervalMap.Values[index];
var leftIntervalKey = leftInterval.Key;
if (leftStart < start && ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyComparer("key", "leftIntervalKey")));
this.Write(@")
{
// Output end edge.
/*
AddToBatch(
start,
leftStart,
ref leftIntervalMap.Values[index].Key,
ref leftIntervalMap.Values[index].Payload,
hash);
*/
");
AddToBatch("start", "leftStart", "leftIntervalKey", "leftIntervalMap.Values[index]", "hash");
this.Write(@"
// Remove from heap and map.
leftEndPointHeap.Remove(leftIntervalMap.Values[index].HeapIndex);
leftIntervals.Remove();
}
}
// Remove any matching left edges.
var leftEdges = leftEdgeMap.Find(hash);
while (leftEdges.Next(out index))
{
var leftEdge = leftEdgeMap.Values[index];
long leftStart = leftEdge.Start;
var leftEdgeKey = leftEdge.Key;
if (leftStart < start && ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyComparer("key", "leftEdgeKey")));
this.Write(@")
{
// Output end edge.
/*
AddToBatch(
start,
leftStart,
ref leftEdgeMap.Values[index].Key,
ref leftEdgeMap.Values[index].Payload,
hash);
*/
");
AddToBatch("start", "leftStart", "leftEdgeKey", "leftEdge", "hash");
this.Write(@"
// Remove left event.
leftEdges.Remove();
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ReachTime()
{
// Carry-out all interval endpoints for left intervals that end prior or at new current time.
long endPointTime;
int index;
while (leftEndPointHeap.TryGetNextInclusive(currTime, out endPointTime, out index))
{
// Output end edge.
/*
AddToBatch(
endPointTime,
leftIntervalMap.Values[index].Start,
ref leftIntervalMap.Values[index].Key,
ref leftIntervalMap.Values[index].Payload,
leftIntervalMap.GetHash(index));
*/
var leftInterval = leftIntervalMap.Values[index];
var leftIntervalKey = leftInterval.Key;
var leftIntervalStart = leftInterval.Start;
var leftIntervalHash = leftIntervalMap.GetHash(index);
");
AddToBatch("endPointTime", "leftIntervalStart", "leftIntervalKey", "leftInterval", "leftIntervalHash");
this.Write(@"
// Remove from leftMap.
leftIntervalMap.Remove(index);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void AddPunctuationToBatch(long start)
{
if (start > lastCTI)
{
lastCTI = start;
int index = output.Count++;
output.vsync.col[index] = start;
output.vother.col[index] = StreamEvent.PunctuationOtherTime;
output.key.col[index] = default;
");
foreach (var f in this.leftFields)
{
if (f.OptimizeString())
{
this.Write(" output.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".AddString(string.Empty);\r\n");
}
else
{
this.Write(" output.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col[index] = default;\r\n");
}
}
this.Write(@" output.hash.col[index] = 0;
output.bitvector.col[index >> 6] |= (1L << (index & 0x3f));
if (output.Count == Config.DataBatchSize) FlushContents();
}
}
/*
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void AddToBatch(long start, long end, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(@" payload, int hash)
{
int index = output.Count++;
output.vsync.col[index] = start;
output.vother.col[index] = end;
output.key.col[index] = key;
output[index] = payload;
output.hash.col[index] = hash;
if (output.Count == Config.DataBatchSize)
{
this.Observer.OnNext(output);
pool.Get(out output);
output.Allocate();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool AreSame(long start, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(@" payload, ref LeftEdge active)
{
return start == active.Start && keyComparerEquals(key, active.Key) && leftComparerEquals(payload, active.Payload);
}
*/
protected override void FlushContents()
{
if (output.Count == 0) return;
this.Observer.OnNext(output);
");
GetOutputBatch();
this.Write(" }\r\n");
if (!noFields && !this.leftType.GetTypeInfo().IsValueType)
{
this.Write(" [DataContract]\r\n private struct ");
this.Write(this.ToStringHelper.ToStringWithCulture(ActiveEventType));
this.Write("\r\n {\r\n");
foreach (var f in this.leftFields)
{
this.Write(" [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax()));
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n");
}
this.Write(" }\r\n");
}
this.Write("/*\r\n [DataContract]\r\n private struct LeftInterval\r\n {\r\n [DataMemb" +
"er]\r\n public long Start;\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(" Payload;\r\n [DataMember]\r\n public int HeapIndex;\r\n\r\n [Method" +
"Impl(MethodImplOptions.AggressiveInlining)]\r\n public void Initialize(long" +
" start, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(@" payload, int heapIndex)
{
Start = start;
Key = key;
Payload = payload;
HeapIndex = heapIndex;
}
public override string ToString()
{
return ""[Start="" + Start + "", Key='"" + Key + ""', Payload='"" + Payload + ""', HeapIndex="" + HeapIndex + ""]"";
}
}
*/
[DataContract]
private struct LeftInterval
{
[DataMember]
public long Start;
[DataMember]
public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n [DataMember]\r\n public int HeapIndex;\r\n [DataMember]\r" +
"\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(ActiveEventType));
this.Write(" Payload;\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " +
"public void Initialize(long start, ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TLeft));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(" batch, int index, int heapIndex)\r\n {\r\n Start = start;\r\n " +
" Key = batch.key.col[index];\r\n");
foreach (var f in this.leftFields)
{
this.Write(" this.Payload.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(" = ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index")));
this.Write(";\r\n");
}
this.Write(@" HeapIndex = heapIndex;
}
public override string ToString()
{
//return string.Format(""[Start={0}, Key='{1}', Payload='{2}', HeapIndex={3}"", Start, Key, Batch[Index], HeapIndex);
System.Text.StringBuilder sb = new System.Text.StringBuilder();
sb.AppendFormat(""[Start={0}"", Start);
sb.AppendFormat("", Key='{0}'"", Key);
");
foreach (var f in this.leftFields)
{
this.Write(" sb.AppendFormat(\", ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write("={0}, \", Payload.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(");\r\n ");
}
this.Write(@" sb.AppendFormat("", Key='{0}'"", HeapIndex);
sb.Append(""]"");
return sb.ToString();
}
}
/*
[DataContract]
private struct LeftEdge
{
[DataMember]
public long Start;
[DataMember]
public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(" Payload;\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " +
"public void Populate(long start, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(@" payload)
{
Start = start;
Key = key;
Payload = payload;
}
public override string ToString()
{
return ""[Start="" + Start + "", Key='"" + Key + ""', Payload='"" + Payload + ""]"";
}
}
*/
[DataContract]
private struct LeftEdge
{
[DataMember]
public long Start;
[DataMember]
public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(ActiveEventType));
this.Write(" Payload;\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " +
"public void Populate(long start, ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TLeft));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(" batch, int index)\r\n {\r\n Start = start;\r\n Key = batc" +
"h.key.col[index];\r\n");
foreach (var f in this.leftFields)
{
this.Write(" this.Payload.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(" = ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index")));
this.Write(";\r\n");
}
this.Write(@" }
public override string ToString()
{
System.Text.StringBuilder sb = new System.Text.StringBuilder();
sb.AppendFormat(""[Start={0}"", Start);
sb.AppendFormat("", Key='{0}'"", Key);
");
foreach (var f in this.leftFields)
{
this.Write(" sb.AppendFormat(\", ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write("={0}, \", Payload.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(");\r\n ");
}
this.Write(" sb.Append(\"]\");\r\n return sb.ToString();\r\n //ret" +
"urn string.Format(\"[Start={0}, Key=\'{1}\', Payload=\'{2}\'\", Start, KeyColumn.col[R" +
"owIndex], Batch[RowIndex]);\r\n }\r\n }\r\n}\r\n\r\n");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using System.Threading;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Aggregates;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
[assembly: IgnoresAccessChecksTo(""Microsoft.StreamProcessing"")]
// TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TLeft: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("\r\n// TRight: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("\r\n\r\n");
var leftBatchTypeName = Transformer.GetBatchClassName(this.keyType, this.leftType) + this.TKeyTLeftGenericParameters;
var rightBatchTypeName = Transformer.GetBatchClassName(this.keyType, this.rightType) + this.TKeyTRightGenericParameters;
this.Write("\r\n[DataContract]\r\n[KnownType(typeof(EndPointHeap))]\r\n[KnownType(typeof(EndPointQu" +
"eue))]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : BinaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(">, IBinaryObserver\r\n{\r\n private const long NotActive = long.MaxValue;\r\n pri" +
"vate readonly MemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> pool;\r\n\r\n [SchemaSerialization]\r\n private readonly Expression<Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", bool>> keyComparer;\r\n [SchemaSerialization]\r\n private readonly Expression" +
"<Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(", bool>> leftComparer;\r\n\r\n private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> genericOutputBatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TLeft));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(@" output;
[DataMember]
private FastMap<LeftEvent> leftIntervalMap = new FastMap<LeftEvent>();
[DataMember]
private FastMap<LeftEvent> leftEdgeMap = new FastMap<LeftEvent>();
[DataMember]
private IEndPointOrderer leftEndPointHeap;
[DataMember]
private FastMap<RightEvent> rightMap = new FastMap<RightEvent>();
[DataMember]
private FastStack<QueuedEndEdge> rightEndEdges = new FastStack<QueuedEndEdge>();
[DataMember]
private IEndPointOrderer rightEndPointHeap;
[DataMember]
private long nextLeftTime = long.MinValue;
[DataMember]
private long nextRightTime = long.MinValue;
[DataMember]
private long currTime = long.MinValue;
private readonly Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator;
");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n [Obsolete(\"Used only by serialization. Do not call directly.\")]\r\n publ" +
"ic ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> observer,\r\n IEqualityComparerExpression<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("> keyEqualityComparer,\r\n IEqualityComparerExpression<");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(@"> leftComparer,
Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator,
bool leftIsConstantDuration,
bool rightIsConstantDuration)
: base(stream, observer)
{
this.queryPlanGenerator = queryPlanGenerator;
this.keyComparer = keyEqualityComparer.GetEqualsExpr();
this.leftComparer = leftComparer.GetEqualsExpr();
if (leftIsConstantDuration) this.leftEndPointHeap = new EndPointQueue();
else this.leftEndPointHeap = new EndPointHeap();
if (rightIsConstantDuration) this.rightEndPointHeap = new EndPointQueue();
else this.rightEndPointHeap = new EndPointHeap();
this.pool = MemoryManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(">(true);\r\n GetOutputBatch();\r\n }\r\n\r\n public override int CurrentlyBu" +
"fferedOutputCount => this.output.Count;\r\n public override int CurrentlyBuffer" +
"edLeftInputCount => base.CurrentlyBufferedLeftInputCount + this.leftEdgeMap.Coun" +
"t + this.leftIntervalMap.Count;\r\n public override int CurrentlyBufferedRightI" +
"nputCount => base.CurrentlyBufferedRightInputCount + this.rightEndEdges.Count + " +
"this.rightMap.Count;\r\n\r\n protected override void ProduceBinaryQueryPlan(PlanN" +
"ode left, PlanNode right)\r\n {\r\n Observer.ProduceQueryPlan(queryPlanGen" +
"erator(left, right, this));\r\n }\r\n\r\n protected override void DisposeState()" +
"\r\n {\r\n this.output.Free();\r\n\r\n var allLeftEdgeTraverser = this." +
"leftEdgeMap.Traverse();\r\n allLeftEdgeTraverser.currIndex = 0;\r\n in" +
"t x, y;\r\n while (allLeftEdgeTraverser.Next(out x, out y))\r\n {\r\n " +
" var leftInterval = this.leftEdgeMap.Values[x];\r\n }\r\n var " +
"allLeftEdgeInvisibleTraverser = this.leftEdgeMap.TraverseInvisible();\r\n w" +
"hile (allLeftEdgeInvisibleTraverser.Next(out x, out y))\r\n {\r\n " +
"var leftEdge = this.leftEdgeMap.Values[x];\r\n }\r\n\r\n var allLeftInte" +
"rvalTraverser = this.leftIntervalMap.Traverse();\r\n allLeftIntervalTravers" +
"er.currIndex = 0;\r\n while (allLeftIntervalTraverser.Next(out x, out y))\r\n" +
" {\r\n var leftInterval = this.leftIntervalMap.Values[x];\r\n " +
" }\r\n var allLeftIntervalInvisibleTraverser = this.leftIntervalMap.Trave" +
"rseInvisible();\r\n while (allLeftIntervalInvisibleTraverser.Next(out x, ou" +
"t y))\r\n {\r\n var leftInterval = this.leftIntervalMap.Values[x];" +
"\r\n }\r\n }\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " +
" protected override void ProcessBothBatches(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> genericLeftBatch, StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("> genericRightBatch, out bool leftBatchDone, out bool rightBatchDone, out bool le" +
"ftBatchFree, out bool rightBatchFree)\r\n {\r\n var leftBatch = genericLef" +
"tBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(";\r\n var rightBatch = genericRightBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName));
this.Write(";\r\n\r\n leftBatchFree = rightBatchFree = true;\r\n if (!GoToVisibleRow(" +
"leftBatch))\r\n {\r\n leftBatchDone = true;\r\n rightBatc" +
"hDone = false;\r\n return;\r\n }\r\n\r\n this.nextLeftTime = le" +
"ftBatch.vsync.col[leftBatch.iter];\r\n\r\n if (!GoToVisibleRow(rightBatch))\r\n" +
" {\r\n leftBatchDone = false;\r\n rightBatchDone = true" +
";\r\n return;\r\n }\r\n\r\n this.nextRightTime = rightBatch.vsy" +
"nc.col[rightBatch.iter];\r\n\r\n while (true)\r\n {\r\n if (thi" +
"s.nextLeftTime <= this.nextRightTime)\r\n {\r\n UpdateTime" +
"(this.nextLeftTime);\r\n\r\n ProcessLeftEvent(\r\n t" +
"his.nextLeftTime,\r\n leftBatch.vother.col[leftBatch.iter],\r\n " +
" ref leftBatch.key.col[leftBatch.iter],\r\n le" +
"ftBatch,\r\n leftBatch.iter,\r\n leftBatch.has" +
"h.col[leftBatch.iter]);\r\n\r\n leftBatch.iter++;\r\n\r\n " +
"if (!GoToVisibleRow(leftBatch))\r\n {\r\n leftBatc" +
"hDone = true;\r\n rightBatchDone = false;\r\n " +
"return;\r\n }\r\n\r\n this.nextLeftTime = leftBatch.vsyn" +
"c.col[leftBatch.iter];\r\n }\r\n else\r\n {\r\n " +
" UpdateTime(this.nextRightTime);\r\n\r\n ProcessRightEvent(\r\n " +
" this.nextRightTime,\r\n rightBatch.vother.co" +
"l[rightBatch.iter],\r\n ref rightBatch.key.col[rightBatch.iter]" +
",\r\n rightBatch.hash.col[rightBatch.iter]);\r\n\r\n " +
" rightBatch.iter++;\r\n\r\n if (!GoToVisibleRow(rightBatch))\r\n " +
" {\r\n leftBatchDone = false;\r\n righ" +
"tBatchDone = true;\r\n return;\r\n }\r\n\r\n " +
" this.nextRightTime = rightBatch.vsync.col[rightBatch.iter];\r\n }" +
"\r\n }\r\n }\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " +
" protected override void ProcessLeftBatch(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> genericBatch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var b" +
"atch = genericBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(@";
isBatchFree = true;
while (true)
{
if (!GoToVisibleRow(batch))
{
isBatchDone = true;
return;
}
this.nextLeftTime = batch.vsync.col[batch.iter];
if (this.nextLeftTime > this.nextRightTime)
{
isBatchDone = false;
return;
}
UpdateTime(this.nextLeftTime);
ProcessLeftEvent(
this.nextLeftTime,
batch.vother.col[batch.iter],
ref batch.key.col[batch.iter],
batch,
batch.iter,
batch.hash.col[batch.iter]);
batch.iter++;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessRightBatch(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("> genericBatch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var b" +
"atch = genericBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName));
this.Write(@";
isBatchFree = true;
while (true)
{
if (!GoToVisibleRow(batch))
{
isBatchDone = true;
return;
}
this.nextRightTime = batch.vsync.col[batch.iter];
if (this.nextRightTime > this.nextLeftTime)
{
isBatchDone = false;
return;
}
UpdateTime(this.nextRightTime);
ProcessRightEvent(
this.nextRightTime,
batch.vother.col[batch.iter],
ref batch.key.col[batch.iter],
batch.hash.col[batch.iter]);
batch.iter++;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool GoToVisibleRow<TPayload>(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", TPayload> batch)
{
while (batch.iter < batch.Count && (batch.bitvector.col[batch.iter >> 6] & (1L << (batch.iter & 0x3f))) != 00 && batch.vother.col[batch.iter] >= 0)
{
batch.iter++;
}
if (batch.iter == batch.Count)
{
return false;
}
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void UpdateTime(long time)
{
if (time != this.currTime)
{
LeaveTime();
this.currTime = time;
ReachTime();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ProcessLeftEvent(long start, long end, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(" batch, int batchIndex, int hash)\r\n {\r\n if (start < end)\r\n {\r\n " +
" // Row is a start edge or interval.\r\n bool isProcessable = " +
"this.nextRightTime > start;\r\n bool isInterval = end < StreamEvent.Inf" +
"initySyncTime;\r\n var map = isInterval ? this.leftIntervalMap : this.l" +
"eftEdgeMap;\r\n if (isProcessable)\r\n {\r\n int " +
"matchingRight;\r\n if (FindOnRight(ref key, hash, out matchingRight" +
"))\r\n {\r\n // Row joins with something on right," +
" so not currently visible.\r\n int index = map.Insert(hash);\r\n " +
" map.Values[index].Populate(start, NotActive, end, ref key, ba" +
"tch, batchIndex);\r\n if (isInterval)\r\n {\r\n " +
" this.leftEndPointHeap.Insert(end, index);\r\n " +
" }\r\n }\r\n else\r\n {\r\n " +
" // Row does not join (and it is processable).\r\n bool" +
" isFullyOutputtable = isInterval && this.nextRightTime >= end;\r\n " +
" if (isFullyOutputtable)\r\n {\r\n // W" +
"ill never join because right has advanced beyond endtime, so output interval.\r\n " +
" AddToBatch(start, end, ref key, batch, batchIndex, hash);" +
"\r\n }\r\n else\r\n {\r\n " +
" // Output start edge.\r\n int index = map" +
".Insert(hash);\r\n map.Values[index].Populate(start, start," +
" end, ref key, batch, batchIndex);\r\n AddToBatch(start, St" +
"reamEvent.InfinitySyncTime, ref key, batch, batchIndex, hash);\r\n " +
" if (isInterval)\r\n {\r\n " +
"this.leftEndPointHeap.Insert(end, index);\r\n }\r\n " +
" }\r\n }\r\n }\r\n else\r\n {\r\n" +
" // Row is not yet processable, so insert as invisible.\r\n " +
" int index = map.InsertInvisible(hash);\r\n map.Values[index" +
"].Populate(start, NotActive, end, ref key, batch, batchIndex);\r\n }\r\n " +
" }\r\n else if (end == StreamEvent.PunctuationOtherTime)\r\n {\r\n" +
" AddPunctuationToBatch(start);\r\n }\r\n else\r\n {\r\n " +
" // Row is an end edge.\r\n\r\n // Remove from this.leftEdgeMap" +
".\r\n var leftEvents = this.leftEdgeMap.Find(hash);\r\n int in" +
"dex;\r\n while (leftEvents.Next(out index))\r\n {\r\n " +
" var leftEdge = this.leftEdgeMap.Values[index];\r\n if (AreSame" +
"(end, StreamEvent.InfinitySyncTime, ref key, batch, batchIndex, ref leftEdge))\r\n" +
" {\r\n long currentStart = leftEdge.CurrentStart" +
";\r\n if (currentStart != NotActive)\r\n {\r\n " +
" // Matching left start edge is currently visible, so outpu" +
"t end edge.\r\n AddToBatch(start, currentStart, ref key, ba" +
"tch, batchIndex, hash);\r\n }\r\n\r\n leftEvents" +
".Remove();\r\n break;\r\n }\r\n }\r\n " +
" }\r\n }\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n privat" +
"e void ProcessRightEvent(long start, long end, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, int hash)\r\n {\r\n if (start < end)\r\n {\r\n // Row i" +
"s a start edge or interval.\r\n int index;\r\n if (FindOnRight" +
"(ref key, hash, out index))\r\n {\r\n // Corresponding key" +
" already exists in map, so any joining on left and already not active.\r\n " +
" this.rightMap.Values[index].Count++;\r\n }\r\n else\r\n " +
" {\r\n // First instance of this key, so insert and make " +
"any joining left entries not active.\r\n index = this.rightMap.Inse" +
"rt(hash);\r\n this.rightMap.Values[index].Initialize(ref key);\r\n " +
" MakeMatchingLeftInvisible(start, ref key, hash);\r\n }\r\n\r\n" +
" if (end != StreamEvent.InfinitySyncTime)\r\n {\r\n " +
" // Row is an interval, so schedule removal of interval.\r\n th" +
"is.rightEndPointHeap.Insert(end, index);\r\n }\r\n }\r\n else" +
" if (end == StreamEvent.PunctuationOtherTime)\r\n {\r\n AddPunctua" +
"tionToBatch(start);\r\n }\r\n else\r\n {\r\n // Row is a" +
"n end edge.\r\n\r\n // Queue for removal when time advances.\r\n " +
" int index = this.rightEndEdges.Push();\r\n this.rightEndEdges.Values[i" +
"ndex].Populate(ref key, hash);\r\n }\r\n }\r\n\r\n [MethodImpl(MethodImplOp" +
"tions.AggressiveInlining)]\r\n private void LeaveTime()\r\n {\r\n // Carr" +
"y-out all queued end edges for right events.\r\n int index;\r\n int ha" +
"sh;\r\n for (int i = 0; i < this.rightEndEdges.Count; i++)\r\n {\r\n " +
" hash = this.rightEndEdges.Values[i].Hash;\r\n if (FindOnRight(r" +
"ef this.rightEndEdges.Values[i].Key, hash, out index))\r\n {\r\n " +
" int count = this.rightMap.Values[index].Count - 1;\r\n if (c" +
"ount > 0)\r\n {\r\n this.rightMap.Values[index].Co" +
"unt = count;\r\n }\r\n else\r\n {\r\n " +
" MakeMatchingLeftVisible(this.currTime, ref this.rightMap.Values[in" +
"dex].Key, hash);\r\n this.rightMap.Remove(index);\r\n " +
" }\r\n }\r\n }\r\n\r\n this.rightEndEdges.Clear();\r\n\r\n " +
" // Actually insert all pending left start intervals.\r\n var leftEvents =" +
" this.leftIntervalMap.TraverseInvisible();\r\n while (leftEvents.Next(out i" +
"ndex, out hash))\r\n {\r\n var leftInterval = this.leftIntervalMap" +
".Values[index];\r\n long end = leftInterval.End;\r\n int match" +
"ingRight;\r\n if (FindOnRight(ref leftInterval.Key, hash, out matchingR" +
"ight))\r\n {\r\n leftEvents.MakeVisible();\r\n " +
" this.leftEndPointHeap.Insert(end, index);\r\n }\r\n else\r\n " +
" {\r\n // Row does not join.\r\n bool isFully" +
"Outputtable = this.nextRightTime >= end;\r\n if (isFullyOutputtable" +
")\r\n {\r\n AddToBatch(\r\n t" +
"his.currTime,\r\n end,\r\n ref leftInt" +
"erval.Key,\r\n ref leftInterval,\r\n h" +
"ash);\r\n leftEvents.Remove();\r\n }\r\n " +
" else\r\n {\r\n leftEvents.MakeVisible();\r\n " +
" this.leftIntervalMap.Values[index].CurrentStart = this.currTime;" +
"\r\n AddToBatch(\r\n this.currTime,\r\n " +
" StreamEvent.InfinitySyncTime,\r\n ref th" +
"is.leftIntervalMap.Values[index].Key,\r\n ref this.leftInte" +
"rvalMap.Values[index],\r\n hash);\r\n\r\n th" +
"is.leftEndPointHeap.Insert(end, index);\r\n }\r\n }\r\n " +
" }\r\n\r\n // Actually insert all pending left start edges.\r\n leftEv" +
"ents = this.leftEdgeMap.TraverseInvisible();\r\n while (leftEvents.Next(out" +
" index, out hash))\r\n {\r\n int matchingRight;\r\n if (!" +
"FindOnRight(ref this.leftEdgeMap.Values[index].Key, hash, out matchingRight))\r\n " +
" {\r\n // Row does not join, so output start edge.\r\n " +
" this.leftEdgeMap.Values[index].CurrentStart = this.currTime;\r\n " +
" AddToBatch(\r\n this.currTime,\r\n St" +
"reamEvent.InfinitySyncTime,\r\n ref this.leftEdgeMap.Values[ind" +
"ex].Key,\r\n ref this.leftEdgeMap.Values[index],\r\n " +
" hash);\r\n }\r\n\r\n leftEvents.MakeVisible();\r\n }" +
"\r\n }\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n private v" +
"oid ReachTime()\r\n {\r\n // Carry-out all interval endpoints for left int" +
"ervals that end prior to (or at) new current time.\r\n long leftTime;\r\n " +
" int leftIndex;\r\n if (!this.leftEndPointHeap.TryPeekNext(out leftTime," +
" out leftIndex))\r\n {\r\n leftTime = long.MaxValue;\r\n }\r\n\r" +
"\n long rightTime;\r\n int rightIndex;\r\n if (!this.rightEndPoi" +
"ntHeap.TryPeekNext(out rightTime, out rightIndex))\r\n {\r\n right" +
"Time = long.MaxValue;\r\n }\r\n\r\n while (leftTime <= this.currTime || " +
"rightTime < this.currTime)\r\n {\r\n if (leftTime <= rightTime)\r\n " +
" {\r\n // Always process left end-points first as they ma" +
"y end intervals that right end-points may\r\n // try to make visibl" +
"e.\r\n var leftInterval = this.leftIntervalMap.Values[leftIndex];\r\n" +
" long currentStart = leftInterval.CurrentStart;\r\n " +
"if (currentStart != NotActive)\r\n {\r\n // Matchi" +
"ng left start edge is currently visible, so output end edge.\r\n " +
" AddToBatch(\r\n leftTime,\r\n curren" +
"tStart,\r\n ref leftInterval.Key,\r\n " +
"ref leftInterval,\r\n this.leftIntervalMap.GetHash(leftInde" +
"x));\r\n }\r\n\r\n this.leftIntervalMap.Remove(leftIndex" +
");\r\n\r\n this.leftEndPointHeap.RemoveTop();\r\n if (!t" +
"his.leftEndPointHeap.TryPeekNext(out leftTime, out leftIndex))\r\n " +
"{\r\n leftTime = long.MaxValue;\r\n }\r\n " +
" }\r\n else\r\n {\r\n // Process right end-point " +
"up to but not including the current time.\r\n int count = this.righ" +
"tMap.Values[rightIndex].Count - 1;\r\n if (count > 0)\r\n " +
" {\r\n this.rightMap.Values[rightIndex].Count = count;\r\n " +
" }\r\n else\r\n {\r\n Make" +
"MatchingLeftVisible(rightTime, ref this.rightMap.Values[rightIndex].Key, this.ri" +
"ghtMap.GetHash(rightIndex));\r\n this.rightMap.Remove(rightInde" +
"x);\r\n }\r\n\r\n this.rightEndPointHeap.RemoveTop();\r\n " +
" if (!this.rightEndPointHeap.TryPeekNext(out rightTime, out rightI" +
"ndex))\r\n {\r\n rightTime = long.MaxValue;\r\n " +
" }\r\n }\r\n }\r\n }\r\n\r\n [MethodImpl(MethodImplOptio" +
"ns.AggressiveInlining)]\r\n private bool FindOnRight(ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, int hash, out int index)\r\n {\r\n int rightIndex;\r\n var right" +
"Events = this.rightMap.Find(hash);\r\n while (rightEvents.Next(out rightInd" +
"ex))\r\n {\r\n if (");
this.Write(this.ToStringHelper.ToStringWithCulture(keyComparer("key", "this.rightMap.Values[rightIndex].Key")));
this.Write(@")
{
index = rightIndex;
return true;
}
}
index = 0;
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void MakeMatchingLeftInvisible(long time, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, int hash)\r\n {\r\n // Make matching left intervals invisible.\r\n " +
" int index;\r\n var leftEvents = this.leftIntervalMap.Find(hash);\r\n " +
" while (leftEvents.Next(out index))\r\n {\r\n if (");
this.Write(this.ToStringHelper.ToStringWithCulture(keyComparer("key", "this.leftIntervalMap.Values[index].Key")));
this.Write(@")
{
// Output end edge.
AddToBatch(
time,
this.leftIntervalMap.Values[index].CurrentStart,
ref this.leftIntervalMap.Values[index].Key,
ref this.leftIntervalMap.Values[index],
hash);
// Mark left event as not visible.
this.leftIntervalMap.Values[index].CurrentStart = NotActive;
}
}
// Make matching left edges invisible.
leftEvents = this.leftEdgeMap.Find(hash);
while (leftEvents.Next(out index))
{
if (");
this.Write(this.ToStringHelper.ToStringWithCulture(keyComparer("key", "this.leftEdgeMap.Values[index].Key")));
this.Write(@")
{
// Output end edge.
AddToBatch(
time,
this.leftEdgeMap.Values[index].CurrentStart,
ref this.leftEdgeMap.Values[index].Key,
ref this.leftEdgeMap.Values[index],
hash);
// Mark left event as not visible.
this.leftEdgeMap.Values[index].CurrentStart = NotActive;
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void MakeMatchingLeftVisible(long time, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, int hash)\r\n {\r\n // Make matching left intervals visible.\r\n " +
" int index;\r\n var leftEvents = this.leftIntervalMap.Find(hash);\r\n " +
"while (leftEvents.Next(out index))\r\n {\r\n if (");
this.Write(this.ToStringHelper.ToStringWithCulture(keyComparer("key", "this.leftIntervalMap.Values[index].Key")));
this.Write(@")
{
long end = this.leftIntervalMap.Values[index].End;
bool isFullyOutputtable = this.nextRightTime >= end;
if (isFullyOutputtable)
{
// Output interval.
AddToBatch(
time,
end,
ref this.leftIntervalMap.Values[index].Key,
ref this.leftIntervalMap.Values[index],
hash);
// Mark left event as not visible so that an end-edge is not outputted when the interval actually endss.
this.leftIntervalMap.Values[index].CurrentStart = NotActive;
}
else
{
// Output start edge.
AddToBatch(
time,
StreamEvent.InfinitySyncTime,
ref this.leftIntervalMap.Values[index].Key,
ref this.leftIntervalMap.Values[index],
hash);
// Mark left event as visible.
this.leftIntervalMap.Values[index].CurrentStart = time;
}
}
}
// Make matching left edges visible.
leftEvents = this.leftEdgeMap.Find(hash);
while (leftEvents.Next(out index))
{
if (");
this.Write(this.ToStringHelper.ToStringWithCulture(keyComparer("key", "this.leftEdgeMap.Values[index].Key")));
this.Write(@")
{
// Output start edge.
AddToBatch(
time,
StreamEvent.InfinitySyncTime,
ref this.leftEdgeMap.Values[index].Key,
ref this.leftEdgeMap.Values[index],
hash);
// Mark left event as visible.
this.leftEdgeMap.Values[index].CurrentStart = time;
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void AddPunctuationToBatch(long start)
{
if (start > lastCTI)
{
lastCTI = start;
int index = this.output.Count++;
this.output.vsync.col[index] = start;
this.output.vother.col[index] = StreamEvent.PunctuationOtherTime;
this.output.key.col[index] = default;
");
foreach (var f in this.leftFields)
{
if (f.OptimizeString())
{
this.Write(" this.output.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".AddString(string.Empty);\r\n");
}
else
{
this.Write(" this.output.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col[index] = default;\r\n");
}
}
this.Write(@" this.output.hash.col[index] = 0;
this.output.bitvector.col[index >> 6] |= (1L << (index & 0x3f));
if (this.output.Count == Config.DataBatchSize) FlushContents();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void AddToBatch(long start, long end, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(" batch, int batchIndex, int hash)\r\n {\r\n int index = this.output.Count++" +
";\r\n this.output.vsync.col[index] = start;\r\n this.output.vother.col" +
"[index] = end;\r\n this.output.key.col[index] = key;\r\n");
foreach (var f in this.leftFields)
{
if (f.OptimizeString())
{
this.Write(" this.output.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".AddString(batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[batchIndex]);\r\n");
}
else
{
this.Write(" this.output.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col[index] = batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col[batchIndex];\r\n");
}
}
this.Write(@" this.output.hash.col[index] = hash;
if (this.output.Count == Config.DataBatchSize)
{
this.output.Seal();
this.Observer.OnNext(this.output);
GetOutputBatch();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void AddToBatch(long start, long end, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ref LeftEvent leftEvent, int hash)\r\n {\r\n int index = this.output." +
"Count++;\r\n this.output.vsync.col[index] = start;\r\n this.output.vot" +
"her.col[index] = end;\r\n this.output.key.col[index] = key;\r\n");
foreach (var f in this.leftFields)
{
string source;
if (noLeftFields)
{
source = "leftEvent.Payload";
}
else
{
source = "leftEvent.Payload." + f.OriginalName;
}
if (f.OptimizeString())
{
this.Write(" this.output.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".AddString(");
this.Write(this.ToStringHelper.ToStringWithCulture(source));
this.Write(");\r\n");
}
else
{
this.Write(" this.output.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col[index] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(source));
this.Write(";\r\n");
}
}
this.Write(" this.output.hash.col[index] = hash;\r\n\r\n if (this.output.Count == C" +
"onfig.DataBatchSize) FlushContents();\r\n }\r\n\r\n [MethodImpl(MethodImplOption" +
"s.AggressiveInlining)]\r\n private bool AreSame(long start, long end, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(" batch, int index, ref LeftEvent active)\r\n {\r\n return start == active.S" +
"tart &&\r\n end == active.End &&\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(keyComparer("key", "active.Key")));
this.Write(" &&\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftComparer("batch", "active.Payload")));
this.Write(@";
}
protected override void FlushContents()
{
if (this.output.Count == 0) return;
this.output.Seal();
this.Observer.OnNext(this.output);
GetOutputBatch();
}
private void GetOutputBatch()
{
this.pool.Get(out this.genericOutputBatch);
this.genericOutputBatch.Allocate();
this.output = (");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(")this.genericOutputBatch;\r\n");
foreach (var f in this.leftFields.Where(fld => fld.OptimizeString()))
{
this.Write(" this.output.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".Initialize();\r\n");
}
this.Write(" }\r\n\r\n");
if (!noLeftFields && !this.leftType.GetTypeInfo().IsValueType)
{
this.Write(" [DataContract]\r\n private struct ");
this.Write(this.ToStringHelper.ToStringWithCulture(ActiveEventType));
this.Write("\r\n {\r\n ");
foreach (var f in this.leftFields)
{
this.Write(" [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax()));
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n ");
}
this.Write(" }\r\n");
}
this.Write("\r\n [DataContract]\r\n private struct LeftEvent\r\n {\r\n [DataMember]\r\n" +
" public long Start;\r\n [DataMember]\r\n public long CurrentSta" +
"rt;\r\n [DataMember]\r\n public long End;\r\n [DataMember]\r\n " +
" public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(ActiveEventType));
this.Write(" Payload;\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n " +
"public void Populate(long start, long currentStart, long end, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(" batch, int index)\r\n {\r\n Start = start;\r\n CurrentSta" +
"rt = currentStart;\r\n End = end;\r\n Key = key;\r\n");
if (noLeftFields)
{
this.Write(" this.Payload = batch.payload.col[index];\r\n");
}
else
{
foreach (var f in leftFields)
{
this.Write(" this.Payload.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(" = ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index")));
this.Write(";\r\n");
}
}
this.Write(" }\r\n\r\n public override string ToString()\r\n {\r\n Sy" +
"stem.Text.StringBuilder sb = new System.Text.StringBuilder();\r\n sb.Ap" +
"pendFormat(\"[Start={0}\", Start);\r\n sb.AppendFormat(\", Key=\'{0}\'\", Key" +
");\r\n");
if (noLeftFields)
{
this.Write(" sb.AppendFormat(\", payload={0}, \", this.Payload);\r\n");
}
else
{
foreach (var f in leftFields)
{
this.Write(" sb.AppendFormat(\", ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write("={0}, \", this.Payload.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(");\r\n");
}
}
this.Write(" sb.Append(\"]\");\r\n return sb.ToString();\r\n }\r\n }\r" +
"\n\r\n [DataContract]\r\n private struct RightEvent\r\n {\r\n [DataMember" +
"]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n [DataMember]\r\n public int Count;\r\n\r\n [MethodImpl(Met" +
"hodImplOptions.AggressiveInlining)]\r\n public void Initialize(ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@" key)
{
Key = key;
Count = 1;
}
public override string ToString()
{
return ""[Key='"" + Key + ""', Count="" + Count + ""]"";
}
}
[DataContract]
private struct QueuedEndEdge
{
[DataMember]
public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n [DataMember]\r\n public int Hash;\r\n\r\n [MethodImpl(Meth" +
"odImplOptions.AggressiveInlining)]\r\n public void Populate(ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, int hash)\r\n {\r\n Key = key;\r\n Hash = hash;\r\n " +
" }\r\n\r\n public override string ToString()\r\n {\r\n retu" +
"rn \"[Key=\'\" + Key + \"\', Hash=\" + Hash + \"]\";\r\n }\r\n }\r\n}\r\n");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write("\r\n");
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
using Microsoft.StreamProcessing.Aggregates;
");
List <string> genericParamList = new List <string>();
int oldCount = 0;
var TKey = keyType.GetCSharpSourceSyntax(ref genericParamList);
var keyGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TInput = inputType.GetCSharpSourceSyntax(ref genericParamList);
var inputGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TState = stateType.GetCSharpSourceSyntax(ref genericParamList);
var stateGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TOutput = outputType.GetCSharpSourceSyntax(ref genericParamList);
var outputGenericParameters = new List <string>(genericParamList.Skip(oldCount));
var genericParameters = genericParamList.BracketedCommaSeparatedString();
var TKeyTInputGenericParameters = keyGenericParameters.Concat(inputGenericParameters).BracketedCommaSeparatedString();
var TKeyTOutputGenericParameters = keyGenericParameters.Concat(outputGenericParameters).BracketedCommaSeparatedString();
var BatchGeneratedFrom_TKey_TInput = Transformer.GetBatchClassName(keyType, inputType);
var genericParameters2 = string.Format("<{0}, {1}>", TKey, TOutput);
if (!keyType.KeyTypeNeedsGeneratedMemoryPool() && outputType.MemoryPoolHasGetMethodFor())
{
genericParameters2 = string.Empty;
}
else if (!outputType.CanRepresentAsColumnar())
{
genericParameters2 = string.Empty;
}
Func <string, string> assignToOutput = rhs =>
this.outputType.IsAnonymousType()
?
rhs
:
(
this.outputFields.Count() == 1
?
string.Format("this.batch.{0}.col[c] = {1};", this.outputFields.First().Name, rhs)
:
"temporaryOutput = " + rhs + ";\r\n" + String.Join("\r\n", this.outputFields.Select(f => "dest_" + f.Name + "[c] = temporaryOutput." + f.OriginalName + ";")))
;
var getOutputBatch = string.Format("this.pool.Get(out genericOutputbatch); this.batch = ({0}{1})genericOutputbatch;",
Transformer.GetBatchClassName(keyType, outputType),
TKeyTOutputGenericParameters);
this.Write("[assembly: IgnoresAccessChecksTo(\"Microsoft.StreamProcessing\")]\r\n\r\n// genericPara" +
"ms2 = \"");
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write("\"\r\n\r\n[DataContract]\r\nstruct StateAndActive\r\n{\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(" state;\r\n [DataMember]\r\n public ulong active;\r\n}\r\n\r\n[DataContract]\r\nstruct " +
"HeldStateStruct\r\n{\r\n [DataMember]\r\n public long timestamp;\r\n [DataMembe" +
"r]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(" state;\r\n}\r\n\r\n[DataContract]\r\nsealed class HeldState\r\n{\r\n [DataMember]\r\n pu" +
"blic long timestamp;\r\n [DataMember]\r\n public StateAndActive state;\r\n}\r\n\r\n/" +
"/ TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TInput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("\r\n// TState: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("\r\n// TOutput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("\r\n/// <summary>\r\n/// Operator that uses a full-fledged priority queue as ECQ\r\n///" +
" </summary>\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : UnaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(">\r\n{\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGenera" +
"tor;\r\n private readonly IAggregate<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> aggregate;\r\n\r\n [DataContract]\r\n class EcqState\r\n {\r\n [DataMembe" +
"r]\r\n public long timestamp;\r\n [DataMember]\r\n public FastDic" +
"tionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", StateAndActive> states;\r\n }\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(" pool;\r\n\r\n private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> genericOutputbatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTOutputGenericParameters));
this.Write(" batch;\r\n\r\n");
if (this.useCompiledInitialState)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> initialState;\r\n");
}
if (this.useCompiledAccumulate)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", long, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> accumulate;\r\n");
}
if (this.useCompiledDeaccumulate)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", long, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> deaccumulate;\r\n");
}
if (this.useCompiledDifference)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> difference;\r\n");
}
if (this.useCompiledComputeResult)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> computeResult;\r\n");
}
this.Write("\r\n private readonly IEqualityComparerExpression<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("> keyComparer;\r\n private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", bool> keyComparerEquals;\r\n private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", int> keyComparerGetHashCode;\r\n\r\n [DataMember]\r\n private FastDictionary2<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldState> aggregateByKey;\r\n [DataMember]\r\n private FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldState> heldAggregates;\r\n [DataMember]\r\n private SortedDictionary<long" +
", EcqState> ecq;\r\n [DataMember]\r\n private long lastSyncTime = long.MinValu" +
"e;\r\n\r\n private HeldState currentState;\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" currentKey;\r\n private int currentHash;\r\n\r\n private Func<FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", StateAndActive>> stateDictGenerator;\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n Streamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator,\r\n" +
" IAggregate<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> aggregate)\r\n : base(stream, observer)\r\n {\r\n this.aggregate = a" +
"ggregate;\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n");
if (this.useCompiledInitialState)
{
this.Write(" initialState = aggregate.InitialState().Compile();\r\n");
}
if (this.useCompiledAccumulate)
{
this.Write(" accumulate = aggregate.Accumulate().Compile();\r\n");
}
if (this.useCompiledDeaccumulate)
{
this.Write(" deaccumulate = aggregate.Deaccumulate().Compile();\r\n");
}
if (this.useCompiledDifference)
{
this.Write(" difference = aggregate.Difference().Compile();\r\n");
}
if (this.useCompiledComputeResult)
{
this.Write(" computeResult = aggregate.ComputeResult().Compile();\r\n");
}
this.Write(@"
this.keyComparer = stream.Properties.KeyEqualityComparer;
this.keyComparerEquals = this.keyComparer.GetEqualsExpr().Compile();
this.keyComparerGetHashCode = this.keyComparer.GetGetHashCodeExpr().Compile();
this.stateDictGenerator = this.keyComparer.CreateFastDictionaryGenerator<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", StateAndActive>(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream." +
"Properties.QueryContainer);\r\n\r\n this.pool = MemoryManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(">() as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(";\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n\r\n this.aggregateByKey = this.keyCompare" +
"r.CreateFastDictionary2Generator<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldState>(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Prope" +
"rties.QueryContainer).Invoke();\r\n this.heldAggregates = this.keyComparer." +
"CreateFastDictionaryGenerator<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldState>(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Properties.QueryContainer).Invoke();
this.ecq = new SortedDictionary<long, EcqState>();
}
public override void ProduceQueryPlan(PlanNode previous)
=> Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));
protected override void FlushContents()
{
if (this.batch == null || this.batch.Count == 0) return;
this.batch.Seal();
this.Observer.OnNext(this.batch);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write(@"
this.batch.Allocate();
}
protected override void DisposeState() => this.batch.Free();
public override int CurrentlyBufferedOutputCount => this.batch.Count;
public override int CurrentlyBufferedInputCount => this.aggregateByKey.Count + this.ecq.Values.Select(o => o.states.Count).Sum();
public override unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("> inputBatch)\r\n {\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters));
this.Write(" generatedBatch = inputBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters));
this.Write(";\r\n\r\n this.batch.iter = generatedBatch.iter;\r\n\r\n var count = genera" +
"tedBatch.Count;\r\n\r\n");
if (this.outputFields.Count() > 1)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(" temporaryOutput;\r\n");
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the input batch.\r\n");
foreach (var f in this.inputFields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = generatedBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = generatedBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the output batch.\r\n");
foreach (var f in this.outputFields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n fixed (long* col_vsync = generatedBatch.vsync.col)\r\n fixed (long" +
"* col_vother = generatedBatch.vother.col)\r\n fixed (int* col_hash = genera" +
"tedBatch.hash.col)\r\n fixed (long* col_bv = generatedBatch.bitvector.col)\r" +
"\n for (int i = 0; i < count; i++)\r\n {\r\n if ((col_bv[i >" +
"> 6] & (1L << (i & 0x3f))) != 0)\r\n {\r\n if (col_vother[" +
"i] == long.MinValue)\r\n {\r\n // We have found a " +
"row that corresponds to punctuation\r\n OnPunctuation(col_vsync" +
"[i]);\r\n\r\n int c = this.batch.Count;\r\n this" +
".batch.vsync.col[c] = col_vsync[i];\r\n this.batch.vother.col[c" +
"] = long.MinValue;\r\n this.batch.key.col[c] = default;\r\n " +
" this.batch[c] = default;\r\n this.batch.hash.col[" +
"c] = 0;\r\n this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f" +
");\r\n this.batch.Count++;\r\n if (this.batch." +
"Count == Config.DataBatchSize) FlushContents();\r\n }\r\n " +
" continue;\r\n }\r\n\r\n var syncTime = col_vsync[i];\r\n " +
" var key_i = generatedBatch.key.col[i];\r\n\r\n HeldState heldState;" +
"\r\n bool cachedState = false;\r\n\r\n // Handle time moving for" +
"ward\r\n if (syncTime > this.lastSyncTime)\r\n {\r\n " +
" /* Issue start edges for held aggregates */\r\n if (currentStat" +
"e != null && this.heldAggregates.Count == 1)\r\n {\r\n " +
" // there is just one held aggregate, and currentState is set\r\n " +
" // so currentState has to be the held aggregate\r\n cach" +
"edState = true;\r\n\r\n if (currentState.state.active > 0)\r\n " +
" {\r\n int c = this.batch.Count;\r\n " +
" this.batch.vsync.col[c] = currentState.timestamp;\r\n " +
" this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;\r\n " +
" ");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize)
{
this.batch.iter = generatedBatch.iter;
FlushContents();
this.batch.iter = generatedBatch.iter;
}
}
else
{
this.aggregateByKey.Remove(currentKey, currentHash);
currentState = null;
}
}
else
{
int iter1 = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldState>.IteratorStart;
while (this.heldAggregates.Iterate(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
if (iter1entry.value.state.active > 0)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = iter1entry.value.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value.state.state"))));
this.Write("\r\n this.batch.key.col[c] = iter1entry.key;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("iter1entry.key")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
else
{
this.aggregateByKey.Remove(iter1entry.key); // , (currentKey, currentHash);
currentState = null;
}
}
// Time has moved forward, clear the held aggregates
this.heldAggregates.Clear();
currentState = null;
}
/* Process the ECQ up until the new sync time */
long ve;
EcqState ecqState;
while (this.ecq.Count > 0 && this.ecq.TryGetFirst(out ve, out ecqState) && ve <= syncTime)
{
this.ecq.Remove(ve);
int iter = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(">.IteratorStart;\r\n\r\n while (ecqState.states.Iterate(ref iter))" +
"\r\n {\r\n if (currentState == null || !");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "ecqState.states.entries[iter].key")));
this.Write(@")
{
int index;
this.aggregateByKey.Lookup(ecqState.states.entries[iter].key, out index);
heldState = this.aggregateByKey.entries[index].value;
}
else
heldState = currentState;
if (heldState.state.active > 0)
{
// Issue end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = ecqState.timestamp;
this.batch.vother.col[c] = heldState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries[ite" +
"r].key;\r\n this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
// Update aggregate
heldState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(difference("heldState.state.state", "ecqState.states.entries[iter].value.state")));
this.Write(@";
heldState.state.active -= ecqState.states.entries[iter].value.active;
if (ecqState.timestamp < syncTime)
{
if (heldState.state.active > 0)
{
// Issue start edge
int c = this.batch.Count;
this.batch.vsync.col[c] = ecqState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries" +
"[iter].key;\r\n this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key")));
this.Write(";\r\n this.batch.Count++;\r\n " +
" if (this.batch.Count == Config.DataBatchSize) FlushContents();\r\n " +
" }\r\n else\r\n " +
" {\r\n // remove from aggregateByKey\r\n " +
" this.aggregateByKey.Remove(ecqState.states.entries[iter].k" +
"ey);\r\n currentState = null;\r\n " +
" }\r\n }\r\n else\r\n " +
" {\r\n if (cachedState)\r\n " +
" {\r\n cachedState = false;\r\n " +
" if (heldState != currentState)\r\n " +
" {\r\n this.heldAggregates.Clear();\r\n " +
" currentState = null;\r\n " +
" int index;\r\n this.heldAggregates.Lookup(" +
"ecqState.states.entries[iter].key, out index);\r\n " +
" this.heldAggregates.Insert(ref index, ecqState.states.entries[iter].key, hel" +
"dState);\r\n }\r\n }\r\n " +
" else\r\n {\r\n " +
" int index;\r\n this.heldAggregates.Loo" +
"kup(ecqState.states.entries[iter].key, out index);\r\n " +
" this.heldAggregates.Insert(ref index, ecqState.states.entries[iter].key, hel" +
"dState);\r\n }\r\n }\r\n\r\n " +
" // Update timestamp\r\n heldState.timestamp =" +
" ecqState.timestamp;\r\n }\r\n }\r\n\r\n " +
" // Since sync time changed, set this.lastSyncTime\r\n this.lastSy" +
"ncTime = syncTime;\r\n }\r\n\r\n if (currentState == null || cur" +
"rentHash != col_hash[i] || !");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "key_i")));
this.Write(@")
{
if (cachedState)
{
cachedState = false;
this.heldAggregates.Clear();
}
// Need to retrieve the key from the dictionary
currentKey = key_i;
currentHash = col_hash[i];
int index;
if (!this.heldAggregates.Lookup(currentKey, currentHash, out index))
{
// First time group is active for this time
int aggindex;
if (!this.aggregateByKey.Lookup(currentKey, currentHash, out aggindex))
{
// New group. Create new state
currentState = new HeldState();
currentState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@";
currentState.timestamp = syncTime;
this.aggregateByKey.Insert(currentKey, currentState, currentHash);
// No output because initial state is empty
}
else
{
currentState = this.aggregateByKey.entries[aggindex].value;
if (syncTime > currentState.timestamp)
{
if (currentState.state.active > 0)
{
// Output end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = syncTime;
this.batch.vother.col[c] = currentState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
currentState.timestamp = syncTime;
}
}
this.heldAggregates.Insert(ref index, currentKey, currentState);
}
else
{
// read new currentState from _heldAgg index
currentState = this.heldAggregates.entries[index].value;
}
}
else
{
if (syncTime > currentState.timestamp)
{
if (currentState.state.active > 0)
{
// Output end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = syncTime;
this.batch.vother.col[c] = currentState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
currentState.timestamp = syncTime;
}
}
if (col_vsync[i] < col_vother[i]) // insert event
{
currentState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("currentState.state.state", "col_vsync[i]" /*, "col_payload[i]"*/)));
this.Write(@";
currentState.state.active++;
// Update ECQ
if (col_vother[i] < StreamEvent.InfinitySyncTime)
{
EcqState eState;
int index;
if (this.ecq.Count > 0)
{
if (!this.ecq.TryGetValue(col_vother[i], out eState))
{
eState = new EcqState();
eState.timestamp = col_vother[i];
eState.states = this.stateDictGenerator.Invoke();
eState.states.Lookup(key_i, col_hash[i], out index);
eState.states.Insert(ref index, key_i, new StateAndActive { state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@" });
this.ecq.Add(col_vother[i], eState);
}
else
{
if (!eState.states.Lookup(key_i, col_hash[i], out index))
{
eState.states.Insert(ref index, key_i, new StateAndActive { state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@" });
}
}
}
else
{
eState = new EcqState();
eState.timestamp = col_vother[i];
eState.states = this.stateDictGenerator.Invoke();
eState.states.Lookup(key_i, col_hash[i], out index);
eState.states.Insert(ref index, key_i, new StateAndActive { state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(" });\r\n this.ecq.Add(col_vother[i], eState);\r\n " +
" }\r\n\r\n eState.states.entries[index].value.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("eState.states.entries[index].value.state", "col_vsync[i]")));
this.Write(";\r\n eState.states.entries[index].value.active++;\r\n " +
" }\r\n }\r\n else // is a retraction\r\n {\r\n " +
" currentState.state.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(deaccumulate("currentState.state.state", "col_vsync[i]" /*, "col_payload[i]"*/)));
this.Write(";\r\n currentState.state.active--;\r\n }\r\n }\r\n\r\n");
foreach (var f in this.inputFields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
foreach (var f in this.outputFields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
this.Write("\r\n generatedBatch.Release();\r\n generatedBatch.Return();\r\n }\r\n\r\n\r" +
"\n public void OnPunctuation(long syncTime)\r\n {\r\n HeldState heldStat" +
"e;\r\n\r\n");
if (this.outputFields.Count() > 1)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(" temporaryOutput;\r\n");
foreach (var f in this.outputFields)
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write(@"
// Handle time moving forward
if (syncTime > this.lastSyncTime)
{
/* Issue start edges for held aggregates */
if (currentState != null && this.heldAggregates.Count == 1)
{
// there is just one held aggregate, and currentState is set
// so currentState has to be the held aggregate
if (currentState.state.active > 0)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = currentState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = currentKey;\r\n th" +
"is.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("currentKey")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
else
{
this.aggregateByKey.Remove(currentKey, currentHash);
currentState = null;
}
}
else
{
int iter1 = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldState>.IteratorStart;
while (this.heldAggregates.Iterate(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
if (iter1entry.value.state.active > 0)
{
int c = this.batch.Count;
this.batch.vsync.col[c] = iter1entry.value.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value.state.state"))));
this.Write("\r\n this.batch.key.col[c] = iter1entry.key;\r\n " +
" this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("iter1entry.key")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
else
{
this.aggregateByKey.Remove(iter1entry.key);
currentState = null;
}
}
}
// Time has moved forward, clear the held aggregates
this.heldAggregates.Clear();
currentState = null;
/* Process the ECQ up until the new sync time */
long ve;
EcqState ecqState;
while (this.ecq.Count > 0 && this.ecq.TryGetFirst(out ve, out ecqState) && ve <= syncTime)
{
this.ecq.Remove(ve);
int iter = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(">.IteratorStart;\r\n\r\n while (ecqState.states.Iterate(ref iter))\r\n " +
" {\r\n if (currentState == null || !");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "ecqState.states.entries[iter].key")));
this.Write(@")
{
int index;
this.aggregateByKey.Lookup(ecqState.states.entries[iter].key, out index);
heldState = this.aggregateByKey.entries[index].value;
}
else
heldState = currentState;
if (heldState.state.active > 0)
{
// Issue end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = ecqState.timestamp;
this.batch.vother.col[c] = heldState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries[iter].k" +
"ey;\r\n this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key")));
this.Write(";\r\n this.batch.Count++;\r\n if (this." +
"batch.Count == Config.DataBatchSize) FlushContents();\r\n }\r\n\r\n" +
" // Update aggregate\r\n heldState.state.sta" +
"te = ");
this.Write(this.ToStringHelper.ToStringWithCulture(difference("heldState.state.state", "ecqState.states.entries[iter].value.state")));
this.Write(@";
heldState.state.active -= ecqState.states.entries[iter].value.active;
if (ecqState.timestamp < syncTime)
{
if (heldState.state.active > 0)
{
// Issue start edge
int c = this.batch.Count;
this.batch.vsync.col[c] = ecqState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("heldState.state.state"))));
this.Write("\r\n this.batch.key.col[c] = ecqState.states.entries[ite" +
"r].key;\r\n this.batch.hash.col[c] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerGetHashCode("ecqState.states.entries[iter].key")));
this.Write(@";
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
else
{
// remove from aggregateByKey
this.aggregateByKey.Remove(ecqState.states.entries[iter].key);
currentState = null;
}
}
else
{
int index;
this.heldAggregates.Lookup(ecqState.states.entries[iter].key, out index);
this.heldAggregates.Insert(ref index, ecqState.states.entries[iter].key, heldState);
}
// Update timestamp
heldState.timestamp = ecqState.timestamp;
}
}
// Since sync time changed, set this.lastSyncTime
this.lastSyncTime = syncTime;
}
}
protected override void UpdatePointers()
{
int iter1 = FastDictionary<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldState>.IteratorStart;\r\n while (this.heldAggregates.Iterate(ref iter1" +
"))\r\n {\r\n int iter2;\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@" key = this.heldAggregates.entries[iter1].key;
bool found = this.aggregateByKey.Lookup(key, out iter2);
if (!found) throw new InvalidOperationException();
this.heldAggregates.entries[iter1].value = this.aggregateByKey.entries[iter2].value;
}
}
}
");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using System.Threading;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Aggregates;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
// TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TLeft: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("\r\n// TRight: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("\r\n// TResult: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : BinaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">\r\n{\r\n private const int DefaultCapacity = 64;\r\n private readonly MemoryPoo" +
"l<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(@"> pool;
private readonly Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator;
[DataMember]
private FastMap<ActiveEventLeft> leftEdgeMap = new FastMap<ActiveEventLeft>(DefaultCapacity);
[DataMember]
private FastMap<ActiveEventRight> rightEdgeMap = new FastMap<ActiveEventRight>(DefaultCapacity);
private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> genericOutputBatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TResult));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters));
this.Write(@" output;
[DataMember]
private long nextLeftTime = long.MinValue;
[DataMember]
private bool isLeftComplete;
[DataMember]
private long nextRightTime = long.MinValue;
[DataMember]
private bool isRightComplete;
");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> observer,\r\n Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> qu" +
"eryPlanGenerator)\r\n : base(stream, observer)\r\n {\r\n pool = Memor" +
"yManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">(true /*stream.Properties.IsColumnar*/);\r\n ");
GetOutputBatch();
this.Write(" this.queryPlanGenerator = queryPlanGenerator;\r\n }\r\n\r\n protected ove" +
"rride void FlushContents()\r\n {\r\n if (output.Count == 0) return;\r\n " +
" output.Seal();\r\n this.Observer.OnNext(output);\r\n ");
GetOutputBatch();
this.Write(@" }
public override int CurrentlyBufferedOutputCount => output.Count;
public override int CurrentlyBufferedLeftInputCount => base.CurrentlyBufferedLeftInputCount + leftEdgeMap.Count;
public override int CurrentlyBufferedRightInputCount => base.CurrentlyBufferedRightInputCount + rightEdgeMap.Count;
protected override void ProduceBinaryQueryPlan(PlanNode left, PlanNode right)
{
Observer.ProduceQueryPlan(queryPlanGenerator(left, right, this));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void DisposeState() => this.output.Free();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessBothBatches(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> leftBatch, StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("> rightBatch, out bool leftBatchDone, out bool rightBatchDone, out bool leftBatch" +
"Free, out bool rightBatchFree)\r\n {\r\n var generatedLeftBatch = leftBatc" +
"h as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, leftType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(";\r\n var generatedRightBatch = rightBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, rightType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTRightGenericParameters));
this.Write(@";
leftBatchFree = rightBatchFree = true;
if (!GoToVisibleRow(leftBatch))
{
leftBatchDone = true;
rightBatchDone = false;
return;
}
UpdateNextLeftTime(leftBatch.vsync.col[leftBatch.iter]);
if (!GoToVisibleRow(rightBatch))
{
leftBatchDone = false;
rightBatchDone = true;
return;
}
UpdateNextRightTime(rightBatch.vsync.col[rightBatch.iter]);
FastMap<ActiveEventRight>.FindTraverser rightEdges = default;
while (true)
{
bool leftPunctuation = leftBatch.vother.col[leftBatch.iter] == StreamEvent.PunctuationOtherTime;
bool rightPunctuation = rightBatch.vother.col[rightBatch.iter] == StreamEvent.PunctuationOtherTime;
if (nextLeftTime <= nextRightTime)
{
if (leftPunctuation)
{
AddPunctuationToBatch(nextLeftTime);
}
else
{
// ProcessLeftStartEdge
if (true)
{
bool first = true;
");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@" key = default;
var hash = leftBatch.hash.col[leftBatch.iter];
if (rightEdgeMap.Find(hash, ref rightEdges))
{
int rightIndex;
while (rightEdges.Next(out rightIndex))
{
if (first) { key = leftBatch.key.col[leftBatch.iter]; first = false; }
//if (keyComparer(key, rightEdgeMap.Values[rightIndex].Key))
var activeEventRight = rightEdgeMap.Values[rightIndex];
var rightKey = activeEventRight.Key;
if (");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyComparer("key", "rightKey")));
this.Write(@")
{
//payload = leftBatch[leftBatch.iter];
//OutputStartEdge(nextLeftTime, ref key, ref payload, ref rightEdgeMap.Values[rightIndex].Payload, hash);
");
OutputStartEdgeWithActiveEventRight("nextLeftTime", "key", "generatedLeftBatch", "leftBatch.iter", "activeEventRight", "hash");
this.Write(@" }
}
}
if (!isRightComplete)
{
int newIndex = leftEdgeMap.Insert(hash);
if (first) key = leftBatch.key.col[leftBatch.iter];
leftEdgeMap.Values[newIndex].Populate(generatedLeftBatch, leftBatch.iter);
}
}
}
leftBatch.iter++;
if (!GoToVisibleRow(leftBatch))
{
leftBatchDone = true;
rightBatchDone = false;
return;
}
UpdateNextLeftTime(leftBatch.vsync.col[leftBatch.iter]);
}
else
{
if (rightPunctuation)
{
AddPunctuationToBatch(nextRightTime);
}
else
{
// ProcessRightStartEdge
");
ProcessRightStartEdge2(
"nextRightTime",
"generatedRightBatch",
"rightBatch.iter",
"rightBatch.hash.col[rightBatch.iter]");
this.Write(@" }
rightBatch.iter++;
if (!GoToVisibleRow(rightBatch))
{
leftBatchDone = false;
rightBatchDone = true;
return;
}
UpdateNextRightTime(rightBatch.vsync.col[rightBatch.iter]);
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessLeftBatch(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> batch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var generate" +
"dLeftBatch = batch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, leftType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTLeftGenericParameters));
this.Write(@";
isBatchFree = true;
while (true)
{
if (!GoToVisibleRow(batch))
{
isBatchDone = true;
return;
}
UpdateNextLeftTime(batch.vsync.col[batch.iter]);
if (nextLeftTime > nextRightTime)
{
isBatchDone = false;
return;
}
if (batch.vother.col[batch.iter] == StreamEvent.PunctuationOtherTime)
{
AddPunctuationToBatch(batch.vsync.col[batch.iter]);
batch.iter++;
continue;
}
// ProcessLeftStartEdge
FastMap<ActiveEventRight>.FindTraverser rightEdges = default;
if (true)
{
bool first = true;
");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@" key = default;
var hash = batch.hash.col[batch.iter];
if (rightEdgeMap.Find(hash, ref rightEdges))
{
int rightIndex;
while (rightEdges.Next(out rightIndex))
{
if (first) { key = batch.key.col[batch.iter]; first = false; }
//if (keyComparer(key, rightEdgeMap.Values[rightIndex].Key))
var activeEventRight = rightEdgeMap.Values[rightIndex];
var rightKey = activeEventRight.Key;
if (");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyComparer("key", "rightKey")));
this.Write(")\r\n {\r\n //payload = batch[batch" +
".iter];\r\n //OutputStartEdge(nextLeftTime, ref key, re" +
"f payload, ref rightEdgeMap.Values[rightIndex].Payload, hash);\r\n " +
" ");
OutputStartEdgeWithActiveEventRight("nextLeftTime", "key", "generatedLeftBatch", "batch.iter", "activeEventRight", "hash");
this.Write(@" }
}
}
if (!isRightComplete)
{
int newIndex = leftEdgeMap.Insert(hash);
if (first) key = batch.key.col[batch.iter];
//payload = batch[batch.iter]; // potential rare recomputation
//leftEdgeMap.Values[newIndex].Populate(ref key, ref payload);
leftEdgeMap.Values[newIndex].Populate(generatedLeftBatch, batch.iter);
}
}
batch.iter++;
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessRightBatch(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("> batch, out bool isBatchDone, out bool isBatchFree)\r\n {\r\n var generate" +
"dBatch = batch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, rightType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTRightGenericParameters));
this.Write(@";
isBatchFree = true;
while (true)
{
if (!GoToVisibleRow(batch))
{
isBatchDone = true;
return;
}
UpdateNextRightTime(batch.vsync.col[batch.iter]);
if (nextRightTime > nextLeftTime)
{
isBatchDone = false;
return;
}
if (batch.vother.col[batch.iter] == StreamEvent.PunctuationOtherTime)
{
AddPunctuationToBatch(batch.vsync.col[batch.iter]);
batch.iter++;
continue;
}
// ProcessRightStartEdge
");
ProcessRightStartEdge2(
"nextRightTime",
"generatedBatch",
"batch.iter",
"batch.hash.col[batch.iter]");
this.Write("\r\n batch.iter++;\r\n }\r\n }\r\n\r\n\r\n [MethodImpl(MethodImplOpti" +
"ons.AggressiveInlining)]\r\n private bool GoToVisibleRow<TPayload>(StreamMessag" +
"e<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", TPayload> batch)\r\n {\r\n while (batch.iter < batch.Count && (batch.bitv" +
"ector.col[batch.iter >> 6] & (1L << (batch.iter & 0x3f))) != 0 && batch.vother.c" +
"ol[batch.iter] >= 0)\r\n {\r\n batch.iter++;\r\n }\r\n\r\n " +
" return (batch.iter != batch.Count);\r\n }\r\n\r\n [MethodImpl(MethodImplOptions" +
".AggressiveInlining)]\r\n private void UpdateNextLeftTime(long time)\r\n {\r\n " +
" nextLeftTime = time;\r\n if (nextLeftTime == StreamEvent.InfinitySync" +
"Time)\r\n {\r\n isLeftComplete = true;\r\n }\r\n }\r\n\r\n [M" +
"ethodImpl(MethodImplOptions.AggressiveInlining)]\r\n private void UpdateNextRig" +
"htTime(long time)\r\n {\r\n nextRightTime = time;\r\n if (nextRightTi" +
"me == StreamEvent.InfinitySyncTime)\r\n {\r\n isRightComplete = tr" +
"ue;\r\n }\r\n }\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r" +
"\n private void AddPunctuationToBatch(long start)\r\n {\r\n if (start > " +
"lastCTI)\r\n {\r\n lastCTI = start;\r\n\r\n int index = out" +
"put.Count++;\r\n output.vsync.col[index] = start;\r\n output.v" +
"other.col[index] = StreamEvent.PunctuationOtherTime;\r\n output.key.col" +
"[index] = default;\r\n output[index] = default;\r\n output.has" +
"h.col[index] = 0;\r\n output.bitvector.col[index >> 6] |= (1L << (index" +
" & 0x3f));\r\n\r\n if (output.Count == Config.DataBatchSize) FlushContent" +
"s();\r\n }\r\n }\r\n\r\n [DataContract]\r\n private struct ActiveEventLeft" +
"\r\n {\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n ");
foreach (var f in this.leftFields)
{
this.Write("\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax()));
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n ");
}
this.Write("\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n public vo" +
"id Populate(");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TLeft));
this.Write(" batch, int index)\r\n {\r\n Key = batch.key.col[index];\r\n " +
" ");
foreach (var f in this.leftFields)
{
this.Write("\r\n this.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(" = ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index")));
this.Write(";\r\n ");
}
this.Write("\r\n }\r\n\r\n public override string ToString()\r\n {\r\n " +
"return \"[Payload=\'\" + \"\']\";\r\n }\r\n }\r\n\r\n [DataContract]\r\n private" +
" struct ActiveEventRight\r\n {\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" Key;\r\n ");
foreach (var f in this.rightFields)
{
this.Write("\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax()));
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n ");
}
this.Write("\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n public void" +
" Populate(");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TRight));
this.Write(" batch, int index)\r\n {\r\n Key = batch.key.col[index];\r\n " +
" ");
foreach (var f in this.rightFields)
{
this.Write("\r\n this.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(" = ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index")));
this.Write(";\r\n ");
}
this.Write("\r\n }\r\n\r\n public override string ToString()\r\n {\r\n " +
"return \"[Payload=\'\" + \"\']\";\r\n }\r\n }\r\n}\r\n\r\n");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
");
if (this.keyType.Namespace != null)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.keyType.Namespace));
this.Write(";\r\n");
}
if (this.payloadType.Namespace != null && this.payloadType.Namespace != this.keyType.Namespace)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.payloadType.Namespace));
this.Write(";\r\n");
}
if (this.resultType.Namespace != null && this.resultType.Namespace != this.keyType.Namespace && this.resultType.Namespace != this.payloadType.Namespace)
{
this.Write("using ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.resultType.Namespace));
this.Write(";\r\n");
}
this.Write("[assembly: IgnoresAccessChecksTo(\"Microsoft.StreamProcessing\")]\r\n\r\n// Source Fiel" +
"ds: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.fields.Select(f => f.OriginalName))));
this.Write("\r\n// Destination Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.destinationFields.Select(f => f.OriginalName))));
this.Write("\r\n// Computed Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.computedFields.Keys.Select(f => f.OriginalName))));
this.Write("\r\n// Swinging Fields: ");
this.Write(this.ToStringHelper.ToStringWithCulture(String.Join(",", this.swingingFields.Select(tup => string.Format("<{0},{1}>", tup.Item1.OriginalName, tup.Item2.Name)))));
this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : UnaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">\r\n{\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(MemoryPoolGenericParameters));
this.Write(" pool;\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGene" +
"rator;\r\n");
foreach (var f in this.unassignedFields)
{
if (!f.OptimizeString())
{
this.Write("\r\n private readonly ColumnBatch<");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("> sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
}
this.Write("\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator)\r\n" +
" : base(stream, observer)\r\n {\r\n pool = MemoryManager.GetMemoryP" +
"ool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">() as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(MemoryPoolGenericParameters));
this.Write(";\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n");
foreach (var f in this.unassignedFields.Where(fld => !fld.OptimizeString()))
{
this.Write("\r\n pool.Get(out sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(");\r\n Array.Clear(sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col, 0, sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col.Length);\r\n");
}
this.Write("\r\n }\r\n\r\n public override void ProduceQueryPlan(PlanNode previous)\r\n {\r\n " +
" Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));\r\n }\r\n\r\n" +
" protected override void DisposeState()\r\n {\r\n");
foreach (var f in this.unassignedFields.Where(fld => !fld.OptimizeString()))
{
this.Write(" this.sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".Return();\r\n");
}
this.Write(" }\r\n\r\n public override unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TPayload));
this.Write("> _inBatch)\r\n {\r\n var sourceBatch = _inBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TPayload));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTPayloadGenericParameters));
this.Write(";\r\n\r\n StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> _genBatch; // Need this type to call Get with so the right subtype will be retu" +
"rned\r\n pool.Get(out _genBatch);\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters));
this.Write(" resultBatch = _genBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters));
this.Write(@";
var count = sourceBatch.Count;
resultBatch.vsync = sourceBatch.vsync;
resultBatch.vother = sourceBatch.vother;
resultBatch.key = sourceBatch.key;
resultBatch.hash = sourceBatch.hash;
resultBatch.bitvector = sourceBatch.bitvector;
");
if (resultType.CanContainNull())
{
this.Write(" pool.GetBV(out resultBatch._nullnessvector);\r\n");
}
this.Write(@"
// Get memory pools for the result columns.
// When no transformation was done to the query, then needed for all fields in the result type.
// When the query was transformed, then this is needed only for computed fields, since
// any field that is just assigned a field from the source type will get its value by
// swinging a pointer for the corresponding column.
");
foreach (var f in (this.ProjectionReturningResultInstance != null ? this.destinationFields : this.computedFields.Keys))
{
this.Write(" pool.Get(out resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(");\r\n");
}
this.Write("\r\n");
foreach (var f in this.unassignedFields)
{
if (!f.OptimizeString())
{
this.Write(" this.sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".IncrementRefCount(1);\r\n");
}
this.Write(" resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.sharedDefaultColumnFor_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the destination batch.\r\n\r\n");
foreach (var f in this.computedFields.Keys)
{
if (f.OptimizeString())
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
else
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the source batch.\r\n");
if (this.ProjectionReturningResultInstance != null || 0 < this.computedFields.Count())
{
foreach (var f in this.fields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = sourceBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = sourceBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n fixed (long* bv = sourceBatch.bitvector.col)\r\n {\r\n fo" +
"r (int i = 0; i < count; i++)\r\n {\r\n if ((bv[i >> 6] & " +
"(1L << (i & 0x3f))) == 0)\r\n {\r\n");
if (this.StartEdgeParameterName != null)
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.StartEdgeParameterName));
this.Write(" = sourceBatch.vsync.col[i] < sourceBatch.vother.col[i] ? sourceBatch.vsync.col[i" +
"] : sourceBatch.vother.col[i];\r\n");
}
if (this.needSourceInstance)
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(PARAMETER));
this.Write(" = sourceBatch[i];\r\n");
}
if (this.ProjectionReturningResultInstance != null)
{
this.Write(" resultBatch[i] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.ProjectionReturningResultInstance));
this.Write(";\r\n");
}
else
{
foreach (var kv in this.computedFields)
{
var f = kv.Key;
var v = kv.Value.ExpressionToCSharp();
if (f.OptimizeString())
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".AddString(");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(");\r\n");
}
else
{
this.Write(" dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("[i] = ");
this.Write(this.ToStringHelper.ToStringWithCulture(v));
this.Write(";\r\n");
}
}
}
this.Write(" }\r\n }\r\n }\r\n\r\n");
foreach (var f in this.fields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
}
this.Write("\r\n #region MultiString Vector Operations (if any)\r\n");
foreach (var mso in this.multiStringOperations)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(mso));
this.Write("\r\n");
}
this.Write(@" #endregion
#region Swinging Fields
// When the query is transformed, then any fields in the result batch that
// are just assigned from a field in the source batch are computed by just
// swinging the pointer for the corresponding column.
");
foreach (var tuple in this.swingingFields)
{
var destField = tuple.Item1.Name;
var sourceField = tuple.Item2.Name;
this.Write("\r\n // Special case for a field in the output being assigned the key field\r" +
"\n // when both are strings and the output field is a MultiString. The key" +
"\r\n // field is a CB<string> so we need to call MultiString.FromColumnBatc" +
"h\r\n");
if (tuple.Item1.OptimizeString() && sourceField.Equals("key"))
{
this.Write(" resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(destField));
this.Write(" = MultiString.FromColumnBatch(sourceBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceField));
this.Write(", sourceBatch.bitvector, pool.charArrayPool, pool.intPool, pool.shortPool, pool.b" +
"itvectorPool);\r\n");
}
else
{
this.Write(" resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(destField));
this.Write(" = sourceBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceField));
this.Write(";\r\n");
}
}
this.Write(@"
// We avoid incrementing the ref counts on the ouput batch because for the
// most part we are throwing away our reference to the columns from the source batch
// so the reference count doesn't change. But the same field might get swung to more
// than one column in the output batch. In that case, we need to increment the ref count.
");
foreach (var kv in this.swungFieldsCount)
{
var sourceField = kv.Key;
var count = kv.Value;
if (sourceField.Name.Equals("key") && !sourceField.OptimizeString())
{
// special case: we "pointer swing" the key field once just because the key
// of the output batch is the key of the source batch
// (Note that this really applies only to SelectKey since Select is not able
// to reference the key field in the selector.)
count++;
}
if (count == 1)
{
continue;
}
this.Write(" sourceBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceField.Name));
this.Write(".IncrementRefCount(");
this.Write(this.ToStringHelper.ToStringWithCulture(count - 1));
this.Write(");\r\n");
}
this.Write(@" #endregion
resultBatch.Count = count;
resultBatch.Seal();
// Return source columns as necessary.
// When the query is transformed, this is the ""non-swinging"" fields.
// Otherwise it is all source fields
");
foreach (var sourceField in this.nonSwingingFields)
{
if (sourceField.OptimizeString())
{
this.Write(" sourceBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceField.Name));
this.Write(".Dispose();\r\n");
}
else
{
this.Write(" sourceBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceField.Name));
this.Write(".Return();\r\n");
}
}
if (payloadType.CanContainNull())
{
this.Write(" sourceBatch._nullnessvector.ReturnClear();\r\n");
}
this.Write("\r\n sourceBatch.Return();\r\n this.Observer.OnNext(resultBatch);\r\n " +
"}\r\n\r\n public override int CurrentlyBufferedOutputCount => 0;\r\n\r\n public ov" +
"erride int CurrentlyBufferedInputCount => 0;\r\n}\r\n");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write("// *********************************************************************\r\n// Copy" +
"right (c) Microsoft Corporation. All rights reserved.\r\n// Licensed under the MI" +
"T License\r\n// ******************************************************************" +
"***\r\n");
var leftBatchTypeName = string.Format("{0}{1}",
Transformer.GetBatchClassName(keyType, leftType),
!String.IsNullOrWhiteSpace(TKeyTLeftGenericParameters) ? "<" + TKeyTLeftGenericParameters + ">" : string.Empty);
var rightBatchTypeName = string.Format("{0}{1}",
Transformer.GetBatchClassName(keyType, rightType),
!String.IsNullOrWhiteSpace(TKeyTRightGenericParameters) ? "<" + TKeyTRightGenericParameters + ">" : string.Empty);
this.Write(@"
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using System.Threading;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Aggregates;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
// TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TLeft: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("\r\n// TRight: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("\r\n// TResult: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : BinaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">\r\n{\r\n private const int DefaultCapacity = 64;\r\n private readonly MemoryPoo" +
"l<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> pool;\r\n private readonly Func<PlanNode, PlanNode, IBinaryObserver, BinaryPla" +
"nNode> queryPlanGenerator;\r\n\r\n /* Comparer for ordering of join key - should " +
"eventually be a stream property */\r\n\r\n private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> genericOutputBatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, resultType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTResultGenericParameters));
this.Write(" output;\r\n\r\n [DataMember]\r\n private long nextLeftTime = long.MinValue;\r\n " +
" [DataMember]\r\n private long nextRightTime = long.MinValue;\r\n [DataMember]" +
"\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" nextLeftKey;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" nextRightKey;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" currentRightKey;\r\n [DataMember]\r\n private List<ActiveEventRight> currentRi" +
"ghtList;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" currentLeftKey;\r\n [DataMember]\r\n private List<ActiveEventLeft> currentLeft" +
"List;\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(@"> observer,
Func<PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> queryPlanGenerator)
: base(stream, observer)
{
currentLeftList = new List<ActiveEventLeft>();
currentRightList = new List<ActiveEventRight>();
pool = MemoryManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TResult));
this.Write(">(true /*stream.Properties.IsColumnar*/);\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write(@"
output.Allocate();
this.queryPlanGenerator = queryPlanGenerator;
}
protected override void FlushContents()
{
if (output.Count == 0) return;
output.Seal();
this.Observer.OnNext(output);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write(@"
output.Allocate();
}
public override int CurrentlyBufferedOutputCount => output.Count;
public override int CurrentlyBufferedLeftInputCount => base.CurrentlyBufferedLeftInputCount + currentLeftList.Count;
public override int CurrentlyBufferedRightInputCount => base.CurrentlyBufferedRightInputCount + currentRightList.Count;
protected override void ProduceBinaryQueryPlan(PlanNode left, PlanNode right)
{
Observer.ProduceQueryPlan(queryPlanGenerator(left, right, this));
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void DisposeState() => output.Free();
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessBothBatches(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write("> leftBatch, StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("> rightBatch, out bool leftBatchDone, out bool rightBatchDone, out bool leftBatch" +
"Free, out bool rightBatchFree)\r\n {\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(" generatedLeftBatch = null;\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName));
this.Write(" generatedRightBatch = null;\r\n\r\n leftBatchFree = rightBatchFree = true;\r\n " +
" generatedLeftBatch = (");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(@") leftBatch;
if (!GoToVisibleRow(leftBatch))
{
leftBatchDone = true;
rightBatchDone = false;
return;
}
UpdateNextLeftTime(leftBatch.vsync.col[leftBatch.iter]);
nextLeftKey = leftBatch.key.col[leftBatch.iter];
generatedRightBatch = (");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName));
this.Write(@") rightBatch;
if (!GoToVisibleRow(rightBatch))
{
leftBatchDone = false;
rightBatchDone = true;
return;
}
UpdateNextRightTime(rightBatch.vsync.col[rightBatch.iter]);
nextRightKey = rightBatch.key.col[rightBatch.iter];
while (true)
{
bool leftPunctuation = leftBatch.vother.col[leftBatch.iter] == StreamEvent.PunctuationOtherTime;
bool rightPunctuation = rightBatch.vother.col[rightBatch.iter] == StreamEvent.PunctuationOtherTime;
int compare = (leftPunctuation || rightPunctuation) ? 0 : ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "nextRightKey")));
this.Write(@";
if (compare == 0)
{
if (nextLeftTime <= nextRightTime)
{
// process left
if (leftPunctuation)
{
AddPunctuationToBatch(nextLeftTime);
}
else
{
#region ProcessLeftStartEdge
/*
ProcessLeftStartEdge(
nextLeftTime,
ref leftBatch.key.col[leftBatch.iter],
leftBatch[leftBatch.iter],
leftBatch.hash.col[leftBatch.iter], compare);
*/
{
if (currentRightList.Count > 0)
{
int compare2 = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "currentRightKey")));
this.Write(@";
Contract.Assert(compare2 >= 0, ""Unexpected comparison in sort-ordered join"");
if (compare2 == 0)
{
// perform the join
for (int i = 0; i < currentRightList.Count; i++)
{
var t = currentRightList[i];
OutputStartEdge(nextLeftTime > t.Timestamp ? nextLeftTime : t.Timestamp, ref nextLeftKey, generatedLeftBatch, leftBatch.iter, t, leftBatch.hash.col[leftBatch.iter]);
}
}
else
{
// clear the right array
currentRightList.Clear();
}
}
if (compare >= 0)
{
// update the left array
if ((currentLeftList.Count != 0) && (");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "currentLeftKey")));
this.Write(" != 0))\r\n {\r\n C" +
"ontract.Assert(");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "currentLeftKey")));
this.Write(" > 0);\r\n currentLeftList.Clear();\r\n " +
" }\r\n currentLeftKey = nextLef" +
"tKey;\r\n var leftAE = new ActiveEventLeft();\r\n " +
" leftAE.Populate(ref nextLeftTime, generatedLeftBatch" +
", leftBatch.iter);\r\n currentLeftList.Add(leftAE);" +
"\r\n }\r\n }\r\n " +
" #endregion\r\n }\r\n\r\n leftBatch.iter++;\r\n" +
"\r\n if (!GoToVisibleRow(leftBatch))\r\n {\r\n " +
" leftBatchDone = true;\r\n rightBatchD" +
"one = false;\r\n return;\r\n }\r\n\r\n " +
" nextLeftTime = leftBatch.vsync.col[leftBatch.iter];\r\n " +
" nextLeftKey = leftBatch.key.col[leftBatch.iter];\r\n }\r\n " +
" else\r\n {\r\n // process right\r\n " +
" if (rightPunctuation)\r\n {\r\n " +
" AddPunctuationToBatch(nextRightTime);\r\n }\r\n " +
" else\r\n {\r\n #region ProcessRightSt" +
"artEdge\r\n /* Inlined version of:\r\n " +
" ProcessRightStartEdge(\r\n nextRightTime,\r\n " +
" ref rightBatch.key.col[rightBatch.iter],\r\n " +
" rightBatch[rightBatch.iter],\r\n rightBatch." +
"hash.col[rightBatch.iter], compare);\r\n */\r\n\r\n " +
" if (currentLeftList.Count > 0)\r\n {\r\n " +
" int compare2 = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextRightKey", "currentLeftKey")));
this.Write(@";
Contract.Assert(compare2 >= 0, ""Unexpected comparison in sort-ordered join"");
if (compare2 == 0)
{
// perform the join
for (int i = 0; i < currentLeftList.Count; i++)
{
var t = currentLeftList[i];
#region OutputStartEdge
/* OutputStartEdge(nextRightTime > t.Timestamp ? nextRightTime : t.Timestamp,
ref nextRightKey, ref t.Payload, ref rightP, rightBatch.hash.col[rightBatch.iter]); */
int index = output.Count++;
output.vsync.col[index] = nextRightTime > t.Timestamp ? nextRightTime : t.Timestamp;
output.vother.col[index] = StreamEvent.InfinitySyncTime;
output.key.col[index] = nextRightKey;
");
this.Write(this.ToStringHelper.ToStringWithCulture(this.leftMessageRepresentation.noFields
? this.rightBatchSelector("t.payload", "generatedRightBatch", "rightBatch.iter")
: this.rightBatchSelector("t", "generatedRightBatch", "rightBatch.iter")));
this.Write(@"
output.hash.col[index] = rightBatch.hash.col[rightBatch.iter];
if (output.Count == Config.DataBatchSize)
{
output.Seal();
this.Observer.OnNext(output);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write(@"
output.Allocate();
}
#endregion
}
}
else
{
// clear the left array
currentLeftList.Clear();
}
}
if (compare <= 0)
{
// update the right array
if ((currentRightList.Count != 0) && (");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("rightBatch.key.col[rightBatch.iter]", "currentRightKey")));
this.Write(" != 0))\r\n {\r\n Contract." +
"Assert(");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("rightBatch.key.col[rightBatch.iter]", "currentRightKey")));
this.Write(" > 0);\r\n currentRightList.Clear();\r\n " +
" }\r\n currentRightKey = rightBatch.key.co" +
"l[rightBatch.iter];\r\n var rightAE = new ActiveEventRi" +
"ght();\r\n rightAE.Populate(ref nextRightTime, generate" +
"dRightBatch, rightBatch.iter);\r\n currentRightList.Add" +
"(rightAE);\r\n }\r\n #endregion\r\n " +
" }\r\n\r\n rightBatch.iter++;\r\n\r\n " +
" #region GoToVisibleRow\r\n /* Inlined version of:\r\n " +
" if (!GoToVisibleRow(rightBatch))\r\n {\r\n " +
" leftBatchDone = false;\r\n right" +
"BatchDone = true;\r\n return;\r\n " +
"}\r\n */\r\n while (rightBatch.iter < rightBat" +
"ch.Count &&\r\n (rightBatch.bitvector.col[rightBatch.iter >" +
"> 6] & (1L << (rightBatch.iter & 0x3f))) != 0 &&\r\n rightB" +
"atch.vother.col[rightBatch.iter] >= 0)\r\n {\r\n " +
" rightBatch.iter++;\r\n }\r\n if (rightB" +
"atch.iter == rightBatch.Count)\r\n {\r\n l" +
"eftBatchDone = false;\r\n rightBatchDone = true;\r\n " +
" return;\r\n }\r\n #endregion\r\n\r" +
"\n nextRightTime = rightBatch.vsync.col[rightBatch.iter];\r\n " +
" nextRightKey = rightBatch.key.col[rightBatch.iter];\r\n " +
" }\r\n }\r\n else if (compare < 0)\r\n {\r\n " +
" // process left\r\n #region ProcessLeftStartEdge\r\n " +
" /* Inlined version of:\r\n ProcessLeftStartEdge(\r\n " +
" nextLeftTime,\r\n ref leftBatch.key.col" +
"[leftBatch.iter],\r\n leftBatch[leftBatch.iter],\r\n " +
" leftBatch.hash.col[leftBatch.iter], compare);\r\n */" +
"\r\n {\r\n\r\n\r\n if (currentRightList.Count > 0)\r\n " +
" {\r\n int compare2 = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "currentRightKey")));
this.Write(";\r\n\r\n Contract.Assert(compare2 >= 0, \"Unexpected compariso" +
"n in sort-ordered join\");\r\n\r\n if (compare2 == 0)\r\n " +
" {\r\n // perform the join\r\n " +
" for (int i = 0; i < currentRightList.Count; i++)\r\n " +
" {\r\n var t = currentRightList[i];\r\n" +
" OutputStartEdge(nextLeftTime > t.Timestamp ? nex" +
"tLeftTime : t.Timestamp, ref nextLeftKey, generatedLeftBatch, leftBatch.iter, t," +
" leftBatch.hash.col[leftBatch.iter]);\r\n }\r\n " +
" }\r\n else\r\n {\r\n " +
" // clear the right array\r\n curr" +
"entRightList.Clear();\r\n }\r\n }\r\n " +
" }\r\n #endregion\r\n\r\n leftBatch.iter++;\r\n\r\n " +
" #region GoToVisibleRow\r\n /* Inlined version of:\r\n " +
" if (!GoToVisibleRow(leftBatch))\r\n {\r\n " +
" leftBatchDone = true;\r\n rightBatchDone" +
" = false;\r\n return;\r\n }\r\n " +
" */\r\n while (leftBatch.iter < leftBatch.Count &&\r\n " +
" (leftBatch.bitvector.col[leftBatch.iter >> 6] & (1L << (leftBatch.iter & " +
"0x3f))) != 0 &&\r\n leftBatch.vother.col[leftBatch.iter] >= 0)\r" +
"\n {\r\n leftBatch.iter++;\r\n }\r\n " +
" if (leftBatch.iter == leftBatch.Count)\r\n {\r\n " +
" leftBatchDone = true;\r\n rightBatchDone = false;\r" +
"\n return;\r\n }\r\n #endregion\r\n\r\n\r" +
"\n #region UpdateNextLeftTime\r\n nextLeftTime = left" +
"Batch.vsync.col[leftBatch.iter];\r\n #endregion\r\n\r\n " +
"nextLeftKey = leftBatch.key.col[leftBatch.iter];\r\n\r\n }\r\n e" +
"lse // hot path if larger right side of join matches very few things on the left" +
" side\r\n {\r\n // process right\r\n #region " +
"ProcessRightStartEdge\r\n /* Inlined version of:\r\n P" +
"rocessRightStartEdge(\r\n nextRightTime,\r\n r" +
"ef rightBatch.key.col[rightBatch.iter],\r\n rightBatch.payload." +
"col[rightBatch.iter],\r\n rightBatch.hash.col[rightBatch.iter]," +
" compare);\r\n */\r\n if (currentLeftList.Count > 0)\r\n" +
" {\r\n int compare2 = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextRightKey", "currentLeftKey")));
this.Write(@";
Contract.Assert(compare2 >= 0, ""Unexpected comparison in sort-ordered join"");
if (compare2 == 0)
{
// perform the join
for (int i = 0; i < currentLeftList.Count; i++)
{
var t = currentLeftList[i];
#region OutputStartEdge
/* OutputStartEdge(nextRightTime > t.Timestamp ? nextRightTime : t.Timestamp,
ref nextRightKey, ref t.Payload, ref rightP, rightBatch.hash.col[rightBatch.iter]); */
int index = output.Count++;
output.vsync.col[index] = nextRightTime > t.Timestamp ? nextRightTime : t.Timestamp;
output.vother.col[index] = StreamEvent.InfinitySyncTime;
output.key.col[index] = nextRightKey;
");
this.Write(this.ToStringHelper.ToStringWithCulture(this.leftMessageRepresentation.noFields
? this.rightBatchSelector("t.payload", "generatedRightBatch", "rightBatch.iter")
: this.rightBatchSelector("t", "generatedRightBatch", "rightBatch.iter")));
this.Write(@"
output.hash.col[index] = rightBatch.hash.col[rightBatch.iter];
if (output.Count == Config.DataBatchSize)
{
output.Seal();
this.Observer.OnNext(output);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write(@"
output.Allocate();
}
#endregion
}
}
else
{
// clear the left array
currentLeftList.Clear();
}
}
/*
if (compare <= 0)
{
// update the right array
if ((currentRightList.Count != 0) && (");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("rightBatch.key.col[rightBatch.iter]", "currentRightKey")));
this.Write(" != 0))\r\n {\r\n Contract.Assert(");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("rightBatch.key.col[rightBatch.iter]", "currentRightKey")));
this.Write(" > 0);\r\n currentRightList.Clear();\r\n " +
" }\r\n currentRightKey = rightBatch.key.col[rightBatch" +
".iter];\r\n ActiveEvent<");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write("> rightAE = new ActiveEvent<");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write(">();\r\n rightAE.Populate(ref nextRightTime, ref rightBatch." +
"payload.col[rightBatch.iter]);\r\n currentRightList.Add(rig" +
"htAE);\r\n }\r\n */\r\n #endregion\r\n\r" +
"\n rightBatch.iter++;\r\n\r\n #region GoToVisibleRow\r\n " +
" /* Inlined version of:\r\n if (!GoToVisibleRow(r" +
"ightBatch))\r\n {\r\n leftBatchDone = fals" +
"e;\r\n rightBatchDone = true;\r\n retu" +
"rn;\r\n }\r\n */\r\n while (rightBatc" +
"h.iter < rightBatch.Count &&\r\n (rightBatch.bitvector.col[righ" +
"tBatch.iter >> 6] & (1L << (rightBatch.iter & 0x3f))) != 0 &&\r\n " +
" rightBatch.vother.col[rightBatch.iter] >= 0)\r\n {\r\n " +
" rightBatch.iter++;\r\n }\r\n if (rightBatch.it" +
"er == rightBatch.Count)\r\n {\r\n leftBatchDone = " +
"false;\r\n rightBatchDone = true;\r\n return;\r" +
"\n }\r\n #endregion\r\n\r\n #region Update" +
"NextRightTime\r\n /* Inlined version of: UpdateNextRightTime(rightB" +
"atch.vsync.col[rightBatch.iter]); */\r\n nextRightTime = rightBatch" +
".vsync.col[rightBatch.iter];\r\n #endregion\r\n\r\n next" +
"RightKey = rightBatch.key.col[rightBatch.iter];\r\n }\r\n }\r\n }" +
"\r\n\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n protected overri" +
"de void ProcessLeftBatch(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TLeft));
this.Write(@"> batch, out bool isBatchDone, out bool isBatchFree)
{
isBatchFree = true;
while (true)
{
if (!GoToVisibleRow(batch))
{
isBatchDone = true;
return;
}
UpdateNextLeftTime(batch.vsync.col[batch.iter]);
if (batch.vother.col[batch.iter] == StreamEvent.PunctuationOtherTime)
{
if (nextLeftTime > nextRightTime)
{
isBatchDone = false;
return;
}
AddPunctuationToBatch(batch.vsync.col[batch.iter]);
batch.iter++;
continue;
}
nextLeftKey = batch.key.col[batch.iter];
int compare = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "nextRightKey")));
this.Write(";\r\n if ((compare == 0) && (nextLeftTime <= nextRightTime))\r\n " +
" {\r\n ProcessLeftStartEdge(\r\n nextLeftTime,\r\n " +
" ref batch.key.col[batch.iter],\r\n (");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(@")batch, batch.iter,
batch.hash.col[batch.iter], compare);
batch.iter++;
}
else
{
isBatchDone = false;
return;
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
protected override void ProcessRightBatch(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TRight));
this.Write(@"> batch, out bool isBatchDone, out bool isBatchFree)
{
isBatchFree = true;
while (true)
{
if (!GoToVisibleRow(batch))
{
isBatchDone = true;
return;
}
UpdateNextRightTime(batch.vsync.col[batch.iter]);
if (batch.vother.col[batch.iter] == StreamEvent.PunctuationOtherTime)
{
if (nextRightTime > nextLeftTime)
{
isBatchDone = false;
return;
}
AddPunctuationToBatch(batch.vsync.col[batch.iter]);
batch.iter++;
continue;
}
nextRightKey = batch.key.col[batch.iter];
int compare = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("nextLeftKey", "nextRightKey")));
this.Write(";\r\n if ((compare == 0) && (nextRightTime <= nextLeftTime))\r\n " +
" {\r\n ProcessRightStartEdge(\r\n nextRightTime,\r" +
"\n ref batch.key.col[batch.iter],\r\n (");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName));
this.Write(@")batch,batch.iter,
batch.hash.col[batch.iter], compare);
batch.iter++;
}
else
{
isBatchDone = false;
return;
}
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool GoToVisibleRow<TPayload>(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", TPayload> batch)
{
while (batch.iter < batch.Count && (batch.bitvector.col[batch.iter >> 6] & (1L << (batch.iter & 0x3f))) != 0 && batch.vother.col[batch.iter] >= 0)
{
batch.iter++;
}
if (batch.iter == batch.Count)
{
return false;
}
return true;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void UpdateNextLeftTime(long time)
{
nextLeftTime = time;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void UpdateNextRightTime(long time)
{
nextRightTime = time;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ProcessLeftStartEdge(long start, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(" leftBatch, int leftIndex, int hash, int nextLeftRightCompareResult)\r\n {\r\n " +
" if (currentRightList.Count > 0)\r\n {\r\n int compare = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentRightKey")));
this.Write(@";
Contract.Assert(compare >= 0, ""Unexpected comparison in sort-ordered join"");
if (compare == 0)
{
// perform the join
for (int i = 0; i < currentRightList.Count; i++)
{
var t = currentRightList[i];
OutputStartEdge(start > t.Timestamp ? start : t.Timestamp, ref key, leftBatch, leftIndex, t, hash);
}
}
else
{
// clear the right array
currentRightList.Clear();
}
}
if (nextLeftRightCompareResult >= 0)
{
// update the left array
if ((currentLeftList.Count != 0) && (");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentLeftKey")));
this.Write(" != 0))\r\n {\r\n Contract.Assert(");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentLeftKey")));
this.Write(@" > 0);
currentLeftList.Clear();
}
currentLeftKey = key;
var leftAE = new ActiveEventLeft();
leftAE.Populate(ref start, leftBatch, leftIndex);
currentLeftList.Add(leftAE);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ProcessRightStartEdge(long start, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName));
this.Write(" rightBatch, int rightIndex, int hash, int nextLeftRightCompareResult)\r\n {\r\n " +
" if (currentLeftList.Count > 0)\r\n {\r\n int compare = ");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentLeftKey")));
this.Write(@";
Contract.Assert(compare >= 0, ""Unexpected comparison in sort-ordered join"");
if (compare == 0)
{
// perform the join
for (int i = 0; i < currentLeftList.Count; i++)
{
var t = currentLeftList[i];
OutputStartEdge(start > t.Timestamp ? start : t.Timestamp, ref key, t, rightBatch, rightIndex, hash);
}
}
else
{
// clear the left array
currentLeftList.Clear();
}
}
if (nextLeftRightCompareResult <= 0)
{
// update the right array
if ((currentRightList.Count != 0) && (");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentRightKey")));
this.Write(" != 0))\r\n {\r\n Contract.Assert(");
this.Write(this.ToStringHelper.ToStringWithCulture(joinKeyOrderComparer("key", "currentRightKey")));
this.Write(@" > 0);
currentRightList.Clear();
}
currentRightKey = key;
var rightAE = new ActiveEventRight();
rightAE.Populate(ref start, rightBatch, rightIndex);
currentRightList.Add(rightAE);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void AddPunctuationToBatch(long start)
{
if (start > lastCTI)
{
lastCTI = start;
int index = output.Count++;
output.vsync.col[index] = start;
output.vother.col[index] = StreamEvent.PunctuationOtherTime;
output.key.col[index] = default;
output[index] = default;
output.hash.col[index] = 0;
output.bitvector.col[index >> 6] |= (1L << (index & 0x3f));
if (output.Count == Config.DataBatchSize) FlushContents();
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void OutputStartEdge(long start, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ActiveEventLeft leftEvent, ");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName));
this.Write(" rightBatch, int rightIndex, int hash)\r\n {\r\n int index = output.Count++" +
";\r\n output.vsync.col[index] = start;\r\n output.vother.col[index] = " +
"StreamEvent.InfinitySyncTime;\r\n output.key.col[index] = key;\r\n");
if (this.leftMessageRepresentation.noFields)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchSelector("leftEvent.payload", "rightBatch", "rightIndex")));
this.Write("\r\n");
}
else
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchSelector("leftEvent", "rightBatch", "rightIndex")));
this.Write("\r\n");
}
this.Write(" output.hash.col[index] = hash;\r\n\r\n if (output.Count == Config.Data" +
"BatchSize) FlushContents();\r\n }\r\n [MethodImpl(MethodImplOptions.Aggressive" +
"Inlining)]\r\n private void OutputStartEdge(long start, ref ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(" key, ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(@" leftBatch, int leftIndex, ActiveEventRight rightEvent, int hash)
{
int index = output.Count++;
output.vsync.col[index] = start;
output.vother.col[index] = StreamEvent.InfinitySyncTime;
output.key.col[index] = key;
");
if (this.rightMessageRepresentation.noFields)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchSelector("leftBatch", "leftIndex", "rightEvent.payload")));
this.Write("\r\n");
}
else
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchSelector("leftBatch", "leftIndex", "rightEvent")));
this.Write("\r\n");
}
this.Write(" output.hash.col[index] = hash;\r\n\r\n if (output.Count == Config.Data" +
"BatchSize) FlushContents();\r\n }\r\n\r\n [DataContract]\r\n private struct Act" +
"iveEventLeft\r\n {\r\n [DataMember]\r\n public long Timestamp;\r\n\r\n " +
" ");
foreach (var f in this.leftFields)
{
this.Write("\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax()));
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n ");
}
this.Write("\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n public void" +
" Populate(ref long timestamp, ");
this.Write(this.ToStringHelper.ToStringWithCulture(leftBatchTypeName));
this.Write(" batch, int index)\r\n {\r\n Timestamp = timestamp;\r\n ");
foreach (var f in this.leftFields)
{
this.Write("\r\n this.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(" = ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index")));
this.Write(";\r\n ");
}
this.Write(@"
}
public override string ToString()
{
return ""[Timestamp='"" + Timestamp + ""', Payload='"" + ""']"";
}
}
[DataContract]
private struct ActiveEventRight
{
[DataMember]
public long Timestamp;
");
foreach (var f in this.rightFields)
{
this.Write("\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Type.GetCSharpSourceSyntax()));
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(";\r\n ");
}
this.Write("\r\n [MethodImpl(MethodImplOptions.AggressiveInlining)]\r\n public void" +
" Populate(ref long timestamp, ");
this.Write(this.ToStringHelper.ToStringWithCulture(rightBatchTypeName));
this.Write(" batch, int index)\r\n {\r\n Timestamp = timestamp;\r\n ");
foreach (var f in this.rightFields)
{
this.Write("\r\n this.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.OriginalName));
this.Write(" = ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.AccessExpressionForRowValue("batch", "index")));
this.Write(";\r\n ");
}
this.Write("\r\n }\r\n\r\n public override string ToString()\r\n {\r\n " +
"return \"[Timestamp=\'\" + Timestamp + \"\', Payload=\'\" + \"\']\";\r\n }\r\n }\r\n}");
return(this.GenerationEnvironment.ToString());
}
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new StartEdgeEquiJoinTemplate($"GeneratedStartEdgeEquiJoin_{StartEdgeEquiJoinSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult));
var gps = template.tm.GenericTypeVariables(template.keyType, template.leftType, template.rightType, template.resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType);
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType);
template.rightFields = template.rightMessageRepresentation.AllFields;
var resultRepresentation = new ColumnarRepresentation(template.resultType);
#region Key Comparer
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType);
template.TKeyTLeftGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType);
template.TKeyTRightGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.rightType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType);
template.TKeyTResultGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString();
template.outputFields = resultRepresentation.AllFields;
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
#region LeftBatchSelector
{
var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var parameterMap = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex },
{ selector.Parameters[1], rightEvent }
};
if (projectionResult.ProjectionReturningResultInstance != null)
{
return($"this.output[index] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
else
{
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendLine($"this.output.{f.Name}.AddString({e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)});");
}
else
{
sb.AppendLine($"this.output.{f.Name}.col[index] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
}
sb.AppendLine("}");
return(sb.ToString());
}
};
}
#endregion
#region RightBatchSelector
{
var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var parameterMap = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex }
};
if (projectionResult.ProjectionReturningResultInstance != null)
{
return($"this.output[index] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
else
{
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendLine($"this.output.{f.Name}.AddString({e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)});");
}
else
{
sb.AppendLine($"this.output.{f.Name}.col[index] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
}
sb.AppendLine("}");
return(sb.ToString());
}
};
}
#endregion
return(template.Generate <TKey, TLeft, TRight, TResult>(selector));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Generate a batch class definition to be used as a Clip pipe.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TLeft">The payload type for the left side.</typeparam>
/// <typeparam name="TRight">The payload type for the right side.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TKey"/>, <typeparamref name="TRight"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight>(ClipJoinStreamable <TKey, TLeft, TRight> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TLeft>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new ClipJoinTemplate();
var keyType = template.keyType = typeof(TKey);
var leftType = template.leftType = typeof(TLeft);
var rightType = template.rightType = typeof(TRight);
var tm = new TypeMapper(keyType, leftType, rightType);
template.TKey = tm.CSharpNameFor(keyType);
template.TLeft = tm.CSharpNameFor(leftType);
template.TRight = tm.CSharpNameFor(rightType);
var gps = tm.GenericTypeVariables(keyType, leftType, rightType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.className = string.Format("GeneratedClip_{0}", ClipSequenceNumber++);
var resultRepresentation = new ColumnarRepresentation(leftType);
template.ActiveEventType = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event";
#region Key Comparer
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
#region Left Comparer
var leftComparer = stream.LeftComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftComparer, "index_ProcessLeftEvent", 0);
template.leftComparer = (left, right) => newLambda.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
template.TKeyTLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.TKeyTRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString();
template.leftFields = resultRepresentation.AllFields;
template.noFields = resultRepresentation.noFields;
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(leftComparer));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType);
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(leftType.GetAnonymousTypes());
list.AddRange(rightType.GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write("\r\n");
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
using Microsoft.StreamProcessing.Aggregates;
");
List <string> genericParamList = new List <string>();
int oldCount = 0;
var TKey = keyType.GetCSharpSourceSyntax(ref genericParamList);
var keyGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TInput = inputType.GetCSharpSourceSyntax(ref genericParamList);
var inputGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TState = stateType.GetCSharpSourceSyntax(ref genericParamList);
var stateGenericParameters = new List <string>(genericParamList.Skip(oldCount));
oldCount = genericParamList.Count;
var TOutput = outputType.GetCSharpSourceSyntax(ref genericParamList);
var outputGenericParameters = new List <string>(genericParamList.Skip(oldCount));
var genericParameters = genericParamList.BracketedCommaSeparatedString();
var TKeyTInputGenericParameters = keyGenericParameters.Concat(inputGenericParameters).BracketedCommaSeparatedString();
var TKeyTOutputGenericParameters = keyGenericParameters.Concat(outputGenericParameters).BracketedCommaSeparatedString();
var BatchGeneratedFrom_TKey_TInput = Transformer.GetBatchClassName(keyType, inputType);
var genericParameters2 = string.Format("<{0}, {1}>", TKey, TOutput);
if (!keyType.KeyTypeNeedsGeneratedMemoryPool() && outputType.MemoryPoolHasGetMethodFor())
{
genericParameters2 = string.Empty;
}
else if (!outputType.CanRepresentAsColumnar())
{
genericParameters2 = string.Empty;
}
Func <string, string> assignToOutput = rhs =>
this.outputType.IsAnonymousType()
?
rhs
:
(
this.outputFields.Count() == 1
?
string.Format("this.batch.{0}.col[c] = {1};", this.outputFields.First().Name, rhs)
:
"temporaryOutput = " + rhs + ";\r\n" + String.Join("\r\n", this.outputFields.Select(f => "dest_" + f.Name + "[c] = temporaryOutput." + f.OriginalName + ";")))
;
var getOutputBatch = string.Format("this.pool.Get(out genericOutputbatch); this.batch = ({0}{1})genericOutputbatch;",
Transformer.GetBatchClassName(keyType, outputType),
TKeyTOutputGenericParameters);
this.Write("[assembly: IgnoresAccessChecksTo(\"Microsoft.StreamProcessing\")]\r\n\r\n// genericPara" +
"ms2 = \"");
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write("\"\r\n\r\n[DataContract]\r\nstruct StateAndActive\r\n{\r\n [DataMember]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(" state;\r\n [DataMember]\r\n public ulong active;\r\n}\r\n\r\n[DataContract]\r\nstruct " +
"HeldStateStruct\r\n{\r\n [DataMember]\r\n public long timestamp;\r\n [DataMembe" +
"r]\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(" state;\r\n}\r\n\r\n[DataContract]\r\nsealed class HeldState\r\n{\r\n [DataMember]\r\n pu" +
"blic long timestamp;\r\n [DataMember]\r\n public StateAndActive state;\r\n}\r\n\r\n/" +
"/ TKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("\r\n// TInput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("\r\n// TState: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("\r\n// TOutput: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("\r\n/// <summary>\r\n/// Operator only has to deal with start edges\r\n/// </summary>\r\n" +
"[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" : UnaryPipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(">\r\n{\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanGenera" +
"tor;\r\n private readonly IAggregate<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> aggregate;\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(" pool;\r\n\r\n private StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> genericOutputbatch;\r\n [DataMember]\r\n private ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetBatchClassName(keyType, outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTOutputGenericParameters));
this.Write(" batch;\r\n\r\n");
if (this.useCompiledInitialState)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> initialState;\r\n");
}
if (this.useCompiledAccumulate)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", long, ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write("> accumulate;\r\n");
}
if (this.useCompiledComputeResult)
{
this.Write(" private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> computeResult;\r\n");
}
this.Write("\r\n private readonly IEqualityComparerExpression<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write("> keyComparer;\r\n\r\n");
if (!this.isUngrouped)
{
this.Write(" [DataMember]\r\n private FastDictionary3<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldStateStruct> heldAggregates;\r\n");
}
else
{
this.Write(" [DataMember]\r\n private HeldStateStruct currentState;\r\n [DataMember]\r\n " +
" private bool currentStateExists;\r\n [DataMember]\r\n private bool isDirty;\r" +
"\n");
}
this.Write("\r\n [DataMember]\r\n private long lastSyncTime = long.MinValue;\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n Streamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> observer,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerator,\r\n" +
" IAggregate<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TState));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write("> aggregate)\r\n : base(stream, observer)\r\n {\r\n this.aggregate = a" +
"ggregate;\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n\r\n");
if (this.useCompiledInitialState)
{
this.Write(" initialState = aggregate.InitialState().Compile();\r\n");
}
if (this.useCompiledAccumulate)
{
this.Write(" accumulate = aggregate.Accumulate().Compile();\r\n");
}
if (this.useCompiledComputeResult)
{
this.Write(" computeResult = aggregate.ComputeResult().Compile();\r\n");
}
this.Write("\r\n this.keyComparer = stream.Properties.KeyEqualityComparer;\r\n\r\n th" +
"is.pool = MemoryManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(">() as ");
this.Write(this.ToStringHelper.ToStringWithCulture(Transformer.GetMemoryPoolClassName(this.keyType, this.outputType)));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters2));
this.Write(";\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n\r\n");
if (!this.isUngrouped)
{
this.Write(" var generator = this.keyComparer.CreateFastDictionary3Generator<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", HeldStateStruct>(1, this.keyComparer.GetEqualsExpr().Compile(), this.keyCompare" +
"r.GetGetHashCodeExpr().Compile(), stream.Properties.QueryContainer);\r\n th" +
"is.heldAggregates = generator.Invoke();\r\n");
}
else
{
this.Write(" isDirty = false;\r\n");
}
this.Write(@" }
public override void ProduceQueryPlan(PlanNode previous)
=> Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));
protected override void FlushContents()
{
if (this.batch == null || this.batch.Count == 0) return;
this.batch.Seal();
this.Observer.OnNext(this.batch);
");
this.Write(this.ToStringHelper.ToStringWithCulture(getOutputBatch));
this.Write("\r\n this.batch.Allocate();\r\n }\r\n\r\n protected override void DisposeSta" +
"te() => this.batch.Free();\r\n\r\n public override int CurrentlyBufferedOutputCou" +
"nt => this.batch.Count;\r\n\r\n public override int CurrentlyBufferedInputCount =" +
"> ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.isUngrouped ? "0" : "this.heldAggregates.Count"));
this.Write(";\r\n\r\n public override unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInput));
this.Write("> inputBatch)\r\n {\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters));
this.Write(" generatedBatch = inputBatch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(BatchGeneratedFrom_TKey_TInput));
this.Write(this.ToStringHelper.ToStringWithCulture(TKeyTInputGenericParameters));
this.Write(";\r\n\r\n var count = generatedBatch.Count;\r\n\r\n");
if (this.outputFields.Count() > 1)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(" temporaryOutput;\r\n");
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the input batch.\r\n");
foreach (var f in this.inputFields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = generatedBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = generatedBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n // Create locals that point directly to the arrays within the columns i" +
"n the output batch.\r\n");
foreach (var f in this.outputFields)
{
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write(@" var col_key = generatedBatch.key.col;
fixed (long* col_vsync = generatedBatch.vsync.col)
fixed (long* col_vother = generatedBatch.vother.col)
fixed (int* col_hash = generatedBatch.hash.col)
fixed (long* col_bv = generatedBatch.bitvector.col)
for (int i = 0; i < count; i++)
{
if ((col_bv[i >> 6] & (1L << (i & 0x3f))) != 0)
{
if (col_vother[i] == long.MinValue)
{
// We have found a row that corresponds to punctuation
OnPunctuation(col_vsync[i]);
int c = this.batch.Count;
this.batch.vsync.col[c] = col_vsync[i];
this.batch.vother.col[c] = long.MinValue;
this.batch.key.col[c] = default;
this.batch[c] = default;
this.batch.hash.col[c] = 0;
this.batch.bitvector.col[c >> 6] |= 1L << (c & 0x3f);
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
continue;
}
var syncTime = col_vsync[i];
// Handle time moving forward
if (syncTime > this.lastSyncTime)
{
");
if (this.isUngrouped)
{
this.Write(@" if (currentStateExists && isDirty) // there exists earlier state
{
int c = this.batch.Count;
this.batch.vsync.col[c] = currentState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state"))));
this.Write("\r\n this.batch.hash.col[c] = 0;\r\n this.batch" +
".Count++;\r\n if (this.batch.Count == Config.DataBatchSize) Flu" +
"shContents();\r\n }\r\n isDirty = false;\r\n");
}
else
{
this.Write(" int iter1 = FastDictionary3<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldStateStruct>.IteratorStart;
while (this.heldAggregates.IterateDirty(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
int c = this.batch.Count;
this.batch.vsync.col[c] = iter1entry.value.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value.state"))));
this.Write(@"
this.batch.key.col[c] = iter1entry.key;
this.batch.hash.col[c] = iter1entry.hash;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
// Time has moved forward, clean the held aggregates
this.heldAggregates.Clean();
");
}
this.Write("\r\n // Since sync time changed, set this.lastSyncTime\r\n " +
" this.lastSyncTime = syncTime;\r\n }\r\n\r\n");
if (this.isUngrouped)
{
this.Write(" if (!currentStateExists)\r\n {\r\n currentState" +
".state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@";
currentState.timestamp = syncTime;
currentStateExists = true;
isDirty = true;
}
else
{
if (!isDirty)
{
// Output end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = syncTime;
this.batch.vother.col[c] = currentState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state"))));
this.Write(@"
this.batch.hash.col[c] = 0;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
currentState.timestamp = syncTime;
isDirty = true;
}
}
currentState.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("currentState.state", "col_vsync[i]")));
this.Write(";\r\n");
}
else
{
this.Write(@" // Retrieve the key from the dictionary
var currentKey = col_key[i];
var currentHash = col_hash[i];
int index;
bool heldAggsLookup = false;
{
int num = currentHash & 0x7fffffff;
index = num % this.heldAggregates.Size;
do
{
if ((this.heldAggregates.bitvector[index >> 3] & (0x1 << (index & 0x7))) == 0)
{
heldAggsLookup = false;
break;
}
if ((currentHash == this.heldAggregates.entries[index].hash) && (");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedKeyComparerEquals("currentKey", "this.heldAggregates.entries[index].key")));
this.Write(@"))
{
heldAggsLookup = true;
break;
}
index++;
if (index == this.heldAggregates.Size)
index = 0;
} while (true);
}
if (!heldAggsLookup)
// if (!this.heldAggregates.Lookup(currentKey, currentHash, out index))
{
// New group. Create new state
HeldStateStruct currentState;
currentState.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(initialState));
this.Write(@";
currentState.timestamp = syncTime;
// No output because initial state is empty
this.heldAggregates.Insert(ref index, currentKey, currentState, currentHash);
}
else
{
// read new currentState from _heldAgg index
var currentState = this.heldAggregates.entries[index].value;
if (this.heldAggregates.IsClean(ref index))
{
// Output end edge
int c = this.batch.Count;
this.batch.vsync.col[c] = syncTime;
this.batch.vother.col[c] = currentState.timestamp;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state"))));
this.Write(@"
this.batch.key.col[c] = currentKey;
this.batch.hash.col[c] = currentHash;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
currentState.timestamp = syncTime;
this.heldAggregates.SetDirty(ref index);
}
}
this.heldAggregates.entries[index].value.state = ");
this.Write(this.ToStringHelper.ToStringWithCulture(accumulate("this.heldAggregates.entries[index].value.state", "col_vsync[i]")));
this.Write(";\r\n");
}
this.Write(" }\r\n\r\n");
foreach (var f in this.inputFields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
foreach (var f in this.outputFields.Where(fld => fld.canBeFixed))
{
this.Write(" }\r\n");
}
this.Write("\r\n generatedBatch.Release();\r\n generatedBatch.Return();\r\n }\r\n\r\n " +
" public void OnPunctuation(long syncTime)\r\n {\r\n");
if (this.outputFields.Count() > 1)
{
this.Write(" ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOutput));
this.Write(" temporaryOutput;\r\n");
foreach (var f in this.outputFields)
{
this.Write(" var dest_");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = this.batch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
this.Write("\r\n // Handle time moving forward\r\n if (syncTime > this.lastSyncTime" +
")\r\n {\r\n");
if (this.isUngrouped)
{
this.Write(@" if (currentStateExists && isDirty) // need to send start edge if state is dirty
{
int c = this.batch.Count;
this.batch.vsync.col[c] = currentState.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("currentState.state"))));
this.Write("\r\n\r\n this.batch.hash.col[c] = 0;\r\n this.batch.Count" +
"++;\r\n if (this.batch.Count == Config.DataBatchSize) FlushContents" +
"();\r\n }\r\n isDirty = false;\r\n");
}
else
{
this.Write(" int iter1 = FastDictionary3<");
this.Write(this.ToStringHelper.ToStringWithCulture(TKey));
this.Write(@", HeldStateStruct>.IteratorStart;
while (this.heldAggregates.IterateDirty(ref iter1))
{
var iter1entry = this.heldAggregates.entries[iter1];
int c = this.batch.Count;
this.batch.vsync.col[c] = iter1entry.value.timestamp;
this.batch.vother.col[c] = StreamEvent.InfinitySyncTime;
");
this.Write(this.ToStringHelper.ToStringWithCulture(assignToOutput(computeResult("iter1entry.value.state"))));
this.Write(@"
this.batch.key.col[c] = iter1entry.key;
this.batch.hash.col[c] = iter1entry.hash;
this.batch.Count++;
if (this.batch.Count == Config.DataBatchSize) FlushContents();
}
// Time has moved forward, clean the held aggregates
this.heldAggregates.Clean();
");
}
this.Write("\r\n // Since sync time changed, set this.lastSyncTime\r\n this" +
".lastSyncTime = syncTime;\r\n }\r\n }\r\n}//\r\n");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Generate a batch class definition to be used as an aggreate definition.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for the aggregate.</typeparam>
/// <typeparam name="TInput">The input type for the aggregate.</typeparam>
/// <typeparam name="TOutput">The output type for the aggregate.</typeparam>
/// <typeparam name="TState">The type for the accumulated state held by the aggregate.</typeparam>
/// <typeparam name="TResult">The type of the result.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of UnaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TInput"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TInput, TState, TOutput, TResult>(
GroupedWindowStreamable <TKey, TInput, TState, TOutput, TResult> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(IStreamObserver <Empty, TInput>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
string expandedCode;
var template = new GroupedWindowTemplate();
var keyType = template.keyType = typeof(TKey);
var inputType = template.inputType = typeof(TInput);
var stateType = template.stateType = typeof(TState);
var outputType = template.outputType = typeof(TOutput);
var resultType = template.resultType = typeof(TResult);
template.TResult = resultType.GetCSharpSourceSyntax(); // BUGBUG: need to get any generic parameters needed
template.isUngrouped = (keyType == typeof(Empty));
template.className = string.Format("GeneratedGroupedAggregate_{0}", GroupedAggregateSequenceNumber++);
var inputMessageRepresentation = new ColumnarRepresentation(inputType);
var resultRepresentation = new ColumnarRepresentation(resultType);
var assemblyReferences = new List <Assembly>();
#region Key Selector
var keySelector = stream.KeySelector;
string transformedKeySelectorAsString;
if (keyType.IsAnonymousTypeName())
{
Contract.Assume(keySelector.Body is NewExpression);
var transformedFunction = Extensions.TransformUnaryFunction <TKey, TInput>(keySelector);
var newBody = (NewExpression)transformedFunction.Body;
transformedKeySelectorAsString = string.Join(",", newBody.Arguments);
}
else
{
var transformedFunction = Extensions.TransformUnaryFunction <Empty, TInput>(keySelector).Body;
if (transformedFunction == null)
{
return(null);
}
transformedKeySelectorAsString = transformedFunction.ExpressionToCSharp();
}
template.keySelector = transformedKeySelectorAsString;
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector));
#endregion
#region Key Comparer and HashCode
var keyComparer = EqualityComparerExpression <TKey> .Default;
template.keyComparerEquals =
(left, right) =>
keyComparer.GetEqualsExpr().Inline(left, right);
template.keyComparerGetHashCode =
(x) =>
keyComparer.GetGetHashCodeExpr().Inline(x);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyComparer.GetEqualsExpr()));
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyComparer.GetGetHashCodeExpr()));
#endregion
#region Aggregate functions
var initialStateLambda = stream.Aggregate.InitialState();
if (ConstantExpressionFinder.IsClosedExpression(initialStateLambda))
{
template.initialState = initialStateLambda.Body.ExpressionToCSharp();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(initialStateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the initial state lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledInitialState = true;
template.initialState = "initialState()";
}
}
var accumulateLambda = stream.Aggregate.Accumulate();
if (ConstantExpressionFinder.IsClosedExpression(accumulateLambda))
{
var accTransformedLambda = Extensions.TransformFunction <TKey, TInput>(accumulateLambda, 2);
template.accumulate = (stateArg, longArg) => accTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(accumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the accumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledAccumulate = true;
template.accumulate = (s1, s2) => string.Format("accumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var deaccumulateLambda = stream.Aggregate.Deaccumulate();
if (ConstantExpressionFinder.IsClosedExpression(deaccumulateLambda))
{
var deaccumulateTransformedLambda = Extensions.TransformFunction <TKey, TInput>(deaccumulateLambda, 2);
template.deaccumulate = (stateArg, longArg) => deaccumulateTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(deaccumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
throw new InvalidOperationException("Code Generation couldn't inline a lambda!");
}
else
{
template.useCompiledDeaccumulate = true;
template.deaccumulate = (s1, s2) => string.Format("deaccumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var differenceLambda = stream.Aggregate.Difference();
if (ConstantExpressionFinder.IsClosedExpression(differenceLambda))
{
template.difference = (stateArg1, stateArg2) => differenceLambda.Inline(stateArg1, stateArg2);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(differenceLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the deaccumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledDifference = true;
template.deaccumulate = (s1, s2) => string.Format("difference({0}, {1});", s1, s2);
}
}
var computeResultLambda = stream.Aggregate.ComputeResult();
if (ConstantExpressionFinder.IsClosedExpression(computeResultLambda))
{
if (outputType.IsAnonymousType())
{
if (computeResultLambda.Body is NewExpression newExpression)
{
errorMessages = "Code Generation for GroupedWindow: result selector must be a new expression for anonymous types";
throw new NotImplementedException(errorMessages);
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(computeResultLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the result selector lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledComputeResult = true;
template.computeResult = (stateArg) => "computeResult(" + stateArg + ")";
}
}
#endregion
template.BatchGeneratedFrom_Unit_TInput = Transformer.GetBatchClassName(typeof(Empty), inputType);
template.UnitTInputGenericParameters = string.Empty; // BUGBUG
template.UnitTResultGenericParameters = string.Empty; // BUGBUG
template.inputFields = inputMessageRepresentation.AllFields;
template.outputFields = resultRepresentation.AllFields;
var resultSelector = stream.ResultSelector;
var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); // dont want the parameters substituted for at all
var projectionResult = SelectTransformer.Transform(resultSelector, parameterSubstitutions, resultRepresentation, true);
if (projectionResult.Error)
{
return(null);
}
template.finalResultSelector =
(key, aggregateResult) =>
{
var parameters = new Dictionary <ParameterExpression, string>
{
{ resultSelector.Parameters.ElementAt(0), key }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
sb.AppendLine(string.Format("var {0} = {1};\n", resultSelector.Parameters.ElementAt(1).Name, aggregateResult));
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
sb.AppendFormat("this.batch.{0}.col[_c] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(parameters));
}
sb.AppendLine("}");
return(sb.ToString());
};
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(resultSelector));
template.staticCtor = Transformer.StaticCtor(template.className);
expandedCode = template.TransformText();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(typeof(Empty), typeof(TKey), typeof(TInput), typeof(TState), typeof(TOutput), typeof(FastDictionaryGenerator3)));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TInput>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <Empty, TResult>());
var assembly = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = assembly.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(typeof(TInput).GetAnonymousTypes());
list.AddRange(typeof(TState).GetAnonymousTypes());
list.AddRange(typeof(TOutput).GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new FixedIntervalEquiJoinTemplate($"GeneratedFixedIntervalEquiJoin_{EquiJoinSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult))
{
leftDuration = stream.Left.Properties.ConstantDurationLength.Value,
rightDuration = stream.Right.Properties.ConstantDurationLength.Value
};
var keyAndLeftGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType).BracketedCommaSeparatedString();
var keyAndRightGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.rightType).BracketedCommaSeparatedString();
template.TKeyTResultGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString();
template.genericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType, template.rightType, template.resultType).BracketedCommaSeparatedString();
template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType);
template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType);
var resultMessageRepresentation = new ColumnarRepresentation(template.resultType);
var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType);
var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType);
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType);
template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters;
template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters;
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightFields = template.rightMessageRepresentation.AllFields;
template.resultFields = resultMessageRepresentation.AllFields;
template.ActiveEventTypeLeft = template.leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left";
template.ActiveEventTypeRight = template.rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right";
#region Key Equals
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparerEquals =
(left, right) =>
keyComparer.Inline(left, right);
if (template.keyType.IsAnonymousType())
{
template.keyComparerEquals =
(left, right) => $"keyComparerEquals({left}, {right})";
}
#endregion
#region Left Payload Equals
{
var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0);
template.leftComparerEquals = (left, right) => newLambda.Inline(left, right);
}
#endregion
#region Right Payload Equals
{
var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0);
template.rightComparerEquals = (left, right) => newLambda.Inline(left, right);
}
#endregion
#region Result Selector
{
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
errorMessages = "result selector is not a closed expression";
throw new InvalidOperationException();
}
#region LeftBatchSelector
{
var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region ActiveSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >();
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.activeSelector = (leftEvent, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
}
#endregion
return(template.Generate <TKey, TLeft, TRight, TResult>());
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
private static Tuple <Type, string> Generate <TKey, TSource, TResult>(
string ingressType,
string partitionString,
Expression <Func <TSource, TKey> > partitionExpression,
Expression <Func <TSource, long> > startEdgeExpression,
Expression <Func <TSource, long> > endEdgeExpression,
string latencyOption,
string diagnosticOption,
FuseModule fuseModule)
{
var template = new TemporalIngressTemplate(
$"GeneratedTemporalIngress_{TemporalIngressSequenceNumber++}",
typeof(TKey), typeof(TSource), fuseModule?.OutputType ?? typeof(TResult));
var tm = new TypeMapper(template.keyType, template.payloadType, template.resultType);
var gps = tm.GenericTypeVariables(template.keyType, template.payloadType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.payloadOrResult = tm.CSharpNameFor(template.resultRepresentation.RepresentationFor);
var batchGeneratedFrom_TKey_TPayload = string.IsNullOrEmpty(partitionString)
? Transformer.GetBatchClassName(template.keyType, template.resultType)
: Transformer.GetBatchClassName(typeof(PartitionKey <TKey>), template.resultType);
var keyAndPayloadGenericParameters = tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString();
template.GeneratedBatchName = batchGeneratedFrom_TKey_TPayload + keyAndPayloadGenericParameters;
template.keyOrNothing = string.IsNullOrEmpty(partitionString) ? string.Empty : template.TKey + ", ";
template.diagnosticOption = diagnosticOption;
template.needsStreamEvent = ingressType == "StreamEvent";
template.genericArguments = (string.IsNullOrEmpty(partitionString) ? string.Empty : template.TKey + ", ") + template.TPayload;
template.adjustedGenericArgs = (string.IsNullOrEmpty(partitionString) ? string.Empty : template.TKey + ", ") + (!fuseModule.IsEmpty && Config.AllowFloatingReorderPolicy ? template.TResult : template.TPayload);
template.emptyOrPartition = string.IsNullOrEmpty(partitionString) ? "Microsoft.StreamProcessing.Empty.Default" : "new PartitionKey<" + template.TKey + ">(value.PartitionKey)";
template.partitionString = partitionString;
template.baseStructure = partitionString + "StreamEvent<" + template.genericArguments + ">";
template.inheritBase = (ingressType != "StreamEvent") ? template.TPayload : template.baseStructure;
template.ingressType = ingressType;
template.latencyOption = latencyOption;
template.partitionFunction = x => partitionExpression == null ?
string.Empty : partitionExpression.Body.ExpressionToCSharpStringWithParameterSubstitution(
new Dictionary <ParameterExpression, string>
{
{ partitionExpression.Parameters.Single(), x }
});
template.startEdgeFunction = x => startEdgeExpression == null ?
string.Empty : startEdgeExpression.Body.ExpressionToCSharpStringWithParameterSubstitution(
new Dictionary <ParameterExpression, string>
{
{ startEdgeExpression.Parameters.Single(), x }
});
template.endEdgeFunction = x => endEdgeExpression == null ?
"StreamEvent.InfinitySyncTime" :
endEdgeExpression.Body.ExpressionToCSharpStringWithParameterSubstitution(
new Dictionary <ParameterExpression, string>
{
{ endEdgeExpression.Parameters.Single(), x }
});
template.fusionOption = fuseModule.IsEmpty ? "Simple" :
(Config.AllowFloatingReorderPolicy ? "Disordered" : "Fused");
template.resultMightBeNull = template.resultType.CanContainNull();
Expression[] expressions = null;
if (!fuseModule.IsEmpty)
{
template.valueString = fuseModule.Coalesce <TSource, TResult, TKey>("value.SyncTime", "value.OtherTime", "value.Payload", template.emptyOrPartition, out template.leadingText, out template.trailingText);
template.generatedEndTimeVariable = "generatedEndTimeVariable";
if (Config.AllowFloatingReorderPolicy)
{
template.valueString = "value.Payload";
template.generatedEndTimeVariable = "value.OtherTime";
}
expressions = fuseModule.GetCodeGenExpressions();
}
return(template.Generate <TKey, TSource, TResult>(new Type[] { typeof(IStreamable <,>) }, expressions));
}
/// <summary>
/// Generate a batch class definition to be used as StartEdgeEquiJoin operator.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TLeft">The payload type for the left side.</typeparam>
/// <typeparam name="TRight">The payload type for the right side.</typeparam>
/// <typeparam name="TResult">The payload type for the resulting stream.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TRight"/>, <typeparamref name="TKey"/>, <typeparamref name="TResult"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new IncreasingOrderEquiJoinTemplate($"GeneratedIncreasingOrderEquiJoin_{IOOEJSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult));
template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType);
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType);
template.rightFields = template.rightMessageRepresentation.AllFields;
var resultRepresentation = new ColumnarRepresentation(template.resultType);
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
#region Key Comparer
var keyComparer = stream.Left.Properties.KeyComparer.GetCompareExpr();
if (!ConstantExpressionFinder.IsClosedExpression(keyComparer))
{
return(null);
}
template.joinKeyOrderComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType);
template.TKeyTLeftGenericParameters = string.Empty; // BUGBUG
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType);
template.TKeyTRightGenericParameters = string.Empty; // BUGBUG
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType);
template.TKeyTResultGenericParameters = string.Empty; // BUGBUG
template.outputFields = resultRepresentation.AllFields;
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
return(null);
}
#region LeftBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), "i"), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), "j"), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
template.getOutputBatch = string.Format(
"pool.Get(out genericOutputBatch); output = ({0}{1})genericOutputBatch;",
Transformer.GetBatchClassName(template.keyType, template.resultType),
template.TKeyTResultGenericParameters);
return(template.Generate <TKey, TLeft, TRight, TResult>(selector));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Create the template output
/// </summary>
public override string TransformText()
{
this.Write("\r\n");
this.Write(@"// *********************************************************************
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License
// *********************************************************************
using System;
using System.Collections.Generic;
using System.Linq.Expressions;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using Microsoft.StreamProcessing;
using Microsoft.StreamProcessing.Internal;
using Microsoft.StreamProcessing.Internal.Collections;
[assembly: IgnoresAccessChecksTo(""Microsoft.StreamProcessing"")]
");
List <string> genericParamList = new List <string>();
var TOuterKey = outerKeyType.GetCSharpSourceSyntax(ref genericParamList);
var outerKeyGenericParameterCount = genericParamList.Count;
var TSource = sourceType.GetCSharpSourceSyntax(ref genericParamList);
var sourceGenericParameterCount = genericParamList.Count;
var sourceGenericParameters = 0 < genericParamList.Count ? "<" + String.Join(",", genericParamList) + ">" : string.Empty;
var TInnerKey = innerKeyType.GetCSharpSourceSyntax(ref genericParamList);
var genericParameters = 0 < genericParamList.Count ? "<" + String.Join(",", genericParamList) + ">" : string.Empty;
var outKeyInnerKeyGenericParameterList = genericParamList.Take(outerKeyGenericParameterCount).Concat(genericParamList.Skip(sourceGenericParameterCount));
var TOuterKeyTInnerKeyGenericParameters = 0 < outKeyInnerKeyGenericParameterList.Count() ? "<" + String.Join(",", outKeyInnerKeyGenericParameterList) + ">" : string.Empty;
var resultBatchGenericParameters = genericParameters;
var resultBatchClassType = isFirstLevelGroup
? Transformer.GetBatchClassName(innerKeyType, sourceType)
: Transformer.GetBatchClassName(typeof(CompoundGroupKey <,>).MakeGenericType(outerKeyType, innerKeyType), sourceType);
var sourceBatchClassType = Transformer.GetBatchClassName(outerKeyType, sourceType);
var innerKeyIsAnonymous = innerKeyType.IsAnonymousTypeName();
if (innerKeyIsAnonymous)
{
transformedKeySelectorAsString = string.Format("({0})Activator.CreateInstance(typeof({0}), {1} )", TInnerKey, transformedKeySelectorAsString);
}
var outputKey = !isFirstLevelGroup ? "CompoundGroupKey<" + TOuterKey + ", " + TInnerKey + ">" : TInnerKey;
var nestedInfix = !isFirstLevelGroup ? "Nested" : string.Empty;
var ungroupingToUnit = isFirstLevelGroup && innerKeyType == typeof(Empty);
var swingingKeyFromField = isFirstLevelGroup && !this.swingingField.Equals(default(MyFieldInfo));
this.Write("\r\n// TOuterKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write("\r\n// TInnerKey: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey));
this.Write("\r\n// TSource: ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write("\r\n\r\n[DataContract]\r\ninternal sealed class ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write(this.ToStringHelper.ToStringWithCulture(genericParameters));
this.Write(" :\r\n Pipe<");
this.Write(this.ToStringHelper.ToStringWithCulture(outputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write(">, IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write(">\r\n{\r\n");
if (innerKeyIsAnonymous)
{
this.Write(" [SchemaSerialization]\r\n private readonly Expression<Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey));
this.Write(", int>> keyComparer;\r\n private readonly Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey));
this.Write(", int> innerHashCode;\r\n");
}
this.Write(" private readonly MemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(outputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write("> l1Pool;\r\n private readonly Func<PlanNode, IQueryObject, PlanNode> queryPlanG" +
"enerator;\r\n\r\n");
if (ungroupingToUnit)
{
this.Write("\r\n private readonly ColumnBatch<Microsoft.StreamProcessing.Empty> unitColumn;\r" +
"\n private readonly ColumnBatch<int> unitHashColumn;\r\n");
}
this.Write("\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(staticCtor));
this.Write("\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("() { }\r\n\r\n public ");
this.Write(this.ToStringHelper.ToStringWithCulture(className));
this.Write("(\r\n IStreamable<");
this.Write(this.ToStringHelper.ToStringWithCulture(outputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write("> stream,\r\n IStreamObserver<");
this.Write(this.ToStringHelper.ToStringWithCulture(outputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write("> observer,\r\n Expression<Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey));
this.Write(", int>> keyComparer,\r\n Expression<Func<");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TInnerKey));
this.Write(">> keySelector,\r\n Func<PlanNode, IQueryObject, PlanNode> queryPlanGenerato" +
"r)\r\n : base(stream, observer)\r\n {\r\n");
if (innerKeyIsAnonymous)
{
if (isFirstLevelGroup)
{
this.Write(" var _keySelector = keySelector.Compile();\r\n");
}
this.Write(" this.keyComparer = keyComparer;\r\n this.innerHashCode = keyComparer" +
".Compile();\r\n");
}
this.Write(" this.l1Pool = MemoryManager.GetMemoryPool<");
this.Write(this.ToStringHelper.ToStringWithCulture(outputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write(">(true);\r\n this.queryPlanGenerator = queryPlanGenerator;\r\n");
if (ungroupingToUnit)
{
this.Write("\r\n this.l1Pool.GetKey(out this.unitColumn);\r\n this.l1Pool.Get(out t" +
"his.unitHashColumn);\r\n Array.Clear(this.unitHashColumn.col, 0, this.unitH" +
"ashColumn.col.Length);\r\n");
}
this.Write(@" }
public override void ProduceQueryPlan(PlanNode previous)
{
Observer.ProduceQueryPlan(queryPlanGenerator(previous, this));
}
protected override void FlushContents() { }
public override int CurrentlyBufferedOutputCount => 0;
public override int CurrentlyBufferedInputCount => 0;
protected override void DisposeState()
{
");
if (ungroupingToUnit)
{
this.Write(" this.unitColumn.Return();\r\n this.unitHashColumn.Return();\r\n");
}
this.Write(" }\r\n\r\n public unsafe void OnNext(StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(TOuterKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write("> batch)\r\n {\r\n StreamMessage<");
this.Write(this.ToStringHelper.ToStringWithCulture(outputKey));
this.Write(", ");
this.Write(this.ToStringHelper.ToStringWithCulture(TSource));
this.Write("> resultBatchUncast; // Need this type to call Get with so the right subtype will" +
" be returned\r\n this.l1Pool.Get(out resultBatchUncast);\r\n\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchGenericParameters));
this.Write(" resultBatch = resultBatchUncast as ");
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(resultBatchGenericParameters));
this.Write(";\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(sourceGenericParameters));
this.Write(" inputBatch = batch as ");
this.Write(this.ToStringHelper.ToStringWithCulture(sourceBatchClassType));
this.Write(this.ToStringHelper.ToStringWithCulture(sourceGenericParameters));
this.Write(";\r\n\r\n resultBatch.vsync = batch.vsync;\r\n resultBatch.vother = batch" +
".vother;\r\n");
foreach (var f in this.fields)
{
this.Write(" resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(" = inputBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
if (this.payloadMightBeNull)
{
this.Write(" resultBatch._nullnessvector = inputBatch._nullnessvector;\r\n");
}
this.Write("\r\n");
if (!ungroupingToUnit && !swingingHashColumn)
{
this.Write("\r\n resultBatch.hash = batch.hash.MakeWritable(this.l1Pool.intPool);\r\n");
}
this.Write(" resultBatch.bitvector = batch.bitvector;\r\n\r\n");
if (!ungroupingToUnit && (!isFirstLevelGroup || this.swingingField.Equals(default(MyFieldInfo))))
{
this.Write(" this.l1Pool.GetKey(out resultBatchUncast.key);\r\n");
}
this.Write("\r\n resultBatch.Count = batch.Count;\r\n\r\n var count = batch.Count;\r\n\r" +
"\n");
if (ungroupingToUnit)
{
this.Write(" this.unitColumn.IncrementRefCount(1);\r\n resultBatch.key = unitColu" +
"mn;\r\n this.unitHashColumn.IncrementRefCount(1);\r\n resultBatch.hash" +
" = unitHashColumn;\r\n");
}
else
{
this.Write("\r\n var src_key = batch.key.col;\r\n");
if (!isFirstLevelGroup || this.swingingField.Equals(default(MyFieldInfo)))
{
this.Write("\r\n var dest_key = resultBatch.key.col;\r\n");
}
this.Write(" var dest_payload = resultBatch;\r\n\r\n");
if (swingingKeyFromField)
{
this.Write(" // The key selector was just e => e.f *and* isFirstLevelGroup so just swi" +
"ng the key field\r\n");
if (this.swingingField.OptimizeString())
{
this.Write("\r\n resultBatch.key = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(this.swingingField.Name));
this.Write(".ToColumnBatch(this.l1Pool.stringPool, resultBatch.bitvector);\r\n");
}
else
{
this.Write("\r\n resultBatch.key = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(this.swingingField.Name));
this.Write(";\r\n resultBatch.key.IncrementRefCount(1);\r\n");
}
}
this.Write("\r\n\r\n");
foreach (var f in this.fields)
{
this.Write("\r\n");
if (f.canBeFixed)
{
this.Write(" fixed (");
this.Write(this.ToStringHelper.ToStringWithCulture(f.TypeName));
this.Write("* ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col)\r\n {\r\n");
}
else
{
this.Write("\r\n");
if (f.OptimizeString())
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(";\r\n");
}
else
{
this.Write(" var ");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write("_col = resultBatch.");
this.Write(this.ToStringHelper.ToStringWithCulture(f.Name));
this.Write(".col;\r\n");
}
}
}
this.Write("\r\n ");
this.Write(this.ToStringHelper.ToStringWithCulture(vectorHashCodeInitialization));
this.Write("\r\n\r\n");
if (!swingingKeyFromField || !swingingHashColumn)
{
this.Write("\r\n fixed (long* src_bv = batch.bitvector.col) {\r\n");
if (!ungroupingToUnit)
{
this.Write(" fixed (int* dest_hash = resultBatch.hash.col) {\r\n");
}
this.Write(" for (int i = 0; i < count; i++)\r\n {\r\n if ((" +
"src_bv[i >> 6] & (1L << (i & 0x3f))) != 0) continue;\r\n\r\n");
if (!swingingKeyFromField || !this.swingingField.OptimizeString())
{
this.Write("\r\n var key = ");
this.Write(this.ToStringHelper.ToStringWithCulture(this.transformedKeySelectorAsString));
this.Write(";\r\n");
}
if (innerKeyIsAnonymous)
{
this.Write(" var innerHash = this.innerHashCode(key);\r\n");
}
else if (swingingHashColumn)
{
this.Write("\r\n // don\'t compute hash because that was done by calling MultiStr" +
"ing.GetHashCode\r\n");
}
else
{
this.Write(" var innerHash = ");
this.Write(this.ToStringHelper.ToStringWithCulture(inlinedHashCodeComputation));
this.Write(";\r\n");
}
this.Write("\r\n");
if (isFirstLevelGroup)
{
if (!this.swingingField.Equals(default(MyFieldInfo)))
{
this.Write(" // no assignment to key, pointer was swung!\r\n");
}
else
{
this.Write(" dest_key[i] = key;\r\n");
}
if (!swingingHashColumn)
{
this.Write("\r\n dest_hash[i] = innerHash;\r\n");
}
}
else
{
this.Write(" var hash = dest_hash[i] ^ innerHash;\r\n dest_key[i]" +
".outerGroup = src_key[i];\r\n dest_key[i].innerGroup = key;\r\n " +
" dest_key[i].hashCode = hash;\r\n dest_hash[i] = hash;\r\n");
}
this.Write(" }\r\n");
if (!ungroupingToUnit)
{
this.Write(" } // end of fixed for dest_hash\r\n");
}
this.Write(" } // end of fixed for src_bv\r\n");
}
this.Write("\r\n ");
if (!String.IsNullOrWhiteSpace(vectorHashCodeInitialization) && !swingingHashColumn)
{
this.Write("\r\n this.hashCodeVector.Return();\r\n ");
}
this.Write("\r\n");
foreach (var f in this.fields.Where(fld => fld.canBeFixed))
{
this.Write("\r\n }\r\n");
}
this.Write("\r\n");
}
this.Write(" batch.ReleaseKey();\r\n");
if (ungroupingToUnit || swingingHashColumn)
{
this.Write("\r\n batch.hash.Return();\r\n");
}
this.Write("\r\n batch.Return();\r\n\r\n this.Observer.OnNext(resultBatch);\r\n }\r\n\r" +
"\n public void OnError(Exception error)\r\n {\r\n Observer.OnError(error" +
");\r\n }\r\n}\r\n");
return(this.GenerationEnvironment.ToString());
}
/// <summary>
/// Generate a batch class definition to be used as a Union pipe.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// union pipe class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TPayload">The payload type.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TPayload"/>, <typeparamref name="TPayload"/>, <typeparamref name="TKey"/>, <typeparamref name="TPayload"/>>.
/// </returns>
internal static Tuple <Type, string> GenerateUnionPipeClass <TKey, TPayload>(
UnionStreamable <TKey, TPayload> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TPayload, TPayload, TPayload>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
#if CODEGEN_TIMING
Stopwatch sw = new Stopwatch();
sw.Start();
#endif
string errorMessages = null;
try
{
var template = new UnionTemplate();
var keyType = template.keyType = typeof(TKey);
var payloadType = template.payloadType = typeof(TPayload);
var tm = new TypeMapper(keyType, payloadType);
template.TKey = tm.CSharpNameFor(keyType);
template.TPayload = tm.CSharpNameFor(payloadType);
var gps = tm.GenericTypeVariables(keyType, payloadType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.className = string.Format("GeneratedUnion_{0}", UnionSequenceNumber++);
var resultRepresentation = new ColumnarRepresentation(payloadType);
var batchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName(keyType, payloadType);
var keyAndPayloadGenericParameters = tm.GenericTypeVariables(keyType, payloadType).BracketedCommaSeparatedString();
template.GeneratedBatchName = batchGeneratedFrom_TKey_TPayload + keyAndPayloadGenericParameters;
template.fields = resultRepresentation.AllFields;
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, payloadType);
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(payloadType.GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new StartEdgeEquiJoinTemplate();
var keyType = template.keyType = typeof(TKey);
var leftType = template.leftType = typeof(TLeft);
var rightType = template.rightType = typeof(TRight);
var resultType = template.resultType = typeof(TResult);
var tm = new TypeMapper(keyType, leftType, rightType, resultType);
template.TKey = tm.CSharpNameFor(keyType);
template.TLeft = tm.CSharpNameFor(leftType);
template.TRight = tm.CSharpNameFor(rightType);
template.TResult = tm.CSharpNameFor(resultType);
var gps = tm.GenericTypeVariables(keyType, leftType, rightType, resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.className = string.Format("GeneratedStartEdgeEquiJoin_{0}", StartEdgeEquiJoinSequenceNumber++);
template.leftMessageRepresentation = new ColumnarRepresentation(leftType);
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightMessageRepresentation = new ColumnarRepresentation(rightType);
template.rightFields = template.rightMessageRepresentation.AllFields;
var outputMessageRepresentation = new ColumnarRepresentation(resultType);
var resultRepresentation = outputMessageRepresentation;
#region Key Comparer
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
template.TKeyTLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.TKeyTRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType);
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.outputFields = resultRepresentation.AllFields;
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
#region LeftBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), "i"), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), "j"), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType, resultType);
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(leftType.GetAnonymousTypes());
list.AddRange(rightType.GetAnonymousTypes());
list.AddRange(resultType.GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}