protected CommonUnaryTemplate(string className, Type keyType, Type payloadType, Type resultType, bool suppressPayloadBatch = false)
: base(className)
{
this.keyType = keyType;
this.payloadType = payloadType;
this.resultType = resultType;
var tm = new TypeMapper(keyType, payloadType, resultType);
this.TKey = tm.CSharpNameFor(keyType);
this.TPayload = tm.CSharpNameFor(payloadType);
this.TResult = tm.CSharpNameFor(resultType);
if (!suppressPayloadBatch)
{
this.TKeyTPayloadGenericParameters = tm.GenericTypeVariables(keyType, payloadType).BracketedCommaSeparatedString();
this.BatchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName(keyType, payloadType);
}
this.payloadRepresentation = new ColumnarRepresentation(payloadType);
this.fields = this.payloadRepresentation.AllFields;
this.noFields = this.payloadRepresentation.noFields;
this.resultRepresentation = new ColumnarRepresentation(resultType);
}
private TemporalEgressTemplate(Type tKey, Type tPayload, Type tResult, string partitionString, string ingressType, bool isColumnar)
: base($"GeneratedTemporalEgress_{TemporalEgressSequenceNumber++}")
{
var tm = new TypeMapper(tKey, tPayload, tResult);
this.payloadRepresentation = new ColumnarRepresentation(tPayload);
this.BatchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName(
tKey == typeof(Empty)
? tKey
: typeof(PartitionKey <>).MakeGenericType(tKey), tPayload);
this.TKeyTPayloadGenericParameters = tm.GenericTypeVariables(tKey, tPayload).BracketedCommaSeparatedString();
this.fields = this.payloadRepresentation.AllFields;
this.TKey = tm.CSharpNameFor(tKey);
this.TPayload = tm.CSharpNameFor(tPayload);
this.TResult = tm.CSharpNameFor(tResult);
this.partitionString = partitionString;
this.ingressType = ingressType;
this.partitionKeyArgument = !string.IsNullOrEmpty(partitionString) ? "colkey[i].Key, " : string.Empty;
this.genericArguments = string.IsNullOrEmpty(partitionString) ? this.TPayload : this.TKey + ", " + this.TPayload;
this.egress = (ingressType != "StreamEvent")
? this.TResult
: partitionString + "StreamEvent<" + this.genericArguments + ">";
this.inputKey = string.IsNullOrEmpty(partitionString) ? this.TKey : "PartitionKey<" + this.TKey + ">";
this.isColumnar = isColumnar;
}
internal MemoryPoolTemplate(ColumnarRepresentation keyRepresentation, ColumnarRepresentation payloadRepresentation)
{
this.assemblyReferences = new List <Assembly>();
var keyType = keyRepresentation == null ? typeof(Empty) : keyRepresentation.RepresentationFor;
Contract.Assume(Transformer.IsValidKeyType(keyType));
this.assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyType));
this.keyType = keyType;
#region Decompose TPayload into columns
var payloadType = payloadRepresentation.RepresentationFor;
this.assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(payloadType));
this.payloadType = payloadType;
var payloadTypes = payloadRepresentation.AllFields.Select(f => f.Type).Where(t => !t.MemoryPoolHasGetMethodFor());
this.types = new HashSet <Type>(payloadTypes.Distinct());
this.assemblyReferences.AddRange(this.types.SelectMany(t => Transformer.AssemblyReferencesNeededFor(t)));
#endregion
this.generatedClassName = Transformer.GetMemoryPoolClassName(keyType, payloadType);
this.className = this.generatedClassName.CleanUpIdentifierName();
this.generatedClassName += "`2";
this.expandedCode = TransformText();
}
/// <summary>
/// Generate a batch class definition to be used as a Union pipe.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// union pipe class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TPayload">The payload type.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TPayload"/>, <typeparamref name="TPayload"/>, <typeparamref name="TKey"/>, <typeparamref name="TPayload"/>>.
/// </returns>
internal static Tuple <Type, string> GenerateUnionPipeClass <TKey, TPayload>(
UnionStreamable <TKey, TPayload> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TPayload, TPayload, TPayload>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
#if CODEGEN_TIMING
Stopwatch sw = new Stopwatch();
sw.Start();
#endif
var template = new UnionTemplate($"GeneratedUnion_{UnionSequenceNumber++}", typeof(TKey), typeof(TPayload));
var gps = template.tm.GenericTypeVariables(template.keyType, template.resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
var resultRepresentation = new ColumnarRepresentation(template.resultType);
var batchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName(template.keyType, template.resultType);
var keyAndPayloadGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString();
template.GeneratedBatchName = batchGeneratedFrom_TKey_TPayload + keyAndPayloadGenericParameters;
template.fields = resultRepresentation.AllFields;
return(template.Generate <TKey, TPayload>());
}
public static void GetGeneratedCode(Type keyType, Type payloadType, out string generatedClassName, out string expandedCode, out List <Assembly> assemblyReferences)
{
var template = new SafeBatchTemplate();
assemblyReferences = new List <Assembly>
{
Transformer.SystemRuntimeSerializationDll // needed for [DataContract] and [DataMember] in generated code
};
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyType));
template.keyType = keyType;
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(payloadType));
template.payloadType = payloadType;
template.needsPolymorphismCheck =
!payloadType.GetTypeInfo().IsValueType&&
!payloadType.IsAnonymousTypeName() &&
!payloadType.GetTypeInfo().IsSealed;
template.payloadMightBeNull = payloadType.CanContainNull();
generatedClassName = Transformer.GetBatchClassName(keyType, payloadType);
template.CLASSNAME = generatedClassName.CleanUpIdentifierName();
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(keyType, payloadType);
var payloadRepresentation = new ColumnarRepresentation(payloadType);
template.fields = payloadRepresentation.AllFields;
template.noPublicFields = payloadRepresentation.noFields;
expandedCode = template.TransformText();
}
internal static SelectTransformationResult Transform(
LambdaExpression function,
IEnumerable <Tuple <ParameterExpression, SelectParameterInformation> > substitutionInformation,
ColumnarRepresentation resultTypeInformation,
bool noSwingingFields = false,
bool hasStartEdge = false)
{
Contract.Requires(function != null);
Contract.Requires(substitutionInformation != null);
Contract.Requires(resultTypeInformation != null);
var me = new SelectTransformer(function, substitutionInformation, resultTypeInformation, noSwingingFields,
Config.UseMultiString && ((Config.MultiStringTransforms & Config.CodegenOptions.MultiStringFlags.VectorOperations) != 0), hasStartEdge);
// Need to find any unmentioned fields from the result type
var unmentionedFields = new List <MyFieldInfo>();
if (me.ProjectionReturningResultInstance == null)
{
foreach (var kv in resultTypeInformation.Fields)
{
var fieldInfo = kv.Value;
if (me.swingingFields.Any(t => t.Item1.Equals(fieldInfo)))
{
continue;
}
if (me.computedFields.Keys.Any(fi => fi.Equals(fieldInfo)))
{
continue;
}
if (me.multiStringResultFields.Any(f => f.Equals(fieldInfo)))
{
continue;
}
unmentionedFields.Add(fieldInfo);
}
}
return(new SelectTransformationResult()
{
Error = me.error,
ComputedFields = me.computedFields,
SwingingFields = me.swingingFields,
UnmentionedFields = unmentionedFields,
ProjectionReturningResultInstance = me.ProjectionReturningResultInstance,
MultiStringOperations = me.multiStringOperations,
NeedsSourceInstance = me.needsSourceInstance,
});
}
private static LambdaExpression /*?*/ MakeOneParameterColumnar <TKey, TPayload>(LambdaExpression f, int index, string indexVariableName)
{
Contract.Requires(index >= 0 && index < f.Parameters.Count);
var batchType = StreamMessageManager.GetStreamMessageType <TKey, TPayload>();
var d = new Dictionary <ParameterExpression, ColumnOriented.SubstitutionInformation>();
var payloadRepresentation = new ColumnarRepresentation(typeof(TPayload));
d.Add(
f.Parameters[index],
new ColumnOriented.SubstitutionInformation
{
columnarRepresentation = payloadRepresentation,
nameForIndexVariable = indexVariableName,
typeOfBatchVariable = batchType,
});
return(ColumnOriented.Transform(f, d));
}
/// <summary>
/// Generate a batch class definition to be used as a Clip pipe.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TLeft">The payload type for the left side.</typeparam>
/// <typeparam name="TRight">The payload type for the right side.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TKey"/>, <typeparamref name="TRight"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight>(ClipJoinStreamable <TKey, TLeft, TRight> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TLeft>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
var template = new ClipJoinTemplate($"GeneratedClip_{ClipSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight));
var keyType = typeof(TKey);
var leftType = typeof(TLeft);
var rightType = typeof(TRight);
var gps = template.tm.GenericTypeVariables(keyType, leftType, rightType);
template.genericParameters = gps.BracketedCommaSeparatedString();
var resultRepresentation = new ColumnarRepresentation(leftType);
template.ActiveEventType = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event";
#region Key Comparer
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
#region Left Comparer
var leftComparer = stream.LeftComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftComparer, "index_ProcessLeftEvent", 0);
template.leftComparer = (left, right) => newLambda.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
template.TKeyTLeftGenericParameters = template.tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.TKeyTRightGenericParameters = template.tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString();
template.leftFields = resultRepresentation.AllFields;
template.noFields = resultRepresentation.noFields;
return(template.Generate <TKey, TLeft, TRight>(leftComparer));
}
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new StartEdgeEquiJoinTemplate($"GeneratedStartEdgeEquiJoin_{StartEdgeEquiJoinSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult));
var gps = template.tm.GenericTypeVariables(template.keyType, template.leftType, template.rightType, template.resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType);
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType);
template.rightFields = template.rightMessageRepresentation.AllFields;
var resultRepresentation = new ColumnarRepresentation(template.resultType);
#region Key Comparer
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType);
template.TKeyTLeftGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType);
template.TKeyTRightGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.rightType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType);
template.TKeyTResultGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString();
template.outputFields = resultRepresentation.AllFields;
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
#region LeftBatchSelector
{
var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var parameterMap = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex },
{ selector.Parameters[1], rightEvent }
};
if (projectionResult.ProjectionReturningResultInstance != null)
{
return($"this.output[index] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
else
{
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendLine($"this.output.{f.Name}.AddString({e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)});");
}
else
{
sb.AppendLine($"this.output.{f.Name}.col[index] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
}
sb.AppendLine("}");
return(sb.ToString());
}
};
}
#endregion
#region RightBatchSelector
{
var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var parameterMap = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex }
};
if (projectionResult.ProjectionReturningResultInstance != null)
{
return($"this.output[index] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
else
{
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendLine($"this.output.{f.Name}.AddString({e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)});");
}
else
{
sb.AppendLine($"this.output.{f.Name}.col[index] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
}
sb.AppendLine("}");
return(sb.ToString());
}
};
}
#endregion
return(template.Generate <TKey, TLeft, TRight, TResult>(selector));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TOuterKey, TSource, TInnerKey>(
Expression <Func <TSource, TInnerKey> > keySelector,
string inlinedHashCodeComputation,
bool nested,
bool powerOf2)
{
string errorMessages = null;
try
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTSource = typeof(TSource);
var typeOfTInnerKey = typeof(TInnerKey);
var generatedClassName = string.Format(CultureInfo.InvariantCulture, "ShuffleStreamablePipeGeneratedFrom_{0}_{1}_{2}_{3}", typeOfTOuterKey.GetValidIdentifier(), typeOfTSource.GetValidIdentifier(), typeOfTInnerKey.GetValidIdentifier(), shuffleCounter++);
var inputMessageRepresentation = new ColumnarRepresentation(typeOfTSource);
var template = new ShuffleTemplate(
generatedClassName,
typeOfTOuterKey,
typeOfTSource,
typeOfTInnerKey,
inlinedHashCodeComputation,
nested, powerOf2)
{
fields = inputMessageRepresentation.AllFields
};
var innerKeyIsAnonymous = typeOfTInnerKey.IsAnonymousTypeName();
if (keySelector != null)
{
var transformedKeySelector = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector);
var keySelectorAsNewExpression = keySelector.Body as NewExpression;
if (innerKeyIsAnonymous && keySelectorAsNewExpression != null)
{
var newPrime = (NewExpression)transformedKeySelector.Body;
template.transformedKeySelectorAsString = string.Format(CultureInfo.InvariantCulture, "({0})Activator.CreateInstance(typeof({0}), {1})",
template.TInnerKey,
string.Join(",", newPrime.Arguments.Select(m => m.ExpressionToCSharp())));
}
else
{
template.transformedKeySelectorAsString = transformedKeySelector.Body.ExpressionToCSharp();
if (Config.UseMultiString &&
typeOfTInnerKey.Equals(typeof(string)) &&
keySelector.IsSimpleFieldOrPropertyAccess())
{
template.inlinedHashCodeComputation = "hashCodeVector.col[i]";
var fieldName = ((MemberExpression)(keySelector.Body)).Member.Name;
template.vectorHashCodeInitialization = string.Format(CultureInfo.InvariantCulture, "var hashCodeVector = {0}{1}_col.GetHashCode(batch.bitvector);", Transformer.ColumnFieldPrefix, fieldName);
}
}
}
else
{
template.transformedKeySelectorAsString = string.Empty;
}
template.innerKeyIsAnonymous = innerKeyIsAnonymous;
template.staticCtor = Transformer.StaticCtor(template.CLASSNAME);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TSource>());
if (nested)
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
}
else
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TInnerKey, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TInnerKey, TSource>());
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (typeOfTInnerKey.IsAnonymousTypeName())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The inner key type for Shuffle is anonymous, causing the use of Activator.CreateInstance in an inner loop. This will lead to poor performance.\n";
}
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
var t = a.GetType(generatedClassName);
return(Tuple.Create(t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey), errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Fifth component of tuple is needed when the projection body must be evaluated into a local to hold the result value.
/// That is the case when the body looks like "new T(...)" or is the result of a method call.
/// </summary>
private SelectTransformer(
LambdaExpression function,
IEnumerable <Tuple <ParameterExpression, SelectParameterInformation> > substitutionInformation,
ColumnarRepresentation resultTypeInformation,
bool noSwingingFields,
bool doMultiStringTransform,
bool hasStartEdge)
{
this.parameterInformation = new Dictionary <ParameterExpression, SelectParameterInformation>();
foreach (var tup in substitutionInformation)
{
this.parameterInformation.Add(tup.Item1, tup.Item2);
}
this.resultTypeInformation = resultTypeInformation;
this.noSwingingFields = noSwingingFields;
this.doMultiStringTransform = doMultiStringTransform;
this.swingingFields = new List <Tuple <MyFieldInfo, MyFieldInfo> >();
this.computedFields = new Dictionary <MyFieldInfo, Expression>();
this.multiStringOperations = new List <string>();
this.multiStringResultFields = new List <MyFieldInfo>();
var body = function.Body;
// The projection might just be (e_1, e_2, ..., e_n) => e_i, i.e., projecting just the single parameter e_i.
if (body is ParameterExpression parameter)
{
TransformSingleParameterSelect(parameter, hasStartEdge);
return;
}
// The projection might just be (e_1, e_2, ..., e_n) => e_i.f, i.e., projecting just the single field f from one of the parameters e_i.
if (body is MemberExpression memberExpression)
{
TransformSingleFieldSelect(memberExpression);
return;
}
// Case: projection is (e_1, e_2, ..., e_n) => new { f1 = ..., f2 = ..., ...}), i.e., projecting into an anonymous type
if (body is NewExpression newExpression && newExpression.Type.IsAnonymousType())
{
Contract.Assume(newExpression.Type == resultTypeInformation.RepresentationFor);
// REVIEW: Should these be turned into part of the if-test?
Contract.Assume(newExpression.Arguments != null);
Contract.Assume(newExpression.Members != null);
Contract.Assume(newExpression.Arguments.Count == newExpression.Members.Count);
TransformAnonymousTypeSelect(newExpression);
return;
}
// Case: projection is (e_1, e_2, ..., e_n) => new T{ f1 = ..., f2 = ..., ... }), i.e., T is *not* an anonymous type
// TODO: See if this can be unified with the code above for anonymous types.
if (body is MemberInitExpression && !this.resultTypeInformation.noFields)
{
Visit(body);
return;
}
// Case: projection is (e_1, e_2, ..., e_n) => f(e_1, e_2, ..., e_n)) *AND* the result type is a non-decomposable type, e.g., int.
// In that case, the expression does not have to be evaluated into a local, but can just be assigned to the result
// column's pseudo-field, "payload".
// Note that f is either a real method call or else just an expression
// that computes a result value (e.g., a type cast which shows up as a unary expression).
if (this.resultTypeInformation.noFields)
{
if (this.doMultiStringTransform && IsMultiStringCall(body, out string s))
{
this.multiStringOperations.Add($"resultBatch.{this.resultTypeInformation.PseudoField.Name} = {s};");
}
else
{
var transformedBody = Visit(body);
this.computedFields.Add(this.resultTypeInformation.PseudoField, transformedBody);
}
return;
}
// Otherwise, degenerate case: need to just evaluate the expression (with source field access transformed to columnar).
// That expression will be passed to the setter for the indexer on the generated batch.
// REVIEW: this is where something should be signalled so the user knows it isn't as fast as it could be.
var transformedBody2 = Visit(body);
this.ProjectionReturningResultInstance = transformedBody2.ExpressionToCSharp();
return;
}
protected static EdgeInfo CreateListEdgeInfo <TKey, TPayload, TRegister, TAccumulator>(AfaStreamable <TKey, TPayload, TRegister, TAccumulator> stream, ColumnarRepresentation payloadRepresentation, int targetNodeNumber, ListElementArc <TPayload, TRegister> edge, string indexVariableName)
{
var edgeInfo = new EdgeInfo()
{
Type = EdgeInfo.EdgeType.List,
EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNodeNumber),
SourceNode = targetNodeNumber,
Fence = (ts, evs, reg) => edge.Fence.Inline(ts, evs, reg),
Transfer = edge.Transfer == null ? ((Func <string, string, string, string>)null) : (ts, evs, reg) => edge.Transfer.Inline(ts, evs, reg),
};
return(edgeInfo);
}
public static Tuple <Type, string> Generate <TKey, TPayload, TResult>(SelectStreamable <TKey, TPayload, TResult> stream)
{
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string generatedClassName;
string errorMessages = null;
try
{
var keyType = typeof(TKey);
var payloadType = typeof(TPayload);
var resultType = typeof(TResult);
generatedClassName = string.Format("Select_{0}_{1}_{2}_{3}", keyType.GetValidIdentifier(), payloadType.GetValidIdentifier(), resultType.GetValidIdentifier(), sequenceNumber++);
var template = new SelectTemplate(generatedClassName, keyType, payloadType, resultType);
var tm = new TypeMapper(keyType, payloadType, resultType);
var gps = tm.GenericTypeVariables(keyType, payloadType, resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.numberOfGenericParameters = gps.Count();
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.MemoryPoolGenericParameters = $"<{template.TKey}, {template.TResult}>";
if (resultType == typeof(int) || resultType == typeof(long) || resultType == typeof(string))
{
template.MemoryPoolGenericParameters = string.Empty;
}
var payloadParameterIndex = 0;
if (stream.HasKey && stream.HasStartEdge)
{
payloadParameterIndex = 2;
}
else if (stream.HasKey || stream.HasStartEdge)
{
payloadParameterIndex = 1;
}
template.PARAMETER = stream.Selector.Parameters[payloadParameterIndex].Name;
template.resultPayloadRepresentation = new ColumnarRepresentation(resultType);
template.destinationFields = template.resultPayloadRepresentation.AllFields;
if (template.numberOfGenericParameters > 0)
{
generatedClassName = generatedClassName + "`" + template.numberOfGenericParameters.ToString(CultureInfo.InvariantCulture);
}
var resultSelector = stream.Selector;
var sourceMessageType = StreamMessageManager.GetStreamMessageType <TKey, TPayload>();
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >();
// Don't create a parameter substitution for the start edge parameter. That will just remain in the
// body of the result selector and will be set as a local variable in the generated code.
if (stream.HasKey)
{
var keyRepresentation = new ColumnarRepresentation(keyType, "key");
var keyParameterIndex = stream.HasStartEdge ? 1 : 0;
parameterSubsitutions.Add(Tuple.Create(
resultSelector.Parameters.ElementAt(keyParameterIndex),
new SelectParameterInformation()
{
BatchName = "sourceBatch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = keyRepresentation,
}));
}
parameterSubsitutions.Add(Tuple.Create(
resultSelector.Parameters.ElementAt(payloadParameterIndex),
new SelectParameterInformation()
{
BatchName = "sourceBatch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = new ColumnarRepresentation(payloadType)
}));
var projectionResult = SelectTransformer.Transform(resultSelector, parameterSubsitutions, template.resultPayloadRepresentation, false, stream.HasStartEdge);
if (projectionResult.Error)
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
throw new InvalidOperationException("Code Generation couldn't transform a selector!");
}
return(Tuple.Create((Type)null, errorMessages));
}
template.StartEdgeParameterName = stream.HasStartEdge ? resultSelector.Parameters.ElementAt(0).Name : null;
template.computedFields = projectionResult.ComputedFields;
template.swingingFields = projectionResult.SwingingFields;
template.unassignedFields = projectionResult.UnmentionedFields;
template.ProjectionReturningResultInstance = projectionResult.ProjectionReturningResultInstance;
template.multiStringOperations = projectionResult.MultiStringOperations;
template.needSourceInstance = projectionResult.NeedsSourceInstance;
var d = new Dictionary <MyFieldInfo, int>();
foreach (var f in template.swingingFields)
{
var target = f.Item1;
var source = f.Item2;
if (!d.ContainsKey(source))
{
d.Add(source, 0);
}
d[source] = d[source] + 1;
}
template.swungFieldsCount = d;
template.nonSwingingFields = template.fields.Where(sf => !template.swingingFields.Any(swingingField => swingingField.Item2.Equals(sf)));
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(
typeof(TKey), typeof(TPayload), typeof(TResult), typeof(IStreamable <,>));
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(stream.Selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeof(TKey), typeof(TPayload), typeof(TResult));
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Generate a batch class definition to be used as a Clip pipe.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TLeft">The payload type for the left side.</typeparam>
/// <typeparam name="TRight">The payload type for the right side.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TKey"/>, <typeparamref name="TRight"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight>(ClipJoinStreamable <TKey, TLeft, TRight> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TLeft>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new ClipJoinTemplate();
var keyType = template.keyType = typeof(TKey);
var leftType = template.leftType = typeof(TLeft);
var rightType = template.rightType = typeof(TRight);
var tm = new TypeMapper(keyType, leftType, rightType);
template.TKey = tm.CSharpNameFor(keyType);
template.TLeft = tm.CSharpNameFor(leftType);
template.TRight = tm.CSharpNameFor(rightType);
var gps = tm.GenericTypeVariables(keyType, leftType, rightType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.className = string.Format("GeneratedClip_{0}", ClipSequenceNumber++);
var resultRepresentation = new ColumnarRepresentation(leftType);
template.ActiveEventType = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event";
#region Key Comparer
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
#region Left Comparer
var leftComparer = stream.LeftComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftComparer, "index_ProcessLeftEvent", 0);
template.leftComparer = (left, right) => newLambda.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
template.TKeyTLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.TKeyTRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString();
template.leftFields = resultRepresentation.AllFields;
template.noFields = resultRepresentation.noFields;
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(leftComparer));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType);
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(leftType.GetAnonymousTypes());
list.AddRange(rightType.GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> GenerateAFA <TKey, TPayload, TRegister, TAccumulator>(
AfaStreamable <TKey, TPayload, TRegister, TAccumulator> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
var className = string.Format("Generated{1}AfaMultiEventList_{0}", AFASequenceNumber++, typeof(TKey).Equals(typeof(Empty)) ? string.Empty : "Grouped");
var template = new AfaMultiEventListTemplate(className, typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(TAccumulator))
{
TKey = typeof(TKey).GetCSharpSourceSyntax()
};
var payloadRepresentation = new ColumnarRepresentation(typeof(TPayload));
template.isFinal = stream.afa.isFinal;
template.hasOutgoingArcs = stream.afa.hasOutgoingArcs;
template.startStates = stream.afa.startStates;
template.AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances;
template.payloadIsAnon = typeof(TPayload).IsAnonymousTypeName();
template.payloadHasNoFields = payloadRepresentation.noFields;
var d1 = stream.afa.uncompiledAfa.transitionInfo;
var orderedKeys = d1.Keys.OrderBy(e => e).ToArray();
for (int i = 0; i < orderedKeys.Length; i++)
{
var sourceNodeNumber = orderedKeys[i];
var outgoingEdgesDictionary = d1[sourceNodeNumber];
var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray();
var edgeList = new List <EdgeInfo>();
for (int j = 0; j < orderedTargetNodes.Length; j++)
{
var targetNodeNumber = orderedTargetNodes[j];
var edge = outgoingEdgesDictionary[targetNodeNumber];
if (edge is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc)
{
var multiEdgeInfo = new MultiEdgeInfo()
{
SourceNode = sourceNodeNumber,
TargetNode = targetNodeNumber,
fromStartState = stream.afa.startStates.Any(n => n == sourceNodeNumber),
EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNodeNumber),
Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg),
Accumulate = (ts, ev, reg, acc) => multiArc.Accumulate.Inline(ts, ev, reg, acc),
Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg),
};
multiEdgeInfo.Transfer = multiArc.Transfer == null
? (Func <string, string, string, string>)null
: ((ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg));
if (multiArc.Dispose == null)
{
multiEdgeInfo.Dispose = (acc) => "// no dispose function";
}
else
{
multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc);
}
multiEdgeInfo.SkipToEnd = multiArc.SkipToEnd == null
? (Func <string, string, string, string>)null
: ((ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc));
edgeList.Add(multiEdgeInfo);
continue;
}
if (edge is SingleElementArc <TPayload, TRegister> singleArc)
{
var edgeInfo = CreateSingleEdgeInfo(stream, targetNodeNumber, singleArc, "payloadoffset");
edgeList.Add(edgeInfo);
continue;
}
if (edge is ListElementArc <TPayload, TRegister> listArc)
{
var edgeInfo = CreateListEdgeInfo(stream, payloadRepresentation, targetNodeNumber, listArc, "payloadoffset");
edgeList.Add(edgeInfo);
continue;
}
}
template.edgeInfos.Add(Tuple.Create(sourceNodeNumber, edgeList));
}
for (int i = 0; i < stream.afa.startStates.Length; i++)
{
var startState = stream.afa.startStates[i];
var edgeList2 = new List <EdgeInfo>();
var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState];
foreach (var edge in outgoingEdgeDictionary)
{
var targetNode = edge.Key;
var arc = edge.Value;
if (arc is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc)
{
var eps = EpsilonClosure(stream.afa, targetNode);
var multiEdgeInfo = new MultiEdgeInfo()
{
SourceNode = startState,
TargetNode = targetNode,
fromStartState = true,
EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNode),
Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg),
Accumulate = (ts, ev, reg, acc) => multiArc.Accumulate.Inline(ts, ev, reg, acc),
Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg),
};
multiEdgeInfo.Transfer = multiArc.Transfer == null
? (Func <string, string, string, string>)null
: ((ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg));
if (multiArc.Dispose == null)
{
multiEdgeInfo.Dispose = (acc) => "// no dispose function";
}
else
{
multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc);
}
multiEdgeInfo.SkipToEnd = multiArc.SkipToEnd == null
? (Func <string, string, string, string>)null
: ((ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc));
edgeList2.Add(multiEdgeInfo);
continue;
}
if (arc is SingleElementArc <TPayload, TRegister> singleArc)
{
var edgeInfo = CreateSingleEdgeInfo(stream, targetNode, singleArc, "payloadoffset");
edgeList2.Add(edgeInfo);
continue;
}
if (arc is ListElementArc <TPayload, TRegister> listArc)
{
var edgeInfo = CreateListEdgeInfo(stream, payloadRepresentation, targetNode, listArc, "payloadoffset");
edgeList2.Add(edgeInfo);
continue;
}
}
template.startEdgeInfos.Add(Tuple.Create(startState, edgeList2));
}
template.isSyncTimeSimultaneityFree = true; // The handwritten version doesn't make a distinction.
template.keyEqualityComparer =
(left, right) =>
stream.Properties.KeyEqualityComparer.GetEqualsExpr().Inline(left, right);
return(template.Generate <TKey, TPayload, TRegister, TAccumulator>());
}
private static Tuple <Type, string> GenerateInternal <TOuterKey, TInnerKey, TInnerResult, TResult>(Expression <Func <TInnerKey, TInnerResult, TResult> > resultSelector, bool isFirstLevelGroup)
{
Contract.Ensures(Contract.Result <Tuple <Type, string> >() != null);
Contract.Ensures(typeof(Pipe <TOuterKey, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTInnerKey = typeof(TInnerKey);
var typeofTInnerResult = typeof(TInnerResult);
var typeofTResult = typeof(TResult);
string expandedCode;
List <Assembly> assemblyReferences;
int numberOfGenericParameters;
// inline: var ok = UngroupTemplate.GetGeneratedCode<TOuterKey, TInnerKey, TInnerResult, TResult>(resultSelector, false, out generatedClassName, out expandedCode, out assemblyReferences, out numberOfGenericParameters);
var generatedClassName = string.Format(
"UngroupPipeGeneratedFrom_{0}_{1}_{2}_{3}_{4}",
typeOfTOuterKey.GetValidIdentifier(),
typeOfTInnerKey.GetValidIdentifier(),
typeofTInnerResult.GetValidIdentifier(),
typeofTResult.GetValidIdentifier(),
UngroupSequenceNumber++);
var inputMessageType = StreamMessageManager.GetStreamMessageType <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult>();
var innerResultRepresentation = new ColumnarRepresentation(typeofTInnerResult);
var resultRepresentation = new ColumnarRepresentation(typeofTResult);
var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >
{
// Leave the key parameter to the selector unchanged, so no substitution for parameters[0]
Tuple.Create(resultSelector.Parameters[1], new SelectParameterInformation()
{
BatchName = "inputBatch", BatchType = inputMessageType, IndexVariableName = "i", parameterRepresentation = innerResultRepresentation,
})
};
var result = SelectTransformer.Transform(resultSelector, parameterSubstitutions, resultRepresentation);
if (result.Error)
{
return(Tuple.Create((Type)null, errorMessages));
}
var template = new UngroupTemplate(generatedClassName, !isFirstLevelGroup, typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult)
{
innerResultRepresentation = innerResultRepresentation,
swingingFields = result.SwingingFields,
computedFields = result.ComputedFields,
unassignedFields = result.UnmentionedFields,
keyParameter = resultSelector.Parameters.First()
};
template.staticCtor = Transformer.StaticCtor(template.CLASSNAME);
expandedCode = template.TransformText();
assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
// input messages
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult>());
// output messages
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TResult>());
// memory pool
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TOuterKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(resultSelector));
numberOfGenericParameters = template.numberOfGenericParameters;
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (numberOfGenericParameters > 0)
{
generatedClassName = generatedClassName + "`" + numberOfGenericParameters.ToString(CultureInfo.InvariantCulture);
}
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult);
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> GenerateAFA <TPayload, TRegister, TAccumulator>(
AfaStreamable <Empty, TPayload, TRegister, TAccumulator> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <Empty, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var className = string.Format("GeneratedUngroupedAfa_{0}", AFASequenceNumber++);
var template = new UngroupedAfaTemplate(className, typeof(Empty), typeof(TPayload), typeof(TRegister), typeof(TAccumulator));
var payloadRepresentation = new ColumnarRepresentation(typeof(TPayload));
var registerRepresentation = new ColumnarRepresentation(typeof(TRegister));
template.isFinal = stream.afa.isFinal;
template.hasOutgoingArcs = stream.afa.hasOutgoingArcs;
template.startStates = stream.afa.startStates;
template.AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances;
var d1 = stream.afa.uncompiledAfa.transitionInfo;
var orderedKeys = d1.Keys.OrderBy(e => e).ToArray();
for (int i = 0; i < orderedKeys.Length; i++)
{
var sourceNodeNumber = orderedKeys[i];
var outgoingEdgesDictionary = d1[sourceNodeNumber];
var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray();
var edgeList1 = new List <EdgeInfo>();
for (int j = 0; j < orderedTargetNodes.Length; j++)
{
var targetNodeNumber = orderedTargetNodes[j];
var edge = outgoingEdgesDictionary[targetNodeNumber];
if (edge is SingleElementArc <TPayload, TRegister> searc)
{
var edgeInfo = CreateSingleEdgeInfo(stream, targetNodeNumber, searc, "i");
edgeList1.Add(edgeInfo);
}
}
template.currentlyActiveInfo.Add(Tuple.Create(sourceNodeNumber, edgeList1));
}
for (int i = 0; i < stream.afa.startStates.Length; i++)
{
var startState = stream.afa.startStates[i];
var edgeList2 = new List <EdgeInfo>();
var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState];
foreach (var edge in outgoingEdgeDictionary)
{
var targetNode = edge.Key;
var arc = edge.Value;
if (arc is SingleElementArc <TPayload, TRegister> searc)
{
var edgeInfo = CreateSingleEdgeInfo(stream, targetNode, searc, "i");
edgeList2.Add(edgeInfo);
}
}
template.newActivationInfo.Add(Tuple.Create(startState, edgeList2));
}
template.isSyncTimeSimultaneityFree = stream.Properties.IsSyncTimeSimultaneityFree;
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TPayload), typeof(TRegister), typeof(Stack <>));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TRegister>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <Empty, TRegister>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = new List <Type>();
list.AddRange(template.payloadType.GetAnonymousTypes());
list.AddRange(template.registerType.GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new FixedIntervalEquiJoinTemplate($"GeneratedFixedIntervalEquiJoin_{EquiJoinSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult))
{
leftDuration = stream.Left.Properties.ConstantDurationLength.Value,
rightDuration = stream.Right.Properties.ConstantDurationLength.Value
};
var keyAndLeftGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType).BracketedCommaSeparatedString();
var keyAndRightGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.rightType).BracketedCommaSeparatedString();
template.TKeyTResultGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString();
template.genericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType, template.rightType, template.resultType).BracketedCommaSeparatedString();
template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType);
template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType);
var resultMessageRepresentation = new ColumnarRepresentation(template.resultType);
var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType);
var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType);
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType);
template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters;
template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters;
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightFields = template.rightMessageRepresentation.AllFields;
template.resultFields = resultMessageRepresentation.AllFields;
template.ActiveEventTypeLeft = template.leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left";
template.ActiveEventTypeRight = template.rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right";
#region Key Equals
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparerEquals =
(left, right) =>
keyComparer.Inline(left, right);
if (template.keyType.IsAnonymousType())
{
template.keyComparerEquals =
(left, right) => $"keyComparerEquals({left}, {right})";
}
#endregion
#region Left Payload Equals
{
var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0);
template.leftComparerEquals = (left, right) => newLambda.Inline(left, right);
}
#endregion
#region Right Payload Equals
{
var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0);
template.rightComparerEquals = (left, right) => newLambda.Inline(left, right);
}
#endregion
#region Result Selector
{
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
errorMessages = "result selector is not a closed expression";
throw new InvalidOperationException();
}
#region LeftBatchSelector
{
var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region ActiveSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >();
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.activeSelector = (leftEvent, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
}
#endregion
return(template.Generate <TKey, TLeft, TRight, TResult>());
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Generate a batch class definition to be used as an aggreate definition.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for the aggregate.</typeparam>
/// <typeparam name="TInput">The input type for the aggregate.</typeparam>
/// <typeparam name="TOutput">The output type for the aggregate.</typeparam>
/// <typeparam name="TState">The type for the accumulated state held by the aggregate.</typeparam>
/// <typeparam name="TResult">The type of the result.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of UnaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TInput"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TInput, TState, TOutput, TResult>(
GroupedWindowStreamable <TKey, TInput, TState, TOutput, TResult> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(IStreamObserver <Empty, TInput>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
string expandedCode;
var template = new GroupedWindowTemplate();
var keyType = template.keyType = typeof(TKey);
var inputType = template.inputType = typeof(TInput);
var stateType = template.stateType = typeof(TState);
var outputType = template.outputType = typeof(TOutput);
var resultType = template.resultType = typeof(TResult);
template.TResult = resultType.GetCSharpSourceSyntax(); // BUGBUG: need to get any generic parameters needed
template.isUngrouped = (keyType == typeof(Empty));
template.className = string.Format("GeneratedGroupedAggregate_{0}", GroupedAggregateSequenceNumber++);
var inputMessageRepresentation = new ColumnarRepresentation(inputType);
var resultRepresentation = new ColumnarRepresentation(resultType);
var assemblyReferences = new List <Assembly>();
#region Key Selector
var keySelector = stream.KeySelector;
string transformedKeySelectorAsString;
if (keyType.IsAnonymousTypeName())
{
Contract.Assume(keySelector.Body is NewExpression);
var transformedFunction = Extensions.TransformUnaryFunction <TKey, TInput>(keySelector);
var newBody = (NewExpression)transformedFunction.Body;
transformedKeySelectorAsString = string.Join(",", newBody.Arguments);
}
else
{
var transformedFunction = Extensions.TransformUnaryFunction <Empty, TInput>(keySelector).Body;
if (transformedFunction == null)
{
return(null);
}
transformedKeySelectorAsString = transformedFunction.ExpressionToCSharp();
}
template.keySelector = transformedKeySelectorAsString;
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector));
#endregion
#region Key Comparer and HashCode
var keyComparer = EqualityComparerExpression <TKey> .Default;
template.keyComparerEquals =
(left, right) =>
keyComparer.GetEqualsExpr().Inline(left, right);
template.keyComparerGetHashCode =
(x) =>
keyComparer.GetGetHashCodeExpr().Inline(x);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyComparer.GetEqualsExpr()));
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyComparer.GetGetHashCodeExpr()));
#endregion
#region Aggregate functions
var initialStateLambda = stream.Aggregate.InitialState();
if (ConstantExpressionFinder.IsClosedExpression(initialStateLambda))
{
template.initialState = initialStateLambda.Body.ExpressionToCSharp();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(initialStateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the initial state lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledInitialState = true;
template.initialState = "initialState()";
}
}
var accumulateLambda = stream.Aggregate.Accumulate();
if (ConstantExpressionFinder.IsClosedExpression(accumulateLambda))
{
var accTransformedLambda = Extensions.TransformFunction <TKey, TInput>(accumulateLambda, 2);
template.accumulate = (stateArg, longArg) => accTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(accumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the accumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledAccumulate = true;
template.accumulate = (s1, s2) => string.Format("accumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var deaccumulateLambda = stream.Aggregate.Deaccumulate();
if (ConstantExpressionFinder.IsClosedExpression(deaccumulateLambda))
{
var deaccumulateTransformedLambda = Extensions.TransformFunction <TKey, TInput>(deaccumulateLambda, 2);
template.deaccumulate = (stateArg, longArg) => deaccumulateTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(deaccumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
throw new InvalidOperationException("Code Generation couldn't inline a lambda!");
}
else
{
template.useCompiledDeaccumulate = true;
template.deaccumulate = (s1, s2) => string.Format("deaccumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var differenceLambda = stream.Aggregate.Difference();
if (ConstantExpressionFinder.IsClosedExpression(differenceLambda))
{
template.difference = (stateArg1, stateArg2) => differenceLambda.Inline(stateArg1, stateArg2);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(differenceLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the deaccumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledDifference = true;
template.deaccumulate = (s1, s2) => string.Format("difference({0}, {1});", s1, s2);
}
}
var computeResultLambda = stream.Aggregate.ComputeResult();
if (ConstantExpressionFinder.IsClosedExpression(computeResultLambda))
{
if (outputType.IsAnonymousType())
{
if (computeResultLambda.Body is NewExpression newExpression)
{
errorMessages = "Code Generation for GroupedWindow: result selector must be a new expression for anonymous types";
throw new NotImplementedException(errorMessages);
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(computeResultLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the result selector lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledComputeResult = true;
template.computeResult = (stateArg) => "computeResult(" + stateArg + ")";
}
}
#endregion
template.BatchGeneratedFrom_Unit_TInput = Transformer.GetBatchClassName(typeof(Empty), inputType);
template.UnitTInputGenericParameters = string.Empty; // BUGBUG
template.UnitTResultGenericParameters = string.Empty; // BUGBUG
template.inputFields = inputMessageRepresentation.AllFields;
template.outputFields = resultRepresentation.AllFields;
var resultSelector = stream.ResultSelector;
var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); // dont want the parameters substituted for at all
var projectionResult = SelectTransformer.Transform(resultSelector, parameterSubstitutions, resultRepresentation, true);
if (projectionResult.Error)
{
return(null);
}
template.finalResultSelector =
(key, aggregateResult) =>
{
var parameters = new Dictionary <ParameterExpression, string>
{
{ resultSelector.Parameters.ElementAt(0), key }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
sb.AppendLine(string.Format("var {0} = {1};\n", resultSelector.Parameters.ElementAt(1).Name, aggregateResult));
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
sb.AppendFormat("this.batch.{0}.col[_c] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(parameters));
}
sb.AppendLine("}");
return(sb.ToString());
};
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(resultSelector));
template.staticCtor = Transformer.StaticCtor(template.className);
expandedCode = template.TransformText();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(typeof(Empty), typeof(TKey), typeof(TInput), typeof(TState), typeof(TOutput), typeof(FastDictionaryGenerator3)));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TInput>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <Empty, TResult>());
var assembly = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = assembly.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(typeof(TInput).GetAnonymousTypes());
list.AddRange(typeof(TState).GetAnonymousTypes());
list.AddRange(typeof(TOutput).GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new EquiJoinTemplate();
var keyType = template.keyType = typeof(TKey);
var leftType = template.leftType = typeof(TLeft);
var rightType = template.rightType = typeof(TRight);
var resultType = template.resultType = typeof(TResult);
var tm = new TypeMapper(keyType, leftType, rightType, resultType);
template.TKey = tm.CSharpNameFor(keyType);
template.TLeft = tm.CSharpNameFor(leftType);
template.TRight = tm.CSharpNameFor(rightType);
template.TResult = tm.CSharpNameFor(resultType);
var keyAndLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString();
var keyAndRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString();
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.genericParameters = tm.GenericTypeVariables(keyType, leftType, rightType, resultType).BracketedCommaSeparatedString();
template.className = string.Format("GeneratedEquiJoin_{0}", EquiJoinSequenceNumber++);
template.leftMessageRepresentation = new ColumnarRepresentation(leftType);
template.rightMessageRepresentation = new ColumnarRepresentation(rightType);
var resultMessageRepresentation = new ColumnarRepresentation(resultType);
var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType);
template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters;
template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters;
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightFields = template.rightMessageRepresentation.AllFields;
template.resultFields = resultMessageRepresentation.AllFields;
template.ActiveEventTypeLeft = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left";
template.ActiveEventTypeRight = rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right";
#region Key Equals
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparerEquals =
(left, right) =>
keyComparer.Inline(left, right);
if (keyType.IsAnonymousType())
{
template.keyComparerEquals =
(left, right) => string.Format("keyComparerEquals({0}, {1})", left, right);
}
#endregion
#region Left Payload Equals
{
var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0);
template.leftComparerEquals = (left, right) => newLambda.Inline(left, right);
}
#endregion
#region Right Payload Equals
{
var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0);
template.rightComparerEquals = (left, right) => newLambda.Inline(left, right);
}
#endregion
#region Result Selector
{
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
errorMessages = "result selector is not a closed expression";
throw new InvalidOperationException();
}
#region LeftBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), "i"), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), "j"), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region ActiveSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >();
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.activeSelector = (leftEvent, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
}
#endregion
if (stream.Left.Properties.IsIntervalFree && stream.Right.Properties.IsConstantDuration)
{
template.endPointHeap = "EndPointQueue";
}
else if (stream.Right.Properties.IsIntervalFree && stream.Left.Properties.IsConstantDuration)
{
template.endPointHeap = "EndPointQueue";
}
else if (stream.Left.Properties.IsConstantDuration && stream.Right.Properties.IsConstantDuration &&
stream.Left.Properties.ConstantDurationLength == stream.Right.Properties.ConstantDurationLength)
{
template.endPointHeap = "EndPointQueue";
}
else
{
template.endPointHeap = "EndPointHeap";
}
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(keyType, leftType, rightType, resultType);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (keyType.IsAnonymousType())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The key type for an equi-join is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n";
}
var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType, resultType);
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(leftType.GetAnonymousTypes());
list.AddRange(rightType.GetAnonymousTypes());
list.AddRange(resultType.GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Generate a batch class definition to be used as a Union pipe.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// union pipe class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TPayload">The payload type.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TPayload"/>, <typeparamref name="TPayload"/>, <typeparamref name="TKey"/>, <typeparamref name="TPayload"/>>.
/// </returns>
internal static Tuple <Type, string> GenerateUnionPipeClass <TKey, TPayload>(
UnionStreamable <TKey, TPayload> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TPayload, TPayload, TPayload>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
#if CODEGEN_TIMING
Stopwatch sw = new Stopwatch();
sw.Start();
#endif
string errorMessages = null;
try
{
var template = new UnionTemplate();
var keyType = template.keyType = typeof(TKey);
var payloadType = template.payloadType = typeof(TPayload);
var tm = new TypeMapper(keyType, payloadType);
template.TKey = tm.CSharpNameFor(keyType);
template.TPayload = tm.CSharpNameFor(payloadType);
var gps = tm.GenericTypeVariables(keyType, payloadType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.className = string.Format("GeneratedUnion_{0}", UnionSequenceNumber++);
var resultRepresentation = new ColumnarRepresentation(payloadType);
var batchGeneratedFrom_TKey_TPayload = Transformer.GetBatchClassName(keyType, payloadType);
var keyAndPayloadGenericParameters = tm.GenericTypeVariables(keyType, payloadType).BracketedCommaSeparatedString();
template.GeneratedBatchName = batchGeneratedFrom_TKey_TPayload + keyAndPayloadGenericParameters;
template.fields = resultRepresentation.AllFields;
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, payloadType);
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(payloadType.GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> GenerateAFA <TKey, TPayload, TRegister, TAccumulator>(
AfaStreamable <TKey, TPayload, TRegister, TAccumulator> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var className = string.Format("GeneratedGroupedAfaMultiEvent_{0}", AFASequenceNumber++);
var template = new GroupedAfaMultiEventTemplate(className, typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(TAccumulator));
var payloadRepresentation = new ColumnarRepresentation(typeof(TPayload));
template.isFinal = stream.afa.isFinal;
template.hasOutgoingArcs = stream.afa.hasOutgoingArcs;
template.startStates = stream.afa.startStates;
template.AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances;
var d1 = stream.afa.uncompiledAfa.transitionInfo;
var orderedKeys = d1.Keys.OrderBy(e => e).ToArray();
for (int i = 0; i < orderedKeys.Length; i++)
{
var sourceNodeNumber = orderedKeys[i];
var outgoingEdgesDictionary = d1[sourceNodeNumber];
var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray();
var edgeList = new List <MultiEdgeInfo>();
for (int j = 0; j < orderedTargetNodes.Length; j++)
{
var targetNodeNumber = orderedTargetNodes[j];
var edge = outgoingEdgesDictionary[targetNodeNumber];
if (edge is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc)
{
var multiEdgeInfo = new MultiEdgeInfo()
{
SourceNode = sourceNodeNumber,
TargetNode = targetNodeNumber,
fromStartState = stream.afa.startStates.Any(n => n == sourceNodeNumber),
EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNodeNumber),
Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg),
Accumulate = (ts, ev, reg, acc) =>
{
var transformedAccumulate = Extensions.TransformFunction <TKey, TPayload>(multiArc.Accumulate, 1, batchVariableName: "sourceBatch");
return(transformedAccumulate.Inline(ts, reg, acc));
},
Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg),
};
if (multiArc.Transfer == null)
{
multiEdgeInfo.Transfer = null;
}
else
{
multiEdgeInfo.Transfer = (ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg);
}
if (multiArc.Dispose == null)
{
multiEdgeInfo.Dispose = (acc) => "// no dispose function";
}
else
{
multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc);
}
if (multiArc.SkipToEnd == null)
{
multiEdgeInfo.SkipToEnd = null;
}
else
{
multiEdgeInfo.SkipToEnd = (ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc);
}
edgeList.Add(multiEdgeInfo);
}
}
template.edgeInfos.Add(Tuple.Create(sourceNodeNumber, edgeList));
}
for (int i = 0; i < stream.afa.startStates.Length; i++)
{
var startState = stream.afa.startStates[i];
var edgeList2 = new List <MultiEdgeInfo>();
var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState];
foreach (var edge in outgoingEdgeDictionary)
{
var targetNode = edge.Key;
var arc = edge.Value;
if (arc is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc)
{
var eps = EpsilonClosure(stream.afa, targetNode);
var multiEdgeInfo = new MultiEdgeInfo()
{
SourceNode = startState,
TargetNode = targetNode,
fromStartState = true,
EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNode),
Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg),
Accumulate = (ts, ev, reg, acc) =>
{
var transformedAccumulate = Extensions.TransformFunction <TKey, TPayload>(multiArc.Accumulate, 1, batchVariableName: "sourceBatch");
return(transformedAccumulate.Inline(ts, reg, acc));
},
Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg),
};
if (multiArc.Transfer == null)
{
multiEdgeInfo.Transfer = null;
}
else
{
multiEdgeInfo.Transfer = (ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg);
}
if (multiArc.Dispose == null)
{
multiEdgeInfo.Dispose = (acc) => "// no dispose function";
}
else
{
multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc);
}
if (multiArc.SkipToEnd == null)
{
multiEdgeInfo.SkipToEnd = null;
}
else
{
multiEdgeInfo.SkipToEnd = (ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc);
}
edgeList2.Add(multiEdgeInfo);
}
}
template.startEdgeInfos.Add(Tuple.Create(startState, edgeList2));
}
template.isSyncTimeSimultaneityFree = true; // The handwritten version doesn't make a distinction.
template.keyEqualityComparer =
(left, right) =>
stream.Properties.KeyEqualityComparer.GetEqualsExpr().Inline(left, right);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TRegister));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRegister>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TRegister>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(template.payloadType.GetAnonymousTypes());
list.AddRange(template.registerType.GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Generate a batch class definition to be used as StartEdgeEquiJoin operator.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TLeft">The payload type for the left side.</typeparam>
/// <typeparam name="TRight">The payload type for the right side.</typeparam>
/// <typeparam name="TResult">The payload type for the resulting stream.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TRight"/>, <typeparamref name="TKey"/>, <typeparamref name="TResult"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new IncreasingOrderEquiJoinTemplate($"GeneratedIncreasingOrderEquiJoin_{IOOEJSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult));
template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType);
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType);
template.rightFields = template.rightMessageRepresentation.AllFields;
var resultRepresentation = new ColumnarRepresentation(template.resultType);
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
#region Key Comparer
var keyComparer = stream.Left.Properties.KeyComparer.GetCompareExpr();
if (!ConstantExpressionFinder.IsClosedExpression(keyComparer))
{
return(null);
}
template.joinKeyOrderComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType);
template.TKeyTLeftGenericParameters = string.Empty; // BUGBUG
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType);
template.TKeyTRightGenericParameters = string.Empty; // BUGBUG
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType);
template.TKeyTResultGenericParameters = string.Empty; // BUGBUG
template.outputFields = resultRepresentation.AllFields;
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
return(null);
}
#region LeftBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), "i"), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), "j"), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
template.getOutputBatch = string.Format(
"pool.Get(out genericOutputBatch); output = ({0}{1})genericOutputBatch;",
Transformer.GetBatchClassName(template.keyType, template.resultType),
template.TKeyTResultGenericParameters);
return(template.Generate <TKey, TLeft, TRight, TResult>(selector));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TOuterKey, TSource, TInnerKey>(
Expression <Func <TInnerKey, int> > hashComparer,
Expression <Func <TSource, TInnerKey> > keySelector,
bool nested)
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTSource = typeof(TSource);
var typeOfTInnerKey = typeof(TInnerKey);
string expandedCode;
List <Assembly> assemblyReferences;
string errorMessages = null;
try
{
string generatedClassName = $"GeneratedGroupStreamable_{GroupStreamableSequenceNumber++}";
string transformedKeySelectorAsString;
MyFieldInfo swingingField = default;
if (typeOfTInnerKey.IsAnonymousTypeName())
{
Contract.Assume(keySelector.Body is NewExpression);
var transformedFunction = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector);
var newBody = (NewExpression)transformedFunction.Body;
transformedKeySelectorAsString = string.Join(",", newBody.Arguments.Select(arg => arg.ExpressionToCSharp()));
}
else
{
var body = keySelector.Body;
if (!nested && body is MemberExpression singleFieldProjection)
{
if (singleFieldProjection.Expression is ParameterExpression dereferencedObject)
{
// then can just swing a pointer to set the key of the result message
Contract.Assume(dereferencedObject == keySelector.Parameters.ElementAt(0));
var f = singleFieldProjection.Member;
var sourceMessageRepresentation = new ColumnarRepresentation(typeOfTSource);
var fieldToSwingFrom = sourceMessageRepresentation.Fields[f.Name];
swingingField = fieldToSwingFrom;
transformedKeySelectorAsString = swingingField.Name + "_col[i]";
}
else
{
var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body;
if (transformedPredicate == null)
{
return(Tuple.Create((Type)null, errorMessages));
}
transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp();
}
}
else
{
var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body;
if (transformedPredicate == null)
{
return(Tuple.Create((Type)null, errorMessages));
}
transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp();
}
}
var inlinedHashCodeComputation = hashComparer.Inline("key");
var template = new GroupTemplate(
generatedClassName,
typeOfTOuterKey,
typeOfTSource,
typeOfTInnerKey,
transformedKeySelectorAsString,
inlinedHashCodeComputation,
nested)
{
swingingField = swingingField,
payloadMightBeNull = typeOfTSource.CanContainNull()
};
if (!nested && Config.UseMultiString &&
typeOfTInnerKey.Equals(typeof(string)) &&
keySelector.IsSimpleFieldOrPropertyAccess())
{
var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body;
template.transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp();
template.inlinedHashCodeComputation = "hashCodeVector.col[i]";
var fieldName = ((MemberExpression)keySelector.Body).Member.Name;
template.vectorHashCodeInitialization = $"resultBatch.hash = {Transformer.ColumnFieldPrefix}{fieldName}_col.GetHashCode(batch.bitvector);";
template.swingingHashColumn = true;
}
template.fields = new ColumnarRepresentation(typeOfTSource).AllFields;
expandedCode = template.TransformText();
assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TSource>());
if (nested)
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
}
else
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TInnerKey, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TInnerKey, TSource>());
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (typeOfTInnerKey.IsAnonymousTypeName())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The inner key type for Group is anonymous, causing the use of Activator.CreateInstance in an inner loop. This will lead to poor performance.\n";
}
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new StartEdgeEquiJoinTemplate();
var keyType = template.keyType = typeof(TKey);
var leftType = template.leftType = typeof(TLeft);
var rightType = template.rightType = typeof(TRight);
var resultType = template.resultType = typeof(TResult);
var tm = new TypeMapper(keyType, leftType, rightType, resultType);
template.TKey = tm.CSharpNameFor(keyType);
template.TLeft = tm.CSharpNameFor(leftType);
template.TRight = tm.CSharpNameFor(rightType);
template.TResult = tm.CSharpNameFor(resultType);
var gps = tm.GenericTypeVariables(keyType, leftType, rightType, resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.className = string.Format("GeneratedStartEdgeEquiJoin_{0}", StartEdgeEquiJoinSequenceNumber++);
template.leftMessageRepresentation = new ColumnarRepresentation(leftType);
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightMessageRepresentation = new ColumnarRepresentation(rightType);
template.rightFields = template.rightMessageRepresentation.AllFields;
var outputMessageRepresentation = new ColumnarRepresentation(resultType);
var resultRepresentation = outputMessageRepresentation;
#region Key Comparer
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
template.TKeyTLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.TKeyTRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType);
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.outputFields = resultRepresentation.AllFields;
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
#region LeftBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), "i"), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), "j"), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType, resultType);
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(leftType.GetAnonymousTypes());
list.AddRange(rightType.GetAnonymousTypes());
list.AddRange(resultType.GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
