/// <summary>
/// Generate a batch class definition from the types <typeparamref name="TKey"/> and <typeparamref name="TPayload"/>.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// batch class.
/// </summary>
/// <typeparam name="TKey">
/// The key type for the batch class.
/// </typeparam>
/// <typeparam name="TPayload">
/// The payload type for which a batch class is generated. The batch class will have a field F of type T[] for every field
/// F of type T in the type <typeparamref name="TPayload"/>.
/// </typeparam>
/// <returns>
/// A type that is defined to be a subtype of Batch<<typeparamref name="TKey"/>,<typeparamref name="TPayload"/>>.
/// </returns>
public static Type GenerateBatchClass <TKey, TPayload>()
{
#if CODEGEN_TIMING
Stopwatch sw = new Stopwatch();
sw.Start();
#endif
var keyType = typeof(TKey);
var payloadType = typeof(TPayload);
SafeBatchTemplate.GetGeneratedCode(keyType, payloadType, out string generatedClassName, out string expandedCode, out List <Assembly> assemblyReferences);
assemblyReferences.Add(MemoryManager.GetMemoryPool <TKey, TPayload>().GetType().GetTypeInfo().Assembly);
assemblyReferences.Add(SystemRuntimeSerializationDll);
if (keyType != typeof(Empty))
{
assemblyReferences.Add(typeof(Empty).GetTypeInfo().Assembly);
assemblyReferences.Add(StreamMessageManager.GetStreamMessageType <Empty, TPayload>().GetTypeInfo().Assembly);
}
var a = CompileSourceCode(expandedCode, assemblyReferences, out string errorMessages);
var t = a.GetType(generatedClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(payloadType.GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
#if CODEGEN_TIMING
sw.Stop();
Console.WriteLine("Time to generate and instantiate a batch for {0},{1}: {2}ms",
keyType.GetCSharpSourceSyntax(), payload.GetCSharpSourceSyntax(), sw.ElapsedMilliseconds);
#endif
return(t);
}
private void Ingress(IStreamObserver <TKey, TPayload> observer)
{
int messages = 0;
try
{
var serializer = StreamableSerializer.Create <QueuedMessage <StreamMessage <TKey, TPayload> > >(
new SerializerSettings()
{
KnownTypes = StreamMessageManager.GeneratedTypes()
});
while (true)
{
var message = serializer.Deserialize(this.stream);
if (message.Kind != MessageKind.Completed)
{
observer.OnNext(message.Message);
messages++;
}
if (message.Kind == MessageKind.Completed ||
(this.numMessages != 0 && messages == this.numMessages))
{
observer.OnCompleted();
break;
}
}
}
catch (Exception e)
{
observer.OnError(e);
}
this.onSubscriptionCompleted();
}
public BinaryStreamObserver(StreamProperties <TKey, TPayload> streamProperties, Stream stream)
{
this.serializer = StreamSerializer.Create <QueuedMessage <StreamMessage <TKey, TPayload> > >(
new SerializerSettings()
{
KnownTypes = StreamMessageManager.GeneratedTypes()
});
this.stream = stream;
}
private static LambdaExpression /*?*/ MakeOneParameterColumnar <TKey, TPayload>(LambdaExpression f, int index, string indexVariableName)
{
Contract.Requires(index >= 0 && index < f.Parameters.Count);
var batchType = StreamMessageManager.GetStreamMessageType <TKey, TPayload>();
var d = new Dictionary <ParameterExpression, ColumnOriented.SubstitutionInformation>();
var payloadRepresentation = new ColumnarRepresentation(typeof(TPayload));
d.Add(
f.Parameters[index],
new ColumnOriented.SubstitutionInformation
{
columnarRepresentation = payloadRepresentation,
nameForIndexVariable = indexVariableName,
typeOfBatchVariable = batchType,
});
return(ColumnOriented.Transform(f, d));
}
public static Tuple <Type, string> Generate <TKey, TPayload, TResult>(SelectManyStreamable <TKey, TPayload, TResult> stream)
{
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string generatedClassName;
string expandedCode;
string errorMessages = null;
try
{
generatedClassName = $"SelectMany_{sequenceNumber++}";
var keyType = typeof(TKey);
var payloadType = typeof(TPayload);
var resultType = typeof(TResult);
var template = new SelectManyTemplate(generatedClassName, keyType, payloadType, resultType);
var tm = new TypeMapper(keyType, payloadType, resultType);
var gps = tm.GenericTypeVariables(keyType, payloadType, resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.numberOfGenericParameters = gps.Count();
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.MemoryPoolGenericParameters = $"<{template.TKey}, {template.TResult}>";
if (resultType == typeof(int) || resultType == typeof(long) || resultType == typeof(string))
{
template.MemoryPoolGenericParameters = string.Empty;
}
var payloadParameterIndex = 0;
if (stream.HasKey && stream.HasStartEdge)
{
payloadParameterIndex = 2;
}
else if (stream.HasKey || stream.HasStartEdge)
{
payloadParameterIndex = 1;
}
var selector = stream.Selector;
var payloadParameter = selector.Parameters.ElementAt(payloadParameterIndex);
template.PARAMETER = payloadParameter.Name;
template.resultPayloadRepresentation = new ColumnarRepresentation(resultType);
template.resultFields = template.resultPayloadRepresentation.AllFields;
if (template.numberOfGenericParameters > 0)
{
generatedClassName = generatedClassName + "`" + template.numberOfGenericParameters.ToString(CultureInfo.InvariantCulture);
}
expandedCode = string.Empty;
Expression transformedSelector = selector;
// No substitutions are made for the start edge parameter or key parameter. Both just remain in the
// body of the result selector and are set as local variables in the generated code.
var keyParameterIndex = stream.HasStartEdge ? 1 : 0;
var tuple = OptimizeSelectMany(selector.Body);
if (tuple != null)
{
template.enumerableRepeatSelector = true;
var resultSelector = stream.Selector;
var sourceMessageType = StreamMessageManager.GetStreamMessageType <TKey, TPayload>();
var pseudoLambdaParameters = new ParameterExpression[stream.HasKey ? 2 : 1];
var pseudoLambdaIndex = 0;
if (stream.HasKey)
{
pseudoLambdaParameters[pseudoLambdaIndex++] = selector.Parameters[keyParameterIndex];
}
pseudoLambdaParameters[pseudoLambdaIndex] = payloadParameter;
var pseudoLambda = Expression.Lambda(tuple.Item1, pseudoLambdaParameters);
var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >
{
Tuple.Create(payloadParameter, new SelectParameterInformation()
{
BatchName = "batch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = new ColumnarRepresentation(payloadType)
})
};
var projectionResult = SelectTransformer.Transform(pseudoLambda, parameterSubstitutions, template.resultPayloadRepresentation, true, stream.HasStartEdge);
template.computedFields = projectionResult.ComputedFields;
template.useEnumerator = false;
var loopCounter = tuple.Item2;
var newParameters = new ParameterExpression[stream.HasKey ? 2 : 1];
newParameters[0] = payloadParameter;
if (stream.HasKey)
{
newParameters[1] = selector.Parameters[keyParameterIndex];
}
var loopCounterLambda = Expression.Lambda(loopCounter, payloadParameter);
var transformedLoopCounter = Extensions.TransformFunction <TKey, TPayload>(loopCounterLambda, 0);
template.loopCounter = transformedLoopCounter.Body.ExpressionToCSharp();
// REVIEW: Alternative: use Inline to replace occurrences of the key parameter
// with "batch.key.col[i]".
if (stream.HasKey)
{
template.keyParameterName = selector.Parameters[keyParameterIndex].Name;
}
}
else
{
transformedSelector = Extensions.TransformFunction <TKey, TPayload>(stream.Selector, payloadParameterIndex).Body;
if (transformedSelector == null)
{
template.useEnumerator = true;
template.transformedSelectorAsSource = stream.Selector.ExpressionToCSharp();
}
else
{
var tuple2 = OptimizeSelectMany(transformedSelector);
if (tuple2 != null)
{
template.useEnumerator = false;
template.loopCounter = tuple2.Item2.ExpressionToCSharp();
template.transformedSelectorAsSource = tuple2.Item1.ExpressionToCSharp();
}
else
{
template.useEnumerator = true;
template.transformedSelectorAsSource = transformedSelector.ExpressionToCSharp();
}
}
}
template.StartEdgeParameterName = stream.HasStartEdge ? selector.Parameters.ElementAt(0).Name : null;
template.hasKey = stream.HasKey;
expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TResult));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(stream.Selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeof(TKey), typeof(TPayload), typeof(TResult));
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TKey, TPayload, TResult>(SelectStreamable <TKey, TPayload, TResult> stream)
{
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string generatedClassName;
string errorMessages = null;
try
{
var keyType = typeof(TKey);
var payloadType = typeof(TPayload);
var resultType = typeof(TResult);
generatedClassName = string.Format("Select_{0}_{1}_{2}_{3}", keyType.GetValidIdentifier(), payloadType.GetValidIdentifier(), resultType.GetValidIdentifier(), sequenceNumber++);
var template = new SelectTemplate(generatedClassName, keyType, payloadType, resultType);
var tm = new TypeMapper(keyType, payloadType, resultType);
var gps = tm.GenericTypeVariables(keyType, payloadType, resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.numberOfGenericParameters = gps.Count();
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.MemoryPoolGenericParameters = $"<{template.TKey}, {template.TResult}>";
if (resultType == typeof(int) || resultType == typeof(long) || resultType == typeof(string))
{
template.MemoryPoolGenericParameters = string.Empty;
}
var payloadParameterIndex = 0;
if (stream.HasKey && stream.HasStartEdge)
{
payloadParameterIndex = 2;
}
else if (stream.HasKey || stream.HasStartEdge)
{
payloadParameterIndex = 1;
}
template.PARAMETER = stream.Selector.Parameters[payloadParameterIndex].Name;
template.resultPayloadRepresentation = new ColumnarRepresentation(resultType);
template.destinationFields = template.resultPayloadRepresentation.AllFields;
if (template.numberOfGenericParameters > 0)
{
generatedClassName = generatedClassName + "`" + template.numberOfGenericParameters.ToString(CultureInfo.InvariantCulture);
}
var resultSelector = stream.Selector;
var sourceMessageType = StreamMessageManager.GetStreamMessageType <TKey, TPayload>();
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >();
// Don't create a parameter substitution for the start edge parameter. That will just remain in the
// body of the result selector and will be set as a local variable in the generated code.
if (stream.HasKey)
{
var keyRepresentation = new ColumnarRepresentation(keyType, "key");
var keyParameterIndex = stream.HasStartEdge ? 1 : 0;
parameterSubsitutions.Add(Tuple.Create(
resultSelector.Parameters.ElementAt(keyParameterIndex),
new SelectParameterInformation()
{
BatchName = "sourceBatch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = keyRepresentation,
}));
}
parameterSubsitutions.Add(Tuple.Create(
resultSelector.Parameters.ElementAt(payloadParameterIndex),
new SelectParameterInformation()
{
BatchName = "sourceBatch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = new ColumnarRepresentation(payloadType)
}));
var projectionResult = SelectTransformer.Transform(resultSelector, parameterSubsitutions, template.resultPayloadRepresentation, false, stream.HasStartEdge);
if (projectionResult.Error)
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
throw new InvalidOperationException("Code Generation couldn't transform a selector!");
}
return(Tuple.Create((Type)null, errorMessages));
}
template.StartEdgeParameterName = stream.HasStartEdge ? resultSelector.Parameters.ElementAt(0).Name : null;
template.computedFields = projectionResult.ComputedFields;
template.swingingFields = projectionResult.SwingingFields;
template.unassignedFields = projectionResult.UnmentionedFields;
template.ProjectionReturningResultInstance = projectionResult.ProjectionReturningResultInstance;
template.multiStringOperations = projectionResult.MultiStringOperations;
template.needSourceInstance = projectionResult.NeedsSourceInstance;
var d = new Dictionary <MyFieldInfo, int>();
foreach (var f in template.swingingFields)
{
var target = f.Item1;
var source = f.Item2;
if (!d.ContainsKey(source))
{
d.Add(source, 0);
}
d[source] = d[source] + 1;
}
template.swungFieldsCount = d;
template.nonSwingingFields = template.fields.Where(sf => !template.swingingFields.Any(swingingField => swingingField.Item2.Equals(sf)));
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(
typeof(TKey), typeof(TPayload), typeof(TResult), typeof(IStreamable <,>));
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(stream.Selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeof(TKey), typeof(TPayload), typeof(TResult));
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new FixedIntervalEquiJoinTemplate($"GeneratedFixedIntervalEquiJoin_{EquiJoinSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult))
{
leftDuration = stream.Left.Properties.ConstantDurationLength.Value,
rightDuration = stream.Right.Properties.ConstantDurationLength.Value
};
var keyAndLeftGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType).BracketedCommaSeparatedString();
var keyAndRightGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.rightType).BracketedCommaSeparatedString();
template.TKeyTResultGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString();
template.genericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType, template.rightType, template.resultType).BracketedCommaSeparatedString();
template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType);
template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType);
var resultMessageRepresentation = new ColumnarRepresentation(template.resultType);
var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType);
var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType);
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType);
template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters;
template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters;
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightFields = template.rightMessageRepresentation.AllFields;
template.resultFields = resultMessageRepresentation.AllFields;
template.ActiveEventTypeLeft = template.leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left";
template.ActiveEventTypeRight = template.rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right";
#region Key Equals
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparerEquals =
(left, right) =>
keyComparer.Inline(left, right);
if (template.keyType.IsAnonymousType())
{
template.keyComparerEquals =
(left, right) => $"keyComparerEquals({left}, {right})";
}
#endregion
#region Left Payload Equals
{
var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0);
template.leftComparerEquals = (left, right) => newLambda.Inline(left, right);
}
#endregion
#region Right Payload Equals
{
var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0);
template.rightComparerEquals = (left, right) => newLambda.Inline(left, right);
}
#endregion
#region Result Selector
{
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
errorMessages = "result selector is not a closed expression";
throw new InvalidOperationException();
}
#region LeftBatchSelector
{
var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region ActiveSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >();
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.activeSelector = (leftEvent, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
}
#endregion
return(template.Generate <TKey, TLeft, TRight, TResult>());
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new StartEdgeEquiJoinTemplate($"GeneratedStartEdgeEquiJoin_{StartEdgeEquiJoinSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult));
var gps = template.tm.GenericTypeVariables(template.keyType, template.leftType, template.rightType, template.resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType);
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType);
template.rightFields = template.rightMessageRepresentation.AllFields;
var resultRepresentation = new ColumnarRepresentation(template.resultType);
#region Key Comparer
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType);
template.TKeyTLeftGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.leftType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType);
template.TKeyTRightGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.rightType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType);
template.TKeyTResultGenericParameters = template.tm.GenericTypeVariables(template.keyType, template.resultType).BracketedCommaSeparatedString();
template.outputFields = resultRepresentation.AllFields;
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
#region LeftBatchSelector
{
var leftBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("i");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = leftBatchIndexVariable, parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var parameterMap = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), leftBatchIndexVariable), leftIndex },
{ selector.Parameters[1], rightEvent }
};
if (projectionResult.ProjectionReturningResultInstance != null)
{
return($"this.output[index] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
else
{
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendLine($"this.output.{f.Name}.AddString({e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)});");
}
else
{
sb.AppendLine($"this.output.{f.Name}.col[index] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
}
sb.AppendLine("}");
return(sb.ToString());
}
};
}
#endregion
#region RightBatchSelector
{
var rightBatchIndexVariable = selector.Parameters.GenerateFreshVariableName("j");
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = rightBatchIndexVariable, parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var parameterMap = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), rightBatchIndexVariable), rightIndex }
};
if (projectionResult.ProjectionReturningResultInstance != null)
{
return($"this.output[index] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
else
{
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendLine($"this.output.{f.Name}.AddString({e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)});");
}
else
{
sb.AppendLine($"this.output.{f.Name}.col[index] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameterMap)};");
}
}
sb.AppendLine("}");
return(sb.ToString());
}
};
}
#endregion
return(template.Generate <TKey, TLeft, TRight, TResult>(selector));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Assembly GeneratedStreamMessageAssembly <TKey, TPayload>() => StreamMessageManager.GetStreamMessageType <TKey, TPayload>().GetTypeInfo().Assembly;
/// <summary>
/// Generate a batch class definition to be used as StartEdgeEquiJoin operator.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TLeft">The payload type for the left side.</typeparam>
/// <typeparam name="TRight">The payload type for the right side.</typeparam>
/// <typeparam name="TResult">The payload type for the resulting stream.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TRight"/>, <typeparamref name="TKey"/>, <typeparamref name="TResult"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new IncreasingOrderEquiJoinTemplate($"GeneratedIncreasingOrderEquiJoin_{IOOEJSequenceNumber++}", typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult));
template.leftMessageRepresentation = new ColumnarRepresentation(template.leftType);
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightMessageRepresentation = new ColumnarRepresentation(template.rightType);
template.rightFields = template.rightMessageRepresentation.AllFields;
var resultRepresentation = new ColumnarRepresentation(template.resultType);
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
#region Key Comparer
var keyComparer = stream.Left.Properties.KeyComparer.GetCompareExpr();
if (!ConstantExpressionFinder.IsClosedExpression(keyComparer))
{
return(null);
}
template.joinKeyOrderComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(template.keyType, template.leftType);
template.TKeyTLeftGenericParameters = string.Empty; // BUGBUG
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(template.keyType, template.rightType);
template.TKeyTRightGenericParameters = string.Empty; // BUGBUG
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(template.keyType, template.resultType);
template.TKeyTResultGenericParameters = string.Empty; // BUGBUG
template.outputFields = resultRepresentation.AllFields;
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
return(null);
}
#region LeftBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), "i"), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), "j"), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
template.getOutputBatch = string.Format(
"pool.Get(out genericOutputBatch); output = ({0}{1})genericOutputBatch;",
Transformer.GetBatchClassName(template.keyType, template.resultType),
template.TKeyTResultGenericParameters);
return(template.Generate <TKey, TLeft, TRight, TResult>(selector));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new EquiJoinTemplate();
var keyType = template.keyType = typeof(TKey);
var leftType = template.leftType = typeof(TLeft);
var rightType = template.rightType = typeof(TRight);
var resultType = template.resultType = typeof(TResult);
var tm = new TypeMapper(keyType, leftType, rightType, resultType);
template.TKey = tm.CSharpNameFor(keyType);
template.TLeft = tm.CSharpNameFor(leftType);
template.TRight = tm.CSharpNameFor(rightType);
template.TResult = tm.CSharpNameFor(resultType);
var keyAndLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString();
var keyAndRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString();
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.genericParameters = tm.GenericTypeVariables(keyType, leftType, rightType, resultType).BracketedCommaSeparatedString();
template.className = string.Format("GeneratedEquiJoin_{0}", EquiJoinSequenceNumber++);
template.leftMessageRepresentation = new ColumnarRepresentation(leftType);
template.rightMessageRepresentation = new ColumnarRepresentation(rightType);
var resultMessageRepresentation = new ColumnarRepresentation(resultType);
var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType);
template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters;
template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters;
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightFields = template.rightMessageRepresentation.AllFields;
template.resultFields = resultMessageRepresentation.AllFields;
template.ActiveEventTypeLeft = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left";
template.ActiveEventTypeRight = rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right";
#region Key Equals
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparerEquals =
(left, right) =>
keyComparer.Inline(left, right);
if (keyType.IsAnonymousType())
{
template.keyComparerEquals =
(left, right) => string.Format("keyComparerEquals({0}, {1})", left, right);
}
#endregion
#region Left Payload Equals
{
var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0);
template.leftComparerEquals = (left, right) => newLambda.Inline(left, right);
}
#endregion
#region Right Payload Equals
{
var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0);
template.rightComparerEquals = (left, right) => newLambda.Inline(left, right);
}
#endregion
#region Result Selector
{
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
errorMessages = "result selector is not a closed expression";
throw new InvalidOperationException();
}
#region LeftBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), "i"), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), "j"), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region ActiveSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >();
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.activeSelector = (leftEvent, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
}
#endregion
if (stream.Left.Properties.IsIntervalFree && stream.Right.Properties.IsConstantDuration)
{
template.endPointHeap = "EndPointQueue";
}
else if (stream.Right.Properties.IsIntervalFree && stream.Left.Properties.IsConstantDuration)
{
template.endPointHeap = "EndPointQueue";
}
else if (stream.Left.Properties.IsConstantDuration && stream.Right.Properties.IsConstantDuration &&
stream.Left.Properties.ConstantDurationLength == stream.Right.Properties.ConstantDurationLength)
{
template.endPointHeap = "EndPointQueue";
}
else
{
template.endPointHeap = "EndPointHeap";
}
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(keyType, leftType, rightType, resultType);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (keyType.IsAnonymousType())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The key type for an equi-join is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n";
}
var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType, resultType);
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(leftType.GetAnonymousTypes());
list.AddRange(rightType.GetAnonymousTypes());
list.AddRange(resultType.GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
private static Tuple <Type, string> GenerateInternal <TOuterKey, TInnerKey, TInnerResult, TResult>(Expression <Func <TInnerKey, TInnerResult, TResult> > resultSelector, bool isFirstLevelGroup)
{
Contract.Ensures(Contract.Result <Tuple <Type, string> >() != null);
Contract.Ensures(typeof(Pipe <TOuterKey, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTInnerKey = typeof(TInnerKey);
var typeofTInnerResult = typeof(TInnerResult);
var typeofTResult = typeof(TResult);
string expandedCode;
List <Assembly> assemblyReferences;
int numberOfGenericParameters;
// inline: var ok = UngroupTemplate.GetGeneratedCode<TOuterKey, TInnerKey, TInnerResult, TResult>(resultSelector, false, out generatedClassName, out expandedCode, out assemblyReferences, out numberOfGenericParameters);
var generatedClassName = string.Format(
"UngroupPipeGeneratedFrom_{0}_{1}_{2}_{3}_{4}",
typeOfTOuterKey.GetValidIdentifier(),
typeOfTInnerKey.GetValidIdentifier(),
typeofTInnerResult.GetValidIdentifier(),
typeofTResult.GetValidIdentifier(),
UngroupSequenceNumber++);
var inputMessageType = StreamMessageManager.GetStreamMessageType <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult>();
var innerResultRepresentation = new ColumnarRepresentation(typeofTInnerResult);
var resultRepresentation = new ColumnarRepresentation(typeofTResult);
var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >
{
// Leave the key parameter to the selector unchanged, so no substitution for parameters[0]
Tuple.Create(resultSelector.Parameters[1], new SelectParameterInformation()
{
BatchName = "inputBatch", BatchType = inputMessageType, IndexVariableName = "i", parameterRepresentation = innerResultRepresentation,
})
};
var result = SelectTransformer.Transform(resultSelector, parameterSubstitutions, resultRepresentation);
if (result.Error)
{
return(Tuple.Create((Type)null, errorMessages));
}
var template = new UngroupTemplate(generatedClassName, !isFirstLevelGroup, typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult)
{
innerResultRepresentation = innerResultRepresentation,
swingingFields = result.SwingingFields,
computedFields = result.ComputedFields,
unassignedFields = result.UnmentionedFields,
keyParameter = resultSelector.Parameters.First()
};
template.staticCtor = Transformer.StaticCtor(template.CLASSNAME);
expandedCode = template.TransformText();
assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
// input messages
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult>());
// output messages
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TResult>());
// memory pool
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TOuterKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(resultSelector));
numberOfGenericParameters = template.numberOfGenericParameters;
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (numberOfGenericParameters > 0)
{
generatedClassName = generatedClassName + "`" + numberOfGenericParameters.ToString(CultureInfo.InvariantCulture);
}
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult);
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TKey, TInput, TState, TOutput>(SnapshotWindowStreamable <TKey, TInput, TState, TOutput> stream, AggregatePipeType pipeType)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(IStreamObserver <TKey, TInput>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
var container = stream.Properties.QueryContainer;
string generatedClassName;
string expandedCode;
List <Assembly> assemblyReferences;
string errorMessages = null;
try
{
AggregateTemplate template;
switch (pipeType)
{
case AggregatePipeType.StartEdge:
template = new SnapshotWindowStartEdgeTemplate();
break;
case AggregatePipeType.PriorityQueue:
template = new SnapshotWindowPriorityQueueTemplate();
break;
case AggregatePipeType.Tumbling:
template = new SnapshotWindowTumblingTemplate();
break;
case AggregatePipeType.Sliding:
template = new SnapshotWindowSlidingTemplate();
break;
case AggregatePipeType.Hopping:
template = new SnapshotWindowHoppingTemplate
{
hopsPerDuration = ((int)(stream.Source.Properties.ConstantDurationLength.Value / stream.Source.Properties.ConstantHopLength) + 1).ToString()
};
break;
default:
Contract.Assert(false, "case meant to be exhaustive");
throw new InvalidOperationException("case meant to be exhaustive");
}
template.isUngrouped = (typeof(TKey) == typeof(Empty));
var keyType = template.keyType = typeof(TKey);
var inputType = template.inputType = typeof(TInput);
var stateType = template.stateType = typeof(TState);
var outputType = template.outputType = typeof(TOutput);
template.inputFields = new ColumnarRepresentation(inputType).AllFields;
template.outputFields = new ColumnarRepresentation(outputType).AllFields;
assemblyReferences = new List <Assembly>();
#region Key Comparer
IEqualityComparerExpression <TKey> keyComparer;
keyComparer = stream.Properties.KeyEqualityComparer;
var equalsExpression = keyComparer.GetEqualsExpr();
var getHashcodeExpression = keyComparer.GetGetHashCodeExpr();
template.inlinedKeyComparerEquals =
(left, right) =>
string.Format("({0})", equalsExpression.Inline(left, right));
template.inlinedKeyComparerGetHashCode =
(x) =>
string.Format("({0}/* inlined GetHashCode */)", getHashcodeExpression.Inline(x));
if (keyType.IsAnonymousType())
{
template.inlinedKeyComparerEquals =
(left, right) => string.Format("keyComparerEquals({0}, {1})", left, right);
template.inlinedKeyComparerGetHashCode =
(x) => string.Format("keyComparerGetHashCode({0})", x);
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(equalsExpression));
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(getHashcodeExpression));
#endregion
#region Aggregate Functions
var initialStateLambda = stream.Aggregate.InitialState();
if (ConstantExpressionFinder.IsClosedExpression(initialStateLambda))
{
template.initialState = initialStateLambda.Body.ExpressionToCSharp();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(initialStateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the initial state lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledInitialState = true;
template.initialState = "initialState()";
}
}
var accumulateLambda = stream.Aggregate.Accumulate();
if (ConstantExpressionFinder.IsClosedExpression(accumulateLambda))
{
var accTransformedLambda = Extensions.TransformFunction <TKey, TInput>(accumulateLambda, 2);
template.accumulate = (stateArg, longArg) => accTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(accumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the accumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledAccumulate = true;
template.accumulate = (s1, s2) => string.Format("accumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var deaccumulateLambda = stream.Aggregate.Deaccumulate();
if (ConstantExpressionFinder.IsClosedExpression(deaccumulateLambda))
{
var deaccumulateTransformedLambda = Extensions.TransformFunction <TKey, TInput>(deaccumulateLambda, 2);
template.deaccumulate = (stateArg, longArg) => deaccumulateTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(deaccumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the deaccumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledDeaccumulate = true;
template.deaccumulate = (s1, s2) => string.Format("deaccumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var differenceLambda = stream.Aggregate.Difference();
if (ConstantExpressionFinder.IsClosedExpression(differenceLambda))
{
template.difference = (stateArg1, stateArg2) => differenceLambda.Inline(stateArg1, stateArg2);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(differenceLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the difference lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledDifference = true;
template.difference = (s1, s2) => string.Format("difference({0}, {1});", s1, s2);
}
}
var computeResultLambda = stream.Aggregate.ComputeResult();
if (ConstantExpressionFinder.IsClosedExpression(computeResultLambda))
{
if (outputType.IsAnonymousType())
{
if (computeResultLambda.Body is NewExpression newExpression)
{
var outputBatchType = StreamMessageManager.GetStreamMessageType <TKey, TOutput>();
var foo = Transform(newExpression, outputBatchType);
template.computeResult = (stateArg) => Expression.Lambda(foo, computeResultLambda.Parameters.ToArray()).Inline(stateArg);
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(computeResultLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar || outputType.IsAnonymousType())
{
// second disjunct is because if we aren't inlining the computeResult function and
// the output type is anonymous, calling the compiled computeResult function returns
// a value of the anonymous type and since the generated operator represents the anonymous
// type as a generic parameter, it can't use the "field" (i.e., property) names to
// get the individual pieces to assign to each column of the output message.
errorMessages = "Code Generation for Aggregate: couldn't inline the compute result lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledComputeResult = true;
template.computeResult = (stateArg) => "computeResult(" + stateArg + ");";
}
}
#endregion
generatedClassName = template.className = string.Format("Aggregate_{0}", sequenceNumber++);
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(keyType, inputType, stateType, outputType);
template.staticCtor = Transformer.StaticCtor(template.className);
expandedCode = template.TransformText();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TInput), typeof(TState), typeof(TOutput), typeof(FastDictionaryGenerator), typeof(SortedDictionary <,>)));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TInput>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TOutput>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TOutput>());
if (container != null)
{
assemblyReferences.AddRange(container.CollectedGeneratedTypes.Select(o => o.GetTypeInfo().Assembly));
}
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (keyType.IsAnonymousType())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The key type for an aggregate is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n";
}
var t = a.GetType(generatedClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(typeof(TInput).GetAnonymousTypes());
list.AddRange(typeof(TState).GetAnonymousTypes());
list.AddRange(typeof(TOutput).GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new StartEdgeEquiJoinTemplate();
var keyType = template.keyType = typeof(TKey);
var leftType = template.leftType = typeof(TLeft);
var rightType = template.rightType = typeof(TRight);
var resultType = template.resultType = typeof(TResult);
var tm = new TypeMapper(keyType, leftType, rightType, resultType);
template.TKey = tm.CSharpNameFor(keyType);
template.TLeft = tm.CSharpNameFor(leftType);
template.TRight = tm.CSharpNameFor(rightType);
template.TResult = tm.CSharpNameFor(resultType);
var gps = tm.GenericTypeVariables(keyType, leftType, rightType, resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.className = string.Format("GeneratedStartEdgeEquiJoin_{0}", StartEdgeEquiJoinSequenceNumber++);
template.leftMessageRepresentation = new ColumnarRepresentation(leftType);
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightMessageRepresentation = new ColumnarRepresentation(rightType);
template.rightFields = template.rightMessageRepresentation.AllFields;
var outputMessageRepresentation = new ColumnarRepresentation(resultType);
var resultRepresentation = outputMessageRepresentation;
#region Key Comparer
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
template.TKeyTLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.TKeyTRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType);
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.outputFields = resultRepresentation.AllFields;
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
#region LeftBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), "i"), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), "j"), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType, resultType);
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(leftType.GetAnonymousTypes());
list.AddRange(rightType.GetAnonymousTypes());
list.AddRange(resultType.GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}