protected Tuple <Type, string> Generate <TKey, TPayload, TRegister, TAccumulator>()
{
string errorMessages = null;
try
{
var expandedCode = TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(
typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(Stack <>), typeof(IStreamable <,>));
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRegister>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TRegister>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(this.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(this.payloadType.GetAnonymousTypes());
list.AddRange(this.registerType.GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> Generate <TKey, TPayload>(RowToColumnStreamable <TKey, TPayload> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() != null);
Contract.Ensures(typeof(UnaryPipe <TKey, TPayload, TPayload>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
var keyType = typeof(TKey);
var payloadType = typeof(TPayload);
var generatedClassName = string.Format(CultureInfo.InvariantCulture, "RowToColumnUnaryPipeGeneratedFrom_{0}_{1}_{2}", keyType.GetValidIdentifier(), payloadType.GetValidIdentifier(), RowToColumnSequenceNumber++);
var template = new RowToColumnTemplate(generatedClassName, keyType, payloadType);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(keyType, payloadType);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TPayload>());
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(keyType, payloadType);
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out string errorMessages);
var t = a.GetType(generatedClassName);
return(Tuple.Create(t.InstantiateAsNecessary(typeof(TKey), typeof(TPayload)), errorMessages));
}
public static Tuple <Type, string> Generate <TOuterKey, TSource, TInnerKey>(
Expression <Func <TInnerKey, int> > hashComparer,
Expression <Func <TSource, TInnerKey> > keySelector,
bool nested)
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTSource = typeof(TSource);
var typeOfTInnerKey = typeof(TInnerKey);
string expandedCode;
List <Assembly> assemblyReferences;
string errorMessages = null;
try
{
string generatedClassName = $"GeneratedGroupStreamable_{GroupStreamableSequenceNumber++}";
string transformedKeySelectorAsString;
MyFieldInfo swingingField = default;
if (typeOfTInnerKey.IsAnonymousTypeName())
{
Contract.Assume(keySelector.Body is NewExpression);
var transformedFunction = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector);
var newBody = (NewExpression)transformedFunction.Body;
transformedKeySelectorAsString = string.Join(",", newBody.Arguments.Select(arg => arg.ExpressionToCSharp()));
}
else
{
var body = keySelector.Body;
if (!nested && body is MemberExpression singleFieldProjection)
{
if (singleFieldProjection.Expression is ParameterExpression dereferencedObject)
{
// then can just swing a pointer to set the key of the result message
Contract.Assume(dereferencedObject == keySelector.Parameters.ElementAt(0));
var f = singleFieldProjection.Member;
var sourceMessageRepresentation = new ColumnarRepresentation(typeOfTSource);
var fieldToSwingFrom = sourceMessageRepresentation.Fields[f.Name];
swingingField = fieldToSwingFrom;
transformedKeySelectorAsString = swingingField.Name + "_col[i]";
}
else
{
var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body;
if (transformedPredicate == null)
{
return(Tuple.Create((Type)null, errorMessages));
}
transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp();
}
}
else
{
var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body;
if (transformedPredicate == null)
{
return(Tuple.Create((Type)null, errorMessages));
}
transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp();
}
}
var inlinedHashCodeComputation = hashComparer.Inline("key");
var template = new GroupTemplate(
generatedClassName,
typeOfTOuterKey,
typeOfTSource,
typeOfTInnerKey,
transformedKeySelectorAsString,
inlinedHashCodeComputation,
nested)
{
swingingField = swingingField,
payloadMightBeNull = typeOfTSource.CanContainNull()
};
if (!nested && Config.UseMultiString &&
typeOfTInnerKey.Equals(typeof(string)) &&
keySelector.IsSimpleFieldOrPropertyAccess())
{
var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body;
template.transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp();
template.inlinedHashCodeComputation = "hashCodeVector.col[i]";
var fieldName = ((MemberExpression)keySelector.Body).Member.Name;
template.vectorHashCodeInitialization = $"resultBatch.hash = {Transformer.ColumnFieldPrefix}{fieldName}_col.GetHashCode(batch.bitvector);";
template.swingingHashColumn = true;
}
template.fields = new ColumnarRepresentation(typeOfTSource).AllFields;
expandedCode = template.TransformText();
assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TSource>());
if (nested)
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
}
else
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TInnerKey, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TInnerKey, TSource>());
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (typeOfTInnerKey.IsAnonymousTypeName())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The inner key type for Group is anonymous, causing the use of Activator.CreateInstance in an inner loop. This will lead to poor performance.\n";
}
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TKey, TPayload, TResult>(SelectManyStreamable <TKey, TPayload, TResult> stream)
{
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string generatedClassName;
string expandedCode;
string errorMessages = null;
try
{
generatedClassName = $"SelectMany_{sequenceNumber++}";
var keyType = typeof(TKey);
var payloadType = typeof(TPayload);
var resultType = typeof(TResult);
var template = new SelectManyTemplate(generatedClassName, keyType, payloadType, resultType);
var tm = new TypeMapper(keyType, payloadType, resultType);
var gps = tm.GenericTypeVariables(keyType, payloadType, resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.numberOfGenericParameters = gps.Count();
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.MemoryPoolGenericParameters = $"<{template.TKey}, {template.TResult}>";
if (resultType == typeof(int) || resultType == typeof(long) || resultType == typeof(string))
{
template.MemoryPoolGenericParameters = string.Empty;
}
var payloadParameterIndex = 0;
if (stream.HasKey && stream.HasStartEdge)
{
payloadParameterIndex = 2;
}
else if (stream.HasKey || stream.HasStartEdge)
{
payloadParameterIndex = 1;
}
var selector = stream.Selector;
var payloadParameter = selector.Parameters.ElementAt(payloadParameterIndex);
template.PARAMETER = payloadParameter.Name;
template.resultPayloadRepresentation = new ColumnarRepresentation(resultType);
template.resultFields = template.resultPayloadRepresentation.AllFields;
if (template.numberOfGenericParameters > 0)
{
generatedClassName = generatedClassName + "`" + template.numberOfGenericParameters.ToString(CultureInfo.InvariantCulture);
}
expandedCode = string.Empty;
Expression transformedSelector = selector;
// No substitutions are made for the start edge parameter or key parameter. Both just remain in the
// body of the result selector and are set as local variables in the generated code.
var keyParameterIndex = stream.HasStartEdge ? 1 : 0;
var tuple = OptimizeSelectMany(selector.Body);
if (tuple != null)
{
template.enumerableRepeatSelector = true;
var resultSelector = stream.Selector;
var sourceMessageType = StreamMessageManager.GetStreamMessageType <TKey, TPayload>();
var pseudoLambdaParameters = new ParameterExpression[stream.HasKey ? 2 : 1];
var pseudoLambdaIndex = 0;
if (stream.HasKey)
{
pseudoLambdaParameters[pseudoLambdaIndex++] = selector.Parameters[keyParameterIndex];
}
pseudoLambdaParameters[pseudoLambdaIndex] = payloadParameter;
var pseudoLambda = Expression.Lambda(tuple.Item1, pseudoLambdaParameters);
var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >
{
Tuple.Create(payloadParameter, new SelectParameterInformation()
{
BatchName = "batch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = new ColumnarRepresentation(payloadType)
})
};
var projectionResult = SelectTransformer.Transform(pseudoLambda, parameterSubstitutions, template.resultPayloadRepresentation, true, stream.HasStartEdge);
template.computedFields = projectionResult.ComputedFields;
template.useEnumerator = false;
var loopCounter = tuple.Item2;
var newParameters = new ParameterExpression[stream.HasKey ? 2 : 1];
newParameters[0] = payloadParameter;
if (stream.HasKey)
{
newParameters[1] = selector.Parameters[keyParameterIndex];
}
var loopCounterLambda = Expression.Lambda(loopCounter, payloadParameter);
var transformedLoopCounter = Extensions.TransformFunction <TKey, TPayload>(loopCounterLambda, 0);
template.loopCounter = transformedLoopCounter.Body.ExpressionToCSharp();
// REVIEW: Alternative: use Inline to replace occurrences of the key parameter
// with "batch.key.col[i]".
if (stream.HasKey)
{
template.keyParameterName = selector.Parameters[keyParameterIndex].Name;
}
}
else
{
transformedSelector = Extensions.TransformFunction <TKey, TPayload>(stream.Selector, payloadParameterIndex).Body;
if (transformedSelector == null)
{
template.useEnumerator = true;
template.transformedSelectorAsSource = stream.Selector.ExpressionToCSharp();
}
else
{
var tuple2 = OptimizeSelectMany(transformedSelector);
if (tuple2 != null)
{
template.useEnumerator = false;
template.loopCounter = tuple2.Item2.ExpressionToCSharp();
template.transformedSelectorAsSource = tuple2.Item1.ExpressionToCSharp();
}
else
{
template.useEnumerator = true;
template.transformedSelectorAsSource = transformedSelector.ExpressionToCSharp();
}
}
}
template.StartEdgeParameterName = stream.HasStartEdge ? selector.Parameters.ElementAt(0).Name : null;
template.hasKey = stream.HasKey;
expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TResult));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(stream.Selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeof(TKey), typeof(TPayload), typeof(TResult));
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> GenerateAFA <TKey, TPayload, TRegister, TAccumulator>(
AfaStreamable <TKey, TPayload, TRegister, TAccumulator> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var className = string.Format("GeneratedGroupedAfaMultiEvent_{0}", AFASequenceNumber++);
var template = new GroupedAfaMultiEventTemplate(className, typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(TAccumulator));
var payloadRepresentation = new ColumnarRepresentation(typeof(TPayload));
template.isFinal = stream.afa.isFinal;
template.hasOutgoingArcs = stream.afa.hasOutgoingArcs;
template.startStates = stream.afa.startStates;
template.AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances;
var d1 = stream.afa.uncompiledAfa.transitionInfo;
var orderedKeys = d1.Keys.OrderBy(e => e).ToArray();
for (int i = 0; i < orderedKeys.Length; i++)
{
var sourceNodeNumber = orderedKeys[i];
var outgoingEdgesDictionary = d1[sourceNodeNumber];
var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray();
var edgeList = new List <MultiEdgeInfo>();
for (int j = 0; j < orderedTargetNodes.Length; j++)
{
var targetNodeNumber = orderedTargetNodes[j];
var edge = outgoingEdgesDictionary[targetNodeNumber];
if (edge is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc)
{
var multiEdgeInfo = new MultiEdgeInfo()
{
SourceNode = sourceNodeNumber,
TargetNode = targetNodeNumber,
fromStartState = stream.afa.startStates.Any(n => n == sourceNodeNumber),
EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNodeNumber),
Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg),
Accumulate = (ts, ev, reg, acc) =>
{
var transformedAccumulate = Extensions.TransformFunction <TKey, TPayload>(multiArc.Accumulate, 1, batchVariableName: "sourceBatch");
return(transformedAccumulate.Inline(ts, reg, acc));
},
Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg),
};
if (multiArc.Transfer == null)
{
multiEdgeInfo.Transfer = null;
}
else
{
multiEdgeInfo.Transfer = (ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg);
}
if (multiArc.Dispose == null)
{
multiEdgeInfo.Dispose = (acc) => "// no dispose function";
}
else
{
multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc);
}
if (multiArc.SkipToEnd == null)
{
multiEdgeInfo.SkipToEnd = null;
}
else
{
multiEdgeInfo.SkipToEnd = (ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc);
}
edgeList.Add(multiEdgeInfo);
}
}
template.edgeInfos.Add(Tuple.Create(sourceNodeNumber, edgeList));
}
for (int i = 0; i < stream.afa.startStates.Length; i++)
{
var startState = stream.afa.startStates[i];
var edgeList2 = new List <MultiEdgeInfo>();
var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState];
foreach (var edge in outgoingEdgeDictionary)
{
var targetNode = edge.Key;
var arc = edge.Value;
if (arc is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc)
{
var eps = EpsilonClosure(stream.afa, targetNode);
var multiEdgeInfo = new MultiEdgeInfo()
{
SourceNode = startState,
TargetNode = targetNode,
fromStartState = true,
EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNode),
Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg),
Accumulate = (ts, ev, reg, acc) =>
{
var transformedAccumulate = Extensions.TransformFunction <TKey, TPayload>(multiArc.Accumulate, 1, batchVariableName: "sourceBatch");
return(transformedAccumulate.Inline(ts, reg, acc));
},
Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg),
};
if (multiArc.Transfer == null)
{
multiEdgeInfo.Transfer = null;
}
else
{
multiEdgeInfo.Transfer = (ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg);
}
if (multiArc.Dispose == null)
{
multiEdgeInfo.Dispose = (acc) => "// no dispose function";
}
else
{
multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc);
}
if (multiArc.SkipToEnd == null)
{
multiEdgeInfo.SkipToEnd = null;
}
else
{
multiEdgeInfo.SkipToEnd = (ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc);
}
edgeList2.Add(multiEdgeInfo);
}
}
template.startEdgeInfos.Add(Tuple.Create(startState, edgeList2));
}
template.isSyncTimeSimultaneityFree = true; // The handwritten version doesn't make a distinction.
template.keyEqualityComparer =
(left, right) =>
stream.Properties.KeyEqualityComparer.GetEqualsExpr().Inline(left, right);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TRegister));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRegister>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TRegister>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(template.payloadType.GetAnonymousTypes());
list.AddRange(template.registerType.GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Generate a batch class definition to be used as an aggreate definition.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for the aggregate.</typeparam>
/// <typeparam name="TInput">The input type for the aggregate.</typeparam>
/// <typeparam name="TOutput">The output type for the aggregate.</typeparam>
/// <typeparam name="TState">The type for the accumulated state held by the aggregate.</typeparam>
/// <typeparam name="TResult">The type of the result.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of UnaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TInput"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TInput, TState, TOutput, TResult>(
GroupedWindowStreamable <TKey, TInput, TState, TOutput, TResult> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(IStreamObserver <Empty, TInput>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
string expandedCode;
var template = new GroupedWindowTemplate();
var keyType = template.keyType = typeof(TKey);
var inputType = template.inputType = typeof(TInput);
var stateType = template.stateType = typeof(TState);
var outputType = template.outputType = typeof(TOutput);
var resultType = template.resultType = typeof(TResult);
template.TResult = resultType.GetCSharpSourceSyntax(); // BUGBUG: need to get any generic parameters needed
template.isUngrouped = (keyType == typeof(Empty));
template.className = string.Format("GeneratedGroupedAggregate_{0}", GroupedAggregateSequenceNumber++);
var inputMessageRepresentation = new ColumnarRepresentation(inputType);
var resultRepresentation = new ColumnarRepresentation(resultType);
var assemblyReferences = new List <Assembly>();
#region Key Selector
var keySelector = stream.KeySelector;
string transformedKeySelectorAsString;
if (keyType.IsAnonymousTypeName())
{
Contract.Assume(keySelector.Body is NewExpression);
var transformedFunction = Extensions.TransformUnaryFunction <TKey, TInput>(keySelector);
var newBody = (NewExpression)transformedFunction.Body;
transformedKeySelectorAsString = string.Join(",", newBody.Arguments);
}
else
{
var transformedFunction = Extensions.TransformUnaryFunction <Empty, TInput>(keySelector).Body;
if (transformedFunction == null)
{
return(null);
}
transformedKeySelectorAsString = transformedFunction.ExpressionToCSharp();
}
template.keySelector = transformedKeySelectorAsString;
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector));
#endregion
#region Key Comparer and HashCode
var keyComparer = EqualityComparerExpression <TKey> .Default;
template.keyComparerEquals =
(left, right) =>
keyComparer.GetEqualsExpr().Inline(left, right);
template.keyComparerGetHashCode =
(x) =>
keyComparer.GetGetHashCodeExpr().Inline(x);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyComparer.GetEqualsExpr()));
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyComparer.GetGetHashCodeExpr()));
#endregion
#region Aggregate functions
var initialStateLambda = stream.Aggregate.InitialState();
if (ConstantExpressionFinder.IsClosedExpression(initialStateLambda))
{
template.initialState = initialStateLambda.Body.ExpressionToCSharp();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(initialStateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the initial state lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledInitialState = true;
template.initialState = "initialState()";
}
}
var accumulateLambda = stream.Aggregate.Accumulate();
if (ConstantExpressionFinder.IsClosedExpression(accumulateLambda))
{
var accTransformedLambda = Extensions.TransformFunction <TKey, TInput>(accumulateLambda, 2);
template.accumulate = (stateArg, longArg) => accTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(accumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the accumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledAccumulate = true;
template.accumulate = (s1, s2) => string.Format("accumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var deaccumulateLambda = stream.Aggregate.Deaccumulate();
if (ConstantExpressionFinder.IsClosedExpression(deaccumulateLambda))
{
var deaccumulateTransformedLambda = Extensions.TransformFunction <TKey, TInput>(deaccumulateLambda, 2);
template.deaccumulate = (stateArg, longArg) => deaccumulateTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(deaccumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
throw new InvalidOperationException("Code Generation couldn't inline a lambda!");
}
else
{
template.useCompiledDeaccumulate = true;
template.deaccumulate = (s1, s2) => string.Format("deaccumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var differenceLambda = stream.Aggregate.Difference();
if (ConstantExpressionFinder.IsClosedExpression(differenceLambda))
{
template.difference = (stateArg1, stateArg2) => differenceLambda.Inline(stateArg1, stateArg2);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(differenceLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the deaccumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledDifference = true;
template.deaccumulate = (s1, s2) => string.Format("difference({0}, {1});", s1, s2);
}
}
var computeResultLambda = stream.Aggregate.ComputeResult();
if (ConstantExpressionFinder.IsClosedExpression(computeResultLambda))
{
if (outputType.IsAnonymousType())
{
if (computeResultLambda.Body is NewExpression newExpression)
{
errorMessages = "Code Generation for GroupedWindow: result selector must be a new expression for anonymous types";
throw new NotImplementedException(errorMessages);
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(computeResultLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the result selector lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledComputeResult = true;
template.computeResult = (stateArg) => "computeResult(" + stateArg + ")";
}
}
#endregion
template.BatchGeneratedFrom_Unit_TInput = Transformer.GetBatchClassName(typeof(Empty), inputType);
template.UnitTInputGenericParameters = string.Empty; // BUGBUG
template.UnitTResultGenericParameters = string.Empty; // BUGBUG
template.inputFields = inputMessageRepresentation.AllFields;
template.outputFields = resultRepresentation.AllFields;
var resultSelector = stream.ResultSelector;
var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); // dont want the parameters substituted for at all
var projectionResult = SelectTransformer.Transform(resultSelector, parameterSubstitutions, resultRepresentation, true);
if (projectionResult.Error)
{
return(null);
}
template.finalResultSelector =
(key, aggregateResult) =>
{
var parameters = new Dictionary <ParameterExpression, string>
{
{ resultSelector.Parameters.ElementAt(0), key }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
sb.AppendLine(string.Format("var {0} = {1};\n", resultSelector.Parameters.ElementAt(1).Name, aggregateResult));
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
sb.AppendFormat("this.batch.{0}.col[_c] = {1};\n", f.Name, e.ExpressionToCSharpStringWithParameterSubstitution(parameters));
}
sb.AppendLine("}");
return(sb.ToString());
};
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(resultSelector));
template.staticCtor = Transformer.StaticCtor(template.className);
expandedCode = template.TransformText();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(typeof(Empty), typeof(TKey), typeof(TInput), typeof(TState), typeof(TOutput), typeof(FastDictionaryGenerator3)));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TInput>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <Empty, TResult>());
var assembly = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = assembly.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(typeof(TInput).GetAnonymousTypes());
list.AddRange(typeof(TState).GetAnonymousTypes());
list.AddRange(typeof(TOutput).GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TKey, TPayload, TResult>(SelectStreamable <TKey, TPayload, TResult> stream)
{
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string generatedClassName;
string errorMessages = null;
try
{
var keyType = typeof(TKey);
var payloadType = typeof(TPayload);
var resultType = typeof(TResult);
generatedClassName = string.Format("Select_{0}_{1}_{2}_{3}", keyType.GetValidIdentifier(), payloadType.GetValidIdentifier(), resultType.GetValidIdentifier(), sequenceNumber++);
var template = new SelectTemplate(generatedClassName, keyType, payloadType, resultType);
var tm = new TypeMapper(keyType, payloadType, resultType);
var gps = tm.GenericTypeVariables(keyType, payloadType, resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.numberOfGenericParameters = gps.Count();
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.MemoryPoolGenericParameters = $"<{template.TKey}, {template.TResult}>";
if (resultType == typeof(int) || resultType == typeof(long) || resultType == typeof(string))
{
template.MemoryPoolGenericParameters = string.Empty;
}
var payloadParameterIndex = 0;
if (stream.HasKey && stream.HasStartEdge)
{
payloadParameterIndex = 2;
}
else if (stream.HasKey || stream.HasStartEdge)
{
payloadParameterIndex = 1;
}
template.PARAMETER = stream.Selector.Parameters[payloadParameterIndex].Name;
template.resultPayloadRepresentation = new ColumnarRepresentation(resultType);
template.destinationFields = template.resultPayloadRepresentation.AllFields;
if (template.numberOfGenericParameters > 0)
{
generatedClassName = generatedClassName + "`" + template.numberOfGenericParameters.ToString(CultureInfo.InvariantCulture);
}
var resultSelector = stream.Selector;
var sourceMessageType = StreamMessageManager.GetStreamMessageType <TKey, TPayload>();
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >();
// Don't create a parameter substitution for the start edge parameter. That will just remain in the
// body of the result selector and will be set as a local variable in the generated code.
if (stream.HasKey)
{
var keyRepresentation = new ColumnarRepresentation(keyType, "key");
var keyParameterIndex = stream.HasStartEdge ? 1 : 0;
parameterSubsitutions.Add(Tuple.Create(
resultSelector.Parameters.ElementAt(keyParameterIndex),
new SelectParameterInformation()
{
BatchName = "sourceBatch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = keyRepresentation,
}));
}
parameterSubsitutions.Add(Tuple.Create(
resultSelector.Parameters.ElementAt(payloadParameterIndex),
new SelectParameterInformation()
{
BatchName = "sourceBatch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = new ColumnarRepresentation(payloadType)
}));
var projectionResult = SelectTransformer.Transform(resultSelector, parameterSubsitutions, template.resultPayloadRepresentation, false, stream.HasStartEdge);
if (projectionResult.Error)
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
throw new InvalidOperationException("Code Generation couldn't transform a selector!");
}
return(Tuple.Create((Type)null, errorMessages));
}
template.StartEdgeParameterName = stream.HasStartEdge ? resultSelector.Parameters.ElementAt(0).Name : null;
template.computedFields = projectionResult.ComputedFields;
template.swingingFields = projectionResult.SwingingFields;
template.unassignedFields = projectionResult.UnmentionedFields;
template.ProjectionReturningResultInstance = projectionResult.ProjectionReturningResultInstance;
template.multiStringOperations = projectionResult.MultiStringOperations;
template.needSourceInstance = projectionResult.NeedsSourceInstance;
var d = new Dictionary <MyFieldInfo, int>();
foreach (var f in template.swingingFields)
{
var target = f.Item1;
var source = f.Item2;
if (!d.ContainsKey(source))
{
d.Add(source, 0);
}
d[source] = d[source] + 1;
}
template.swungFieldsCount = d;
template.nonSwingingFields = template.fields.Where(sf => !template.swingingFields.Any(swingingField => swingingField.Item2.Equals(sf)));
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(
typeof(TKey), typeof(TPayload), typeof(TResult), typeof(IStreamable <,>));
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(stream.Selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeof(TKey), typeof(TPayload), typeof(TResult));
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> GenerateAFA <TKey, TPayload, TRegister, TAccumulator>(
AfaStreamable <TKey, TPayload, TRegister, TAccumulator> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var className = string.Format("GeneratedUngroupedDAfa_{0}", AFASequenceNumber++);
var template = new UngroupedDAfaTemplate(className, typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(TAccumulator))
{
TKey = typeof(TKey).GetCSharpSourceSyntax()
};
template.isFinal = stream.afa.isFinal;
template.hasOutgoingArcs = stream.afa.hasOutgoingArcs;
template.startStates = stream.afa.startStates;
template.AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances;
var d1 = stream.afa.uncompiledAfa.transitionInfo;
var orderedKeys = d1.Keys.OrderBy(e => e).ToArray();
for (int i = 0; i < orderedKeys.Length; i++)
{
var sourceNodeNumber = orderedKeys[i];
var outgoingEdgesDictionary = d1[sourceNodeNumber];
var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray();
var edgeList1 = new List <EdgeInfo>();
for (int j = 0; j < orderedTargetNodes.Length; j++)
{
var targetNodeNumber = orderedTargetNodes[j];
var edge = outgoingEdgesDictionary[targetNodeNumber];
if (edge is SingleElementArc <TPayload, TRegister> searc)
{
var edgeInfo = CreateSingleEdgeInfo(stream, targetNodeNumber, searc, "i");
edgeList1.Add(edgeInfo);
}
}
template.currentlyActiveInfo.Add(Tuple.Create(sourceNodeNumber, edgeList1));
}
for (int i = 0; i < stream.afa.startStates.Length; i++)
{
var startState = stream.afa.startStates[i];
var edgeList2 = new List <EdgeInfo>();
var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState];
foreach (var edge in outgoingEdgeDictionary)
{
var targetNode = edge.Key;
var arc = edge.Value;
if (arc is SingleElementArc <TPayload, TRegister> searc)
{
var edgeInfo = CreateSingleEdgeInfo(stream, targetNode, searc, "i");
edgeList2.Add(edgeInfo);
}
}
template.newActivationInfo.Add(Tuple.Create(startState, edgeList2));
}
template.isSyncTimeSimultaneityFree = stream.Properties.IsSyncTimeSimultaneityFree;
template.keyEqualityComparer =
(left, right) =>
stream.Properties.KeyEqualityComparer.GetEqualsExpr().Inline(left, right);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(Stack <>));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRegister>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TRegister>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(template.payloadType.GetAnonymousTypes());
list.AddRange(template.registerType.GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TOuterKey, TSource, TInnerKey>(
Expression <Func <TSource, TInnerKey> > keySelector,
string inlinedHashCodeComputation,
bool nested,
bool powerOf2)
{
string errorMessages = null;
try
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTSource = typeof(TSource);
var typeOfTInnerKey = typeof(TInnerKey);
var generatedClassName = string.Format(CultureInfo.InvariantCulture, "ShuffleStreamablePipeGeneratedFrom_{0}_{1}_{2}_{3}", typeOfTOuterKey.GetValidIdentifier(), typeOfTSource.GetValidIdentifier(), typeOfTInnerKey.GetValidIdentifier(), shuffleCounter++);
var inputMessageRepresentation = new ColumnarRepresentation(typeOfTSource);
var template = new ShuffleTemplate(
generatedClassName,
typeOfTOuterKey,
typeOfTSource,
typeOfTInnerKey,
inlinedHashCodeComputation,
nested, powerOf2)
{
fields = inputMessageRepresentation.AllFields
};
var innerKeyIsAnonymous = typeOfTInnerKey.IsAnonymousTypeName();
if (keySelector != null)
{
var transformedKeySelector = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector);
var keySelectorAsNewExpression = keySelector.Body as NewExpression;
if (innerKeyIsAnonymous && keySelectorAsNewExpression != null)
{
var newPrime = (NewExpression)transformedKeySelector.Body;
template.transformedKeySelectorAsString = string.Format(CultureInfo.InvariantCulture, "({0})Activator.CreateInstance(typeof({0}), {1})",
template.TInnerKey,
string.Join(",", newPrime.Arguments.Select(m => m.ExpressionToCSharp())));
}
else
{
template.transformedKeySelectorAsString = transformedKeySelector.Body.ExpressionToCSharp();
if (Config.UseMultiString &&
typeOfTInnerKey.Equals(typeof(string)) &&
keySelector.IsSimpleFieldOrPropertyAccess())
{
template.inlinedHashCodeComputation = "hashCodeVector.col[i]";
var fieldName = ((MemberExpression)(keySelector.Body)).Member.Name;
template.vectorHashCodeInitialization = string.Format(CultureInfo.InvariantCulture, "var hashCodeVector = {0}{1}_col.GetHashCode(batch.bitvector);", Transformer.ColumnFieldPrefix, fieldName);
}
}
}
else
{
template.transformedKeySelectorAsString = string.Empty;
}
template.innerKeyIsAnonymous = innerKeyIsAnonymous;
template.staticCtor = Transformer.StaticCtor(template.CLASSNAME);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TSource>());
if (nested)
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
}
else
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TInnerKey, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TInnerKey, TSource>());
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (typeOfTInnerKey.IsAnonymousTypeName())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The inner key type for Shuffle is anonymous, causing the use of Activator.CreateInstance in an inner loop. This will lead to poor performance.\n";
}
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
var t = a.GetType(generatedClassName);
return(Tuple.Create(t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey), errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
private static Tuple <Type, string> GenerateInternal <TOuterKey, TInnerKey, TInnerResult, TResult>(Expression <Func <TInnerKey, TInnerResult, TResult> > resultSelector, bool isFirstLevelGroup)
{
Contract.Ensures(Contract.Result <Tuple <Type, string> >() != null);
Contract.Ensures(typeof(Pipe <TOuterKey, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTInnerKey = typeof(TInnerKey);
var typeofTInnerResult = typeof(TInnerResult);
var typeofTResult = typeof(TResult);
string expandedCode;
List <Assembly> assemblyReferences;
int numberOfGenericParameters;
// inline: var ok = UngroupTemplate.GetGeneratedCode<TOuterKey, TInnerKey, TInnerResult, TResult>(resultSelector, false, out generatedClassName, out expandedCode, out assemblyReferences, out numberOfGenericParameters);
var generatedClassName = string.Format(
"UngroupPipeGeneratedFrom_{0}_{1}_{2}_{3}_{4}",
typeOfTOuterKey.GetValidIdentifier(),
typeOfTInnerKey.GetValidIdentifier(),
typeofTInnerResult.GetValidIdentifier(),
typeofTResult.GetValidIdentifier(),
UngroupSequenceNumber++);
var inputMessageType = StreamMessageManager.GetStreamMessageType <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult>();
var innerResultRepresentation = new ColumnarRepresentation(typeofTInnerResult);
var resultRepresentation = new ColumnarRepresentation(typeofTResult);
var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >
{
// Leave the key parameter to the selector unchanged, so no substitution for parameters[0]
Tuple.Create(resultSelector.Parameters[1], new SelectParameterInformation()
{
BatchName = "inputBatch", BatchType = inputMessageType, IndexVariableName = "i", parameterRepresentation = innerResultRepresentation,
})
};
var result = SelectTransformer.Transform(resultSelector, parameterSubstitutions, resultRepresentation);
if (result.Error)
{
return(Tuple.Create((Type)null, errorMessages));
}
var template = new UngroupTemplate(generatedClassName, !isFirstLevelGroup, typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult)
{
innerResultRepresentation = innerResultRepresentation,
swingingFields = result.SwingingFields,
computedFields = result.ComputedFields,
unassignedFields = result.UnmentionedFields,
keyParameter = resultSelector.Parameters.First()
};
template.staticCtor = Transformer.StaticCtor(template.CLASSNAME);
expandedCode = template.TransformText();
assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
// input messages
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult>());
// output messages
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TResult>());
// memory pool
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TOuterKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(resultSelector));
numberOfGenericParameters = template.numberOfGenericParameters;
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (numberOfGenericParameters > 0)
{
generatedClassName = generatedClassName + "`" + numberOfGenericParameters.ToString(CultureInfo.InvariantCulture);
}
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult);
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TKey, TInput, TState, TOutput>(SnapshotWindowStreamable <TKey, TInput, TState, TOutput> stream, AggregatePipeType pipeType)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(IStreamObserver <TKey, TInput>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
var container = stream.Properties.QueryContainer;
string generatedClassName;
string expandedCode;
List <Assembly> assemblyReferences;
string errorMessages = null;
try
{
AggregateTemplate template;
switch (pipeType)
{
case AggregatePipeType.StartEdge:
template = new SnapshotWindowStartEdgeTemplate();
break;
case AggregatePipeType.PriorityQueue:
template = new SnapshotWindowPriorityQueueTemplate();
break;
case AggregatePipeType.Tumbling:
template = new SnapshotWindowTumblingTemplate();
break;
case AggregatePipeType.Sliding:
template = new SnapshotWindowSlidingTemplate();
break;
case AggregatePipeType.Hopping:
template = new SnapshotWindowHoppingTemplate
{
hopsPerDuration = ((int)(stream.Source.Properties.ConstantDurationLength.Value / stream.Source.Properties.ConstantHopLength) + 1).ToString()
};
break;
default:
Contract.Assert(false, "case meant to be exhaustive");
throw new InvalidOperationException("case meant to be exhaustive");
}
template.isUngrouped = (typeof(TKey) == typeof(Empty));
var keyType = template.keyType = typeof(TKey);
var inputType = template.inputType = typeof(TInput);
var stateType = template.stateType = typeof(TState);
var outputType = template.outputType = typeof(TOutput);
template.inputFields = new ColumnarRepresentation(inputType).AllFields;
template.outputFields = new ColumnarRepresentation(outputType).AllFields;
assemblyReferences = new List <Assembly>();
#region Key Comparer
IEqualityComparerExpression <TKey> keyComparer;
keyComparer = stream.Properties.KeyEqualityComparer;
var equalsExpression = keyComparer.GetEqualsExpr();
var getHashcodeExpression = keyComparer.GetGetHashCodeExpr();
template.inlinedKeyComparerEquals =
(left, right) =>
string.Format("({0})", equalsExpression.Inline(left, right));
template.inlinedKeyComparerGetHashCode =
(x) =>
string.Format("({0}/* inlined GetHashCode */)", getHashcodeExpression.Inline(x));
if (keyType.IsAnonymousType())
{
template.inlinedKeyComparerEquals =
(left, right) => string.Format("keyComparerEquals({0}, {1})", left, right);
template.inlinedKeyComparerGetHashCode =
(x) => string.Format("keyComparerGetHashCode({0})", x);
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(equalsExpression));
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(getHashcodeExpression));
#endregion
#region Aggregate Functions
var initialStateLambda = stream.Aggregate.InitialState();
if (ConstantExpressionFinder.IsClosedExpression(initialStateLambda))
{
template.initialState = initialStateLambda.Body.ExpressionToCSharp();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(initialStateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the initial state lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledInitialState = true;
template.initialState = "initialState()";
}
}
var accumulateLambda = stream.Aggregate.Accumulate();
if (ConstantExpressionFinder.IsClosedExpression(accumulateLambda))
{
var accTransformedLambda = Extensions.TransformFunction <TKey, TInput>(accumulateLambda, 2);
template.accumulate = (stateArg, longArg) => accTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(accumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the accumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledAccumulate = true;
template.accumulate = (s1, s2) => string.Format("accumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var deaccumulateLambda = stream.Aggregate.Deaccumulate();
if (ConstantExpressionFinder.IsClosedExpression(deaccumulateLambda))
{
var deaccumulateTransformedLambda = Extensions.TransformFunction <TKey, TInput>(deaccumulateLambda, 2);
template.deaccumulate = (stateArg, longArg) => deaccumulateTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(deaccumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the deaccumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledDeaccumulate = true;
template.deaccumulate = (s1, s2) => string.Format("deaccumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var differenceLambda = stream.Aggregate.Difference();
if (ConstantExpressionFinder.IsClosedExpression(differenceLambda))
{
template.difference = (stateArg1, stateArg2) => differenceLambda.Inline(stateArg1, stateArg2);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(differenceLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the difference lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledDifference = true;
template.difference = (s1, s2) => string.Format("difference({0}, {1});", s1, s2);
}
}
var computeResultLambda = stream.Aggregate.ComputeResult();
if (ConstantExpressionFinder.IsClosedExpression(computeResultLambda))
{
if (outputType.IsAnonymousType())
{
if (computeResultLambda.Body is NewExpression newExpression)
{
var outputBatchType = StreamMessageManager.GetStreamMessageType <TKey, TOutput>();
var foo = Transform(newExpression, outputBatchType);
template.computeResult = (stateArg) => Expression.Lambda(foo, computeResultLambda.Parameters.ToArray()).Inline(stateArg);
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(computeResultLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar || outputType.IsAnonymousType())
{
// second disjunct is because if we aren't inlining the computeResult function and
// the output type is anonymous, calling the compiled computeResult function returns
// a value of the anonymous type and since the generated operator represents the anonymous
// type as a generic parameter, it can't use the "field" (i.e., property) names to
// get the individual pieces to assign to each column of the output message.
errorMessages = "Code Generation for Aggregate: couldn't inline the compute result lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledComputeResult = true;
template.computeResult = (stateArg) => "computeResult(" + stateArg + ");";
}
}
#endregion
generatedClassName = template.className = string.Format("Aggregate_{0}", sequenceNumber++);
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(keyType, inputType, stateType, outputType);
template.staticCtor = Transformer.StaticCtor(template.className);
expandedCode = template.TransformText();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TInput), typeof(TState), typeof(TOutput), typeof(FastDictionaryGenerator), typeof(SortedDictionary <,>)));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TInput>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TOutput>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TOutput>());
if (container != null)
{
assemblyReferences.AddRange(container.CollectedGeneratedTypes.Select(o => o.GetTypeInfo().Assembly));
}
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (keyType.IsAnonymousType())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The key type for an aggregate is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n";
}
var t = a.GetType(generatedClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(typeof(TInput).GetAnonymousTypes());
list.AddRange(typeof(TState).GetAnonymousTypes());
list.AddRange(typeof(TOutput).GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Generate a batch class definition to be used as StartEdgeEquiJoin operator.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TLeft">The payload type for the left side.</typeparam>
/// <typeparam name="TRight">The payload type for the right side.</typeparam>
/// <typeparam name="TResult">The payload type for the resulting stream.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TRight"/>, <typeparamref name="TKey"/>, <typeparamref name="TResult"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new IncreasingOrderEquiJoinTemplate();
var keyType = template.keyType = typeof(TKey);
var leftType = template.leftType = typeof(TLeft);
var rightType = template.rightType = typeof(TRight);
var resultType = template.resultType = typeof(TResult);
template.TKey = keyType.GetCSharpSourceSyntax();
template.TLeft = leftType.GetCSharpSourceSyntax();
template.TRight = rightType.GetCSharpSourceSyntax();
template.TResult = resultType.GetCSharpSourceSyntax(); // BUGBUG: need to get any generic parameters needed
template.className = string.Format("GeneratedIncreasingOrderEquiJoin_{0}", IOOEJSequenceNumber++);
template.leftMessageRepresentation = new ColumnarRepresentation(leftType);
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightMessageRepresentation = new ColumnarRepresentation(rightType);
template.rightFields = template.rightMessageRepresentation.AllFields;
var outputMessageRepresentation = new ColumnarRepresentation(resultType);
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
var resultRepresentation = outputMessageRepresentation;
#region Key Comparer
var keyComparer = stream.Left.Properties.KeyComparer.GetCompareExpr();
if (!ConstantExpressionFinder.IsClosedExpression(keyComparer))
{
return(null);
}
template.joinKeyOrderComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
template.TKeyTLeftGenericParameters = string.Empty; // BUGBUG
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.TKeyTRightGenericParameters = string.Empty; // BUGBUG
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType);
template.TKeyTResultGenericParameters = string.Empty; // BUGBUG
template.outputFields = resultRepresentation.AllFields;
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
return(null);
}
#region LeftBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), "i"), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), "j"), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
template.getOutputBatch = string.Format(
"pool.Get(out genericOutputBatch); output = ({0}{1})genericOutputBatch;",
Transformer.GetBatchClassName(keyType, resultType),
template.TKeyTResultGenericParameters);
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(leftType.GetAnonymousTypes());
list.AddRange(rightType.GetAnonymousTypes());
list.AddRange(resultType.GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
