internal static Tuple <Type, string> GenerateAFA <TPayload, TRegister, TAccumulator>(
AfaStreamable <Empty, TPayload, TRegister, TAccumulator> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <Empty, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var className = string.Format("GeneratedUngroupedAfa_{0}", AFASequenceNumber++);
var template = new UngroupedAfaTemplate(className, typeof(Empty), typeof(TPayload), typeof(TRegister), typeof(TAccumulator));
template.isFinal = stream.afa.isFinal;
template.hasOutgoingArcs = stream.afa.hasOutgoingArcs;
template.startStates = stream.afa.startStates;
template.AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances;
var d1 = stream.afa.uncompiledAfa.transitionInfo;
var orderedKeys = d1.Keys.OrderBy(e => e).ToArray();
for (int i = 0; i < orderedKeys.Length; i++)
{
var sourceNodeNumber = orderedKeys[i];
var outgoingEdgesDictionary = d1[sourceNodeNumber];
var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray();
var edgeList1 = new List <EdgeInfo>();
for (int j = 0; j < orderedTargetNodes.Length; j++)
{
var targetNodeNumber = orderedTargetNodes[j];
var edge = outgoingEdgesDictionary[targetNodeNumber];
if (edge is SingleElementArc <TPayload, TRegister> searc)
{
var edgeInfo = CreateSingleEdgeInfo(stream, targetNodeNumber, searc, "i");
edgeList1.Add(edgeInfo);
}
}
template.currentlyActiveInfo.Add(Tuple.Create(sourceNodeNumber, edgeList1));
}
for (int i = 0; i < stream.afa.startStates.Length; i++)
{
var startState = stream.afa.startStates[i];
var edgeList2 = new List <EdgeInfo>();
var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState];
foreach (var edge in outgoingEdgeDictionary)
{
var targetNode = edge.Key;
var arc = edge.Value;
if (arc is SingleElementArc <TPayload, TRegister> searc)
{
var edgeInfo = CreateSingleEdgeInfo(stream, targetNode, searc, "i");
edgeList2.Add(edgeInfo);
}
}
template.newActivationInfo.Add(Tuple.Create(startState, edgeList2));
}
template.isSyncTimeSimultaneityFree = stream.Properties.IsSyncTimeSimultaneityFree;
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TPayload), typeof(TRegister), typeof(Stack <>));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TRegister>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <Empty, TRegister>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = new List <Type>();
list.AddRange(template.payloadType.GetAnonymousTypes());
list.AddRange(template.registerType.GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TOuterKey, TSource, TInnerKey>(
Expression <Func <TSource, TInnerKey> > keySelector,
string inlinedHashCodeComputation,
bool nested,
bool powerOf2)
{
string errorMessages = null;
try
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTSource = typeof(TSource);
var typeOfTInnerKey = typeof(TInnerKey);
var generatedClassName = string.Format(CultureInfo.InvariantCulture, "ShuffleStreamablePipeGeneratedFrom_{0}_{1}_{2}_{3}", typeOfTOuterKey.GetValidIdentifier(), typeOfTSource.GetValidIdentifier(), typeOfTInnerKey.GetValidIdentifier(), shuffleCounter++);
var inputMessageRepresentation = new ColumnarRepresentation(typeOfTSource);
var template = new ShuffleTemplate(
generatedClassName,
typeOfTOuterKey,
typeOfTSource,
typeOfTInnerKey,
inlinedHashCodeComputation,
nested, powerOf2)
{
fields = inputMessageRepresentation.AllFields
};
var innerKeyIsAnonymous = typeOfTInnerKey.IsAnonymousTypeName();
if (keySelector != null)
{
var transformedKeySelector = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector);
var keySelectorAsNewExpression = keySelector.Body as NewExpression;
if (innerKeyIsAnonymous && keySelectorAsNewExpression != null)
{
var newPrime = (NewExpression)transformedKeySelector.Body;
template.transformedKeySelectorAsString = string.Format(CultureInfo.InvariantCulture, "({0})Activator.CreateInstance(typeof({0}), {1})",
template.TInnerKey,
string.Join(",", newPrime.Arguments.Select(m => m.ExpressionToCSharp())));
}
else
{
template.transformedKeySelectorAsString = transformedKeySelector.Body.ExpressionToCSharp();
if (Config.UseMultiString &&
typeOfTInnerKey.Equals(typeof(string)) &&
keySelector.IsSimpleFieldOrPropertyAccess())
{
template.inlinedHashCodeComputation = "hashCodeVector.col[i]";
var fieldName = ((MemberExpression)keySelector.Body).Member.Name;
template.vectorHashCodeInitialization = $"var hashCodeVector = {Transformer.ColumnFieldPrefix}{fieldName}_col.GetHashCode(batch.bitvector);";
}
}
}
else
{
template.transformedKeySelectorAsString = string.Empty;
}
template.innerKeyIsAnonymous = innerKeyIsAnonymous;
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TSource>());
if (nested)
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
}
else
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TInnerKey, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TInnerKey, TSource>());
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (typeOfTInnerKey.IsAnonymousTypeName())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The inner key type for Shuffle is anonymous, causing the use of Activator.CreateInstance in an inner loop. This will lead to poor performance.\n";
}
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
var t = a.GetType(generatedClassName);
return(Tuple.Create(t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey), errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> GenerateAFA <TKey, TPayload, TRegister, TAccumulator>(
AfaStreamable <TKey, TPayload, TRegister, TAccumulator> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var className = string.Format("Generated{1}AfaMultiEventList_{0}", AFASequenceNumber++, typeof(TKey).Equals(typeof(Empty)) ? string.Empty : "Grouped");
var template = new AfaMultiEventListTemplate(className, typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(TAccumulator))
{
TKey = typeof(TKey).GetCSharpSourceSyntax()
};
var payloadRepresentation =
Config.ForceRowBasedExecution
?
new ColumnarRepresentation(typeof(TPayload))
:
new ColumnarRepresentation(typeof(TPayload))
;
template.isFinal = stream.afa.isFinal;
template.hasOutgoingArcs = stream.afa.hasOutgoingArcs;
template.startStates = stream.afa.startStates;
template.AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances;
template.payloadIsAnon = typeof(TPayload).IsAnonymousTypeName();
template.payloadHasNoFields = payloadRepresentation.noFields;
var d1 = stream.afa.uncompiledAfa.transitionInfo;
var orderedKeys = d1.Keys.OrderBy(e => e).ToArray();
for (int i = 0; i < orderedKeys.Length; i++)
{
var sourceNodeNumber = orderedKeys[i];
var outgoingEdgesDictionary = d1[sourceNodeNumber];
var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray();
var edgeList = new List <EdgeInfo>();
for (int j = 0; j < orderedTargetNodes.Length; j++)
{
var targetNodeNumber = orderedTargetNodes[j];
var edge = outgoingEdgesDictionary[targetNodeNumber];
if (edge is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc)
{
var multiEdgeInfo = new MultiEdgeInfo()
{
SourceNode = sourceNodeNumber,
TargetNode = targetNodeNumber,
fromStartState = stream.afa.startStates.Any(n => n == sourceNodeNumber),
EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNodeNumber),
Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg),
Accumulate = (ts, ev, reg, acc) => multiArc.Accumulate.Inline(ts, ev, reg, acc),
Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg),
};
if (multiArc.Transfer == null)
{
multiEdgeInfo.Transfer = null;
}
else
{
multiEdgeInfo.Transfer = (ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg);
}
if (multiArc.Dispose == null)
{
multiEdgeInfo.Dispose = (acc) => "// no dispose function";
}
else
{
multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc);
}
if (multiArc.SkipToEnd == null)
{
multiEdgeInfo.SkipToEnd = null;
}
else
{
multiEdgeInfo.SkipToEnd = (ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc);
}
edgeList.Add(multiEdgeInfo);
continue;
}
if (edge is SingleElementArc <TPayload, TRegister> singleArc)
{
var edgeInfo = CreateSingleEdgeInfo(stream, targetNodeNumber, singleArc, "payloadoffset");
edgeList.Add(edgeInfo);
continue;
}
if (edge is ListElementArc <TPayload, TRegister> listArc)
{
var edgeInfo = CreateListEdgeInfo(stream, payloadRepresentation, targetNodeNumber, listArc, "payloadoffset");
edgeList.Add(edgeInfo);
continue;
}
}
template.edgeInfos.Add(Tuple.Create(sourceNodeNumber, edgeList));
}
for (int i = 0; i < stream.afa.startStates.Length; i++)
{
var startState = stream.afa.startStates[i];
var edgeList2 = new List <EdgeInfo>();
var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState];
foreach (var edge in outgoingEdgeDictionary)
{
var targetNode = edge.Key;
var arc = edge.Value;
if (arc is MultiElementArc <TPayload, TRegister, TAccumulator> multiArc)
{
var eps = EpsilonClosure(stream.afa, targetNode);
var multiEdgeInfo = new MultiEdgeInfo()
{
SourceNode = startState,
TargetNode = targetNode,
fromStartState = true,
EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNode),
Initialize = (ts, reg) => multiArc.Initialize.Inline(ts, reg),
Accumulate = (ts, ev, reg, acc) => multiArc.Accumulate.Inline(ts, ev, reg, acc),
Fence = (ts, acc, reg) => multiArc.Fence.Inline(ts, acc, reg),
};
if (multiArc.Transfer == null)
{
multiEdgeInfo.Transfer = null;
}
else
{
multiEdgeInfo.Transfer = (ts, acc, reg) => multiArc.Transfer.Inline(ts, acc, reg);
}
if (multiArc.Dispose == null)
{
multiEdgeInfo.Dispose = (acc) => "// no dispose function";
}
else
{
multiEdgeInfo.Dispose = (acc) => multiArc.Dispose.Inline(acc);
}
if (multiArc.SkipToEnd == null)
{
multiEdgeInfo.SkipToEnd = null;
}
else
{
multiEdgeInfo.SkipToEnd = (ts, ev, acc) => multiArc.SkipToEnd.Inline(ts, ev, acc);
}
edgeList2.Add(multiEdgeInfo);
continue;
}
if (arc is SingleElementArc <TPayload, TRegister> singleArc)
{
var edgeInfo = CreateSingleEdgeInfo(stream, targetNode, singleArc, "payloadoffset");
edgeList2.Add(edgeInfo);
continue;
}
if (arc is ListElementArc <TPayload, TRegister> listArc)
{
var edgeInfo = CreateListEdgeInfo(stream, payloadRepresentation, targetNode, listArc, "payloadoffset");
edgeList2.Add(edgeInfo);
continue;
}
}
template.startEdgeInfos.Add(Tuple.Create(startState, edgeList2));
}
template.isSyncTimeSimultaneityFree = true; // The handwritten version doesn't make a distinction.
template.keyEqualityComparer =
(left, right) =>
stream.Properties.KeyEqualityComparer.GetEqualsExpr().Inline(left, right);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(Queue <>));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRegister>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TRegister>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(template.payloadType.GetAnonymousTypes());
list.AddRange(template.registerType.GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> Generate <TKey, TPayload>(WhereStreamable <TKey, TPayload> stream)
{
string generatedClassName;
string expandedCode;
var assemblyReferences = new List <Assembly>();
string errorMessages = null;
try
{
var keyType = typeof(TKey);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyType));
var sourceType = typeof(TPayload);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(sourceType));
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(stream.Predicate));
generatedClassName = string.Format("WhereUnaryPipeGeneratedFrom_{0}_{1}_{2}", keyType.GetValidIdentifier(), sourceType.GetValidIdentifier(), sequenceNumber++);
var noTransformation = false;
var p = stream.Predicate.Body;
var transformedPredicate = p;
var multiStringInit = string.Empty;
var multiStringReturns = string.Empty;
var multiStringWrapperInit = string.Empty;
string vectorOperations = null;
if (Config.UseMultiString && Config.MultiStringTransforms != Config.CodegenOptions.MultiStringFlags.None)
{
var result = MultiStringTransformer.Transform(sourceType, transformedPredicate);
if (result.transformedExpression != null)
{
transformedPredicate = result.transformedExpression;
}
multiStringInit = string.Join("\n", result.wrapperTable.Select(e => string.Format(
"var {0} = new MultiString.MultiStringWrapper({1}{2}_col);",
e.Value.Name, Transformer.ColumnFieldPrefix, e.Key.Name)));
multiStringWrapperInit = string.Join("\n", result.wrapperTable.Select(e => string.Format("{0}.rowIndex = i;", e.Value.Name)));
vectorOperations = result.vectorOperation;
}
Contract.Assume(stream.Predicate.Parameters.Count() == 1);
var parameter = stream.Predicate.Parameters.First();
var x = Extensions.TransformUnaryFunction <TKey, TPayload>(Expression.Lambda(transformedPredicate, parameter));
noTransformation = x == null;
transformedPredicate = noTransformation ? p : x.Body;
var template = new WhereTemplate(
generatedClassName,
keyType,
sourceType,
transformedPredicate.ExpressionToCSharp(),
noTransformation,
stream.Predicate.Parameters[0].ToString(),
multiStringInit,
multiStringReturns)
{
multiStringWrapperInit = multiStringWrapperInit,
vectorOperations = vectorOperations
};
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(keyType, sourceType);
expandedCode = template.TransformText();
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeof(TKey), typeof(TPayload));
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TKey, TPayload, TResult>(SelectManyStreamable <TKey, TPayload, TResult> stream)
{
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string generatedClassName;
string expandedCode;
string errorMessages = null;
try
{
generatedClassName = string.Format("SelectMany_{0}", sequenceNumber++);
var keyType = typeof(TKey);
var payloadType = typeof(TPayload);
var resultType = typeof(TResult);
var template = new SelectManyTemplate(generatedClassName, keyType, payloadType, resultType);
var tm = new TypeMapper(keyType, payloadType, resultType);
var gps = tm.GenericTypeVariables(keyType, payloadType, resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.numberOfGenericParameters = gps.Count();
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.MemoryPoolGenericParameters = string.Format("<{0}, {1}>", template.TKey, template.TResult);
if (resultType == typeof(int) || resultType == typeof(long) || resultType == typeof(string))
{
template.MemoryPoolGenericParameters = string.Empty;
}
var payloadParameterIndex = 0;
if (stream.HasKey && stream.HasStartEdge)
{
payloadParameterIndex = 2;
}
else if (stream.HasKey || stream.HasStartEdge)
{
payloadParameterIndex = 1;
}
var selector = stream.Selector;
var payloadParameter = selector.Parameters.ElementAt(payloadParameterIndex);
template.PARAMETER = payloadParameter.Name;
template.resultPayloadRepresentation = new ColumnarRepresentation(resultType);
template.resultFields = template.resultPayloadRepresentation.AllFields;
if (template.numberOfGenericParameters > 0)
{
generatedClassName = generatedClassName + "`" + template.numberOfGenericParameters.ToString(CultureInfo.InvariantCulture);
}
expandedCode = string.Empty;
Expression transformedSelector = selector;
// No substitutions are made for the start edge parameter or key parameter. Both just remain in the
// body of the result selector and are set as local variables in the generated code.
var keyParameterIndex = stream.HasStartEdge ? 1 : 0;
var tuple = OptimizeSelectMany(selector.Body);
if (tuple != null)
{
template.enumerableRepeatSelector = true;
var resultSelector = stream.Selector;
var sourceMessageType = StreamMessageManager.GetStreamMessageType <TKey, TPayload>();
var pseudoLambdaParameters = new ParameterExpression[stream.HasKey ? 2 : 1];
var pseudoLambdaIndex = 0;
if (stream.HasKey)
{
pseudoLambdaParameters[pseudoLambdaIndex++] = selector.Parameters[keyParameterIndex];
}
pseudoLambdaParameters[pseudoLambdaIndex] = payloadParameter;
var pseudoLambda = Expression.Lambda(tuple.Item1, pseudoLambdaParameters);
var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >
{
Tuple.Create(payloadParameter, new SelectParameterInformation()
{
BatchName = "batch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = new ColumnarRepresentation(payloadType)
})
};
var projectionResult = SelectTransformer.Transform(pseudoLambda, parameterSubstitutions, template.resultPayloadRepresentation, true, stream.HasStartEdge);
template.computedFields = projectionResult.ComputedFields;
template.useEnumerator = false;
var loopCounter = tuple.Item2;
var newParameters = new ParameterExpression[stream.HasKey ? 2 : 1];
newParameters[0] = payloadParameter;
if (stream.HasKey)
{
newParameters[1] = selector.Parameters[keyParameterIndex];
}
var loopCounterLambda = Expression.Lambda(loopCounter, payloadParameter);
var transformedLoopCounter = Extensions.TransformFunction <TKey, TPayload>(loopCounterLambda, 0);
template.loopCounter = transformedLoopCounter.Body.ExpressionToCSharp();
// REVIEW: Alternative: use Inline to replace occurrences of the key parameter
// with "batch.key.col[i]".
if (stream.HasKey)
{
template.keyParameterName = selector.Parameters[keyParameterIndex].Name;
}
}
else
{
transformedSelector = Extensions.TransformFunction <TKey, TPayload>(stream.Selector, payloadParameterIndex).Body;
if (transformedSelector == null)
{
template.useEnumerator = true;
template.transformedSelectorAsSource = stream.Selector.ExpressionToCSharp();
}
else
{
var tuple2 = OptimizeSelectMany(transformedSelector);
if (tuple2 != null)
{
template.useEnumerator = false;
template.loopCounter = tuple2.Item2.ExpressionToCSharp();
template.transformedSelectorAsSource = tuple2.Item1.ExpressionToCSharp();
}
else
{
template.useEnumerator = true;
template.transformedSelectorAsSource = transformedSelector.ExpressionToCSharp();
}
}
}
template.StartEdgeParameterName = stream.HasStartEdge ? selector.Parameters.ElementAt(0).Name : null;
template.hasKey = stream.HasKey;
expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TResult));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(stream.Selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeof(TKey), typeof(TPayload), typeof(TResult));
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new EquiJoinTemplate();
var keyType = template.keyType = typeof(TKey);
var leftType = template.leftType = typeof(TLeft);
var rightType = template.rightType = typeof(TRight);
var resultType = template.resultType = typeof(TResult);
var tm = new TypeMapper(keyType, leftType, rightType, resultType);
template.TKey = tm.CSharpNameFor(keyType);
template.TLeft = tm.CSharpNameFor(leftType);
template.TRight = tm.CSharpNameFor(rightType);
template.TResult = tm.CSharpNameFor(resultType);
var keyAndLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString();
var keyAndRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString();
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.genericParameters = tm.GenericTypeVariables(keyType, leftType, rightType, resultType).BracketedCommaSeparatedString();
template.className = string.Format("GeneratedEquiJoin_{0}", EquiJoinSequenceNumber++);
template.leftMessageRepresentation = new ColumnarRepresentation(leftType);
template.rightMessageRepresentation = new ColumnarRepresentation(rightType);
var resultMessageRepresentation = new ColumnarRepresentation(resultType);
var batchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
var batchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType);
template.LeftBatchType = batchGeneratedFrom_TKey_TLeft + keyAndLeftGenericParameters;
template.RightBatchType = batchGeneratedFrom_TKey_TRight + keyAndRightGenericParameters;
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightFields = template.rightMessageRepresentation.AllFields;
template.resultFields = resultMessageRepresentation.AllFields;
template.ActiveEventTypeLeft = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event_Left";
template.ActiveEventTypeRight = rightType.GetTypeInfo().IsValueType ? template.TRight : "Active_Event_Right";
#region Key Equals
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparerEquals =
(left, right) =>
keyComparer.Inline(left, right);
if (keyType.IsAnonymousType())
{
template.keyComparerEquals =
(left, right) => string.Format("keyComparerEquals({0}, {1})", left, right);
}
#endregion
#region Left Payload Equals
{
var leftPayloadComparer = stream.Left.Properties.PayloadEqualityComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftPayloadComparer, "leftIndex", 0);
template.leftComparerEquals = (left, right) => newLambda.Inline(left, right);
}
#endregion
#region Right Payload Equals
{
var rightPayloadComparer = stream.Right.Properties.PayloadEqualityComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TRight>(rightPayloadComparer, "rightIndex", 0);
template.rightComparerEquals = (left, right) => newLambda.Inline(left, right);
}
#endregion
#region Result Selector
{
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
errorMessages = "result selector is not a closed expression";
throw new InvalidOperationException();
}
#region LeftBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), "i"), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), "j"), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region ActiveSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >();
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultMessageRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.activeSelector = (leftEvent, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
}
#endregion
if (stream.Left.Properties.IsIntervalFree && stream.Right.Properties.IsConstantDuration)
{
template.endPointHeap = "EndPointQueue";
}
else if (stream.Right.Properties.IsIntervalFree && stream.Left.Properties.IsConstantDuration)
{
template.endPointHeap = "EndPointQueue";
}
else if (stream.Left.Properties.IsConstantDuration && stream.Right.Properties.IsConstantDuration &&
stream.Left.Properties.ConstantDurationLength == stream.Right.Properties.ConstantDurationLength)
{
template.endPointHeap = "EndPointQueue";
}
else
{
template.endPointHeap = "EndPointHeap";
}
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(keyType, leftType, rightType, resultType);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (keyType.IsAnonymousType())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The key type for an equi-join is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n";
}
var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType, resultType);
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(leftType.GetAnonymousTypes());
list.AddRange(rightType.GetAnonymousTypes());
list.AddRange(resultType.GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
private static Tuple <Type, string> GenerateInternal <TOuterKey, TInnerKey, TInnerResult, TResult>(Expression <Func <TInnerKey, TInnerResult, TResult> > resultSelector, bool isFirstLevelGroup)
{
Contract.Ensures(Contract.Result <Tuple <Type, string> >() != null);
Contract.Ensures(typeof(Pipe <TOuterKey, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTInnerKey = typeof(TInnerKey);
var typeofTInnerResult = typeof(TInnerResult);
var typeofTResult = typeof(TResult);
string expandedCode;
List <Assembly> assemblyReferences;
int numberOfGenericParameters;
// inline: var ok = UngroupTemplate.GetGeneratedCode<TOuterKey, TInnerKey, TInnerResult, TResult>(resultSelector, false, out generatedClassName, out expandedCode, out assemblyReferences, out numberOfGenericParameters);
var generatedClassName = string.Format(
"UngroupPipeGeneratedFrom_{0}_{1}_{2}_{3}_{4}",
typeOfTOuterKey.GetValidIdentifier(),
typeOfTInnerKey.GetValidIdentifier(),
typeofTInnerResult.GetValidIdentifier(),
typeofTResult.GetValidIdentifier(),
UngroupSequenceNumber++);
var inputMessageType = StreamMessageManager.GetStreamMessageType <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult>();
var innerResultRepresentation = new ColumnarRepresentation(typeofTInnerResult);
var resultRepresentation = new ColumnarRepresentation(typeofTResult);
var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >
{
// Leave the key parameter to the selector unchanged, so no substitution for parameters[0]
Tuple.Create(resultSelector.Parameters[1], new SelectParameterInformation()
{
BatchName = "inputBatch", BatchType = inputMessageType, IndexVariableName = "i", parameterRepresentation = innerResultRepresentation,
})
};
var result = SelectTransformer.Transform(resultSelector, parameterSubstitutions, resultRepresentation);
if (result.Error)
{
return(Tuple.Create((Type)null, errorMessages));
}
var template = new UngroupTemplate(generatedClassName, !isFirstLevelGroup, typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult)
{
innerResultRepresentation = innerResultRepresentation,
swingingFields = result.SwingingFields,
computedFields = result.ComputedFields,
unassignedFields = result.UnmentionedFields,
keyParameter = resultSelector.Parameters.First()
};
template.staticCtor = Transformer.StaticCtor(template.CLASSNAME);
expandedCode = template.TransformText();
assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
// input messages
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TInnerResult>());
// output messages
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TResult>());
// memory pool
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TOuterKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(resultSelector));
numberOfGenericParameters = template.numberOfGenericParameters;
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (numberOfGenericParameters > 0)
{
generatedClassName = generatedClassName + "`" + numberOfGenericParameters.ToString(CultureInfo.InvariantCulture);
}
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTInnerKey, typeofTInnerResult, typeofTResult);
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Generate a batch class definition to be used as a Clip pipe.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TLeft">The payload type for the left side.</typeparam>
/// <typeparam name="TRight">The payload type for the right side.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TKey"/>, <typeparamref name="TRight"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight>(ClipJoinStreamable <TKey, TLeft, TRight> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TLeft>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new ClipJoinTemplate();
var keyType = template.keyType = typeof(TKey);
var leftType = template.leftType = typeof(TLeft);
var rightType = template.rightType = typeof(TRight);
var tm = new TypeMapper(keyType, leftType, rightType);
template.TKey = tm.CSharpNameFor(keyType);
template.TLeft = tm.CSharpNameFor(leftType);
template.TRight = tm.CSharpNameFor(rightType);
var gps = tm.GenericTypeVariables(keyType, leftType, rightType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.className = string.Format("GeneratedClip_{0}", ClipSequenceNumber++);
var resultRepresentation = new ColumnarRepresentation(leftType);
template.ActiveEventType = leftType.GetTypeInfo().IsValueType ? template.TLeft : "Active_Event";
#region Key Comparer
var keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
template.keyComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
#region Left Comparer
var leftComparer = stream.LeftComparer.GetEqualsExpr();
var newLambda = Extensions.TransformFunction <TKey, TLeft>(leftComparer, "index_ProcessLeftEvent", 0);
template.leftComparer = (left, right) => newLambda.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
template.TKeyTLeftGenericParameters = tm.GenericTypeVariables(keyType, leftType).BracketedCommaSeparatedString();
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.TKeyTRightGenericParameters = tm.GenericTypeVariables(keyType, rightType).BracketedCommaSeparatedString();
template.leftFields = resultRepresentation.AllFields;
template.noFields = resultRepresentation.noFields;
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(leftComparer));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var realClassName = template.className.AddNumberOfNecessaryGenericArguments(keyType, leftType, rightType);
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(leftType.GetAnonymousTypes());
list.AddRange(rightType.GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TKey, TPayload, TResult>(SelectStreamable <TKey, TPayload, TResult> stream)
{
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string generatedClassName;
string errorMessages = null;
try
{
var keyType = typeof(TKey);
var payloadType = typeof(TPayload);
var resultType = typeof(TResult);
generatedClassName = string.Format("Select_{0}_{1}_{2}_{3}", keyType.GetValidIdentifier(), payloadType.GetValidIdentifier(), resultType.GetValidIdentifier(), sequenceNumber++);
var template = new SelectTemplate(generatedClassName, keyType, payloadType, resultType);
var tm = new TypeMapper(keyType, payloadType, resultType);
var gps = tm.GenericTypeVariables(keyType, payloadType, resultType);
template.genericParameters = gps.BracketedCommaSeparatedString();
template.numberOfGenericParameters = gps.Count();
template.TKeyTResultGenericParameters = tm.GenericTypeVariables(keyType, resultType).BracketedCommaSeparatedString();
template.MemoryPoolGenericParameters = string.Format("<{0}, {1}>", template.TKey, template.TResult);
if (resultType == typeof(int) || resultType == typeof(long) || resultType == typeof(string))
{
template.MemoryPoolGenericParameters = string.Empty;
}
var payloadParameterIndex = 0;
if (stream.HasKey && stream.HasStartEdge)
{
payloadParameterIndex = 2;
}
else if (stream.HasKey || stream.HasStartEdge)
{
payloadParameterIndex = 1;
}
template.PARAMETER = stream.Selector.Parameters[payloadParameterIndex].Name;
template.resultPayloadRepresentation = new ColumnarRepresentation(resultType);
template.destinationFields = template.resultPayloadRepresentation.AllFields;
if (template.numberOfGenericParameters > 0)
{
generatedClassName = generatedClassName + "`" + template.numberOfGenericParameters.ToString(CultureInfo.InvariantCulture);
}
var resultSelector = stream.Selector;
var sourceMessageType = StreamMessageManager.GetStreamMessageType <TKey, TPayload>();
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >();
// Don't create a parameter substitution for the start edge parameter. That will just remain in the
// body of the result selector and will be set as a local variable in the generated code.
if (stream.HasKey)
{
var keyRepresentation = new ColumnarRepresentation(keyType, "key");
var keyParameterIndex = stream.HasStartEdge ? 1 : 0;
parameterSubsitutions.Add(Tuple.Create(
resultSelector.Parameters.ElementAt(keyParameterIndex),
new SelectParameterInformation()
{
BatchName = "sourceBatch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = keyRepresentation,
}));
}
parameterSubsitutions.Add(Tuple.Create(
resultSelector.Parameters.ElementAt(payloadParameterIndex),
new SelectParameterInformation()
{
BatchName = "sourceBatch", BatchType = sourceMessageType, IndexVariableName = "i", parameterRepresentation = new ColumnarRepresentation(payloadType)
}));
var projectionResult = SelectTransformer.Transform(resultSelector, parameterSubsitutions, template.resultPayloadRepresentation, false, stream.HasStartEdge);
if (projectionResult.Error)
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
throw new InvalidOperationException("Code Generation couldn't transform a selector!");
}
return(Tuple.Create((Type)null, errorMessages));
}
template.StartEdgeParameterName = stream.HasStartEdge ? resultSelector.Parameters.ElementAt(0).Name : null;
template.computedFields = projectionResult.ComputedFields;
template.swingingFields = projectionResult.SwingingFields;
template.unassignedFields = projectionResult.UnmentionedFields;
template.ProjectionReturningResultInstance = projectionResult.ProjectionReturningResultInstance;
template.multiStringOperations = projectionResult.MultiStringOperations;
template.needSourceInstance = projectionResult.NeedsSourceInstance;
var d = new Dictionary <MyFieldInfo, int>();
foreach (var f in template.swingingFields)
{
var target = f.Item1;
var source = f.Item2;
if (!d.ContainsKey(source))
{
d.Add(source, 0);
}
d[source] = d[source] + 1;
}
template.swungFieldsCount = d;
template.nonSwingingFields = template.fields.Where(sf => !template.swingingFields.Any(swingingField => swingingField.Item2.Equals(sf)));
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TResult));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(stream.Selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeof(TKey), typeof(TPayload), typeof(TResult));
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Generate a batch class definition to be used as an aggregate definition.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for the aggregate.</typeparam>
/// <typeparam name="TInput">The input type for the aggregate.</typeparam>
/// <typeparam name="TOutput">The output type for the aggregate.</typeparam>
/// <typeparam name="TState">The type for the accumulated state held by the aggregate.</typeparam>
/// <typeparam name="TResult">The type of the result.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of UnaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TInput"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TInput, TState, TOutput, TResult>(
GroupedWindowStreamable <TKey, TInput, TState, TOutput, TResult> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(IStreamObserver <Empty, TInput>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
string expandedCode;
var template = new GroupedWindowTemplate($"GeneratedGroupedAggregate_{GroupedAggregateSequenceNumber++}");
var keyType = template.keyType = typeof(TKey);
var inputType = template.inputType = typeof(TInput);
var stateType = template.stateType = typeof(TState);
var outputType = template.outputType = typeof(TOutput);
var resultType = template.resultType = typeof(TResult);
template.TResult = resultType.GetCSharpSourceSyntax(); // BUGBUG: need to get any generic parameters needed
template.isUngrouped = (keyType == typeof(Empty));
var inputMessageRepresentation = new ColumnarRepresentation(inputType);
var resultRepresentation = new ColumnarRepresentation(resultType);
var assemblyReferences = new List <Assembly>();
#region Key Selector
var keySelector = stream.KeySelector;
string transformedKeySelectorAsString;
if (keyType.IsAnonymousTypeName())
{
Contract.Assume(keySelector.Body is NewExpression);
var transformedFunction = Extensions.TransformUnaryFunction <TKey, TInput>(keySelector);
var newBody = (NewExpression)transformedFunction.Body;
transformedKeySelectorAsString = string.Join(",", newBody.Arguments);
}
else
{
var transformedFunction = Extensions.TransformUnaryFunction <Empty, TInput>(keySelector).Body;
if (transformedFunction == null)
{
return(null);
}
transformedKeySelectorAsString = transformedFunction.ExpressionToCSharp();
}
template.keySelector = transformedKeySelectorAsString;
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector));
#endregion
#region Key Comparer and HashCode
var keyComparer = EqualityComparerExpression <TKey> .Default;
template.keyComparerEquals =
(left, right) =>
keyComparer.GetEqualsExpr().Inline(left, right);
template.keyComparerGetHashCode =
(x) =>
keyComparer.GetGetHashCodeExpr().Inline(x);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyComparer.GetEqualsExpr()));
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keyComparer.GetGetHashCodeExpr()));
#endregion
#region Aggregate functions
var initialStateLambda = stream.Aggregate.InitialState();
if (ConstantExpressionFinder.IsClosedExpression(initialStateLambda))
{
template.initialState = initialStateLambda.Body.ExpressionToCSharp();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(initialStateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the initial state lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledInitialState = true;
template.initialState = "initialState()";
}
}
var accumulateLambda = stream.Aggregate.Accumulate();
if (ConstantExpressionFinder.IsClosedExpression(accumulateLambda))
{
var accTransformedLambda = Extensions.TransformFunction <TKey, TInput>(accumulateLambda, 2);
template.accumulate = (stateArg, longArg) => accTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(accumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the accumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledAccumulate = true;
template.accumulate = (s1, s2) => $"accumulate({s1}, {s2}, batch[i]);";
}
}
var deaccumulateLambda = stream.Aggregate.Deaccumulate();
if (ConstantExpressionFinder.IsClosedExpression(deaccumulateLambda))
{
var deaccumulateTransformedLambda = Extensions.TransformFunction <TKey, TInput>(deaccumulateLambda, 2);
template.deaccumulate = (stateArg, longArg) => deaccumulateTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(deaccumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
throw new InvalidOperationException("Code Generation couldn't inline a lambda!");
}
else
{
template.useCompiledDeaccumulate = true;
template.deaccumulate = (s1, s2) => $"deaccumulate({s1}, {s2}, batch[i]);";
}
}
var differenceLambda = stream.Aggregate.Difference();
if (ConstantExpressionFinder.IsClosedExpression(differenceLambda))
{
template.difference = (stateArg1, stateArg2) => differenceLambda.Inline(stateArg1, stateArg2);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(differenceLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the deaccumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledDifference = true;
template.deaccumulate = (s1, s2) => $"difference({s1}, {s2});";
}
}
var computeResultLambda = stream.Aggregate.ComputeResult();
if (ConstantExpressionFinder.IsClosedExpression(computeResultLambda))
{
if (outputType.IsAnonymousType())
{
if (computeResultLambda.Body is NewExpression newExpression)
{
errorMessages = "Code Generation for GroupedWindow: result selector must be a new expression for anonymous types";
throw new NotImplementedException(errorMessages);
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(computeResultLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for GroupedWindow: couldn't inline the result selector lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledComputeResult = true;
template.computeResult = (stateArg) => $"computeResult({stateArg})";
}
}
#endregion
template.BatchGeneratedFrom_Unit_TInput = Transformer.GetBatchClassName(typeof(Empty), inputType);
template.UnitTInputGenericParameters = string.Empty; // BUGBUG
template.UnitTResultGenericParameters = string.Empty; // BUGBUG
template.inputFields = inputMessageRepresentation.AllFields;
template.outputFields = resultRepresentation.AllFields;
var resultSelector = stream.ResultSelector;
var parameterSubstitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >(); // dont want the parameters substituted for at all
var projectionResult = SelectTransformer.Transform(resultSelector, parameterSubstitutions, resultRepresentation, true);
if (projectionResult.Error)
{
return(null);
}
template.finalResultSelector =
(key, aggregateResult) =>
{
var parameters = new Dictionary <ParameterExpression, string>
{
{ resultSelector.Parameters.ElementAt(0), key }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
sb.AppendLine($"var {resultSelector.Parameters.ElementAt(1).Name} = {aggregateResult};\n");
if (projectionResult.ProjectionReturningResultInstance != null)
{
sb.AppendLine($"this.batch[_c] = {projectionResult.ProjectionReturningResultInstance.ExpressionToCSharp()};");
}
else
{
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
sb.AppendLine($"this.batch.{f.Name}.col[_c] = {e.ExpressionToCSharpStringWithParameterSubstitution(parameters)};");
}
}
sb.AppendLine("}");
return(sb.ToString());
};
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(resultSelector));
expandedCode = template.TransformText();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(typeof(Empty), typeof(TKey), typeof(TInput), typeof(TState), typeof(TOutput), typeof(FastDictionaryGenerator3)));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TInput>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <Empty, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <Empty, TResult>());
var assembly = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = assembly.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(typeof(TInput).GetAnonymousTypes());
list.AddRange(typeof(TState).GetAnonymousTypes());
list.AddRange(typeof(TOutput).GetAnonymousTypes());
return(Tuple.Create(t.MakeGenericType(list.ToArray()), errorMessages));
}
else
{
return(Tuple.Create(t, errorMessages));
}
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TKey, TInput, TState, TOutput>(SnapshotWindowStreamable <TKey, TInput, TState, TOutput> stream, AggregatePipeType pipeType)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(IStreamObserver <TKey, TInput>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
var container = stream.Properties.QueryContainer;
string generatedClassName;
string expandedCode;
List <Assembly> assemblyReferences;
string errorMessages = null;
try
{
AggregateTemplate template;
switch (pipeType)
{
case AggregatePipeType.StartEdge:
template = new SnapshotWindowStartEdgeTemplate();
break;
case AggregatePipeType.PriorityQueue:
template = new SnapshotWindowPriorityQueueTemplate();
break;
case AggregatePipeType.Tumbling:
template = new SnapshotWindowTumblingTemplate();
break;
case AggregatePipeType.Sliding:
template = new SnapshotWindowSlidingTemplate();
break;
case AggregatePipeType.Hopping:
template = new SnapshotWindowHoppingTemplate
{
hopsPerDuration = ((int)(stream.Source.Properties.ConstantDurationLength.Value / stream.Source.Properties.ConstantHopLength) + 1).ToString()
};
break;
default:
Contract.Assert(false, "case meant to be exhaustive");
throw new InvalidOperationException("case meant to be exhaustive");
}
template.isUngrouped = (typeof(TKey) == typeof(Empty));
var keyType = template.keyType = typeof(TKey);
var inputType = template.inputType = typeof(TInput);
var stateType = template.stateType = typeof(TState);
var outputType = template.outputType = typeof(TOutput);
template.inputFields = new ColumnarRepresentation(inputType).AllFields;
template.outputFields = new ColumnarRepresentation(outputType).AllFields;
assemblyReferences = new List <Assembly>();
#region Key Comparer
IEqualityComparerExpression <TKey> keyComparer;
keyComparer = stream.Properties.KeyEqualityComparer;
var equalsExpression = keyComparer.GetEqualsExpr();
var getHashcodeExpression = keyComparer.GetGetHashCodeExpr();
template.inlinedKeyComparerEquals =
(left, right) =>
string.Format("({0})", equalsExpression.Inline(left, right));
template.inlinedKeyComparerGetHashCode =
(x) =>
string.Format("({0}/* inlined GetHashCode */)", getHashcodeExpression.Inline(x));
if (keyType.IsAnonymousType())
{
template.inlinedKeyComparerEquals =
(left, right) => string.Format("keyComparerEquals({0}, {1})", left, right);
template.inlinedKeyComparerGetHashCode =
(x) => string.Format("keyComparerGetHashCode({0})", x);
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(equalsExpression));
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(getHashcodeExpression));
#endregion
#region Aggregate Functions
var initialStateLambda = stream.Aggregate.InitialState();
if (ConstantExpressionFinder.IsClosedExpression(initialStateLambda))
{
template.initialState = initialStateLambda.Body.ExpressionToCSharp();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(initialStateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the initial state lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledInitialState = true;
template.initialState = "initialState()";
}
}
var accumulateLambda = stream.Aggregate.Accumulate();
if (ConstantExpressionFinder.IsClosedExpression(accumulateLambda))
{
var accTransformedLambda = Extensions.TransformFunction <TKey, TInput>(accumulateLambda, 2);
template.accumulate = (stateArg, longArg) => accTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(accumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the accumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledAccumulate = true;
template.accumulate = (s1, s2) => string.Format("accumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var deaccumulateLambda = stream.Aggregate.Deaccumulate();
if (ConstantExpressionFinder.IsClosedExpression(deaccumulateLambda))
{
var deaccumulateTransformedLambda = Extensions.TransformFunction <TKey, TInput>(deaccumulateLambda, 2);
template.deaccumulate = (stateArg, longArg) => deaccumulateTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(deaccumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the deaccumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledDeaccumulate = true;
template.deaccumulate = (s1, s2) => string.Format("deaccumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var differenceLambda = stream.Aggregate.Difference();
if (ConstantExpressionFinder.IsClosedExpression(differenceLambda))
{
template.difference = (stateArg1, stateArg2) => differenceLambda.Inline(stateArg1, stateArg2);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(differenceLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the difference lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledDifference = true;
template.difference = (s1, s2) => string.Format("difference({0}, {1});", s1, s2);
}
}
var computeResultLambda = stream.Aggregate.ComputeResult();
if (ConstantExpressionFinder.IsClosedExpression(computeResultLambda))
{
if (outputType.IsAnonymousType())
{
if (computeResultLambda.Body is NewExpression newExpression)
{
var outputBatchType = StreamMessageManager.GetStreamMessageType <TKey, TOutput>();
var foo = Transform(newExpression, outputBatchType);
template.computeResult = (stateArg) => Expression.Lambda(foo, computeResultLambda.Parameters.ToArray()).Inline(stateArg);
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(computeResultLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar || outputType.IsAnonymousType())
{
// second disjunct is because if we aren't inlining the computeResult function and
// the output type is anonymous, calling the compiled computeResult function returns
// a value of the anonymous type and since the generated operator represents the anonymous
// type as a generic parameter, it can't use the "field" (i.e., property) names to
// get the individual pieces to assign to each column of the output message.
errorMessages = "Code Generation for Aggregate: couldn't inline the compute result lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledComputeResult = true;
template.computeResult = (stateArg) => "computeResult(" + stateArg + ");";
}
}
#endregion
generatedClassName = template.className = string.Format("Aggregate_{0}", sequenceNumber++);
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(keyType, inputType, stateType, outputType);
template.staticCtor = Transformer.StaticCtor(template.className);
expandedCode = template.TransformText();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TInput), typeof(TState), typeof(TOutput), typeof(FastDictionaryGenerator), typeof(SortedDictionary <,>)));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TInput>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TOutput>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TOutput>());
if (container != null)
{
assemblyReferences.AddRange(container.CollectedGeneratedTypes.Select(o => o.GetTypeInfo().Assembly));
}
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (keyType.IsAnonymousType())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The key type for an aggregate is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n";
}
var t = a.GetType(generatedClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(typeof(TInput).GetAnonymousTypes());
list.AddRange(typeof(TState).GetAnonymousTypes());
list.AddRange(typeof(TOutput).GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
public static Tuple <Type, string> Generate <TOuterKey, TSource, TInnerKey>(
Expression <Func <TInnerKey, int> > hashComparer,
Expression <Func <TSource, TInnerKey> > keySelector,
bool nested)
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTSource = typeof(TSource);
var typeOfTInnerKey = typeof(TInnerKey);
string expandedCode;
List <Assembly> assemblyReferences;
string errorMessages = null;
try
{
string generatedClassName = $"GeneratedGroupStreamable_{GroupStreamableSequenceNumber++}";
string transformedKeySelectorAsString;
MyFieldInfo swingingField = default;
if (typeOfTInnerKey.IsAnonymousTypeName())
{
Contract.Assume(keySelector.Body is NewExpression);
var transformedFunction = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector);
var newBody = (NewExpression)transformedFunction.Body;
transformedKeySelectorAsString = string.Join(",", newBody.Arguments.Select(arg => arg.ExpressionToCSharp()));
}
else
{
var body = keySelector.Body;
var singleFieldProjection = body as MemberExpression;
if (!nested && singleFieldProjection != null)
{
if (singleFieldProjection.Expression is ParameterExpression dereferencedObject)
{
Contract.Assume(dereferencedObject == keySelector.Parameters.ElementAt(0));
// then can just swing a pointer to set the key of the result message
var f = singleFieldProjection.Member;
var sourceMessageRepresentation = new ColumnarRepresentation(typeOfTSource);
var fieldToSwingFrom = sourceMessageRepresentation.Fields[f.Name];
swingingField = fieldToSwingFrom;
transformedKeySelectorAsString = swingingField.Name + "_col[i]";
}
else
{
var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body;
if (transformedPredicate == null)
{
return(Tuple.Create((Type)null, errorMessages));
}
transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp();
}
}
else
{
var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body;
if (transformedPredicate == null)
{
return(Tuple.Create((Type)null, errorMessages));
}
transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp();
}
}
var inlinedHashCodeComputation = hashComparer.Inline("key");
var template = new GroupTemplate(
generatedClassName,
typeOfTOuterKey,
typeOfTSource,
typeOfTInnerKey,
transformedKeySelectorAsString,
inlinedHashCodeComputation,
nested)
{
swingingField = swingingField,
payloadMightBeNull = typeOfTSource.CanContainNull()
};
if (!nested && Config.UseMultiString &&
typeOfTInnerKey.Equals(typeof(string)) &&
keySelector.IsSimpleFieldOrPropertyAccess())
{
var transformedPredicate = Extensions.TransformUnaryFunction <TOuterKey, TSource>(keySelector).Body;
template.transformedKeySelectorAsString = transformedPredicate.ExpressionToCSharp();
template.inlinedHashCodeComputation = "hashCodeVector.col[i]";
var fieldName = ((MemberExpression)keySelector.Body).Member.Name;
template.vectorHashCodeInitialization = $"resultBatch.hash = {Transformer.ColumnFieldPrefix}{fieldName}_col.GetHashCode(batch.bitvector);";
template.swingingHashColumn = true;
}
template.fields = new ColumnarRepresentation(typeOfTSource).AllFields;
template.innerKeyFieldRepresentation = new ColumnarRepresentation(typeOfTInnerKey);
expandedCode = template.TransformText();
assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TOuterKey, TSource>());
if (nested)
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <CompoundGroupKey <TOuterKey, TInnerKey>, TSource>());
}
else
{
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TInnerKey, TSource>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TInnerKey, TSource>());
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(keySelector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (typeOfTInnerKey.IsAnonymousTypeName())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The inner key type for Group is anonymous, causing the use of Activator.CreateInstance in an inner loop. This will lead to poor performance.\n";
}
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
var t = a.GetType(generatedClassName);
t = t.InstantiateAsNecessary(typeOfTOuterKey, typeOfTSource, typeOfTInnerKey);
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
private Tuple <Type, string> GenerateInternal <TKey, TLeft, TRight, TResult>(int numParameters, Expression expression)
{
string errorMessages = null;
try
{
var expandedCode = TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(this.keyType, this.leftType, this.rightType, this.resultType);
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
if (expression != null)
{
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(expression));
}
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (this.keyType.IsAnonymousType())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The key type for a binary operator is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n";
}
var types = new List <Type> {
this.keyType, this.leftType
};
if (numParameters > 2)
{
types.Add(this.rightType);
}
if (numParameters == 4)
{
types.Add(this.resultType);
}
var realClassName = this.className.AddNumberOfNecessaryGenericArguments(types.ToArray());
var t = a.GetType(realClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = this.keyType.GetAnonymousTypes();
list.AddRange(this.leftType.GetAnonymousTypes());
if (numParameters > 2)
{
list.AddRange(this.rightType.GetAnonymousTypes());
}
if (numParameters == 4)
{
list.AddRange(this.resultType.GetAnonymousTypes());
}
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
/// <summary>
/// Generate a batch class definition to be used as StartEdgeEquiJoin operator.
/// Compile the definition, dynamically load the assembly containing it, and return the Type representing the
/// aggregate class.
/// </summary>
/// <typeparam name="TKey">The key type for both sides.</typeparam>
/// <typeparam name="TLeft">The payload type for the left side.</typeparam>
/// <typeparam name="TRight">The payload type for the right side.</typeparam>
/// <typeparam name="TResult">The payload type for the resulting stream.</typeparam>
/// <returns>
/// A type that is defined to be a subtype of BinaryPipe<<typeparamref name="TKey"/>,<typeparamref name="TLeft"/>, <typeparamref name="TRight"/>, <typeparamref name="TKey"/>, <typeparamref name="TResult"/>>.
/// </returns>
internal static Tuple <Type, string> Generate <TKey, TLeft, TRight, TResult>(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(BinaryPipe <TKey, TLeft, TRight, TResult>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var template = new IncreasingOrderEquiJoinTemplate();
var keyType = template.keyType = typeof(TKey);
var leftType = template.leftType = typeof(TLeft);
var rightType = template.rightType = typeof(TRight);
var resultType = template.resultType = typeof(TResult);
template.TKey = keyType.GetCSharpSourceSyntax();
template.TLeft = leftType.GetCSharpSourceSyntax();
template.TRight = rightType.GetCSharpSourceSyntax();
template.TResult = resultType.GetCSharpSourceSyntax(); // BUGBUG: need to get any generic parameters needed
template.className = string.Format("GeneratedIncreasingOrderEquiJoin_{0}", IOOEJSequenceNumber++);
template.leftMessageRepresentation = new ColumnarRepresentation(leftType);
template.leftFields = template.leftMessageRepresentation.AllFields;
template.rightMessageRepresentation = new ColumnarRepresentation(rightType);
template.rightFields = template.rightMessageRepresentation.AllFields;
var outputMessageRepresentation = new ColumnarRepresentation(resultType);
var leftMessageType = StreamMessageManager.GetStreamMessageType <TKey, TLeft>();
var rightMessageType = StreamMessageManager.GetStreamMessageType <TKey, TRight>();
var resultRepresentation = outputMessageRepresentation;
#region Key Comparer
var keyComparer = stream.Left.Properties.KeyComparer.GetCompareExpr();
if (!ConstantExpressionFinder.IsClosedExpression(keyComparer))
{
return(null);
}
template.joinKeyOrderComparer =
(left, right) =>
keyComparer.Inline(left, right);
#endregion
template.BatchGeneratedFrom_TKey_TLeft = Transformer.GetBatchClassName(keyType, leftType);
template.TKeyTLeftGenericParameters = string.Empty; // BUGBUG
template.BatchGeneratedFrom_TKey_TRight = Transformer.GetBatchClassName(keyType, rightType);
template.TKeyTRightGenericParameters = string.Empty; // BUGBUG
template.BatchGeneratedFrom_TKey_TResult = Transformer.GetBatchClassName(keyType, resultType);
template.TKeyTResultGenericParameters = string.Empty; // BUGBUG
template.outputFields = resultRepresentation.AllFields;
if (!ConstantExpressionFinder.IsClosedExpression(selector))
{
return(null);
}
#region LeftBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[0], new SelectParameterInformation()
{
BatchName = "leftBatch", BatchType = leftMessageType, IndexVariableName = "i", parameterRepresentation = template.leftMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.leftBatchSelector = (leftBatch, leftIndex, rightEvent) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ Expression.Variable(leftMessageType, "leftBatch"), leftBatch },
{ Expression.Variable(typeof(int), "i"), leftIndex },
{ selector.Parameters[1], rightEvent }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
#region RightBatchSelector
{
var parameterSubsitutions = new List <Tuple <ParameterExpression, SelectParameterInformation> >()
{
Tuple.Create(selector.Parameters[1], new SelectParameterInformation()
{
BatchName = "rightBatch", BatchType = rightMessageType, IndexVariableName = "j", parameterRepresentation = template.rightMessageRepresentation,
}),
};
var projectionResult = SelectTransformer.Transform(selector, parameterSubsitutions, resultRepresentation, true);
if (projectionResult.Error)
{
errorMessages = "error while transforming the result selector";
throw new InvalidOperationException();
}
template.rightBatchSelector = (leftEvent, rightBatch, rightIndex) =>
{
var d = new Dictionary <ParameterExpression, string>
{
{ selector.Parameters[0], leftEvent },
{ Expression.Variable(rightMessageType, "rightBatch"), rightBatch },
{ Expression.Variable(typeof(int), "j"), rightIndex }
};
var sb = new System.Text.StringBuilder();
sb.AppendLine("{");
foreach (var kv in projectionResult.ComputedFields)
{
var f = kv.Key;
var e = kv.Value;
if (f.OptimizeString())
{
sb.AppendFormat(
"output.{0}.AddString({1});\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
else
{
sb.AppendFormat(
"output.{0}.col[index] = {1};\n",
f.Name,
e.ExpressionToCSharpStringWithParameterSubstitution(d));
}
}
sb.AppendLine("}");
return(sb.ToString());
};
}
#endregion
template.getOutputBatch = string.Format(
"pool.Get(out genericOutputBatch); output = ({0}{1})genericOutputBatch;",
Transformer.GetBatchClassName(keyType, resultType),
template.TKeyTResultGenericParameters);
template.staticCtor = Transformer.StaticCtor(template.className);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TLeft), typeof(TRight), typeof(TResult));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TLeft>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRight>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TResult>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TResult>());
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(selector));
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = keyType.GetAnonymousTypes();
list.AddRange(leftType.GetAnonymousTypes());
list.AddRange(rightType.GetAnonymousTypes());
list.AddRange(resultType.GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
internal static Tuple <Type, string> GenerateAFA <TKey, TPayload, TRegister, TAccumulator>(
AfaStreamable <TKey, TPayload, TRegister, TAccumulator> stream)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(UnaryPipe <TKey, TPayload, TRegister>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
string errorMessages = null;
try
{
var className = string.Format("GeneratedGroupedAfaEventList_{0}", AFASequenceNumber++);
var template = new GroupedAfaEventListTemplate(className, typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(TAccumulator))
{
TKey = typeof(TKey).GetCSharpSourceSyntax(),
isFinal = stream.afa.isFinal,
hasOutgoingArcs = stream.afa.hasOutgoingArcs,
startStates = stream.afa.startStates,
AllowOverlappingInstances = stream.afa.uncompiledAfa.AllowOverlappingInstances
};
var d1 = stream.afa.uncompiledAfa.transitionInfo;
var orderedKeys = d1.Keys.OrderBy(e => e).ToArray();
for (int i = 0; i < orderedKeys.Length; i++)
{
var sourceNodeNumber = orderedKeys[i];
var outgoingEdgesDictionary = d1[sourceNodeNumber];
var orderedTargetNodes = outgoingEdgesDictionary.Keys.OrderBy(e => e).ToArray();
var edgeList1 = new List <EdgeInfo>();
for (int j = 0; j < orderedTargetNodes.Length; j++)
{
var targetNodeNumber = orderedTargetNodes[j];
var edge = outgoingEdgesDictionary[targetNodeNumber];
if (edge is ListElementArc <TPayload, TRegister> leArc)
{
var edgeInfo = new EdgeInfo()
{
Type = EdgeInfo.EdgeType.List,
EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNodeNumber),
Fence = (ts, evs, reg) => leArc.Fence.Inline(ts, evs, reg),
Transfer = leArc.Transfer == null ? ((Func <string, string, string, string>)null) : (ts, evs, reg) => leArc.Transfer.Inline(ts, evs, reg),
};
edgeList1.Add(edgeInfo);
}
}
template.currentlyActiveInfo.Add(Tuple.Create(sourceNodeNumber, edgeList1));
}
for (int i = 0; i < stream.afa.startStates.Length; i++)
{
var startState = stream.afa.startStates[i];
var edgeList2 = new List <EdgeInfo>();
var outgoingEdgeDictionary = stream.afa.uncompiledAfa.transitionInfo[startState];
foreach (var edge in outgoingEdgeDictionary)
{
var targetNode = edge.Key;
var arc = edge.Value;
if (arc is ListElementArc <TPayload, TRegister> leArc)
{
var edgeInfo = new EdgeInfo()
{
Type = EdgeInfo.EdgeType.List,
EpsilonReachableNodes = EpsilonClosure(stream.afa, targetNode),
Fence = (ts, evs, reg) => leArc.Fence.Inline(ts, evs, reg),
Transfer = leArc.Transfer == null ? ((Func <string, string, string, string>)null) : (ts, evs, reg) => leArc.Transfer.Inline(ts, evs, reg),
};
edgeList2.Add(edgeInfo);
}
}
template.newActivationInfo.Add(Tuple.Create(startState, edgeList2));
}
template.isSyncTimeSimultaneityFree = true; // The handwritten version doesn't make a distinction.
template.keyEqualityComparer =
(left, right) =>
stream.Properties.KeyEqualityComparer.GetEqualsExpr().Inline(left, right);
var expandedCode = template.TransformText();
var assemblyReferences = Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TPayload), typeof(TRegister), typeof(FastDictionaryGenerator));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TPayload>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TRegister>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TRegister>());
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
var t = a.GetType(template.className);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(template.payloadType.GetAnonymousTypes());
list.AddRange(template.registerType.GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}