public PartitionedSnapshotWindowStartEdgePipe(
SnapshotWindowStreamable <TKey, TInput, TState, TOutput> stream,
IStreamObserver <TKey, TOutput> observer)
: base(stream, observer)
{
this.aggregate = stream.Aggregate;
this.initialStateExpr = this.aggregate.InitialState();
this.initialState = this.initialStateExpr.Compile();
this.accumulateExpr = this.aggregate.Accumulate();
this.accumulate = this.accumulateExpr.Compile();
this.computeResultExpr = this.aggregate.ComputeResult();
this.computeResult = this.computeResultExpr.Compile();
var comparer = stream.Properties.KeyEqualityComparer;
this.keyComparerEqualsExpr = comparer.GetEqualsExpr();
this.keyComparerEquals = this.keyComparerEqualsExpr.Compile();
this.keyComparerGetHashCodeExpr = comparer.GetGetHashCodeExpr();
this.keyComparerGetHashCode = this.keyComparerGetHashCodeExpr.Compile();
this.errorMessages = stream.ErrorMessages;
this.pool = MemoryManager.GetMemoryPool <TKey, TOutput>(false);
this.pool.Get(out this.batch);
this.batch.Allocate();
var generator = comparer.CreateFastDictionary2Generator <TKey, HeldState <TState> >(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Properties.QueryContainer);
this.aggregateByKey = generator.Invoke();
}
public PartitionedSnapshotWindowHoppingPipe(
SnapshotWindowStreamable <TKey, TInput, TState, TOutput> stream,
IStreamObserver <TKey, TOutput> observer)
: base(stream, observer)
{
this.aggregate = stream.Aggregate;
this.initialStateExpr = this.aggregate.InitialState();
this.initialState = this.initialStateExpr.Compile();
this.accumulateExpr = this.aggregate.Accumulate();
this.accumulate = this.accumulateExpr.Compile();
this.deaccumulateExpr = this.aggregate.Deaccumulate();
this.deaccumulate = this.deaccumulateExpr.Compile();
this.differenceExpr = this.aggregate.Difference();
this.difference = this.differenceExpr.Compile();
this.computeResultExpr = this.aggregate.ComputeResult();
this.computeResult = this.computeResultExpr.Compile();
var comparer = stream.Properties.KeyEqualityComparer;
this.keyComparerEqualsExpr = comparer.GetEqualsExpr();
this.keyComparerEquals = this.keyComparerEqualsExpr.Compile();
this.keyComparerGetHashCodeExpr = comparer.GetGetHashCodeExpr();
this.keyComparerGetHashCode = this.keyComparerGetHashCodeExpr.Compile();
this.errorMessages = stream.ErrorMessages;
this.pool = MemoryManager.GetMemoryPool <TKey, TOutput>(false);
this.pool.Get(out this.batch);
this.batch.Allocate();
this.aggregateByKey = comparer.CreateFastDictionary2Generator <TKey, HeldState <TState> >(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Properties.QueryContainer).Invoke();
var stateDictGenerator = comparer.CreateFastDictionaryGenerator <TKey, StateAndActive <TState> >(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Properties.QueryContainer);
this.ecqEntryPool = new DataStructurePool <FastDictionary <TKey, StateAndActive <TState> > >(() => stateDictGenerator.Invoke());
this.hopsPerDuration = (int)(stream.Source.Properties.ConstantDurationLength.Value / stream.Source.Properties.ConstantHopLength) + 1;
}
public SnapshotWindowHoppingPipeSimple(
SnapshotWindowStreamable <Empty, TInput, TState, TOutput> stream,
IStreamObserver <Empty, TOutput> observer)
: base(stream, observer)
{
this.aggregate = stream.Aggregate;
this.initialStateExpr = this.aggregate.InitialState();
this.initialState = this.initialStateExpr.Compile();
this.accumulateExpr = this.aggregate.Accumulate();
this.accumulate = this.accumulateExpr.Compile();
this.deaccumulateExpr = this.aggregate.Deaccumulate();
this.deaccumulate = this.deaccumulateExpr.Compile();
this.differenceExpr = this.aggregate.Difference();
this.difference = this.differenceExpr.Compile();
this.computeResultExpr = this.aggregate.ComputeResult();
this.computeResult = this.computeResultExpr.Compile();
this.errorMessages = stream.ErrorMessages;
this.pool = MemoryManager.GetMemoryPool <Empty, TOutput>(false);
this.pool.Get(out this.batch);
this.batch.Allocate();
var hopsPerDuration = (int)(stream.Source.Properties.ConstantDurationLength.Value / stream.Source.Properties.ConstantHopLength) + 1;
this.ecq = new CircularBuffer <HeldState <TState> >(hopsPerDuration);
}
public SnapshotWindowStartEdgePipeSimple(
SnapshotWindowStreamable <Empty, TInput, TState, TOutput> stream,
IStreamObserver <Empty, TOutput> observer)
: base(stream, observer)
{
this.aggregate = stream.Aggregate;
this.initialStateExpr = this.aggregate.InitialState();
this.initialState = this.initialStateExpr.Compile();
this.accumulateExpr = this.aggregate.Accumulate();
this.accumulate = this.accumulateExpr.Compile();
this.computeResultExpr = this.aggregate.ComputeResult();
this.computeResult = this.computeResultExpr.Compile();
this.errorMessages = stream.ErrorMessages;
this.pool = MemoryManager.GetMemoryPool <Empty, TOutput>(false);
this.pool.Get(out this.batch);
this.batch.Allocate();
}
public PartitionedSnapshotWindowTumblingPipeSimple(
SnapshotWindowStreamable <PartitionKey <TPartitionKey>, TInput, TState, TOutput> stream,
IStreamObserver <PartitionKey <TPartitionKey>, TOutput> observer, long hop)
: base(stream, observer)
{
this.aggregate = stream.Aggregate;
this.initialStateExpr = this.aggregate.InitialState();
this.initialState = this.initialStateExpr.Compile();
this.accumulateExpr = this.aggregate.Accumulate();
this.accumulate = this.accumulateExpr.Compile();
this.computeResultExpr = this.aggregate.ComputeResult();
this.computeResult = this.computeResultExpr.Compile();
this.errorMessages = stream.ErrorMessages;
this.pool = MemoryManager.GetMemoryPool <PartitionKey <TPartitionKey>, TOutput>(false);
this.pool.Get(out this.batch);
this.batch.Allocate();
this.hop = hop;
}
public SnapshotWindowSlidingPipeSimple(
SnapshotWindowStreamable <Empty, TInput, TState, TOutput> stream,
IStreamObserver <Empty, TOutput> observer)
: base(stream, observer)
{
this.aggregate = stream.Aggregate;
this.initialStateExpr = this.aggregate.InitialState();
this.initialState = this.initialStateExpr.Compile();
this.accumulateExpr = this.aggregate.Accumulate();
this.accumulate = this.accumulateExpr.Compile();
this.deaccumulateExpr = this.aggregate.Deaccumulate();
this.deaccumulate = this.deaccumulateExpr.Compile();
this.differenceExpr = this.aggregate.Difference();
this.difference = this.differenceExpr.Compile();
this.computeResultExpr = this.aggregate.ComputeResult();
this.computeResult = this.computeResultExpr.Compile();
this.errorMessages = stream.ErrorMessages;
this.pool = MemoryManager.GetMemoryPool <Empty, TOutput>(false);
this.pool.Get(out this.batch);
this.batch.Allocate();
this.ecq = new ElasticCircularBuffer <HeldState <TState> >();
}
public static Tuple <Type, string> Generate <TKey, TInput, TState, TOutput>(SnapshotWindowStreamable <TKey, TInput, TState, TOutput> stream, AggregatePipeType pipeType)
{
Contract.Requires(stream != null);
Contract.Ensures(Contract.Result <Tuple <Type, string> >() == null || typeof(IStreamObserver <TKey, TInput>).GetTypeInfo().IsAssignableFrom(Contract.Result <Tuple <Type, string> >().Item1));
var container = stream.Properties.QueryContainer;
string generatedClassName;
string expandedCode;
List <Assembly> assemblyReferences;
string errorMessages = null;
try
{
AggregateTemplate template;
switch (pipeType)
{
case AggregatePipeType.StartEdge:
template = new SnapshotWindowStartEdgeTemplate();
break;
case AggregatePipeType.PriorityQueue:
template = new SnapshotWindowPriorityQueueTemplate();
break;
case AggregatePipeType.Tumbling:
template = new SnapshotWindowTumblingTemplate();
break;
case AggregatePipeType.Sliding:
template = new SnapshotWindowSlidingTemplate();
break;
case AggregatePipeType.Hopping:
template = new SnapshotWindowHoppingTemplate
{
hopsPerDuration = ((int)(stream.Source.Properties.ConstantDurationLength.Value / stream.Source.Properties.ConstantHopLength) + 1).ToString()
};
break;
default:
Contract.Assert(false, "case meant to be exhaustive");
throw new InvalidOperationException("case meant to be exhaustive");
}
template.isUngrouped = (typeof(TKey) == typeof(Empty));
var keyType = template.keyType = typeof(TKey);
var inputType = template.inputType = typeof(TInput);
var stateType = template.stateType = typeof(TState);
var outputType = template.outputType = typeof(TOutput);
template.inputFields = new ColumnarRepresentation(inputType).AllFields;
template.outputFields = new ColumnarRepresentation(outputType).AllFields;
assemblyReferences = new List <Assembly>();
#region Key Comparer
IEqualityComparerExpression <TKey> keyComparer;
keyComparer = stream.Properties.KeyEqualityComparer;
var equalsExpression = keyComparer.GetEqualsExpr();
var getHashcodeExpression = keyComparer.GetGetHashCodeExpr();
template.inlinedKeyComparerEquals =
(left, right) =>
string.Format("({0})", equalsExpression.Inline(left, right));
template.inlinedKeyComparerGetHashCode =
(x) =>
string.Format("({0}/* inlined GetHashCode */)", getHashcodeExpression.Inline(x));
if (keyType.IsAnonymousType())
{
template.inlinedKeyComparerEquals =
(left, right) => string.Format("keyComparerEquals({0}, {1})", left, right);
template.inlinedKeyComparerGetHashCode =
(x) => string.Format("keyComparerGetHashCode({0})", x);
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(equalsExpression));
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(getHashcodeExpression));
#endregion
#region Aggregate Functions
var initialStateLambda = stream.Aggregate.InitialState();
if (ConstantExpressionFinder.IsClosedExpression(initialStateLambda))
{
template.initialState = initialStateLambda.Body.ExpressionToCSharp();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(initialStateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the initial state lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledInitialState = true;
template.initialState = "initialState()";
}
}
var accumulateLambda = stream.Aggregate.Accumulate();
if (ConstantExpressionFinder.IsClosedExpression(accumulateLambda))
{
var accTransformedLambda = Extensions.TransformFunction <TKey, TInput>(accumulateLambda, 2);
template.accumulate = (stateArg, longArg) => accTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(accumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the accumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledAccumulate = true;
template.accumulate = (s1, s2) => string.Format("accumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var deaccumulateLambda = stream.Aggregate.Deaccumulate();
if (ConstantExpressionFinder.IsClosedExpression(deaccumulateLambda))
{
var deaccumulateTransformedLambda = Extensions.TransformFunction <TKey, TInput>(deaccumulateLambda, 2);
template.deaccumulate = (stateArg, longArg) => deaccumulateTransformedLambda.Inline(stateArg, longArg);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(deaccumulateLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the deaccumulate lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledDeaccumulate = true;
template.deaccumulate = (s1, s2) => string.Format("deaccumulate({0}, {1}, batch[i]);", s1, s2);
}
}
var differenceLambda = stream.Aggregate.Difference();
if (ConstantExpressionFinder.IsClosedExpression(differenceLambda))
{
template.difference = (stateArg1, stateArg2) => differenceLambda.Inline(stateArg1, stateArg2);
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(differenceLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar)
{
errorMessages = "Code Generation for Aggregate: couldn't inline the difference lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledDifference = true;
template.difference = (s1, s2) => string.Format("difference({0}, {1});", s1, s2);
}
}
var computeResultLambda = stream.Aggregate.ComputeResult();
if (ConstantExpressionFinder.IsClosedExpression(computeResultLambda))
{
if (outputType.IsAnonymousType())
{
if (computeResultLambda.Body is NewExpression newExpression)
{
var outputBatchType = StreamMessageManager.GetStreamMessageType <TKey, TOutput>();
var foo = Transform(newExpression, outputBatchType);
template.computeResult = (stateArg) => Expression.Lambda(foo, computeResultLambda.Parameters.ToArray()).Inline(stateArg);
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
}
else
{
template.computeResult = (stateArg) => computeResultLambda.Inline(stateArg);
}
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(computeResultLambda));
}
else
{
if (Config.CodegenOptions.SuperStrictColumnar || outputType.IsAnonymousType())
{
// second disjunct is because if we aren't inlining the computeResult function and
// the output type is anonymous, calling the compiled computeResult function returns
// a value of the anonymous type and since the generated operator represents the anonymous
// type as a generic parameter, it can't use the "field" (i.e., property) names to
// get the individual pieces to assign to each column of the output message.
errorMessages = "Code Generation for Aggregate: couldn't inline the compute result lambda!";
throw new InvalidOperationException(errorMessages);
}
else
{
template.useCompiledComputeResult = true;
template.computeResult = (stateArg) => "computeResult(" + stateArg + ");";
}
}
#endregion
generatedClassName = template.className = string.Format("Aggregate_{0}", sequenceNumber++);
generatedClassName = generatedClassName.AddNumberOfNecessaryGenericArguments(keyType, inputType, stateType, outputType);
template.staticCtor = Transformer.StaticCtor(template.className);
expandedCode = template.TransformText();
assemblyReferences.AddRange(Transformer.AssemblyReferencesNeededFor(typeof(TKey), typeof(TInput), typeof(TState), typeof(TOutput), typeof(FastDictionaryGenerator), typeof(SortedDictionary <,>)));
assemblyReferences.Add(typeof(IStreamable <,>).GetTypeInfo().Assembly);
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TInput>());
assemblyReferences.Add(Transformer.GeneratedStreamMessageAssembly <TKey, TOutput>());
assemblyReferences.Add(Transformer.GeneratedMemoryPoolAssembly <TKey, TOutput>());
if (container != null)
{
assemblyReferences.AddRange(container.CollectedGeneratedTypes.Select(o => o.GetTypeInfo().Assembly));
}
var a = Transformer.CompileSourceCode(expandedCode, assemblyReferences, out errorMessages);
if (keyType.IsAnonymousType())
{
if (errorMessages == null)
{
errorMessages = string.Empty;
}
errorMessages += "\nCodegen Warning: The key type for an aggregate is an anonymous type (or contains an anonymous type), preventing the inlining of the key equality and hashcode functions. This may lead to poor performance.\n";
}
var t = a.GetType(generatedClassName);
if (t.GetTypeInfo().IsGenericType)
{
var list = typeof(TKey).GetAnonymousTypes();
list.AddRange(typeof(TInput).GetAnonymousTypes());
list.AddRange(typeof(TState).GetAnonymousTypes());
list.AddRange(typeof(TOutput).GetAnonymousTypes());
t = t.MakeGenericType(list.ToArray());
}
return(Tuple.Create(t, errorMessages));
}
catch
{
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new InvalidOperationException("Code Generation failed when it wasn't supposed to!");
}
return(Tuple.Create((Type)null, errorMessages));
}
}
