private bool CanGenerateColumnar()
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTSource = typeof(TSource);
var typeOfTInnerKey = typeof(TInnerKey);
if (!typeOfTSource.CanRepresentAsColumnar())
{
return(false);
}
if (typeOfTOuterKey.GetPartitionType() != null)
{
return(false);
}
if (typeOfTInnerKey.GetPartitionType() != null)
{
return(false);
}
// For now, restrict the inner key to be anything other than an anonymous type since those can't be ungrouped without using reflection.
if (typeOfTInnerKey.IsAnonymousType())
{
return(false);
}
var lookupKey = CacheKey.Create(KeySelector.ToString());
var comparer = Properties.KeyEqualityComparer.GetGetHashCodeExpr();
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => GroupTemplate.Generate <TOuterKey, TSource, TInnerKey>(comparer, KeySelector, false));
errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
private bool CanGenerateColumnar()
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTSource = typeof(TSource);
var typeOfTInnerKey = typeof(TInnerKey);
if (!typeOfTSource.CanRepresentAsColumnar())
{
return(false);
}
if (typeOfTOuterKey.GetPartitionType() != null)
{
return(false);
}
if (typeOfTInnerKey.GetPartitionType() != null)
{
return(false);
}
var lookupKey = CacheKey.Create(KeySelector.ToString());
var comparer = (Properties.KeyEqualityComparer as CompoundGroupKeyEqualityComparer <TOuterKey, TInnerKey>).innerComparer.GetGetHashCodeExpr();
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => GroupTemplate.Generate <TOuterKey, TSource, TInnerKey>(comparer, KeySelector, true));
errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
protected override bool CanGenerateColumnar()
{
var typeOfTKey = typeof(TKey);
var typeOfTLeft = typeof(TLeft);
var typeOfTRight = typeof(TRight);
if (!typeOfTLeft.CanRepresentAsColumnar())
{
return(false);
}
if (!typeOfTRight.CanRepresentAsColumnar())
{
return(false);
}
if (typeOfTKey.GetPartitionType() != null)
{
return(false);
}
var lookupKey = CacheKey.Create(this.Properties.KeyEqualityComparer.GetEqualsExpr().ToString(), this.LeftComparer.GetEqualsExpr().ToString());
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => ClipJoinTemplate.Generate(this));
this.errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
protected override bool CanGenerateColumnar()
{
var typeOfTKey = typeof(TKey);
var typeOfTInput = typeof(TInput);
var typeOfTOutput = typeof(TOutput);
if (!typeOfTInput.CanRepresentAsColumnar())
{
return(false);
}
if (typeOfTKey.GetPartitionType() != null)
{
return(false);
}
if (!typeOfTOutput.CanRepresentAsColumnar())
{
return(false);
}
var lookupKey = CacheKey.Create(this.apt, GetAggregateFunctionsHashCode(this.Aggregate));
var tuple = cachedPipes.GetOrAdd(lookupKey, key => AggregateTemplate.Generate(this, this.apt));
var generatedPipeType = tuple.Item1;
this.errorMessages = tuple.Item2;
return(generatedPipeType != null);
}
protected override bool CanGenerateColumnar()
{
var typeOfTKey = typeof(TKey);
var typeOfTSource = typeof(TPayload);
var typeOfTResult = typeof(TResult);
if (!typeOfTSource.CanRepresentAsColumnar())
{
return(false);
}
if (!typeOfTResult.CanRepresentAsColumnar())
{
return(false);
}
if (typeOfTKey.GetPartitionType() != null)
{
return(false);
}
var lookupKey = CacheKey.Create(this.Selector.ExpressionToCSharp(), this.HasStartEdge, this.HasKey);
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => SelectTemplate.Generate(this));
this.errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
public static MemoryPool <TKey, TPayload> GetMemoryPool <TKey, TPayload>(bool isColumnar = true)
{
if (Config.ForceRowBasedExecution)
{
isColumnar = false;
}
var typeOfTKey = typeof(TKey);
var typeOfTPayload = typeof(TPayload);
var cacheKey = CacheKey.Create(typeof(TKey), typeof(TPayload), isColumnar);
if (!isColumnar ||
(!typeOfTKey.KeyTypeNeedsGeneratedMemoryPool() &&
typeOfTPayload.MemoryPoolHasGetMethodFor()))
{
return((MemoryPool <TKey, TPayload>)memoryPools.GetOrAdd(cacheKey, key => new MemoryPool <TKey, TPayload>(isColumnar)));
}
if (!typeOfTPayload.CanRepresentAsColumnar())
{
return((MemoryPool <TKey, TPayload>)memoryPools.GetOrAdd(cacheKey, new MemoryPool <TKey, TPayload>(false)));
}
var lookupKey = CacheKey.Create(typeOfTKey, typeOfTPayload);
Type generatedMemoryPool = cachedMemoryPools.GetOrAdd(lookupKey, key => Transformer.GenerateMemoryPoolClass <TKey, TPayload>());
return((MemoryPool <TKey, TPayload>)memoryPools.GetOrAdd(cacheKey, t => Activator.CreateInstance(generatedMemoryPool)));
}
private bool CanGenerateColumnar()
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTInnerKey = typeof(TInnerKey);
var typeOfTResult = typeof(TResult);
if (!typeOfTResult.CanRepresentAsColumnar())
{
return(false);
}
if (typeOfTOuterKey.GetPartitionType() != null)
{
return(false);
}
if (typeOfTInnerKey.GetPartitionType() != null)
{
return(false);
}
// For now, restrict the inner key to be anything other than an anonymous type since those can't be ungrouped without using reflection.
if (typeOfTInnerKey.IsAnonymousType())
{
return(false);
}
var lookupKey = CacheKey.Create(ResultSelector.ExpressionToCSharp());
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => UngroupTemplate.Generate <TOuterKey, TInnerKey, TInnerResult, TResult>(ResultSelector));
errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
protected override bool CanGenerateColumnar()
{
var lookupKey = CacheKey.Create();
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => SessionWindowTemplate.Generate(this));
this.errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
protected override bool CanGenerateColumnar()
{
if (!typeof(TResult).CanRepresentAsColumnar())
{
return(false);
}
var lookupKey = CacheKey.Create(this.joinKind, this.Properties.KeyEqualityComparer.GetEqualsExpr().ExpressionToCSharp(), this.Left.Properties.PayloadEqualityComparer.GetEqualsExpr().ExpressionToCSharp(), this.Right.Properties.PayloadEqualityComparer.GetEqualsExpr().ExpressionToCSharp(), this.Selector.ExpressionToCSharp());
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, this.columnarGenerator);
this.errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
protected override bool CanGenerateColumnar()
{
var typeOfTKey = typeof(TKey);
if (typeOfTKey.GetPartitionType() != null)
{
return(false);
}
// This operator can handle a pseudo-columnar payload type, e.g., string.
var lookupKey = CacheKey.Create(this.Properties.KeyEqualityComparer.GetEqualsExpr().ExpressionToCSharp(), this.Properties.PayloadEqualityComparer.GetEqualsExpr().ExpressionToCSharp());
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => BeatTemplate.Generate(this));
this.errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
private bool CanGenerateColumnar()
{
if (typeof(TPayload).IsAnonymousTypeName())
{
return(false);
}
if (!typeof(TPayload).GetTypeInfo().IsVisible)
{
return(false);
}
var lookupKey = CachedPipeLookupKey();
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => TemporalEgressTemplate.Generate(this));
this.errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
public static ColumnPool <T> GetColumnPool <T>(int size = -1)
{
if (size == -1)
{
size = Config.DataBatchSize;
}
return((ColumnPool <T>)columnPools.GetOrAdd(CacheKey.Create(typeof(T), size), new ColumnPool <T>(size)));
}
private bool CanGenerateColumnar()
{
if (typeof(TKey).IsAnonymousTypeName() || typeof(TPayload).IsAnonymousTypeName() || typeof(TResult).IsAnonymousTypeName())
{
return(false);
}
if (!typeof(TKey).GetTypeInfo().IsVisible || !typeof(TPayload).GetTypeInfo().IsVisible || !typeof(TResult).GetTypeInfo().IsVisible)
{
return(false);
}
var lookupKey = CacheKey.Create(this.constructor.Body.ExpressionToCSharp(), this.source.Properties.IsColumnar);
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => TemporalEgressTemplate.Generate(this));
this.errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
protected override bool CanGenerateColumnar()
{
var typeOfTKey = typeof(TKey);
var typeOfTPayload = typeof(TPayload);
if (typeOfTKey.GetPartitionType() != null)
{
return(false);
}
if (!typeOfTPayload.CanRepresentAsColumnar())
{
return(false);
}
var lookupKey = CacheKey.Create(this.duration);
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => ExtendLifetimeBaseTemplate.Generate(this, this.duration));
this.errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
protected override bool CanGenerateColumnar()
{
var typeOfTKey = typeof(TKey);
var typeOfTPayload = typeof(TPayload);
if (typeOfTKey.GetPartitionType() != null)
{
return(false);
}
if (!typeOfTPayload.CanRepresentAsColumnar())
{
return(false);
}
var lookupKey = CacheKey.Create(this.Properties.KeyEqualityComparer.GetEqualsExpr().ExpressionToCSharp(), this.Properties.PayloadEqualityComparer.GetEqualsExpr().ExpressionToCSharp());
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => StitchTemplate.Generate(this));
this.errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
internal override IStreamObserver <TKey, TPayload> CreatePipe(IStreamObserver <TKey, TPayload> observer)
{
var lookupKey = CacheKey.Create();
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => ColumnToRowTemplate.Generate(this));
Func <PlanNode, IQueryObject, PlanNode> planNode = ((PlanNode p, IQueryObject o) => new ColumnToRowPlanNode(p, o, typeof(TKey), typeof(TPayload)));
var instance = Activator.CreateInstance(generatedPipeType.Item1, this, observer, planNode);
var returnValue = (UnaryPipe <TKey, TPayload, TPayload>)instance;
return(returnValue);
}
protected override bool CanGenerateColumnar()
{
var typeOfTKey = typeof(TKey);
var typeOfTInput = typeof(TInput);
if (!typeOfTInput.CanRepresentAsColumnar())
{
return(false);
}
if (typeOfTKey.GetPartitionType() != null)
{
return(false);
}
var lookupKey = CacheKey.Create(this.KeySelector.ExpressionToCSharp(), this.ResultSelector.ExpressionToCSharp());
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => GroupedWindowTemplate.Generate(this));
this.errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
private UnaryPipe <TKey, TPayload, TPayload> GetPipe(IStreamObserver <TKey, TPayload> observer)
{
var lookupKey = CacheKey.Create();
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => QuantizeLifetimeTemplate.Generate(this));
Func <PlanNode, IQueryObject, PlanNode> planNode = ((PlanNode p, IQueryObject o) => new PointAtEndPlanNode(p, o, typeof(TKey), typeof(TPayload), true, generatedPipeType.Item2, false));
var instance = Activator.CreateInstance(generatedPipeType.Item1, this, observer, this.width, this.skip, this.offset, planNode);
var returnValue = (UnaryPipe <TKey, TPayload, TPayload>)instance;
return(returnValue);
}
internal static string GetBatchClassName(Type keyType, Type payloadType)
{
if (!payloadType.CanRepresentAsColumnar())
{
return($"StreamMessage<{keyType.GetCSharpSourceSyntax()}, {payloadType.GetCSharpSourceSyntax()}>");
}
var dictionaryKey = CacheKey.Create(keyType, payloadType);
return(batchType2Name.GetOrAdd(
dictionaryKey,
key => $"GeneratedBatch_{BatchClassSequenceNumber++}"));
}
private bool CanGenerateColumnar()
{
var typeOfTOuterKey = typeof(TOuterKey);
var typeOfTSource = typeof(TSource);
var typeOfTInnerKey = typeof(TInnerKey);
if (!typeOfTSource.CanRepresentAsColumnar())
{
return(false);
}
if (typeOfTOuterKey.GetPartitionType() != null)
{
return(false);
}
if (typeOfTInnerKey.GetPartitionType() != null)
{
return(false);
}
var keyEqComparer = Properties.KeyEqualityComparer;
string inlinedHashCodeComputation;
if (keyEqComparer is CompoundGroupKeyEqualityComparer <TOuterKey, TInnerKey> comparer)
{
var y = comparer.innerComparer.GetGetHashCodeExpr();
inlinedHashCodeComputation = y.Inline("key");
}
else
{
inlinedHashCodeComputation = keyEqComparer.GetGetHashCodeExpr().Inline("key");
}
var lookupKey = CacheKey.Create(inlinedHashCodeComputation, powerOf2);
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => ShuffleTemplate.Generate <TOuterKey, TSource, TInnerKey>(null, inlinedHashCodeComputation, false, this.powerOf2));
errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
protected override bool CanGenerateColumnar()
{
// This operator uses the equality method on payloads
if (this.Properties.IsColumnar && !this.LeftComparer.CanUsePayloadEquality())
{
this.errorMessages = $"The payload type, '{typeof(TLeft).FullName}', to Left Antisemijoin does not implement the interface {nameof(IEqualityComparerExpression<TLeft>)}. This interface is needed for code generation of this operator for columnar mode. Furthermore, the equality expression in the interface can only refer to input variables if used in field or property references.";
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new StreamProcessingException(this.errorMessages);
}
return(false);
}
var typeOfTKey = typeof(TKey);
var typeOfTLeft = typeof(TLeft);
var typeOfTRight = typeof(TRight);
if (!typeOfTLeft.CanRepresentAsColumnar())
{
return(false);
}
if (!typeOfTRight.CanRepresentAsColumnar())
{
return(false);
}
if (typeOfTKey.GetPartitionType() != null)
{
return(false);
}
var lookupKey = CacheKey.Create(this.Properties.KeyEqualityComparer.GetEqualsExpr().ToString(), this.LeftComparer.GetEqualsExpr().ToString());
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => LeftAntiSemiJoinTemplate.Generate(this));
this.errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
protected override bool CanGenerateColumnar()
{
// This operator uses the equality method on payloads
if (this.Left.Properties.IsColumnar && !this.Left.Properties.IsStartEdgeOnly && !this.Left.Properties.PayloadEqualityComparer.CanUsePayloadEquality())
{
this.errorMessages = $"The left input payload type, '{typeof(TLeft).FullName}', to Equijoin does not implement the interface {nameof(IEqualityComparerExpression<TLeft>)}. This interface is needed for code generation of this operator for columnar mode. Furthermore, the equality expression in the interface can only refer to input variables if used in field or property references.";
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new StreamProcessingException(this.errorMessages);
}
return(false);
}
// This operator uses the equality method on payloads
if (this.Right.Properties.IsColumnar && !this.Right.Properties.IsStartEdgeOnly && !this.Right.Properties.PayloadEqualityComparer.CanUsePayloadEquality())
{
this.errorMessages = $"The right input payload type, '{typeof(TRight).FullName}', to Equijoin does not implement the interface {nameof(IEqualityComparerExpression<TRight>)}. This interface is needed for code generation of this operator for columnar mode. Furthermore, the equality expression in the interface can only refer to input variables if used in field or property references.";
if (Config.CodegenOptions.DontFallBackToRowBasedExecution)
{
throw new StreamProcessingException(this.errorMessages);
}
return(false);
}
if (!typeof(TResult).CanRepresentAsColumnar())
{
return(false);
}
var lookupKey = CacheKey.Create(this.joinKind, this.Properties.KeyEqualityComparer.GetEqualsExpr().ExpressionToCSharp(), this.Left.Properties.PayloadEqualityComparer.GetEqualsExpr().ExpressionToCSharp(), this.Right.Properties.PayloadEqualityComparer.GetEqualsExpr().ExpressionToCSharp(), this.Selector.ExpressionToCSharp());
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, this.columnarGenerator);
this.errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
private BinaryPipe <TKey, TPayload, TPayload, TPayload> GetPipe(IStreamObserver <TKey, TPayload> observer)
{
var lookupKey = CacheKey.Create();
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => UnionTemplate.GenerateUnionPipeClass(this));
Func <PlanNode, PlanNode, IBinaryObserver, BinaryPlanNode> planNode = ((PlanNode left, PlanNode right, IBinaryObserver o) =>
{
var node = new UnionPlanNode(
left, right, o, typeof(TKey), typeof(TPayload), false, true, this.ErrorMessages);
return(node);
});
var instance = Activator.CreateInstance(generatedPipeType.Item1, this, observer, planNode);
var returnValue = (BinaryPipe <TKey, TPayload, TPayload, TPayload>)instance;
return(returnValue);
}
public static Type GetStreamMessageType <TKey, TPayload>()
{
if (Config.ForceRowBasedExecution)
{
return(typeof(StreamMessage <TKey, TPayload>));
}
if (!typeof(TPayload).CanRepresentAsColumnar())
{
return(typeof(StreamMessage <TKey, TPayload>));
}
var typeOfTKey = typeof(TKey);
var typeOfTPayload = typeof(TPayload);
var lookupKey = CacheKey.Create(typeOfTKey, typeOfTPayload);
return(cachedObjects.GetOrAdd(lookupKey, key => Transformer.GenerateBatchClass <TKey, TPayload>()));
}
internal static ColumnPool <long> GetBVPool(int size)
{
return((ColumnPool <long>)bitvectorPools.GetOrAdd(CacheKey.Create(size), new ColumnPool <long>(size)));
}
internal static ColumnPool <long> GetBVPool(int size) => bitvectorPools.GetOrAdd(CacheKey.Create(size), key => new ColumnPool <long>(size));
internal static DoublingArrayPool <T> GetDoublingArrayPool <T>()
{
return((DoublingArrayPool <T>)doublingArrayPools.GetOrAdd(CacheKey.Create(typeof(T)), new DoublingArrayPool <T>()));
}
internal static CharArrayPool GetCharArrayPool()
{
return((CharArrayPool)charArrayPools.GetOrAdd(CacheKey.Create(), new CharArrayPool()));
}
internal static StreamMessagePool <TKey, TPayload> GetStreamMessagePool <TKey, TPayload>(MemoryPool <TKey, TPayload> memoryPool, bool isColumnar)
{
return((StreamMessagePool <TKey, TPayload>)eventBatchPools.GetOrAdd(CacheKey.Create(typeof(TKey), typeof(TPayload), isColumnar), new StreamMessagePool <TKey, TPayload>(memoryPool, isColumnar)));
}
private bool CanGenerateUngroupedAFAPipe()
{
if (!typeof(TPayload).CanRepresentAsColumnar())
{
return(false);
}
if (!typeof(TRegister).CanRepresentAsColumnar())
{
return(false);
}
var lookupKey = CacheKey.Create((object)this.afa);
var downcast = this as AfaStreamable <Empty, TPayload, TRegister, TAccumulator>;
var generatedPipeType = cachedPipes.GetOrAdd(lookupKey, key => UngroupedAfaTemplate.GenerateAFA(downcast));
this.errorMessages = generatedPipeType.Item2;
return(generatedPipeType.Item1 != null);
}
