public PartitionedSnapshotWindowHoppingPipe(
SnapshotWindowStreamable <TKey, TInput, TState, TOutput> stream,
IStreamObserver <TKey, TOutput> observer)
: base(stream, observer)
{
this.aggregate = stream.Aggregate;
this.initialStateExpr = this.aggregate.InitialState();
this.initialState = this.initialStateExpr.Compile();
this.accumulateExpr = this.aggregate.Accumulate();
this.accumulate = this.accumulateExpr.Compile();
this.deaccumulateExpr = this.aggregate.Deaccumulate();
this.deaccumulate = this.deaccumulateExpr.Compile();
this.differenceExpr = this.aggregate.Difference();
this.difference = this.differenceExpr.Compile();
this.computeResultExpr = this.aggregate.ComputeResult();
this.computeResult = this.computeResultExpr.Compile();
var comparer = stream.Properties.KeyEqualityComparer;
this.keyComparerEqualsExpr = comparer.GetEqualsExpr();
this.keyComparerEquals = this.keyComparerEqualsExpr.Compile();
this.keyComparerGetHashCodeExpr = comparer.GetGetHashCodeExpr();
this.keyComparerGetHashCode = this.keyComparerGetHashCodeExpr.Compile();
this.errorMessages = stream.ErrorMessages;
this.pool = MemoryManager.GetMemoryPool <TKey, TOutput>(false);
this.pool.Get(out this.batch);
this.batch.Allocate();
this.aggregateByKey = comparer.CreateFastDictionary2Generator <TKey, HeldState <TState> >(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Properties.QueryContainer).Invoke();
var stateDictGenerator = comparer.CreateFastDictionaryGenerator <TKey, StateAndActive <TState> >(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Properties.QueryContainer);
this.ecqEntryPool = new DataStructurePool <FastDictionary <TKey, StateAndActive <TState> > >(() => stateDictGenerator.Invoke());
this.hopsPerDuration = (int)(stream.Source.Properties.ConstantDurationLength.Value / stream.Source.Properties.ConstantHopLength) + 1;
}
public DelayedSubscription(
StreamMessageIngressBase <TKey, TPayload> current,
IStreamObserver <TKey, TPayload> observer)
{
this.current = current;
this.observer = observer;
}
public override IDisposable Subscribe(IStreamObserver <TInnerKey, TSource> observer)
{
if (this.totalBranchesL2 <= 1)
{
return(this.Source.Subscribe(observer));
}
this.numBranches++;
if (this.pipe == null)
{
this.pipe = CreatePipe(observer);
}
var o = observer;
this.pipe.AddObserver(o);
var d = o as IDisposable;
if (this.numBranches < this.totalBranchesL2)
{
return(d ?? Utility.EmptyDisposable);
}
else
{
// Reset status for next set of subscribe calls
var oldpipe = this.pipe;
this.pipe = null;
this.numBranches = 0;
return(d == null
? this.Source.Subscribe(oldpipe)
: Utility.CreateDisposable(this.Source.Subscribe(oldpipe), d));
}
}
public PartitionedSnapshotWindowStartEdgePipe(
SnapshotWindowStreamable <TKey, TInput, TState, TOutput> stream,
IStreamObserver <TKey, TOutput> observer)
: base(stream, observer)
{
this.aggregate = stream.Aggregate;
this.initialStateExpr = this.aggregate.InitialState();
this.initialState = this.initialStateExpr.Compile();
this.accumulateExpr = this.aggregate.Accumulate();
this.accumulate = this.accumulateExpr.Compile();
this.computeResultExpr = this.aggregate.ComputeResult();
this.computeResult = this.computeResultExpr.Compile();
var comparer = stream.Properties.KeyEqualityComparer;
this.keyComparerEqualsExpr = comparer.GetEqualsExpr();
this.keyComparerEquals = this.keyComparerEqualsExpr.Compile();
this.keyComparerGetHashCodeExpr = comparer.GetGetHashCodeExpr();
this.keyComparerGetHashCode = this.keyComparerGetHashCodeExpr.Compile();
this.errorMessages = stream.ErrorMessages;
this.pool = MemoryManager.GetMemoryPool <TKey, TOutput>(false);
this.pool.Get(out this.batch);
this.batch.Allocate();
var generator = comparer.CreateFastDictionary2Generator <TKey, HeldState <TState> >(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Properties.QueryContainer);
this.aggregateByKey = generator.Invoke();
}
public StitchAggregatePipe(StitchAggregateStreamable <TKey, TPayload, TState, TResult> stream,
IStreamObserver <TKey, TResult> observer)
: base(stream, observer)
{
this.errorMessages = stream.ErrorMessages;
this.pool = MemoryManager.GetMemoryPool <TKey, TResult>(stream.Properties.IsColumnar);
this.pool.Get(out this.batch);
this.batch.Allocate();
this.aggregate = stream.aggregate;
this.initialState = this.aggregate.InitialState().Compile();
this.accumulate = this.aggregate.Accumulate().Compile();
this.computeResult = this.aggregate.ComputeResult().Compile();
var khpcomparer = EqualityComparerExtensions.GetEqualityComparerExpression <KHP, TKey>(
e => e.Key, stream.Properties.KeyEqualityComparer);
var equals = khpcomparer.GetEqualsExpr().Compile();
var getHashCode = khpcomparer.GetGetHashCodeExpr().Compile();
var generator = khpcomparer.CreateFastDictionary2Generator <KHP, List <ActiveEvent> >(1, equals, getHashCode, stream.Properties.QueryContainer);
this.dictPool = new DataStructurePool <FastDictionary2 <KHP, List <ActiveEvent> > >(() => generator.Invoke());
this.outputCount = 0;
this.CurrentTimeOpenEventBuffer = khpcomparer.CreateFastDictionary2Generator <KHP, int>(1, equals, getHashCode, stream.Properties.QueryContainer).Invoke();
this.OpenEvents = khpcomparer.CreateFastDictionary2Generator <KHP, List <ActiveEventExt> >(1, equals, getHashCode, stream.Properties.QueryContainer).Invoke();
}
protected CompiledAfaPipeBase(IStreamable <TKey, TRegister> stream, IStreamObserver <TKey, TRegister> observer, object afa, long maxDuration, bool isGenerated)
: base(stream, observer)
{
var compiledAfa = (CompiledAfa <TPayload, TRegister, TAccumulator>)afa;
this.isFinal = compiledAfa.isFinal;
this.hasOutgoingArcs = compiledAfa.hasOutgoingArcs;
this.singleEventStateMap = compiledAfa.singleEventStateMap;
this.eventListStateMap = compiledAfa.eventListStateMap;
this.multiEventStateMap = compiledAfa.multiEventStateMap;
this.epsilonStateMap = compiledAfa.epsilonStateMap;
this.startStates = compiledAfa.startStates;
this.numStartStates = compiledAfa.numStartStates;
this.defaultRegister = compiledAfa.defaultRegister;
this.AllowOverlappingInstances = compiledAfa.uncompiledAfa.AllowOverlappingInstances;
this.IsDeterministic = compiledAfa.uncompiledAfa.IsDeterministic;
this.MaxDuration = maxDuration;
this.IsSyncTimeSimultaneityFree = stream.Properties.IsSyncTimeSimultaneityFree;
this.pool = MemoryManager.GetMemoryPool <TKey, TRegister>(stream.Properties.IsColumnar);
this.pool.Get(out this.batch);
this.batch.Allocate();
this.batch.iter = 0;
this.iter = 0;
this.errorMessages = stream.ErrorMessages;
this.keyEqualityComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr().Compile();
}
public override IDisposable Subscribe(IStreamObserver <TKey, TPayload> observer)
{
Contract.EnsuresOnThrow <IngressException>(true);
this.container?.RegisterIngressPipe(this.IngressSiteIdentifier, this);
return(base.Subscribe(observer));
}
public SnapshotWindowHoppingPipeSimple(
SnapshotWindowStreamable <Empty, TInput, TState, TOutput> stream,
IStreamObserver <Empty, TOutput> observer)
: base(stream, observer)
{
this.aggregate = stream.Aggregate;
this.initialStateExpr = this.aggregate.InitialState();
this.initialState = this.initialStateExpr.Compile();
this.accumulateExpr = this.aggregate.Accumulate();
this.accumulate = this.accumulateExpr.Compile();
this.deaccumulateExpr = this.aggregate.Deaccumulate();
this.deaccumulate = this.deaccumulateExpr.Compile();
this.differenceExpr = this.aggregate.Difference();
this.difference = this.differenceExpr.Compile();
this.computeResultExpr = this.aggregate.ComputeResult();
this.computeResult = this.computeResultExpr.Compile();
this.errorMessages = stream.ErrorMessages;
this.pool = MemoryManager.GetMemoryPool <Empty, TOutput>(false);
this.pool.Get(out this.batch);
this.batch.Allocate();
var hopsPerDuration = (int)(stream.Source.Properties.ConstantDurationLength.Value / stream.Source.Properties.ConstantHopLength) + 1;
this.ecq = new CircularBuffer <HeldState <TState> >(hopsPerDuration);
}
public PartitionedFixedIntervalEquiJoinPipeCompound(
EquiJoinStreamable <CompoundGroupKey <PartitionKey <TPartitionKey>, TGroupKey>, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector,
IStreamObserver <CompoundGroupKey <PartitionKey <TPartitionKey>, TGroupKey>, TResult> observer)
: base(stream, observer)
{
this.selector = selector.Compile();
this.leftDuration = stream.Left.Properties.ConstantDurationLength.Value;
this.rightDuration = stream.Right.Properties.ConstantDurationLength.Value;
this.keyComparer = EqualityComparerExpression <TGroupKey> .Default.GetEqualsExpr();
this.keyComparerEquals = this.keyComparer.Compile();
if (this.leftDuration == this.rightDuration)
{
this.endpointGenerator = () => new EndPointQueue();
}
else
{
this.endpointGenerator = () => new EndPointHeap();
}
this.pool = MemoryManager.GetMemoryPool <CompoundGroupKey <PartitionKey <TPartitionKey>, TGroupKey>, TResult>(stream.Properties.IsColumnar);
this.pool.Get(out this.output);
this.output.Allocate();
}
public IntervalArraySubscriptionThrowNone(
IObservable <ArraySegment <TPayload> > observable,
Expression <Func <TPayload, long> > startEdgeExtractor,
Expression <Func <TPayload, long> > endEdgeExtractor,
string identifier,
Streamable <Empty, TPayload> streamable,
IStreamObserver <Empty, TPayload> observer,
OnCompletedPolicy onCompletedPolicy,
IObserver <OutOfOrderStreamEvent <TPayload> > diagnosticOutput)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Throw(),
FlushPolicy.FlushOnPunctuation,
PeriodicPunctuationPolicy.None(),
onCompletedPolicy,
diagnosticOutput)
{
this.startEdgeExtractor = startEdgeExtractor;
this.startEdgeFunction = startEdgeExtractor.Compile();
this.endEdgeExtractor = endEdgeExtractor;
this.endEdgeFunction = endEdgeExtractor.Compile();
}
public PartitionedIntervalArraySubscriptionThrowNone(
IObservable <ArraySegment <TPayload> > observable,
Expression <Func <TPayload, TKey> > partitionExtractor,
Expression <Func <TPayload, long> > startEdgeExtractor,
Expression <Func <TPayload, long> > endEdgeExtractor,
string identifier,
Streamable <PartitionKey <TKey>, TPayload> streamable,
IStreamObserver <PartitionKey <TKey>, TPayload> observer,
OnCompletedPolicy onCompletedPolicy,
IObserver <OutOfOrderPartitionedStreamEvent <TKey, TPayload> > diagnosticOutput)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Throw(),
PartitionedFlushPolicy.FlushOnLowWatermark,
PeriodicPunctuationPolicy.None(),
PeriodicLowWatermarkPolicy.None(),
onCompletedPolicy,
diagnosticOutput)
{
this.partitionExtractor = partitionExtractor;
this.partitionFunction = partitionExtractor.Compile();
this.startEdgeExtractor = startEdgeExtractor;
this.startEdgeFunction = startEdgeExtractor.Compile();
this.endEdgeExtractor = endEdgeExtractor;
this.endEdgeFunction = endEdgeExtractor.Compile();
}
private IDisposable Subscribe(IStreamObserver <TKey, TSource> observer, int index)
{
IDisposable child;
lock (this.subscriptionLock)
{
if (this.toSubscribe.Add(index))
{
child = new ChildDisposable(this.connectableStream.Subscribe(observer), this.crew, index);
}
else
{
throw new InvalidOperationException("Cannot subscribe to the same child streamable more than once.");
}
if (this.toSubscribe.Count == this.outputCount)
{
this.crew.SetListDisposable(this.connectableStream.Connect());
this.crew = new DisposableManager(this.outputCount);
this.connectableStream = new ConnectableStreamable <TKey, TSource>(this.source);
this.toSubscribe.Clear();
}
}
return(child);
}
public PartitionedStitchPipe(IStreamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer)
: base(stream, observer)
{
this.errorMessages = stream.ErrorMessages;
this.outputCount = 0;
this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar);
this.pool.Get(out this.batch);
this.batch.Allocate();
var khpcomparer = EqualityComparerExtensions.GetCompoundEqualityComparerExpression <KHP, TKey, TPayload>(
e => e.Key, stream.Properties.KeyEqualityComparer,
e => e.Payload, stream.Properties.PayloadEqualityComparer);
var equals = khpcomparer.GetEqualsExpr().Compile();
var getHashCode = khpcomparer.GetGetHashCodeExpr().Compile();
var generator1 = khpcomparer.CreateFastDictionary2Generator <KHP, List <ActiveEvent> >(1, equals, getHashCode, stream.Properties.QueryContainer);
this.dictPool = new DataStructurePool <FastDictionary2 <KHP, List <ActiveEvent> > >(() => generator1.Invoke());
var generator2 = khpcomparer.CreateFastDictionary2Generator <KHP, int>(1, equals, getHashCode, stream.Properties.QueryContainer);
this.CurrentTimeOpenEventBufferGenerator = () => generator2.Invoke();
var generator3 = khpcomparer.CreateFastDictionary2Generator <KHP, List <ActiveEventExt> >(1, equals, getHashCode, stream.Properties.QueryContainer);
this.OpenEventsGenerator = () => generator3.Invoke();
}
internal override IStreamObserver <TKey, TPayload> CreatePipe(IStreamObserver <TKey, TPayload> observer)
{
var t = typeof(TKey).GetPartitionType();
if (t == null)
{
if (this.Source.Properties.IsColumnar)
{
return(GetPipe(observer));
}
if (typeof(TKey) == typeof(Empty))
{
return(new SessionWindowPipeStateless <TKey, TPayload>(this, observer, this.sessionTimeout, this.maximumDuration));
}
else
{
return(new SessionWindowPipe <TKey, TPayload>(this, observer, this.sessionTimeout, this.maximumDuration));
}
}
var outputType = typeof(PartitionedSessionWindowPipe <, ,>).MakeGenericType(
typeof(TKey),
typeof(TPayload),
t);
return((IStreamObserver <TKey, TPayload>)Activator.CreateInstance(outputType, this, observer, this.sessionTimeout, this.maximumDuration));
}
private void Ingress(IStreamObserver <TKey, TPayload> observer)
{
bool done = false;
int messages = 0;
try
{
var serializer = StreamableSerializer.Create <QueuedMessage <StreamMessage <TKey, TPayload> > >(new SerializerSettings());
while (!done)
{
var message = serializer.Deserialize(this.stream);
if (message.Kind == MessageKind.Completed)
{
done = true;
}
observer.OnNext(message.Message);
messages++;
if (this.numMessages != 0 && messages == this.numMessages)
{
if (message.Kind != MessageKind.Completed)
{
observer.OnCompleted();
break;
}
}
}
}
catch (Exception e)
{
observer.OnError(e);
}
}
private void Ingress(IStreamObserver <TKey, TPayload> observer)
{
int messages = 0;
try
{
var serializer = StreamableSerializer.Create <QueuedMessage <StreamMessage <TKey, TPayload> > >(
new SerializerSettings()
{
KnownTypes = StreamMessageManager.GeneratedTypes()
});
while (true)
{
var message = serializer.Deserialize(this.stream);
if (message.Kind != MessageKind.Completed)
{
observer.OnNext(message.Message);
messages++;
}
if (message.Kind == MessageKind.Completed ||
(this.numMessages != 0 && messages == this.numMessages))
{
observer.OnCompleted();
break;
}
}
}
catch (Exception e)
{
observer.OnError(e);
}
this.onSubscriptionCompleted();
}
public override IDisposable Subscribe(IStreamObserver <PartitionKey <TPartitionKey>, TPayload> observer)
{
Contract.EnsuresOnThrow <IngressException>(true);
var subscription = PartitionedIntervalArraySubscriptionCreator <TPartitionKey, TPayload> .CreateSubscription(
this.observable,
this.partitionExtractor,
this.startEdgeExtractor,
this.endEdgeExtractor,
this.IngressSiteIdentifier,
this,
observer,
onCompletedPolicy,
diagnosticOutput);
if (delayed)
{
container.RegisterIngressPipe(this.IngressSiteIdentifier, subscription);
return(subscription.DelayedDisposable);
}
else
{
subscription.Enable();
return(subscription);
}
}
public override IDisposable Subscribe(IStreamObserver <Empty, TPayload> observer)
{
Contract.EnsuresOnThrow <IngressException>(true);
IIngressStreamObserver subscription;
if (this.timelinePolicy.timelineEnum == TimelineEnum.WallClock)
{
subscription = new MonotonicSubscriptionWallClock <TPayload>(this.observable, this.IngressSiteIdentifier,
this,
observer, this.flushPolicy, this.punctuationPolicy, this.onCompletedPolicy, this.timelinePolicy);
}
else
{
subscription = new MonotonicSubscriptionSequence <TPayload>(this.observable, this.IngressSiteIdentifier,
this,
observer, this.flushPolicy, this.punctuationPolicy, this.onCompletedPolicy, this.timelinePolicy);
}
if (this.delayed)
{
this.container.RegisterIngressPipe(this.IngressSiteIdentifier, subscription);
return(subscription.DelayedDisposable);
}
else
{
subscription.Enable();
return(subscription);
}
}
public MonotonicSubscriptionWallClock(
IObservable <TPayload> observable,
string identifier,
Streamable <Empty, TPayload> streamable,
IStreamObserver <Empty, TPayload> observer,
FlushPolicy flushPolicy,
PeriodicPunctuationPolicy punctuationPolicy,
OnCompletedPolicy onCompletedPolicy,
TimelinePolicy timelinePolicy)
: base(
observable,
identifier,
streamable,
observer,
DisorderPolicy.Adjust(),
flushPolicy,
punctuationPolicy,
onCompletedPolicy,
null)
{
if (timelinePolicy.punctuationInterval > default(TimeSpan))
{
this.timer = new Timer(EmitPunctuation, null, new TimeSpan(0), timelinePolicy.punctuationInterval);
}
}
public FixedIntervalEquiJoinPipe(
BinaryStreamable <TKey, TLeft, TRight, TResult> stream,
Expression <Func <TLeft, TRight, TResult> > selector,
IStreamObserver <TKey, TResult> observer)
: base(stream, observer)
{
this.selector = selector.Compile();
this.leftDuration = stream.Left.Properties.ConstantDurationLength.Value;
this.rightDuration = stream.Right.Properties.ConstantDurationLength.Value;
this.keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr();
this.keyComparerEquals = this.keyComparer.Compile();
if (this.leftDuration == this.rightDuration)
{
this.endPointHeap = new EndPointQueue();
}
else
{
this.endPointHeap = new EndPointHeap();
}
this.pool = MemoryManager.GetMemoryPool <TKey, TResult>(stream.Properties.IsColumnar);
this.pool.Get(out this.output);
this.output.Allocate();
}
public WrapperStreamObserver(IStreamObserver <TK, TP> o, OwnedThreadsScheduler scheduler, Guid classId, TaskEntry t)
{
this.o = o;
this.scheduler = scheduler;
this.classId = classId;
this.te = t;
}
private IStreamObserver <TOuterKey, TSource> CreatePipe(IStreamObserver <CompoundGroupKey <TOuterKey, TInnerKey>, TSource> observer)
{
if (typeof(TOuterKey).GetPartitionType() == null)
{
return(new GroupNestedPipe <TOuterKey, TSource, TInnerKey>(this, observer));
}
return(new PartitionedGroupNestedPipe <TOuterKey, TSource, TInnerKey>(this, observer));
}
public override IDisposable Subscribe(IStreamObserver <CompoundGroupKey <TOuterKey, TInnerKey>, TSource> observer)
{
var pipe = Properties.IsColumnar
? GetPipe(observer)
: CreatePipe(observer);
return(Source.Subscribe(pipe));
}
public SelectPipe(SelectStreamable <TKey, TPayload, TResult> stream, IStreamObserver <TKey, TResult> observer)
: base(stream, observer)
{
this.selector = (Expression <Func <TPayload, TResult> >)stream.Selector;
this.selectorFunc = this.selector.Compile();
this.pool = MemoryManager.GetMemoryPool <TKey, TResult>(stream.Properties.IsColumnar);
this.errorMessages = stream.ErrorMessages;
}
private Tuple <Type, string> GetPipe(IStreamObserver <TKey, TOutput> observer)
{
var lookupKey = CacheKey.Create(this.apt, GetAggregateFunctionsHashCode(this.Aggregate));
var tuple = cachedPipes.GetOrAdd(lookupKey, key => AggregateTemplate.Generate(this, this.apt));
return(tuple);
}
public StatelessQuantizeLifetimePipe(IStreamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer, long width, long skip, long offset)
: base(stream, observer)
{
this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar);
this.width = width;
this.skip = skip;
this.offset = offset;
}
public UnionPipe(Streamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer)
: base(stream, observer)
{
this.errorMessages = stream.ErrorMessages;
this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar);
this.pool.Get(out this.output);
this.output.Allocate();
}
public StatelessPointAtEndPipe(IStreamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer, long constantDuration)
: base(stream, observer)
{
this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar);
this.errorMessages = stream.ErrorMessages;
this.constantDuration = constantDuration;
this.isPartitioned = typeof(TKey).GetPartitionType() != null;
}
public ExtendLifetimePipe(ExtendLifetimeStreamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer, long duration)
: base(stream, observer)
{
this.duration = duration;
this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar);
this.errorMessages = stream.ErrorMessages;
this.pool.Get(out this.output);
this.output.Allocate();
}
public ClipByConstantPipe(ClipByConstantStreamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer, long limit)
: base(stream, observer)
{
this.limit = limit;
this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar);
this.errorMessages = stream.ErrorMessages;
this.pool.Get(out this.output);
this.output.Allocate();
}
