public PartitionedSnapshotWindowHoppingPipe( SnapshotWindowStreamable <TKey, TInput, TState, TOutput> stream, IStreamObserver <TKey, TOutput> observer) : base(stream, observer) { this.aggregate = stream.Aggregate; this.initialStateExpr = this.aggregate.InitialState(); this.initialState = this.initialStateExpr.Compile(); this.accumulateExpr = this.aggregate.Accumulate(); this.accumulate = this.accumulateExpr.Compile(); this.deaccumulateExpr = this.aggregate.Deaccumulate(); this.deaccumulate = this.deaccumulateExpr.Compile(); this.differenceExpr = this.aggregate.Difference(); this.difference = this.differenceExpr.Compile(); this.computeResultExpr = this.aggregate.ComputeResult(); this.computeResult = this.computeResultExpr.Compile(); var comparer = stream.Properties.KeyEqualityComparer; this.keyComparerEqualsExpr = comparer.GetEqualsExpr(); this.keyComparerEquals = this.keyComparerEqualsExpr.Compile(); this.keyComparerGetHashCodeExpr = comparer.GetGetHashCodeExpr(); this.keyComparerGetHashCode = this.keyComparerGetHashCodeExpr.Compile(); this.errorMessages = stream.ErrorMessages; this.pool = MemoryManager.GetMemoryPool <TKey, TOutput>(false); this.pool.Get(out this.batch); this.batch.Allocate(); this.aggregateByKey = comparer.CreateFastDictionary2Generator <TKey, HeldState <TState> >(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Properties.QueryContainer).Invoke(); var stateDictGenerator = comparer.CreateFastDictionaryGenerator <TKey, StateAndActive <TState> >(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Properties.QueryContainer); this.ecqEntryPool = new DataStructurePool <FastDictionary <TKey, StateAndActive <TState> > >(() => stateDictGenerator.Invoke()); this.hopsPerDuration = (int)(stream.Source.Properties.ConstantDurationLength.Value / stream.Source.Properties.ConstantHopLength) + 1; }
public DelayedSubscription( StreamMessageIngressBase <TKey, TPayload> current, IStreamObserver <TKey, TPayload> observer) { this.current = current; this.observer = observer; }
public override IDisposable Subscribe(IStreamObserver <TInnerKey, TSource> observer) { if (this.totalBranchesL2 <= 1) { return(this.Source.Subscribe(observer)); } this.numBranches++; if (this.pipe == null) { this.pipe = CreatePipe(observer); } var o = observer; this.pipe.AddObserver(o); var d = o as IDisposable; if (this.numBranches < this.totalBranchesL2) { return(d ?? Utility.EmptyDisposable); } else { // Reset status for next set of subscribe calls var oldpipe = this.pipe; this.pipe = null; this.numBranches = 0; return(d == null ? this.Source.Subscribe(oldpipe) : Utility.CreateDisposable(this.Source.Subscribe(oldpipe), d)); } }
public PartitionedSnapshotWindowStartEdgePipe( SnapshotWindowStreamable <TKey, TInput, TState, TOutput> stream, IStreamObserver <TKey, TOutput> observer) : base(stream, observer) { this.aggregate = stream.Aggregate; this.initialStateExpr = this.aggregate.InitialState(); this.initialState = this.initialStateExpr.Compile(); this.accumulateExpr = this.aggregate.Accumulate(); this.accumulate = this.accumulateExpr.Compile(); this.computeResultExpr = this.aggregate.ComputeResult(); this.computeResult = this.computeResultExpr.Compile(); var comparer = stream.Properties.KeyEqualityComparer; this.keyComparerEqualsExpr = comparer.GetEqualsExpr(); this.keyComparerEquals = this.keyComparerEqualsExpr.Compile(); this.keyComparerGetHashCodeExpr = comparer.GetGetHashCodeExpr(); this.keyComparerGetHashCode = this.keyComparerGetHashCodeExpr.Compile(); this.errorMessages = stream.ErrorMessages; this.pool = MemoryManager.GetMemoryPool <TKey, TOutput>(false); this.pool.Get(out this.batch); this.batch.Allocate(); var generator = comparer.CreateFastDictionary2Generator <TKey, HeldState <TState> >(1, this.keyComparerEquals, this.keyComparerGetHashCode, stream.Properties.QueryContainer); this.aggregateByKey = generator.Invoke(); }
public StitchAggregatePipe(StitchAggregateStreamable <TKey, TPayload, TState, TResult> stream, IStreamObserver <TKey, TResult> observer) : base(stream, observer) { this.errorMessages = stream.ErrorMessages; this.pool = MemoryManager.GetMemoryPool <TKey, TResult>(stream.Properties.IsColumnar); this.pool.Get(out this.batch); this.batch.Allocate(); this.aggregate = stream.aggregate; this.initialState = this.aggregate.InitialState().Compile(); this.accumulate = this.aggregate.Accumulate().Compile(); this.computeResult = this.aggregate.ComputeResult().Compile(); var khpcomparer = EqualityComparerExtensions.GetEqualityComparerExpression <KHP, TKey>( e => e.Key, stream.Properties.KeyEqualityComparer); var equals = khpcomparer.GetEqualsExpr().Compile(); var getHashCode = khpcomparer.GetGetHashCodeExpr().Compile(); var generator = khpcomparer.CreateFastDictionary2Generator <KHP, List <ActiveEvent> >(1, equals, getHashCode, stream.Properties.QueryContainer); this.dictPool = new DataStructurePool <FastDictionary2 <KHP, List <ActiveEvent> > >(() => generator.Invoke()); this.outputCount = 0; this.CurrentTimeOpenEventBuffer = khpcomparer.CreateFastDictionary2Generator <KHP, int>(1, equals, getHashCode, stream.Properties.QueryContainer).Invoke(); this.OpenEvents = khpcomparer.CreateFastDictionary2Generator <KHP, List <ActiveEventExt> >(1, equals, getHashCode, stream.Properties.QueryContainer).Invoke(); }
protected CompiledAfaPipeBase(IStreamable <TKey, TRegister> stream, IStreamObserver <TKey, TRegister> observer, object afa, long maxDuration, bool isGenerated) : base(stream, observer) { var compiledAfa = (CompiledAfa <TPayload, TRegister, TAccumulator>)afa; this.isFinal = compiledAfa.isFinal; this.hasOutgoingArcs = compiledAfa.hasOutgoingArcs; this.singleEventStateMap = compiledAfa.singleEventStateMap; this.eventListStateMap = compiledAfa.eventListStateMap; this.multiEventStateMap = compiledAfa.multiEventStateMap; this.epsilonStateMap = compiledAfa.epsilonStateMap; this.startStates = compiledAfa.startStates; this.numStartStates = compiledAfa.numStartStates; this.defaultRegister = compiledAfa.defaultRegister; this.AllowOverlappingInstances = compiledAfa.uncompiledAfa.AllowOverlappingInstances; this.IsDeterministic = compiledAfa.uncompiledAfa.IsDeterministic; this.MaxDuration = maxDuration; this.IsSyncTimeSimultaneityFree = stream.Properties.IsSyncTimeSimultaneityFree; this.pool = MemoryManager.GetMemoryPool <TKey, TRegister>(stream.Properties.IsColumnar); this.pool.Get(out this.batch); this.batch.Allocate(); this.batch.iter = 0; this.iter = 0; this.errorMessages = stream.ErrorMessages; this.keyEqualityComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr().Compile(); }
public override IDisposable Subscribe(IStreamObserver <TKey, TPayload> observer) { Contract.EnsuresOnThrow <IngressException>(true); this.container?.RegisterIngressPipe(this.IngressSiteIdentifier, this); return(base.Subscribe(observer)); }
public SnapshotWindowHoppingPipeSimple( SnapshotWindowStreamable <Empty, TInput, TState, TOutput> stream, IStreamObserver <Empty, TOutput> observer) : base(stream, observer) { this.aggregate = stream.Aggregate; this.initialStateExpr = this.aggregate.InitialState(); this.initialState = this.initialStateExpr.Compile(); this.accumulateExpr = this.aggregate.Accumulate(); this.accumulate = this.accumulateExpr.Compile(); this.deaccumulateExpr = this.aggregate.Deaccumulate(); this.deaccumulate = this.deaccumulateExpr.Compile(); this.differenceExpr = this.aggregate.Difference(); this.difference = this.differenceExpr.Compile(); this.computeResultExpr = this.aggregate.ComputeResult(); this.computeResult = this.computeResultExpr.Compile(); this.errorMessages = stream.ErrorMessages; this.pool = MemoryManager.GetMemoryPool <Empty, TOutput>(false); this.pool.Get(out this.batch); this.batch.Allocate(); var hopsPerDuration = (int)(stream.Source.Properties.ConstantDurationLength.Value / stream.Source.Properties.ConstantHopLength) + 1; this.ecq = new CircularBuffer <HeldState <TState> >(hopsPerDuration); }
public PartitionedFixedIntervalEquiJoinPipeCompound( EquiJoinStreamable <CompoundGroupKey <PartitionKey <TPartitionKey>, TGroupKey>, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector, IStreamObserver <CompoundGroupKey <PartitionKey <TPartitionKey>, TGroupKey>, TResult> observer) : base(stream, observer) { this.selector = selector.Compile(); this.leftDuration = stream.Left.Properties.ConstantDurationLength.Value; this.rightDuration = stream.Right.Properties.ConstantDurationLength.Value; this.keyComparer = EqualityComparerExpression <TGroupKey> .Default.GetEqualsExpr(); this.keyComparerEquals = this.keyComparer.Compile(); if (this.leftDuration == this.rightDuration) { this.endpointGenerator = () => new EndPointQueue(); } else { this.endpointGenerator = () => new EndPointHeap(); } this.pool = MemoryManager.GetMemoryPool <CompoundGroupKey <PartitionKey <TPartitionKey>, TGroupKey>, TResult>(stream.Properties.IsColumnar); this.pool.Get(out this.output); this.output.Allocate(); }
public IntervalArraySubscriptionThrowNone( IObservable <ArraySegment <TPayload> > observable, Expression <Func <TPayload, long> > startEdgeExtractor, Expression <Func <TPayload, long> > endEdgeExtractor, string identifier, Streamable <Empty, TPayload> streamable, IStreamObserver <Empty, TPayload> observer, OnCompletedPolicy onCompletedPolicy, IObserver <OutOfOrderStreamEvent <TPayload> > diagnosticOutput) : base( observable, identifier, streamable, observer, DisorderPolicy.Throw(), FlushPolicy.FlushOnPunctuation, PeriodicPunctuationPolicy.None(), onCompletedPolicy, diagnosticOutput) { this.startEdgeExtractor = startEdgeExtractor; this.startEdgeFunction = startEdgeExtractor.Compile(); this.endEdgeExtractor = endEdgeExtractor; this.endEdgeFunction = endEdgeExtractor.Compile(); }
public PartitionedIntervalArraySubscriptionThrowNone( IObservable <ArraySegment <TPayload> > observable, Expression <Func <TPayload, TKey> > partitionExtractor, Expression <Func <TPayload, long> > startEdgeExtractor, Expression <Func <TPayload, long> > endEdgeExtractor, string identifier, Streamable <PartitionKey <TKey>, TPayload> streamable, IStreamObserver <PartitionKey <TKey>, TPayload> observer, OnCompletedPolicy onCompletedPolicy, IObserver <OutOfOrderPartitionedStreamEvent <TKey, TPayload> > diagnosticOutput) : base( observable, identifier, streamable, observer, DisorderPolicy.Throw(), PartitionedFlushPolicy.FlushOnLowWatermark, PeriodicPunctuationPolicy.None(), PeriodicLowWatermarkPolicy.None(), onCompletedPolicy, diagnosticOutput) { this.partitionExtractor = partitionExtractor; this.partitionFunction = partitionExtractor.Compile(); this.startEdgeExtractor = startEdgeExtractor; this.startEdgeFunction = startEdgeExtractor.Compile(); this.endEdgeExtractor = endEdgeExtractor; this.endEdgeFunction = endEdgeExtractor.Compile(); }
private IDisposable Subscribe(IStreamObserver <TKey, TSource> observer, int index) { IDisposable child; lock (this.subscriptionLock) { if (this.toSubscribe.Add(index)) { child = new ChildDisposable(this.connectableStream.Subscribe(observer), this.crew, index); } else { throw new InvalidOperationException("Cannot subscribe to the same child streamable more than once."); } if (this.toSubscribe.Count == this.outputCount) { this.crew.SetListDisposable(this.connectableStream.Connect()); this.crew = new DisposableManager(this.outputCount); this.connectableStream = new ConnectableStreamable <TKey, TSource>(this.source); this.toSubscribe.Clear(); } } return(child); }
public PartitionedStitchPipe(IStreamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer) : base(stream, observer) { this.errorMessages = stream.ErrorMessages; this.outputCount = 0; this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar); this.pool.Get(out this.batch); this.batch.Allocate(); var khpcomparer = EqualityComparerExtensions.GetCompoundEqualityComparerExpression <KHP, TKey, TPayload>( e => e.Key, stream.Properties.KeyEqualityComparer, e => e.Payload, stream.Properties.PayloadEqualityComparer); var equals = khpcomparer.GetEqualsExpr().Compile(); var getHashCode = khpcomparer.GetGetHashCodeExpr().Compile(); var generator1 = khpcomparer.CreateFastDictionary2Generator <KHP, List <ActiveEvent> >(1, equals, getHashCode, stream.Properties.QueryContainer); this.dictPool = new DataStructurePool <FastDictionary2 <KHP, List <ActiveEvent> > >(() => generator1.Invoke()); var generator2 = khpcomparer.CreateFastDictionary2Generator <KHP, int>(1, equals, getHashCode, stream.Properties.QueryContainer); this.CurrentTimeOpenEventBufferGenerator = () => generator2.Invoke(); var generator3 = khpcomparer.CreateFastDictionary2Generator <KHP, List <ActiveEventExt> >(1, equals, getHashCode, stream.Properties.QueryContainer); this.OpenEventsGenerator = () => generator3.Invoke(); }
internal override IStreamObserver <TKey, TPayload> CreatePipe(IStreamObserver <TKey, TPayload> observer) { var t = typeof(TKey).GetPartitionType(); if (t == null) { if (this.Source.Properties.IsColumnar) { return(GetPipe(observer)); } if (typeof(TKey) == typeof(Empty)) { return(new SessionWindowPipeStateless <TKey, TPayload>(this, observer, this.sessionTimeout, this.maximumDuration)); } else { return(new SessionWindowPipe <TKey, TPayload>(this, observer, this.sessionTimeout, this.maximumDuration)); } } var outputType = typeof(PartitionedSessionWindowPipe <, ,>).MakeGenericType( typeof(TKey), typeof(TPayload), t); return((IStreamObserver <TKey, TPayload>)Activator.CreateInstance(outputType, this, observer, this.sessionTimeout, this.maximumDuration)); }
private void Ingress(IStreamObserver <TKey, TPayload> observer) { bool done = false; int messages = 0; try { var serializer = StreamableSerializer.Create <QueuedMessage <StreamMessage <TKey, TPayload> > >(new SerializerSettings()); while (!done) { var message = serializer.Deserialize(this.stream); if (message.Kind == MessageKind.Completed) { done = true; } observer.OnNext(message.Message); messages++; if (this.numMessages != 0 && messages == this.numMessages) { if (message.Kind != MessageKind.Completed) { observer.OnCompleted(); break; } } } } catch (Exception e) { observer.OnError(e); } }
private void Ingress(IStreamObserver <TKey, TPayload> observer) { int messages = 0; try { var serializer = StreamableSerializer.Create <QueuedMessage <StreamMessage <TKey, TPayload> > >( new SerializerSettings() { KnownTypes = StreamMessageManager.GeneratedTypes() }); while (true) { var message = serializer.Deserialize(this.stream); if (message.Kind != MessageKind.Completed) { observer.OnNext(message.Message); messages++; } if (message.Kind == MessageKind.Completed || (this.numMessages != 0 && messages == this.numMessages)) { observer.OnCompleted(); break; } } } catch (Exception e) { observer.OnError(e); } this.onSubscriptionCompleted(); }
public override IDisposable Subscribe(IStreamObserver <PartitionKey <TPartitionKey>, TPayload> observer) { Contract.EnsuresOnThrow <IngressException>(true); var subscription = PartitionedIntervalArraySubscriptionCreator <TPartitionKey, TPayload> .CreateSubscription( this.observable, this.partitionExtractor, this.startEdgeExtractor, this.endEdgeExtractor, this.IngressSiteIdentifier, this, observer, onCompletedPolicy, diagnosticOutput); if (delayed) { container.RegisterIngressPipe(this.IngressSiteIdentifier, subscription); return(subscription.DelayedDisposable); } else { subscription.Enable(); return(subscription); } }
public override IDisposable Subscribe(IStreamObserver <Empty, TPayload> observer) { Contract.EnsuresOnThrow <IngressException>(true); IIngressStreamObserver subscription; if (this.timelinePolicy.timelineEnum == TimelineEnum.WallClock) { subscription = new MonotonicSubscriptionWallClock <TPayload>(this.observable, this.IngressSiteIdentifier, this, observer, this.flushPolicy, this.punctuationPolicy, this.onCompletedPolicy, this.timelinePolicy); } else { subscription = new MonotonicSubscriptionSequence <TPayload>(this.observable, this.IngressSiteIdentifier, this, observer, this.flushPolicy, this.punctuationPolicy, this.onCompletedPolicy, this.timelinePolicy); } if (this.delayed) { this.container.RegisterIngressPipe(this.IngressSiteIdentifier, subscription); return(subscription.DelayedDisposable); } else { subscription.Enable(); return(subscription); } }
public MonotonicSubscriptionWallClock( IObservable <TPayload> observable, string identifier, Streamable <Empty, TPayload> streamable, IStreamObserver <Empty, TPayload> observer, FlushPolicy flushPolicy, PeriodicPunctuationPolicy punctuationPolicy, OnCompletedPolicy onCompletedPolicy, TimelinePolicy timelinePolicy) : base( observable, identifier, streamable, observer, DisorderPolicy.Adjust(), flushPolicy, punctuationPolicy, onCompletedPolicy, null) { if (timelinePolicy.punctuationInterval > default(TimeSpan)) { this.timer = new Timer(EmitPunctuation, null, new TimeSpan(0), timelinePolicy.punctuationInterval); } }
public FixedIntervalEquiJoinPipe( BinaryStreamable <TKey, TLeft, TRight, TResult> stream, Expression <Func <TLeft, TRight, TResult> > selector, IStreamObserver <TKey, TResult> observer) : base(stream, observer) { this.selector = selector.Compile(); this.leftDuration = stream.Left.Properties.ConstantDurationLength.Value; this.rightDuration = stream.Right.Properties.ConstantDurationLength.Value; this.keyComparer = stream.Properties.KeyEqualityComparer.GetEqualsExpr(); this.keyComparerEquals = this.keyComparer.Compile(); if (this.leftDuration == this.rightDuration) { this.endPointHeap = new EndPointQueue(); } else { this.endPointHeap = new EndPointHeap(); } this.pool = MemoryManager.GetMemoryPool <TKey, TResult>(stream.Properties.IsColumnar); this.pool.Get(out this.output); this.output.Allocate(); }
public WrapperStreamObserver(IStreamObserver <TK, TP> o, OwnedThreadsScheduler scheduler, Guid classId, TaskEntry t) { this.o = o; this.scheduler = scheduler; this.classId = classId; this.te = t; }
private IStreamObserver <TOuterKey, TSource> CreatePipe(IStreamObserver <CompoundGroupKey <TOuterKey, TInnerKey>, TSource> observer) { if (typeof(TOuterKey).GetPartitionType() == null) { return(new GroupNestedPipe <TOuterKey, TSource, TInnerKey>(this, observer)); } return(new PartitionedGroupNestedPipe <TOuterKey, TSource, TInnerKey>(this, observer)); }
public override IDisposable Subscribe(IStreamObserver <CompoundGroupKey <TOuterKey, TInnerKey>, TSource> observer) { var pipe = Properties.IsColumnar ? GetPipe(observer) : CreatePipe(observer); return(Source.Subscribe(pipe)); }
public SelectPipe(SelectStreamable <TKey, TPayload, TResult> stream, IStreamObserver <TKey, TResult> observer) : base(stream, observer) { this.selector = (Expression <Func <TPayload, TResult> >)stream.Selector; this.selectorFunc = this.selector.Compile(); this.pool = MemoryManager.GetMemoryPool <TKey, TResult>(stream.Properties.IsColumnar); this.errorMessages = stream.ErrorMessages; }
private Tuple <Type, string> GetPipe(IStreamObserver <TKey, TOutput> observer) { var lookupKey = CacheKey.Create(this.apt, GetAggregateFunctionsHashCode(this.Aggregate)); var tuple = cachedPipes.GetOrAdd(lookupKey, key => AggregateTemplate.Generate(this, this.apt)); return(tuple); }
public StatelessQuantizeLifetimePipe(IStreamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer, long width, long skip, long offset) : base(stream, observer) { this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar); this.width = width; this.skip = skip; this.offset = offset; }
public UnionPipe(Streamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer) : base(stream, observer) { this.errorMessages = stream.ErrorMessages; this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar); this.pool.Get(out this.output); this.output.Allocate(); }
public StatelessPointAtEndPipe(IStreamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer, long constantDuration) : base(stream, observer) { this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar); this.errorMessages = stream.ErrorMessages; this.constantDuration = constantDuration; this.isPartitioned = typeof(TKey).GetPartitionType() != null; }
public ExtendLifetimePipe(ExtendLifetimeStreamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer, long duration) : base(stream, observer) { this.duration = duration; this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar); this.errorMessages = stream.ErrorMessages; this.pool.Get(out this.output); this.output.Allocate(); }
public ClipByConstantPipe(ClipByConstantStreamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer, long limit) : base(stream, observer) { this.limit = limit; this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar); this.errorMessages = stream.ErrorMessages; this.pool.Get(out this.output); this.output.Allocate(); }