Example #1
0
 public MonotonicSubscriptionWallClock(
     IObservable <TPayload> observable,
     string identifier,
     Streamable <Empty, TPayload> streamable,
     IStreamObserver <Empty, TPayload> observer,
     FlushPolicy flushPolicy,
     PeriodicPunctuationPolicy punctuationPolicy,
     OnCompletedPolicy onCompletedPolicy,
     TimelinePolicy timelinePolicy)
     : base(
         observable,
         identifier,
         streamable,
         observer,
         DisorderPolicy.Adjust(),
         flushPolicy,
         punctuationPolicy,
         onCompletedPolicy,
         null)
 {
     if (timelinePolicy.punctuationInterval > default(TimeSpan))
     {
         this.timer = new Timer(EmitPunctuation, null, new TimeSpan(0), timelinePolicy.punctuationInterval);
     }
 }
Example #2
0
 public IntervalArraySubscriptionThrowNone(
     IObservable <ArraySegment <TPayload> > observable,
     Expression <Func <TPayload, long> > startEdgeExtractor,
     Expression <Func <TPayload, long> > endEdgeExtractor,
     string identifier,
     Streamable <Empty, TPayload> streamable,
     IStreamObserver <Empty, TPayload> observer,
     OnCompletedPolicy onCompletedPolicy,
     IObserver <OutOfOrderStreamEvent <TPayload> > diagnosticOutput)
     : base(
         observable,
         identifier,
         streamable,
         observer,
         DisorderPolicy.Throw(),
         FlushPolicy.FlushOnPunctuation,
         PeriodicPunctuationPolicy.None(),
         onCompletedPolicy,
         diagnosticOutput)
 {
     this.startEdgeExtractor = startEdgeExtractor;
     this.startEdgeFunction  = startEdgeExtractor.Compile();
     this.endEdgeExtractor   = endEdgeExtractor;
     this.endEdgeFunction    = endEdgeExtractor.Compile();
 }
Example #3
0
 public CompiledPartitionedAfaPipe_MultiEventList(Streamable <TKey, TRegister> stream, IStreamObserver <TKey, TRegister> observer, object afa, long maxDuration)
     : base(stream, observer, afa, maxDuration, false)
 {
     this.getPartitionKey     = GetPartitionExtractor <TPartitionKey, TKey>();
     this.activeStates        = new FastMap <GroupedActiveState <TKey, TRegister> >();
     this.activeFindTraverser = new FastMap <GroupedActiveState <TKey, TRegister> > .FindTraverser(this.activeStates);
 }
Example #4
0
 public PartitionedIntervalArraySubscriptionThrowNone(
     IObservable <ArraySegment <TPayload> > observable,
     Expression <Func <TPayload, TKey> > partitionExtractor,
     Expression <Func <TPayload, long> > startEdgeExtractor,
     Expression <Func <TPayload, long> > endEdgeExtractor,
     string identifier,
     Streamable <PartitionKey <TKey>, TPayload> streamable,
     IStreamObserver <PartitionKey <TKey>, TPayload> observer,
     OnCompletedPolicy onCompletedPolicy,
     IObserver <OutOfOrderPartitionedStreamEvent <TKey, TPayload> > diagnosticOutput)
     : base(
         observable,
         identifier,
         streamable,
         observer,
         DisorderPolicy.Throw(),
         PartitionedFlushPolicy.FlushOnLowWatermark,
         PeriodicPunctuationPolicy.None(),
         PeriodicLowWatermarkPolicy.None(),
         onCompletedPolicy,
         diagnosticOutput)
 {
     this.partitionExtractor = partitionExtractor;
     this.partitionFunction  = partitionExtractor.Compile();
     this.startEdgeExtractor = startEdgeExtractor;
     this.startEdgeFunction  = startEdgeExtractor.Compile();
     this.endEdgeExtractor   = endEdgeExtractor;
     this.endEdgeFunction    = endEdgeExtractor.Compile();
 }
Example #5
0
 public UnionPipe(Streamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer)
     : base(stream, observer)
 {
     this.errorMessages = stream.ErrorMessages;
     this.pool          = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar);
     this.pool.Get(out this.output);
     this.output.Allocate();
 }
Example #6
0
        public CompiledUngroupedAfaPipe_EventList(Streamable <Empty, TRegister> stream, IStreamObserver <Empty, TRegister> observer, object afa, long maxDuration)
            : base(stream, observer, afa, maxDuration, false)
        {
            this.activeStates          = new FastLinkedList <GroupedActiveState <Empty, TRegister> >();
            this.activeStatesTraverser = new FastLinkedList <GroupedActiveState <Empty, TRegister> > .ListTraverser(this.activeStates);

            this.currentList  = new List <TPayload>();
            this.lastSyncTime = -1;
        }
Example #7
0
 public PartitionedExtendLifetimePipe(Streamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer, long duration)
     : base(stream, observer)
 {
     this.duration      = duration;
     this.pool          = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar);
     this.errorMessages = stream.ErrorMessages;
     this.pool.Get(out this.output);
     this.output.Allocate();
 }
Example #8
0
 public static PartitionedObserverSubscriptionBase <TKey, ArraySegment <PartitionedStreamEvent <TKey, TPayload> >, TPayload, TPayload> CreateSubscription(
     IObservable <ArraySegment <PartitionedStreamEvent <TKey, TPayload> > > observable,
     string identifier,
     Streamable <PartitionKey <TKey>, TPayload> streamable,
     IStreamObserver <PartitionKey <TKey>, TPayload> observer,
     OnCompletedPolicy onCompletedPolicy,
     IObserver <OutOfOrderPartitionedStreamEvent <TKey, TPayload> > diagnosticOutput)
 {
     return(new PartitionedStreamEventArraySubscriptionThrowNone <TKey, TPayload>(observable, identifier, streamable, observer, onCompletedPolicy, diagnosticOutput));
 }
Example #9
0
 public static ObserverSubscriptionBase <ArraySegment <TPayload>, TPayload, TPayload> CreateSubscription(
     IObservable <ArraySegment <TPayload> > observable,
     Expression <Func <TPayload, long> > startEdgeExtractor,
     Expression <Func <TPayload, long> > endEdgeExtractor,
     string identifier,
     Streamable <Empty, TPayload> streamable,
     IStreamObserver <Empty, TPayload> observer,
     OnCompletedPolicy onCompletedPolicy,
     IObserver <OutOfOrderStreamEvent <TPayload> > diagnosticOutput)
 {
     return(new IntervalArraySubscriptionThrowNone <TPayload>(observable, startEdgeExtractor, endEdgeExtractor, identifier, streamable, observer, onCompletedPolicy, diagnosticOutput));
 }
 public CompiledPartitionedAfaPipe_SingleEvent(Streamable <TKey, TRegister> stream, IStreamObserver <TKey, TRegister> observer, object afa, long maxDuration)
     : base(stream, observer, afa, maxDuration)
 {
     if (!this.IsSyncTimeSimultaneityFree)
     {
         var comparer        = stream.Properties.KeyEqualityComparer;
         var equalsFunc      = comparer.GetEqualsExpr().Compile();
         var getHashCodeFunc = comparer.GetGetHashCodeExpr().Compile();
         this.seenEvent       = comparer.CreateFastDictionary2Generator <TKey, byte>(10, equalsFunc, getHashCodeFunc, stream.Properties.QueryContainer).Invoke();
         this.tentativeOutput = new FastDictionary2 <TPartitionKey, FastMap <OutputEvent <TKey, TRegister> > >();
     }
 }
Example #11
0
        public CompiledUngroupedAfaPipe(Streamable <Empty, TRegister> stream, IStreamObserver <Empty, TRegister> observer, object afa, long maxDuration)
            : base(stream, observer, afa, maxDuration)
        {
            this.activeStates = new FastLinkedList <GroupedActiveState <Empty, TRegister> >();

            if (!this.IsSyncTimeSimultaneityFree)
            {
                this.seenEvent       = 0;
                this.tentativeOutput = new FastLinkedList <OutputEvent <Empty, TRegister> >();
                this.lastSyncTime    = -1;
            }
        }
        public PartitionedQuantizeLifetimePipe(Streamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer, long width, long skip, long progress, long offset)
            : base(stream, observer)
        {
            this.pool          = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar);
            this.errorMessages = stream.ErrorMessages;
            this.pool.Get(out this.output);
            this.output.Allocate();

            this.width    = width;
            this.skip     = skip;
            this.progress = progress;
            this.offset   = offset;
        }
        public CompiledGroupedAfaPipe_MultiEvent(Streamable <TKey, TRegister> stream, IStreamObserver <TKey, TRegister> observer, object afa, long maxDuration)
            : base(stream, observer, afa, maxDuration)
        {
            this.activeStates        = new FastMap <GroupedActiveStateAccumulator <TKey, TPayload, TRegister, TAccumulator> >(1);
            this.activeFindTraverser = new FastMap <GroupedActiveStateAccumulator <TKey, TPayload, TRegister, TAccumulator> > .FindTraverser(this.activeStates);

            this.keyHeads = new FastMap <TKey>(1);
            this.keyHeadsFindTraverser = new FastMap <TKey> .FindTraverser(this.keyHeads);

            this.keyHeadsVisibleTraverser = new FastMap <TKey> .VisibleTraverser(this.keyHeads);

            this.lastSyncTime = -1;
        }
Example #14
0
        public CompiledGroupedAfaPipe_MultiEventList(Streamable <TKey, TRegister> stream, IStreamObserver <TKey, TRegister> observer, object afa, long maxDuration)
            : base(stream, observer, afa, maxDuration)
        {
            this.activeStates        = new FastMap <GroupedActiveState <TKey, TRegister> >();
            this.activeFindTraverser = new FastMap <GroupedActiveState <TKey, TRegister> > .FindTraverser(this.activeStates);

            this.currentTimestampEventList = new FastMap <SavedEventList <TKey, TPayload> >();
            this.eventListTraverser        = new FastMap <SavedEventList <TKey, TPayload> > .FindTraverser(this.currentTimestampEventList);

            this.allEventListTraverser = new FastMap <SavedEventList <TKey, TPayload> > .VisibleTraverser(this.currentTimestampEventList);

            this.lastSyncTime = -1;
        }
Example #15
0
 public StreamMessageIngressSubscription(
     IObservable <StreamMessage <Empty, TPayload> > observable,
     string identifier,
     Streamable <Empty, TPayload> stream,
     IStreamObserver <Empty, TPayload> observer)
     : base(
         observable,
         identifier,
         stream,
         observer,
         DisorderPolicy.Drop(),
         FlushPolicy.FlushOnPunctuation,
         PeriodicPunctuationPolicy.None(),
         OnCompletedPolicy.None,
         null)
 {
 }
 public MonotonicArraySubscriptionSequence(
     IObservable <ArraySegment <TPayload> > observable,
     string identifier,
     Streamable <Empty, TPayload> streamable,
     IStreamObserver <Empty, TPayload> observer,
     OnCompletedPolicy onCompletedPolicy,
     TimelinePolicy timelinePolicy)
     : base(
         observable,
         identifier,
         streamable,
         observer,
         DisorderPolicy.Throw(),
         FlushPolicy.FlushOnPunctuation,
         PeriodicPunctuationPolicy.None(),
         onCompletedPolicy,
         null)
     => this.eventsPerSample = timelinePolicy.sampleSize;
Example #17
0
 public StreamEventArraySubscriptionThrowNone(
     IObservable <ArraySegment <StreamEvent <TPayload> > > observable,
     string identifier,
     Streamable <Empty, TPayload> streamable,
     IStreamObserver <Empty, TPayload> observer,
     OnCompletedPolicy onCompletedPolicy,
     IObserver <OutOfOrderStreamEvent <TPayload> > diagnosticOutput)
     : base(
         observable,
         identifier,
         streamable,
         observer,
         DisorderPolicy.Throw(),
         FlushPolicy.FlushOnPunctuation,
         PeriodicPunctuationPolicy.None(),
         onCompletedPolicy,
         diagnosticOutput)
 {
 }
Example #18
0
        public BinaryIngressReader(
            string identifier,
            Streamable <TKey, TPayload> streamable,
            IStreamObserver <TKey, TPayload> observer,
            int numMessages,
            Stream stream,
            IIngressScheduler scheduler,
            bool delayed)
            : base(identifier, streamable, observer)
        {
            this.pool        = MemoryManager.GetMemoryPool <TKey, TPayload>();
            this.numMessages = numMessages;
            this.stream      = stream;
            this.scheduler   = scheduler;

            if (!delayed)
            {
                this.subscription.Enable();
            }
        }
Example #19
0
 public PartitionedStreamEventArraySubscriptionThrowNone(
     IObservable <ArraySegment <PartitionedStreamEvent <TKey, TPayload> > > observable,
     string identifier,
     Streamable <PartitionKey <TKey>, TPayload> streamable,
     IStreamObserver <PartitionKey <TKey>, TPayload> observer,
     OnCompletedPolicy onCompletedPolicy,
     IObserver <OutOfOrderPartitionedStreamEvent <TKey, TPayload> > diagnosticOutput)
     : base(
         observable,
         identifier,
         streamable,
         observer,
         DisorderPolicy.Throw(),
         PartitionedFlushPolicy.FlushOnLowWatermark,
         PeriodicPunctuationPolicy.None(),
         PeriodicLowWatermarkPolicy.None(),
         onCompletedPolicy,
         diagnosticOutput)
 {
 }
Example #20
0
 public MonotonicSubscriptionSequence(
     IObservable <TPayload> observable,
     string identifier,
     Streamable <Empty, TPayload> streamable,
     IStreamObserver <Empty, TPayload> observer,
     FlushPolicy flushPolicy,
     PeriodicPunctuationPolicy punctuationPolicy,
     OnCompletedPolicy onCompletedPolicy,
     TimelinePolicy timelinePolicy)
     : base(
         observable,
         identifier,
         streamable,
         observer,
         DisorderPolicy.Adjust(),
         flushPolicy,
         punctuationPolicy,
         onCompletedPolicy,
         null)
 {
     this.eventsPerSample = timelinePolicy.sampleSize;
     this.currentTime     = 0;
 }
Example #21
0
 public DisjointUnionPipe(Streamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer)
     : base(stream, observer)
 {
 }
Example #22
0
        public override IDisposable Subscribe(IStreamObserver <TMapKey, TOutput> observer)
        {
            // asymmetric mapper implies that we have to have a 2-input mapper
            Contract.Assert((!this.leftAsymmetric1) || (this.sourceRight1 != null));
            Contract.Assert((!this.leftAsymmetric2) || (this.sourceRight2 != null));

            var mapArity    = Config.MapArity;
            var reduceArity = Config.ReduceArity;

            // process mapper #1
            Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>[] shuffleL2Results1;

            if (this.sourceRight1 != null) // two-input mapper
            {
                // DEAD
                shuffleL2Results1 = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> [reduceArity];

                // [1] spray batches into L1 physical cores
                var importLeft1  = new SprayGroupImportStreamable <TMapKey, TMapInputLeft1>(this.sourceLeft1, mapArity, this.leftAsymmetric1);
                var importRight1 = new SprayGroupImportStreamable <TMapKey, TMapInputRight1>(this.sourceRight1, mapArity);

                // [2] perform the spray lambda on each L1 core
                var sprayResults1 = new BinaryMulticastStreamable <TMapKey, TMapInputLeft1, TMapInputRight1, TReduceInput1> [mapArity];
                for (int i = 0; i < mapArity; i++)
                {
                    sprayResults1[i] = new BinaryMulticastStreamable <TMapKey, TMapInputLeft1, TMapInputRight1, TReduceInput1>(importLeft1, importRight1, this.mapper1);
                }

                // [3] apply shuffle on the result of each spray
                Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>[] shuffleL1Results1 = new ShuffleNestedStreamable <TMapKey, TReduceInput1, TReduceKey> [mapArity];
                for (int i = 0; i < mapArity; i++)
                {
                    shuffleL1Results1[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput1, TReduceKey>(sprayResults1[i], this.keySelector1, reduceArity, i);
                }

                // [4] Union the shuffled data by group key
                MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> .l2index = 0;
                for (int i = 0; i < reduceArity; i++)
                {
                    shuffleL2Results1[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>(shuffleL1Results1);
                }
            }
            else // single-input mapper
            {
                // [1] spray batches into L1 physical cores
                var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft1>
                                     (this.sourceLeft1, mapArity, false, this.sprayComparer1);

                // [2] perform the spray lambda on each L1 core
                var sprayResults1 = new MulticastStreamable <TMapKey, TMapInputLeft1, TReduceInput1> [mapArity];
                for (int i = 0; i < mapArity; i++)
                {
                    sprayResults1[i] = new MulticastStreamable <TMapKey, TMapInputLeft1, TReduceInput1>(importLeft, a => this.mapper1(a, null));
                }

                if (this.reduceInMap || (this.reduceOptions == OperationalHint.Asymmetric))
                {
                    shuffleL2Results1 = new Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> [mapArity];
                    for (int i = 0; i < mapArity; i++)
                    {
                        shuffleL2Results1[i] = new GroupNestedStreamable <TMapKey, TReduceInput1, TReduceKey>(sprayResults1[i], this.keySelector1);
                    }
                }
                else
                {
                    shuffleL2Results1 = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> [reduceArity];

                    // [3] apply shuffle on the result of each spray
                    Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>[] shuffleL1Results1 = new ShuffleNestedStreamable <TMapKey, TReduceInput1, TReduceKey> [mapArity];
                    for (int i = 0; i < mapArity; i++)
                    {
                        shuffleL1Results1[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput1, TReduceKey>(sprayResults1[i], this.keySelector1, reduceArity, i);
                    }

                    // [4] Union the shuffled data by group key
                    MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1> .l2index = 0;
                    for (int i = 0; i < reduceArity; i++)
                    {
                        shuffleL2Results1[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1>(shuffleL1Results1);
                    }
                }
            }

            // process mapper #2
            Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>[] shuffleL2Results2;
            if (this.sourceRight2 != null) // two-input mapper
            {
                // DEAD
                shuffleL2Results2 = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> [reduceArity];

                // [1] spray batches into L1 physical cores
                var importLeft2  = new SprayGroupImportStreamable <TMapKey, TMapInputLeft2>(this.sourceLeft2, mapArity, this.leftAsymmetric2);
                var importRight2 = new SprayGroupImportStreamable <TMapKey, TMapInputRight2>(this.sourceRight2, mapArity);

                // [2] perform the spray lambda on each L1 core
                var sprayResults2 = new BinaryMulticastStreamable <TMapKey, TMapInputLeft2, TMapInputRight2, TReduceInput2> [mapArity];
                for (int i = 0; i < mapArity; i++)
                {
                    sprayResults2[i] = new BinaryMulticastStreamable <TMapKey, TMapInputLeft2, TMapInputRight2, TReduceInput2>(importLeft2, importRight2, this.mapper2);
                }

                // [3] apply shuffle on the result of each spray
                Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>[] shuffleL1Results2 = new ShuffleNestedStreamable <TMapKey, TReduceInput2, TReduceKey> [mapArity];
                for (int i = 0; i < mapArity; i++)
                {
                    shuffleL1Results2[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput2, TReduceKey>(sprayResults2[i], this.keySelector2, reduceArity, i);
                }

                // [4] Union the shuffled data by group key
                MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> .l2index = 0;
                for (int i = 0; i < reduceArity; i++)
                {
                    shuffleL2Results2[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>(shuffleL1Results2);
                }
            }
            else // single-input mapper
            {
                // [1] spray batches into L1 physical cores
                var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft2>(this.sourceLeft2, mapArity, this.reduceOptions == OperationalHint.Asymmetric);

                // [2] perform the spray lambda on each L1 core
                var sprayResults2 = new MulticastStreamable <TMapKey, TMapInputLeft2, TReduceInput2> [mapArity];
                for (int i = 0; i < mapArity; i++)
                {
                    sprayResults2[i] = new MulticastStreamable <TMapKey, TMapInputLeft2, TReduceInput2>(importLeft, a => this.mapper2(a, null));
                }

                if (this.reduceInMap || (this.reduceOptions == OperationalHint.Asymmetric))
                {
                    shuffleL2Results2 = new Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> [mapArity];
                    for (int i = 0; i < mapArity; i++)
                    {
                        shuffleL2Results2[i] = new GroupNestedStreamable <TMapKey, TReduceInput2, TReduceKey>(sprayResults2[i], this.keySelector2);
                    }
                }
                else
                {
                    shuffleL2Results2 = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> [reduceArity];

                    // [3] apply shuffle on the result of each spray
                    var shuffleL1Results2 = new ShuffleNestedStreamable <TMapKey, TReduceInput2, TReduceKey> [mapArity];
                    for (int i = 0; i < mapArity; i++)
                    {
                        shuffleL1Results2[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput2, TReduceKey>(sprayResults2[i], this.keySelector2, reduceArity, i);
                    }

                    // [4] Union the shuffled data by group key
                    MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2> .l2index = 0;
                    for (int i = 0; i < reduceArity; i++)
                    {
                        shuffleL2Results2[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput2>(shuffleL1Results2);
                    }
                }
            }

            // process 2-input reducer
            // [5] perform the apply lambda on each L2 core
            var innerResults
                = new BinaryMulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1, TReduceInput2, TBind> [shuffleL2Results1.Length];
            var ungroupInnerResults
                = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput> [shuffleL2Results1.Length];

            for (int i = 0; i < shuffleL2Results1.Length; i++)
            {
                innerResults[i]        = new BinaryMulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput1, TReduceInput2, TBind>(shuffleL2Results1[i], shuffleL2Results2[i], this.reducer);
                ungroupInnerResults[i] = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput>(this.sourceLeft1.Properties.KeyEqualityComparer, innerResults[i], this.resultSelector);
            }

            // [6] final single merging union
            var union = new MultiUnionStreamable <TMapKey, TOutput>(ungroupInnerResults, false);

            return(union.Subscribe(observer));
        }
Example #23
0
 public DisjointUnionPipe(Streamable <TKey, TPayload> stream, IStreamObserver <TKey, TPayload> observer)
     : base(stream, observer)
 {
     this.pool = MemoryManager.GetMemoryPool <TKey, TPayload>(stream.Properties.IsColumnar);
 }
 public CompiledPartitionedAfaPipe_MultiEvent(Streamable <TKey, TRegister> stream, IStreamObserver <TKey, TRegister> observer, object afa, long maxDuration)
     : base(stream, observer, afa, maxDuration)
 {
     this.activeStates        = new FastMap <GroupedActiveStateAccumulator <TKey, TPayload, TRegister, TAccumulator> >(1);
     this.activeFindTraverser = new FastMap <GroupedActiveStateAccumulator <TKey, TPayload, TRegister, TAccumulator> > .FindTraverser(this.activeStates);
 }
Example #25
0
        public override IDisposable Subscribe(IStreamObserver <TMapKey, TOutput> observer)
        {
            // asymmetric mapper implies that we have to have a 2-input mapper
            Contract.Assert((!this.leftAsymmetric) || (this.sourceRight != null));

            var mapArity    = this.isMulticore ? 1 : Config.MapArity;
            var reduceArity = this.isMulticore ? 1 : Config.ReduceArity;

            if (this.keySelector != null)
            {
                if (this.sourceRight != null) // two-input mapper
                {
                    // [1] spray batches into L1 physical cores
                    var importLeft  = new SprayGroupImportStreamable <TMapKey, TMapInputLeft>(this.sourceLeft, mapArity, this.leftAsymmetric);
                    var importRight = new SprayGroupImportStreamable <TMapKey, TMapInputRight>(this.sourceRight, mapArity);

                    // [2] perform the spray lambda on each L1 core
                    var sprayResults = new BinaryMulticastStreamable <TMapKey, TMapInputLeft, TMapInputRight, TReduceInput> [mapArity];
                    for (int i = 0; i < mapArity; i++)
                    {
                        sprayResults[i] = new BinaryMulticastStreamable <TMapKey, TMapInputLeft, TMapInputRight, TReduceInput>(importLeft, importRight, this.mapper);
                    }

                    // [3] apply shuffle on the result of each spray
                    Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>[] shuffleL1Results = new ShuffleNestedStreamable <TMapKey, TReduceInput, TReduceKey> [mapArity];
                    for (int i = 0; i < mapArity; i++)
                    {
                        shuffleL1Results[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput, TReduceKey>(this.keyComparer, sprayResults[i], this.keySelector, reduceArity, i);
                    }

                    // [4] Union the shuffled data by group key
                    MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> .l2index = 0;
                    var shuffleL2Results = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> [reduceArity];
                    for (int i = 0; i < reduceArity; i++)
                    {
                        shuffleL2Results[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>(shuffleL1Results);
                    }

                    // [5] perform the apply lambda on each L2 core
                    var innerResults        = new MulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput, TBind> [reduceArity];
                    var ungroupInnerResults = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput> [reduceArity];

                    for (int i = 0; i < reduceArity; i++)
                    {
                        innerResults[i]        = new MulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput, TBind>(shuffleL2Results[i], this.reducer);
                        ungroupInnerResults[i] = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput>(this.sourceLeft.Properties.KeyEqualityComparer, innerResults[i], this.resultSelector);
                    }
                    // [6] final single merging union
                    var union = new MultiUnionStreamable <TMapKey, TOutput>(ungroupInnerResults, false);

                    return(union.Subscribe(observer));
                }
                else // single-input mapper
                {
                    // [1] spray batches into L1 physical cores
                    var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft>(this.sourceLeft, mapArity, this.leftAsymmetric, this.sprayComparer);

                    // [2] perform the spray lambda on each L1 core
                    var sprayResults = new MulticastStreamable <TMapKey, TMapInputLeft, TReduceInput> [mapArity];
                    for (int i = 0; i < mapArity; i++)
                    {
                        sprayResults[i] = new MulticastStreamable <TMapKey, TMapInputLeft, TReduceInput>(importLeft, a => this.mapper(a, null));
                    }

                    Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>[] mergeInputs;
                    if (this.reduceInMap) // apply reducer in map phase itself
                    {
                        // [3] apply shuffle on the result of each spray
                        mergeInputs = new Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> [mapArity];
                        for (int i = 0; i < mapArity; i++)
                        {
                            mergeInputs[i] = new GroupNestedStreamable <TMapKey, TReduceInput, TReduceKey>(this.keyComparer, sprayResults[i], this.keySelector);
                        }
                    }
                    else
                    {
                        // [3] apply shuffle on the result of each spray
                        Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>[] shuffleL1Results = new ShuffleNestedStreamable <TMapKey, TReduceInput, TReduceKey> [mapArity];
                        for (int i = 0; i < mapArity; i++)
                        {
                            shuffleL1Results[i] = new ShuffleNestedStreamable <TMapKey, TReduceInput, TReduceKey>(this.keyComparer, sprayResults[i], this.keySelector, reduceArity, i);
                        }

                        // [4] Union the shuffled data by group key
                        MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> .l2index = 0;

                        mergeInputs = new Streamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> [reduceArity];
                        mergeInputs = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput> [reduceArity];
                        for (int i = 0; i < reduceArity; i++)
                        {
                            mergeInputs[i] = new MultiUnionStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput>(shuffleL1Results);
                        }
                    }

                    // [5] perform the apply lambda on each L2 core
                    var innerResults
                        = new MulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput, TBind> [mergeInputs.Length];
                    var ungroupInnerResults
                        = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput> [mergeInputs.Length];

                    for (int i = 0; i < mergeInputs.Length; i++)
                    {
                        innerResults[i]        = new MulticastStreamable <CompoundGroupKey <TMapKey, TReduceKey>, TReduceInput, TBind>(mergeInputs[i], this.reducer);
                        ungroupInnerResults[i] = new UngroupStreamable <TMapKey, TReduceKey, TBind, TOutput>(this.sourceLeft.Properties.KeyEqualityComparer, innerResults[i], this.resultSelector);
                    }

                    // [6] final single merging union
                    var union = new MultiUnionStreamable <TMapKey, TOutput>(ungroupInnerResults, false);

                    return(union.Subscribe(observer));
                }
            }
            else
            {
                if (this.sourceRight != null) // two-input mapper
                {
                    // [1] spray batches into L1 physical cores
                    var importLeft  = new SprayGroupImportStreamable <TMapKey, TMapInputLeft>(this.sourceLeft, mapArity, this.leftAsymmetric);
                    var importRight = new SprayGroupImportStreamable <TMapKey, TMapInputRight>(this.sourceRight, mapArity);

                    // [2] perform the spray lambda on each L1 core
                    var sprayResults = new BinaryMulticastStreamable <TMapKey, TMapInputLeft, TMapInputRight, TReduceInput> [mapArity];
                    for (int i = 0; i < mapArity; i++)
                    {
                        sprayResults[i] = new BinaryMulticastStreamable <TMapKey, TMapInputLeft, TMapInputRight, TReduceInput>(importLeft, importRight, this.mapper);
                    }

                    // [4] Union the shuffled data by group key
                    MultiUnionStreamable <TMapKey, TReduceInput> .l2index = 0;
                    var shuffleL2Result
                        = new MultiUnionStreamable <TMapKey, TReduceInput>(sprayResults, false) as MultiUnionStreamable <TMapKey, TOutput>;

                    return(shuffleL2Result.Subscribe(observer));
                }
                else // single-input mapper
                {
                    // [1] spray batches into L1 physical cores
                    var importLeft = new SprayGroupImportStreamable <TMapKey, TMapInputLeft>(this.sourceLeft, mapArity);

                    // [2] perform the spray lambda on each L1 core
                    var sprayResults = new MulticastStreamable <TMapKey, TMapInputLeft, TReduceInput> [mapArity];
                    for (int i = 0; i < mapArity; i++)
                    {
                        sprayResults[i] = new MulticastStreamable <TMapKey, TMapInputLeft, TReduceInput>(importLeft, a => this.mapper(a, null));
                    }

                    // [4] Union the shuffled data by group key
                    MultiUnionStreamable <TMapKey, TReduceInput> .l2index = 0;
                    var shuffleL2Result
                        = new MultiUnionStreamable <TMapKey, TReduceInput>(sprayResults, false) as MultiUnionStreamable <TMapKey, TOutput>;

                    return(shuffleL2Result.Subscribe(observer));
                }
            }
        }