示例#1
0
        public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch)
        {
            var count = batch.Count;

            var srckey = batch.key.col;

            SavedEventList <TKey, TPayload> sevref = default;

            fixed(long *src_bv = batch.bitvector.col, src_vsync = batch.vsync.col)
            {
                fixed(int *src_hash = batch.hash.col)
                {
                    for (int i = 0; i < count; i++)
                    {
                        if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0 || batch.vother.col[i] < 0)
                        {
                            long synctime = src_vsync[i];

                            int index;

                            if (synctime > this.lastSyncTime) // move time forward
                            {
                                ProcessCurrentTimestamp();
                                this.lastSyncTime = synctime;
                            }

                            if (batch.vother.col[i] < 0)
                            {
                                OnPunctuation(synctime);
                                continue;
                            }

                            bool done = false;

                            if (this.eventListTraverser.Find(src_hash[i]))
                            {
                                while (this.eventListTraverser.Next(out index))
                                {
                                    var state = this.currentTimestampEventList.Values[index];

                                    if (this.keyEqualityComparer(state.key, srckey[i]))
                                    {
                                        state.payloads.Add(batch.payload.col[i]);

                                        done = true;
                                        break;
                                    }
                                }
                            }

                            if (!done)
                            {
                                index           = this.currentTimestampEventList.Insert(src_hash[i]);
                                sevref.payloads = new List <TPayload>(10)
                                {
                                    batch.payload.col[i]
                                };

                                sevref.key = srckey[i];
                                this.currentTimestampEventList.Values[index] = sevref;
                            }
                        }
                    }
                }
            }

            batch.Free();
        }
        public override unsafe void OnNext(StreamMessage <TKey, TPayload> batch)
        {
            var count = batch.Count;

            var srckey = batch.key.col;

            SavedEventList <TKey, TPayload> sevref = default;

            fixed(long *src_bv = batch.bitvector.col, src_vsync = batch.vsync.col, src_vother = batch.vother.col)
            {
                fixed(int *src_hash = batch.hash.col)
                {
                    for (int i = 0; i < count; i++)
                    {
                        if ((src_bv[i >> 6] & (1L << (i & 0x3f))) == 0)
                        {
                            var partitionKey   = this.getPartitionKey(srckey[i]);
                            int partitionIndex = EnsurePartition(partitionKey);

                            long synctime = src_vsync[i];

                            int index;

                            if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward
                            {
                                ProcessCurrentTimestamp(partitionIndex);
                                this.lastSyncTime.entries[partitionIndex].value = synctime;
                            }

                            bool done = false;

                            var eventListTraverser = new FastMap <SavedEventList <TKey, TPayload> > .FindTraverser(this.currentTimestampEventList.entries[partitionIndex].value);

                            if (eventListTraverser.Find(src_hash[i]))
                            {
                                while (eventListTraverser.Next(out index))
                                {
                                    var state = this.currentTimestampEventList.entries[partitionIndex].value.Values[index];

                                    if (this.keyEqualityComparer(state.key, srckey[i]))
                                    {
                                        state.payloads.Add(batch.payload.col[i]);

                                        done = true;
                                        break;
                                    }
                                }
                            }

                            if (!done)
                            {
                                index           = this.currentTimestampEventList.entries[partitionIndex].value.Insert(src_hash[i]);
                                sevref.payloads = new List <TPayload>(10)
                                {
                                    batch.payload.col[i]
                                };

                                sevref.key = srckey[i];
                                this.currentTimestampEventList.entries[partitionIndex].value.Values[index] = sevref;
                            }
                        }
                        else if (src_vother[i] == PartitionedStreamEvent.PunctuationOtherTime)
                        {
                            int  partitionIndex = FastDictionary2 <TPartitionKey, List <TKey> > .IteratorStart;
                            long synctime       = src_vsync[i];
                            while (this.lastSyncTime.Iterate(ref partitionIndex))
                            {
                                if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward
                                {
                                    ProcessCurrentTimestamp(partitionIndex);
                                    this.lastSyncTime.entries[partitionIndex].value = synctime;
                                }
                            }

                            OnLowWatermark(synctime);
                        }
                        else if (src_vother[i] == PartitionedStreamEvent.PunctuationOtherTime)
                        {
                            var partitionKey   = this.getPartitionKey(srckey[i]);
                            int partitionIndex = EnsurePartition(partitionKey);

                            long synctime = src_vsync[i];

                            if (synctime > this.lastSyncTime.entries[partitionIndex].value) // move time forward
                            {
                                ProcessCurrentTimestamp(partitionIndex);
                                this.lastSyncTime.entries[partitionIndex].value = synctime;
                            }
                        }
                    }
                }
            }

            batch.Free();
        }