Exemple #1
0
        public override void Send(Pair <S, T> record, RemotePostbox from)
        {
            if (this.serializer == null)
            {
                this.serializer = AutoSerialization.GetSerializer <Pair <S, T> >();
            }

            SendBufferPage page = this.messagesFromLocalThreads[from.ThreadIndex];

            if (page == null)
            {
                this.messagesFromLocalThreads[from.ThreadIndex] = new SendBufferPage(GlobalBufferPool <byte> .pool, this.pageSize);
                page = this.messagesFromLocalThreads[from.ThreadIndex];
                page.WriteHeader(new MessageHeader(from.ShardID, /* TODO FIXME: this.sendSequenceNumbers[destProcessID, destShardID]++ */ 0, this.Id, this.ShardId, SerializedMessageType.Data));
            }

            if (!page.WriteRecord(this.serializer, record))
            {
                this.Flush(from);
                this.messagesFromLocalThreads[from.ThreadIndex] = new SendBufferPage(GlobalBufferPool <byte> .pool, this.pageSize);
                page = this.messagesFromLocalThreads[from.ThreadIndex];
                page.WriteHeader(new MessageHeader(from.ShardID, /* TODO FIXME: this.sendSequenceNumbers[destProcessID, destShardID]++ */ 0, this.Id, this.ShardId, SerializedMessageType.Data));

                bool success = page.WriteRecord(this.serializer, record);
                if (!success)
                {
                    throw new Exception("Record too long to spill");
                }
            }
        }
Exemple #2
0
        public override void DeliverSerializedMessage(SerializedMessage message, RemotePostbox from)
        {
            byte[] headerBuffer = new byte[MessageHeader.SizeOf];
            MessageHeader.WriteHeaderToBuffer(headerBuffer, 0, message.Header);
            lock (this)
            {
                if (message.Header.Length == 0)
                {
                    this.spillStream.Flush(true);
                    this.pagesFlushed = this.pagesWritten;
                    this.dirty        = true;
                }
                else
                {
                    Debug.Assert(this.spillStream.Position % this.pageSize == 0);
                    this.spillStream.Write(headerBuffer, 0, headerBuffer.Length);
                    int count = message.Body.End - message.Body.CurrentPos;
                    this.spillStream.Write(message.Body.Buffer, message.Body.CurrentPos, count);
                    int padEnd = this.pageSize - (count + MessageHeader.SizeOf);
                    if (padEnd != 0)
                    {
                        this.spillStream.Seek(padEnd, SeekOrigin.Current);
                    }

                    ++this.pagesWritten;
                    Debug.Assert(this.spillStream.Position % this.pageSize == 0);
                }
            }
            message.Dispose();
            this.postOffice.Signal();
        }
Exemple #3
0
            public void DeliverSerializedMessage(SerializedMessage message, RemotePostbox from)
            {
                lock (this)
                {
                    if (message.Header.SequenceNumber == this.nextSequenceNumbers[from.ShardID])
                    {
                        //Console.Error.WriteLine("Delivering message {0} L = {1} A = {2}", message.Header.SequenceNumber, message.Header.Length, message.Body.Available);
                        //foreach (Pair<Int64, Pointstamp> currentRecord in this.decoder.Elements(message))
                        //    Console.Error.WriteLine("-- {0}", currentRecord);

                        //this.nextSequenceNumber++;
                        this.nextSequenceNumbers[from.ShardID]++;

                        var counter = 0;
                        foreach (Pair <Int64, Pointstamp> currentRecord in this.decoder.Elements(message))
                        {
                            this.consumer.ProcessCountChange(currentRecord.v2, currentRecord.v1);
                            counter++;
                        }

                        this.Flush(from);
                    }
                    else
                    {
                        //Console.Error.WriteLine("Discarding message {0} (expecting {1}) L = {2} A = {3}", message.Header.SequenceNumber, this.nextSequenceNumber, message.Header.Length, message.Body.Available);
                        //foreach (Pair<Int64, Pointstamp> currentRecord in this.decoder.Elements(message))
                        //    Console.Error.WriteLine("-- {0}", currentRecord);
                    }
                }
            }
Exemple #4
0
            public void DeliverSerializedMessage(SerializedMessage message, RemotePostbox from)
            {
                foreach (Pair <Int64, Pointstamp> currentRecord in this.decoder.Elements(message))
                {
                    this.consumer.ProcessCountChange(currentRecord.v2, currentRecord.v1);
                }

                this.Flush(from);
            }
Exemple #5
0
        /// <summary>
        /// Flushes the queue of local messages.
        /// Called by the postbox.
        /// </summary>
        /// <param name="from">Postbox of the upstream operator</param>
        public override void Flush(RemotePostbox from)
        {
            int i = from.ThreadIndex;

            if (this.messagesFromLocalShards[i].length > 0)
            {
                if (this.endpoint.Vertex.Scheduler.Index == from.ThreadIndex && this.endpoint.AvailableEntrancy >= 0)
                {
                    for (int j = 0; j < this.messagesFromLocalShards[from.ThreadIndex].length; j++)
                    {
                        if (progressBuffer != null)
                        {
                            progressBuffer.Update(this.messagesFromLocalShards[from.ThreadIndex].payload[j].v2, -1);
                        }
                    }


                    // capturing buffer to avoid writing to it in reentrancy.
                    var buffer = this.messagesFromLocalShards[from.ThreadIndex];

                    this.messagesFromLocalShards[from.ThreadIndex].Disable();

                    this.endpoint.AvailableEntrancy = this.endpoint.AvailableEntrancy - 1;

                    this.endpoint.MessageReceived(buffer, from);

                    this.Drain();

                    this.endpoint.AvailableEntrancy = this.endpoint.AvailableEntrancy + 1;

                    ThreadLocalBufferPools <Pair <S, T> > .pool.Value.CheckIn(buffer.payload);

                    buffer.Disable();
                }
                else
                {
                    this.Post(this.messagesFromLocalShards[i]);
                    this.messagesFromLocalShards[i].Disable();
                }
            }

            if (this.ShardId == from.ShardID)
            {
                if (this.endpoint.AvailableEntrancy >= 0)
                {
                    this.endpoint.AvailableEntrancy = this.endpoint.AvailableEntrancy - 1;
                    endpoint.Flush();
                    this.endpoint.AvailableEntrancy = this.endpoint.AvailableEntrancy + 1;
                }

                this.progressBuffer.Flush();
            }
        }
Exemple #6
0
 private void SpillLocalMessage(RemotePostbox from)
 {
     lock (this)
     {
         Debug.Assert(this.spillStream.Position % this.pageSize == 0);
         this.messagesFromLocalThreads[from.ThreadIndex].FinalizeLastMessage();
         using (BufferSegment segment = this.messagesFromLocalThreads[from.ThreadIndex].Consume())
             this.SpillBufferSegment(segment, 0);
         this.messagesFromLocalThreads[from.ThreadIndex].Release();
         this.messagesFromLocalThreads[from.ThreadIndex] = null;
         Debug.Assert(this.spillStream.Position % this.pageSize == 0);
     }
 }
Exemple #7
0
 public override void Flush(RemotePostbox from)
 {
     if (this.messagesFromLocalThreads[from.ThreadIndex] != null)
     {
         lock (this)
         {
             this.SpillLocalMessage(from);
             this.spillStream.Flush(true);
             Debug.Assert(this.spillStream.Position % this.pageSize == 0);
             this.pagesFlushed = this.pagesWritten;
             this.dirty        = true;
         }
         this.postOffice.Signal();
     }
 }
Exemple #8
0
        /// <summary>
        /// Enqueues the incoming network message on the shared queue, called by the network channel.
        /// </summary>
        /// <param name="message"></param>
        public override void DeliverSerializedMessage(SerializedMessage message, RemotePostbox from)
        {
            this.sharedSerializedQueue.Enqueue(message);

            if (this.sharedSerializedQueue.Count > SerializedQueueHighWatermark)
            {
                SerializedQueueHighWatermark *= 2;
            }

            if (!this.dirty)
            {
                this.dirty = true;
            }

            this.postOffice.Signal();
        }
Exemple #9
0
        public override void Send(Pair <S, T> record, RemotePostbox from)
        {
            if (this.messagesFromLocalShards[from.ThreadIndex].payload.Length == 0)
            {
                // Message has been disabled to conserve memory.
                this.messagesFromLocalShards[from.ThreadIndex].Enable();
            }

            this.messagesFromLocalShards[from.ThreadIndex].payload[this.messagesFromLocalShards[from.ThreadIndex].length++] = record;
            if (this.messagesFromLocalShards[from.ThreadIndex].length == this.messagesFromLocalShards[from.ThreadIndex].payload.Length)
            {
                if (this.endpoint.Vertex.Scheduler.Index == from.ThreadIndex && this.endpoint.AvailableEntrancy >= 0)
                {
                    for (int i = 0; i < this.messagesFromLocalShards[from.ThreadIndex].length; i++)
                    {
                        if (progressBuffer != null)
                        {
                            progressBuffer.Update(this.messagesFromLocalShards[from.ThreadIndex].payload[i].v2, -1);
                        }
                    }

                    // capturing buffer to avoid writing to it in reentrancy.
                    var buffer = this.messagesFromLocalShards[from.ThreadIndex];

                    this.messagesFromLocalShards[from.ThreadIndex].Disable();

                    this.endpoint.AvailableEntrancy = this.endpoint.AvailableEntrancy - 1;

                    this.endpoint.MessageReceived(buffer, from);

                    ThreadLocalBufferPools <Pair <S, T> > .pool.Value.CheckIn(buffer.payload);

                    buffer.Disable();

                    this.Drain();

                    this.endpoint.AvailableEntrancy = this.endpoint.AvailableEntrancy + 1;
                }
                else
                {
                    this.Post(this.messagesFromLocalShards[from.ThreadIndex]);
                    this.messagesFromLocalShards[from.ThreadIndex].Disable();
                }
            }
        }
Exemple #10
0
 public void Flush(RemotePostbox from)
 {
 }
Exemple #11
0
 public abstract void Send(Pair <S, T> record, RemotePostbox from);
Exemple #12
0
 public abstract void Flush(RemotePostbox from);
Exemple #13
0
 public void DeliverSerializedMessage(SerializedMessage message, RemotePostbox from)
 {
     throw new NotImplementedException("Attempted to deliver message from a remote sender to a remote recipient, which is not currently supported.");
 }
Exemple #14
0
 public void Send(Pair <S, T> record, RemotePostbox from)
 {
     this.encodersFromLocalShards[from.ThreadIndex].Write(record, from.ShardID);
 }
Exemple #15
0
 public void Flush(RemotePostbox from)
 {
     this.encodersFromLocalShards[from.ThreadIndex].Flush();
 }
Exemple #16
0
 /// <summary>
 /// "Sends" a progress message by poking the consumer directly.
 /// </summary>
 /// <param name="record"></param>
 /// <param name="from"></param>
 public void Send(Pair <Int64, Pointstamp> record, RemotePostbox from)
 {
     this.recordsReceived += 1;
     this.consumer.scheduler.statistics[(int)RuntimeStatistic.ProgressLocalRecords] += 1;
     this.consumer.ProcessCountChange(record.v2, record.v1);
 }
Exemple #17
0
 public abstract void DeliverSerializedMessage(SerializedMessage message, RemotePostbox from);