public override void Send(Pair <S, T> record, RemotePostbox from) { if (this.serializer == null) { this.serializer = AutoSerialization.GetSerializer <Pair <S, T> >(); } SendBufferPage page = this.messagesFromLocalThreads[from.ThreadIndex]; if (page == null) { this.messagesFromLocalThreads[from.ThreadIndex] = new SendBufferPage(GlobalBufferPool <byte> .pool, this.pageSize); page = this.messagesFromLocalThreads[from.ThreadIndex]; page.WriteHeader(new MessageHeader(from.ShardID, /* TODO FIXME: this.sendSequenceNumbers[destProcessID, destShardID]++ */ 0, this.Id, this.ShardId, SerializedMessageType.Data)); } if (!page.WriteRecord(this.serializer, record)) { this.Flush(from); this.messagesFromLocalThreads[from.ThreadIndex] = new SendBufferPage(GlobalBufferPool <byte> .pool, this.pageSize); page = this.messagesFromLocalThreads[from.ThreadIndex]; page.WriteHeader(new MessageHeader(from.ShardID, /* TODO FIXME: this.sendSequenceNumbers[destProcessID, destShardID]++ */ 0, this.Id, this.ShardId, SerializedMessageType.Data)); bool success = page.WriteRecord(this.serializer, record); if (!success) { throw new Exception("Record too long to spill"); } } }
public override void DeliverSerializedMessage(SerializedMessage message, RemotePostbox from) { byte[] headerBuffer = new byte[MessageHeader.SizeOf]; MessageHeader.WriteHeaderToBuffer(headerBuffer, 0, message.Header); lock (this) { if (message.Header.Length == 0) { this.spillStream.Flush(true); this.pagesFlushed = this.pagesWritten; this.dirty = true; } else { Debug.Assert(this.spillStream.Position % this.pageSize == 0); this.spillStream.Write(headerBuffer, 0, headerBuffer.Length); int count = message.Body.End - message.Body.CurrentPos; this.spillStream.Write(message.Body.Buffer, message.Body.CurrentPos, count); int padEnd = this.pageSize - (count + MessageHeader.SizeOf); if (padEnd != 0) { this.spillStream.Seek(padEnd, SeekOrigin.Current); } ++this.pagesWritten; Debug.Assert(this.spillStream.Position % this.pageSize == 0); } } message.Dispose(); this.postOffice.Signal(); }
public void DeliverSerializedMessage(SerializedMessage message, RemotePostbox from) { lock (this) { if (message.Header.SequenceNumber == this.nextSequenceNumbers[from.ShardID]) { //Console.Error.WriteLine("Delivering message {0} L = {1} A = {2}", message.Header.SequenceNumber, message.Header.Length, message.Body.Available); //foreach (Pair<Int64, Pointstamp> currentRecord in this.decoder.Elements(message)) // Console.Error.WriteLine("-- {0}", currentRecord); //this.nextSequenceNumber++; this.nextSequenceNumbers[from.ShardID]++; var counter = 0; foreach (Pair <Int64, Pointstamp> currentRecord in this.decoder.Elements(message)) { this.consumer.ProcessCountChange(currentRecord.v2, currentRecord.v1); counter++; } this.Flush(from); } else { //Console.Error.WriteLine("Discarding message {0} (expecting {1}) L = {2} A = {3}", message.Header.SequenceNumber, this.nextSequenceNumber, message.Header.Length, message.Body.Available); //foreach (Pair<Int64, Pointstamp> currentRecord in this.decoder.Elements(message)) // Console.Error.WriteLine("-- {0}", currentRecord); } } }
public void DeliverSerializedMessage(SerializedMessage message, RemotePostbox from) { foreach (Pair <Int64, Pointstamp> currentRecord in this.decoder.Elements(message)) { this.consumer.ProcessCountChange(currentRecord.v2, currentRecord.v1); } this.Flush(from); }
/// <summary> /// Flushes the queue of local messages. /// Called by the postbox. /// </summary> /// <param name="from">Postbox of the upstream operator</param> public override void Flush(RemotePostbox from) { int i = from.ThreadIndex; if (this.messagesFromLocalShards[i].length > 0) { if (this.endpoint.Vertex.Scheduler.Index == from.ThreadIndex && this.endpoint.AvailableEntrancy >= 0) { for (int j = 0; j < this.messagesFromLocalShards[from.ThreadIndex].length; j++) { if (progressBuffer != null) { progressBuffer.Update(this.messagesFromLocalShards[from.ThreadIndex].payload[j].v2, -1); } } // capturing buffer to avoid writing to it in reentrancy. var buffer = this.messagesFromLocalShards[from.ThreadIndex]; this.messagesFromLocalShards[from.ThreadIndex].Disable(); this.endpoint.AvailableEntrancy = this.endpoint.AvailableEntrancy - 1; this.endpoint.MessageReceived(buffer, from); this.Drain(); this.endpoint.AvailableEntrancy = this.endpoint.AvailableEntrancy + 1; ThreadLocalBufferPools <Pair <S, T> > .pool.Value.CheckIn(buffer.payload); buffer.Disable(); } else { this.Post(this.messagesFromLocalShards[i]); this.messagesFromLocalShards[i].Disable(); } } if (this.ShardId == from.ShardID) { if (this.endpoint.AvailableEntrancy >= 0) { this.endpoint.AvailableEntrancy = this.endpoint.AvailableEntrancy - 1; endpoint.Flush(); this.endpoint.AvailableEntrancy = this.endpoint.AvailableEntrancy + 1; } this.progressBuffer.Flush(); } }
private void SpillLocalMessage(RemotePostbox from) { lock (this) { Debug.Assert(this.spillStream.Position % this.pageSize == 0); this.messagesFromLocalThreads[from.ThreadIndex].FinalizeLastMessage(); using (BufferSegment segment = this.messagesFromLocalThreads[from.ThreadIndex].Consume()) this.SpillBufferSegment(segment, 0); this.messagesFromLocalThreads[from.ThreadIndex].Release(); this.messagesFromLocalThreads[from.ThreadIndex] = null; Debug.Assert(this.spillStream.Position % this.pageSize == 0); } }
public override void Flush(RemotePostbox from) { if (this.messagesFromLocalThreads[from.ThreadIndex] != null) { lock (this) { this.SpillLocalMessage(from); this.spillStream.Flush(true); Debug.Assert(this.spillStream.Position % this.pageSize == 0); this.pagesFlushed = this.pagesWritten; this.dirty = true; } this.postOffice.Signal(); } }
/// <summary> /// Enqueues the incoming network message on the shared queue, called by the network channel. /// </summary> /// <param name="message"></param> public override void DeliverSerializedMessage(SerializedMessage message, RemotePostbox from) { this.sharedSerializedQueue.Enqueue(message); if (this.sharedSerializedQueue.Count > SerializedQueueHighWatermark) { SerializedQueueHighWatermark *= 2; } if (!this.dirty) { this.dirty = true; } this.postOffice.Signal(); }
public override void Send(Pair <S, T> record, RemotePostbox from) { if (this.messagesFromLocalShards[from.ThreadIndex].payload.Length == 0) { // Message has been disabled to conserve memory. this.messagesFromLocalShards[from.ThreadIndex].Enable(); } this.messagesFromLocalShards[from.ThreadIndex].payload[this.messagesFromLocalShards[from.ThreadIndex].length++] = record; if (this.messagesFromLocalShards[from.ThreadIndex].length == this.messagesFromLocalShards[from.ThreadIndex].payload.Length) { if (this.endpoint.Vertex.Scheduler.Index == from.ThreadIndex && this.endpoint.AvailableEntrancy >= 0) { for (int i = 0; i < this.messagesFromLocalShards[from.ThreadIndex].length; i++) { if (progressBuffer != null) { progressBuffer.Update(this.messagesFromLocalShards[from.ThreadIndex].payload[i].v2, -1); } } // capturing buffer to avoid writing to it in reentrancy. var buffer = this.messagesFromLocalShards[from.ThreadIndex]; this.messagesFromLocalShards[from.ThreadIndex].Disable(); this.endpoint.AvailableEntrancy = this.endpoint.AvailableEntrancy - 1; this.endpoint.MessageReceived(buffer, from); ThreadLocalBufferPools <Pair <S, T> > .pool.Value.CheckIn(buffer.payload); buffer.Disable(); this.Drain(); this.endpoint.AvailableEntrancy = this.endpoint.AvailableEntrancy + 1; } else { this.Post(this.messagesFromLocalShards[from.ThreadIndex]); this.messagesFromLocalShards[from.ThreadIndex].Disable(); } } }
public void Flush(RemotePostbox from) { }
public abstract void Send(Pair <S, T> record, RemotePostbox from);
public abstract void Flush(RemotePostbox from);
public void DeliverSerializedMessage(SerializedMessage message, RemotePostbox from) { throw new NotImplementedException("Attempted to deliver message from a remote sender to a remote recipient, which is not currently supported."); }
public void Send(Pair <S, T> record, RemotePostbox from) { this.encodersFromLocalShards[from.ThreadIndex].Write(record, from.ShardID); }
public void Flush(RemotePostbox from) { this.encodersFromLocalShards[from.ThreadIndex].Flush(); }
/// <summary> /// "Sends" a progress message by poking the consumer directly. /// </summary> /// <param name="record"></param> /// <param name="from"></param> public void Send(Pair <Int64, Pointstamp> record, RemotePostbox from) { this.recordsReceived += 1; this.consumer.scheduler.statistics[(int)RuntimeStatistic.ProgressLocalRecords] += 1; this.consumer.ProcessCountChange(record.v2, record.v1); }
public abstract void DeliverSerializedMessage(SerializedMessage message, RemotePostbox from);