/// <summary> /// Reads the specified number of bits into a preallocated array /// </summary> /// <param name="into">The destination array</param> /// <param name="offset">The offset where to start writing in the destination array</param> /// <param name="numberOfBits">The number of bits to read</param> public void ReadBits(byte[] into, int offset, int numberOfBits) { NetException.Assert(m_bitLength - m_readPosition >= numberOfBits, c_readOverflowError); NetException.Assert(offset + NetUtility.BytesToHoldBits(numberOfBits) <= into.Length); int numberOfWholeBytes = numberOfBits / 8; int extraBits = numberOfBits - (numberOfWholeBytes * 8); NetBitWriter.ReadBytes(m_data, numberOfWholeBytes, m_readPosition, into, offset); m_readPosition += (8 * numberOfWholeBytes); if (extraBits > 0) { into[offset + numberOfWholeBytes] = ReadByte(extraBits); } return; }
public UInt64 PeekUInt64(int numberOfBits) { NetException.Assert((numberOfBits > 0 && numberOfBits <= 64), "ReadUInt() can only read between 1 and 64 bits"); NetException.Assert(m_bitLength - m_readPosition >= numberOfBits, c_readOverflowError); ulong retval; if (numberOfBits <= 32) { retval = (ulong)NetBitWriter.ReadUInt32(m_data, numberOfBits, m_readPosition); } else { retval = NetBitWriter.ReadUInt32(m_data, 32, m_readPosition); retval |= NetBitWriter.ReadUInt32(m_data, numberOfBits - 32, m_readPosition) << 32; } return(retval); }
private void ExecuteSend(float now, NetOutgoingMessage message) { int seqNr = m_sendStart; m_sendStart = (m_sendStart + 1) % NetConstants.NumSequenceNumbers; m_connection.QueueSendMessage(message, seqNr); int storeIndex = seqNr % m_windowSize; NetException.Assert(m_storedMessages[storeIndex].Message == null); m_storedMessages[storeIndex].NumSent++; m_storedMessages[storeIndex].Message = message; m_storedMessages[storeIndex].LastSent = now; return; }
/// <summary> /// Send a message to a specific connection /// </summary> /// <param name="msg">The message to send</param> /// <param name="recipient">The recipient connection</param> /// <param name="method">How to deliver the message</param> /// <param name="sequenceChannel">Sequence channel within the delivery method</param> public NetSendResult SendMessage(NetOutgoingMessage msg, NetConnection recipient, NetDeliveryMethod method, int sequenceChannel) { if (msg == null) { throw new ArgumentNullException("msg"); } if (recipient == null) { throw new ArgumentNullException("recipient"); } if (sequenceChannel >= NetConstants.NetChannelsPerDeliveryMethod) { throw new ArgumentOutOfRangeException("sequenceChannel"); } NetException.Assert( ((method != NetDeliveryMethod.Unreliable && method != NetDeliveryMethod.ReliableUnordered) || ((method == NetDeliveryMethod.Unreliable || method == NetDeliveryMethod.ReliableUnordered) && sequenceChannel == 0)), "Delivery method " + method + " cannot use sequence channels other than 0!" ); NetException.Assert(method != NetDeliveryMethod.Unknown, "Bad delivery method!"); if (msg.m_isSent) { throw new NetException("This message has already been sent! Use NetPeer.SendMessage() to send to multiple recipients efficiently"); } msg.m_isSent = true; int len = NetConstants.UnfragmentedMessageHeaderSize + msg.LengthBytes; // headers + length, faster than calling msg.GetEncodedSize if (len <= recipient.m_currentMTU) { Interlocked.Increment(ref msg.m_recyclingCount); return(recipient.EnqueueMessage(msg, method, sequenceChannel)); } else { // message must be fragmented! SendFragmentedMessage(msg, new NetConnection[] { recipient }, method, sequenceChannel); return(NetSendResult.Queued); // could be different for each connection; Queued is "most true" } }
private NetReceiverChannelBase CreateReceiverChannel(NetMessageType tp) { m_peer.VerifyNetworkThread(); // create receiver channel NetReceiverChannelBase chan; NetDeliveryMethod method = NetUtility.GetDeliveryMethod(tp); switch (method) { case NetDeliveryMethod.Unreliable: chan = new NetUnreliableUnorderedReceiver(this); break; case NetDeliveryMethod.ReliableOrdered: chan = new NetReliableOrderedReceiver(this, NetConstants.ReliableOrderedWindowSize); break; case NetDeliveryMethod.UnreliableSequenced: chan = new NetUnreliableSequencedReceiver(this); break; case NetDeliveryMethod.ReliableUnordered: chan = new NetReliableUnorderedReceiver(this, NetConstants.ReliableOrderedWindowSize); break; case NetDeliveryMethod.ReliableSequenced: chan = new NetReliableSequencedReceiver(this, NetConstants.ReliableSequencedWindowSize); break; default: throw new NetException("Unhandled NetDeliveryMethod!"); } int channelSlot = (int)tp - 1; NetException.Assert(m_receiveChannels[channelSlot] == null); m_receiveChannels[channelSlot] = chan; return(chan); }
/// <summary> /// Reads a string written using Write(string) /// </summary> public string ReadString() { int byteLen = (int)ReadVariableUInt32(); if (byteLen == 0) { return(String.Empty); } NetException.Assert(m_bitLength - m_readPosition >= (byteLen * 8), c_readOverflowError); if ((m_readPosition & 7) == 0) { // read directly string retval = System.Text.Encoding.UTF8.GetString(m_data, m_readPosition >> 3, byteLen); m_readPosition += (8 * byteLen); return(retval); } byte[] bytes = ReadBytes(byteLen); return(System.Text.Encoding.UTF8.GetString(bytes, 0, bytes.Length)); }
internal void Write(Int32 source, int numberOfBits) { NetException.Assert((numberOfBits > 0 && numberOfBits <= 32), "Write(int, numberOfBits) can only write between 1 and 32 bits"); InternalEnsureBufferSize(m_bitLength + numberOfBits); if (numberOfBits != 32) { // make first bit sign int signBit = 1 << (numberOfBits - 1); if (source < 0) { source = (-source - 1) | signBit; } else { source &= (~signBit); } } NetBitWriter.WriteUInt32((uint)source, numberOfBits, m_data, m_bitLength); m_bitLength += numberOfBits; }
/// <summary> /// Sets or clears the bit/bool at the specified index /// </summary> public void Set(int bitIndex, bool value) { NetException.Assert(bitIndex >= 0 && bitIndex < m_capacity); int idx = bitIndex / 32; if (value) { if ((m_data[idx] & (1 << (bitIndex % 32))) == 0) { m_numBitsSet++; } m_data[idx] |= (1 << (bitIndex % 32)); } else { if ((m_data[idx] & (1 << (bitIndex % 32))) != 0) { m_numBitsSet--; } m_data[idx] &= (~(1 << (bitIndex % 32))); } }
/// <summary> /// Write a byte consisting of 1-8 bits to a buffer; assumes buffer is previously allocated /// </summary> public static void WriteByte(byte source, int numberOfBits, byte[] destination, int destBitOffset) { NetException.Assert(((numberOfBits >= 1) && (numberOfBits <= 8)), "Must write between 1 and 8 bits!"); // mask out unwanted bits in the source byte isrc = (byte)((uint)source & ((~(uint)0) >> (8 - numberOfBits))); int bytePtr = destBitOffset >> 3; int localBitLen = (destBitOffset % 8); if (localBitLen == 0) { destination[bytePtr] = (byte)isrc; return; } //destination[bytePtr] &= (byte)(255 >> (8 - localBitLen)); // clear before writing //destination[bytePtr] |= (byte)(isrc << localBitLen); // write first half destination[bytePtr] = (byte)( (uint)(destination[bytePtr] & (255 >> (8 - localBitLen))) | (uint)(isrc << localBitLen) ); // need write into next byte? if (localBitLen + numberOfBits > 8) { //destination[bytePtr + 1] &= (byte)(255 << localBitLen); // clear before writing //destination[bytePtr + 1] |= (byte)(isrc >> (8 - localBitLen)); // write second half destination[bytePtr + 1] = (byte)( (uint)(destination[bytePtr + 1] & (255 << localBitLen)) | (uint)(isrc >> (8 - localBitLen)) ); } return; }
private void Heartbeat() { VerifyNetworkThread(); double dnow = NetTime.Now; float now = (float)dnow; double delta = dnow - m_lastHeartbeat; int maxCHBpS = 1250 - m_connections.Count; if (maxCHBpS < 250) { maxCHBpS = 250; } if (delta > (1.0 / (double)maxCHBpS)) // max connection heartbeats/second max { m_frameCounter++; m_lastHeartbeat = dnow; // do handshake heartbeats if ((m_frameCounter % 3) == 0) { foreach (var kvp in m_handshakes) { NetConnection conn = kvp.Value as NetConnection; #if DEBUG // sanity check if (kvp.Key != kvp.Key) { LogWarning("Sanity fail! Connection in handshake list under wrong key!"); } #endif conn.UnconnectedHeartbeat(now); if (conn.m_status == NetConnectionStatus.Connected || conn.m_status == NetConnectionStatus.Disconnected) { #if DEBUG // sanity check if (conn.m_status == NetConnectionStatus.Disconnected && m_handshakes.ContainsKey(conn.RemoteEndpoint)) { LogWarning("Sanity fail! Handshakes list contained disconnected connection!"); m_handshakes.Remove(conn.RemoteEndpoint); } #endif break; // collection has been modified } } } #if DEBUG SendDelayedPackets(); #endif // do connection heartbeats lock (m_connections) { foreach (NetConnection conn in m_connections) { conn.Heartbeat(now, m_frameCounter); if (conn.m_status == NetConnectionStatus.Disconnected) { // // remove connection // m_connections.Remove(conn); m_connectionLookup.Remove(conn.RemoteEndpoint); break; // can't continue iteration here } } } // send unsent unconnected messages NetTuple <IPEndPoint, NetOutgoingMessage> unsent; while (m_unsentUnconnectedMessages.TryDequeue(out unsent)) { NetOutgoingMessage om = unsent.Item2; #if DEBUG if (om.m_messageType == NetMessageType.NatPunchMessage) { LogDebug("Sending Nat Punch Message to " + unsent.Item1.ToString()); } #endif bool connReset; int len = om.Encode(m_sendBuffer, 0, 0); SendPacket(len, unsent.Item1, 1, out connReset); Interlocked.Decrement(ref om.m_recyclingCount); if (om.m_recyclingCount <= 0) { Recycle(om); } } } // // read from socket // if (m_socket == null) { return; } if (!m_socket.Poll(1000)) // wait up to 1 ms for data to arrive { return; } //if (m_socket == null || m_socket.Available < 1) // return; do { int bytesReceived = 0; try { bytesReceived = m_socket.ReceiveFrom(m_receiveBuffer, 0, m_receiveBuffer.Length, ref m_senderRemote); } catch (SocketException sx) { if (sx.SocketErrorCode == SocketError.ConnectionReset) { // connection reset by peer, aka connection forcibly closed aka "ICMP port unreachable" // we should shut down the connection; but m_senderRemote seemingly cannot be trusted, so which connection should we shut down?! // So, what to do? return; } LogWarning(sx.ToString()); return; } if (bytesReceived < NetConstants.HeaderByteSize) { return; } //LogVerbose("Received " + bytesReceived + " bytes"); IPEndPoint ipsender = (IPEndPoint)m_senderRemote; if (ipsender.Port == 1900) { // UPnP response try { string resp = System.Text.Encoding.UTF8.GetString(m_receiveBuffer, 0, bytesReceived); if (resp.Contains("upnp:rootdevice")) { resp = resp.Substring(resp.ToLower().IndexOf("location:") + 9); resp = resp.Substring(0, resp.IndexOf("\r")).Trim(); m_upnp.ExtractServiceUrl(resp); return; } } catch { } } NetConnection sender = null; m_connectionLookup.TryGetValue(ipsender, out sender); double receiveTime = NetTime.Now; // // parse packet into messages // int numMessages = 0; int ptr = 0; while ((bytesReceived - ptr) >= NetConstants.HeaderByteSize) { // decode header // 8 bits - NetMessageType // 1 bit - Fragment? // 15 bits - Sequence number // 16 bits - Payload length in bits numMessages++; NetMessageType tp = (NetMessageType)m_receiveBuffer[ptr++]; byte low = m_receiveBuffer[ptr++]; byte high = m_receiveBuffer[ptr++]; bool isFragment = ((low & 1) == 1); ushort sequenceNumber = (ushort)((low >> 1) | (((int)high) << 7)); ushort payloadBitLength = (ushort)(m_receiveBuffer[ptr++] | (m_receiveBuffer[ptr++] << 8)); int payloadByteLength = NetUtility.BytesToHoldBits(payloadBitLength); if (bytesReceived - ptr < payloadByteLength) { LogWarning("Malformed packet; stated payload length " + payloadByteLength + ", remaining bytes " + (bytesReceived - ptr)); return; } try { NetException.Assert(tp <NetMessageType.Unused1 || tp> NetMessageType.Unused29); if (tp >= NetMessageType.LibraryError) { if (sender != null) { sender.ReceivedLibraryMessage(tp, ptr, payloadByteLength); } else { ReceivedUnconnectedLibraryMessage(receiveTime, ipsender, tp, ptr, payloadByteLength); } } else { if (sender == null && !m_configuration.IsMessageTypeEnabled(NetIncomingMessageType.UnconnectedData)) { return; // dropping unconnected message since it's not enabled } NetIncomingMessage msg = CreateIncomingMessage(NetIncomingMessageType.Data, payloadByteLength); msg.m_isFragment = isFragment; msg.m_receiveTime = receiveTime; msg.m_sequenceNumber = sequenceNumber; msg.m_receivedMessageType = tp; msg.m_senderConnection = sender; msg.m_senderEndpoint = ipsender; msg.m_bitLength = payloadBitLength; Buffer.BlockCopy(m_receiveBuffer, ptr, msg.m_data, 0, payloadByteLength); if (sender != null) { if (tp == NetMessageType.Unconnected) { // We're connected; but we can still send unconnected messages to this peer msg.m_incomingMessageType = NetIncomingMessageType.UnconnectedData; ReleaseMessage(msg); } else { // connected application (non-library) message sender.ReceivedMessage(msg); } } else { // at this point we know the message type is enabled // unconnected application (non-library) message msg.m_incomingMessageType = NetIncomingMessageType.UnconnectedData; ReleaseMessage(msg); } } } catch (Exception ex) { LogError("Packet parsing error: " + ex.Message + " from " + ipsender); } ptr += payloadByteLength; } m_statistics.PacketReceived(bytesReceived, numMessages); if (sender != null) { sender.m_statistics.PacketReceived(bytesReceived, numMessages); } } while (m_socket.Available > 0); }
private void HandleReleasedFragment(NetIncomingMessage im) { // // read fragmentation header and combine fragments // int group; int totalBits; int chunkByteSize; int chunkNumber; int ptr = NetFragmentationHelper.ReadHeader( im.m_data, 0, out group, out totalBits, out chunkByteSize, out chunkNumber ); NetException.Assert(im.LengthBytes > ptr); NetException.Assert(group > 0); NetException.Assert(totalBits > 0); NetException.Assert(chunkByteSize > 0); int totalBytes = NetUtility.BytesToHoldBits((int)totalBits); int totalNumChunks = totalBytes / chunkByteSize; if (totalNumChunks * chunkByteSize < totalBytes) { totalNumChunks++; } NetException.Assert(chunkNumber < totalNumChunks); if (chunkNumber >= totalNumChunks) { LogWarning("Index out of bounds for chunk " + chunkNumber + " (total chunks " + totalNumChunks + ")"); return; } Dictionary <int, ReceivedFragmentGroup> groups; if (!m_receivedFragmentGroups.TryGetValue(im.SenderConnection, out groups)) { groups = new Dictionary <int, ReceivedFragmentGroup>(); m_receivedFragmentGroups[im.SenderConnection] = groups; } ReceivedFragmentGroup info; if (!groups.TryGetValue(group, out info)) { info = new ReceivedFragmentGroup(); info.Data = new byte[totalBytes]; info.ReceivedChunks = new NetBitVector(totalNumChunks); groups[group] = info; } info.ReceivedChunks[chunkNumber] = true; info.LastReceived = (float)NetTime.Now; // copy to data int offset = (chunkNumber * chunkByteSize); Buffer.BlockCopy(im.m_data, ptr, info.Data, offset, im.LengthBytes - ptr); int cnt = info.ReceivedChunks.Count(); //LogVerbose("Found fragment #" + chunkNumber + " in group " + group + " offset " + offset + " of total bits " + totalBits + " (total chunks done " + cnt + ")"); LogVerbose("Received fragment " + chunkNumber + " of " + totalNumChunks + " (" + cnt + " chunks received)"); if (info.ReceivedChunks.Count() == totalNumChunks) { // Done! Transform this incoming message im.m_data = info.Data; im.m_bitLength = (int)totalBits; im.m_isFragment = false; LogVerbose("Fragment group #" + group + " fully received in " + totalNumChunks + " chunks (" + totalBits + " bits)"); ReleaseMessage(im); } else { // data has been copied; recycle this incoming message Recycle(im); } return; }
/// <summary> /// Sets all bits/booleans to zero/false /// </summary> public void Clear() { Array.Clear(m_data, 0, m_data.Length); m_numBitsSet = 0; NetException.Assert(this.IsEmpty()); }
/// <summary> /// Gets the bit/bool at the specified index /// </summary> public bool Get(int bitIndex) { NetException.Assert(bitIndex >= 0 && bitIndex < m_capacity); return((m_data[bitIndex / 32] & (1 << (bitIndex % 32))) != 0); }
// remoteWindowStart is remote expected sequence number; everything below this has arrived properly // seqNr is the actual nr received internal override void ReceiveAcknowledge(float now, int seqNr) { // late (dupe), on time or early ack? int relate = NetUtility.RelativeSequenceNumber(seqNr, m_windowStart); if (relate < 0) { //m_connection.m_peer.LogDebug("Received late/dupe ack for #" + seqNr); return; // late/duplicate ack } if (relate == 0) { //m_connection.m_peer.LogDebug("Received right-on-time ack for #" + seqNr); // ack arrived right on time NetException.Assert(seqNr == m_windowStart); m_receivedAcks[m_windowStart] = false; DestoreMessage(m_windowStart % m_windowSize); m_windowStart = (m_windowStart + 1) % NetConstants.NumSequenceNumbers; // advance window if we already have early acks while (m_receivedAcks.Get(m_windowStart)) { //m_connection.m_peer.LogDebug("Using early ack for #" + m_windowStart + "..."); m_receivedAcks[m_windowStart] = false; NetException.Assert(m_storedMessages[m_windowStart % m_windowSize].Message == null); // should already be destored m_windowStart = (m_windowStart + 1) % NetConstants.NumSequenceNumbers; //m_connection.m_peer.LogDebug("Advancing window to #" + m_windowStart); } return; } // // early ack... (if it has been sent!) // // If it has been sent either the m_windowStart message was lost // ... or the ack for that message was lost // //m_connection.m_peer.LogDebug("Received early ack for #" + seqNr); int sendRelate = NetUtility.RelativeSequenceNumber(seqNr, m_sendStart); if (sendRelate <= 0) { // yes, we've sent this message - it's an early (but valid) ack if (m_receivedAcks[seqNr]) { // we've already destored/been acked for this message } else { DestoreMessage(seqNr % m_windowSize); m_receivedAcks[seqNr] = true; } } else if (sendRelate > 0) { // uh... we haven't sent this message yet? Weird, dupe or error... return; } // Ok, lets resend all missing acks int rnr = seqNr; do { rnr--; if (rnr < 0) { rnr = NetConstants.NumSequenceNumbers - 1; } if (m_receivedAcks[rnr]) { // m_connection.m_peer.LogDebug("Not resending #" + rnr + " (since we got ack)"); } else { int slot = rnr % m_windowSize; NetException.Assert(m_storedMessages[slot].Message != null); if (m_storedMessages[slot].NumSent == 1) { // just sent once; resend immediately since we found gap in ack sequence NetOutgoingMessage rmsg = m_storedMessages[slot].Message; //m_connection.m_peer.LogVerbose("Resending #" + rnr + " (" + rmsg + ")"); if (now - m_storedMessages[slot].LastSent < (m_resendDelay * 0.35f)) { // already resent recently } else { m_storedMessages[slot].LastSent = now; m_storedMessages[slot].NumSent++; m_connection.m_statistics.MessageResent(MessageResendReason.HoleInSequence); m_connection.QueueSendMessage(rmsg, rnr); } } } } while (rnr != m_windowStart); }
internal void Heartbeat(float now, uint frameCounter) { m_peer.VerifyNetworkThread(); NetException.Assert(m_status != NetConnectionStatus.InitiatedConnect && m_status != NetConnectionStatus.RespondedConnect); if ((frameCounter % 5) == 0) { if (now > m_timeoutDeadline) { // // connection timed out // m_peer.LogVerbose("Connection timed out at " + now + " deadline was " + m_timeoutDeadline); ExecuteDisconnect("Connection timed out", true); } // send ping? if (m_status == NetConnectionStatus.Connected) { if (now > m_sentPingTime + m_peer.m_configuration.m_pingInterval) { SendPing(); } // handle expand mtu MTUExpansionHeartbeat(now); } if (m_disconnectRequested) { ExecuteDisconnect(m_disconnectMessage, true); return; } } bool connectionReset; // TODO: handle connection reset // // Note: at this point m_sendBufferWritePtr and m_sendBufferNumMessages may be non-null; resends may already be queued up // byte[] sendBuffer = m_peer.m_sendBuffer; int mtu = m_currentMTU; if ((frameCounter % 3) == 0) // coalesce a few frames { // // send ack messages // while (m_queuedAcks.Count > 0) { int acks = (mtu - (m_sendBufferWritePtr + 5)) / 3; // 3 bytes per actual ack if (acks > m_queuedAcks.Count) { acks = m_queuedAcks.Count; } NetException.Assert(acks > 0); m_sendBufferNumMessages++; // write acks header sendBuffer[m_sendBufferWritePtr++] = (byte)NetMessageType.Acknowledge; sendBuffer[m_sendBufferWritePtr++] = 0; // no sequence number sendBuffer[m_sendBufferWritePtr++] = 0; // no sequence number int len = (acks * 3) * 8; // bits sendBuffer[m_sendBufferWritePtr++] = (byte)len; sendBuffer[m_sendBufferWritePtr++] = (byte)(len >> 8); // write acks for (int i = 0; i < acks; i++) { NetTuple <NetMessageType, int> tuple; m_queuedAcks.TryDequeue(out tuple); //m_peer.LogVerbose("Sending ack for " + tuple.Item1 + "#" + tuple.Item2); sendBuffer[m_sendBufferWritePtr++] = (byte)tuple.Item1; sendBuffer[m_sendBufferWritePtr++] = (byte)tuple.Item2; sendBuffer[m_sendBufferWritePtr++] = (byte)(tuple.Item2 >> 8); } if (m_queuedAcks.Count > 0) { // send packet and go for another round of acks NetException.Assert(m_sendBufferWritePtr > 0 && m_sendBufferNumMessages > 0); m_peer.SendPacket(m_sendBufferWritePtr, m_remoteEndpoint, m_sendBufferNumMessages, out connectionReset); m_statistics.PacketSent(m_sendBufferWritePtr, 1); m_sendBufferWritePtr = 0; m_sendBufferNumMessages = 0; } } } // // send queued messages // for (int i = m_sendChannels.Length - 1; i >= 0; i--) // Reverse order so reliable messages are sent first { var channel = m_sendChannels[i]; NetException.Assert(m_sendBufferWritePtr < 1 || m_sendBufferNumMessages > 0); if (channel != null) { channel.SendQueuedMessages(now); } NetException.Assert(m_sendBufferWritePtr < 1 || m_sendBufferNumMessages > 0); } // // Put on wire data has been written to send buffer but not yet sent // if (m_sendBufferWritePtr > 0) { m_peer.VerifyNetworkThread(); NetException.Assert(m_sendBufferWritePtr > 0 && m_sendBufferNumMessages > 0); m_peer.SendPacket(m_sendBufferWritePtr, m_remoteEndpoint, m_sendBufferNumMessages, out connectionReset); m_statistics.PacketSent(m_sendBufferWritePtr, m_sendBufferNumMessages); m_sendBufferWritePtr = 0; m_sendBufferNumMessages = 0; } }
/// <summary> /// Reads a signed integer stored in 1 to 64 bits, written using Write(Int64, Int32) /// </summary> public Int64 ReadInt64(int numberOfBits) { NetException.Assert(((numberOfBits > 0) && (numberOfBits <= 64)), "ReadInt64(bits) can only read between 1 and 64 bits"); return((long)ReadUInt64(numberOfBits)); }
private void SendCallBack(IAsyncResult res) { NetException.Assert(res.IsCompleted == true); m_socket.EndSendTo(res); }