public bool EnqueueOutgoing(OutgoingPacket packet)
{
int category = (int)packet.Category;
if (category >= 0 && category < m_packetOutboxes.Length)
{
LocklessQueue <OutgoingPacket> queue = m_packetOutboxes[category];
TokenBucket bucket = m_throttleCategories[category];
if (bucket.RemoveTokens(packet.Buffer.DataLength))
{
// Enough tokens were removed from the bucket, the packet will not be queued
return(false);
}
else
{
// Not enough tokens in the bucket, queue this packet
queue.Enqueue(packet);
return(true);
}
}
else
{
// We don't have a token bucket for this category, so it will not be queued
return(false);
}
}
public void ResendUnacked(LLAgent agent)
{
//FIXME: Make this an .ini setting
const int AGENT_TIMEOUT_MS = 1000 * 60;
// Disconnect an agent if no packets are received for some time
if (Util.TickCount() - agent.TickLastPacketReceived > AGENT_TIMEOUT_MS)
{
m_log.Warn("Ack timeout, disconnecting " + agent.ID);
ShutdownClient(agent);
return;
}
// Get a list of all of the packets that have been sitting unacked longer than udpClient.RTO
List <OutgoingPacket> expiredPackets = agent.NeedAcks.GetExpiredPackets(agent.RTO);
if (expiredPackets != null)
{
//m_log.Debug("[LLUDPSERVER]: Resending " + expiredPackets.Count + " packets to " + udpClient.AgentID + ", RTO=" + udpClient.RTO);
// Exponential backoff of the retransmission timeout
agent.BackoffRTO();
// Resend packets
for (int i = 0; i < expiredPackets.Count; i++)
{
OutgoingPacket outgoingPacket = expiredPackets[i];
//m_log.DebugFormat("[LLUDPSERVER]: Resending packet #{0} (attempt {1}), {2}ms have passed",
// outgoingPacket.SequenceNumber, outgoingPacket.ResendCount, Environment.TickCount - outgoingPacket.TickCount);
// Set the resent flag
outgoingPacket.Buffer.Data[0] = (byte)(outgoingPacket.Buffer.Data[0] | Helpers.MSG_RESENT);
outgoingPacket.Category = ThrottleCategory.Resend;
// Bump up the resend count on this packet
Interlocked.Increment(ref outgoingPacket.ResendCount);
Interlocked.Increment(ref m_packetsResent);
// Requeue or resend the packet
if (!agent.EnqueueOutgoing(outgoingPacket))
{
SendPacketFinal(outgoingPacket);
}
}
}
}
/// <summary>
/// Add an unacked packet to the collection
/// </summary>
/// <param name="packet">Packet that is awaiting acknowledgement</param>
/// <returns>True if the packet was successfully added, false if the
/// packet already existed in the collection</returns>
/// <remarks>This does not immediately add the ACK to the collection,
/// it only queues it so it can be added in a thread-safe way later</remarks>
public void Add(OutgoingPacket packet)
{
m_pendingAdds.Enqueue(packet);
}
internal void SendPacketFinal(OutgoingPacket outgoingPacket)
{
const int MAX_APPENDED_ACKS = 250;
UDPPacketBuffer buffer = outgoingPacket.Buffer;
byte flags = buffer.Data[0];
bool isResend = (flags & Helpers.MSG_RESENT) != 0;
bool isReliable = (flags & Helpers.MSG_RELIABLE) != 0;
bool isZerocoded = (flags & Helpers.MSG_ZEROCODED) != 0;
LLAgent agent = outgoingPacket.Agent;
if (!agent.IsConnected || buffer.RemoteEndPoint == null)
{
m_log.Debug("Dropping " + buffer.DataLength + " byte packet to client we have no route to");
return;
}
int dataLength = buffer.DataLength;
#region ACK Appending
// NOTE: I'm seeing problems with some viewers when ACKs are appended to zerocoded packets so I've disabled that here
if (!isZerocoded && outgoingPacket.Type != PacketType.PacketAck)
{
// Keep appending ACKs until there is no room left in the buffer or there are
// no more ACKs to append
byte ackCount = 0;
uint ack;
while (dataLength + 5 < buffer.Data.Length && ackCount < MAX_APPENDED_ACKS && agent.PendingAcks.TryDequeue(out ack))
{
Utils.UIntToBytesBig(ack, buffer.Data, dataLength);
dataLength += 4;
++ackCount;
}
if (ackCount > 0)
{
// Set the last byte of the packet equal to the number of appended ACKs
buffer.Data[dataLength++] = ackCount;
// Set the appended ACKs flag on this packet
buffer.Data[0] |= Helpers.MSG_APPENDED_ACKS;
}
}
#endregion ACK Appending
buffer.DataLength = dataLength;
#region Sequence Number Assignment
if (!isResend)
{
// Not a resend, assign a new sequence number
uint sequenceNumber = (uint)Interlocked.Increment(ref agent.CurrentSequence);
Utils.UIntToBytesBig(sequenceNumber, buffer.Data, 1);
outgoingPacket.SequenceNumber = sequenceNumber;
if (isReliable)
{
// Add this packet to the list of ACK responses we are waiting on from the server
agent.NeedAcks.Add(outgoingPacket);
}
}
#endregion Sequence Number Assignment
// Stats tracking
Interlocked.Increment(ref agent.PacketsSent);
Interlocked.Increment(ref m_packetsSent);
if (isReliable)
Interlocked.Add(ref agent.UnackedBytes, outgoingPacket.Buffer.DataLength);
if (LoggingEnabled)
m_log.Debug("<-- (" + buffer.RemoteEndPoint + ") " + outgoingPacket.Type);
// Put the UDP payload on the wire
Send(buffer);
// Keep track of when this packet was sent out (right now)
outgoingPacket.TickCount = Util.TickCount();
//m_log.Debug("Sent " + outgoingPacket.Buffer.DataLength + " byte " + outgoingPacket.Category + " packet");
}
public void SendPacketData(LLAgent agent, byte[] data, PacketType type, ThrottleCategory category)
{
int dataLength = data.Length;
bool doZerocode = (data[0] & Helpers.MSG_ZEROCODED) != 0;
bool doCopy = true;
// Frequency analysis of outgoing packet sizes shows a large clump of packets at each end of the spectrum.
// The vast majority of packets are less than 200 bytes, although due to asset transfers and packet splitting
// there are a decent number of packets in the 1000-1140 byte range. We allocate one of two sizes of data here
// to accomodate for both common scenarios and provide ample room for ACK appending in both
int bufferSize = (dataLength > 180)
? LLUDPServer.MTU
: 200;
UDPPacketBuffer buffer = new UDPPacketBuffer(agent.RemoteEndPoint, bufferSize);
// Zerocode if needed
if (doZerocode)
{
try
{
dataLength = Helpers.ZeroEncode(data, dataLength, buffer.Data);
doCopy = false;
}
catch (IndexOutOfRangeException)
{
// The packet grew larger than the bufferSize while zerocoding.
// Remove the MSG_ZEROCODED flag and send the unencoded data
// instead
m_log.Debug("Packet exceeded buffer size during zerocoding for " + type + ". DataLength=" + dataLength +
" and BufferLength=" + buffer.Data.Length + ". Removing MSG_ZEROCODED flag");
data[0] = (byte)(data[0] & ~Helpers.MSG_ZEROCODED);
}
}
// If the packet data wasn't already copied during zerocoding, copy it now
if (doCopy)
{
if (dataLength > buffer.Data.Length)
{
m_log.Error("Packet exceeded buffer size! This could be an indication of packet assembly not obeying the MTU. Type=" +
type + ", DataLength=" + dataLength + ", BufferLength=" + buffer.Data.Length);
buffer.Data = new byte[dataLength];
}
Buffer.BlockCopy(data, 0, buffer.Data, 0, dataLength);
}
buffer.DataLength = dataLength;
#region Queue or Send
OutgoingPacket outgoingPacket = new OutgoingPacket(agent, buffer, category, type);
if (!agent.EnqueueOutgoing(outgoingPacket))
SendPacketFinal(outgoingPacket);
#endregion Queue or Send
}
/// <summary>
/// Add an unacked packet to the collection
/// </summary>
/// <param name="packet">Packet that is awaiting acknowledgement</param>
/// <returns>True if the packet was successfully added, false if the
/// packet already existed in the collection</returns>
/// <remarks>This does not immediately add the ACK to the collection,
/// it only queues it so it can be added in a thread-safe way later</remarks>
public void Add(OutgoingPacket packet)
{
m_pendingAdds.Enqueue(packet);
}
public bool EnqueueOutgoing(OutgoingPacket packet)
{
int category = (int)packet.Category;
if (category >= 0 && category < m_packetOutboxes.Length)
{
LocklessQueue<OutgoingPacket> queue = m_packetOutboxes[category];
TokenBucket bucket = m_throttleCategories[category];
if (bucket.RemoveTokens(packet.Buffer.DataLength))
{
// Enough tokens were removed from the bucket, the packet will not be queued
return false;
}
else
{
// Not enough tokens in the bucket, queue this packet
queue.Enqueue(packet);
return true;
}
}
else
{
// We don't have a token bucket for this category, so it will not be queued
return false;
}
}
internal void SendPacketFinal(OutgoingPacket outgoingPacket)
{
const int MAX_APPENDED_ACKS = 250;
UDPPacketBuffer buffer = outgoingPacket.Buffer;
byte flags = buffer.Data[0];
bool isResend = (flags & Helpers.MSG_RESENT) != 0;
bool isReliable = (flags & Helpers.MSG_RELIABLE) != 0;
bool isZerocoded = (flags & Helpers.MSG_ZEROCODED) != 0;
LLAgent agent = outgoingPacket.Agent;
if (!agent.IsConnected || buffer.RemoteEndPoint == null)
{
m_log.Debug("Dropping " + buffer.DataLength + " byte packet to client we have no route to");
return;
}
int dataLength = buffer.DataLength;
#region ACK Appending
// NOTE: I'm seeing problems with some viewers when ACKs are appended to zerocoded packets so I've disabled that here
if (!isZerocoded && outgoingPacket.Type != PacketType.PacketAck)
{
// Keep appending ACKs until there is no room left in the buffer or there are
// no more ACKs to append
byte ackCount = 0;
uint ack;
while (dataLength + 5 < buffer.Data.Length && ackCount < MAX_APPENDED_ACKS && agent.PendingAcks.TryDequeue(out ack))
{
Utils.UIntToBytesBig(ack, buffer.Data, dataLength);
dataLength += 4;
++ackCount;
}
if (ackCount > 0)
{
// Set the last byte of the packet equal to the number of appended ACKs
buffer.Data[dataLength++] = ackCount;
// Set the appended ACKs flag on this packet
buffer.Data[0] |= Helpers.MSG_APPENDED_ACKS;
}
}
#endregion ACK Appending
buffer.DataLength = dataLength;
#region Sequence Number Assignment
if (!isResend)
{
// Not a resend, assign a new sequence number
uint sequenceNumber = (uint)Interlocked.Increment(ref agent.CurrentSequence);
Utils.UIntToBytesBig(sequenceNumber, buffer.Data, 1);
outgoingPacket.SequenceNumber = sequenceNumber;
if (isReliable)
{
// Add this packet to the list of ACK responses we are waiting on from the server
agent.NeedAcks.Add(outgoingPacket);
}
}
#endregion Sequence Number Assignment
// Stats tracking
Interlocked.Increment(ref agent.PacketsSent);
Interlocked.Increment(ref m_packetsSent);
if (isReliable)
{
Interlocked.Add(ref agent.UnackedBytes, outgoingPacket.Buffer.DataLength);
}
if (LoggingEnabled)
{
m_log.Debug("<-- (" + buffer.RemoteEndPoint + ") " + outgoingPacket.Type);
}
// Put the UDP payload on the wire
Send(buffer);
// Keep track of when this packet was sent out (right now)
outgoingPacket.TickCount = Util.TickCount();
//m_log.Debug("Sent " + outgoingPacket.Buffer.DataLength + " byte " + outgoingPacket.Category + " packet");
}
public void SendPacketData(LLAgent agent, byte[] data, PacketType type, ThrottleCategory category)
{
int dataLength = data.Length;
bool doZerocode = (data[0] & Helpers.MSG_ZEROCODED) != 0;
bool doCopy = true;
// Frequency analysis of outgoing packet sizes shows a large clump of packets at each end of the spectrum.
// The vast majority of packets are less than 200 bytes, although due to asset transfers and packet splitting
// there are a decent number of packets in the 1000-1140 byte range. We allocate one of two sizes of data here
// to accomodate for both common scenarios and provide ample room for ACK appending in both
int bufferSize = (dataLength > 180)
? LLUDPServer.MTU
: 200;
UDPPacketBuffer buffer = new UDPPacketBuffer(agent.RemoteEndPoint, bufferSize);
// Zerocode if needed
if (doZerocode)
{
try
{
dataLength = Helpers.ZeroEncode(data, dataLength, buffer.Data);
doCopy = false;
}
catch (IndexOutOfRangeException)
{
// The packet grew larger than the bufferSize while zerocoding.
// Remove the MSG_ZEROCODED flag and send the unencoded data
// instead
m_log.Debug("Packet exceeded buffer size during zerocoding for " + type + ". DataLength=" + dataLength +
" and BufferLength=" + buffer.Data.Length + ". Removing MSG_ZEROCODED flag");
data[0] = (byte)(data[0] & ~Helpers.MSG_ZEROCODED);
}
}
// If the packet data wasn't already copied during zerocoding, copy it now
if (doCopy)
{
if (dataLength > buffer.Data.Length)
{
m_log.Error("Packet exceeded buffer size! This could be an indication of packet assembly not obeying the MTU. Type=" +
type + ", DataLength=" + dataLength + ", BufferLength=" + buffer.Data.Length);
buffer.Data = new byte[dataLength];
}
Buffer.BlockCopy(data, 0, buffer.Data, 0, dataLength);
}
buffer.DataLength = dataLength;
#region Queue or Send
OutgoingPacket outgoingPacket = new OutgoingPacket(agent, buffer, category, type);
if (!agent.EnqueueOutgoing(outgoingPacket))
{
SendPacketFinal(outgoingPacket);
}
#endregion Queue or Send
}
/// <summary>
/// Loops through all of the packet queues for this client and tries to send
/// any outgoing packets, obeying the throttling bucket limits
/// </summary>
/// <remarks>This function is only called from a synchronous loop in the
/// UDPServer so we don't need to bother making this thread safe</remarks>
/// <returns>True if any packets were sent, otherwise false</returns>
public bool DequeueOutgoing()
{
OutgoingPacket packet;
LocklessQueue <OutgoingPacket> queue;
TokenBucket bucket;
bool packetSent = false;
ThrottleCategoryFlags emptyCategories = 0;
//string queueDebugOutput = String.Empty; // Serious debug business
for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++)
{
bucket = m_throttleCategories[i];
//queueDebugOutput += m_packetOutboxes[i].Count + " "; // Serious debug business
if (m_nextPackets[i] != null)
{
// This bucket was empty the last time we tried to send a packet,
// leaving a dequeued packet still waiting to be sent out. Try to
// send it again
OutgoingPacket nextPacket = m_nextPackets[i];
if (bucket.RemoveTokens(nextPacket.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(nextPacket);
m_nextPackets[i] = null;
packetSent = true;
}
}
else
{
// No dequeued packet waiting to be sent, try to pull one off
// this queue
queue = m_packetOutboxes[i];
if (queue.TryDequeue(out packet))
{
// A packet was pulled off the queue. See if we have
// enough tokens in the bucket to send it out
if (bucket.RemoveTokens(packet.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(packet);
packetSent = true;
}
else
{
// Save the dequeued packet for the next iteration
m_nextPackets[i] = packet;
}
// If the queue is empty after this dequeue, fire the queue
// empty callback now so it has a chance to fill before we
// get back here
if (queue.Count == 0)
{
emptyCategories |= CategoryToFlag(i);
}
}
else
{
// No packets in this queue. Fire the queue empty callback
// if it has not been called recently
emptyCategories |= CategoryToFlag(i);
}
}
}
if (emptyCategories != 0)
{
BeginFireQueueEmpty(emptyCategories);
}
//m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business
return(packetSent);
}
