public bool EnqueueOutgoing(OutgoingPacket packet)
{
int category = (int)packet.Category;
if (category >= 0 && category < m_packetOutboxes.Length)
{
OpenSim.Framework.LocklessQueue <OutgoingPacket> queue = m_packetOutboxes[category];
TokenBucket bucket = m_throttleCategories[category];
if (bucket.RemoveTokens(packet.Buffer.DataLength))
{
// Enough tokens were removed from the bucket, the packet will not be queued
return(false);
}
else
{
// Not enough tokens in the bucket, queue this packet
queue.Enqueue(packet);
return(true);
}
}
else
{
// We don't have a token bucket for this category, so it will not be queued
return(false);
}
}
public void TestReportCriticalPacketDrop(OutgoingPacket packet)
{
lock (_dropReportLock)
{
if (DateTime.Now - _lastDropReport > TimeSpan.FromSeconds(MIN_DROP_REPORT_INTERVAL))
{
_lastDropReport = DateTime.Now;
m_log.WarnFormat("[LLUDP] Reliable packets are being dropped for {0} due to overfilled outbound queue. Last packet type {1}",
AgentID, packet.Type);
}
}
}
public void ResendUnacked(LLUDPClient udpClient)
{
if (!udpClient.IsConnected)
{
return;
}
// Disconnect an agent if no packets are received for some time
//FIXME: Make 60 an .ini setting
if ((Environment.TickCount & Int32.MaxValue) - udpClient.TickLastPacketReceived > 1000 * 60)
{
m_log.Warn("[LLUDPSERVER]: Ack timeout, disconnecting " + udpClient.AgentID);
RemoveClient(udpClient);
return;
}
// Get a list of all of the packets that have been sitting unacked longer than udpClient.RTO
List <OutgoingPacket> expiredPackets = udpClient.NeedAcks.GetExpiredPackets(udpClient.RTO);
if (expiredPackets != null)
{
//m_log.Debug("[LLUDPSERVER]: Resending " + expiredPackets.Count + " packets to " + udpClient.AgentID + ", RTO=" + udpClient.RTO);
// Exponential backoff of the retransmission timeout
udpClient.BackoffRTO();
// Resend packets
for (int i = 0; i < expiredPackets.Count; i++)
{
OutgoingPacket outgoingPacket = expiredPackets[i];
//m_log.DebugFormat("[LLUDPSERVER]: Resending packet #{0} (attempt {1}), {2}ms have passed",
// outgoingPacket.SequenceNumber, outgoingPacket.ResendCount, Environment.TickCount - outgoingPacket.TickCount);
// Set the resent flag
outgoingPacket.Buffer.Data[0] = (byte)(outgoingPacket.Buffer.Data[0] | Helpers.MSG_RESENT);
outgoingPacket.Category = ThrottleOutPacketType.Resend;
// Bump up the resend count on this packet
Interlocked.Increment(ref outgoingPacket.ResendCount);
//Interlocked.Increment(ref Stats.ResentPackets);
// Requeue or resend the packet
if (!outgoingPacket.Client.EnqueueOutgoing(outgoingPacket))
{
SendPacketFinal(outgoingPacket);
}
}
}
}
public bool EnqueueOutgoing(OutgoingPacket packet)
{
int category = (int)packet.Category;
if (category >= 0 && category < (int)ThrottleOutPacketType.Count)
{
//All packets are enqueued, except those that don't have a queue
int prio = MapCatsToPriority[category];
m_outbox.Enqueue(prio, packet);
return(true);
}
// We don't have a token bucket for this category,
// so it will not be queued and sent immediately
return(false);
}
void AsyncEndSend(IAsyncResult result)
{
OutgoingPacket packet = (OutgoingPacket)result.AsyncState;
try
{
m_udpSocket.EndSendTo(result);
}
catch (SocketException) { }
catch (ObjectDisposedException) { }
finally
{
this.SendCompleted(packet);
}
}
/// <summary>
/// Queue an outgoing packet if appropriate.
/// </summary>
/// <param name="packet"></param>
/// <param name="forceQueue">Always queue the packet if at all possible.</param>
/// <returns>
/// true if the packet has been queued,
/// false if the packet has not been queued and should be sent immediately.
/// </returns>
public bool EnqueueOutgoing(OutgoingPacket packet)
{
int category = (int)packet.Category;
if (category >= 0 && category < m_packetOutboxes.Length)
{
DoubleLocklessQueue <OutgoingPacket> queue = m_packetOutboxes[category];
queue.Enqueue(packet, false);
return(true);
}
else
{
// We don't have a token bucket for this category, so it will not be queued
return(false);
}
}
public bool EnqueueOutgoing(OutgoingPacket packet)
{
int category = (int)packet.Category;
int prio;
if (category >= 0 && category < m_packetOutboxes.Length )
{
//All packets are enqueued, except those that don't have a queue
prio = MapCatsToPriority[category];
m_outbox.Enqueue(prio, (object)packet);
return true;
}
else
{
// all known packs should have a known
// We don't have a token bucket for this category, so it will not be queued
return false;
}
}
public bool EnqueueOutgoing(OutgoingPacket packet, bool forceQueue, bool highPriority)
{
int category = (int)packet.Category;
if (category >= 0 && category < m_packetOutboxes.Length)
{
DoubleLocklessQueue <OutgoingPacket> queue = m_packetOutboxes[category];
if (m_deliverPackets == false)
{
queue.Enqueue(packet, highPriority);
return(true);
}
TokenBucket bucket = m_throttleCategories[category];
// Don't send this packet if queue is not empty
if (queue.Count > 0 || m_nextPackets[category] != null)
{
queue.Enqueue(packet, highPriority);
return(true);
}
if (!forceQueue && bucket.CheckTokens(packet.Buffer.DataLength))
{
// enough tokens so it can be sent imediatly by caller
bucket.RemoveTokens(packet.Buffer.DataLength);
return(false);
}
else
{
// Force queue specified or not enough tokens in the bucket, queue this packet
queue.Enqueue(packet, highPriority);
return(true);
}
}
else
{
// We don't have a token bucket for this category, so it will not be queued
return(false);
}
}
public void AsyncBeginSend(OutgoingPacket packet)
{
if (!m_shutdownFlag)
{
try
{
packet.AddRef();
m_udpSocket.BeginSendTo(
packet.Buffer,
0,
packet.DataSize,
SocketFlags.None,
packet.Destination,
AsyncEndSend,
packet);
}
catch (SocketException) { }
catch (ObjectDisposedException) { }
}
}
public bool Dequeue(out OutgoingPacket pack)
{
object o;
int i = nlevels;
while (--i >= 0) // go down levels looking for data
{
if (queues[i].Dequeue(out o))
{
if (o is OutgoingPacket)
{
pack = (OutgoingPacket)o;
Interlocked.Decrement(ref count);
return true;
}
// else do call to a funtion that will return the packet or whatever
}
}
pack=null;
return false;
}
/// <summary>
/// Queue an outgoing packet if appropriate.
/// </summary>
/// <param name="packet"></param>
/// <param name="forceQueue">Always queue the packet if at all possible.</param>
/// <returns>
/// true if the packet has been queued,
/// false if the packet has not been queued and should be sent immediately.
/// </returns>
public bool EnqueueOutgoing(OutgoingPacket packet, bool forceQueue)
{
int category = (int)packet.Category;
if (category >= 0 && category < m_packetOutboxes.Length)
{
ThreadedClasses.NonblockingQueue <OutgoingPacket> queue = m_packetOutboxes[category];
TokenBucket bucket = m_throttleCategories[category];
// Don't send this packet if there is already a packet waiting in the queue
// even if we have the tokens to send it, tokens should go to the already
// queued packets
if (queue.Count > 0)
{
queue.Enqueue(packet);
return(true);
}
if (!forceQueue && bucket.RemoveTokens(packet.Buffer.DataLength))
{
// Enough tokens were removed from the bucket, the packet will not be queued
return(false);
}
else
{
// Force queue specified or not enough tokens in the bucket, queue this packet
queue.Enqueue(packet);
return(true);
}
}
else
{
// We don't have a token bucket for this category, so it will not be queued
return(false);
}
}
public bool Dequeue(out OutgoingPacket pack)
{
int i = nlevels;
while (--i >= 0) // go down levels looking for data
{
object o;
if (!queues[i].Dequeue(out o))
{
continue;
}
if (!(o is OutgoingPacket))
{
continue;
}
pack = (OutgoingPacket)o;
Interlocked.Decrement(ref Count);
return(true);
// else do call to a funtion that will return the packet or whatever
}
pack = null;
return(false);
}
/// <summary>
/// Queue an outgoing packet if appropriate.
/// </summary>
/// <param name="packet"></param>
/// <param name="forceQueue">Always queue the packet if at all possible.</param>
/// <returns>
/// true if the packet has been queued,
/// false if the packet has not been queued and should be sent immediately.
/// </returns>
public bool EnqueueOutgoing(OutgoingPacket packet, bool forceQueue)
{
return(EnqueueOutgoing(packet, forceQueue, false));
}
private void SendAckImmediate(IPEndPoint remoteEndpoint, uint sequenceNumber)
{
PacketAckPacket ack = new PacketAckPacket();
ack.Header.Reliable = false;
ack.Packets = new PacketAckPacket.PacketsBlock[1];
ack.Packets[0] = new PacketAckPacket.PacketsBlock();
ack.Packets[0].ID = sequenceNumber;
byte[] packetData = ack.ToBytes();
int length = packetData.Length;
OutgoingPacket outgoingPacket = new OutgoingPacket(null, packetData, 0, length,
remoteEndpoint, false, PacketType.PacketAck);
AsyncBeginSend(outgoingPacket);
}
/// <summary>
/// Loops through all of the packet queues for this client and tries to send
/// an outgoing packet from each, obeying the throttling bucket limits
/// </summary>
///
/// Packet queues are inspected in ascending numerical order starting from 0. Therefore, queues with a lower
/// ThrottleOutPacketType number will see their packet get sent first (e.g. if both Land and Wind queues have
/// packets, then the packet at the front of the Land queue will be sent before the packet at the front of the
/// wind queue).
///
/// <remarks>This function is only called from a synchronous loop in the
/// UDPServer so we don't need to bother making this thread safe</remarks>
/// <returns>True if any packets were sent, otherwise false</returns>
public bool DequeueOutgoing(int MaxNPacks)
{
OutgoingPacket packet;
bool packetSent = false;
for (int i = 0; i < MaxNPacks; i++)
{
if (m_nextOutPackets != null)
{
OutgoingPacket nextPacket = m_nextOutPackets;
if (m_throttle.RemoveTokens(nextPacket.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(nextPacket);
m_nextOutPackets = null;
packetSent = true;
this.PacketsSent++;
}
else
{
int prio = MapCatsToPriority[(int)nextPacket.Category];
prio++;
if (prio > MapCatsToPriority.Length)
{
prio--;
}
m_outbox.Enqueue(prio, nextPacket);
m_nextOutPackets = null;
}
}
else
{
// No dequeued packet waiting to be sent, try to pull one off
// this queue
if (m_outbox.Dequeue(out packet))
{
// A packet was pulled off the queue. See if we have
// enough tokens in the bucket to send it out
if (packet.Category == ThrottleOutPacketType.OutBand || m_throttle.RemoveTokens(packet.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(packet);
packetSent = true;
if (m_throttle.MaxBurst < TotalRateRequested)
{
float tmp = (float)m_throttle.MaxBurst * 1.005f;
m_throttle.DripRate = (int)tmp;
m_throttle.MaxBurst = (int)tmp;
}
this.PacketsSent++;
}
else
{
m_nextOutPackets = packet;
}
}
else
{
break;
}
}
}
if (m_outbox.count < 100 || m_lastEmptyUpdates > 10)
{
//if(m_outbox.count > 100)
// MainConsole.Instance.Output(m_outbox.count.ToString(), log4net.Core.Level.Alert);
m_lastEmptyUpdates = 0;
BeginFireQueueEmpty();
}
else
{
m_lastEmptyUpdates++;
}
//m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business
return(packetSent);
/*
*
* OutgoingPacket packet;
* OpenSim.Framework.LocklessQueue<OutgoingPacket> queue;
* TokenBucket bucket;
* bool packetSent = false;
* ThrottleOutPacketTypeFlags emptyCategories = 0;
*
* //string queueDebugOutput = String.Empty; // Serious debug business
* int npacksTosent = MaxNPacks;
*
* int i = m_lastthrottleCategoryChecked;
* for (int j = 0; j < (int)ThrottleOutPacketType.OutBand; j++) // don't check OutBand
* {
*
* bucket = m_throttleCategories[i];
* //queueDebugOutput += m_packetOutboxes[i].Count + " "; // Serious debug business
*
* if (m_nextPackets[i] != null)
* {
* // This bucket was empty the last time we tried to send a packet,
* // leaving a dequeued packet still waiting to be sent out. Try to
* // send it again
* OutgoingPacket nextPacket = m_nextPackets[i];
* if (bucket.RemoveTokens(nextPacket.Buffer.DataLength))
* {
* // Send the packet
* m_udpServer.SendPacketFinal(nextPacket);
* m_nextPackets[i] = null;
* packetSent = true;
* this.PacketsSent++;
* }
* }
* else
* {
* // No dequeued packet waiting to be sent, try to pull one off
* // this queue
* queue = m_packetOutboxes[i];
* if (queue.Dequeue(out packet))
* {
* // A packet was pulled off the queue. See if we have
* // enough tokens in the bucket to send it out
* if (bucket.RemoveTokens(packet.Buffer.DataLength))
* {
* // Send the packet
* m_udpServer.SendPacketFinal(packet);
* packetSent = true;
* this.PacketsSent++;
* }
* else
* {
* // Save the dequeued packet for the next iteration
* m_nextPackets[i] = packet;
* }
*
* // If the queue is empty after this dequeue, fire the queue
* // empty callback now so it has a chance to fill before we
* // get back here
* if (queue.Count == 0)
* emptyCategories |= CategoryToFlag(i);
* }
* else
* {
* // No packets in this queue. Fire the queue empty callback
* // if it has not been called recently
* emptyCategories |= CategoryToFlag(i);
* }
* }
*
* if (++i >= (int)ThrottleOutPacketType.OutBand)
* i = 0;
*
* if (--npacksTosent <= 0)
* break;
* }
*
* m_lastthrottleCategoryChecked = i;
*
* // send at least one outband packet
*
* if (npacksTosent <= 0)
* npacksTosent = 1;
*
* i = (int)ThrottleOutPacketType.OutBand;
*
* while (npacksTosent > 0)
* {
* //outband has no tokens checking and no throttle
* // No dequeued packet waiting to be sent, try to pull one off
* // this queue
* queue = m_packetOutboxes[i];
* if (queue.Dequeue(out packet))
* {
* // A packet was pulled off the queue. See if we have
* // enough tokens in the bucket to send it out
*
* // if (bucket.RemoveTokens(packet.Buffer.DataLength))
* // {
* // Send the packet
* m_udpServer.SendPacketFinal(packet);
* packetSent = true;
* this.PacketsSent++;
* // }
* // else
* // {
* // Save the dequeued packet for the next iteration
* // m_nextPackets[i] = packet;
* // }
*
* // If the queue is empty after this dequeue, fire the queue
* // empty callback now so it has a chance to fill before we
* // get back here
* if (queue.Count == 0)
* emptyCategories |= CategoryToFlag(i);
* }
* else
* {
* // No packets in this queue. Fire the queue empty callback
* // if it has not been called recently
* emptyCategories |= CategoryToFlag(i);
* break;
* }
* npacksTosent--;
* }
*
* if (emptyCategories != 0)
* BeginFireQueueEmpty(emptyCategories);
*
* //m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business
* return packetSent;
*/
}
public bool EnqueueOutgoing(OutgoingPacket packet)
{
int category = (int)packet.Category;
if (category >= 0 && category < m_packetOutboxes.Length)
{
OpenSim.Framework.LocklessQueue<OutgoingPacket> queue = m_packetOutboxes[category];
TokenBucket bucket = m_throttleCategories[category];
if (bucket.RemoveTokens(packet.Buffer.DataLength))
{
// Enough tokens were removed from the bucket, the packet will not be queued
return false;
}
else
{
// Not enough tokens in the bucket, queue this packet
queue.Enqueue(packet);
return true;
}
}
else
{
// We don't have a token bucket for this category, so it will not be queued
return false;
}
}
/// <summary>
/// tries to send queued packets
/// </summary>
/// <remarks>
/// This function is only called from a synchronous loop in the
/// UDPServer so we don't need to bother making this thread safe
/// </remarks>
/// <returns>True if any packets were sent, otherwise false</returns>
public bool DequeueOutgoing(int MaxNPacks)
{
bool packetSent = false;
for (int i = 0; i < MaxNPacks; i++)
{
OutgoingPacket packet;
if (m_nextOutPacket != null)
{
packet = m_nextOutPacket;
if (m_throttle.RemoveTokens(packet.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(packet);
m_nextOutPacket = null;
packetSent = true;
}
}
// No dequeued packet waiting to be sent, try to pull one off
// this queue
else if (m_outbox.Dequeue(out packet))
{
MainConsole.Instance.Output(AgentID + " - " + packet.Packet.Type, "Verbose");
// A packet was pulled off the queue. See if we have
// enough tokens in the bucket to send it out
if (packet.Category == ThrottleOutPacketType.OutBand ||
m_throttle.RemoveTokens(packet.Buffer.DataLength))
{
packetSent = true;
//Send the packet
PacketsCounts[(int)packet.Category] += packet.Packet.Length;
m_udpServer.SendPacketFinal(packet);
PacketsSent++;
}
else
{
m_nextOutPacket = packet;
break;
}
}
else
{
break;
}
}
if (packetSent)
{
if (m_throttle.MaxBurst < TotalRateRequested)
{
float tmp = m_throttle.MaxBurst * 1.005f;
m_throttle.DripRate = (int)tmp;
m_throttle.MaxBurst = (int)tmp;
}
}
if (m_nextOnQueueEmpty != 0 && Util.EnvironmentTickCountSubtract(m_nextOnQueueEmpty) >= 0)
{
// Use a value of 0 to signal that FireQueueEmpty is running
m_nextOnQueueEmpty = 0;
// Asynchronously run the callback
int ptmp = m_outbox.queues[MapCatsToPriority[(int)ThrottleOutPacketType.Task]].Count;
int atmp = m_outbox.queues[MapCatsToPriority[(int)ThrottleOutPacketType.AvatarInfo]].Count;
int ttmp = m_outbox.queues[MapCatsToPriority[(int)ThrottleOutPacketType.Texture]].Count;
int[] arg = { ptmp, atmp, ttmp };
Util.FireAndForget(FireQueueEmpty, arg);
}
return(packetSent);
}
/// <summary>
/// Add an unacked packet to the collection
/// </summary>
/// <param name="packet">Packet that is awaiting acknowledgement</param>
/// <returns>True if the packet was successfully added, false if the
/// packet already existed in the collection</returns>
/// <remarks>This does not immediately add the ACK to the collection,
/// it only queues it so it can be added in a thread-safe way later</remarks>
public void Add(OutgoingPacket packet)
{
m_pendingAdds.Enqueue(packet);
Interlocked.Add(ref packet.Client.UnackedBytes, packet.Buffer.DataLength);
}
public bool EnqueueOutgoing(OutgoingPacket packet)
{
int category = (int) packet.Category;
if (category >= 0 && category < (int) ThrottleOutPacketType.Count)
{
//All packets are enqueued, except those that don't have a queue
int prio = MapCatsToPriority[category];
m_outbox.Enqueue(prio, packet);
return true;
}
// We don't have a token bucket for this category,
// so it will not be queued and sent immediately
return false;
}
protected override void SendCompleted(OutgoingPacket packet)
{
packet.DecRef(_bufferPool);
}
/// <summary> /// Called when the base is through with the given packet /// </summary> /// <param name="packet"></param> protected abstract void SendCompleted(OutgoingPacket packet);
/// <summary>
/// Loops through all of the packet queues for this client and tries to send
/// an outgoing packet from each, obeying the throttling bucket limits
/// </summary>
///
/// <remarks>
/// Packet queues are inspected in ascending numerical order starting from 0. Therefore, queues with a lower
/// ThrottleOutPacketType number will see their packet get sent first (e.g. if both Land and Wind queues have
/// packets, then the packet at the front of the Land queue will be sent before the packet at the front of the
/// wind queue).
///
/// This function is only called from a synchronous loop in the
/// UDPServer so we don't need to bother making this thread safe
/// </remarks>
///
/// <returns>True if any packets were sent, otherwise false</returns>
public bool DequeueOutgoing()
{
OutgoingPacket packet;
OpenSim.Framework.LocklessQueue <OutgoingPacket> queue;
TokenBucket bucket;
bool packetSent = false;
ThrottleOutPacketTypeFlags emptyCategories = 0;
//string queueDebugOutput = String.Empty; // Serious debug business
for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++)
{
bucket = m_throttleCategories[i];
//queueDebugOutput += m_packetOutboxes[i].Count + " "; // Serious debug business
if (m_nextPackets[i] != null)
{
// This bucket was empty the last time we tried to send a packet,
// leaving a dequeued packet still waiting to be sent out. Try to
// send it again
OutgoingPacket nextPacket = m_nextPackets[i];
if (bucket.RemoveTokens(nextPacket.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(nextPacket);
m_nextPackets[i] = null;
packetSent = true;
}
}
else
{
// No dequeued packet waiting to be sent, try to pull one off
// this queue
queue = m_packetOutboxes[i];
if (queue.Dequeue(out packet))
{
// A packet was pulled off the queue. See if we have
// enough tokens in the bucket to send it out
if (bucket.RemoveTokens(packet.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(packet);
packetSent = true;
}
else
{
// Save the dequeued packet for the next iteration
m_nextPackets[i] = packet;
}
// If the queue is empty after this dequeue, fire the queue
// empty callback now so it has a chance to fill before we
// get back here
if (queue.Count == 0)
{
emptyCategories |= CategoryToFlag(i);
}
}
else
{
// No packets in this queue. Fire the queue empty callback
// if it has not been called recently
emptyCategories |= CategoryToFlag(i);
}
}
}
if (emptyCategories != 0)
{
BeginFireQueueEmpty(emptyCategories);
}
//m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business
return(packetSent);
}
/// <summary>
/// Actually send a packet to a client.
/// </summary>
/// <param name="outgoingPacket"></param>
internal void SendPacketFinal(OutgoingPacket outgoingPacket)
{
UDPPacketBuffer buffer = outgoingPacket.Buffer;
byte flags = buffer.Data[0];
bool isResend = (flags & Helpers.MSG_RESENT) != 0;
bool isReliable = (flags & Helpers.MSG_RELIABLE) != 0;
bool isZerocoded = (flags & Helpers.MSG_ZEROCODED) != 0;
LLUDPClient udpClient = outgoingPacket.Client;
if (!udpClient.IsConnected)
{
return;
}
#region ACK Appending
int dataLength = buffer.DataLength;
// NOTE: I'm seeing problems with some viewers when ACKs are appended to zerocoded packets so I've disabled that here
if (!isZerocoded)
{
// Keep appending ACKs until there is no room left in the buffer or there are
// no more ACKs to append
uint ackCount = 0;
uint ack;
while (dataLength + 5 < buffer.Data.Length && udpClient.PendingAcks.Dequeue(out ack))
{
Utils.UIntToBytesBig(ack, buffer.Data, dataLength);
dataLength += 4;
++ackCount;
}
if (ackCount > 0)
{
// Set the last byte of the packet equal to the number of appended ACKs
buffer.Data[dataLength++] = (byte)ackCount;
// Set the appended ACKs flag on this packet
buffer.Data[0] = (byte)(buffer.Data[0] | Helpers.MSG_APPENDED_ACKS);
}
}
buffer.DataLength = dataLength;
#endregion ACK Appending
#region Sequence Number Assignment
if (!isResend)
{
// Not a resend, assign a new sequence number
uint sequenceNumber = (uint)Interlocked.Increment(ref udpClient.CurrentSequence);
Utils.UIntToBytesBig(sequenceNumber, buffer.Data, 1);
outgoingPacket.SequenceNumber = sequenceNumber;
if (isReliable)
{
// Add this packet to the list of ACK responses we are waiting on from the server
udpClient.NeedAcks.Add(outgoingPacket);
}
}
#endregion Sequence Number Assignment
// Stats tracking
Interlocked.Increment(ref udpClient.PacketsSent);
if (isReliable)
{
Interlocked.Add(ref udpClient.UnackedBytes, outgoingPacket.Buffer.DataLength);
}
// Put the UDP payload on the wire
AsyncBeginSend(buffer);
// Keep track of when this packet was sent out (right now)
outgoingPacket.TickCount = Environment.TickCount & Int32.MaxValue;
}
/// <summary>
/// Loops through all of the packet queues for this client and tries to send
/// an outgoing packet from each, obeying the throttling bucket limits
/// </summary>
///
/// <remarks>
/// Packet queues are inspected in ascending numerical order starting from 0. Therefore, queues with a lower
/// ThrottleOutPacketType number will see their packet get sent first (e.g. if both Land and Wind queues have
/// packets, then the packet at the front of the Land queue will be sent before the packet at the front of the
/// wind queue).
///
/// This function is only called from a synchronous loop in the
/// UDPServer so we don't need to bother making this thread safe
/// </remarks>
///
/// <returns>True if any packets were sent, otherwise false</returns>
public bool DequeueOutgoing()
{
// if (m_deliverPackets == false) return false;
OutgoingPacket packet = null;
DoubleLocklessQueue <OutgoingPacket> queue;
TokenBucket bucket;
bool packetSent = false;
ThrottleOutPacketTypeFlags emptyCategories = 0;
//string queueDebugOutput = String.Empty; // Serious debug business
for (int i = 0; i < THROTTLE_CATEGORY_COUNT; i++)
{
bucket = m_throttleCategories[i];
//queueDebugOutput += m_packetOutboxes[i].Count + " "; // Serious debug business
if (m_nextPackets[i] != null)
{
// This bucket was empty the last time we tried to send a packet,
// leaving a dequeued packet still waiting to be sent out. Try to
// send it again
OutgoingPacket nextPacket = m_nextPackets[i];
if (nextPacket.Buffer == null)
{
if (m_packetOutboxes[i].Count < 5)
{
emptyCategories |= CategoryToFlag(i);
}
continue;
}
if (bucket.RemoveTokens(nextPacket.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(nextPacket);
m_nextPackets[i] = null;
packetSent = true;
if (m_packetOutboxes[i].Count < 5)
{
emptyCategories |= CategoryToFlag(i);
}
}
}
else
{
// No dequeued packet waiting to be sent, try to pull one off
// this queue
queue = m_packetOutboxes[i];
if (queue != null)
{
bool success = false;
try
{
success = queue.Dequeue(out packet);
}
catch
{
m_packetOutboxes[i] = new DoubleLocklessQueue <OutgoingPacket>();
}
if (success)
{
// A packet was pulled off the queue. See if we have
// enough tokens in the bucket to send it out
if (packet.Buffer == null)
{
// packet canceled elsewhere (by a ack for example)
if (queue.Count < 5)
{
emptyCategories |= CategoryToFlag(i);
}
}
else
{
if (bucket.RemoveTokens(packet.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(packet);
packetSent = true;
if (queue.Count < 5)
{
emptyCategories |= CategoryToFlag(i);
}
}
else
{
// Save the dequeued packet for the next iteration
m_nextPackets[i] = packet;
}
}
}
else
{
// No packets in this queue. Fire the queue empty callback
// if it has not been called recently
emptyCategories |= CategoryToFlag(i);
}
}
else
{
m_packetOutboxes[i] = new DoubleLocklessQueue <OutgoingPacket>();
emptyCategories |= CategoryToFlag(i);
}
}
}
if (emptyCategories != 0)
{
BeginFireQueueEmpty(emptyCategories);
}
//m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business
return(packetSent);
}
public void SendPacketData(LLUDPClient udpClient, byte[] data, PacketType type, ThrottleOutPacketType category)
{
int dataLength = data.Length;
bool doZerocode = (data[0] & Helpers.MSG_ZEROCODED) != 0;
bool doCopy = true;
// Frequency analysis of outgoing packet sizes shows a large clump of packets at each end of the spectrum.
// The vast majority of packets are less than 200 bytes, although due to asset transfers and packet splitting
// there are a decent number of packets in the 1000-1140 byte range. We allocate one of two sizes of data here
// to accomodate for both common scenarios and provide ample room for ACK appending in both
int bufferSize = (dataLength > 180) ? LLUDPServer.MTU : 200;
UDPPacketBuffer buffer = new UDPPacketBuffer(udpClient.RemoteEndPoint, bufferSize);
// Zerocode if needed
if (doZerocode)
{
try
{
dataLength = Helpers.ZeroEncode(data, dataLength, buffer.Data);
doCopy = false;
}
catch (IndexOutOfRangeException)
{
// The packet grew larger than the bufferSize while zerocoding.
// Remove the MSG_ZEROCODED flag and send the unencoded data
// instead
m_log.Debug("[LLUDPSERVER]: Packet exceeded buffer size during zerocoding for " + type + ". DataLength=" + dataLength +
" and BufferLength=" + buffer.Data.Length + ". Removing MSG_ZEROCODED flag");
data[0] = (byte)(data[0] & ~Helpers.MSG_ZEROCODED);
}
}
// If the packet data wasn't already copied during zerocoding, copy it now
if (doCopy)
{
if (dataLength <= buffer.Data.Length)
{
Buffer.BlockCopy(data, 0, buffer.Data, 0, dataLength);
}
else
{
bufferSize = dataLength;
buffer = new UDPPacketBuffer(udpClient.RemoteEndPoint, bufferSize);
// m_log.Error("[LLUDPSERVER]: Packet exceeded buffer size! This could be an indication of packet assembly not obeying the MTU. Type=" +
// type + ", DataLength=" + dataLength + ", BufferLength=" + buffer.Data.Length + ". Dropping packet");
Buffer.BlockCopy(data, 0, buffer.Data, 0, dataLength);
}
}
buffer.DataLength = dataLength;
#region Queue or Send
OutgoingPacket outgoingPacket = new OutgoingPacket(udpClient, buffer, category);
if (!outgoingPacket.Client.EnqueueOutgoing(outgoingPacket))
{
SendPacketFinal(outgoingPacket);
}
#endregion Queue or Send
}
public void SendPacketData(LLUDPClient udpClient, byte[] data, int dataLength, PacketType type,
ThrottleOutPacketType category, bool bufferAcquiredFromPool)
{
bool doZerocode = (data[0] & Helpers.MSG_ZEROCODED) != 0;
bool zeroCoded = false;
byte[] outBuffer = null;
// Zerocode if needed
if (doZerocode)
{
// Frequency analysis of outgoing packet sizes shows a large clump of packets at each end of the spectrum.
// The vast majority of packets are less than 200 bytes, although due to asset transfers and packet splitting
// there are a decent number of packets in the 1000-1140 byte range. We allocate one of two sizes of data here
// to accomodate for both common scenarios and provide ample room for ACK appending in both
int bufferSize = (dataLength > 180) ? LLUDPServer.MTU : 200;
try
{
//zerocode and return the current buffer to the pool if necessary
outBuffer = _bufferPool.LeaseBytes(bufferSize);
dataLength = Helpers.ZeroEncode(data, dataLength, outBuffer);
zeroCoded = true;
if (bufferAcquiredFromPool)
{
_bufferPool.ReturnBytes(data);
}
//now the buffer is from a pool most definitely
bufferAcquiredFromPool = true;
}
catch (IndexOutOfRangeException)
{
//TODO: Throwing an exception here needs to be revisted. I've seen an issue with
//70+ avatars where a very common high freq packet hits this code everytime
//that packet either needs to be split, or this needs to be revised to not throw
//and instead check the buffer size and return an error condition
// The packet grew larger than the bufferSize while zerocoding.
// Remove the MSG_ZEROCODED flag and send the unencoded data
// instead
m_log.Debug("[LLUDPSERVER]: Packet exceeded buffer size during zerocoding for " + type + ". DataLength=" + dataLength +
" and BufferLength=" + outBuffer.Length + ". Removing MSG_ZEROCODED flag");
data[0] = (byte)(data[0] & ~Helpers.MSG_ZEROCODED);
_bufferPool.ReturnBytes(outBuffer);
}
}
if (! zeroCoded)
{
outBuffer = data;
}
#region Queue or Send
OutgoingPacket outgoingPacket = new OutgoingPacket(udpClient, outBuffer, (int)category, dataLength,
udpClient.RemoteEndPoint, bufferAcquiredFromPool, type);
if (!udpClient.EnqueueOutgoing(outgoingPacket))
SendPacketFinal(outgoingPacket);
#endregion Queue or Send
}
public void SendPacketData(LLUDPClient udpClient, byte[] data, Packet packet, ThrottleOutPacketType category, UnackedPacketMethod resendMethod, UnackedPacketMethod finishedMethod)
{
int dataLength = data.Length;
bool doZerocode = (data[0] & Helpers.MSG_ZEROCODED) != 0;
bool doCopy = true;
// Frequency analysis of outgoing packet sizes shows a large clump of packets at each end of the spectrum.
// The vast majority of packets are less than 200 bytes, although due to asset transfers and packet splitting
// there are a decent number of packets in the 1000-1140 byte range. We allocate one of two sizes of data here
// to accomodate for both common scenarios and provide ample room for ACK appending in both
int bufferSize = dataLength * 2;
UDPPacketBuffer buffer = new UDPPacketBuffer(udpClient.RemoteEndPoint, bufferSize);
// Zerocode if needed
if (doZerocode)
{
try
{
dataLength = Helpers.ZeroEncode(data, dataLength, buffer.Data);
doCopy = false;
}
catch (IndexOutOfRangeException)
{
// The packet grew larger than the bufferSize while zerocoding.
// Remove the MSG_ZEROCODED flag and send the unencoded data
// instead
m_log.Info("[LLUDPSERVER]: Packet exceeded buffer size during zerocoding for " + packet.Type + ". DataLength=" + dataLength +
" and BufferLength=" + buffer.Data.Length + ". Removing MSG_ZEROCODED flag");
data[0] = (byte)(data[0] & ~Helpers.MSG_ZEROCODED);
}
}
// If the packet data wasn't already copied during zerocoding, copy it now
if (doCopy)
{
if (dataLength <= buffer.Data.Length)
{
Buffer.BlockCopy(data, 0, buffer.Data, 0, dataLength);
}
else
{
bufferSize = dataLength;
buffer = new UDPPacketBuffer(udpClient.RemoteEndPoint, bufferSize);
// m_log.Error("[LLUDPSERVER]: Packet exceeded buffer size! This could be an indication of packet assembly not obeying the MTU. Type=" +
// type + ", DataLength=" + dataLength + ", BufferLength=" + buffer.Data.Length + ". Dropping packet");
Buffer.BlockCopy(data, 0, buffer.Data, 0, dataLength);
}
}
buffer.DataLength = dataLength;
#region Queue or Send
OutgoingPacket outgoingPacket = new OutgoingPacket(udpClient, buffer, category, resendMethod, finishedMethod, packet);
if (!outgoingPacket.Client.EnqueueOutgoing(outgoingPacket))
SendPacketFinal(outgoingPacket);
#endregion Queue or Send
}
/// <summary>
/// Loops through all of the packet queues for this client and tries to send
/// an outgoing packet from each, obeying the throttling bucket limits
/// </summary>
///
/// <remarks>
/// Packet queues are inspected in ascending numerical order starting from 0. Therefore, queues with a lower
/// ThrottleOutPacketType number will see their packet get sent first (e.g. if both Land and Wind queues have
/// packets, then the packet at the front of the Land queue will be sent before the packet at the front of the
/// wind queue).
///
/// This function is only called from a synchronous loop in the
/// UDPServer so we don't need to bother making this thread safe
/// </remarks>
///
/// <returns>True if any packets were sent, otherwise false</returns>
public bool DequeueOutgoing()
{
// if (m_deliverPackets == false) return false;
OutgoingPacket packet = null;
DoubleLocklessQueue <OutgoingPacket> queue;
bool packetSent = false;
ThrottleOutPacketTypeFlags emptyCategories = 0;
//string queueDebugOutput = String.Empty; // Serious debug business
// do resends
packet = m_nextPackets[0];
if (packet != null)
{
if (packet.Buffer != null)
{
if (m_throttleCategories[0].RemoveTokens(packet.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(packet);
packetSent = true;
m_nextPackets[0] = null;
}
}
else
{
m_nextPackets[0] = null;
}
}
else
{
queue = m_packetOutboxes[0];
if (queue != null)
{
if (queue.Dequeue(out packet))
{
// A packet was pulled off the queue. See if we have
// enough tokens in the bucket to send it out
if (packet.Buffer != null)
{
if (m_throttleCategories[0].RemoveTokens(packet.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(packet);
packetSent = true;
}
else
{
// Save the dequeued packet for the next iteration
m_nextPackets[0] = packet;
}
}
}
}
else
{
m_packetOutboxes[0] = new DoubleLocklessQueue <OutgoingPacket>();
}
}
if (NeedAcks.Count() > 50)
{
Interlocked.Increment(ref AckStalls);
return(true);
}
for (int i = 1; i < THROTTLE_CATEGORY_COUNT; i++)
{
//queueDebugOutput += m_packetOutboxes[i].Count + " "; // Serious debug business
packet = m_nextPackets[i];
if (packet != null)
{
if (packet.Buffer == null)
{
if (m_packetOutboxes[i].Count < 5)
{
emptyCategories |= CategoryToFlag(i);
}
m_nextPackets[i] = null;
continue;
}
if (m_throttleCategories[i].RemoveTokens(packet.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(packet);
m_nextPackets[i] = null;
packetSent = true;
if (m_packetOutboxes[i].Count < 5)
{
emptyCategories |= CategoryToFlag(i);
}
}
}
else
{
// No dequeued packet waiting to be sent, try to pull one off
// this queue
queue = m_packetOutboxes[i];
if (queue.Dequeue(out packet))
{
if (packet.Buffer == null)
{
// packet canceled elsewhere (by a ack for example)
if (queue.Count < 5)
{
emptyCategories |= CategoryToFlag(i);
}
continue;
}
if (m_throttleCategories[i].RemoveTokens(packet.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(packet);
packetSent = true;
if (queue.Count < 5)
{
emptyCategories |= CategoryToFlag(i);
}
}
else
{
// Save the dequeued packet for the next iteration
m_nextPackets[i] = packet;
}
}
else
{
// No packets in this queue. Fire the queue empty callback
// if it has not been called recently
emptyCategories |= CategoryToFlag(i);
}
}
}
if (emptyCategories != 0)
{
BeginFireQueueEmpty(emptyCategories);
}
//m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business
return(packetSent);
}
public bool EnqueueOutgoing(OutgoingPacket packet)
{
int category = (int)packet.Category;
if (category >= 0 && category < m_packetOutboxes.Length)
{
OpenSim.Framework.LocklessQueue<OutgoingPacket> queue = m_packetOutboxes[category];
// Not enough tokens in the bucket, queue this packet
//check the queue
//Dont drop resends this can mess up the buffer pool as well as make the connection situation much worse
if (_currentOutboundQueueSize > MAX_TOTAL_QUEUE_SIZE && (packet.Buffer[0] & Helpers.MSG_RESENT) == 0)
{
//queue already has too much data in it..
//can we drop this packet?
byte flags = packet.Buffer[0];
bool isReliable = (flags & Helpers.MSG_RELIABLE) != 0;
if (!isReliable
&& packet.Type != PacketType.PacketAck
&& packet.Type != PacketType.CompletePingCheck)
{
//packet is unreliable and will be dropped
this.TestReportPacketDrop(packet);
packet.DecRef(m_udpServer.ByteBufferPool);
}
else
{
if (_currentOutboundQueueSize < MAX_TOTAL_QUEUE_SIZE * 1.5)
{
this.TestReportPacketShouldDrop(packet);
Interlocked.Add(ref _currentOutboundQueueSize, packet.DataSize);
packet.AddRef();
queue.Enqueue(packet);
}
else
{
//this connection is in a pretty critical state and probably will never catch up.
//drop all packets until we start to catch up. This includes acks which will disconnect
//the client eventually anyways
this.TestReportCriticalPacketDrop(packet);
packet.DecRef(m_udpServer.ByteBufferPool);
}
}
}
else
{
Interlocked.Add(ref _currentOutboundQueueSize, packet.DataSize);
packet.AddRef();
queue.Enqueue(packet);
}
return true;
}
else
{
// We don't have a token bucket for this category, so it will not be queued
return false;
}
}
public void ResendUnacked(LLUDPClient udpClient)
{
if (!udpClient.IsConnected)
{
return;
}
// Disconnect an agent if no packets are received for some time
if ((Environment.TickCount & Int32.MaxValue) - udpClient.TickLastPacketReceived > 1000 * ClientTimeOut && !udpClient.IsPaused)
{
m_log.Warn("[LLUDPSERVER]: Ack timeout, disconnecting " + udpClient.AgentID);
ILoginMonitor monitor = (ILoginMonitor)m_scene.RequestModuleInterface <IMonitorModule>().GetMonitor("", "LoginMonitor");
if (monitor != null)
{
monitor.AddAbnormalClientThreadTermination();
}
RemoveClient(udpClient);
return;
}
// Get a list of all of the packets that have been sitting unacked longer than udpClient.RTO
List <OutgoingPacket> expiredPackets = udpClient.NeedAcks.GetExpiredPackets(udpClient.RTO);
if (expiredPackets != null)
{
//m_log.Debug("[LLUDPSERVER]: Resending " + expiredPackets.Count + " packets to " + udpClient.AgentID + ", RTO=" + udpClient.RTO);
// Exponential backoff of the retransmission timeout
udpClient.BackoffRTO();
lock (udpClient)
{
udpClient.SlowDownSend();
}
// Resend packets
for (int i = 0; i < expiredPackets.Count; i++)
{
OutgoingPacket outgoingPacket = expiredPackets[i];
//m_log.DebugFormat("[LLUDPSERVER]: Resending packet #{0} (attempt {1}), {2}ms have passed",
// outgoingPacket.SequenceNumber, outgoingPacket.ResendCount, Environment.TickCount - outgoingPacket.TickCount);
// Set the resent flag
outgoingPacket.Buffer.Data[0] = (byte)(outgoingPacket.Buffer.Data[0] | Helpers.MSG_RESENT);
// resend in its original category
// outgoingPacket.Category = ThrottleOutPacketType.Resend;
// Bump up the resend count on this packet
Interlocked.Increment(ref outgoingPacket.ResendCount);
//Interlocked.Increment(ref Stats.ResentPackets);
// Requeue or resend the packet
if (!outgoingPacket.Client.EnqueueOutgoing(outgoingPacket))
{
SendPacketFinal(outgoingPacket);
}
}
}
}
/// <summary>
/// Actually send a packet to a client.
/// </summary>
/// <param name="outgoingPacket"></param>
internal void SendPacketFinal(OutgoingPacket outgoingPacket)
{
UDPPacketBuffer buffer = outgoingPacket.Buffer;
byte flags = buffer.Data[0];
bool isResend = (flags & Helpers.MSG_RESENT) != 0;
bool isReliable = (flags & Helpers.MSG_RELIABLE) != 0;
bool isZerocoded = (flags & Helpers.MSG_ZEROCODED) != 0;
LLUDPClient udpClient = outgoingPacket.Client;
if (!udpClient.IsConnected)
return;
#region ACK Appending
int dataLength = buffer.DataLength;
// NOTE: I'm seeing problems with some viewers when ACKs are appended to zerocoded packets so I've disabled that here
if (!isZerocoded)
{
// Keep appending ACKs until there is no room left in the buffer or there are
// no more ACKs to append
uint ackCount = 0;
uint ack;
while (dataLength + 5 < buffer.Data.Length && udpClient.PendingAcks.Dequeue(out ack))
{
Utils.UIntToBytesBig(ack, buffer.Data, dataLength);
dataLength += 4;
++ackCount;
}
if (ackCount > 0)
{
// Set the last byte of the packet equal to the number of appended ACKs
buffer.Data[dataLength++] = (byte)ackCount;
// Set the appended ACKs flag on this packet
buffer.Data[0] = (byte)(buffer.Data[0] | Helpers.MSG_APPENDED_ACKS);
}
}
buffer.DataLength = dataLength;
#endregion ACK Appending
#region Sequence Number Assignment
if (!isResend)
{
// Not a resend, assign a new sequence number
uint sequenceNumber = (uint)Interlocked.Increment(ref udpClient.CurrentSequence);
Utils.UIntToBytesBig(sequenceNumber, buffer.Data, 1);
outgoingPacket.SequenceNumber = sequenceNumber;
if (isReliable)
{
// Add this packet to the list of ACK responses we are waiting on from the server
udpClient.NeedAcks.Add(outgoingPacket);
}
}
else
{
Interlocked.Increment(ref udpClient.PacketsResent);
}
#endregion Sequence Number Assignment
// Stats tracking
Interlocked.Increment(ref udpClient.PacketsSent);
// Put the UDP payload on the wire
AsyncBeginSend(buffer);
// Keep track of when this packet was sent out (right now)
outgoingPacket.TickCount = Environment.TickCount & Int32.MaxValue;
}
/// <summary>
/// Loops through all of the packet queues for this client and tries to send
/// an outgoing packet from each, obeying the throttling bucket limits
/// </summary>
///
/// Packet queues are inspected in ascending numerical order starting from 0. Therefore, queues with a lower
/// ThrottleOutPacketType number will see their packet get sent first (e.g. if both Land and Wind queues have
/// packets, then the packet at the front of the Land queue will be sent before the packet at the front of the
/// wind queue).
///
/// <remarks>This function is only called from a synchronous loop in the
/// UDPServer so we don't need to bother making this thread safe</remarks>
/// <returns>True if any packets were sent, otherwise false</returns>
public bool DequeueOutgoing (int MaxNPacks)
{
OutgoingPacket packet;
bool packetSent = false;
ThrottleOutPacketTypeFlags emptyCategories = 0;
if (m_nextOutPackets != null)
{
OutgoingPacket nextPacket = m_nextOutPackets;
if (m_throttle.RemoveTokens (nextPacket.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal (nextPacket);
m_nextOutPackets = null;
packetSent = true;
this.PacketsSent++;
}
}
else
{
// No dequeued packet waiting to be sent, try to pull one off
// this queue
if (m_outbox.Dequeue (out packet))
{
// A packet was pulled off the queue. See if we have
// enough tokens in the bucket to send it out
if (packet.Category == ThrottleOutPacketType.OutBand || m_throttle.RemoveTokens (packet.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal (packet);
packetSent = true;
if (m_throttle.MaxBurst < TotalRateRequested)
{
float tmp = (float)m_throttle.MaxBurst * 1.005f;
m_throttle.DripRate = (int)tmp;
m_throttle.MaxBurst = (int)tmp;
}
this.PacketsSent++;
}
else
{
m_nextOutPackets = packet;
}
}
else
{
emptyCategories = (ThrottleOutPacketTypeFlags)0xffff;
}
}
if (m_outbox.count < 100)
{
emptyCategories = (ThrottleOutPacketTypeFlags)0xffff;
BeginFireQueueEmpty (emptyCategories);
}
/*
if (emptyCategories != 0)
BeginFireQueueEmpty(emptyCategories);
else
{
int i = MapCatsToPriority[(int)ThrottleOutPacketType.Texture]; // hack to keep textures flowing for now
if (m_outbox.queues[i].Count < 30)
{
emptyCategories |= ThrottleOutPacketTypeFlags.Texture;
}
}
*/
//m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business
return packetSent;
/*
OutgoingPacket packet;
OpenSim.Framework.LocklessQueue<OutgoingPacket> queue;
TokenBucket bucket;
bool packetSent = false;
ThrottleOutPacketTypeFlags emptyCategories = 0;
//string queueDebugOutput = String.Empty; // Serious debug business
int npacksTosent = MaxNPacks;
int i = m_lastthrottleCategoryChecked;
for (int j = 0; j < (int)ThrottleOutPacketType.OutBand; j++) // don't check OutBand
{
bucket = m_throttleCategories[i];
//queueDebugOutput += m_packetOutboxes[i].Count + " "; // Serious debug business
if (m_nextPackets[i] != null)
{
// This bucket was empty the last time we tried to send a packet,
// leaving a dequeued packet still waiting to be sent out. Try to
// send it again
OutgoingPacket nextPacket = m_nextPackets[i];
if (bucket.RemoveTokens(nextPacket.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(nextPacket);
m_nextPackets[i] = null;
packetSent = true;
this.PacketsSent++;
}
}
else
{
// No dequeued packet waiting to be sent, try to pull one off
// this queue
queue = m_packetOutboxes[i];
if (queue.Dequeue(out packet))
{
// A packet was pulled off the queue. See if we have
// enough tokens in the bucket to send it out
if (bucket.RemoveTokens(packet.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(packet);
packetSent = true;
this.PacketsSent++;
}
else
{
// Save the dequeued packet for the next iteration
m_nextPackets[i] = packet;
}
// If the queue is empty after this dequeue, fire the queue
// empty callback now so it has a chance to fill before we
// get back here
if (queue.Count == 0)
emptyCategories |= CategoryToFlag(i);
}
else
{
// No packets in this queue. Fire the queue empty callback
// if it has not been called recently
emptyCategories |= CategoryToFlag(i);
}
}
if (++i >= (int)ThrottleOutPacketType.OutBand)
i = 0;
if (--npacksTosent <= 0)
break;
}
m_lastthrottleCategoryChecked = i;
// send at least one outband packet
if (npacksTosent <= 0)
npacksTosent = 1;
i = (int)ThrottleOutPacketType.OutBand;
while (npacksTosent > 0)
{
//outband has no tokens checking and no throttle
// No dequeued packet waiting to be sent, try to pull one off
// this queue
queue = m_packetOutboxes[i];
if (queue.Dequeue(out packet))
{
// A packet was pulled off the queue. See if we have
// enough tokens in the bucket to send it out
// if (bucket.RemoveTokens(packet.Buffer.DataLength))
// {
// Send the packet
m_udpServer.SendPacketFinal(packet);
packetSent = true;
this.PacketsSent++;
// }
// else
// {
// Save the dequeued packet for the next iteration
// m_nextPackets[i] = packet;
// }
// If the queue is empty after this dequeue, fire the queue
// empty callback now so it has a chance to fill before we
// get back here
if (queue.Count == 0)
emptyCategories |= CategoryToFlag(i);
}
else
{
// No packets in this queue. Fire the queue empty callback
// if it has not been called recently
emptyCategories |= CategoryToFlag(i);
break;
}
npacksTosent--;
}
if (emptyCategories != 0)
BeginFireQueueEmpty(emptyCategories);
//m_log.Info("[LLUDPCLIENT]: Queues: " + queueDebugOutput); // Serious debug business
return packetSent;
*/
}
/// <summary>
/// Add an unacked packet to the collection
/// </summary>
/// <param name="packet">Packet that is awaiting acknowledgement</param>
/// <returns>True if the packet was successfully added, false if the
/// packet already existed in the collection</returns>
/// <remarks>This does not immediately add the ACK to the collection,
/// it only queues it so it can be added in a thread-safe way later</remarks>
public void Add(OutgoingPacket packet)
{
packet.AddRef();
m_pendingAdds.Enqueue(packet);
}
public void AsyncBeginSend(OutgoingPacket packet)
{
if (!m_shutdownFlag)
{
try
{
packet.AddRef();
m_udpSocket.BeginSendTo(
packet.Buffer.Data,
0,
packet.DataSize,
SocketFlags.None,
packet.Destination,
AsyncEndSend,
packet);
}
catch (SocketException) { }
catch (ObjectDisposedException) { }
}
}
/// <summary> /// Called when the base is through with the given packet /// </summary> /// <param name="packet"></param> protected abstract void SendCompleted(OutgoingPacket packet);
/// <summary>
/// Queue an outgoing packet if appropriate.
/// </summary>
/// <param name="packet"></param>
/// <param name="forceQueue">Always queue the packet if at all possible.</param>
/// <returns>
/// true if the packet has been queued,
/// false if the packet has not been queued and should be sent immediately.
/// </returns>
public bool EnqueueOutgoing(OutgoingPacket packet, bool forceQueue)
{
int category = (int)packet.Category;
if (category >= 0 && category < m_packetOutboxes.Length)
{
OpenSim.Framework.LocklessQueue<OutgoingPacket> queue = m_packetOutboxes[category];
TokenBucket bucket = m_throttleCategories[category];
// Don't send this packet if there is already a packet waiting in the queue
// even if we have the tokens to send it, tokens should go to the already
// queued packets
if (queue.Count > 0)
{
queue.Enqueue(packet);
return true;
}
if (!forceQueue && bucket.RemoveTokens(packet.Buffer.DataLength))
{
// Enough tokens were removed from the bucket, the packet will not be queued
return false;
}
else
{
// Force queue specified or not enough tokens in the bucket, queue this packet
queue.Enqueue(packet);
return true;
}
}
else
{
// We don't have a token bucket for this category, so it will not be queued
return false;
}
}
public bool EnqueueOutgoing(OutgoingPacket packet)
{
int category = (int)packet.Category;
if (category >= 0 && category < m_packetOutboxes.Length)
{
OpenSim.Framework.LocklessQueue <OutgoingPacket> queue = m_packetOutboxes[category];
// Not enough tokens in the bucket, queue this packet
//check the queue
//Dont drop resends this can mess up the buffer pool as well as make the connection situation much worse
if (_currentOutboundQueueSize > MAX_TOTAL_QUEUE_SIZE && (packet.Buffer[0] & Helpers.MSG_RESENT) == 0)
{
//queue already has too much data in it..
//can we drop this packet?
byte flags = packet.Buffer[0];
bool isReliable = (flags & Helpers.MSG_RELIABLE) != 0;
if (!isReliable &&
packet.Type != PacketType.PacketAck &&
packet.Type != PacketType.CompletePingCheck)
{
//packet is unreliable and will be dropped
this.TestReportPacketDrop(packet);
packet.DecRef(m_udpServer.ByteBufferPool);
}
else
{
if (_currentOutboundQueueSize < MAX_TOTAL_QUEUE_SIZE * 1.5)
{
this.TestReportPacketShouldDrop(packet);
Interlocked.Add(ref _currentOutboundQueueSize, packet.DataSize);
packet.AddRef();
queue.Enqueue(packet);
}
else
{
//this connection is in a pretty critical state and probably will never catch up.
//drop all packets until we start to catch up. This includes acks which will disconnect
//the client eventually anyways
this.TestReportCriticalPacketDrop(packet);
packet.DecRef(m_udpServer.ByteBufferPool);
}
}
}
else
{
Interlocked.Add(ref _currentOutboundQueueSize, packet.DataSize);
packet.AddRef();
queue.Enqueue(packet);
}
return(true);
}
else
{
// We don't have a token bucket for this category, so it will not be queued
return(false);
}
}
public bool Dequeue(out OutgoingPacket pack)
{
int i = nlevels;
while (--i >= 0) // go down levels looking for data
{
object o;
if (!queues[i].Dequeue(out o)) continue;
if (!(o is OutgoingPacket)) continue;
pack = (OutgoingPacket) o;
Interlocked.Decrement(ref Count);
return true;
// else do call to a funtion that will return the packet or whatever
}
pack = null;
return false;
}
/// <summary>
/// Queue an outgoing packet if appropriate.
/// </summary>
/// <param name="packet"></param>
/// <param name="forceQueue">Always queue the packet if at all possible.</param>
/// <returns>
/// true if the packet has been queued,
/// false if the packet has not been queued and should be sent immediately.
/// </returns>
public bool EnqueueOutgoing(OutgoingPacket packet)
{
return(EnqueueOutgoing(packet, false));
}
/// <summary>
/// tries to send queued packets
/// </summary>
/// <remarks>
/// This function is only called from a synchronous loop in the
/// UDPServer so we don't need to bother making this thread safe
/// </remarks>
/// <returns>True if any packets were sent, otherwise false</returns>
public bool DequeueOutgoing(int MaxNPacks)
{
bool packetSent = false;
for (int i = 0; i < MaxNPacks; i++)
{
OutgoingPacket packet;
if (m_nextOutPacket != null)
{
packet = m_nextOutPacket;
if (m_throttle.RemoveTokens(packet.Buffer.DataLength))
{
// Send the packet
m_udpServer.SendPacketFinal(packet);
m_nextOutPacket = null;
packetSent = true;
}
}
// No dequeued packet waiting to be sent, try to pull one off
// this queue
else if (m_outbox.Dequeue(out packet))
{
MainConsole.Instance.Output(AgentID + " - " + packet.Packet.Type, "Verbose");
// A packet was pulled off the queue. See if we have
// enough tokens in the bucket to send it out
if (packet.Category == ThrottleOutPacketType.OutBand ||
m_throttle.RemoveTokens(packet.Buffer.DataLength))
{
packetSent = true;
//Send the packet
PacketsCounts[(int) packet.Category] += packet.Packet.Length;
m_udpServer.SendPacketFinal(packet);
PacketsSent++;
}
else
{
m_nextOutPacket = packet;
break;
}
}
else
break;
}
if (packetSent)
{
if (m_throttle.MaxBurst < TotalRateRequested)
{
float tmp = m_throttle.MaxBurst*1.005f;
m_throttle.DripRate = (int) tmp;
m_throttle.MaxBurst = (int) tmp;
}
}
if (m_nextOnQueueEmpty != 0 && Util.EnvironmentTickCountSubtract(m_nextOnQueueEmpty) >= 0)
{
// Use a value of 0 to signal that FireQueueEmpty is running
m_nextOnQueueEmpty = 0;
// Asynchronously run the callback
int ptmp = m_outbox.queues[MapCatsToPriority[(int) ThrottleOutPacketType.Task]].Count;
int atmp = m_outbox.queues[MapCatsToPriority[(int) ThrottleOutPacketType.AvatarInfo]].Count;
int ttmp = m_outbox.queues[MapCatsToPriority[(int) ThrottleOutPacketType.Texture]].Count;
int[] arg = {ptmp, atmp, ttmp};
Util.FireAndForget(FireQueueEmpty, arg);
}
return packetSent;
}
/// <summary>
/// Add an unacked packet to the collection
/// </summary>
/// <param name="packet">Packet that is awaiting acknowledgement</param>
/// <returns>True if the packet was successfully added, false if the
/// packet already existed in the collection</returns>
/// <remarks>This does not immediately add the ACK to the collection,
/// it only queues it so it can be added in a thread-safe way later</remarks>
public void Add(OutgoingPacket packet)
{
m_pendingAdds.Enqueue(packet);
}
public void TestReportCriticalPacketDrop(OutgoingPacket packet)
{
lock (_dropReportLock)
{
if (DateTime.Now - _lastDropReport > TimeSpan.FromSeconds(MIN_DROP_REPORT_INTERVAL))
{
_lastDropReport = DateTime.Now;
m_log.WarnFormat("[LLUDP] Reliable packets are being dropped for {0} due to overfilled outbound queue. Last packet type {1}",
AgentID, packet.Type);
}
}
}
/// <summary>
/// Add an unacked packet to the collection
/// </summary>
/// <param name="packet">Packet that is awaiting acknowledgement</param>
/// <returns>True if the packet was successfully added, false if the
/// packet already existed in the collection</returns>
/// <remarks>This does not immediately add the ACK to the collection,
/// it only queues it so it can be added in a thread-safe way later</remarks>
public void Add(OutgoingPacket packet)
{
m_pendingAdds.Enqueue(packet);
}
public void ResendUnacked(OutgoingPacket outgoingPacket)
{
//m_log.DebugFormat("[LLUDPSERVER]: Resending packet #{0} (attempt {1}), {2}ms have passed",
// outgoingPacket.SequenceNumber, outgoingPacket.ResendCount, Environment.TickCount - outgoingPacket.TickCount);
// Set the resent flag
outgoingPacket.Buffer.Data[0] = (byte)(outgoingPacket.Buffer.Data[0] | Helpers.MSG_RESENT);
outgoingPacket.Category = ThrottleOutPacketType.Resend;
// Bump up the resend count on this packet
Interlocked.Increment(ref outgoingPacket.ResendCount);
// Requeue or resend the packet
if (!outgoingPacket.Client.EnqueueOutgoing(outgoingPacket, false))
SendPacketFinal(outgoingPacket);
}
/// <summary>
/// Add an unacked packet to the collection
/// </summary>
/// <param name="packet">Packet that is awaiting acknowledgement</param>
/// <returns>True if the packet was successfully added, false if the
/// packet already existed in the collection</returns>
/// <remarks>This does not immediately add the ACK to the collection,
/// it only queues it so it can be added in a thread-safe way later</remarks>
public void Add(OutgoingPacket packet)
{
m_pendingAdds.Enqueue(packet);
Interlocked.Add(ref packet.Client.UnackedBytes, packet.Buffer.DataLength);
}
/// <summary>
/// Start the process of sending a packet to the client.
/// </summary>
/// <param name="udpClient"></param>
/// <param name="data"></param>
/// <param name="type"></param>
/// <param name="category"></param>
/// <param name="method">
/// The method to call if the packet is not acked by the client. If null, then a standard
/// resend of the packet is done.
/// </param>
public void SendPacketData(
LLUDPClient udpClient, byte[] data, PacketType type, ThrottleOutPacketType category, UnackedPacketMethod method)
{
int dataLength = data.Length;
bool doZerocode = (data[0] & Helpers.MSG_ZEROCODED) != 0;
bool doCopy = true;
// Frequency analysis of outgoing packet sizes shows a large clump of packets at each end of the spectrum.
// The vast majority of packets are less than 200 bytes, although due to asset transfers and packet splitting
// there are a decent number of packets in the 1000-1140 byte range. We allocate one of two sizes of data here
// to accomodate for both common scenarios and provide ample room for ACK appending in both
int bufferSize = (dataLength > 180) ? LLUDPServer.MTU : 200;
UDPPacketBuffer buffer = new UDPPacketBuffer(udpClient.RemoteEndPoint, bufferSize);
// Zerocode if needed
if (doZerocode)
{
try
{
dataLength = Helpers.ZeroEncode(data, dataLength, buffer.Data);
doCopy = false;
}
catch (IndexOutOfRangeException)
{
// The packet grew larger than the bufferSize while zerocoding.
// Remove the MSG_ZEROCODED flag and send the unencoded data
// instead
m_log.Debug("[LLUDPSERVER]: Packet exceeded buffer size during zerocoding for " + type + ". DataLength=" + dataLength +
" and BufferLength=" + buffer.Data.Length + ". Removing MSG_ZEROCODED flag");
data[0] = (byte)(data[0] & ~Helpers.MSG_ZEROCODED);
}
}
// If the packet data wasn't already copied during zerocoding, copy it now
if (doCopy)
{
if (dataLength <= buffer.Data.Length)
{
Buffer.BlockCopy(data, 0, buffer.Data, 0, dataLength);
}
else
{
bufferSize = dataLength;
buffer = new UDPPacketBuffer(udpClient.RemoteEndPoint, bufferSize);
// m_log.Error("[LLUDPSERVER]: Packet exceeded buffer size! This could be an indication of packet assembly not obeying the MTU. Type=" +
// type + ", DataLength=" + dataLength + ", BufferLength=" + buffer.Data.Length + ". Dropping packet");
Buffer.BlockCopy(data, 0, buffer.Data, 0, dataLength);
}
}
buffer.DataLength = dataLength;
#region Queue or Send
OutgoingPacket outgoingPacket = new OutgoingPacket(udpClient, buffer, category, null);
// If we were not provided a method for handling unacked, use the UDPServer default method
outgoingPacket.UnackedMethod = ((method == null) ? delegate(OutgoingPacket oPacket) { ResendUnacked(oPacket); } : method);
// If a Linden Lab 1.23.5 client receives an update packet after a kill packet for an object, it will
// continue to display the deleted object until relog. Therefore, we need to always queue a kill object
// packet so that it isn't sent before a queued update packet.
bool requestQueue = type == PacketType.KillObject;
if (!outgoingPacket.Client.EnqueueOutgoing(outgoingPacket, requestQueue))
SendPacketFinal(outgoingPacket);
#endregion Queue or Send
}
/// <summary>
/// Add an unacked packet to the collection
/// </summary>
/// <param name="packet">Packet that is awaiting acknowledgement</param>
/// <returns>True if the packet was successfully added, false if the
/// packet already existed in the collection</returns>
/// <remarks>This does not immediately add the ACK to the collection,
/// it only queues it so it can be added in a thread-safe way later</remarks>
public void Add(OutgoingPacket packet)
{
packet.AddRef();
m_pendingAdds.Enqueue(packet);
}
