private void Forward(IxgbeDevice rxDev, int rxQueue, IxgbeDevice txDev, int txQueue, Span <PacketBuffer> buffers)
{
var rxBuffCount = rxDev.RxBatch(rxQueue, buffers);
if (rxBuffCount > 0)
{
var rxBuffers = buffers.Slice(0, rxBuffCount);
//Touch all buffers to simulate a realistic scenario
foreach (var buffer in rxBuffers)
{
buffer.Touch();
}
var txBuffCount = txDev.TxBatch(txQueue, rxBuffers);
_mempool = (_mempool == null) ? Mempool.FindPool(rxBuffers[0].MempoolId) : _mempool;
//Drop unsent packets
foreach (var buffer in rxBuffers.Slice(txBuffCount))
{
_mempool.FreeBuffer(buffer);
}
}
}
public override int TxBatch(int queueId, Span <PacketBuffer> buffers)
{
if (queueId < 0 || queueId >= RxQueues.Length)
{
throw new ArgumentOutOfRangeException("Queue id out of bounds");
}
var queue = TxQueues[queueId] as IxgbeTxQueue;
ushort cleanIndex = queue.CleanIndex;
ushort currentIndex = (ushort)queue.Index;
var cmdTypeFlags = IxgbeDefs.ADVTXD_DCMD_EOP | IxgbeDefs.ADVTXD_DCMD_RS | IxgbeDefs.ADVTXD_DCMD_IFCS |
IxgbeDefs.ADVTXD_DCMD_DEXT | IxgbeDefs.ADVTXD_DTYP_DATA;
//All packet buffers that will be handled here will belong to the same mempool
Mempool pool = null;
//Step 1: Clean up descriptors that were sent out by the hardware and return them to the mempool
//Start by reading step 2 which is done first for each packet
//Cleaning up must be done in batches for performance reasons, so this is unfortunately somewhat complicated
while (true)
{
//currentIndex is always ahead of clean (invariant of our queue)
int cleanable = currentIndex - cleanIndex;
if (cleanable < 0)
{
cleanable = queue.EntriesCount + cleanable;
}
if (cleanable < TxCleanBatch)
{
break;
}
//Calculate the index of the last transcriptor in the clean batch
//We can't check all descriptors for performance reasons
int cleanupTo = cleanIndex + TxCleanBatch - 1;
if (cleanupTo >= queue.EntriesCount)
{
cleanupTo -= queue.EntriesCount;
}
var descAddr = queue.GetDescriptorAddress((ushort)cleanupTo);
uint status = queue.ReadWbStatus(descAddr);
//Hardware sets this flag as soon as it's sent out, we can give back all bufs in the batch back to the mempool
if ((status & IxgbeDefs.ADVTXD_STAT_DD) != 0)
{
int i = cleanIndex;
while (true)
{
var packetBuffer = new PacketBuffer(queue.VirtualAddresses[i]);
if (pool == null)
{
pool = Mempool.FindPool(packetBuffer.MempoolId);
if (pool == null)
{
throw new NullReferenceException("Could not find mempool with id specified by PacketBuffer");
}
}
pool.FreeBufferFast(packetBuffer);
if (i == cleanupTo)
{
break;
}
i = WrapRing(i, queue.EntriesCount);
}
//Next descriptor to be cleaned up is one after the one we just cleaned
cleanIndex = (ushort)WrapRing(cleanupTo, queue.EntriesCount);
}
//Clean the whole batch or nothing. This will leave some packets in the queue forever
//if you stop transmitting but that's not a real concern
else
{
break;
}
}
queue.CleanIndex = cleanIndex;
//Step 2: Send out as many of our packets as possible
int sent;
for (sent = 0; sent < buffers.Length; sent++)
{
var descAddr = queue.GetDescriptorAddress(currentIndex);
ushort nextIndex = WrapRing(currentIndex, (ushort)queue.EntriesCount);
//We are full if the next index is the one we are trying to reclaim
if (cleanIndex == nextIndex)
{
break;
}
var buffer = buffers[sent];
//Remember virtual address to clean it up later
queue.VirtualAddresses[currentIndex] = buffer.VirtualAddress;
queue.Index = WrapRing(queue.Index, queue.EntriesCount);
//NIC reads from here
queue.WriteBufferAddress(descAddr, buffer.PhysicalAddress + PacketBuffer.DataOffset);
//Always the same flags: One buffer (EOP), advanced data descriptor, CRC offload, data length
var bufSize = buffer.Size;
queue.WriteCmdTypeLength(descAddr, cmdTypeFlags | bufSize);
//No fancy offloading - only the total payload length
//implement offloading flags here:
// * ip checksum offloading is trivial: just set the offset
// * tcp/udp checksum offloading is more annoying, you have to precalculate the pseudo-header checksum
queue.WriteOlInfoStatus(descAddr, bufSize << (int)IxgbeDefs.ADVTXD_PAYLEN_SHIFT);
currentIndex = nextIndex;
}
//Send out by advancing tail, i.e. pass control of the bus to the NIC
SetReg(IxgbeDefs.TDT((uint)queueId), (uint)queue.Index);
return(sent);
}