protected IoUringConnectionContext(LinuxSocket socket, EndPoint local, EndPoint remote, TransportThreadContext threadContext)
{
Socket = socket;
LocalEndPoint = local;
RemoteEndPoint = remote;
MemoryPool = threadContext.MemoryPool;
_threadContext = threadContext;
var appScheduler = threadContext.Options.ApplicationSchedulingMode;
var inputOptions = new PipeOptions(MemoryPool, appScheduler, PipeScheduler.Inline, PauseInputWriterThreshold, PauseInputWriterThreshold / 2, useSynchronizationContext: false);
var outputOptions = new PipeOptions(MemoryPool, PipeScheduler.Inline, appScheduler, PauseOutputWriterThreshold, PauseOutputWriterThreshold / 2, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(inputOptions, outputOptions);
Transport = pair.Transport;
Application = pair.Application;
_onOnFlushedToApp = FlushedToAppAsynchronously;
_onReadFromApp = ReadFromAppAsynchronously;
iovec[] vecs = new iovec[ReadIOVecCount + WriteIOVecCount];
var handle = GCHandle.Alloc(vecs, GCHandleType.Pinned);
_iovec = (iovec *)handle.AddrOfPinnedObject();
_iovecHandle = handle;
}
public IoUringConnectionContext(LinuxSocket socket, EndPoint server, EndPoint client, MemoryPool <byte> pool,
TransportThreadContext threadContext)
{
Socket = socket;
LocalEndPoint = server;
RemoteEndPoint = client;
MemoryPool = pool;
_threadContext = threadContext;
var appScheduler = PipeScheduler.ThreadPool; // TODO: configure
var inputOptions = new PipeOptions(MemoryPool, appScheduler, PipeScheduler.Inline, PauseInputWriterThreshold, PauseInputWriterThreshold / 2, useSynchronizationContext: false);
var outputOptions = new PipeOptions(MemoryPool, PipeScheduler.Inline, appScheduler, PauseOutputWriterThreshold, PauseOutputWriterThreshold / 2, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(inputOptions, outputOptions);
Transport = pair.Transport;
Application = pair.Application;
_onOnFlushedToApp = FlushedToAppAsynchronously;
_onReadFromApp = ReadFromAppAsynchronously;
iovec[] vecs = new iovec[ReadIOVecCount + WriteIOVecCount];
var handle = GCHandle.Alloc(vecs, GCHandleType.Pinned);
_iovec = (iovec *)handle.AddrOfPinnedObject();
_iovecHandle = handle;
}
protected IoUringConnection(LinuxSocket socket, EndPoint local, EndPoint remote, MemoryPool <byte> memoryPool, IoUringOptions options, TransportThreadScheduler scheduler)
{
Socket = socket;
LocalEndPoint = local;
RemoteEndPoint = remote;
MemoryPool = memoryPool;
Debug.Assert(MaxBufferSize == MemoryPool.MaxBufferSize);
_scheduler = scheduler;
_connectionClosedTokenSource = new CancellationTokenSource();
ConnectionClosed = _connectionClosedTokenSource.Token;
_waitForConnectionClosedTcs = new TaskCompletionSource <object>(TaskCreationOptions.RunContinuationsAsynchronously);
var appScheduler = options.ApplicationSchedulingMode;
var inputOptions = new PipeOptions(memoryPool, appScheduler, PipeScheduler.Inline, PauseInputWriterThreshold, PauseInputWriterThreshold / 2, useSynchronizationContext: false);
var outputOptions = new PipeOptions(memoryPool, PipeScheduler.Inline, appScheduler, PauseOutputWriterThreshold, PauseOutputWriterThreshold / 2, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(inputOptions, outputOptions);
Transport = pair.Transport;
Application = pair.Application;
_onOnFlushedToApp = () => HandleFlushedToApp();
_onReadFromApp = () => HandleReadFromApp();
_ioVecBytes = GC.AllocateUninitializedArray <byte>(SizeOf.iovec * (ReadIOVecCount + WriteIOVecCount), pinned: true);
unsafe
{
_iovec = (iovec *)MemoryHelper.UnsafeGetAddressOfPinnedArrayData(_ioVecBytes);
}
}
public unsafe ssize_t Send(iovec *ioVectors, int ioVectorLen)
{
var result = TrySend(ioVectors, ioVectorLen);
result.ThrowOnError();
return(result.Value);
}
public TransportThread(IoUringOptions options)
{
int res = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
if (res == -1)
{
throw new ErrnoException(errno);
}
_eventfd = res;
// Pin buffer for eventfd reads via io_uring
byte[] bytes = new byte[8];
_eventfdBytes = GCHandle.Alloc(bytes, GCHandleType.Pinned);
// Pin iovec used for eventfd reads via io_uring
var eventfdIoVec = new iovec
{
iov_base = (void *)_eventfdBytes.AddrOfPinnedObject(),
iov_len = bytes.Length
};
_eventfdIoVecHandle = GCHandle.Alloc(eventfdIoVec, GCHandleType.Pinned);
_eventfdIoVec = (iovec *)_eventfdIoVecHandle.AddrOfPinnedObject();
var memoryPool = new SlabMemoryPool();
_threadContext = new TransportThreadContext(options, memoryPool, _eventfd, s => Read(s), s => Write(s));
_maxBufferSize = memoryPool.MaxBufferSize;
}
/// <summary>
/// Registers a set of buffers with the kernel to reduce per I/O overhead.
/// </summary>
/// <param name="iov">I/O vectors to register</param>
/// <param name="iovcnt">Number of I/O vectors</param>
/// <exception cref="ArgumentOutOfRangeException">If <paramref name="iovcnt"/> is negative</exception>
/// <exception cref="ErrnoException">On failed syscall</exception>
public void RegisterBuffers(iovec *iov, int iovcnt)
{
if (iovcnt < 0)
{
ThrowArgumentOutOfRangeException(ExceptionArgument.iovcnt);
}
Register(IORING_REGISTER_BUFFERS, iov, (uint)iovcnt);
}
/// <summary>
/// Registers a set of buffers with the kernel to reduce per I/O overhead.
/// </summary>
/// <param name="iov">I/O vectors to register</param>
/// <param name="iovcnt">Number of I/O vectors</param>
/// <exception cref="ArgumentOutOfRangeException">If <paramref name="iovcnt"/> is negative</exception>
/// <exception cref="ErrnoException">On failed syscall</exception>
public void RegisterBuffers(iovec *iov, int iovcnt)
{
if (iovcnt < 0)
{
throw new ArgumentOutOfRangeException(nameof(iovcnt), "must be non-negative");
}
Register(IORING_REGISTER_BUFFERS, iov, (uint)iovcnt);
}
public bool TryCompleteReceive(Ring ring, int result, [NotNullWhen(true)] out InboundConnection connection)
{
// Start work-around for https://github.com/axboe/liburing/issues/128
_iov.AsSpan().Clear();
_dummyBuffer.AsSpan().Clear();
_control.AsSpan().Clear();
_header.AsSpan().Clear();
iovec* iov = (iovec*) MemoryHelper.UnsafeGetAddressOfPinnedArrayData(_iov);
iov->iov_base = (byte*) MemoryHelper.UnsafeGetAddressOfPinnedArrayData(_dummyBuffer);
iov->iov_len = 1;
byte* control = (byte*) MemoryHelper.UnsafeGetAddressOfPinnedArrayData(_control);
msghdr* header = (msghdr*) MemoryHelper.UnsafeGetAddressOfPinnedArrayData(_header);
header->msg_iov = iov;
header->msg_iovlen = 1;
header->msg_control = control;
header->msg_controllen = _control.Length;
result = _recipient.RecvMsg(header, (MSG_NOSIGNAL | MSG_CMSG_CLOEXEC));
// End of work-around for https://github.com/axboe/liburing/issues/128
connection = default;
if (result < 0)
{
HandleCompleteReceiveError(ring, result);
return false;
}
bool receivedSocket = false;
LinuxSocket socket = default;
for (cmsghdr* cmsg = CMSG_FIRSTHDR(header); cmsg != null; cmsg = CMSG_NXTHDR(header, cmsg))
{
if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS)
{
int* fdptr = (int*) CMSG_DATA(cmsg);
socket = *fdptr;
socket.SetFlag(O_NONBLOCK);
receivedSocket = true;
break;
}
}
if (!receivedSocket)
{
if (result != 0)
{
PollReceive(ring);
}
return false;
}
connection = new InboundConnection(socket, _endPoint, null, _memoryPool, _options, _scheduler);
return true;
}
/// <summary>
/// Adds a readv, preadv or preadv2 to the Submission Queue without it being submitted.
/// The actual submission can be deferred to avoid unnecessary memory barriers.
/// </summary>
/// <param name="fd">File descriptor to read from</param>
/// <param name="iov">I/O vectors to read to</param>
/// <param name="count">Number of I/O vectors</param>
/// <param name="offset">Offset in bytes into the I/O vectors (as per preadv)</param>
/// <param name="flags">Flags for the I/O (as per preadv2)</param>
/// <param name="userData">User data that will be returned with the respective <see cref="Completion"/></param>
/// <param name="options">Options for the handling of the prepared Submission Queue Entry</param>
/// <exception cref="SubmissionQueueFullException">If no more free space in the Submission Queue is available</exception>
public void PrepareReadV(
int fd, iovec *iov, int count, off_t offset = default, int flags = 0,
ulong userData = 0, SubmissionOption options = SubmissionOption.None)
{
if (!TryPrepareReadV(fd, iov, count, offset, flags, userData, options))
{
ThrowSubmissionQueueFullException();
}
}
public msghdr(void *name, uint namelen, iovec *iov, int iovlen, void *control, int controllen, int flags)
{
msg_name = name;
msg_namelen = (socklen_t)namelen;
msg_iov = iov;
msg_iovlen = iovlen;
msg_control = control;
msg_controllen = (socklen_t)controllen;
msg_flags = flags;
}
public msghdr(void* name, uint namelen, iovec* iov, int iovlen, void* control, int controllen, int flags)
{
msg_name = name;
msg_namelen = (socklen_t)namelen;
msg_iov = iov;
msg_iovlen = (IntPtr)iovlen;
msg_control = control;
msg_controllen = (IntPtr)controllen;
msg_flags = flags;
}
public unsafe RingResult WriteV(int fd, iovec *iov, int count, off_t offset = default, int flags = 0,
SubmissionOption options = SubmissionOption.None)
{
RingResult result = CreateRingResult();
if (!_ring.TryPrepareWriteV(fd, iov, count, offset, flags, result.Handle, options))
{
_pool.Return(result);
return(_ringFullException);
}
return(result);
}
/// <summary>
/// Prepares this Submission Queue Entry as a writev, pwritev or pwritev2.
/// </summary>
/// <param name="fd">File descriptor to write to</param>
/// <param name="iov">I/O vectors to write</param>
/// <param name="count">Number of I/O vectors</param>
/// <param name="offset">Offset at which the I/O is performed (as per pwritev)</param>
/// <param name="flags">Flags for the I/O (as per pwritev2)</param>
/// <param name="userData">User data that will be returned with the respective <see cref="Completion"/></param>
/// <param name="options">Options for the handling of the prepared Submission Queue Entry</param>
/// <param name="personality">The personality to impersonate for this submission</param>
public void PrepareWriteV(int fd, iovec *iov, int count, off_t offset = default, int flags = 0, ulong userData = 0, SubmissionOption options = SubmissionOption.None, ushort personality = 0)
{
var sqe = _sqe;
unchecked
{
sqe->opcode = IORING_OP_WRITEV;
sqe->flags = (byte)options;
sqe->fd = fd;
sqe->off = (ulong)(long)offset;
sqe->addr = (ulong)iov;
sqe->len = (uint)count;
sqe->rw_flags = flags;
sqe->user_data = userData;
sqe->personality = personality;
}
}
public static unsafe PosixResult Receive(int socket, iovec *ioVectors, int ioVectorLen)
{
msghdr hdr = default(msghdr);
hdr.msg_iov = ioVectors;
hdr.msg_iovlen = ioVectorLen;
int flags = MSG_NOSIGNAL;
ssize_t rv;
do
{
rv = recvmsg(socket, &hdr, flags);
} while (rv < 0 && errno == EINTR);
return(PosixResult.FromReturnValue(rv));
}
/// <summary>
/// Prepares this Submission Queue Entry as a readv, preadv or preadv2 with buffer selection.
/// </summary>
/// <param name="fd">File descriptor to read from</param>
/// <param name="iov">I/O vectors to read to</param>
/// <param name="count">Number of I/O vectors</param>
/// <param name="bgid">Group ID for buffer selection</param>
/// <param name="offset">Offset at which the I/O is performed (as per preadv)</param>
/// <param name="flags">Flags for the I/O (as per preadv2)</param>
/// <param name="userData">User data that will be returned with the respective <see cref="Completion"/></param>
/// <param name="options">Options for the handling of the prepared Submission Queue Entry</param>
/// <param name="personality">The personality to impersonate for this submission</param>
public void PrepareReadV(int fd, iovec *iov, int count, int bgid, off_t offset = default, int flags = 0, ulong userData = 0, SubmissionOption options = SubmissionOption.None, ushort personality = 0)
{
var sqe = _sqe;
unchecked
{
sqe->opcode = IORING_OP_READV;
sqe->flags = (byte)((byte)options | IOSQE_BUFFER_SELECT);
sqe->fd = fd;
sqe->off = (ulong)(long)offset;
sqe->addr = (ulong)iov;
sqe->len = (uint)count;
sqe->rw_flags = flags;
sqe->user_data = userData;
sqe->buf_group = (ushort)bgid;
sqe->personality = personality;
}
}
public unsafe IList <ArraySegment <byte> > ReceiveMultipleMessages(IList <ArraySegment <byte> > buffers)
{
if (handle == -1)
{
throw new ObjectDisposedException(objectname);
}
GCHandle[] bufhandles = new GCHandle[buffers.Count];
iovec * iovecs = stackalloc iovec[buffers.Count];
mmsghdr * hdrs = stackalloc mmsghdr[buffers.Count];
for (int i = 0; i < buffers.Count; i++)
{
bufhandles[i] = GCHandle.Alloc(buffers[i].Array, GCHandleType.Pinned);
iovecs[i] = new iovec()
{
iov_base = (byte *)bufhandles[i].AddrOfPinnedObject() + buffers[i].Offset, iov_len = (UIntPtr)(UInt32)buffers[i].Count
};
hdrs[i] = new mmsghdr()
{
msg_hdr = new msghdr()
{
msg_iov = &iovecs[i], msg_iovlen = (UIntPtr)1
}
};
}
int ret = recvmmsg(handle, hdrs, (uint)buffers.Count, MSG_WAITFORONE, null);
for (int i = 0; i < buffers.Count; i++)
{
bufhandles[i].Free();
}
if (ret == -1)
{
throw new PosixException("recvmmsg");
}
ArraySegment <byte>[] retvec = new ArraySegment <byte> [ret];
for (int i = 0; i < ret && i < buffers.Count; i++)
{
retvec[i] = new ArraySegment <byte>(buffers[i].Array, buffers[i].Offset, (int)hdrs[i].msg_len);
}
return(retvec);
}
/// <summary>
/// Attempts to add a readv, preadv or preadv2 to the Submission Queue without it being submitted.
/// The actual submission can be deferred to avoid unnecessary memory barriers.
/// </summary>
/// <param name="fd">File descriptor to read from</param>
/// <param name="iov">I/O vectors to read to</param>
/// <param name="count">Number of I/O vectors</param>
/// <param name="offset">Offset in bytes into the I/O vectors (as per preadv)</param>
/// <param name="flags">Flags for the I/O (as per preadv2)</param>
/// <param name="userData">User data that will be returned with the respective <see cref="Completion"/></param>
/// <param name="options">Options for the handling of the prepared Submission Queue Entry</param>
/// <returns><code>false</code> if the submission queue is full. <code>true</code> otherwise.</returns>
public bool TryPrepareReadV(
int fd, iovec *iov, int count, off_t offset = default, int flags = 0,
ulong userData = 0, SubmissionOption options = SubmissionOption.None)
=> TryPrepareReadWrite(IORING_OP_READV, fd, iov, count, offset, flags, userData, options);
public static unsafe PosixResult Receive(SafeHandle socket, iovec *ioVectors, int ioVectorLen) => Receive(socket.DangerousGetHandle().ToInt32(), ioVectors, ioVectorLen);
public static unsafe PosixResult Send(SafeHandle socket, iovec *ioVectors, int ioVectorLen, int flags = 0) => Send(socket.DangerousGetHandle().ToInt32(), ioVectors, ioVectorLen, flags);
public unsafe PosixResult TrySend(iovec *ioVectors, int ioVectorLen)
{
return(SocketInterop.Send(this, ioVectors, ioVectorLen));
}
public static extern ssize_t vmsplice(int fd, iovec *iov, size_t count, uint flags);
private unsafe void AioSend(List <ScheduledSend> sendQueue)
{
while (sendQueue.Count > 0)
{
int sendCount = 0;
int completedCount = 0;
iocb * aioCbs = AioCbs;
iovec *ioVectors = IoVectorTable;
ReadOnlySequence <byte>[] sendBuffers = _aioSendBuffers;
Span <MemoryHandle> memoryHandles = MemoryHandles;
int memoryHandleCount = 0;
for (int i = 0; i < sendQueue.Count; i++)
{
TSocket socket = sendQueue[i].Socket;
ReadOnlySequence <byte> buffer;
Exception error = socket.GetReadResult(out buffer);
if (error != null)
{
if (error == TransportConstants.StopSentinel)
{
error = null;
}
socket.CompleteOutput(error);
completedCount++;
}
else
{
int ioVectorLength = socket.CalcIOVectorLengthForSend(ref buffer, IoVectorsPerAioSocket);
socket.FillSendIOVector(ref buffer, ioVectors, ioVectorLength, memoryHandles);
memoryHandles = memoryHandles.Slice(ioVectorLength);
memoryHandleCount += ioVectorLength;
aioCbs->aio_fildes = socket.Fd;
aioCbs->aio_data = (ulong)i;
aioCbs->aio_lio_opcode = IOCB_CMD_PWRITEV;
aioCbs->aio_buf = (ulong)ioVectors;
aioCbs->aio_nbytes = (ulong)ioVectorLength;
aioCbs++;
sendBuffers[sendCount] = buffer;
sendCount++;
if (sendCount == EventBufferLength)
{
break;
}
ioVectors += ioVectorLength;
}
}
if (sendCount > 0)
{
PosixResult res = AioInterop.IoSubmit(_aioContext, sendCount, AioCbsTable);
memoryHandles = MemoryHandles;
for (int i = 0; i < memoryHandleCount; i++)
{
memoryHandles[i].Dispose();
}
if (res != sendCount)
{
throw new NotSupportedException("Unexpected IoSubmit Send retval " + res);
}
io_event *aioEvents = AioEvents;
res = AioInterop.IoGetEvents(_aioContext, sendCount, aioEvents);
if (res != sendCount)
{
throw new NotSupportedException("Unexpected IoGetEvents Send retval " + res);
}
io_event *aioEvent = aioEvents;
for (int i = 0; i < sendCount; i++)
{
PosixResult result = new PosixResult((int)aioEvent->res);
int socketIndex = (int)aioEvent->data;
TSocket socket = sendQueue[socketIndex].Socket;
ReadOnlySequence <byte> buffer = sendBuffers[i]; // assumes in-order events
sendBuffers[i] = default;
socket.HandleSendResult(ref buffer, result, loop: false, zerocopy: false, zeroCopyRegistered: false);
aioEvent++;
}
}
sendQueue.RemoveRange(0, sendCount + completedCount);
}
}
private unsafe void AioReceive(List <TSocket> readableSockets)
{
ulong PackReceiveState(int received, int advanced, int iovLength) => ((ulong)received << 32) + (ulong)(advanced << 8) + (ulong)(iovLength);
(int received, int advanced, int iovLength) UnpackReceiveState(ulong data) => ((int)(data >> 32), (int)((data >> 8) & 0xffffff), (int)(data & 0xff));
int readableSocketCount = readableSockets.Count;
iocb * aioCb = AioCbs;
iovec * ioVectors = IoVectorTable;
PosixResult * receiveResults = stackalloc PosixResult[readableSocketCount];
Span <MemoryHandle> receiveMemoryHandles = MemoryHandles;
int receiveMemoryHandleCount = 0;
for (int i = 0; i < readableSocketCount; i++)
{
TSocket socket = readableSockets[i];
var memoryAllocation = socket.DetermineMemoryAllocationForReceive(IoVectorsPerAioSocket);
int advanced = socket.FillReceiveIOVector(memoryAllocation, ioVectors, receiveMemoryHandles);
aioCb->aio_fildes = socket.Fd;
aioCb->aio_data = PackReceiveState(0, advanced, memoryAllocation.IovLength);
aioCb->aio_lio_opcode = IOCB_CMD_PREADV;
aioCb->aio_buf = (ulong)ioVectors;
aioCb->aio_nbytes = (ulong)memoryAllocation.IovLength;
aioCb++;
ioVectors += memoryAllocation.IovLength;
receiveMemoryHandleCount += memoryAllocation.IovLength;
receiveMemoryHandles = receiveMemoryHandles.Slice(memoryAllocation.IovLength);
}
int eAgainCount = 0;
while (readableSocketCount > 0)
{
PosixResult res = AioInterop.IoSubmit(_aioContext, readableSocketCount, AioCbsTable);
if (res != readableSocketCount)
{
throw new NotSupportedException("Unexpected IoSubmit retval " + res);
}
io_event *aioEvents = AioEvents;
res = AioInterop.IoGetEvents(_aioContext, readableSocketCount, aioEvents);
if (res != readableSocketCount)
{
throw new NotSupportedException("Unexpected IoGetEvents retval " + res);
}
int socketsRemaining = readableSocketCount;
bool allEAgain = true;
io_event *aioEvent = aioEvents;
for (int i = 0; i < readableSocketCount; i++)
{
PosixResult result = new PosixResult((int)aioEvent->res);
int socketIndex = i; // assumes in-order events
TSocket socket = readableSockets[socketIndex];
(int received, int advanced, int iovLength) = UnpackReceiveState(aioEvent->data);
(bool done, PosixResult retval) = socket.InterpretReceiveResult(result, ref received, advanced, (iovec *)((iocb *)aioEvent->obj)->aio_buf, iovLength);
if (done)
{
receiveResults[socketIndex] = retval;
socketsRemaining--;
((iocb *)aioEvent->obj)->aio_lio_opcode = IOCB_CMD_NOOP;
allEAgain = false;
}
else if (retval != PosixResult.EAGAIN)
{
((iocb *)aioEvent->obj)->aio_data = PackReceiveState(received, advanced, iovLength);
allEAgain = false;
}
aioEvent++;
}
if (socketsRemaining > 0)
{
if (allEAgain)
{
eAgainCount++;
if (eAgainCount == TransportConstants.MaxEAgainCount)
{
throw new NotSupportedException("Too many EAGAIN, unable to receive available bytes.");
}
}
else
{
aioCb = AioCbs;
iocb *aioCbWriteAt = aioCb;
// The kernel doesn't handle Noop, we need to remove them from the aioCbs
for (int i = 0; i < readableSocketCount; i++)
{
if (aioCb[i].aio_lio_opcode != IOCB_CMD_NOOP)
{
if (aioCbWriteAt != aioCb)
{
*aioCbWriteAt = *aioCb;
}
aioCbWriteAt++;
}
aioCb++;
}
readableSocketCount = socketsRemaining;
eAgainCount = 0;
}
}
else
{
readableSocketCount = 0;
}
}
receiveMemoryHandles = MemoryHandles;
for (int i = 0; i < receiveMemoryHandleCount; i++)
{
receiveMemoryHandles[i].Dispose();
}
for (int i = 0; i < readableSockets.Count; i++)
{
readableSockets[i].OnReceiveFromSocket(receiveResults[i]);
}
readableSockets.Clear();
}
private static extern unsafe IntPtr process_vm_readv(int pid, iovec *local_iov, UIntPtr liovcnt, iovec *remote_iov, UIntPtr riovcnt, UIntPtr flags);
private static extern unsafe int process_vm_readv(int pid,
iovec *local_iov,
ulong liovcnt,
iovec *remote_iov,
ulong riovcnt,
ulong flags);
public unsafe PosixResult TryReceive(iovec *ioVectors, int ioVectorLen)
{
return(SocketInterop.Receive(this, ioVectors, ioVectorLen));
}
private unsafe bool WriteSubmissions()
{
Ring ring = _ring !;
int iovIndex = _iovsUsed;
int sqesAvailable = ring.SubmissionEntriesAvailable;
iovec *iovs = IoVectorTable;
for (int i = 0; (i < _newOperations.Count) && (sqesAvailable > 2) && (iovIndex < _iovsLength); i++)
{
_newOperationsQueued++;
Operation op = _newOperations[i];
int fd = op.Handle !.DangerousGetHandle().ToInt32();
ulong key = CalculateKey(op.Handle, op.Data);
switch (op.OperationType)
{
case OperationType.Read when op.Status == OperationStatus.PollForReadWrite:
{
// Poll first, in case the fd is non-blocking.
sqesAvailable -= 1;
ring.PreparePollAdd(fd, (ushort)POLLIN, key | PollMaskBit);
break;
}
case OperationType.Read when op.Status == OperationStatus.Execution:
{
MemoryHandle handle = op.Memory.Pin();
op.MemoryHandle = handle;
iovec *iov = &iovs[iovIndex++]; // Linux 5.6 doesn't need an iovec (IORING_OP_READ)
* iov = new iovec {
iov_base = handle.Pointer, iov_len = op.Memory.Length
};
sqesAvailable -= 1;
ring.PrepareReadV(fd, iov, 1, userData: key);
break;
}
case OperationType.Write when op.Status == OperationStatus.PollForReadWrite:
{
// Poll first, in case the fd is non-blocking.
sqesAvailable -= 1;
ring.PreparePollAdd(fd, (ushort)POLLOUT, key | PollMaskBit);
break;
}
case OperationType.Write when op.Status == OperationStatus.Execution:
{
MemoryHandle handle = op.Memory.Pin();
op.MemoryHandle = handle;
iovec *iov = &iovs[iovIndex++]; // Linux 5.6 doesn't need an iovec (IORING_OP_WRITE)
* iov = new iovec {
iov_base = handle.Pointer, iov_len = op.Memory.Length
};
sqesAvailable -= 1;
ring.PrepareWriteV(fd, iov, 1, userData: key);
break;
}
case OperationType.PollIn:
{
sqesAvailable -= 1;
ring.PreparePollAdd(fd, (ushort)POLLIN, key);
break;
}
case OperationType.PollOut:
{
sqesAvailable -= 1;
ring.PreparePollAdd(fd, (ushort)POLLOUT, key);
break;
}
case OperationType.Cancel:
{
sqesAvailable -= 2;
// Cancel the operation and possibly associated poll operation.
ring.PrepareCancel(opUserData: key | PollMaskBit, userData: IgnoredData);
ring.PrepareCancel(opUserData: key, userData: IgnoredData);
// Cancel operations aren't added to the dictionary, we can return it now.
ReturnOperation(op);
break;
}
}
}
_iovsUsed = iovIndex;
bool operationsRemaining = (_newOperations.Count - _newOperationsQueued) > 0;
return(operationsRemaining);
}
