protected IoUringConnectionContext(LinuxSocket socket, EndPoint local, EndPoint remote, TransportThreadContext threadContext) { Socket = socket; LocalEndPoint = local; RemoteEndPoint = remote; MemoryPool = threadContext.MemoryPool; _threadContext = threadContext; var appScheduler = threadContext.Options.ApplicationSchedulingMode; var inputOptions = new PipeOptions(MemoryPool, appScheduler, PipeScheduler.Inline, PauseInputWriterThreshold, PauseInputWriterThreshold / 2, useSynchronizationContext: false); var outputOptions = new PipeOptions(MemoryPool, PipeScheduler.Inline, appScheduler, PauseOutputWriterThreshold, PauseOutputWriterThreshold / 2, useSynchronizationContext: false); var pair = DuplexPipe.CreateConnectionPair(inputOptions, outputOptions); Transport = pair.Transport; Application = pair.Application; _onOnFlushedToApp = FlushedToAppAsynchronously; _onReadFromApp = ReadFromAppAsynchronously; iovec[] vecs = new iovec[ReadIOVecCount + WriteIOVecCount]; var handle = GCHandle.Alloc(vecs, GCHandleType.Pinned); _iovec = (iovec *)handle.AddrOfPinnedObject(); _iovecHandle = handle; }
public IoUringConnectionContext(LinuxSocket socket, EndPoint server, EndPoint client, MemoryPool <byte> pool, TransportThreadContext threadContext) { Socket = socket; LocalEndPoint = server; RemoteEndPoint = client; MemoryPool = pool; _threadContext = threadContext; var appScheduler = PipeScheduler.ThreadPool; // TODO: configure var inputOptions = new PipeOptions(MemoryPool, appScheduler, PipeScheduler.Inline, PauseInputWriterThreshold, PauseInputWriterThreshold / 2, useSynchronizationContext: false); var outputOptions = new PipeOptions(MemoryPool, PipeScheduler.Inline, appScheduler, PauseOutputWriterThreshold, PauseOutputWriterThreshold / 2, useSynchronizationContext: false); var pair = DuplexPipe.CreateConnectionPair(inputOptions, outputOptions); Transport = pair.Transport; Application = pair.Application; _onOnFlushedToApp = FlushedToAppAsynchronously; _onReadFromApp = ReadFromAppAsynchronously; iovec[] vecs = new iovec[ReadIOVecCount + WriteIOVecCount]; var handle = GCHandle.Alloc(vecs, GCHandleType.Pinned); _iovec = (iovec *)handle.AddrOfPinnedObject(); _iovecHandle = handle; }
protected IoUringConnection(LinuxSocket socket, EndPoint local, EndPoint remote, MemoryPool <byte> memoryPool, IoUringOptions options, TransportThreadScheduler scheduler) { Socket = socket; LocalEndPoint = local; RemoteEndPoint = remote; MemoryPool = memoryPool; Debug.Assert(MaxBufferSize == MemoryPool.MaxBufferSize); _scheduler = scheduler; _connectionClosedTokenSource = new CancellationTokenSource(); ConnectionClosed = _connectionClosedTokenSource.Token; _waitForConnectionClosedTcs = new TaskCompletionSource <object>(TaskCreationOptions.RunContinuationsAsynchronously); var appScheduler = options.ApplicationSchedulingMode; var inputOptions = new PipeOptions(memoryPool, appScheduler, PipeScheduler.Inline, PauseInputWriterThreshold, PauseInputWriterThreshold / 2, useSynchronizationContext: false); var outputOptions = new PipeOptions(memoryPool, PipeScheduler.Inline, appScheduler, PauseOutputWriterThreshold, PauseOutputWriterThreshold / 2, useSynchronizationContext: false); var pair = DuplexPipe.CreateConnectionPair(inputOptions, outputOptions); Transport = pair.Transport; Application = pair.Application; _onOnFlushedToApp = () => HandleFlushedToApp(); _onReadFromApp = () => HandleReadFromApp(); _ioVecBytes = GC.AllocateUninitializedArray <byte>(SizeOf.iovec * (ReadIOVecCount + WriteIOVecCount), pinned: true); unsafe { _iovec = (iovec *)MemoryHelper.UnsafeGetAddressOfPinnedArrayData(_ioVecBytes); } }
public unsafe ssize_t Send(iovec *ioVectors, int ioVectorLen) { var result = TrySend(ioVectors, ioVectorLen); result.ThrowOnError(); return(result.Value); }
public TransportThread(IoUringOptions options) { int res = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); if (res == -1) { throw new ErrnoException(errno); } _eventfd = res; // Pin buffer for eventfd reads via io_uring byte[] bytes = new byte[8]; _eventfdBytes = GCHandle.Alloc(bytes, GCHandleType.Pinned); // Pin iovec used for eventfd reads via io_uring var eventfdIoVec = new iovec { iov_base = (void *)_eventfdBytes.AddrOfPinnedObject(), iov_len = bytes.Length }; _eventfdIoVecHandle = GCHandle.Alloc(eventfdIoVec, GCHandleType.Pinned); _eventfdIoVec = (iovec *)_eventfdIoVecHandle.AddrOfPinnedObject(); var memoryPool = new SlabMemoryPool(); _threadContext = new TransportThreadContext(options, memoryPool, _eventfd, s => Read(s), s => Write(s)); _maxBufferSize = memoryPool.MaxBufferSize; }
/// <summary> /// Registers a set of buffers with the kernel to reduce per I/O overhead. /// </summary> /// <param name="iov">I/O vectors to register</param> /// <param name="iovcnt">Number of I/O vectors</param> /// <exception cref="ArgumentOutOfRangeException">If <paramref name="iovcnt"/> is negative</exception> /// <exception cref="ErrnoException">On failed syscall</exception> public void RegisterBuffers(iovec *iov, int iovcnt) { if (iovcnt < 0) { ThrowArgumentOutOfRangeException(ExceptionArgument.iovcnt); } Register(IORING_REGISTER_BUFFERS, iov, (uint)iovcnt); }
/// <summary> /// Registers a set of buffers with the kernel to reduce per I/O overhead. /// </summary> /// <param name="iov">I/O vectors to register</param> /// <param name="iovcnt">Number of I/O vectors</param> /// <exception cref="ArgumentOutOfRangeException">If <paramref name="iovcnt"/> is negative</exception> /// <exception cref="ErrnoException">On failed syscall</exception> public void RegisterBuffers(iovec *iov, int iovcnt) { if (iovcnt < 0) { throw new ArgumentOutOfRangeException(nameof(iovcnt), "must be non-negative"); } Register(IORING_REGISTER_BUFFERS, iov, (uint)iovcnt); }
public bool TryCompleteReceive(Ring ring, int result, [NotNullWhen(true)] out InboundConnection connection) { // Start work-around for https://github.com/axboe/liburing/issues/128 _iov.AsSpan().Clear(); _dummyBuffer.AsSpan().Clear(); _control.AsSpan().Clear(); _header.AsSpan().Clear(); iovec* iov = (iovec*) MemoryHelper.UnsafeGetAddressOfPinnedArrayData(_iov); iov->iov_base = (byte*) MemoryHelper.UnsafeGetAddressOfPinnedArrayData(_dummyBuffer); iov->iov_len = 1; byte* control = (byte*) MemoryHelper.UnsafeGetAddressOfPinnedArrayData(_control); msghdr* header = (msghdr*) MemoryHelper.UnsafeGetAddressOfPinnedArrayData(_header); header->msg_iov = iov; header->msg_iovlen = 1; header->msg_control = control; header->msg_controllen = _control.Length; result = _recipient.RecvMsg(header, (MSG_NOSIGNAL | MSG_CMSG_CLOEXEC)); // End of work-around for https://github.com/axboe/liburing/issues/128 connection = default; if (result < 0) { HandleCompleteReceiveError(ring, result); return false; } bool receivedSocket = false; LinuxSocket socket = default; for (cmsghdr* cmsg = CMSG_FIRSTHDR(header); cmsg != null; cmsg = CMSG_NXTHDR(header, cmsg)) { if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) { int* fdptr = (int*) CMSG_DATA(cmsg); socket = *fdptr; socket.SetFlag(O_NONBLOCK); receivedSocket = true; break; } } if (!receivedSocket) { if (result != 0) { PollReceive(ring); } return false; } connection = new InboundConnection(socket, _endPoint, null, _memoryPool, _options, _scheduler); return true; }
/// <summary> /// Adds a readv, preadv or preadv2 to the Submission Queue without it being submitted. /// The actual submission can be deferred to avoid unnecessary memory barriers. /// </summary> /// <param name="fd">File descriptor to read from</param> /// <param name="iov">I/O vectors to read to</param> /// <param name="count">Number of I/O vectors</param> /// <param name="offset">Offset in bytes into the I/O vectors (as per preadv)</param> /// <param name="flags">Flags for the I/O (as per preadv2)</param> /// <param name="userData">User data that will be returned with the respective <see cref="Completion"/></param> /// <param name="options">Options for the handling of the prepared Submission Queue Entry</param> /// <exception cref="SubmissionQueueFullException">If no more free space in the Submission Queue is available</exception> public void PrepareReadV( int fd, iovec *iov, int count, off_t offset = default, int flags = 0, ulong userData = 0, SubmissionOption options = SubmissionOption.None) { if (!TryPrepareReadV(fd, iov, count, offset, flags, userData, options)) { ThrowSubmissionQueueFullException(); } }
public msghdr(void *name, uint namelen, iovec *iov, int iovlen, void *control, int controllen, int flags) { msg_name = name; msg_namelen = (socklen_t)namelen; msg_iov = iov; msg_iovlen = iovlen; msg_control = control; msg_controllen = (socklen_t)controllen; msg_flags = flags; }
public msghdr(void* name, uint namelen, iovec* iov, int iovlen, void* control, int controllen, int flags) { msg_name = name; msg_namelen = (socklen_t)namelen; msg_iov = iov; msg_iovlen = (IntPtr)iovlen; msg_control = control; msg_controllen = (IntPtr)controllen; msg_flags = flags; }
public unsafe RingResult WriteV(int fd, iovec *iov, int count, off_t offset = default, int flags = 0, SubmissionOption options = SubmissionOption.None) { RingResult result = CreateRingResult(); if (!_ring.TryPrepareWriteV(fd, iov, count, offset, flags, result.Handle, options)) { _pool.Return(result); return(_ringFullException); } return(result); }
/// <summary> /// Prepares this Submission Queue Entry as a writev, pwritev or pwritev2. /// </summary> /// <param name="fd">File descriptor to write to</param> /// <param name="iov">I/O vectors to write</param> /// <param name="count">Number of I/O vectors</param> /// <param name="offset">Offset at which the I/O is performed (as per pwritev)</param> /// <param name="flags">Flags for the I/O (as per pwritev2)</param> /// <param name="userData">User data that will be returned with the respective <see cref="Completion"/></param> /// <param name="options">Options for the handling of the prepared Submission Queue Entry</param> /// <param name="personality">The personality to impersonate for this submission</param> public void PrepareWriteV(int fd, iovec *iov, int count, off_t offset = default, int flags = 0, ulong userData = 0, SubmissionOption options = SubmissionOption.None, ushort personality = 0) { var sqe = _sqe; unchecked { sqe->opcode = IORING_OP_WRITEV; sqe->flags = (byte)options; sqe->fd = fd; sqe->off = (ulong)(long)offset; sqe->addr = (ulong)iov; sqe->len = (uint)count; sqe->rw_flags = flags; sqe->user_data = userData; sqe->personality = personality; } }
public static unsafe PosixResult Receive(int socket, iovec *ioVectors, int ioVectorLen) { msghdr hdr = default(msghdr); hdr.msg_iov = ioVectors; hdr.msg_iovlen = ioVectorLen; int flags = MSG_NOSIGNAL; ssize_t rv; do { rv = recvmsg(socket, &hdr, flags); } while (rv < 0 && errno == EINTR); return(PosixResult.FromReturnValue(rv)); }
/// <summary> /// Prepares this Submission Queue Entry as a readv, preadv or preadv2 with buffer selection. /// </summary> /// <param name="fd">File descriptor to read from</param> /// <param name="iov">I/O vectors to read to</param> /// <param name="count">Number of I/O vectors</param> /// <param name="bgid">Group ID for buffer selection</param> /// <param name="offset">Offset at which the I/O is performed (as per preadv)</param> /// <param name="flags">Flags for the I/O (as per preadv2)</param> /// <param name="userData">User data that will be returned with the respective <see cref="Completion"/></param> /// <param name="options">Options for the handling of the prepared Submission Queue Entry</param> /// <param name="personality">The personality to impersonate for this submission</param> public void PrepareReadV(int fd, iovec *iov, int count, int bgid, off_t offset = default, int flags = 0, ulong userData = 0, SubmissionOption options = SubmissionOption.None, ushort personality = 0) { var sqe = _sqe; unchecked { sqe->opcode = IORING_OP_READV; sqe->flags = (byte)((byte)options | IOSQE_BUFFER_SELECT); sqe->fd = fd; sqe->off = (ulong)(long)offset; sqe->addr = (ulong)iov; sqe->len = (uint)count; sqe->rw_flags = flags; sqe->user_data = userData; sqe->buf_group = (ushort)bgid; sqe->personality = personality; } }
public unsafe IList <ArraySegment <byte> > ReceiveMultipleMessages(IList <ArraySegment <byte> > buffers) { if (handle == -1) { throw new ObjectDisposedException(objectname); } GCHandle[] bufhandles = new GCHandle[buffers.Count]; iovec * iovecs = stackalloc iovec[buffers.Count]; mmsghdr * hdrs = stackalloc mmsghdr[buffers.Count]; for (int i = 0; i < buffers.Count; i++) { bufhandles[i] = GCHandle.Alloc(buffers[i].Array, GCHandleType.Pinned); iovecs[i] = new iovec() { iov_base = (byte *)bufhandles[i].AddrOfPinnedObject() + buffers[i].Offset, iov_len = (UIntPtr)(UInt32)buffers[i].Count }; hdrs[i] = new mmsghdr() { msg_hdr = new msghdr() { msg_iov = &iovecs[i], msg_iovlen = (UIntPtr)1 } }; } int ret = recvmmsg(handle, hdrs, (uint)buffers.Count, MSG_WAITFORONE, null); for (int i = 0; i < buffers.Count; i++) { bufhandles[i].Free(); } if (ret == -1) { throw new PosixException("recvmmsg"); } ArraySegment <byte>[] retvec = new ArraySegment <byte> [ret]; for (int i = 0; i < ret && i < buffers.Count; i++) { retvec[i] = new ArraySegment <byte>(buffers[i].Array, buffers[i].Offset, (int)hdrs[i].msg_len); } return(retvec); }
/// <summary> /// Attempts to add a readv, preadv or preadv2 to the Submission Queue without it being submitted. /// The actual submission can be deferred to avoid unnecessary memory barriers. /// </summary> /// <param name="fd">File descriptor to read from</param> /// <param name="iov">I/O vectors to read to</param> /// <param name="count">Number of I/O vectors</param> /// <param name="offset">Offset in bytes into the I/O vectors (as per preadv)</param> /// <param name="flags">Flags for the I/O (as per preadv2)</param> /// <param name="userData">User data that will be returned with the respective <see cref="Completion"/></param> /// <param name="options">Options for the handling of the prepared Submission Queue Entry</param> /// <returns><code>false</code> if the submission queue is full. <code>true</code> otherwise.</returns> public bool TryPrepareReadV( int fd, iovec *iov, int count, off_t offset = default, int flags = 0, ulong userData = 0, SubmissionOption options = SubmissionOption.None) => TryPrepareReadWrite(IORING_OP_READV, fd, iov, count, offset, flags, userData, options);
public static unsafe PosixResult Receive(SafeHandle socket, iovec *ioVectors, int ioVectorLen) => Receive(socket.DangerousGetHandle().ToInt32(), ioVectors, ioVectorLen);
public static unsafe PosixResult Send(SafeHandle socket, iovec *ioVectors, int ioVectorLen, int flags = 0) => Send(socket.DangerousGetHandle().ToInt32(), ioVectors, ioVectorLen, flags);
public unsafe PosixResult TrySend(iovec *ioVectors, int ioVectorLen) { return(SocketInterop.Send(this, ioVectors, ioVectorLen)); }
public static extern ssize_t vmsplice(int fd, iovec *iov, size_t count, uint flags);
private unsafe void AioSend(List <ScheduledSend> sendQueue) { while (sendQueue.Count > 0) { int sendCount = 0; int completedCount = 0; iocb * aioCbs = AioCbs; iovec *ioVectors = IoVectorTable; ReadOnlySequence <byte>[] sendBuffers = _aioSendBuffers; Span <MemoryHandle> memoryHandles = MemoryHandles; int memoryHandleCount = 0; for (int i = 0; i < sendQueue.Count; i++) { TSocket socket = sendQueue[i].Socket; ReadOnlySequence <byte> buffer; Exception error = socket.GetReadResult(out buffer); if (error != null) { if (error == TransportConstants.StopSentinel) { error = null; } socket.CompleteOutput(error); completedCount++; } else { int ioVectorLength = socket.CalcIOVectorLengthForSend(ref buffer, IoVectorsPerAioSocket); socket.FillSendIOVector(ref buffer, ioVectors, ioVectorLength, memoryHandles); memoryHandles = memoryHandles.Slice(ioVectorLength); memoryHandleCount += ioVectorLength; aioCbs->aio_fildes = socket.Fd; aioCbs->aio_data = (ulong)i; aioCbs->aio_lio_opcode = IOCB_CMD_PWRITEV; aioCbs->aio_buf = (ulong)ioVectors; aioCbs->aio_nbytes = (ulong)ioVectorLength; aioCbs++; sendBuffers[sendCount] = buffer; sendCount++; if (sendCount == EventBufferLength) { break; } ioVectors += ioVectorLength; } } if (sendCount > 0) { PosixResult res = AioInterop.IoSubmit(_aioContext, sendCount, AioCbsTable); memoryHandles = MemoryHandles; for (int i = 0; i < memoryHandleCount; i++) { memoryHandles[i].Dispose(); } if (res != sendCount) { throw new NotSupportedException("Unexpected IoSubmit Send retval " + res); } io_event *aioEvents = AioEvents; res = AioInterop.IoGetEvents(_aioContext, sendCount, aioEvents); if (res != sendCount) { throw new NotSupportedException("Unexpected IoGetEvents Send retval " + res); } io_event *aioEvent = aioEvents; for (int i = 0; i < sendCount; i++) { PosixResult result = new PosixResult((int)aioEvent->res); int socketIndex = (int)aioEvent->data; TSocket socket = sendQueue[socketIndex].Socket; ReadOnlySequence <byte> buffer = sendBuffers[i]; // assumes in-order events sendBuffers[i] = default; socket.HandleSendResult(ref buffer, result, loop: false, zerocopy: false, zeroCopyRegistered: false); aioEvent++; } } sendQueue.RemoveRange(0, sendCount + completedCount); } }
private unsafe void AioReceive(List <TSocket> readableSockets) { ulong PackReceiveState(int received, int advanced, int iovLength) => ((ulong)received << 32) + (ulong)(advanced << 8) + (ulong)(iovLength); (int received, int advanced, int iovLength) UnpackReceiveState(ulong data) => ((int)(data >> 32), (int)((data >> 8) & 0xffffff), (int)(data & 0xff)); int readableSocketCount = readableSockets.Count; iocb * aioCb = AioCbs; iovec * ioVectors = IoVectorTable; PosixResult * receiveResults = stackalloc PosixResult[readableSocketCount]; Span <MemoryHandle> receiveMemoryHandles = MemoryHandles; int receiveMemoryHandleCount = 0; for (int i = 0; i < readableSocketCount; i++) { TSocket socket = readableSockets[i]; var memoryAllocation = socket.DetermineMemoryAllocationForReceive(IoVectorsPerAioSocket); int advanced = socket.FillReceiveIOVector(memoryAllocation, ioVectors, receiveMemoryHandles); aioCb->aio_fildes = socket.Fd; aioCb->aio_data = PackReceiveState(0, advanced, memoryAllocation.IovLength); aioCb->aio_lio_opcode = IOCB_CMD_PREADV; aioCb->aio_buf = (ulong)ioVectors; aioCb->aio_nbytes = (ulong)memoryAllocation.IovLength; aioCb++; ioVectors += memoryAllocation.IovLength; receiveMemoryHandleCount += memoryAllocation.IovLength; receiveMemoryHandles = receiveMemoryHandles.Slice(memoryAllocation.IovLength); } int eAgainCount = 0; while (readableSocketCount > 0) { PosixResult res = AioInterop.IoSubmit(_aioContext, readableSocketCount, AioCbsTable); if (res != readableSocketCount) { throw new NotSupportedException("Unexpected IoSubmit retval " + res); } io_event *aioEvents = AioEvents; res = AioInterop.IoGetEvents(_aioContext, readableSocketCount, aioEvents); if (res != readableSocketCount) { throw new NotSupportedException("Unexpected IoGetEvents retval " + res); } int socketsRemaining = readableSocketCount; bool allEAgain = true; io_event *aioEvent = aioEvents; for (int i = 0; i < readableSocketCount; i++) { PosixResult result = new PosixResult((int)aioEvent->res); int socketIndex = i; // assumes in-order events TSocket socket = readableSockets[socketIndex]; (int received, int advanced, int iovLength) = UnpackReceiveState(aioEvent->data); (bool done, PosixResult retval) = socket.InterpretReceiveResult(result, ref received, advanced, (iovec *)((iocb *)aioEvent->obj)->aio_buf, iovLength); if (done) { receiveResults[socketIndex] = retval; socketsRemaining--; ((iocb *)aioEvent->obj)->aio_lio_opcode = IOCB_CMD_NOOP; allEAgain = false; } else if (retval != PosixResult.EAGAIN) { ((iocb *)aioEvent->obj)->aio_data = PackReceiveState(received, advanced, iovLength); allEAgain = false; } aioEvent++; } if (socketsRemaining > 0) { if (allEAgain) { eAgainCount++; if (eAgainCount == TransportConstants.MaxEAgainCount) { throw new NotSupportedException("Too many EAGAIN, unable to receive available bytes."); } } else { aioCb = AioCbs; iocb *aioCbWriteAt = aioCb; // The kernel doesn't handle Noop, we need to remove them from the aioCbs for (int i = 0; i < readableSocketCount; i++) { if (aioCb[i].aio_lio_opcode != IOCB_CMD_NOOP) { if (aioCbWriteAt != aioCb) { *aioCbWriteAt = *aioCb; } aioCbWriteAt++; } aioCb++; } readableSocketCount = socketsRemaining; eAgainCount = 0; } } else { readableSocketCount = 0; } } receiveMemoryHandles = MemoryHandles; for (int i = 0; i < receiveMemoryHandleCount; i++) { receiveMemoryHandles[i].Dispose(); } for (int i = 0; i < readableSockets.Count; i++) { readableSockets[i].OnReceiveFromSocket(receiveResults[i]); } readableSockets.Clear(); }
private static extern unsafe IntPtr process_vm_readv(int pid, iovec *local_iov, UIntPtr liovcnt, iovec *remote_iov, UIntPtr riovcnt, UIntPtr flags);
private static extern unsafe int process_vm_readv(int pid, iovec *local_iov, ulong liovcnt, iovec *remote_iov, ulong riovcnt, ulong flags);
public unsafe PosixResult TryReceive(iovec *ioVectors, int ioVectorLen) { return(SocketInterop.Receive(this, ioVectors, ioVectorLen)); }
private unsafe bool WriteSubmissions() { Ring ring = _ring !; int iovIndex = _iovsUsed; int sqesAvailable = ring.SubmissionEntriesAvailable; iovec *iovs = IoVectorTable; for (int i = 0; (i < _newOperations.Count) && (sqesAvailable > 2) && (iovIndex < _iovsLength); i++) { _newOperationsQueued++; Operation op = _newOperations[i]; int fd = op.Handle !.DangerousGetHandle().ToInt32(); ulong key = CalculateKey(op.Handle, op.Data); switch (op.OperationType) { case OperationType.Read when op.Status == OperationStatus.PollForReadWrite: { // Poll first, in case the fd is non-blocking. sqesAvailable -= 1; ring.PreparePollAdd(fd, (ushort)POLLIN, key | PollMaskBit); break; } case OperationType.Read when op.Status == OperationStatus.Execution: { MemoryHandle handle = op.Memory.Pin(); op.MemoryHandle = handle; iovec *iov = &iovs[iovIndex++]; // Linux 5.6 doesn't need an iovec (IORING_OP_READ) * iov = new iovec { iov_base = handle.Pointer, iov_len = op.Memory.Length }; sqesAvailable -= 1; ring.PrepareReadV(fd, iov, 1, userData: key); break; } case OperationType.Write when op.Status == OperationStatus.PollForReadWrite: { // Poll first, in case the fd is non-blocking. sqesAvailable -= 1; ring.PreparePollAdd(fd, (ushort)POLLOUT, key | PollMaskBit); break; } case OperationType.Write when op.Status == OperationStatus.Execution: { MemoryHandle handle = op.Memory.Pin(); op.MemoryHandle = handle; iovec *iov = &iovs[iovIndex++]; // Linux 5.6 doesn't need an iovec (IORING_OP_WRITE) * iov = new iovec { iov_base = handle.Pointer, iov_len = op.Memory.Length }; sqesAvailable -= 1; ring.PrepareWriteV(fd, iov, 1, userData: key); break; } case OperationType.PollIn: { sqesAvailable -= 1; ring.PreparePollAdd(fd, (ushort)POLLIN, key); break; } case OperationType.PollOut: { sqesAvailable -= 1; ring.PreparePollAdd(fd, (ushort)POLLOUT, key); break; } case OperationType.Cancel: { sqesAvailable -= 2; // Cancel the operation and possibly associated poll operation. ring.PrepareCancel(opUserData: key | PollMaskBit, userData: IgnoredData); ring.PrepareCancel(opUserData: key, userData: IgnoredData); // Cancel operations aren't added to the dictionary, we can return it now. ReturnOperation(op); break; } } } _iovsUsed = iovIndex; bool operationsRemaining = (_newOperations.Count - _newOperationsQueued) > 0; return(operationsRemaining); }