private static unsafe io_event *Copy(io_event *start, io_event *end, io_event *dst) { uint byteCount = (uint)((byte *)end - (byte *)start); Unsafe.CopyBlock(dst, start, byteCount); return((io_event *)((byte *)dst + byteCount)); }
private static unsafe int IoGetEvents(aio_context_t ctx, int nr, io_event *events) { Debug.Assert(nr != 0); // Check the ring buffer to avoid making a syscall. aio_ring *pRing = ctx.ring; if (pRing->magic == 0xa10a10a1 && pRing->incompat_features == 0) { int head = (int)pRing->head; int tail = (int)pRing->tail; int available = tail - head; if (available < 0) { available += (int)pRing->nr; } if (available >= nr) { io_event *ringEvents = (io_event *)((byte *)pRing + pRing->header_length); io_event *start = ringEvents + head; io_event *end = start + nr; if (head + nr > pRing->nr) { end -= pRing->nr; } if (end > start) { Copy(start, end, events); } else { io_event *eventsEnd = Copy(start, ringEvents + pRing->nr, events); Copy(ringEvents, end, eventsEnd); } head += nr; if (head >= pRing->nr) { head -= (int)pRing->nr; } pRing->head = (uint)head; return(nr); } } return(io_getevents(ctx, nr, nr, events, null)); }
public unsafe static PosixResult IoGetEvents(aio_context_t ctx, int nr, io_event *events) { aio_ring *pRing = ctx.ring; if (nr <= 0) { return(new PosixResult(PosixResult.EINVAL)); } if (pRing->magic == 0xa10a10a1 && pRing->incompat_features == 0) { int head = (int)pRing->head; int tail = (int)pRing->tail; int available = tail - head; if (available < 0) { available += (int)pRing->nr; } if (available >= nr) { io_event *ringEvents = (io_event *)((byte *)pRing + pRing->header_length); io_event *start = ringEvents + head; io_event *end = start + nr; if (head + nr > pRing->nr) { end -= pRing->nr; } if (end > start) { Copy(start, end, events); } else { io_event *eventsEnd = Copy(start, ringEvents + pRing->nr, events); Copy(ringEvents, end, eventsEnd); } head += nr; if (head >= pRing->nr) { head -= (int)pRing->nr; } pRing->head = (uint)head; return(new PosixResult(nr)); } } return(IoGetEvents(ctx, nr, nr, events, -1)); }
public unsafe bool ExecuteOperations() { List <Operation>?scheduled = _scheduledOperations; if (scheduled == null) { return(false); } _scheduledOperations = _cachedOperationsList; _cachedOperationsList = null; if (_results == null || _results.Length < scheduled.Count) { _results = new AsyncOperationResult[Math.Max(scheduled.Count, 2 * (_results?.Length ?? 0))]; } AsyncOperationResult[] results = _results; int queueLength = scheduled.Count; int queueOffset = 0; while (queueOffset < queueLength) { int nr = Math.Min((queueLength - queueOffset), IocbLength); try { iocb *aioCbs = AioCbs; for (int i = queueOffset; i < (queueOffset + nr); i++) { Operation op = scheduled[i]; int fd = op.Handle.DangerousGetHandle().ToInt32(); // TODO: make safe MemoryHandle handle = op.Memory.Pin(); _memoryHandles.Add(handle); aioCbs->aio_fildes = fd; aioCbs->aio_data = (ulong)i; aioCbs->aio_lio_opcode = op.IsReadNotWrite ? IOCB_CMD_PREAD : IOCB_CMD_PWRITE; aioCbs->aio_buf = (ulong)handle.Pointer; aioCbs->aio_nbytes = (ulong)op.Memory.Length; aioCbs++; } iocb **iocbpp = AioCbsTable; int toSubmit = nr; while (toSubmit > 0) { int rv = io_submit(_ctx, toSubmit, iocbpp); if (rv == -1) { PlatformException.Throw(); } int toReceive = rv; toSubmit -= rv; iocbpp += rv; while (toReceive > 0) { do { rv = IoGetEvents(_ctx, toReceive, AioEvents); } while (rv == -1 && errno == EINTR); if (rv == -1) { PlatformException.Throw(); } io_event *events = AioEvents; for (int i = 0; i < rv; i++) { results[(int)events->data] = new AsyncOperationResult(events->res); events++; } toReceive -= rv; } } } finally { foreach (var handle in _memoryHandles) { handle.Dispose(); } _memoryHandles.Clear(); } // Callbacks for (int i = queueOffset; i < (queueOffset + nr); i++) { Operation op = scheduled[i]; op.Callback(results[i], op.State, op.Data); } queueOffset += nr; } scheduled.Clear(); if (_scheduledOperations == null) { _scheduledOperations = scheduled; return(false); } else { _cachedOperationsList = scheduled; return(_scheduledOperations.Count > 0); } }
private unsafe void AioSend(List <ScheduledSend> sendQueue) { while (sendQueue.Count > 0) { int sendCount = 0; int completedCount = 0; iocb * aioCbs = AioCbs; iovec *ioVectors = IoVectorTable; ReadOnlySequence <byte>[] sendBuffers = _aioSendBuffers; Span <MemoryHandle> memoryHandles = MemoryHandles; int memoryHandleCount = 0; for (int i = 0; i < sendQueue.Count; i++) { TSocket socket = sendQueue[i].Socket; ReadOnlySequence <byte> buffer; Exception error = socket.GetReadResult(out buffer); if (error != null) { if (error == TransportConstants.StopSentinel) { error = null; } socket.CompleteOutput(error); completedCount++; } else { int ioVectorLength = socket.CalcIOVectorLengthForSend(ref buffer, IoVectorsPerAioSocket); socket.FillSendIOVector(ref buffer, ioVectors, ioVectorLength, memoryHandles); memoryHandles = memoryHandles.Slice(ioVectorLength); memoryHandleCount += ioVectorLength; aioCbs->aio_fildes = socket.Fd; aioCbs->aio_data = (ulong)i; aioCbs->aio_lio_opcode = IOCB_CMD_PWRITEV; aioCbs->aio_buf = (ulong)ioVectors; aioCbs->aio_nbytes = (ulong)ioVectorLength; aioCbs++; sendBuffers[sendCount] = buffer; sendCount++; if (sendCount == EventBufferLength) { break; } ioVectors += ioVectorLength; } } if (sendCount > 0) { PosixResult res = AioInterop.IoSubmit(_aioContext, sendCount, AioCbsTable); memoryHandles = MemoryHandles; for (int i = 0; i < memoryHandleCount; i++) { memoryHandles[i].Dispose(); } if (res != sendCount) { throw new NotSupportedException("Unexpected IoSubmit Send retval " + res); } io_event *aioEvents = AioEvents; res = AioInterop.IoGetEvents(_aioContext, sendCount, aioEvents); if (res != sendCount) { throw new NotSupportedException("Unexpected IoGetEvents Send retval " + res); } io_event *aioEvent = aioEvents; for (int i = 0; i < sendCount; i++) { PosixResult result = new PosixResult((int)aioEvent->res); int socketIndex = (int)aioEvent->data; TSocket socket = sendQueue[socketIndex].Socket; ReadOnlySequence <byte> buffer = sendBuffers[i]; // assumes in-order events sendBuffers[i] = default; socket.HandleSendResult(ref buffer, result, loop: false, zerocopy: false, zeroCopyRegistered: false); aioEvent++; } } sendQueue.RemoveRange(0, sendCount + completedCount); } }
private unsafe void AioReceive(List <TSocket> readableSockets) { ulong PackReceiveState(int received, int advanced, int iovLength) => ((ulong)received << 32) + (ulong)(advanced << 8) + (ulong)(iovLength); (int received, int advanced, int iovLength) UnpackReceiveState(ulong data) => ((int)(data >> 32), (int)((data >> 8) & 0xffffff), (int)(data & 0xff)); int readableSocketCount = readableSockets.Count; iocb * aioCb = AioCbs; iovec * ioVectors = IoVectorTable; PosixResult * receiveResults = stackalloc PosixResult[readableSocketCount]; Span <MemoryHandle> receiveMemoryHandles = MemoryHandles; int receiveMemoryHandleCount = 0; for (int i = 0; i < readableSocketCount; i++) { TSocket socket = readableSockets[i]; var memoryAllocation = socket.DetermineMemoryAllocationForReceive(IoVectorsPerAioSocket); int advanced = socket.FillReceiveIOVector(memoryAllocation, ioVectors, receiveMemoryHandles); aioCb->aio_fildes = socket.Fd; aioCb->aio_data = PackReceiveState(0, advanced, memoryAllocation.IovLength); aioCb->aio_lio_opcode = IOCB_CMD_PREADV; aioCb->aio_buf = (ulong)ioVectors; aioCb->aio_nbytes = (ulong)memoryAllocation.IovLength; aioCb++; ioVectors += memoryAllocation.IovLength; receiveMemoryHandleCount += memoryAllocation.IovLength; receiveMemoryHandles = receiveMemoryHandles.Slice(memoryAllocation.IovLength); } int eAgainCount = 0; while (readableSocketCount > 0) { PosixResult res = AioInterop.IoSubmit(_aioContext, readableSocketCount, AioCbsTable); if (res != readableSocketCount) { throw new NotSupportedException("Unexpected IoSubmit retval " + res); } io_event *aioEvents = AioEvents; res = AioInterop.IoGetEvents(_aioContext, readableSocketCount, aioEvents); if (res != readableSocketCount) { throw new NotSupportedException("Unexpected IoGetEvents retval " + res); } int socketsRemaining = readableSocketCount; bool allEAgain = true; io_event *aioEvent = aioEvents; for (int i = 0; i < readableSocketCount; i++) { PosixResult result = new PosixResult((int)aioEvent->res); int socketIndex = i; // assumes in-order events TSocket socket = readableSockets[socketIndex]; (int received, int advanced, int iovLength) = UnpackReceiveState(aioEvent->data); (bool done, PosixResult retval) = socket.InterpretReceiveResult(result, ref received, advanced, (iovec *)((iocb *)aioEvent->obj)->aio_buf, iovLength); if (done) { receiveResults[socketIndex] = retval; socketsRemaining--; ((iocb *)aioEvent->obj)->aio_lio_opcode = IOCB_CMD_NOOP; allEAgain = false; } else if (retval != PosixResult.EAGAIN) { ((iocb *)aioEvent->obj)->aio_data = PackReceiveState(received, advanced, iovLength); allEAgain = false; } aioEvent++; } if (socketsRemaining > 0) { if (allEAgain) { eAgainCount++; if (eAgainCount == TransportConstants.MaxEAgainCount) { throw new NotSupportedException("Too many EAGAIN, unable to receive available bytes."); } } else { aioCb = AioCbs; iocb *aioCbWriteAt = aioCb; // The kernel doesn't handle Noop, we need to remove them from the aioCbs for (int i = 0; i < readableSocketCount; i++) { if (aioCb[i].aio_lio_opcode != IOCB_CMD_NOOP) { if (aioCbWriteAt != aioCb) { *aioCbWriteAt = *aioCb; } aioCbWriteAt++; } aioCb++; } readableSocketCount = socketsRemaining; eAgainCount = 0; } } else { readableSocketCount = 0; } } receiveMemoryHandles = MemoryHandles; for (int i = 0; i < receiveMemoryHandleCount; i++) { receiveMemoryHandles[i].Dispose(); } for (int i = 0; i < readableSockets.Count; i++) { readableSockets[i].OnReceiveFromSocket(receiveResults[i]); } readableSockets.Clear(); }
public unsafe static PosixResult IoGetEvents(aio_context_t ctx, int min_nr, int nr, io_event *events, int timeoutMs) { timespec timeout = default(timespec); bool hasTimeout = timeoutMs >= 0; if (hasTimeout) { timeout.tv_sec = timeoutMs / 1000; timeout.tv_nsec = 1000 * (timeoutMs % 1000); } int rv; do { rv = io_getevents(ctx, min_nr, nr, events, hasTimeout ? &timeout : null); } while (rv < 0 && errno == EINTR); return(PosixResult.FromReturnValue(rv)); }
public static int io_getevents(aio_context_t ctx, long_t min_nr, long_t nr, io_event *events, timespec *timeout) { int rv = (int)syscall(SYS_io_getevents, ctx.ring, min_nr, nr, events, timeout); return(rv); }
public unsafe static io_event *io_events(aio_ring *ring, int idx) { io_event *ev = (io_event *)(ring + 1); return(ev + idx); }