// private unsafe iovec* IoVectorTable => (iovec*)Align(_ioVectorTableMemory); // TODO
public unsafe LinuxAio() :
base(supportsPolling: false, isThreadSafe: false)
{
_memoryHandles = new List <MemoryHandle>();
try
{
// Memory
_aioEventsMemory = AllocMemory(sizeof(io_event) * IocbLength);
_aioCbsMemory = AllocMemory(sizeof(iocb) * IocbLength);
_aioCbsTableMemory = AllocMemory(IntPtr.Size * IocbLength);
// _ioVectorTableMemory = AllocMemory(SizeOf.iovec * IovsPerIocb * _ioCbLength);
_scheduledOperations = new List <Operation>();
for (int i = 0; i < IocbLength; i++)
{
AioCbsTable[i] = &AioCbs[i];
}
// Aio
aio_context_t ctx;
int rv = io_setup(NrEvents, &ctx);
if (rv == -1)
{
PlatformException.Throw();
}
_ctx = ctx;
}
catch
{
FreeResources();
}
}
void IAsyncExecutionResultHandler.HandleAsyncResult(AsyncOperationResult result)
{
if (result.IsError && result.Errno != EAGAIN)
{
PlatformException.Throw();
}
}
private unsafe void CreateResources(bool useLinuxAio)
{
try
{
if (useLinuxAio)
{
_asyncExecutionQueue = new LinuxAio();
}
_epollFd = epoll_create1(EPOLL_CLOEXEC);
if (_epollFd == -1)
{
PlatformException.Throw();
}
_pipeReadEnd = new CloseSafeHandle();
_pipeWriteEnd = new CloseSafeHandle();
int *pipeFds = stackalloc int[2];
int rv = pipe2(pipeFds, O_CLOEXEC | O_NONBLOCK);
if (rv == -1)
{
PlatformException.Throw();
}
_pipeReadEnd.SetHandle(pipeFds[0]);
_pipeWriteEnd.SetHandle(pipeFds[1]);
Control(EPOLL_CTL_ADD, pipeFds[0], EPOLLIN, PipeKey);
}
catch
{
FreeResources();
throw;
}
}
private unsafe void ReadFromPipe(AsyncExecutionQueue?executionEngine)
{
if (executionEngine == null)
{
Span <byte> buffer = stackalloc byte[128];
int rv = IoPal.Read(_pipeReadEnd !, buffer);
if (rv == -1)
{
if (errno != EAGAIN)
{
PlatformException.Throw();
}
}
}
else
{
if (_dummyReadBuffer == null)
{
_dummyReadBuffer = new byte[128];
}
executionEngine.AddRead(_pipeReadEnd !, _dummyReadBuffer,
(AsyncOperationResult result, object?state, int data) =>
{
if (result.IsError && result.Errno != EAGAIN)
{
PlatformException.Throw();
}
}, state: null, data: 0);
}
}
public unsafe void Control(int op, int fd, int events, int key)
{
epoll_event ev = default;
ev.events = events;
ev.data.fd = key;
int rv = epoll_ctl(_epollFd, op, fd, &ev);
if (rv != 0)
{
PlatformException.Throw();
}
}
private unsafe void WriteToPipe()
{
Span <byte> buffer = stackalloc byte[1];
int rv = IoPal.Write(_pipeWriteEnd !, buffer);
if (rv == -1)
{
if (errno != EAGAIN)
{
PlatformException.Throw();
}
}
}
private unsafe void WriteToEventFd()
{
Span <byte> buffer = stackalloc byte[8];
buffer[7] = 1;
int rv = IoPal.Write(_eventFd !, buffer);
if (rv == -1)
{
if (errno != EAGAIN)
{
PlatformException.Throw();
}
}
}
void IAsyncExecutionResultHandler.HandleAsyncResult(AsyncOperationResult asyncResult)
{
// TODO: do we need to do a volatile read of _disposed?
if (asyncResult.Errno == EAGAIN || (!asyncResult.IsError && asyncResult.Value == 8))
{
AddReadFromEventFd();
}
else if (asyncResult.IsError)
{
PlatformException.Throw(asyncResult.Errno);
}
else
{
ThrowHelper.ThrowIndexOutOfRange(asyncResult.Value);
}
}
public static void SetNonBlocking(SafeHandle handle)
{
int fd = handle.DangerousGetHandle().ToInt32(); // TODO: make safe
int flags = fcntl(fd, F_GETFL, 0);
if (flags == -1)
{
PlatformException.Throw();
}
flags |= O_NONBLOCK;
int rv = fcntl(fd, F_SETFL, flags);
if (rv == -1)
{
PlatformException.Throw();
}
}
private void AddReadFromEventFd()
{
_iouring !.AddRead(_eventFd !, _dummyReadBuffer,
(AsyncOperationResult asyncResult, object?state, int data) =>
{
// TODO: do we need to do a volatile read of _disposed?
if (asyncResult.Errno == EAGAIN || (!asyncResult.IsError && asyncResult.Value == 8))
{
((IOUringThread)state !).AddReadFromEventFd();
}
else if (asyncResult.IsError)
{
PlatformException.Throw(asyncResult.Errno);
}
else
{
ThrowHelper.ThrowIndexOutOfRange(asyncResult.Value);
}
}
, this, 0);
}
private unsafe void ReadFromPipe(AsyncExecutionQueue?executionEngine)
{
if (executionEngine == null)
{
Span <byte> buffer = stackalloc byte[128];
int rv = IoPal.Read(_pipeReadEnd !, buffer);
if (rv == -1)
{
if (errno != EAGAIN)
{
PlatformException.Throw();
}
}
}
else
{
if (_dummyReadBuffer == null)
{
_dummyReadBuffer = new byte[128];
}
executionEngine.AddRead(_pipeReadEnd !, _dummyReadBuffer, this, data: 0);
}
}
private unsafe void CreateResources()
{
try
{
_iouring = new IOUringExecutionQueue();
_eventFd = new CloseSafeHandle();
int rv = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
if (rv == -1)
{
PlatformException.Throw();
}
_eventFd.SetHandle(rv);
AddReadFromEventFd();
}
catch
{
FreeResources();
throw;
}
}
public unsafe bool ExecuteOperations()
{
List <Operation>?scheduled = _scheduledOperations;
if (scheduled == null)
{
return(false);
}
_scheduledOperations = _cachedOperationsList;
_cachedOperationsList = null;
if (_results == null || _results.Length < scheduled.Count)
{
_results = new AsyncOperationResult[Math.Max(scheduled.Count, 2 * (_results?.Length ?? 0))];
}
AsyncOperationResult[] results = _results;
int queueLength = scheduled.Count;
int queueOffset = 0;
while (queueOffset < queueLength)
{
int nr = Math.Min((queueLength - queueOffset), IocbLength);
try
{
iocb *aioCbs = AioCbs;
for (int i = queueOffset; i < (queueOffset + nr); i++)
{
Operation op = scheduled[i];
int fd = op.Handle.DangerousGetHandle().ToInt32(); // TODO: make safe
MemoryHandle handle = op.Memory.Pin();
_memoryHandles.Add(handle);
aioCbs->aio_fildes = fd;
aioCbs->aio_data = (ulong)i;
aioCbs->aio_lio_opcode = op.IsReadNotWrite ? IOCB_CMD_PREAD : IOCB_CMD_PWRITE;
aioCbs->aio_buf = (ulong)handle.Pointer;
aioCbs->aio_nbytes = (ulong)op.Memory.Length;
aioCbs++;
}
iocb **iocbpp = AioCbsTable;
int toSubmit = nr;
while (toSubmit > 0)
{
int rv = io_submit(_ctx, toSubmit, iocbpp);
if (rv == -1)
{
PlatformException.Throw();
}
int toReceive = rv;
toSubmit -= rv;
iocbpp += rv;
while (toReceive > 0)
{
do
{
rv = IoGetEvents(_ctx, toReceive, AioEvents);
} while (rv == -1 && errno == EINTR);
if (rv == -1)
{
PlatformException.Throw();
}
io_event *events = AioEvents;
for (int i = 0; i < rv; i++)
{
results[(int)events->data] = new AsyncOperationResult(events->res);
events++;
}
toReceive -= rv;
}
}
}
finally
{
foreach (var handle in _memoryHandles)
{
handle.Dispose();
}
_memoryHandles.Clear();
}
// Callbacks
for (int i = queueOffset; i < (queueOffset + nr); i++)
{
Operation op = scheduled[i];
op.Callback(results[i], op.State, op.Data);
}
queueOffset += nr;
}
scheduled.Clear();
if (_scheduledOperations == null)
{
_scheduledOperations = scheduled;
return(false);
}
else
{
_cachedOperationsList = scheduled;
return(_scheduledOperations.Count > 0);
}
}
private unsafe void EventLoop()
{
try
{
var eventBuffer = stackalloc epoll_event[EventBufferLength];
List <EPollAsyncContext?> asyncContextsForEvents = new List <EPollAsyncContext?>();
bool running = true;
int epollTimeout = -1;
while (running)
{
int rv = epoll_wait(_epollFd, eventBuffer, EventBufferLength, epollTimeout);
Volatile.Write(ref _blockedState, StateNotBlocked);
if (rv == -1)
{
if (LibC.errno == EINTR)
{
continue;
}
PlatformException.Throw();
}
lock (_asyncContexts)
{
for (int i = 0; i < rv; i++)
{
int key = eventBuffer[i].data.fd;
if (_asyncContexts.TryGetValue(key, out EPollAsyncContext? eventContext))
{
asyncContextsForEvents.Add(eventContext);
}
else
{
if (key == PipeKey)
{
running = !_disposed;
ReadFromPipe(_asyncExecutionQueue);
}
asyncContextsForEvents.Add(null);
}
}
}
for (int i = 0; i < rv; i++)
{
EPollAsyncContext?context = asyncContextsForEvents[i];
if (context != null)
{
context.HandleEvents(eventBuffer[i].events);
}
}
asyncContextsForEvents.Clear();
bool actionsRemaining = false;
// Run this twice, executions cause more executions.
for (int i = 0; i < 2; i++)
{
// First execute scheduled actions, they can add to the exection queue.
ExecuteScheduledActions();
if (_asyncExecutionQueue != null)
{
actionsRemaining = _asyncExecutionQueue.ExecuteOperations();
}
}
if (!actionsRemaining)
{
// Check if there are scheduled actions remaining.
lock (_actionQueueGate)
{
actionsRemaining = _scheduledActions.Count > 0;
if (!actionsRemaining)
{
Volatile.Write(ref _blockedState, StateBlocked);
}
}
}
epollTimeout = actionsRemaining ? 0 : -1;
}
// Execute actions that got posted before we were disposed.
ExecuteScheduledActions();
// Complete pending async operations.
_asyncExecutionQueue?.Dispose();
EPollAsyncContext[] contexts;
lock (_asyncContexts)
{
contexts = new EPollAsyncContext[_asyncContexts.Count];
_asyncContexts.Values.CopyTo(contexts, 0);
_asyncContexts.Clear();
}
foreach (var context in contexts)
{
context.Dispose();
}
FreeResources();
}
catch (Exception e)
{
Environment.FailFast(e.ToString());
}
}