private unsafe bool ReadMemoryReadv(ulong address, Span <byte> buffer, out int bytesRead)
{
int readableBytesCount = GetReadableBytesCount(address, buffer.Length);
if (readableBytesCount <= 0)
{
bytesRead = 0;
return(false);
}
fixed(byte *ptr = buffer)
{
var local = new iovec
{
iov_base = ptr,
iov_len = (IntPtr)readableBytesCount
};
var remote = new iovec
{
iov_base = (void *)address,
iov_len = (IntPtr)readableBytesCount
};
int read = (int)process_vm_readv((int)ProcessId, &local, (UIntPtr)1, &remote, (UIntPtr)1, UIntPtr.Zero).ToInt64();
if (read < 0)
{
bytesRead = 0;
return((Marshal.GetLastWin32Error()) switch
{
EPERM => throw new UnauthorizedAccessException(),
ESRCH => throw new InvalidOperationException("The process has exited"),
_ => false
});
public IoUringConnectionContext(LinuxSocket socket, EndPoint server, EndPoint client, MemoryPool <byte> pool,
TransportThreadContext threadContext)
{
Socket = socket;
LocalEndPoint = server;
RemoteEndPoint = client;
MemoryPool = pool;
_threadContext = threadContext;
var appScheduler = PipeScheduler.ThreadPool; // TODO: configure
var inputOptions = new PipeOptions(MemoryPool, appScheduler, PipeScheduler.Inline, PauseInputWriterThreshold, PauseInputWriterThreshold / 2, useSynchronizationContext: false);
var outputOptions = new PipeOptions(MemoryPool, PipeScheduler.Inline, appScheduler, PauseOutputWriterThreshold, PauseOutputWriterThreshold / 2, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(inputOptions, outputOptions);
Transport = pair.Transport;
Application = pair.Application;
_onOnFlushedToApp = FlushedToAppAsynchronously;
_onReadFromApp = ReadFromAppAsynchronously;
iovec[] vecs = new iovec[ReadIOVecCount + WriteIOVecCount];
var handle = GCHandle.Alloc(vecs, GCHandleType.Pinned);
_iovec = (iovec *)handle.AddrOfPinnedObject();
_iovecHandle = handle;
}
protected IoUringConnectionContext(LinuxSocket socket, EndPoint local, EndPoint remote, TransportThreadContext threadContext)
{
Socket = socket;
LocalEndPoint = local;
RemoteEndPoint = remote;
MemoryPool = threadContext.MemoryPool;
_threadContext = threadContext;
var appScheduler = threadContext.Options.ApplicationSchedulingMode;
var inputOptions = new PipeOptions(MemoryPool, appScheduler, PipeScheduler.Inline, PauseInputWriterThreshold, PauseInputWriterThreshold / 2, useSynchronizationContext: false);
var outputOptions = new PipeOptions(MemoryPool, PipeScheduler.Inline, appScheduler, PauseOutputWriterThreshold, PauseOutputWriterThreshold / 2, useSynchronizationContext: false);
var pair = DuplexPipe.CreateConnectionPair(inputOptions, outputOptions);
Transport = pair.Transport;
Application = pair.Application;
_onOnFlushedToApp = FlushedToAppAsynchronously;
_onReadFromApp = ReadFromAppAsynchronously;
iovec[] vecs = new iovec[ReadIOVecCount + WriteIOVecCount];
var handle = GCHandle.Alloc(vecs, GCHandleType.Pinned);
_iovec = (iovec *)handle.AddrOfPinnedObject();
_iovecHandle = handle;
}
public TransportThread(IoUringOptions options)
{
int res = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
if (res == -1)
{
throw new ErrnoException(errno);
}
_eventfd = res;
// Pin buffer for eventfd reads via io_uring
byte[] bytes = new byte[8];
_eventfdBytes = GCHandle.Alloc(bytes, GCHandleType.Pinned);
// Pin iovec used for eventfd reads via io_uring
var eventfdIoVec = new iovec
{
iov_base = (void *)_eventfdBytes.AddrOfPinnedObject(),
iov_len = bytes.Length
};
_eventfdIoVecHandle = GCHandle.Alloc(eventfdIoVec, GCHandleType.Pinned);
_eventfdIoVec = (iovec *)_eventfdIoVecHandle.AddrOfPinnedObject();
var memoryPool = new SlabMemoryPool();
_threadContext = new TransportThreadContext(options, memoryPool, _eventfd, s => Read(s), s => Write(s));
_maxBufferSize = memoryPool.MaxBufferSize;
}
public unsafe void PackAck()
{
int typeACK = 2;
int msgNo = 100;
int *pMsgNo = &msgNo;
int controlPacket = 1;
byte[] data = { 1, 2, 3, 4 };
fixed(byte *pData = data)
{
Packet packet = new Packet();
packet.pack(typeACK, (void *)pMsgNo, (void *)pData, data.Length);
Assert.AreEqual(controlPacket, packet.getFlag());
Assert.AreEqual(typeACK, packet.getType());
Assert.AreEqual(msgNo, packet.getAckSeqNo());
iovec[] packetVector = packet.getPacketVector();
Assert.AreEqual(2, packetVector.Length);
iovec packetData = packetVector[1];
Assert.AreEqual(data.Length, packetData.iov_len);
uint iov = packetData.iov_base[0];
uint expectedResult = 1 + (2 << 8) + (3 << 16) + (4 << 24);
Assert.AreEqual(expectedResult, iov);
}
}
public unsafe void PackError()
{
int typeError = 8;
int msgNo = 100;
int *pMsgNo = &msgNo;
int controlPacket = 1;
byte[] data = { 1, 2, 3, 4 };
fixed(byte *pData = data)
{
Packet packet = new Packet();
packet.pack(typeError, (void *)pMsgNo, (void *)pData, data.Length);
Assert.AreEqual(controlPacket, packet.getFlag());
Assert.AreEqual(typeError, packet.getType());
Assert.AreEqual(msgNo, packet.getMsgSeq());
iovec[] packetVector = packet.getPacketVector();
Assert.AreEqual(2, packetVector.Length);
iovec packetData = packetVector[1];
Assert.IsNull(packetData.iov_base);
}
}
public msghdr(void* name, uint namelen, iovec* iov, int iovlen, void* control, int controllen, int flags)
{
msg_name = name;
msg_namelen = (socklen_t)namelen;
msg_iov = iov;
msg_iovlen = (IntPtr)iovlen;
msg_control = control;
msg_controllen = (IntPtr)controllen;
msg_flags = flags;
}
public unsafe static PosixResult ReceiveSocket(int fromSocket, out int socket, bool blocking)
{
socket = -1;
byte dummyBuffer = 0;
iovec iov = default(iovec);
iov.iov_base = &dummyBuffer;
iov.iov_len = 1;
int controlLength = CMSG_SPACE(sizeof(int));
byte *control = stackalloc byte[controlLength];
msghdr header = default(msghdr);
header.msg_iov = &iov;
header.msg_iovlen = 1;
header.msg_control = control;
header.msg_controllen = controlLength;
int flags = MSG_NOSIGNAL | MSG_CMSG_CLOEXEC;
ssize_t rv;
do
{
rv = recvmsg(fromSocket, &header, flags);
} while (rv < 0 && errno == EINTR);
if (rv != -1)
{
for (cmsghdr *cmsg = CMSG_FIRSTHDR(&header); cmsg != null; cmsg = CMSG_NXTHDR(&header, cmsg))
{
if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS)
{
int *fdptr = (int *)CMSG_DATA(cmsg);
socket = *fdptr;
flags = fcntl(socket, F_GETFL, 0);
if (blocking)
{
flags &= ~O_NONBLOCK;
}
else
{
flags |= O_NONBLOCK;
}
fcntl(socket, F_SETFL, flags);
break;
}
}
}
return(PosixResult.FromReturnValue(rv));
}
public unsafe PosixResult TrySend(ArraySegment <byte> buffer)
{
ValidateSegment(buffer);
fixed(byte *buf = buffer.Array)
{
iovec ioVector = new iovec()
{
iov_base = buf + buffer.Offset, iov_len = buffer.Count
};
return(SocketInterop.Send(this, &ioVector, 1));
}
}
public unsafe static PosixResult AcceptAndSendHandleTo(Socket fromSocket, int toSocket)
{
int acceptFd = fromSocket.DangerousGetHandle().ToInt32();
ssize_t rv;
do
{
rv = accept4(acceptFd, null, null, SOCK_CLOEXEC);
} while (rv < 0 && errno == EINTR);
if (rv != -1)
{
int acceptedFd = (int)rv;
byte dummyBuffer = 0;
iovec iov = default(iovec);
iov.iov_base = &dummyBuffer;
iov.iov_len = 1;
int controlLength = CMSG_SPACE(sizeof(int));
byte *control = stackalloc byte[controlLength];
msghdr header = default(msghdr);
header.msg_iov = &iov;
header.msg_iovlen = 1;
header.msg_control = control;
header.msg_controllen = controlLength;
cmsghdr *cmsg = CMSG_FIRSTHDR(&header);
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
cmsg->cmsg_len = CMSG_LEN(sizeof(int));
int *fdptr = (int *)CMSG_DATA(cmsg);
* fdptr = acceptedFd;
do
{
rv = sendmsg(toSocket, &header, MSG_NOSIGNAL);
} while (rv < 0 && errno == EINTR);
IOInterop.Close(acceptedFd);
}
return(PosixResult.FromReturnValue(rv));
}
public unsafe IList <ArraySegment <byte> > ReceiveMultipleMessages(IList <ArraySegment <byte> > buffers)
{
if (handle == -1)
{
throw new ObjectDisposedException(objectname);
}
GCHandle[] bufhandles = new GCHandle[buffers.Count];
iovec * iovecs = stackalloc iovec[buffers.Count];
mmsghdr * hdrs = stackalloc mmsghdr[buffers.Count];
for (int i = 0; i < buffers.Count; i++)
{
bufhandles[i] = GCHandle.Alloc(buffers[i].Array, GCHandleType.Pinned);
iovecs[i] = new iovec()
{
iov_base = (byte *)bufhandles[i].AddrOfPinnedObject() + buffers[i].Offset, iov_len = (UIntPtr)(UInt32)buffers[i].Count
};
hdrs[i] = new mmsghdr()
{
msg_hdr = new msghdr()
{
msg_iov = &iovecs[i], msg_iovlen = (UIntPtr)1
}
};
}
int ret = recvmmsg(handle, hdrs, (uint)buffers.Count, MSG_WAITFORONE, null);
for (int i = 0; i < buffers.Count; i++)
{
bufhandles[i].Free();
}
if (ret == -1)
{
throw new PosixException("recvmmsg");
}
ArraySegment <byte>[] retvec = new ArraySegment <byte> [ret];
for (int i = 0; i < ret && i < buffers.Count; i++)
{
retvec[i] = new ArraySegment <byte>(buffers[i].Array, buffers[i].Offset, (int)hdrs[i].msg_len);
}
return(retvec);
}
public unsafe static bool Read <T>(IntPtr address, out T value, Process _process) where T : unmanaged
{
int size = Unsafe.SizeOf <T>();
byte *ptr = stackalloc byte[size];
iovec localIo = new iovec
{
iov_base = ptr,
iov_len = size
};
iovec remoteIo = new iovec
{
iov_base = address.ToPointer(),
iov_len = size
};
var res = process_vm_readv(_process.Id, &localIo, 1, &remoteIo, 1, 0);
value = *(T *)ptr;
return(res != -1);
}
public unsafe static bool ReadArray(IntPtr address, int count, out byte[] value, Process _process)
{
int size = Unsafe.SizeOf <byte>() * count;
byte *ptr = stackalloc byte[size];
iovec localIo = new iovec
{
iov_base = ptr,
iov_len = size
};
iovec remoteIo = new iovec
{
iov_base = address.ToPointer(),
iov_len = size
};
var res = process_vm_readv(_process.Id, &localIo, 1, &remoteIo, 1, 0);
value = new byte[count];
Marshal.Copy((IntPtr)ptr, value, 0, count);
return(res != -1);
}
private unsafe void SetupEventFd()
{
int res = eventfd(0, EFD_SEMAPHORE);
if (res == -1)
{
throw new ErrnoException(errno);
}
_eventfd = res;
// Pin buffer for eventfd reads via io_uring
byte[] bytes = new byte[8];
_eventfdBytes = GCHandle.Alloc(bytes, GCHandleType.Pinned);
// Pin iovec used for eventfd reads via io_uring
var eventfdIoVec = new iovec
{
iov_base = (void *)_eventfdBytes.AddrOfPinnedObject(),
iov_len = bytes.Length
};
_eventfdIoVec = GCHandle.Alloc(eventfdIoVec, GCHandleType.Pinned);
}
private unsafe bool ReadMemoryReadv(ulong address, IntPtr buffer, int bytesRequested, out int bytesRead)
{
bytesRead = 0;
int readableBytesCount = this.GetReadableBytesCount(address, bytesRequested);
if (readableBytesCount <= 0)
{
return(false);
}
var local = new iovec
{
iov_base = (void *)buffer,
iov_len = (IntPtr)readableBytesCount
};
var remote = new iovec
{
iov_base = (void *)address,
iov_len = (IntPtr)readableBytesCount
};
bytesRead = (int)process_vm_readv((int)ProcessId, &local, (UIntPtr)1, &remote, (UIntPtr)1, UIntPtr.Zero).ToInt64();
return(bytesRead > 0);
}
private unsafe bool WriteSubmissions()
{
Ring ring = _ring !;
int iovIndex = _iovsUsed;
int sqesAvailable = ring.SubmissionEntriesAvailable;
iovec *iovs = IoVectorTable;
for (int i = 0; (i < _newOperations.Count) && (sqesAvailable > 2) && (iovIndex < _iovsLength); i++)
{
_newOperationsQueued++;
Operation op = _newOperations[i];
int fd = op.Handle !.DangerousGetHandle().ToInt32();
ulong key = CalculateKey(op.Handle, op.Data);
switch (op.OperationType)
{
case OperationType.Read when op.Status == OperationStatus.PollForReadWrite:
{
// Poll first, in case the fd is non-blocking.
sqesAvailable -= 1;
ring.PreparePollAdd(fd, (ushort)POLLIN, key | PollMaskBit);
break;
}
case OperationType.Read when op.Status == OperationStatus.Execution:
{
MemoryHandle handle = op.Memory.Pin();
op.MemoryHandle = handle;
iovec *iov = &iovs[iovIndex++]; // Linux 5.6 doesn't need an iovec (IORING_OP_READ)
* iov = new iovec {
iov_base = handle.Pointer, iov_len = op.Memory.Length
};
sqesAvailable -= 1;
ring.PrepareReadV(fd, iov, 1, userData: key);
break;
}
case OperationType.Write when op.Status == OperationStatus.PollForReadWrite:
{
// Poll first, in case the fd is non-blocking.
sqesAvailable -= 1;
ring.PreparePollAdd(fd, (ushort)POLLOUT, key | PollMaskBit);
break;
}
case OperationType.Write when op.Status == OperationStatus.Execution:
{
MemoryHandle handle = op.Memory.Pin();
op.MemoryHandle = handle;
iovec *iov = &iovs[iovIndex++]; // Linux 5.6 doesn't need an iovec (IORING_OP_WRITE)
* iov = new iovec {
iov_base = handle.Pointer, iov_len = op.Memory.Length
};
sqesAvailable -= 1;
ring.PrepareWriteV(fd, iov, 1, userData: key);
break;
}
case OperationType.PollIn:
{
sqesAvailable -= 1;
ring.PreparePollAdd(fd, (ushort)POLLIN, key);
break;
}
case OperationType.PollOut:
{
sqesAvailable -= 1;
ring.PreparePollAdd(fd, (ushort)POLLOUT, key);
break;
}
case OperationType.Cancel:
{
sqesAvailable -= 2;
// Cancel the operation and possibly associated poll operation.
ring.PrepareCancel(opUserData: key | PollMaskBit, userData: IgnoredData);
ring.PrepareCancel(opUserData: key, userData: IgnoredData);
// Cancel operations aren't added to the dictionary, we can return it now.
ReturnOperation(op);
break;
}
}
}
_iovsUsed = iovIndex;
bool operationsRemaining = (_newOperations.Count - _newOperationsQueued) > 0;
return(operationsRemaining);
}
