public static unsafe PosixResult GetAvailableBytes(int socket)
{
int availableBytes;
int rv = ioctl(socket, FIONREAD, &availableBytes);
return(PosixResult.FromReturnValue(rv == -1 ? rv : availableBytes));
}
public static unsafe PosixResult GetSockName(int socket, sockaddr_storage *addr)
{
socklen_t sockLen = SizeOf.sockaddr_storage;
int rv = getsockname(socket, (sockaddr *)addr, &sockLen);
return(PosixResult.FromReturnValue(rv));
}
public static unsafe PosixResult SocketPair(int domain, int type, int protocol, bool blocking, out int socket1, out int socket2)
{
int *sv = stackalloc int[2];
type |= SOCK_CLOEXEC;
if (!blocking)
{
type |= SOCK_NONBLOCK;
}
int rv = socketpair(domain, type, protocol, sv);
if (rv == 0)
{
socket1 = sv[0];
socket2 = sv[1];
}
else
{
socket1 = -1;
socket2 = -1;
}
return(PosixResult.FromReturnValue(rv));
}
public unsafe static PosixResult Pipe(out PipeEnd readEnd, out PipeEnd writeEnd, bool blocking)
{
int *fds = stackalloc int[2];
int flags = O_CLOEXEC;
if (!blocking)
{
flags |= O_NONBLOCK;
}
readEnd = new PipeEnd();
writeEnd = new PipeEnd();
int res = pipe2(fds, flags);
if (res == 0)
{
readEnd.SetHandle(fds[0]);
writeEnd.SetHandle(fds[1]);
}
else
{
readEnd = null;
writeEnd = null;
}
return(PosixResult.FromReturnValue(res));
}
public IPEndPointStruct ToIPEndPoint(IPAddress reuseAddress = null)
{
if (Family == AddressFamily.InterNetwork)
{
long value;
fixed(byte *address = Address)
{
value = ((address[3] << 24 | address[2] << 16 | address[1] << 8 | address[0]) & 0x0FFFFFFFF);
}
bool matchesReuseAddress = reuseAddress != null && reuseAddress.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork && reuseAddress.Address == value;
return(new IPEndPointStruct(matchesReuseAddress ? reuseAddress : new IPAddress(value), Port));
}
else if (Family == AddressFamily.InterNetworkV6)
{
// We can't check if we can use reuseAddress without allocating.
const int length = 16;
var bytes = new byte[length];
fixed(byte *address = Address)
{
for (int i = 0; i < length; i++)
{
bytes[i] = address[i];
}
}
return(new IPEndPointStruct(new IPAddress(bytes, ScopeId), Port));
}
else
{
var result = new PosixResult(PosixResult.EAFNOSUPPORT);
throw result.AsException();
}
}
public static unsafe PosixResult Read(SafeHandle handle, byte *buf, int count)
{
bool addedRef = false;
try
{
handle.DangerousAddRef(ref addedRef);
int fd = handle.DangerousGetHandle().ToInt32();
ssize_t rv;
do
{
rv = read(fd, buf, count);
} while (rv < 0 && errno == EINTR);
return(PosixResult.FromReturnValue(rv));
}
finally
{
if (addedRef)
{
handle.DangerousRelease();
}
}
}
public static PosixResult Socket(AddressFamily addressFamily, SocketType socketType, ProtocolType protocolType, bool blocking, out Socket socket)
{
int socketFd;
PosixResult result = Socket(addressFamily, socketType, protocolType, blocking, out socketFd);
socket = result.IsSuccess ? new Socket(socketFd) : null;
return(result);
}
public static PosixResult ReceiveSocket(Socket fromSocket, out Socket socket, bool blocking)
{
int receiveSocketFd;
PosixResult result = ReceiveSocket(fromSocket.DangerousGetHandle().ToInt32(), out receiveSocketFd, blocking);
socket = result.IsSuccess ? new Socket(receiveSocketFd) : null;
return(result);
}
public static unsafe PosixResult Accept(Socket socket, bool blocking, out Socket clientSocket)
{
int clientSocketFd;
PosixResult result = Accept(socket.DangerousGetHandle().ToInt32(), blocking, out clientSocketFd);
clientSocket = result.IsSuccess ? new Socket(clientSocketFd) : null;
return(result);
}
public static PosixResult Socket(int domain, int type, int protocol, bool blocking, out Socket socket)
{
int socketFd;
PosixResult result = Socket(domain, type, protocol, blocking, out socketFd);
socket = result.IsSuccess ? new Socket(socketFd) : null;
return(result);
}
public unsafe PosixResult TryBind(string unixPath)
{
sockaddr_un addr;
GetSockaddrUn(unixPath, out addr);
int rv = bind(DangerousGetHandle().ToInt32(), (sockaddr *)&addr, SizeOf.sockaddr_un);
return(PosixResult.FromReturnValue(rv));
}
public unsafe PosixResult TryBind(IPEndPointStruct endpoint)
{
sockaddr_storage addr;
GetSockaddrInet(endpoint, &addr, out int length);
int rv = bind(DangerousGetHandle().ToInt32(), (sockaddr *)&addr, length);
return(PosixResult.FromReturnValue(rv));
}
public unsafe static PosixResult SetCurrentThreadAffinity(int cpuId)
{
cpu_set_t cpu_set;
CPU_ZERO(&cpu_set);
CPU_SET(cpuId, &cpu_set);
int rv = sched_setaffinity(0, SizeOf.cpu_set_t, &cpu_set);
return(PosixResult.FromReturnValue(rv));
}
public unsafe static PosixResult ReceiveSocket(int fromSocket, out int socket, bool blocking)
{
socket = -1;
byte dummyBuffer = 0;
iovec iov = default(iovec);
iov.iov_base = &dummyBuffer;
iov.iov_len = 1;
int controlLength = CMSG_SPACE(sizeof(int));
byte *control = stackalloc byte[controlLength];
msghdr header = default(msghdr);
header.msg_iov = &iov;
header.msg_iovlen = 1;
header.msg_control = control;
header.msg_controllen = controlLength;
int flags = MSG_NOSIGNAL | MSG_CMSG_CLOEXEC;
ssize_t rv;
do
{
rv = recvmsg(fromSocket, &header, flags);
} while (rv < 0 && errno == EINTR);
if (rv != -1)
{
for (cmsghdr *cmsg = CMSG_FIRSTHDR(&header); cmsg != null; cmsg = CMSG_NXTHDR(&header, cmsg))
{
if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS)
{
int *fdptr = (int *)CMSG_DATA(cmsg);
socket = *fdptr;
flags = fcntl(socket, F_GETFL, 0);
if (blocking)
{
flags &= ~O_NONBLOCK;
}
else
{
flags |= O_NONBLOCK;
}
fcntl(socket, F_SETFL, flags);
break;
}
}
}
return(PosixResult.FromReturnValue(rv));
}
public static unsafe PosixResult EPollWait(int epoll, epoll_event *events, int maxEvents, int timeout)
{
int rv;
do
{
rv = epoll_wait(epoll, events, maxEvents, timeout);
} while (rv < 0 && errno == EINTR);
return(PosixResult.FromReturnValue(rv));
}
public static unsafe PosixResult EPollControl(int epoll, int operation, int fd, int events, int data)
{
epoll_event ev = default(epoll_event);
ev.events = events;
ev.data.fd = data;
int rv = epoll_ctl(epoll, operation, fd, &ev);
return(PosixResult.FromReturnValue(rv));
}
public static unsafe PosixResult Socket(int domain, int type, int protocol, bool blocking, out int s)
{
type |= SOCK_CLOEXEC;
if (!blocking)
{
type |= SOCK_NONBLOCK;
}
s = socket(domain, type, protocol);
return(PosixResult.FromReturnValue(s));
}
public unsafe static PosixResult GetAvailableCpusForProcess()
{
cpu_set_t set;
int rv = sched_getaffinity(getpid(), SizeOf.cpu_set_t, &set);
if (rv == 0)
{
rv = CPU_COUNT(&set);
}
return(PosixResult.FromReturnValue(rv));
}
public unsafe PosixResult TryConnect(string unixPath)
{
sockaddr_un addr;
GetSockaddrUn(unixPath, out addr);
int rv;
do
{
rv = connect(DangerousGetHandle().ToInt32(), (sockaddr *)&addr, SizeOf.sockaddr_un);
} while (rv < 0 && errno == EINTR);
return(PosixResult.FromReturnValue(rv));
}
public unsafe PosixResult TryConnect(IPEndPointStruct endpoint)
{
sockaddr_storage addr;
GetSockaddrInet(endpoint, &addr, out int length);
int rv;
do
{
rv = connect(DangerousGetHandle().ToInt32(), (sockaddr *)&addr, length);
} while (rv < 0 && errno == EINTR);
return(PosixResult.FromReturnValue(rv));
}
public unsafe static PosixResult ClearCurrentThreadAffinity()
{
cpu_set_t cpu_set;
CPU_ZERO(&cpu_set);
for (int cpuId = 0; cpuId < CPU_SETSIZE; cpuId++)
{
CPU_SET(cpuId, &cpu_set);
}
int rv = sched_setaffinity(0, SizeOf.cpu_set_t, &cpu_set);
return(PosixResult.FromReturnValue(rv));
}
public static unsafe PosixResult Disconnect(int socket)
{
sockaddr addr = default(sockaddr);
addr.sa_family = AF_UNSPEC;
int rv;
do
{
rv = connect(socket, &addr, SizeOf.sockaddr);
} while (rv < 0 && errno == EINTR);
return(PosixResult.FromReturnValue(rv));
}
public static PosixResult EPollCreate(out EPoll epoll)
{
epoll = new EPoll();
int rv = epoll_create1(EPOLL_CLOEXEC);
if (rv == -1)
{
epoll = null;
}
else
{
epoll.SetHandle(rv);
}
return(PosixResult.FromReturnValue(rv));
}
public unsafe static PosixResult AcceptAndSendHandleTo(Socket fromSocket, int toSocket)
{
int acceptFd = fromSocket.DangerousGetHandle().ToInt32();
ssize_t rv;
do
{
rv = accept4(acceptFd, null, null, SOCK_CLOEXEC);
} while (rv < 0 && errno == EINTR);
if (rv != -1)
{
int acceptedFd = (int)rv;
byte dummyBuffer = 0;
iovec iov = default(iovec);
iov.iov_base = &dummyBuffer;
iov.iov_len = 1;
int controlLength = CMSG_SPACE(sizeof(int));
byte *control = stackalloc byte[controlLength];
msghdr header = default(msghdr);
header.msg_iov = &iov;
header.msg_iovlen = 1;
header.msg_control = control;
header.msg_controllen = controlLength;
cmsghdr *cmsg = CMSG_FIRSTHDR(&header);
cmsg->cmsg_level = SOL_SOCKET;
cmsg->cmsg_type = SCM_RIGHTS;
cmsg->cmsg_len = CMSG_LEN(sizeof(int));
int *fdptr = (int *)CMSG_DATA(cmsg);
* fdptr = acceptedFd;
do
{
rv = sendmsg(toSocket, &header, MSG_NOSIGNAL);
} while (rv < 0 && errno == EINTR);
IOInterop.Close(acceptedFd);
}
return(PosixResult.FromReturnValue(rv));
}
public static unsafe PosixResult Receive(int socket, iovec *ioVectors, int ioVectorLen)
{
msghdr hdr = default(msghdr);
hdr.msg_iov = ioVectors;
hdr.msg_iovlen = ioVectorLen;
int flags = MSG_NOSIGNAL;
ssize_t rv;
do
{
rv = recvmsg(socket, &hdr, flags);
} while (rv < 0 && errno == EINTR);
return(PosixResult.FromReturnValue(rv));
}
public unsafe static PosixResult CompleteZeroCopy(int socket)
{
msghdr msg = default(msghdr);
int controlLength = 100;
byte * control = stackalloc byte[controlLength];
do
{
msg.msg_control = control;
msg.msg_controllen = controlLength;
ssize_t rv;
do
{
rv = recvmsg(socket, &msg, MSG_NOSIGNAL | MSG_ERRQUEUE);
} while (rv < 0 && errno == EINTR);
if (rv == -1)
{
return(PosixResult.FromReturnValue(rv));
}
cmsghdr *cm = CMSG_FIRSTHDR(&msg);
if (cm == null)
{
continue;
}
if (!((cm->cmsg_level == SOL_IP && cm->cmsg_type == IP_RECVERR) ||
(cm->cmsg_level == SOL_IPV6 && cm->cmsg_type == IPV6_RECVERR)))
{
continue;
}
sock_extended_err *serr = (sock_extended_err *)CMSG_DATA(cm);
if ((serr->ee_origin != SO_EE_ORIGIN_ZEROCOPY) ||
(serr->ee_errno != 0))
{
continue;
}
return(new PosixResult(((serr->ee_code & SO_EE_CODE_ZEROCOPY_COPIED) != 0) ?
ZeroCopyCopied : ZeroCopySuccess));
} while (true);
}
public unsafe static PosixResult IoGetEvents(aio_context_t ctx, int min_nr, int nr, io_event *events, int timeoutMs)
{
timespec timeout = default(timespec);
bool hasTimeout = timeoutMs >= 0;
if (hasTimeout)
{
timeout.tv_sec = timeoutMs / 1000;
timeout.tv_nsec = 1000 * (timeoutMs % 1000);
}
int rv;
do
{
rv = io_getevents(ctx, min_nr, nr, events, hasTimeout ? &timeout : null);
} while (rv < 0 && errno == EINTR);
return(PosixResult.FromReturnValue(rv));
}
public static unsafe PosixResult Accept(int socket, bool blocking, out int clientSocket)
{
int flags = SOCK_CLOEXEC;
if (!blocking)
{
flags |= SOCK_NONBLOCK;
}
int rv;
do
{
rv = accept4(socket, null, null, flags);
} while (rv < 0 && errno == EINTR);
clientSocket = rv;
return(PosixResult.FromReturnValue(rv));
}
private unsafe void AioSend(List <ScheduledSend> sendQueue)
{
while (sendQueue.Count > 0)
{
int sendCount = 0;
int completedCount = 0;
AioCb * aioCbs = AioCbs;
IOVector *ioVectors = IoVectorTable;
ReadOnlySequence <byte>[] sendBuffers = _aioSendBuffers;
Span <MemoryHandle> memoryHandles = MemoryHandles;
int memoryHandleCount = 0;
for (int i = 0; i < sendQueue.Count; i++)
{
TSocket socket = sendQueue[i].Socket;
ReadOnlySequence <byte> buffer;
Exception error = socket.GetReadResult(out buffer);
if (error != null)
{
if (error == TransportConstants.StopSentinel)
{
error = null;
}
socket.CompleteOutput(error);
completedCount++;
}
else
{
int ioVectorLength = socket.CalcIOVectorLengthForSend(ref buffer, IoVectorsPerAioSocket);
socket.FillSendIOVector(ref buffer, ioVectors, ioVectorLength, memoryHandles);
memoryHandles = memoryHandles.Slice(ioVectorLength);
memoryHandleCount += ioVectorLength;
aioCbs->Fd = socket.Fd;
aioCbs->Data = i;
aioCbs->OpCode = AioOpCode.PWritev;
aioCbs->Buffer = ioVectors;
aioCbs->Length = ioVectorLength;
aioCbs++;
sendBuffers[sendCount] = buffer;
sendCount++;
if (sendCount == EventBufferLength)
{
break;
}
ioVectors += ioVectorLength;
}
}
if (sendCount > 0)
{
IntPtr ctxp = _aioContext;
PosixResult res = AioInterop.IoSubmit(ctxp, sendCount, AioCbsTable);
memoryHandles = MemoryHandles;
for (int i = 0; i < memoryHandleCount; i++)
{
memoryHandles[i].Dispose();
}
if (res != sendCount)
{
throw new NotSupportedException("Unexpected IoSubmit Send retval " + res);
}
AioEvent *aioEvents = AioEvents;
res = AioInterop.IoGetEvents(ctxp, sendCount, aioEvents);
if (res != sendCount)
{
throw new NotSupportedException("Unexpected IoGetEvents Send retval " + res);
}
AioEvent *aioEvent = aioEvents;
for (int i = 0; i < sendCount; i++)
{
PosixResult result = aioEvent->Result;
int socketIndex = (int)aioEvent->Data;
TSocket socket = sendQueue[socketIndex].Socket;
ReadOnlySequence <byte> buffer = sendBuffers[i]; // assumes in-order events
sendBuffers[i] = default;
socket.HandleSendResult(ref buffer, result, loop: false, zerocopy: false, zeroCopyRegistered: false);
aioEvent++;
}
}
sendQueue.RemoveRange(0, sendCount + completedCount);
}
}
private unsafe void AioReceive(List <TSocket> readableSockets)
{
long PackReceiveState(int received, int advanced, int iovLength) => ((long)received << 32) + (advanced << 8) + (iovLength);
(int received, int advanced, int iovLength) UnpackReceiveState(long data) => ((int)(data >> 32), (int)((data >> 8) & 0xffffff), (int)(data & 0xff));
int readableSocketCount = readableSockets.Count;
AioCb * aioCb = AioCbs;
IOVector * ioVectors = IoVectorTable;
PosixResult * receiveResults = stackalloc PosixResult[readableSocketCount];
Span <MemoryHandle> receiveMemoryHandles = MemoryHandles;
int receiveMemoryHandleCount = 0;
for (int i = 0; i < readableSocketCount; i++)
{
TSocket socket = readableSockets[i];
var memoryAllocation = socket.DetermineMemoryAllocationForReceive(IoVectorsPerAioSocket);
int advanced = socket.FillReceiveIOVector(memoryAllocation, ioVectors, receiveMemoryHandles);
aioCb->Fd = socket.Fd;
aioCb->Data = PackReceiveState(0, advanced, memoryAllocation.IovLength);
aioCb->OpCode = AioOpCode.PReadv;
aioCb->Buffer = ioVectors;
aioCb->Length = memoryAllocation.IovLength;
aioCb++;
ioVectors += memoryAllocation.IovLength;
receiveMemoryHandleCount += memoryAllocation.IovLength;
receiveMemoryHandles = receiveMemoryHandles.Slice(memoryAllocation.IovLength);
}
int eAgainCount = 0;
while (readableSocketCount > 0)
{
IntPtr ctxp = _aioContext;
PosixResult res = AioInterop.IoSubmit(ctxp, readableSocketCount, AioCbsTable);
if (res != readableSocketCount)
{
throw new NotSupportedException("Unexpected IoSubmit retval " + res);
}
AioEvent *aioEvents = AioEvents;
res = AioInterop.IoGetEvents(ctxp, readableSocketCount, aioEvents);
if (res != readableSocketCount)
{
throw new NotSupportedException("Unexpected IoGetEvents retval " + res);
}
int socketsRemaining = readableSocketCount;
bool allEAgain = true;
AioEvent *aioEvent = aioEvents;
for (int i = 0; i < readableSocketCount; i++)
{
PosixResult result = aioEvent->Result;
int socketIndex = i; // assumes in-order events
TSocket socket = readableSockets[socketIndex];
(int received, int advanced, int iovLength) = UnpackReceiveState(aioEvent->Data);
(bool done, PosixResult retval) = socket.InterpretReceiveResult(result, ref received, advanced, (IOVector *)aioEvent->AioCb->Buffer, iovLength);
if (done)
{
receiveResults[socketIndex] = retval;
socketsRemaining--;
aioEvent->AioCb->OpCode = AioOpCode.Noop;
allEAgain = false;
}
else if (retval != PosixResult.EAGAIN)
{
aioEvent->AioCb->Data = PackReceiveState(received, advanced, iovLength);
allEAgain = false;
}
aioEvent++;
}
if (socketsRemaining > 0)
{
if (allEAgain)
{
eAgainCount++;
if (eAgainCount == TransportConstants.MaxEAgainCount)
{
throw new NotSupportedException("Too many EAGAIN, unable to receive available bytes.");
}
}
else
{
aioCb = AioCbs;
AioCb *aioCbWriteAt = aioCb;
// The kernel doesn't handle Noop, we need to remove them from the aioCbs
for (int i = 0; i < readableSocketCount; i++)
{
if (aioCb[i].OpCode != AioOpCode.Noop)
{
if (aioCbWriteAt != aioCb)
{
*aioCbWriteAt = *aioCb;
}
aioCbWriteAt++;
}
aioCb++;
}
readableSocketCount = socketsRemaining;
eAgainCount = 0;
}
}
else
{
readableSocketCount = 0;
}
}
receiveMemoryHandles = MemoryHandles;
for (int i = 0; i < receiveMemoryHandleCount; i++)
{
receiveMemoryHandles[i].Dispose();
}
for (int i = 0; i < readableSockets.Count; i++)
{
readableSockets[i].OnReceiveFromSocket(receiveResults[i]);
}
readableSockets.Clear();
}
