public unsafe IIOCompletionTarget ReleaseOverlappedAndGetTarget(TaggedOverlapped *overlapped)
{
int entryId = Volatile.Read(ref overlapped->EntryId);
Contract.Assume(entryId != TaggedOverlapped.AvailableMarker, "Attempting to release an available TaggedOverlapped");
Contract.Assume(entryId >= 0 && entryId < PoolNodeSize);
overlapped->Release();
IIOCompletionTarget target = m_targets[entryId];
m_targets[entryId] = null;
Contract.Assume(target != null);
// Set the bit corresponding to this entry ID. This publishes the entry so that TryReserveOverlapped can use it again.
while (true)
{
long availableSigned = Volatile.Read(ref m_availableEntryMask);
ulong newAvailable = unchecked ((ulong)availableSigned) | (1UL << entryId);
if (Interlocked.CompareExchange(
ref m_availableEntryMask,
unchecked ((long)newAvailable),
comparand: availableSigned) == availableSigned)
{
break;
}
}
return(target);
}
private unsafe void TraceStartIfEnabled(TaggedOverlapped *overlapped, IIOCompletionTarget target)
{
IOCompletionTraceHook traceHook = Volatile.Read(ref m_traceHook);
if (traceHook != null)
{
traceHook.TraceStart(overlapped->GetUniqueId(), target);
}
}
public unsafe IIOCompletionTarget ReleaseOverlappedAndGetTarget(TaggedOverlapped *overlapped)
{
int nodeIndex = overlapped->PoolNodeId;
OverlappedPoolNode[] nodes = Volatile.Read(ref m_nodes);
Contract.Assume(nodes != null, "Attempting to release an overlapped on a disposed overlapped pool.");
Contract.Assume(nodeIndex >= 0 && nodeIndex < nodes.Length, "Invalid node ID; note only add node ID (no shrinking)");
return(nodes[nodeIndex].ReleaseOverlappedAndGetTarget(overlapped));
}
/// <inheritdoc />
public unsafe Overlapped *ReadFileOverlapped(
IIOCompletionTarget target,
SafeFileHandle handle,
byte *pinnedBuffer,
int bytesToRead,
long fileOffset)
{
Contract.Requires(target != null);
Contract.Requires(handle != null && !handle.IsInvalid);
Contract.Requires(pinnedBuffer != null);
TaggedOverlapped *overlapped = m_overlappedPool.ReserveOverlappedWithTarget(target);
TraceStartIfEnabled(overlapped, target);
bool needOverlappedRelease = true;
bool overlappedHasBeenReleased = false;
try
{
FileAsyncIOResult result = FileSystemWin.ReadFileOverlapped(
handle,
pinnedBuffer,
bytesToRead,
fileOffset,
(Overlapped *)overlapped);
if (result.Status != FileAsyncIOStatus.Pending)
{
Contract.Assert(
result.Status == FileAsyncIOStatus.Succeeded ||
result.Status == FileAsyncIOStatus.Failed);
// We could call the target directly.
// However, since the target may itself issue more I/O, we need to prevent unbounded stack growth.
// TODO: We could set a recursion-limit to allow some fraction of repeated IOs to complete synchronously, without
// queueing to the threadpool. Sync completions are the common case for files cached in memory.
ReleaseOvelappedAndQueueCompletionNotification(overlapped, result);
overlappedHasBeenReleased = true;
}
// At this point overlapped is either needed (pending status)
// or already released by ReleaseOvelappedAndQueueCompletionNotification
needOverlappedRelease = false;
return(!overlappedHasBeenReleased ? (Overlapped *)overlapped : null);
}
finally
{
if (needOverlappedRelease)
{
IIOCompletionTarget releasedTarget = ReleaseOverlappedAndGetTarget(overlapped);
Contract.Assume(releasedTarget == target);
}
}
}
/// <inheritdoc/>
public unsafe void CancelOverlapped(SafeFileHandle handle, Overlapped *overlapped)
{
Contract.Requires(!handle.IsInvalid);
Contract.Requires(overlapped != null);
TaggedOverlapped *taggedOverlapped = (TaggedOverlapped *)overlapped;
if (taggedOverlapped->EntryId == TaggedOverlapped.AvailableMarker)
{
return;
}
// Best effort.
Analysis.IgnoreResult(FileSystemWin.TryCancelIoWithOverlapped(handle, overlapped, out int _));
}
public unsafe TaggedOverlapped *ReserveOverlappedWithTarget(IIOCompletionTarget target)
{
int nodeIndex = 0;
while (true)
{
OverlappedPoolNode[] nodes = Volatile.Read(ref m_nodes);
Contract.Assume(nodes != null, "Attempting to reserve an overlapped on a disposed overlapped pool.");
if (nodeIndex == nodes.Length)
{
lock (this)
{
Contract.Assume(m_nodes != null, "Attempting to reserve an overlapped on a disposed overlapped pool.");
if (nodeIndex == m_nodes.Length)
{
var newNodes = new OverlappedPoolNode[nodeIndex + 1];
Array.Copy(m_nodes, newNodes, nodeIndex);
newNodes[nodeIndex] = new OverlappedPoolNode(nodeIndex);
m_nodes = newNodes;
}
Contract.Assume(nodeIndex < m_nodes.Length);
nodes = m_nodes;
}
}
for (; nodeIndex < nodes.Length; nodeIndex++)
{
TaggedOverlapped *reserved = nodes[nodeIndex].TryReserveOverlappedWithTarget(target);
if (reserved != null)
{
return(reserved);
}
}
// All nodes exhausted; maybe grow the array and continue from the present index.
}
}
private unsafe void ReleaseOvelappedAndQueueCompletionNotification(TaggedOverlapped *overlapped, FileAsyncIOResult result)
{
// We can now find the external-code callback that needs notification for this request.
// We can't allow it to block this thread, which is dedicated to servicing the completion port.
// Were it to block, we could starve I/O completion; blocking on other I/O completions could be a deadlock.
// So, we guarantee that IIOCompletionTargets run on thread-pool threads - they suffer no special restrictions.
IIOCompletionTarget target = ReleaseOverlappedAndGetTarget(overlapped);
var notificationArgs = new IOCompletionNotificationArgs
{
Result = result,
Target = target,
};
ThreadPool.QueueUserWorkItem(
state =>
{
var stateArgs = (IOCompletionNotificationArgs)state;
stateArgs.Target.OnCompletion(stateArgs.Result);
},
notificationArgs);
}
public unsafe TaggedOverlapped *TryReserveOverlappedWithTarget(IIOCompletionTarget target)
{
// Try to reserve an entry by clearing its available bit. We give up when no bits are set.
int reservedIndex;
while (true)
{
long availableSigned = Volatile.Read(ref m_availableEntryMask);
if (availableSigned == 0)
{
// No entry available; caller should allocate a new node or try again later.
return(null);
}
var available = unchecked ((ulong)availableSigned);
reservedIndex = Bits.FindLowestBitSet(available);
ulong newAvailable = available & ~(1UL << reservedIndex);
if (Interlocked.CompareExchange(
ref m_availableEntryMask,
unchecked ((long)newAvailable),
comparand: availableSigned) == availableSigned)
{
break;
}
}
Contract.Assume(m_targets[reservedIndex] == null);
m_targets[reservedIndex] = target;
fixed(TaggedOverlapped *entries = m_entries)
{
TaggedOverlapped *reservedEntry = &entries[reservedIndex];
reservedEntry->Reserve(reservedIndex);
return(reservedEntry);
}
}
private unsafe IIOCompletionTarget ReleaseOverlappedAndGetTarget(TaggedOverlapped *overlapped)
{
TraceCompletionIfEnabled(overlapped);
return(m_overlappedPool.ReleaseOverlappedAndGetTarget(overlapped));
}
private unsafe void TraceCompletionIfEnabled(TaggedOverlapped *overlapped)
{
IOCompletionTraceHook traceHook = Volatile.Read(ref m_traceHook);
traceHook?.TraceComplete(overlapped->GetUniqueId());
}
