/// <summary>Pull a slot out of a preexisting shared memory segment.</summary>
/// <remarks>
/// Pull a slot out of a preexisting shared memory segment.
/// Must be called with the manager lock held.
/// </remarks>
/// <param name="blockId">The blockId to put inside the Slot object.</param>
/// <returns>
/// null if none of our shared memory segments contain a
/// free slot; the slot object otherwise.
/// </returns>
private ShortCircuitShm.Slot AllocSlotFromExistingShm(ExtendedBlockId blockId)
{
if (this.notFull.IsEmpty())
{
return(null);
}
KeyValuePair <ShortCircuitShm.ShmId, DfsClientShm> entry = this.notFull.FirstEntry
();
DfsClientShm shm = entry.Value;
ShortCircuitShm.ShmId shmId = shm.GetShmId();
ShortCircuitShm.Slot slot = shm.AllocAndRegisterSlot(blockId);
if (shm.IsFull())
{
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": pulled the last slot " + slot.GetSlotIdx(
) + " out of " + shm);
}
DfsClientShm removedShm = Sharpen.Collections.Remove(this.notFull, shmId);
Preconditions.CheckState(removedShm == shm);
this.full[shmId] = shm;
}
else
{
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": pulled slot " + slot.GetSlotIdx() + " out of "
+ shm);
}
}
return(slot);
}
/// <summary>Register a slot.</summary>
/// <remarks>
/// Register a slot.
/// This function looks at a slot which has already been initialized (by
/// another process), and registers it with us. Then, it returns the
/// relevant Slot object.
/// </remarks>
/// <returns>The slot.</returns>
/// <exception cref="Org.Apache.Hadoop.FS.InvalidRequestException">
/// If the slot index we're trying to allocate has not been
/// initialized, or is already in use.
/// </exception>
public ShortCircuitShm.Slot RegisterSlot(int slotIdx, ExtendedBlockId blockId)
{
lock (this)
{
if (slotIdx < 0)
{
throw new InvalidRequestException(this + ": invalid negative slot " + "index " +
slotIdx);
}
if (slotIdx >= slots.Length)
{
throw new InvalidRequestException(this + ": invalid slot " + "index " + slotIdx);
}
if (allocatedSlots.Get(slotIdx))
{
throw new InvalidRequestException(this + ": slot " + slotIdx + " is already in use."
);
}
ShortCircuitShm.Slot slot = new ShortCircuitShm.Slot(this, CalculateSlotAddress(slotIdx
), blockId);
if (!slot.IsValid())
{
throw new InvalidRequestException(this + ": slot " + slotIdx + " is not marked as valid."
);
}
slots[slotIdx] = slot;
allocatedSlots.Set(slotIdx, true);
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": registerSlot " + slotIdx + ": allocatedSlots=" + allocatedSlots
+ StringUtils.GetStackTrace(Sharpen.Thread.CurrentThread()));
}
return(slot);
}
}
/// <summary>Stop tracking a slot.</summary>
/// <remarks>
/// Stop tracking a slot.
/// Must be called with the EndpointShmManager lock held.
/// </remarks>
/// <param name="slot">The slot to release.</param>
internal virtual void FreeSlot(ShortCircuitShm.Slot slot)
{
DfsClientShm shm = (DfsClientShm)slot.GetShm();
shm.UnregisterSlot(slot.GetSlotIdx());
if (shm.IsDisconnected())
{
// Stale shared memory segments should not be tracked here.
Preconditions.CheckState(!this.full.Contains(shm.GetShmId()));
Preconditions.CheckState(!this.notFull.Contains(shm.GetShmId()));
if (shm.IsEmpty())
{
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": freeing empty stale " + shm);
}
shm.Free();
}
}
else
{
ShortCircuitShm.ShmId shmId = shm.GetShmId();
Sharpen.Collections.Remove(this.full, shmId);
// The shm can't be full if we just freed a slot.
if (shm.IsEmpty())
{
Sharpen.Collections.Remove(this.notFull, shmId);
// If the shared memory segment is now empty, we call shutdown(2) on
// the UNIX domain socket associated with it. The DomainSocketWatcher,
// which is watching this socket, will call DfsClientShm#handle,
// cleaning up this shared memory segment.
//
// See #{DfsClientShmManager#domainSocketWatcher} for details about why
// we don't want to call DomainSocketWatcher#remove directly here.
//
// Note that we could experience 'fragmentation' here, where the
// DFSClient allocates a bunch of slots in different shared memory
// segments, and then frees most of them, but never fully empties out
// any segment. We make some attempt to avoid this fragmentation by
// always allocating new slots out of the shared memory segment with the
// lowest ID, but it could still occur. In most workloads,
// fragmentation should not be a major concern, since it doesn't impact
// peak file descriptor usage or the speed of allocation.
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": shutting down UNIX domain socket for " +
"empty " + shm);
}
this.Shutdown(shm);
}
else
{
this.notFull[shmId] = shm;
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestAllocateSlots()
{
FilePath path = new FilePath(TestBase, "testAllocateSlots");
path.Mkdirs();
SharedFileDescriptorFactory factory = SharedFileDescriptorFactory.Create("shm_",
new string[] { path.GetAbsolutePath() });
FileInputStream stream = factory.CreateDescriptor("testAllocateSlots", 4096);
ShortCircuitShm shm = new ShortCircuitShm(ShortCircuitShm.ShmId.CreateRandom(), stream
);
int numSlots = 0;
AList <ShortCircuitShm.Slot> slots = new AList <ShortCircuitShm.Slot>();
while (!shm.IsFull())
{
ShortCircuitShm.Slot slot = shm.AllocAndRegisterSlot(new ExtendedBlockId(123L, "test_bp1"
));
slots.AddItem(slot);
numSlots++;
}
Log.Info("allocated " + numSlots + " slots before running out.");
int slotIdx = 0;
for (IEnumerator <ShortCircuitShm.Slot> iter = shm.SlotIterator(); iter.HasNext();)
{
NUnit.Framework.Assert.IsTrue(slots.Contains(iter.Next()));
}
foreach (ShortCircuitShm.Slot slot_1 in slots)
{
NUnit.Framework.Assert.IsFalse(slot_1.AddAnchor());
NUnit.Framework.Assert.AreEqual(slotIdx++, slot_1.GetSlotIdx());
}
foreach (ShortCircuitShm.Slot slot_2 in slots)
{
slot_2.MakeAnchorable();
}
foreach (ShortCircuitShm.Slot slot_3 in slots)
{
NUnit.Framework.Assert.IsTrue(slot_3.AddAnchor());
}
foreach (ShortCircuitShm.Slot slot_4 in slots)
{
slot_4.RemoveAnchor();
}
foreach (ShortCircuitShm.Slot slot_5 in slots)
{
shm.UnregisterSlot(slot_5.GetSlotIdx());
slot_5.MakeInvalid();
}
shm.Free();
stream.Close();
FileUtil.FullyDelete(path);
}
public virtual void FreeSlot(ShortCircuitShm.Slot slot)
{
Lock.Lock();
try
{
DfsClientShm shm = (DfsClientShm)slot.GetShm();
shm.GetEndpointShmManager().FreeSlot(slot);
}
finally
{
Lock.Unlock();
}
}
/// <exception cref="System.IO.IOException"/>
public ShortCircuitReplica(ExtendedBlockId key, FileInputStream dataStream, FileInputStream
metaStream, ShortCircuitCache cache, long creationTimeMs, ShortCircuitShm.Slot
slot)
{
this.key = key;
this.dataStream = dataStream;
this.metaStream = metaStream;
this.metaHeader = BlockMetadataHeader.PreadHeader(metaStream.GetChannel());
if (metaHeader.GetVersion() != 1)
{
throw new IOException("invalid metadata header version " + metaHeader.GetVersion(
) + ". Can only handle version 1.");
}
this.cache = cache;
this.creationTimeMs = creationTimeMs;
this.slot = slot;
}
/// <exception cref="System.IO.IOException"/>
public void Visit(Dictionary <DatanodeInfo, DfsClientShmManager.PerDatanodeVisitorInfo
> info)
{
NUnit.Framework.Assert.IsTrue(info[datanode].full.IsEmpty());
NUnit.Framework.Assert.IsFalse(info[datanode].disabled);
NUnit.Framework.Assert.AreEqual(1, info[datanode].notFull.Values.Count);
DfsClientShm shm = info[datanode].notFull.Values.GetEnumerator().Next();
for (IEnumerator <ShortCircuitShm.Slot> iter = shm.SlotIterator(); iter.HasNext();)
{
ShortCircuitShm.Slot slot = iter.Next();
if (slot.IsValid())
{
this._enclosing.done.SetValue(false);
}
}
}
/// <summary>
/// Handle the closure of the UNIX domain socket associated with this shared
/// memory segment by marking this segment as stale.
/// </summary>
/// <remarks>
/// Handle the closure of the UNIX domain socket associated with this shared
/// memory segment by marking this segment as stale.
/// If there are no slots associated with this shared memory segment, it will
/// be freed immediately in this function.
/// </remarks>
public virtual bool Handle(DomainSocket sock)
{
manager.UnregisterShm(GetShmId());
lock (this)
{
Preconditions.CheckState(!disconnected);
disconnected = true;
bool hadSlots = false;
for (IEnumerator <ShortCircuitShm.Slot> iter = SlotIterator(); iter.HasNext();)
{
ShortCircuitShm.Slot slot = iter.Next();
slot.MakeInvalid();
hadSlots = true;
}
if (!hadSlots)
{
Free();
}
}
return(true);
}
/// <summary>Allocate a new slot and register it.</summary>
/// <remarks>
/// Allocate a new slot and register it.
/// This function chooses an empty slot, initializes it, and then returns
/// the relevant Slot object.
/// </remarks>
/// <returns>The new slot.</returns>
public ShortCircuitShm.Slot AllocAndRegisterSlot(ExtendedBlockId blockId)
{
lock (this)
{
int idx = allocatedSlots.NextClearBit(0);
if (idx >= slots.Length)
{
throw new RuntimeException(this + ": no more slots are available.");
}
allocatedSlots.Set(idx, true);
ShortCircuitShm.Slot slot = new ShortCircuitShm.Slot(this, CalculateSlotAddress(idx
), blockId);
slot.Clear();
slot.MakeValid();
slots[idx] = slot;
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": allocAndRegisterSlot " + idx + ": allocatedSlots=" + allocatedSlots
+ StringUtils.GetStackTrace(Sharpen.Thread.CurrentThread()));
}
return(slot);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestAllocShm()
{
BlockReaderTestUtil.EnableShortCircuitShmTracing();
TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
Configuration conf = CreateShortCircuitConf("testAllocShm", sockDir);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
ShortCircuitCache cache = fs.GetClient().GetClientContext().GetShortCircuitCache(
);
cache.GetDfsClientShmManager().Visit(new _Visitor_423());
// The ClientShmManager starts off empty
DomainPeer peer = GetDomainPeerToDn(conf);
MutableBoolean usedPeer = new MutableBoolean(false);
ExtendedBlockId blockId = new ExtendedBlockId(123, "xyz");
DatanodeInfo datanode = new DatanodeInfo(cluster.GetDataNodes()[0].GetDatanodeId(
));
// Allocating the first shm slot requires using up a peer.
ShortCircuitShm.Slot slot = cache.AllocShmSlot(datanode, peer, usedPeer, blockId,
"testAllocShm_client");
NUnit.Framework.Assert.IsNotNull(slot);
NUnit.Framework.Assert.IsTrue(usedPeer.BooleanValue());
cache.GetDfsClientShmManager().Visit(new _Visitor_441(datanode));
// The ClientShmManager starts off empty
cache.ScheduleSlotReleaser(slot);
// Wait for the slot to be released, and the shared memory area to be
// closed. Since we didn't register this shared memory segment on the
// server, it will also be a test of how well the server deals with
// bogus client behavior.
GenericTestUtils.WaitFor(new _Supplier_458(cache, datanode), 10, 60000);
cluster.Shutdown();
sockDir.Close();
}
/// <summary>Allocate a new shared memory slot connected to this datanode.</summary>
/// <remarks>
/// Allocate a new shared memory slot connected to this datanode.
/// Must be called with the EndpointShmManager lock held.
/// </remarks>
/// <param name="peer">The peer to use to talk to the DataNode.</param>
/// <param name="usedPeer">
/// (out param) Will be set to true if we used the peer.
/// When a peer is used
/// </param>
/// <param name="clientName">The client name.</param>
/// <param name="blockId">The block ID to use.</param>
/// <returns>
/// null if the DataNode does not support shared memory
/// segments, or experienced an error creating the
/// shm. The shared memory segment itself on success.
/// </returns>
/// <exception cref="System.IO.IOException">If there was an error communicating over the socket.
/// </exception>
internal virtual ShortCircuitShm.Slot AllocSlot(DomainPeer peer, MutableBoolean usedPeer
, string clientName, ExtendedBlockId blockId)
{
while (true)
{
if (this._enclosing.closed)
{
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": the DfsClientShmManager has been closed."
);
}
return(null);
}
if (this.disabled)
{
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": shared memory segment access is disabled."
);
}
return(null);
}
// Try to use an existing slot.
ShortCircuitShm.Slot slot = this.AllocSlotFromExistingShm(blockId);
if (slot != null)
{
return(slot);
}
// There are no free slots. If someone is loading more slots, wait
// for that to finish.
if (this.loading)
{
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": waiting for loading to finish...");
}
this._enclosing.finishedLoading.AwaitUninterruptibly();
}
else
{
// Otherwise, load the slot ourselves.
this.loading = true;
this._enclosing.Lock.Unlock();
DfsClientShm shm;
try
{
shm = this.RequestNewShm(clientName, peer);
if (shm == null)
{
continue;
}
// See #{DfsClientShmManager#domainSocketWatcher} for details
// about why we do this before retaking the manager lock.
this._enclosing.domainSocketWatcher.Add(peer.GetDomainSocket(), shm);
// The DomainPeer is now our responsibility, and should not be
// closed by the caller.
usedPeer.SetValue(true);
}
finally
{
this._enclosing.Lock.Lock();
this.loading = false;
this._enclosing.finishedLoading.SignalAll();
}
if (shm.IsDisconnected())
{
// If the peer closed immediately after the shared memory segment
// was created, the DomainSocketWatcher callback might already have
// fired and marked the shm as disconnected. In this case, we
// obviously don't want to add the SharedMemorySegment to our list
// of valid not-full segments.
if (DfsClientShmManager.Log.IsDebugEnabled())
{
DfsClientShmManager.Log.Debug(this + ": the UNIX domain socket associated with "
+ "this short-circuit memory closed before we could make " + "use of the shm.");
}
}
else
{
this.notFull[shm.GetShmId()] = shm;
}
}
}
}
