/// <summary>Pull a slot out of a preexisting shared memory segment.</summary>
/// <remarks>
/// Pull a slot out of a preexisting shared memory segment.
/// Must be called with the manager lock held.
/// </remarks>
/// <param name="blockId">The blockId to put inside the Slot object.</param>
/// <returns>
/// null if none of our shared memory segments contain a
/// free slot; the slot object otherwise.
/// </returns>
private ShortCircuitShm.Slot AllocSlotFromExistingShm(ExtendedBlockId blockId)
{
if (this.notFull.IsEmpty())
{
return(null);
}
KeyValuePair <ShortCircuitShm.ShmId, DfsClientShm> entry = this.notFull.FirstEntry
();
DfsClientShm shm = entry.Value;
ShortCircuitShm.ShmId shmId = shm.GetShmId();
ShortCircuitShm.Slot slot = shm.AllocAndRegisterSlot(blockId);
if (shm.IsFull())
{
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": pulled the last slot " + slot.GetSlotIdx(
) + " out of " + shm);
}
DfsClientShm removedShm = Sharpen.Collections.Remove(this.notFull, shmId);
Preconditions.CheckState(removedShm == shm);
this.full[shmId] = shm;
}
else
{
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": pulled slot " + slot.GetSlotIdx() + " out of "
+ shm);
}
}
return(slot);
}
/// <summary>
/// Invalidate any slot associated with a blockId that we are invalidating
/// (deleting) from this DataNode.
/// </summary>
/// <remarks>
/// Invalidate any slot associated with a blockId that we are invalidating
/// (deleting) from this DataNode. When a slot is invalid, the DFSClient will
/// not use the corresponding replica for new read or mmap operations (although
/// existing, ongoing read or mmap operations will complete.)
/// </remarks>
/// <param name="blockId">The block ID.</param>
public virtual void ProcessBlockInvalidation(ExtendedBlockId blockId)
{
lock (this)
{
if (!enabled)
{
return;
}
ICollection <ShortCircuitShm.Slot> affectedSlots = ((ICollection <ShortCircuitShm.Slot
>)slots.Get(blockId));
if (!affectedSlots.IsEmpty())
{
StringBuilder bld = new StringBuilder();
string prefix = string.Empty;
bld.Append("Block ").Append(blockId).Append(" has been invalidated. ").Append("Marking short-circuit slots as invalid: "
);
foreach (ShortCircuitShm.Slot slot in affectedSlots)
{
slot.MakeInvalid();
bld.Append(prefix).Append(slot.ToString());
prefix = ", ";
}
Log.Info(bld.ToString());
}
}
}
/// <summary>Register a slot.</summary>
/// <remarks>
/// Register a slot.
/// This function looks at a slot which has already been initialized (by
/// another process), and registers it with us. Then, it returns the
/// relevant Slot object.
/// </remarks>
/// <returns>The slot.</returns>
/// <exception cref="Org.Apache.Hadoop.FS.InvalidRequestException">
/// If the slot index we're trying to allocate has not been
/// initialized, or is already in use.
/// </exception>
public ShortCircuitShm.Slot RegisterSlot(int slotIdx, ExtendedBlockId blockId)
{
lock (this)
{
if (slotIdx < 0)
{
throw new InvalidRequestException(this + ": invalid negative slot " + "index " +
slotIdx);
}
if (slotIdx >= slots.Length)
{
throw new InvalidRequestException(this + ": invalid slot " + "index " + slotIdx);
}
if (allocatedSlots.Get(slotIdx))
{
throw new InvalidRequestException(this + ": slot " + slotIdx + " is already in use."
);
}
ShortCircuitShm.Slot slot = new ShortCircuitShm.Slot(this, CalculateSlotAddress(slotIdx
), blockId);
if (!slot.IsValid())
{
throw new InvalidRequestException(this + ": slot " + slotIdx + " is not marked as valid."
);
}
slots[slotIdx] = slot;
allocatedSlots.Set(slotIdx, true);
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": registerSlot " + slotIdx + ": allocatedSlots=" + allocatedSlots
+ StringUtils.GetStackTrace(Sharpen.Thread.CurrentThread()));
}
return(slot);
}
}
internal Slot(ShortCircuitShm _enclosing, long slotAddress, ExtendedBlockId blockId
)
{
this._enclosing = _enclosing;
this.slotAddress = slotAddress;
this.blockId = blockId;
}
internal virtual void UncacheBlock(string bpid, long blockId)
{
lock (this)
{
ExtendedBlockId key = new ExtendedBlockId(blockId, bpid);
FsDatasetCache.Value prevValue = mappableBlockMap[key];
bool deferred = false;
if (!dataset.datanode.GetShortCircuitRegistry().ProcessBlockMunlockRequest(key))
{
deferred = true;
}
if (prevValue == null)
{
Log.Debug("Block with id {}, pool {} does not need to be uncached, " + "because it is not currently in the mappableBlockMap."
, blockId, bpid);
numBlocksFailedToUncache.IncrementAndGet();
return;
}
switch (prevValue.state)
{
case FsDatasetCache.State.Caching:
{
Log.Debug("Cancelling caching for block with id {}, pool {}.", blockId, bpid);
mappableBlockMap[key] = new FsDatasetCache.Value(prevValue.mappableBlock, FsDatasetCache.State
.CachingCancelled);
break;
}
case FsDatasetCache.State.Cached:
{
mappableBlockMap[key] = new FsDatasetCache.Value(prevValue.mappableBlock, FsDatasetCache.State
.Uncaching);
if (deferred)
{
Log.Debug("{} is anchored, and can't be uncached now. Scheduling it " + "for uncaching in {} "
, key, DurationFormatUtils.FormatDurationHMS(revocationPollingMs));
deferredUncachingExecutor.Schedule(new FsDatasetCache.UncachingTask(this, key, revocationMs
), revocationPollingMs, TimeUnit.Milliseconds);
}
else
{
Log.Debug("{} has been scheduled for immediate uncaching.", key);
uncachingExecutor.Execute(new FsDatasetCache.UncachingTask(this, key, 0));
}
break;
}
default:
{
Log.Debug("Block with id {}, pool {} does not need to be uncached, " + "because it is in state {}."
, blockId, bpid, prevValue.state);
numBlocksFailedToUncache.IncrementAndGet();
break;
}
}
}
}
internal CachingTask(FsDatasetCache _enclosing, ExtendedBlockId key, string blockFileName
, long length, long genstamp)
{
this._enclosing = _enclosing;
this.key = key;
this.blockFileName = blockFileName;
this.length = length;
this.genstamp = genstamp;
}
public virtual bool IsCached(string bpid, long blockId)
{
lock (this)
{
ExtendedBlockId block = new ExtendedBlockId(blockId, bpid);
FsDatasetCache.Value val = mappableBlockMap[block];
return((val != null) && val.state.ShouldAdvertise());
}
}
public _ShortCircuitReplicaCreator_330(ExtendedBlockId key, TestShortCircuitCache.TestFileDescriptorPair
[] pairs, int iVal, ShortCircuitCache cache, long HourInMs)
{
this.key = key;
this.pairs = pairs;
this.iVal = iVal;
this.cache = cache;
this.HourInMs = HourInMs;
}
public void Visit(int numOutstandingMmaps, IDictionary <ExtendedBlockId, ShortCircuitReplica
> replicas, IDictionary <ExtendedBlockId, SecretManager.InvalidToken> failedLoads
, IDictionary <long, ShortCircuitReplica> evictable, IDictionary <long, ShortCircuitReplica
> evictableMmapped)
{
ShortCircuitReplica replica = replicas[ExtendedBlockId.FromExtendedBlock(block)];
NUnit.Framework.Assert.IsNotNull(replica);
NUnit.Framework.Assert.IsFalse(replica.GetSlot().IsValid());
}
public virtual ShortCircuitReplicaInfo CreateShortCircuitReplicaInfo()
{
try
{
ExtendedBlockId key = new ExtendedBlockId(blockId, "test_bp1");
return(new ShortCircuitReplicaInfo(new ShortCircuitReplica(key, pair.GetFileInputStreams
()[0], pair.GetFileInputStreams()[1], cache, Time.MonotonicNow(), null)));
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
internal UncachingTask(FsDatasetCache _enclosing, ExtendedBlockId key, long revocationDelayMs
)
{
this._enclosing = _enclosing;
this.key = key;
if (revocationDelayMs == 0)
{
this.revocationTimeMs = 0;
}
else
{
this.revocationTimeMs = revocationDelayMs + Org.Apache.Hadoop.Util.Time.MonotonicNow
();
}
}
/// <summary>Process a block mlock event from the FsDatasetCache.</summary>
/// <param name="blockId">The block that was mlocked.</param>
public virtual void ProcessBlockMlockEvent(ExtendedBlockId blockId)
{
lock (this)
{
if (!enabled)
{
return;
}
ICollection <ShortCircuitShm.Slot> affectedSlots = ((ICollection <ShortCircuitShm.Slot
>)slots.Get(blockId));
foreach (ShortCircuitShm.Slot slot in affectedSlots)
{
slot.MakeAnchorable();
}
}
}
/// <exception cref="System.IO.IOException"/>
public ShortCircuitReplica(ExtendedBlockId key, FileInputStream dataStream, FileInputStream
metaStream, ShortCircuitCache cache, long creationTimeMs, ShortCircuitShm.Slot
slot)
{
this.key = key;
this.dataStream = dataStream;
this.metaStream = metaStream;
this.metaHeader = BlockMetadataHeader.PreadHeader(metaStream.GetChannel());
if (metaHeader.GetVersion() != 1)
{
throw new IOException("invalid metadata header version " + metaHeader.GetVersion(
) + ". Can only handle version 1.");
}
this.cache = cache;
this.creationTimeMs = creationTimeMs;
this.slot = slot;
}
public virtual string GetClientNames(ExtendedBlockId blockId)
{
lock (this)
{
if (!enabled)
{
return(string.Empty);
}
HashSet <string> clientNames = new HashSet <string>();
ICollection <ShortCircuitShm.Slot> affectedSlots = ((ICollection <ShortCircuitShm.Slot
>)slots.Get(blockId));
foreach (ShortCircuitShm.Slot slot in affectedSlots)
{
clientNames.AddItem(((ShortCircuitRegistry.RegisteredShm)slot.GetShm()).GetClientName
());
}
return(Joiner.On(",").Join(clientNames));
}
}
public void Visit(int numOutstandingMmaps, IDictionary <ExtendedBlockId, ShortCircuitReplica
> replicas, IDictionary <ExtendedBlockId, SecretManager.InvalidToken> failedLoads
, IDictionary <long, ShortCircuitReplica> evictable, IDictionary <long, ShortCircuitReplica
> evictableMmapped)
{
NUnit.Framework.Assert.AreEqual(expectedOutstandingMmaps, numOutstandingMmaps);
ShortCircuitReplica replica = replicas[ExtendedBlockId.FromExtendedBlock(block)];
NUnit.Framework.Assert.IsNotNull(replica);
ShortCircuitShm.Slot slot = replica.GetSlot();
if ((expectedIsAnchorable != slot.IsAnchorable()) || (expectedIsAnchored != slot.
IsAnchored()))
{
TestEnhancedByteBufferAccess.Log.Info("replica " + replica + " has isAnchorable = "
+ slot.IsAnchorable() + ", isAnchored = " + slot.IsAnchored() + ". Waiting for isAnchorable = "
+ expectedIsAnchorable + ", isAnchored = " + expectedIsAnchored);
return;
}
result.SetValue(true);
}
/// <summary>Attempt to begin caching a block.</summary>
internal virtual void CacheBlock(long blockId, string bpid, string blockFileName,
long length, long genstamp, Executor volumeExecutor)
{
lock (this)
{
ExtendedBlockId key = new ExtendedBlockId(blockId, bpid);
FsDatasetCache.Value prevValue = mappableBlockMap[key];
if (prevValue != null)
{
Log.Debug("Block with id {}, pool {} already exists in the " + "FsDatasetCache with state {}"
, blockId, bpid, prevValue.state);
numBlocksFailedToCache.IncrementAndGet();
return;
}
mappableBlockMap[key] = new FsDatasetCache.Value(null, FsDatasetCache.State.Caching
);
volumeExecutor.Execute(new FsDatasetCache.CachingTask(this, key, blockFileName, length
, genstamp));
Log.Debug("Initiating caching for Block with id {}, pool {}", blockId, bpid);
}
}
/// <exception cref="Org.Apache.Hadoop.FS.InvalidRequestException"/>
public virtual void RegisterSlot(ExtendedBlockId blockId, ShortCircuitShm.SlotId
slotId, bool isCached)
{
lock (this)
{
if (!enabled)
{
if (Log.IsTraceEnabled())
{
Log.Trace(this + " can't register a slot because the " + "ShortCircuitRegistry is not enabled."
);
}
throw new NotSupportedException();
}
ShortCircuitShm.ShmId shmId = slotId.GetShmId();
ShortCircuitRegistry.RegisteredShm shm = segments[shmId];
if (shm == null)
{
throw new InvalidRequestException("there is no shared memory segment " + "registered with shmId "
+ shmId);
}
ShortCircuitShm.Slot slot = shm.RegisterSlot(slotId.GetSlotIdx(), blockId);
if (isCached)
{
slot.MakeAnchorable();
}
else
{
slot.MakeUnanchorable();
}
bool added = slots.Put(blockId, slot);
Preconditions.CheckState(added);
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": registered " + blockId + " with slot " + slotId + " (isCached="
+ isCached + ")");
}
}
}
/// <summary>Mark any slots associated with this blockId as unanchorable.</summary>
/// <param name="blockId">The block ID.</param>
/// <returns>True if we should allow the munlock request.</returns>
public virtual bool ProcessBlockMunlockRequest(ExtendedBlockId blockId)
{
lock (this)
{
if (!enabled)
{
return(true);
}
bool allowMunlock = true;
ICollection <ShortCircuitShm.Slot> affectedSlots = ((ICollection <ShortCircuitShm.Slot
>)slots.Get(blockId));
foreach (ShortCircuitShm.Slot slot in affectedSlots)
{
slot.MakeUnanchorable();
if (slot.IsAnchored())
{
allowMunlock = false;
}
}
return(allowMunlock);
}
}
/// <summary>Allocate a new slot and register it.</summary>
/// <remarks>
/// Allocate a new slot and register it.
/// This function chooses an empty slot, initializes it, and then returns
/// the relevant Slot object.
/// </remarks>
/// <returns>The new slot.</returns>
public ShortCircuitShm.Slot AllocAndRegisterSlot(ExtendedBlockId blockId)
{
lock (this)
{
int idx = allocatedSlots.NextClearBit(0);
if (idx >= slots.Length)
{
throw new RuntimeException(this + ": no more slots are available.");
}
allocatedSlots.Set(idx, true);
ShortCircuitShm.Slot slot = new ShortCircuitShm.Slot(this, CalculateSlotAddress(idx
), blockId);
slot.Clear();
slot.MakeValid();
slots[idx] = slot;
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": allocAndRegisterSlot " + idx + ": allocatedSlots=" + allocatedSlots
+ StringUtils.GetStackTrace(Sharpen.Thread.CurrentThread()));
}
return(slot);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestTimeBasedStaleness()
{
// Set up the cache with a short staleness time.
ShortCircuitCache cache = new ShortCircuitCache(2, 10000000, 1, 10000000, 1, 10,
0);
TestShortCircuitCache.TestFileDescriptorPair[] pairs = new TestShortCircuitCache.TestFileDescriptorPair
[] { new TestShortCircuitCache.TestFileDescriptorPair(), new TestShortCircuitCache.TestFileDescriptorPair
() };
ShortCircuitReplicaInfo[] replicaInfos = new ShortCircuitReplicaInfo[] { null, null };
long HourInMs = 60 * 60 * 1000;
for (int i = 0; i < pairs.Length; i++)
{
int iVal = i;
ExtendedBlockId key = new ExtendedBlockId(i, "test_bp1");
replicaInfos[i] = cache.FetchOrCreate(key, new _ShortCircuitReplicaCreator_330(key
, pairs, iVal, cache, HourInMs));
Preconditions.CheckNotNull(replicaInfos[i].GetReplica());
Preconditions.CheckState(replicaInfos[i].GetInvalidTokenException() == null);
pairs[i].CompareWith(replicaInfos[i].GetReplica().GetDataStream(), replicaInfos[i
].GetReplica().GetMetaStream());
}
// Keep trying to getOrCreate block 0 until it goes stale (and we must re-create.)
GenericTestUtils.WaitFor(new _Supplier_351(cache), 500, 60000);
// Make sure that second replica did not go stale.
ShortCircuitReplicaInfo info = cache.FetchOrCreate(new ExtendedBlockId(1, "test_bp1"
), new _ShortCircuitReplicaCreator_371());
info.GetReplica().Unref();
// Clean up
for (int i_1 = 1; i_1 < pairs.Length; i_1++)
{
replicaInfos[i_1].GetReplica().Unref();
}
cache.Close();
}
/// <exception cref="System.IO.IOException"/>
public virtual ShortCircuitShm.Slot AllocSlot(DatanodeInfo datanode, DomainPeer peer
, MutableBoolean usedPeer, ExtendedBlockId blockId, string clientName)
{
Lock.Lock();
try
{
if (closed)
{
Log.Trace(this + ": the DfsClientShmManager isclosed.");
return(null);
}
DfsClientShmManager.EndpointShmManager shmManager = datanodes[datanode];
if (shmManager == null)
{
shmManager = new DfsClientShmManager.EndpointShmManager(this, datanode);
datanodes[datanode] = shmManager;
}
return(shmManager.AllocSlot(peer, usedPeer, clientName, blockId));
}
finally
{
Lock.Unlock();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestAllocShm()
{
BlockReaderTestUtil.EnableShortCircuitShmTracing();
TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
Configuration conf = CreateShortCircuitConf("testAllocShm", sockDir);
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();
cluster.WaitActive();
DistributedFileSystem fs = cluster.GetFileSystem();
ShortCircuitCache cache = fs.GetClient().GetClientContext().GetShortCircuitCache(
);
cache.GetDfsClientShmManager().Visit(new _Visitor_423());
// The ClientShmManager starts off empty
DomainPeer peer = GetDomainPeerToDn(conf);
MutableBoolean usedPeer = new MutableBoolean(false);
ExtendedBlockId blockId = new ExtendedBlockId(123, "xyz");
DatanodeInfo datanode = new DatanodeInfo(cluster.GetDataNodes()[0].GetDatanodeId(
));
// Allocating the first shm slot requires using up a peer.
ShortCircuitShm.Slot slot = cache.AllocShmSlot(datanode, peer, usedPeer, blockId,
"testAllocShm_client");
NUnit.Framework.Assert.IsNotNull(slot);
NUnit.Framework.Assert.IsTrue(usedPeer.BooleanValue());
cache.GetDfsClientShmManager().Visit(new _Visitor_441(datanode));
// The ClientShmManager starts off empty
cache.ScheduleSlotReleaser(slot);
// Wait for the slot to be released, and the shared memory area to be
// closed. Since we didn't register this shared memory segment on the
// server, it will also be a test of how well the server deals with
// bogus client behavior.
GenericTestUtils.WaitFor(new _Supplier_458(cache, datanode), 10, 60000);
cluster.Shutdown();
sockDir.Close();
}
/// <summary>Allocate a new shared memory slot connected to this datanode.</summary>
/// <remarks>
/// Allocate a new shared memory slot connected to this datanode.
/// Must be called with the EndpointShmManager lock held.
/// </remarks>
/// <param name="peer">The peer to use to talk to the DataNode.</param>
/// <param name="usedPeer">
/// (out param) Will be set to true if we used the peer.
/// When a peer is used
/// </param>
/// <param name="clientName">The client name.</param>
/// <param name="blockId">The block ID to use.</param>
/// <returns>
/// null if the DataNode does not support shared memory
/// segments, or experienced an error creating the
/// shm. The shared memory segment itself on success.
/// </returns>
/// <exception cref="System.IO.IOException">If there was an error communicating over the socket.
/// </exception>
internal virtual ShortCircuitShm.Slot AllocSlot(DomainPeer peer, MutableBoolean usedPeer
, string clientName, ExtendedBlockId blockId)
{
while (true)
{
if (this._enclosing.closed)
{
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": the DfsClientShmManager has been closed."
);
}
return(null);
}
if (this.disabled)
{
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": shared memory segment access is disabled."
);
}
return(null);
}
// Try to use an existing slot.
ShortCircuitShm.Slot slot = this.AllocSlotFromExistingShm(blockId);
if (slot != null)
{
return(slot);
}
// There are no free slots. If someone is loading more slots, wait
// for that to finish.
if (this.loading)
{
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": waiting for loading to finish...");
}
this._enclosing.finishedLoading.AwaitUninterruptibly();
}
else
{
// Otherwise, load the slot ourselves.
this.loading = true;
this._enclosing.Lock.Unlock();
DfsClientShm shm;
try
{
shm = this.RequestNewShm(clientName, peer);
if (shm == null)
{
continue;
}
// See #{DfsClientShmManager#domainSocketWatcher} for details
// about why we do this before retaking the manager lock.
this._enclosing.domainSocketWatcher.Add(peer.GetDomainSocket(), shm);
// The DomainPeer is now our responsibility, and should not be
// closed by the caller.
usedPeer.SetValue(true);
}
finally
{
this._enclosing.Lock.Lock();
this.loading = false;
this._enclosing.finishedLoading.SignalAll();
}
if (shm.IsDisconnected())
{
// If the peer closed immediately after the shared memory segment
// was created, the DomainSocketWatcher callback might already have
// fired and marked the shm as disconnected. In this case, we
// obviously don't want to add the SharedMemorySegment to our list
// of valid not-full segments.
if (DfsClientShmManager.Log.IsDebugEnabled())
{
DfsClientShmManager.Log.Debug(this + ": the UNIX domain socket associated with "
+ "this short-circuit memory closed before we could make " + "use of the shm.");
}
}
else
{
this.notFull[shm.GetShmId()] = shm;
}
}
}
}
