/// <summary>Pull a slot out of a preexisting shared memory segment.</summary> /// <remarks> /// Pull a slot out of a preexisting shared memory segment. /// Must be called with the manager lock held. /// </remarks> /// <param name="blockId">The blockId to put inside the Slot object.</param> /// <returns> /// null if none of our shared memory segments contain a /// free slot; the slot object otherwise. /// </returns> private ShortCircuitShm.Slot AllocSlotFromExistingShm(ExtendedBlockId blockId) { if (this.notFull.IsEmpty()) { return(null); } KeyValuePair <ShortCircuitShm.ShmId, DfsClientShm> entry = this.notFull.FirstEntry (); DfsClientShm shm = entry.Value; ShortCircuitShm.ShmId shmId = shm.GetShmId(); ShortCircuitShm.Slot slot = shm.AllocAndRegisterSlot(blockId); if (shm.IsFull()) { if (DfsClientShmManager.Log.IsTraceEnabled()) { DfsClientShmManager.Log.Trace(this + ": pulled the last slot " + slot.GetSlotIdx( ) + " out of " + shm); } DfsClientShm removedShm = Sharpen.Collections.Remove(this.notFull, shmId); Preconditions.CheckState(removedShm == shm); this.full[shmId] = shm; } else { if (DfsClientShmManager.Log.IsTraceEnabled()) { DfsClientShmManager.Log.Trace(this + ": pulled slot " + slot.GetSlotIdx() + " out of " + shm); } } return(slot); }
/// <summary>Register a slot.</summary> /// <remarks> /// Register a slot. /// This function looks at a slot which has already been initialized (by /// another process), and registers it with us. Then, it returns the /// relevant Slot object. /// </remarks> /// <returns>The slot.</returns> /// <exception cref="Org.Apache.Hadoop.FS.InvalidRequestException"> /// If the slot index we're trying to allocate has not been /// initialized, or is already in use. /// </exception> public ShortCircuitShm.Slot RegisterSlot(int slotIdx, ExtendedBlockId blockId) { lock (this) { if (slotIdx < 0) { throw new InvalidRequestException(this + ": invalid negative slot " + "index " + slotIdx); } if (slotIdx >= slots.Length) { throw new InvalidRequestException(this + ": invalid slot " + "index " + slotIdx); } if (allocatedSlots.Get(slotIdx)) { throw new InvalidRequestException(this + ": slot " + slotIdx + " is already in use." ); } ShortCircuitShm.Slot slot = new ShortCircuitShm.Slot(this, CalculateSlotAddress(slotIdx ), blockId); if (!slot.IsValid()) { throw new InvalidRequestException(this + ": slot " + slotIdx + " is not marked as valid." ); } slots[slotIdx] = slot; allocatedSlots.Set(slotIdx, true); if (Log.IsTraceEnabled()) { Log.Trace(this + ": registerSlot " + slotIdx + ": allocatedSlots=" + allocatedSlots + StringUtils.GetStackTrace(Sharpen.Thread.CurrentThread())); } return(slot); } }
/// <summary>Stop tracking a slot.</summary> /// <remarks> /// Stop tracking a slot. /// Must be called with the EndpointShmManager lock held. /// </remarks> /// <param name="slot">The slot to release.</param> internal virtual void FreeSlot(ShortCircuitShm.Slot slot) { DfsClientShm shm = (DfsClientShm)slot.GetShm(); shm.UnregisterSlot(slot.GetSlotIdx()); if (shm.IsDisconnected()) { // Stale shared memory segments should not be tracked here. Preconditions.CheckState(!this.full.Contains(shm.GetShmId())); Preconditions.CheckState(!this.notFull.Contains(shm.GetShmId())); if (shm.IsEmpty()) { if (DfsClientShmManager.Log.IsTraceEnabled()) { DfsClientShmManager.Log.Trace(this + ": freeing empty stale " + shm); } shm.Free(); } } else { ShortCircuitShm.ShmId shmId = shm.GetShmId(); Sharpen.Collections.Remove(this.full, shmId); // The shm can't be full if we just freed a slot. if (shm.IsEmpty()) { Sharpen.Collections.Remove(this.notFull, shmId); // If the shared memory segment is now empty, we call shutdown(2) on // the UNIX domain socket associated with it. The DomainSocketWatcher, // which is watching this socket, will call DfsClientShm#handle, // cleaning up this shared memory segment. // // See #{DfsClientShmManager#domainSocketWatcher} for details about why // we don't want to call DomainSocketWatcher#remove directly here. // // Note that we could experience 'fragmentation' here, where the // DFSClient allocates a bunch of slots in different shared memory // segments, and then frees most of them, but never fully empties out // any segment. We make some attempt to avoid this fragmentation by // always allocating new slots out of the shared memory segment with the // lowest ID, but it could still occur. In most workloads, // fragmentation should not be a major concern, since it doesn't impact // peak file descriptor usage or the speed of allocation. if (DfsClientShmManager.Log.IsTraceEnabled()) { DfsClientShmManager.Log.Trace(this + ": shutting down UNIX domain socket for " + "empty " + shm); } this.Shutdown(shm); } else { this.notFull[shmId] = shm; } } }
/// <exception cref="System.Exception"/> public virtual void TestAllocateSlots() { FilePath path = new FilePath(TestBase, "testAllocateSlots"); path.Mkdirs(); SharedFileDescriptorFactory factory = SharedFileDescriptorFactory.Create("shm_", new string[] { path.GetAbsolutePath() }); FileInputStream stream = factory.CreateDescriptor("testAllocateSlots", 4096); ShortCircuitShm shm = new ShortCircuitShm(ShortCircuitShm.ShmId.CreateRandom(), stream ); int numSlots = 0; AList <ShortCircuitShm.Slot> slots = new AList <ShortCircuitShm.Slot>(); while (!shm.IsFull()) { ShortCircuitShm.Slot slot = shm.AllocAndRegisterSlot(new ExtendedBlockId(123L, "test_bp1" )); slots.AddItem(slot); numSlots++; } Log.Info("allocated " + numSlots + " slots before running out."); int slotIdx = 0; for (IEnumerator <ShortCircuitShm.Slot> iter = shm.SlotIterator(); iter.HasNext();) { NUnit.Framework.Assert.IsTrue(slots.Contains(iter.Next())); } foreach (ShortCircuitShm.Slot slot_1 in slots) { NUnit.Framework.Assert.IsFalse(slot_1.AddAnchor()); NUnit.Framework.Assert.AreEqual(slotIdx++, slot_1.GetSlotIdx()); } foreach (ShortCircuitShm.Slot slot_2 in slots) { slot_2.MakeAnchorable(); } foreach (ShortCircuitShm.Slot slot_3 in slots) { NUnit.Framework.Assert.IsTrue(slot_3.AddAnchor()); } foreach (ShortCircuitShm.Slot slot_4 in slots) { slot_4.RemoveAnchor(); } foreach (ShortCircuitShm.Slot slot_5 in slots) { shm.UnregisterSlot(slot_5.GetSlotIdx()); slot_5.MakeInvalid(); } shm.Free(); stream.Close(); FileUtil.FullyDelete(path); }
public virtual void FreeSlot(ShortCircuitShm.Slot slot) { Lock.Lock(); try { DfsClientShm shm = (DfsClientShm)slot.GetShm(); shm.GetEndpointShmManager().FreeSlot(slot); } finally { Lock.Unlock(); } }
/// <exception cref="System.IO.IOException"/> public ShortCircuitReplica(ExtendedBlockId key, FileInputStream dataStream, FileInputStream metaStream, ShortCircuitCache cache, long creationTimeMs, ShortCircuitShm.Slot slot) { this.key = key; this.dataStream = dataStream; this.metaStream = metaStream; this.metaHeader = BlockMetadataHeader.PreadHeader(metaStream.GetChannel()); if (metaHeader.GetVersion() != 1) { throw new IOException("invalid metadata header version " + metaHeader.GetVersion( ) + ". Can only handle version 1."); } this.cache = cache; this.creationTimeMs = creationTimeMs; this.slot = slot; }
/// <exception cref="System.IO.IOException"/> public void Visit(Dictionary <DatanodeInfo, DfsClientShmManager.PerDatanodeVisitorInfo > info) { NUnit.Framework.Assert.IsTrue(info[datanode].full.IsEmpty()); NUnit.Framework.Assert.IsFalse(info[datanode].disabled); NUnit.Framework.Assert.AreEqual(1, info[datanode].notFull.Values.Count); DfsClientShm shm = info[datanode].notFull.Values.GetEnumerator().Next(); for (IEnumerator <ShortCircuitShm.Slot> iter = shm.SlotIterator(); iter.HasNext();) { ShortCircuitShm.Slot slot = iter.Next(); if (slot.IsValid()) { this._enclosing.done.SetValue(false); } } }
/// <summary> /// Handle the closure of the UNIX domain socket associated with this shared /// memory segment by marking this segment as stale. /// </summary> /// <remarks> /// Handle the closure of the UNIX domain socket associated with this shared /// memory segment by marking this segment as stale. /// If there are no slots associated with this shared memory segment, it will /// be freed immediately in this function. /// </remarks> public virtual bool Handle(DomainSocket sock) { manager.UnregisterShm(GetShmId()); lock (this) { Preconditions.CheckState(!disconnected); disconnected = true; bool hadSlots = false; for (IEnumerator <ShortCircuitShm.Slot> iter = SlotIterator(); iter.HasNext();) { ShortCircuitShm.Slot slot = iter.Next(); slot.MakeInvalid(); hadSlots = true; } if (!hadSlots) { Free(); } } return(true); }
/// <summary>Allocate a new slot and register it.</summary> /// <remarks> /// Allocate a new slot and register it. /// This function chooses an empty slot, initializes it, and then returns /// the relevant Slot object. /// </remarks> /// <returns>The new slot.</returns> public ShortCircuitShm.Slot AllocAndRegisterSlot(ExtendedBlockId blockId) { lock (this) { int idx = allocatedSlots.NextClearBit(0); if (idx >= slots.Length) { throw new RuntimeException(this + ": no more slots are available."); } allocatedSlots.Set(idx, true); ShortCircuitShm.Slot slot = new ShortCircuitShm.Slot(this, CalculateSlotAddress(idx ), blockId); slot.Clear(); slot.MakeValid(); slots[idx] = slot; if (Log.IsTraceEnabled()) { Log.Trace(this + ": allocAndRegisterSlot " + idx + ": allocatedSlots=" + allocatedSlots + StringUtils.GetStackTrace(Sharpen.Thread.CurrentThread())); } return(slot); } }
/// <exception cref="System.Exception"/> public virtual void TestAllocShm() { BlockReaderTestUtil.EnableShortCircuitShmTracing(); TemporarySocketDirectory sockDir = new TemporarySocketDirectory(); Configuration conf = CreateShortCircuitConf("testAllocShm", sockDir); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); DistributedFileSystem fs = cluster.GetFileSystem(); ShortCircuitCache cache = fs.GetClient().GetClientContext().GetShortCircuitCache( ); cache.GetDfsClientShmManager().Visit(new _Visitor_423()); // The ClientShmManager starts off empty DomainPeer peer = GetDomainPeerToDn(conf); MutableBoolean usedPeer = new MutableBoolean(false); ExtendedBlockId blockId = new ExtendedBlockId(123, "xyz"); DatanodeInfo datanode = new DatanodeInfo(cluster.GetDataNodes()[0].GetDatanodeId( )); // Allocating the first shm slot requires using up a peer. ShortCircuitShm.Slot slot = cache.AllocShmSlot(datanode, peer, usedPeer, blockId, "testAllocShm_client"); NUnit.Framework.Assert.IsNotNull(slot); NUnit.Framework.Assert.IsTrue(usedPeer.BooleanValue()); cache.GetDfsClientShmManager().Visit(new _Visitor_441(datanode)); // The ClientShmManager starts off empty cache.ScheduleSlotReleaser(slot); // Wait for the slot to be released, and the shared memory area to be // closed. Since we didn't register this shared memory segment on the // server, it will also be a test of how well the server deals with // bogus client behavior. GenericTestUtils.WaitFor(new _Supplier_458(cache, datanode), 10, 60000); cluster.Shutdown(); sockDir.Close(); }
/// <summary>Allocate a new shared memory slot connected to this datanode.</summary> /// <remarks> /// Allocate a new shared memory slot connected to this datanode. /// Must be called with the EndpointShmManager lock held. /// </remarks> /// <param name="peer">The peer to use to talk to the DataNode.</param> /// <param name="usedPeer"> /// (out param) Will be set to true if we used the peer. /// When a peer is used /// </param> /// <param name="clientName">The client name.</param> /// <param name="blockId">The block ID to use.</param> /// <returns> /// null if the DataNode does not support shared memory /// segments, or experienced an error creating the /// shm. The shared memory segment itself on success. /// </returns> /// <exception cref="System.IO.IOException">If there was an error communicating over the socket. /// </exception> internal virtual ShortCircuitShm.Slot AllocSlot(DomainPeer peer, MutableBoolean usedPeer , string clientName, ExtendedBlockId blockId) { while (true) { if (this._enclosing.closed) { if (DfsClientShmManager.Log.IsTraceEnabled()) { DfsClientShmManager.Log.Trace(this + ": the DfsClientShmManager has been closed." ); } return(null); } if (this.disabled) { if (DfsClientShmManager.Log.IsTraceEnabled()) { DfsClientShmManager.Log.Trace(this + ": shared memory segment access is disabled." ); } return(null); } // Try to use an existing slot. ShortCircuitShm.Slot slot = this.AllocSlotFromExistingShm(blockId); if (slot != null) { return(slot); } // There are no free slots. If someone is loading more slots, wait // for that to finish. if (this.loading) { if (DfsClientShmManager.Log.IsTraceEnabled()) { DfsClientShmManager.Log.Trace(this + ": waiting for loading to finish..."); } this._enclosing.finishedLoading.AwaitUninterruptibly(); } else { // Otherwise, load the slot ourselves. this.loading = true; this._enclosing.Lock.Unlock(); DfsClientShm shm; try { shm = this.RequestNewShm(clientName, peer); if (shm == null) { continue; } // See #{DfsClientShmManager#domainSocketWatcher} for details // about why we do this before retaking the manager lock. this._enclosing.domainSocketWatcher.Add(peer.GetDomainSocket(), shm); // The DomainPeer is now our responsibility, and should not be // closed by the caller. usedPeer.SetValue(true); } finally { this._enclosing.Lock.Lock(); this.loading = false; this._enclosing.finishedLoading.SignalAll(); } if (shm.IsDisconnected()) { // If the peer closed immediately after the shared memory segment // was created, the DomainSocketWatcher callback might already have // fired and marked the shm as disconnected. In this case, we // obviously don't want to add the SharedMemorySegment to our list // of valid not-full segments. if (DfsClientShmManager.Log.IsDebugEnabled()) { DfsClientShmManager.Log.Debug(this + ": the UNIX domain socket associated with " + "this short-circuit memory closed before we could make " + "use of the shm."); } } else { this.notFull[shm.GetShmId()] = shm; } } } }