/// <summary>Allocate a new shared memory slot connected to this datanode.</summary>
/// <remarks>
/// Allocate a new shared memory slot connected to this datanode.
/// Must be called with the EndpointShmManager lock held.
/// </remarks>
/// <param name="peer">The peer to use to talk to the DataNode.</param>
/// <param name="usedPeer">
/// (out param) Will be set to true if we used the peer.
/// When a peer is used
/// </param>
/// <param name="clientName">The client name.</param>
/// <param name="blockId">The block ID to use.</param>
/// <returns>
/// null if the DataNode does not support shared memory
/// segments, or experienced an error creating the
/// shm. The shared memory segment itself on success.
/// </returns>
/// <exception cref="System.IO.IOException">If there was an error communicating over the socket.
/// </exception>
internal virtual ShortCircuitShm.Slot AllocSlot(DomainPeer peer, MutableBoolean usedPeer
, string clientName, ExtendedBlockId blockId)
{
while (true)
{
if (this._enclosing.closed)
{
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": the DfsClientShmManager has been closed."
);
}
return(null);
}
if (this.disabled)
{
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": shared memory segment access is disabled."
);
}
return(null);
}
// Try to use an existing slot.
ShortCircuitShm.Slot slot = this.AllocSlotFromExistingShm(blockId);
if (slot != null)
{
return(slot);
}
// There are no free slots. If someone is loading more slots, wait
// for that to finish.
if (this.loading)
{
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": waiting for loading to finish...");
}
this._enclosing.finishedLoading.AwaitUninterruptibly();
}
else
{
// Otherwise, load the slot ourselves.
this.loading = true;
this._enclosing.Lock.Unlock();
DfsClientShm shm;
try
{
shm = this.RequestNewShm(clientName, peer);
if (shm == null)
{
continue;
}
// See #{DfsClientShmManager#domainSocketWatcher} for details
// about why we do this before retaking the manager lock.
this._enclosing.domainSocketWatcher.Add(peer.GetDomainSocket(), shm);
// The DomainPeer is now our responsibility, and should not be
// closed by the caller.
usedPeer.SetValue(true);
}
finally
{
this._enclosing.Lock.Lock();
this.loading = false;
this._enclosing.finishedLoading.SignalAll();
}
if (shm.IsDisconnected())
{
// If the peer closed immediately after the shared memory segment
// was created, the DomainSocketWatcher callback might already have
// fired and marked the shm as disconnected. In this case, we
// obviously don't want to add the SharedMemorySegment to our list
// of valid not-full segments.
if (DfsClientShmManager.Log.IsDebugEnabled())
{
DfsClientShmManager.Log.Debug(this + ": the UNIX domain socket associated with "
+ "this short-circuit memory closed before we could make " + "use of the shm.");
}
}
else
{
this.notFull[shm.GetShmId()] = shm;
}
}
}
}
/// <summary>Ask the DataNode for a new shared memory segment.</summary>
/// <remarks>
/// Ask the DataNode for a new shared memory segment. This function must be
/// called with the manager lock held. We will release the lock while
/// communicating with the DataNode.
/// </remarks>
/// <param name="clientName">The current client name.</param>
/// <param name="peer">The peer to use to talk to the DataNode.</param>
/// <returns>
/// Null if the DataNode does not support shared memory
/// segments, or experienced an error creating the
/// shm. The shared memory segment itself on success.
/// </returns>
/// <exception cref="System.IO.IOException">
/// If there was an error communicating over the socket.
/// We will not throw an IOException unless the socket
/// itself (or the network) is the problem.
/// </exception>
private DfsClientShm RequestNewShm(string clientName, DomainPeer peer)
{
DataOutputStream @out = new DataOutputStream(new BufferedOutputStream(peer.GetOutputStream
()));
new Sender(@out).RequestShortCircuitShm(clientName);
DataTransferProtos.ShortCircuitShmResponseProto resp = DataTransferProtos.ShortCircuitShmResponseProto
.ParseFrom(PBHelper.VintPrefixed(peer.GetInputStream()));
string error = resp.HasError() ? resp.GetError() : "(unknown)";
switch (resp.GetStatus())
{
case DataTransferProtos.Status.Success:
{
DomainSocket sock = peer.GetDomainSocket();
byte[] buf = new byte[1];
FileInputStream[] fis = new FileInputStream[1];
if (sock.RecvFileInputStreams(fis, buf, 0, buf.Length) < 0)
{
throw new EOFException("got EOF while trying to transfer the " + "file descriptor for the shared memory segment."
);
}
if (fis[0] == null)
{
throw new IOException("the datanode " + this.datanode + " failed to " + "pass a file descriptor for the shared memory segment."
);
}
try
{
DfsClientShm shm = new DfsClientShm(PBHelper.Convert(resp.GetId()), fis[0], this,
peer);
if (DfsClientShmManager.Log.IsTraceEnabled())
{
DfsClientShmManager.Log.Trace(this + ": createNewShm: created " + shm);
}
return(shm);
}
finally
{
IOUtils.Cleanup(DfsClientShmManager.Log, fis[0]);
}
goto case DataTransferProtos.Status.ErrorUnsupported;
}
case DataTransferProtos.Status.ErrorUnsupported:
{
// The DataNode just does not support short-circuit shared memory
// access, and we should stop asking.
DfsClientShmManager.Log.Info(this + ": datanode does not support short-circuit "
+ "shared memory access: " + error);
this.disabled = true;
return(null);
}
default:
{
// The datanode experienced some kind of unexpected error when trying to
// create the short-circuit shared memory segment.
DfsClientShmManager.Log.Warn(this + ": error requesting short-circuit shared memory "
+ "access: " + error);
return(null);
}
}
}
/// <summary>Request file descriptors from a DomainPeer.</summary>
/// <param name="peer">The peer to use for communication.</param>
/// <param name="slot">
/// If non-null, the shared memory slot to associate with the
/// new ShortCircuitReplica.
/// </param>
/// <returns>
/// A ShortCircuitReplica object if we could communicate with the
/// datanode; null, otherwise.
/// </returns>
/// <exception cref="System.IO.IOException">
/// If we encountered an I/O exception while communicating
/// with the datanode.
/// </exception>
private ShortCircuitReplicaInfo RequestFileDescriptors(DomainPeer peer, ShortCircuitShm.Slot
slot)
{
ShortCircuitCache cache = clientContext.GetShortCircuitCache();
DataOutputStream @out = new DataOutputStream(new BufferedOutputStream(peer.GetOutputStream
()));
ShortCircuitShm.SlotId slotId = slot == null ? null : slot.GetSlotId();
new Sender(@out).RequestShortCircuitFds(block, token, slotId, 1, failureInjector.
GetSupportsReceiptVerification());
DataInputStream @in = new DataInputStream(peer.GetInputStream());
DataTransferProtos.BlockOpResponseProto resp = DataTransferProtos.BlockOpResponseProto
.ParseFrom(PBHelper.VintPrefixed(@in));
DomainSocket sock = peer.GetDomainSocket();
failureInjector.InjectRequestFileDescriptorsFailure();
switch (resp.GetStatus())
{
case DataTransferProtos.Status.Success:
{
byte[] buf = new byte[1];
FileInputStream[] fis = new FileInputStream[2];
sock.RecvFileInputStreams(fis, buf, 0, buf.Length);
ShortCircuitReplica replica = null;
try
{
ExtendedBlockId key = new ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId
());
if (buf[0] == DataTransferProtos.ShortCircuitFdResponse.UseReceiptVerification.GetNumber
())
{
Log.Trace("Sending receipt verification byte for slot " + slot);
sock.GetOutputStream().Write(0);
}
replica = new ShortCircuitReplica(key, fis[0], fis[1], cache, Time.MonotonicNow()
, slot);
return(new ShortCircuitReplicaInfo(replica));
}
catch (IOException e)
{
// This indicates an error reading from disk, or a format error. Since
// it's not a socket communication problem, we return null rather than
// throwing an exception.
Log.Warn(this + ": error creating ShortCircuitReplica.", e);
return(null);
}
finally
{
if (replica == null)
{
IOUtils.Cleanup(DFSClient.Log, fis[0], fis[1]);
}
}
goto case DataTransferProtos.Status.ErrorUnsupported;
}
case DataTransferProtos.Status.ErrorUnsupported:
{
if (!resp.HasShortCircuitAccessVersion())
{
Log.Warn("short-circuit read access is disabled for " + "DataNode " + datanode +
". reason: " + resp.GetMessage());
clientContext.GetDomainSocketFactory().DisableShortCircuitForPath(pathInfo.GetPath
());
}
else
{
Log.Warn("short-circuit read access for the file " + fileName + " is disabled for DataNode "
+ datanode + ". reason: " + resp.GetMessage());
}
return(null);
}
case DataTransferProtos.Status.ErrorAccessToken:
{
string msg = "access control error while " + "attempting to set up short-circuit access to "
+ fileName + resp.GetMessage();
if (Log.IsDebugEnabled())
{
Log.Debug(this + ":" + msg);
}
return(new ShortCircuitReplicaInfo(new SecretManager.InvalidToken(msg)));
}
default:
{
Log.Warn(this + ": unknown response code " + resp.GetStatus() + " while attempting to set up short-circuit access. "
+ resp.GetMessage());
clientContext.GetDomainSocketFactory().DisableShortCircuitForPath(pathInfo.GetPath
());
return(null);
}
}
}
/// <summary>Get a RemoteBlockReader that communicates over a UNIX domain socket.</summary>
/// <returns>
/// The new BlockReader, or null if we failed to create the block
/// reader.
/// </returns>
/// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken">
/// If the block token was invalid.
/// Potentially other security-related execptions.
/// </exception>
/// <exception cref="System.IO.IOException"/>
private BlockReader GetRemoteBlockReaderFromDomain()
{
if (pathInfo == null)
{
pathInfo = clientContext.GetDomainSocketFactory().GetPathInfo(inetSocketAddress,
conf);
}
if (!pathInfo.GetPathState().GetUsableForDataTransfer())
{
PerformanceAdvisory.Log.Debug(this + ": not trying to create a " + "remote block reader because the UNIX domain socket at "
+ pathInfo + " is not usable.");
return(null);
}
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": trying to create a remote block reader from the " + "UNIX domain socket at "
+ pathInfo.GetPath());
}
while (true)
{
BlockReaderFactory.BlockReaderPeer curPeer = NextDomainPeer();
if (curPeer == null)
{
break;
}
if (curPeer.fromCache)
{
remainingCacheTries--;
}
DomainPeer peer = (DomainPeer)curPeer.peer;
BlockReader blockReader = null;
try
{
blockReader = GetRemoteBlockReader(peer);
return(blockReader);
}
catch (IOException ioe)
{
IOUtils.Cleanup(Log, peer);
if (IsSecurityException(ioe))
{
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": got security exception while constructing " + "a remote block reader from the unix domain socket at "
+ pathInfo.GetPath(), ioe);
}
throw;
}
if (curPeer.fromCache)
{
// Handle an I/O error we got when using a cached peer. These are
// considered less serious, because the underlying socket may be stale.
if (Log.IsDebugEnabled())
{
Log.Debug("Closed potentially stale domain peer " + peer, ioe);
}
}
else
{
// Handle an I/O error we got when using a newly created domain peer.
// We temporarily disable the domain socket path for a few minutes in
// this case, to prevent wasting more time on it.
Log.Warn("I/O error constructing remote block reader. Disabling " + "domain socket "
+ peer.GetDomainSocket(), ioe);
clientContext.GetDomainSocketFactory().DisableDomainSocketPath(pathInfo.GetPath()
);
return(null);
}
}
finally
{
if (blockReader == null)
{
IOUtils.Cleanup(Log, peer);
}
}
}
return(null);
}
/// <summary>Fetch a pair of short-circuit block descriptors from a local DataNode.</summary>
/// <returns>
/// Null if we could not communicate with the datanode,
/// a new ShortCircuitReplicaInfo object otherwise.
/// ShortCircuitReplicaInfo objects may contain either an InvalidToken
/// exception, or a ShortCircuitReplica object ready to use.
/// </returns>
public virtual ShortCircuitReplicaInfo CreateShortCircuitReplicaInfo()
{
if (createShortCircuitReplicaInfoCallback != null)
{
ShortCircuitReplicaInfo info = createShortCircuitReplicaInfoCallback.CreateShortCircuitReplicaInfo
();
if (info != null)
{
return(info);
}
}
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": trying to create ShortCircuitReplicaInfo.");
}
BlockReaderFactory.BlockReaderPeer curPeer;
while (true)
{
curPeer = NextDomainPeer();
if (curPeer == null)
{
break;
}
if (curPeer.fromCache)
{
remainingCacheTries--;
}
DomainPeer peer = (DomainPeer)curPeer.peer;
ShortCircuitShm.Slot slot = null;
ShortCircuitCache cache = clientContext.GetShortCircuitCache();
try
{
MutableBoolean usedPeer = new MutableBoolean(false);
slot = cache.AllocShmSlot(datanode, peer, usedPeer, new ExtendedBlockId(block.GetBlockId
(), block.GetBlockPoolId()), clientName);
if (usedPeer.BooleanValue())
{
if (Log.IsTraceEnabled())
{
Log.Trace(this + ": allocShmSlot used up our previous socket " + peer.GetDomainSocket
() + ". Allocating a new one...");
}
curPeer = NextDomainPeer();
if (curPeer == null)
{
break;
}
peer = (DomainPeer)curPeer.peer;
}
ShortCircuitReplicaInfo info = RequestFileDescriptors(peer, slot);
clientContext.GetPeerCache().Put(datanode, peer);
return(info);
}
catch (IOException e)
{
if (slot != null)
{
cache.FreeSlot(slot);
}
if (curPeer.fromCache)
{
// Handle an I/O error we got when using a cached socket.
// These are considered less serious, because the socket may be stale.
if (Log.IsDebugEnabled())
{
Log.Debug(this + ": closing stale domain peer " + peer, e);
}
IOUtils.Cleanup(Log, peer);
}
else
{
// Handle an I/O error we got when using a newly created socket.
// We temporarily disable the domain socket path for a few minutes in
// this case, to prevent wasting more time on it.
Log.Warn(this + ": I/O error requesting file descriptors. " + "Disabling domain socket "
+ peer.GetDomainSocket(), e);
IOUtils.Cleanup(Log, peer);
clientContext.GetDomainSocketFactory().DisableDomainSocketPath(pathInfo.GetPath()
);
return(null);
}
}
}
return(null);
}