/* * Replace block */ /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Net.Sockets.SocketException"/> private bool ReplaceBlock(ExtendedBlock block, DatanodeInfo source, DatanodeInfo sourceProxy, DatanodeInfo destination, StorageType targetStorageType) { Socket sock = new Socket(); try { sock.Connect(NetUtils.CreateSocketAddr(destination.GetXferAddr()), HdfsServerConstants .ReadTimeout); sock.SetKeepAlive(true); // sendRequest DataOutputStream @out = new DataOutputStream(sock.GetOutputStream()); new Sender(@out).ReplaceBlock(block, targetStorageType, BlockTokenSecretManager.DummyToken , source.GetDatanodeUuid(), sourceProxy); @out.Flush(); // receiveResponse DataInputStream reply = new DataInputStream(sock.GetInputStream()); DataTransferProtos.BlockOpResponseProto proto = DataTransferProtos.BlockOpResponseProto .ParseDelimitedFrom(reply); while (proto.GetStatus() == DataTransferProtos.Status.InProgress) { proto = DataTransferProtos.BlockOpResponseProto.ParseDelimitedFrom(reply); } return(proto.GetStatus() == DataTransferProtos.Status.Success); } finally { sock.Close(); } }
/// <exception cref="System.IO.IOException"/> public static void CheckBlockOpStatus(DataTransferProtos.BlockOpResponseProto response , string logInfo) { if (response.GetStatus() != DataTransferProtos.Status.Success) { if (response.GetStatus() == DataTransferProtos.Status.ErrorAccessToken) { throw new InvalidBlockTokenException("Got access token error" + ", status message " + response.GetMessage() + ", " + logInfo); } else { throw new IOException("Got error" + ", status message " + response.GetMessage() + ", " + logInfo); } } }
public virtual void TestTransferRbw() { HdfsConfiguration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(Replication ).Build(); try { cluster.WaitActive(); DistributedFileSystem fs = cluster.GetFileSystem(); //create a file, write some data and leave it open. Path p = new Path("/foo"); int size = (1 << 16) + Ran.Next(1 << 16); Log.Info("size = " + size); FSDataOutputStream @out = fs.Create(p, Replication); byte[] bytes = new byte[1024]; for (int remaining = size; remaining > 0;) { Ran.NextBytes(bytes); int len = bytes.Length < remaining ? bytes.Length : remaining; @out.Write(bytes, 0, len); @out.Hflush(); remaining -= len; } //get the RBW ReplicaBeingWritten oldrbw; DataNode newnode; DatanodeInfo newnodeinfo; string bpid = cluster.GetNamesystem().GetBlockPoolId(); { DataNode oldnode = cluster.GetDataNodes()[0]; oldrbw = GetRbw(oldnode, bpid); Log.Info("oldrbw = " + oldrbw); //add a datanode cluster.StartDataNodes(conf, 1, true, null, null); newnode = cluster.GetDataNodes()[Replication]; DatanodeInfo oldnodeinfo; { DatanodeInfo[] datatnodeinfos = cluster.GetNameNodeRpc().GetDatanodeReport(HdfsConstants.DatanodeReportType .Live); NUnit.Framework.Assert.AreEqual(2, datatnodeinfos.Length); int i = 0; for (DatanodeRegistration dnReg = newnode.GetDNRegistrationForBP(bpid); i < datatnodeinfos .Length && !datatnodeinfos[i].Equals(dnReg); i++) { } NUnit.Framework.Assert.IsTrue(i < datatnodeinfos.Length); newnodeinfo = datatnodeinfos[i]; oldnodeinfo = datatnodeinfos[1 - i]; } //transfer RBW ExtendedBlock b = new ExtendedBlock(bpid, oldrbw.GetBlockId(), oldrbw.GetBytesAcked (), oldrbw.GetGenerationStamp()); DataTransferProtos.BlockOpResponseProto s = DFSTestUtil.TransferRbw(b, DFSClientAdapter .GetDFSClient(fs), oldnodeinfo, newnodeinfo); NUnit.Framework.Assert.AreEqual(DataTransferProtos.Status.Success, s.GetStatus()); } //check new rbw ReplicaBeingWritten newrbw = GetRbw(newnode, bpid); Log.Info("newrbw = " + newrbw); NUnit.Framework.Assert.AreEqual(oldrbw.GetBlockId(), newrbw.GetBlockId()); NUnit.Framework.Assert.AreEqual(oldrbw.GetGenerationStamp(), newrbw.GetGenerationStamp ()); NUnit.Framework.Assert.AreEqual(oldrbw.GetVisibleLength(), newrbw.GetVisibleLength ()); Log.Info("DONE"); } finally { cluster.Shutdown(); } }
/// <summary>Request file descriptors from a DomainPeer.</summary> /// <param name="peer">The peer to use for communication.</param> /// <param name="slot"> /// If non-null, the shared memory slot to associate with the /// new ShortCircuitReplica. /// </param> /// <returns> /// A ShortCircuitReplica object if we could communicate with the /// datanode; null, otherwise. /// </returns> /// <exception cref="System.IO.IOException"> /// If we encountered an I/O exception while communicating /// with the datanode. /// </exception> private ShortCircuitReplicaInfo RequestFileDescriptors(DomainPeer peer, ShortCircuitShm.Slot slot) { ShortCircuitCache cache = clientContext.GetShortCircuitCache(); DataOutputStream @out = new DataOutputStream(new BufferedOutputStream(peer.GetOutputStream ())); ShortCircuitShm.SlotId slotId = slot == null ? null : slot.GetSlotId(); new Sender(@out).RequestShortCircuitFds(block, token, slotId, 1, failureInjector. GetSupportsReceiptVerification()); DataInputStream @in = new DataInputStream(peer.GetInputStream()); DataTransferProtos.BlockOpResponseProto resp = DataTransferProtos.BlockOpResponseProto .ParseFrom(PBHelper.VintPrefixed(@in)); DomainSocket sock = peer.GetDomainSocket(); failureInjector.InjectRequestFileDescriptorsFailure(); switch (resp.GetStatus()) { case DataTransferProtos.Status.Success: { byte[] buf = new byte[1]; FileInputStream[] fis = new FileInputStream[2]; sock.RecvFileInputStreams(fis, buf, 0, buf.Length); ShortCircuitReplica replica = null; try { ExtendedBlockId key = new ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId ()); if (buf[0] == DataTransferProtos.ShortCircuitFdResponse.UseReceiptVerification.GetNumber ()) { Log.Trace("Sending receipt verification byte for slot " + slot); sock.GetOutputStream().Write(0); } replica = new ShortCircuitReplica(key, fis[0], fis[1], cache, Time.MonotonicNow() , slot); return(new ShortCircuitReplicaInfo(replica)); } catch (IOException e) { // This indicates an error reading from disk, or a format error. Since // it's not a socket communication problem, we return null rather than // throwing an exception. Log.Warn(this + ": error creating ShortCircuitReplica.", e); return(null); } finally { if (replica == null) { IOUtils.Cleanup(DFSClient.Log, fis[0], fis[1]); } } goto case DataTransferProtos.Status.ErrorUnsupported; } case DataTransferProtos.Status.ErrorUnsupported: { if (!resp.HasShortCircuitAccessVersion()) { Log.Warn("short-circuit read access is disabled for " + "DataNode " + datanode + ". reason: " + resp.GetMessage()); clientContext.GetDomainSocketFactory().DisableShortCircuitForPath(pathInfo.GetPath ()); } else { Log.Warn("short-circuit read access for the file " + fileName + " is disabled for DataNode " + datanode + ". reason: " + resp.GetMessage()); } return(null); } case DataTransferProtos.Status.ErrorAccessToken: { string msg = "access control error while " + "attempting to set up short-circuit access to " + fileName + resp.GetMessage(); if (Log.IsDebugEnabled()) { Log.Debug(this + ":" + msg); } return(new ShortCircuitReplicaInfo(new SecretManager.InvalidToken(msg))); } default: { Log.Warn(this + ": unknown response code " + resp.GetStatus() + " while attempting to set up short-circuit access. " + resp.GetMessage()); clientContext.GetDomainSocketFactory().DisableShortCircuitForPath(pathInfo.GetPath ()); return(null); } } }