private void UpdateDeletedBlockId(ExtendedBlock block) { lock (this) { ICollection <long> blockIds = deletedBlockIds[block.GetBlockPoolId()]; if (blockIds == null) { blockIds = new HashSet <long>(); deletedBlockIds[block.GetBlockPoolId()] = blockIds; } blockIds.AddItem(block.GetBlockId()); numDeletedBlocks++; if (numDeletedBlocks == MaxDeletedBlocks) { foreach (KeyValuePair <string, ICollection <long> > e in deletedBlockIds) { string bpid = e.Key; ICollection <long> bs = e.Value; fsdatasetImpl.RemoveDeletedBlocks(bpid, bs); bs.Clear(); } numDeletedBlocks = 0; } } }
/// <summary>Check if access should be allowed.</summary> /// <remarks> /// Check if access should be allowed. userID is not checked if null. This /// method doesn't check if token password is correct. It should be used only /// when token password has already been verified (e.g., in the RPC layer). /// </remarks> /// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken"/> public virtual void CheckAccess(BlockTokenIdentifier id, string userId, ExtendedBlock block, BlockTokenSecretManager.AccessMode mode) { if (Log.IsDebugEnabled()) { Log.Debug("Checking access for user="******", block=" + block + ", access mode=" + mode + " using " + id.ToString()); } if (userId != null && !userId.Equals(id.GetUserId())) { throw new SecretManager.InvalidToken("Block token with " + id.ToString() + " doesn't belong to user " + userId); } if (!id.GetBlockPoolId().Equals(block.GetBlockPoolId())) { throw new SecretManager.InvalidToken("Block token with " + id.ToString() + " doesn't apply to block " + block); } if (id.GetBlockId() != block.GetBlockId()) { throw new SecretManager.InvalidToken("Block token with " + id.ToString() + " doesn't apply to block " + block); } if (IsExpired(id.GetExpiryDate())) { throw new SecretManager.InvalidToken("Block token with " + id.ToString() + " is expired." ); } if (!id.GetAccessModes().Contains(mode)) { throw new SecretManager.InvalidToken("Block token with " + id.ToString() + " doesn't have " + mode + " permission"); } }
/// <exception cref="System.IO.IOException"/> public static void CheckMetaInfo(ExtendedBlock b, DataNode dn) { Block metainfo = DataNodeTestUtils.GetFSDataset(dn).GetStoredBlock(b.GetBlockPoolId (), b.GetBlockId()); NUnit.Framework.Assert.AreEqual(b.GetBlockId(), metainfo.GetBlockId()); NUnit.Framework.Assert.AreEqual(b.GetNumBytes(), metainfo.GetNumBytes()); }
/// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken"/> private BlockReader GetBlockReaderLocal() { if (Log.IsTraceEnabled()) { Log.Trace(this + ": trying to construct a BlockReaderLocal " + "for short-circuit reads." ); } if (pathInfo == null) { pathInfo = clientContext.GetDomainSocketFactory().GetPathInfo(inetSocketAddress, conf); } if (!pathInfo.GetPathState().GetUsableForShortCircuit()) { PerformanceAdvisory.Log.Debug(this + ": " + pathInfo + " is not " + "usable for short circuit; giving up on BlockReaderLocal." ); return(null); } ShortCircuitCache cache = clientContext.GetShortCircuitCache(); ExtendedBlockId key = new ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId ()); ShortCircuitReplicaInfo info = cache.FetchOrCreate(key, this); SecretManager.InvalidToken exc = info.GetInvalidTokenException(); if (exc != null) { if (Log.IsTraceEnabled()) { Log.Trace(this + ": got InvalidToken exception while trying to " + "construct BlockReaderLocal via " + pathInfo.GetPath()); } throw exc; } if (info.GetReplica() == null) { if (Log.IsTraceEnabled()) { PerformanceAdvisory.Log.Debug(this + ": failed to get " + "ShortCircuitReplica. Cannot construct " + "BlockReaderLocal via " + pathInfo.GetPath()); } return(null); } return(new BlockReaderLocal.Builder(conf).SetFilename(fileName).SetBlock(block).SetStartOffset (startOffset).SetShortCircuitReplica(info.GetReplica()).SetVerifyChecksum(verifyChecksum ).SetCachingStrategy(cachingStrategy).SetStorageType(storageType).Build()); }
/// <summary>Generate a block token for a specified user</summary> /// <exception cref="System.IO.IOException"/> public virtual Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> GenerateToken (string userId, ExtendedBlock block, EnumSet <BlockTokenSecretManager.AccessMode> modes) { BlockTokenIdentifier id = new BlockTokenIdentifier(userId, block.GetBlockPoolId() , block.GetBlockId(), modes); return(new Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier>(id, this)); }
/// <exception cref="System.IO.IOException"/> internal static void CheckSuccess(DataTransferProtos.BlockOpResponseProto status, Peer peer, ExtendedBlock block, string file) { string logInfo = "for OP_READ_BLOCK" + ", self=" + peer.GetLocalAddressString() + ", remote=" + peer.GetRemoteAddressString() + ", for file " + file + ", for pool " + block.GetBlockPoolId() + " block " + block.GetBlockId() + "_" + block.GetGenerationStamp (); DataTransferProtoUtil.CheckBlockOpStatus(status, logInfo); }
/// <exception cref="Org.Apache.Hadoop.Hdfs.Server.Datanode.ReplicaNotFoundException" /// /> private static Replica GetReplica(ExtendedBlock block, DataNode datanode) { Replica replica = datanode.data.GetReplica(block.GetBlockPoolId(), block.GetBlockId ()); if (replica == null) { throw new ReplicaNotFoundException(block); } return(replica); }
/// <summary>Convert an ExtendedBlock to a Json map.</summary> private static IDictionary <string, object> ToJsonMap(ExtendedBlock extendedblock) { if (extendedblock == null) { return(null); } IDictionary <string, object> m = new SortedDictionary <string, object>(); m["blockPoolId"] = extendedblock.GetBlockPoolId(); m["blockId"] = extendedblock.GetBlockId(); m["numBytes"] = extendedblock.GetNumBytes(); m["generationStamp"] = extendedblock.GetGenerationStamp(); return(m); }
public virtual void TestBlockTokenRpc() { Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null); Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken (block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>()); Server server = CreateMockDatanode(sm, token, conf); server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); UserGroupInformation ticket = UserGroupInformation.CreateRemoteUser(block3.ToString ()); ticket.AddToken(token); ClientDatanodeProtocol proxy = null; try { proxy = DFSUtil.CreateClientDatanodeProtocolProxy(addr, ticket, conf, NetUtils.GetDefaultSocketFactory (conf)); NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength (block3)); } finally { server.Stop(); if (proxy != null) { RPC.StopProxy(proxy); } } }
/// <summary> /// sendBlock() is used to read block and its metadata and stream the data to /// either a client or to another datanode. /// </summary> /// <param name="out">stream to which the block is written to</param> /// <param name="baseStream"> /// optional. if non-null, <code>out</code> is assumed to /// be a wrapper over this stream. This enables optimizations for /// sending the data, e.g. /// <see cref="Org.Apache.Hadoop.Net.SocketOutputStream.TransferToFully(Sharpen.FileChannel, long, int) /// "/> /// . /// </param> /// <param name="throttler">for sending data.</param> /// <returns>total bytes read, including checksum data.</returns> /// <exception cref="System.IO.IOException"/> internal virtual long SendBlock(DataOutputStream @out, OutputStream baseStream, DataTransferThrottler throttler) { TraceScope scope = Trace.StartSpan("sendBlock_" + block.GetBlockId(), Sampler.Never ); try { return(DoSendBlock(@out, baseStream, throttler)); } finally { scope.Close(); } }
/// <exception cref="System.IO.IOException"/> private BlockReaderLocalLegacy(DFSClient.Conf conf, string hdfsfile, ExtendedBlock block, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token, long startOffset, long length, BlockLocalPathInfo pathinfo, DataChecksum checksum, bool verifyChecksum, FileInputStream dataIn, long firstChunkOffset, FileInputStream checksumIn) { this.filename = hdfsfile; this.checksum = checksum; this.verifyChecksum = verifyChecksum; this.startOffset = Math.Max(startOffset, 0); this.blockId = block.GetBlockId(); bytesPerChecksum = this.checksum.GetBytesPerChecksum(); checksumSize = this.checksum.GetChecksumSize(); this.dataIn = dataIn; this.checksumIn = checksumIn; this.offsetFromChunkBoundary = (int)(startOffset - firstChunkOffset); int chunksPerChecksumRead = GetSlowReadBufferNumChunks(conf.shortCircuitBufferSize , bytesPerChecksum); slowReadBuff = bufferPool.GetBuffer(bytesPerChecksum * chunksPerChecksumRead); checksumBuff = bufferPool.GetBuffer(checksumSize * chunksPerChecksumRead); // Initially the buffers have nothing to read. slowReadBuff.Flip(); checksumBuff.Flip(); bool success = false; try { // Skip both input streams to beginning of the chunk containing startOffset IOUtils.SkipFully(dataIn, firstChunkOffset); if (checksumIn != null) { long checkSumOffset = (firstChunkOffset / bytesPerChecksum) * checksumSize; IOUtils.SkipFully(checksumIn, checkSumOffset); } success = true; } finally { if (!success) { bufferPool.ReturnBuffer(slowReadBuff); bufferPool.ReturnBuffer(checksumBuff); } } }
/* fill up a cluster with <code>numNodes</code> datanodes * whose used space to be <code>size</code> */ /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> /// <exception cref="Sharpen.TimeoutException"/> private static ExtendedBlock[][] GenerateBlocks(TestBalancerWithMultipleNameNodes.Suite s, long size) { ExtendedBlock[][] blocks = new ExtendedBlock[s.clients.Length][]; for (int n = 0; n < s.clients.Length; n++) { long fileLen = size / s.replication; CreateFile(s, n, fileLen); IList <LocatedBlock> locatedBlocks = s.clients[n].GetBlockLocations(FileName, 0, fileLen ).GetLocatedBlocks(); int numOfBlocks = locatedBlocks.Count; blocks[n] = new ExtendedBlock[numOfBlocks]; for (int i = 0; i < numOfBlocks; i++) { ExtendedBlock b = locatedBlocks[i].GetBlock(); blocks[n][i] = new ExtendedBlock(b.GetBlockPoolId(), b.GetBlockId(), b.GetNumBytes (), b.GetGenerationStamp()); } } return(blocks); }
/// <summary>Create a new BlockReader specifically to satisfy a read.</summary> /// <remarks> /// Create a new BlockReader specifically to satisfy a read. /// This method also sends the OP_READ_BLOCK request. /// </remarks> /// <param name="file">File location</param> /// <param name="block">The block object</param> /// <param name="blockToken">The block token for security</param> /// <param name="startOffset">The read offset, relative to block head</param> /// <param name="len">The number of bytes to read</param> /// <param name="bufferSize">The IO buffer size (not the client buffer size)</param> /// <param name="verifyChecksum">Whether to verify checksum</param> /// <param name="clientName">Client name</param> /// <returns>New BlockReader instance, or null on error.</returns> /// <exception cref="System.IO.IOException"/> public static Org.Apache.Hadoop.Hdfs.RemoteBlockReader NewBlockReader(string file , ExtendedBlock block, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier > blockToken, long startOffset, long len, int bufferSize, bool verifyChecksum, string clientName, Peer peer, DatanodeID datanodeID, PeerCache peerCache, CachingStrategy cachingStrategy) { // in and out will be closed when sock is closed (by the caller) DataOutputStream @out = new DataOutputStream(new BufferedOutputStream(peer.GetOutputStream ())); new Sender(@out).ReadBlock(block, blockToken, clientName, startOffset, len, verifyChecksum , cachingStrategy); // // Get bytes in block, set streams // DataInputStream @in = new DataInputStream(new BufferedInputStream(peer.GetInputStream (), bufferSize)); DataTransferProtos.BlockOpResponseProto status = DataTransferProtos.BlockOpResponseProto .ParseFrom(PBHelper.VintPrefixed(@in)); RemoteBlockReader2.CheckSuccess(status, peer, block, file); DataTransferProtos.ReadOpChecksumInfoProto checksumInfo = status.GetReadOpChecksumInfo (); DataChecksum checksum = DataTransferProtoUtil.FromProto(checksumInfo.GetChecksum( )); //Warning when we get CHECKSUM_NULL? // Read the first chunk offset. long firstChunkOffset = checksumInfo.GetChunkOffset(); if (firstChunkOffset < 0 || firstChunkOffset > startOffset || firstChunkOffset <= (startOffset - checksum.GetBytesPerChecksum())) { throw new IOException("BlockReader: error in first chunk offset (" + firstChunkOffset + ") startOffset is " + startOffset + " for file " + file); } return(new Org.Apache.Hadoop.Hdfs.RemoteBlockReader(file, block.GetBlockPoolId(), block.GetBlockId(), @in, checksum, verifyChecksum, startOffset, firstChunkOffset , len, peer, datanodeID, peerCache)); }
/// <summary>go to each block on the 2nd DataNode until it fails...</summary> /// <param name="path"/> /// <param name="size"/> /// <exception cref="System.IO.IOException"/> private void TriggerFailure(string path, long size) { NamenodeProtocols nn = cluster.GetNameNodeRpc(); IList <LocatedBlock> locatedBlocks = nn.GetBlockLocations(path, 0, size).GetLocatedBlocks (); foreach (LocatedBlock lb in locatedBlocks) { DatanodeInfo dinfo = lb.GetLocations()[1]; ExtendedBlock b = lb.GetBlock(); try { AccessBlock(dinfo, lb); } catch (IOException) { System.Console.Out.WriteLine("Failure triggered, on block: " + b.GetBlockId() + "; corresponding volume should be removed by now" ); break; } } }
public virtual void Handle(ExtendedBlock block, IOException e) { FsVolumeSpi volume = scanner.volume; if (e == null) { Log.Trace("Successfully scanned {} on {}", block, volume.GetBasePath()); return; } // If the block does not exist anymore, then it's not an error. if (!volume.GetDataset().Contains(block)) { Log.Debug("Volume {}: block {} is no longer in the dataset.", volume.GetBasePath( ), block); return; } // If the block exists, the exception may due to a race with write: // The BlockSender got an old block path in rbw. BlockReceiver removed // the rbw block from rbw to finalized but BlockSender tried to open the // file before BlockReceiver updated the VolumeMap. The state of the // block can be changed again now, so ignore this error here. If there // is a block really deleted by mistake, DirectoryScan should catch it. if (e is FileNotFoundException) { Log.Info("Volume {}: verification failed for {} because of " + "FileNotFoundException. This may be due to a race with write." , volume.GetBasePath(), block); return; } Log.Warn("Reporting bad {} on {}", block, volume.GetBasePath()); try { scanner.datanode.ReportBadBlocks(block); } catch (IOException) { // This is bad, but not bad enough to shut down the scanner. Log.Warn("Cannot report bad " + block.GetBlockId(), e); } }
/// <summary>Read from the block file into a buffer.</summary> /// <remarks> /// Read from the block file into a buffer. /// This function overwrites checksumBuf. It will increment dataPos. /// </remarks> /// <param name="buf"> /// The buffer to read into. May be dataBuf. /// The position and limit of this buffer should be set to /// multiples of the checksum size. /// </param> /// <param name="canSkipChecksum">True if we can skip checksumming.</param> /// <returns>Total bytes read. 0 on EOF.</returns> /// <exception cref="System.IO.IOException"/> private int FillBuffer(ByteBuffer buf, bool canSkipChecksum) { lock (this) { TraceScope scope = Trace.StartSpan("BlockReaderLocal#fillBuffer(" + block.GetBlockId () + ")", Sampler.Never); try { int total = 0; long startDataPos = dataPos; int startBufPos = buf.Position(); while (buf.HasRemaining()) { int nRead = dataIn.Read(buf, dataPos); if (nRead < 0) { break; } dataPos += nRead; total += nRead; } if (canSkipChecksum) { FreeChecksumBufIfExists(); return(total); } if (total > 0) { try { buf.Limit(buf.Position()); buf.Position(startBufPos); CreateChecksumBufIfNeeded(); int checksumsNeeded = (total + bytesPerChecksum - 1) / bytesPerChecksum; checksumBuf.Clear(); checksumBuf.Limit(checksumsNeeded * checksumSize); long checksumPos = BlockMetadataHeader.GetHeaderSize() + ((startDataPos / bytesPerChecksum ) * checksumSize); while (checksumBuf.HasRemaining()) { int nRead = checksumIn.Read(checksumBuf, checksumPos); if (nRead < 0) { throw new IOException("Got unexpected checksum file EOF at " + checksumPos + ", block file position " + startDataPos + " for " + "block " + block + " of file " + filename); } checksumPos += nRead; } checksumBuf.Flip(); checksum.VerifyChunkedSums(buf, checksumBuf, filename, startDataPos); } finally { buf.Position(buf.Limit()); } } return(total); } finally { scope.Close(); } } }
public virtual void TestBlockSynchronization() { int OrgFileSize = 3000; Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, BlockSize); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(5).Build(); cluster.WaitActive(); //create a file DistributedFileSystem dfs = cluster.GetFileSystem(); string filestr = "/foo"; Path filepath = new Path(filestr); DFSTestUtil.CreateFile(dfs, filepath, OrgFileSize, ReplicationNum, 0L); NUnit.Framework.Assert.IsTrue(dfs.Exists(filepath)); DFSTestUtil.WaitReplication(dfs, filepath, ReplicationNum); //get block info for the last block LocatedBlock locatedblock = TestInterDatanodeProtocol.GetLastLocatedBlock(dfs.dfs .GetNamenode(), filestr); DatanodeInfo[] datanodeinfos = locatedblock.GetLocations(); NUnit.Framework.Assert.AreEqual(ReplicationNum, datanodeinfos.Length); //connect to data nodes DataNode[] datanodes = new DataNode[ReplicationNum]; for (int i = 0; i < ReplicationNum; i++) { datanodes[i] = cluster.GetDataNode(datanodeinfos[i].GetIpcPort()); NUnit.Framework.Assert.IsTrue(datanodes[i] != null); } //verify Block Info ExtendedBlock lastblock = locatedblock.GetBlock(); DataNode.Log.Info("newblocks=" + lastblock); for (int i_1 = 0; i_1 < ReplicationNum; i_1++) { CheckMetaInfo(lastblock, datanodes[i_1]); } DataNode.Log.Info("dfs.dfs.clientName=" + dfs.dfs.clientName); cluster.GetNameNodeRpc().Append(filestr, dfs.dfs.clientName, new EnumSetWritable < CreateFlag>(EnumSet.Of(CreateFlag.Append))); // expire lease to trigger block recovery. WaitLeaseRecovery(cluster); Block[] updatedmetainfo = new Block[ReplicationNum]; long oldSize = lastblock.GetNumBytes(); lastblock = TestInterDatanodeProtocol.GetLastLocatedBlock(dfs.dfs.GetNamenode(), filestr).GetBlock(); long currentGS = lastblock.GetGenerationStamp(); for (int i_2 = 0; i_2 < ReplicationNum; i_2++) { updatedmetainfo[i_2] = DataNodeTestUtils.GetFSDataset(datanodes[i_2]).GetStoredBlock (lastblock.GetBlockPoolId(), lastblock.GetBlockId()); NUnit.Framework.Assert.AreEqual(lastblock.GetBlockId(), updatedmetainfo[i_2].GetBlockId ()); NUnit.Framework.Assert.AreEqual(oldSize, updatedmetainfo[i_2].GetNumBytes()); NUnit.Framework.Assert.AreEqual(currentGS, updatedmetainfo[i_2].GetGenerationStamp ()); } // verify that lease recovery does not occur when namenode is in safemode System.Console.Out.WriteLine("Testing that lease recovery cannot happen during safemode." ); filestr = "/foo.safemode"; filepath = new Path(filestr); dfs.Create(filepath, (short)1); cluster.GetNameNodeRpc().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter, false); NUnit.Framework.Assert.IsTrue(dfs.dfs.Exists(filestr)); DFSTestUtil.WaitReplication(dfs, filepath, (short)1); WaitLeaseRecovery(cluster); // verify that we still cannot recover the lease LeaseManager lm = NameNodeAdapter.GetLeaseManager(cluster.GetNamesystem()); NUnit.Framework.Assert.IsTrue("Found " + lm.CountLease() + " lease, expected 1", lm.CountLease() == 1); cluster.GetNameNodeRpc().SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave, false); }
public virtual void TestUpdatePipelineAfterDelete() { Configuration conf = new HdfsConfiguration(); Path file = new Path("/test-file"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); try { FileSystem fs = cluster.GetFileSystem(); NamenodeProtocols namenode = cluster.GetNameNodeRpc(); DFSOutputStream @out = null; try { // Create a file and make sure a block is allocated for it. @out = (DFSOutputStream)(fs.Create(file).GetWrappedStream()); @out.Write(1); @out.Hflush(); // Create a snapshot that includes the file. SnapshotTestHelper.CreateSnapshot((DistributedFileSystem)fs, new Path("/"), "s1"); // Grab the block info of this file for later use. FSDataInputStream @in = null; ExtendedBlock oldBlock = null; try { @in = fs.Open(file); oldBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock(); } finally { IOUtils.CloseStream(@in); } // Allocate a new block ID/gen stamp so we can simulate pipeline // recovery. string clientName = ((DistributedFileSystem)fs).GetClient().GetClientName(); LocatedBlock newLocatedBlock = namenode.UpdateBlockForPipeline(oldBlock, clientName ); ExtendedBlock newBlock = new ExtendedBlock(oldBlock.GetBlockPoolId(), oldBlock.GetBlockId (), oldBlock.GetNumBytes(), newLocatedBlock.GetBlock().GetGenerationStamp()); // Delete the file from the present FS. It will still exist the // previously-created snapshot. This will log an OP_DELETE for the // file in question. fs.Delete(file, true); // Simulate a pipeline recovery, wherein a new block is allocated // for the existing block, resulting in an OP_UPDATE_BLOCKS being // logged for the file in question. try { namenode.UpdatePipeline(clientName, oldBlock, newBlock, newLocatedBlock.GetLocations (), newLocatedBlock.GetStorageIDs()); } catch (IOException ioe) { // normal GenericTestUtils.AssertExceptionContains("does not exist or it is not under construction" , ioe); } // Make sure the NN can restart with the edit logs as we have them now. cluster.RestartNameNode(true); } finally { IOUtils.CloseStream(@out); } } finally { cluster.Shutdown(); } }
/// <summary>TC11: Racing rename</summary> /// <exception cref="System.Exception"/> private void TestTC11(bool appendToNewBlock) { Path p = new Path("/TC11/foo" + (appendToNewBlock ? "0" : "1")); System.Console.Out.WriteLine("p=" + p); //a. Create file and write one block of data. Close file. int len1 = (int)BlockSize; { FSDataOutputStream @out = fs.Create(p, false, buffersize, Replication, BlockSize); AppendTestUtil.Write(@out, 0, len1); @out.Close(); } //b. Reopen file in "append" mode. Append half block of data. FSDataOutputStream out_1 = appendToNewBlock ? fs.Append(p, EnumSet.Of(CreateFlag. Append, CreateFlag.NewBlock), 4096, null) : fs.Append(p); int len2 = (int)BlockSize / 2; AppendTestUtil.Write(out_1, len1, len2); out_1.Hflush(); //c. Rename file to file.new. Path pnew = new Path(p + ".new"); NUnit.Framework.Assert.IsTrue(fs.Rename(p, pnew)); //d. Close file handle that was opened in (b). out_1.Close(); //check block sizes long len = fs.GetFileStatus(pnew).GetLen(); LocatedBlocks locatedblocks = fs.dfs.GetNamenode().GetBlockLocations(pnew.ToString (), 0L, len); int numblock = locatedblocks.LocatedBlockCount(); for (int i = 0; i < numblock; i++) { LocatedBlock lb = locatedblocks.Get(i); ExtendedBlock blk = lb.GetBlock(); long size = lb.GetBlockSize(); if (i < numblock - 1) { NUnit.Framework.Assert.AreEqual(BlockSize, size); } foreach (DatanodeInfo datanodeinfo in lb.GetLocations()) { DataNode dn = cluster.GetDataNode(datanodeinfo.GetIpcPort()); Block metainfo = DataNodeTestUtils.GetFSDataset(dn).GetStoredBlock(blk.GetBlockPoolId (), blk.GetBlockId()); NUnit.Framework.Assert.AreEqual(size, metainfo.GetNumBytes()); } } }
public virtual void TestDataTransferProtocol() { Random random = new Random(); int oneMil = 1024 * 1024; Path file = new Path("dataprotocol.dat"); int numDataNodes = 1; Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsReplicationKey, numDataNodes); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes ).Build(); try { cluster.WaitActive(); datanode = cluster.GetFileSystem().GetDataNodeStats(HdfsConstants.DatanodeReportType .Live)[0]; dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr()); FileSystem fileSys = cluster.GetFileSystem(); int fileLen = Math.Min(conf.GetInt(DFSConfigKeys.DfsBlockSizeKey, 4096), 4096); CreateFile(fileSys, file, fileLen); // get the first blockid for the file ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file); string poolId = firstBlock.GetBlockPoolId(); long newBlockId = firstBlock.GetBlockId() + 1; recvBuf.Reset(); sendBuf.Reset(); // bad version recvOut.WriteShort((short)(DataTransferProtocol.DataTransferVersion - 1)); sendOut.WriteShort((short)(DataTransferProtocol.DataTransferVersion - 1)); SendRecvData("Wrong Version", true); // bad ops sendBuf.Reset(); sendOut.WriteShort((short)DataTransferProtocol.DataTransferVersion); sendOut.WriteByte(OP.WriteBlock.code - 1); SendRecvData("Wrong Op Code", true); /* Test OP_WRITE_BLOCK */ sendBuf.Reset(); DataChecksum badChecksum = Org.Mockito.Mockito.Spy(DefaultChecksum); Org.Mockito.Mockito.DoReturn(-1).When(badChecksum).GetBytesPerChecksum(); WriteBlock(poolId, newBlockId, badChecksum); recvBuf.Reset(); SendResponse(DataTransferProtos.Status.Error, null, null, recvOut); SendRecvData("wrong bytesPerChecksum while writing", true); sendBuf.Reset(); recvBuf.Reset(); WriteBlock(poolId, ++newBlockId, DefaultChecksum); PacketHeader hdr = new PacketHeader(4, 0, 100, false, -1 - random.Next(oneMil), false ); // size of packet // offset in block, // seqno // last packet // bad datalen hdr.Write(sendOut); SendResponse(DataTransferProtos.Status.Success, string.Empty, null, recvOut); new PipelineAck(100, new int[] { PipelineAck.CombineHeader(PipelineAck.ECN.Disabled , DataTransferProtos.Status.Error) }).Write(recvOut); SendRecvData("negative DATA_CHUNK len while writing block " + newBlockId, true); // test for writing a valid zero size block sendBuf.Reset(); recvBuf.Reset(); WriteBlock(poolId, ++newBlockId, DefaultChecksum); hdr = new PacketHeader(8, 0, 100, true, 0, false); // size of packet // OffsetInBlock // sequencenumber // lastPacketInBlock // chunk length hdr.Write(sendOut); sendOut.WriteInt(0); // zero checksum sendOut.Flush(); //ok finally write a block with 0 len SendResponse(DataTransferProtos.Status.Success, string.Empty, null, recvOut); new PipelineAck(100, new int[] { PipelineAck.CombineHeader(PipelineAck.ECN.Disabled , DataTransferProtos.Status.Success) }).Write(recvOut); SendRecvData("Writing a zero len block blockid " + newBlockId, false); /* Test OP_READ_BLOCK */ string bpid = cluster.GetNamesystem().GetBlockPoolId(); ExtendedBlock blk = new ExtendedBlock(bpid, firstBlock.GetLocalBlock()); long blkid = blk.GetBlockId(); // bad block id sendBuf.Reset(); recvBuf.Reset(); blk.SetBlockId(blkid - 1); sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen, true , CachingStrategy.NewDefaultStrategy()); SendRecvData("Wrong block ID " + newBlockId + " for read", false); // negative block start offset -1L sendBuf.Reset(); blk.SetBlockId(blkid); sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", -1L, fileLen, true , CachingStrategy.NewDefaultStrategy()); SendRecvData("Negative start-offset for read for block " + firstBlock.GetBlockId( ), false); // bad block start offset sendBuf.Reset(); sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", fileLen, fileLen, true, CachingStrategy.NewDefaultStrategy()); SendRecvData("Wrong start-offset for reading block " + firstBlock.GetBlockId(), false ); // negative length is ok. Datanode assumes we want to read the whole block. recvBuf.Reset(); ((DataTransferProtos.BlockOpResponseProto)DataTransferProtos.BlockOpResponseProto .NewBuilder().SetStatus(DataTransferProtos.Status.Success).SetReadOpChecksumInfo (DataTransferProtos.ReadOpChecksumInfoProto.NewBuilder().SetChecksum(DataTransferProtoUtil .ToProto(DefaultChecksum)).SetChunkOffset(0L)).Build()).WriteDelimitedTo(recvOut ); sendBuf.Reset(); sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, -1L - random. Next(oneMil), true, CachingStrategy.NewDefaultStrategy()); SendRecvData("Negative length for reading block " + firstBlock.GetBlockId(), false ); // length is more than size of block. recvBuf.Reset(); SendResponse(DataTransferProtos.Status.Error, null, "opReadBlock " + firstBlock + " received exception java.io.IOException: " + "Offset 0 and length 4097 don't match block " + firstBlock + " ( blockLen 4096 )", recvOut); sendBuf.Reset(); sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen + 1, true, CachingStrategy.NewDefaultStrategy()); SendRecvData("Wrong length for reading block " + firstBlock.GetBlockId(), false); //At the end of all this, read the file to make sure that succeeds finally. sendBuf.Reset(); sender.ReadBlock(blk, BlockTokenSecretManager.DummyToken, "cl", 0L, fileLen, true , CachingStrategy.NewDefaultStrategy()); ReadFile(fileSys, file, fileLen); } finally { cluster.Shutdown(); } }
/// <summary>The following test first creates a file.</summary> /// <remarks> /// The following test first creates a file. /// It verifies the block information from a datanode. /// Then, it updates the block with new information and verifies again. /// </remarks> /// <param name="useDnHostname">whether DNs should connect to other DNs by hostname</param> /// <exception cref="System.Exception"/> private void CheckBlockMetaDataInfo(bool useDnHostname) { MiniDFSCluster cluster = null; conf.SetBoolean(DFSConfigKeys.DfsDatanodeUseDnHostname, useDnHostname); if (useDnHostname) { // Since the mini cluster only listens on the loopback we have to // ensure the hostname used to access DNs maps to the loopback. We // do this by telling the DN to advertise localhost as its hostname // instead of the default hostname. conf.Set(DFSConfigKeys.DfsDatanodeHostNameKey, "localhost"); } try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).CheckDataNodeHostConfig (true).Build(); cluster.WaitActive(); //create a file DistributedFileSystem dfs = cluster.GetFileSystem(); string filestr = "/foo"; Path filepath = new Path(filestr); DFSTestUtil.CreateFile(dfs, filepath, 1024L, (short)3, 0L); NUnit.Framework.Assert.IsTrue(dfs.Exists(filepath)); //get block info LocatedBlock locatedblock = GetLastLocatedBlock(DFSClientAdapter.GetDFSClient(dfs ).GetNamenode(), filestr); DatanodeInfo[] datanodeinfo = locatedblock.GetLocations(); NUnit.Framework.Assert.IsTrue(datanodeinfo.Length > 0); //connect to a data node DataNode datanode = cluster.GetDataNode(datanodeinfo[0].GetIpcPort()); InterDatanodeProtocol idp = DataNodeTestUtils.CreateInterDatanodeProtocolProxy(datanode , datanodeinfo[0], conf, useDnHostname); // Stop the block scanners. datanode.GetBlockScanner().RemoveAllVolumeScanners(); //verify BlockMetaDataInfo ExtendedBlock b = locatedblock.GetBlock(); InterDatanodeProtocol.Log.Info("b=" + b + ", " + b.GetType()); CheckMetaInfo(b, datanode); long recoveryId = b.GetGenerationStamp() + 1; idp.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock(b, locatedblock. GetLocations(), recoveryId)); //verify updateBlock ExtendedBlock newblock = new ExtendedBlock(b.GetBlockPoolId(), b.GetBlockId(), b. GetNumBytes() / 2, b.GetGenerationStamp() + 1); idp.UpdateReplicaUnderRecovery(b, recoveryId, b.GetBlockId(), newblock.GetNumBytes ()); CheckMetaInfo(newblock, datanode); // Verify correct null response trying to init recovery for a missing block ExtendedBlock badBlock = new ExtendedBlock("fake-pool", b.GetBlockId(), 0, 0); NUnit.Framework.Assert.IsNull(idp.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock (badBlock, locatedblock.GetLocations(), recoveryId))); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary>Scan a block.</summary> /// <param name="cblock">The block to scan.</param> /// <param name="bytesPerSec">The bytes per second to scan at.</param> /// <returns> /// The length of the block that was scanned, or /// -1 if the block could not be scanned. /// </returns> private long ScanBlock(ExtendedBlock cblock, long bytesPerSec) { // 'cblock' has a valid blockId and block pool id, but we don't yet know the // genstamp the block is supposed to have. Ask the FsDatasetImpl for this // information. ExtendedBlock block = null; try { Block b = volume.GetDataset().GetStoredBlock(cblock.GetBlockPoolId(), cblock.GetBlockId ()); if (b == null) { Log.Info("FileNotFound while finding block {} on volume {}", cblock, volume.GetBasePath ()); } else { block = new ExtendedBlock(cblock.GetBlockPoolId(), b); } } catch (FileNotFoundException) { Log.Info("FileNotFoundException while finding block {} on volume {}", cblock, volume .GetBasePath()); } catch (IOException) { Log.Warn("I/O error while finding block {} on volume {}", cblock, volume.GetBasePath ()); } if (block == null) { return(-1); } // block not found. BlockSender blockSender = null; try { blockSender = new BlockSender(block, 0, -1, false, true, true, datanode, null, CachingStrategy .NewDropBehind()); throttler.SetBandwidth(bytesPerSec); long bytesRead = blockSender.SendBlock(nullStream, null, throttler); resultHandler.Handle(block, null); return(bytesRead); } catch (IOException e) { resultHandler.Handle(block, e); } finally { IOUtils.Cleanup(null, blockSender); } return(-1); }
/// <exception cref="System.IO.IOException"/> private int GetTrueReplication(MiniDFSCluster cluster, ExtendedBlock block) { int count = 0; foreach (DataNode dn in cluster.GetDataNodes()) { if (DataNodeTestUtils.GetFSDataset(dn).GetStoredBlock(block.GetBlockPoolId(), block .GetBlockId()) != null) { count++; } } return(count); }
/// <summary>try to access a block on a data node.</summary> /// <remarks>try to access a block on a data node. If fails - throws exception</remarks> /// <param name="datanode"/> /// <param name="lblock"/> /// <exception cref="System.IO.IOException"/> private void AccessBlock(DatanodeInfo datanode, LocatedBlock lblock) { IPEndPoint targetAddr = null; ExtendedBlock block = lblock.GetBlock(); targetAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr()); BlockReader blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)).SetInetSocketAddress (targetAddr).SetBlock(block).SetFileName(BlockReaderFactory.GetFileName(targetAddr , "test-blockpoolid", block.GetBlockId())).SetBlockToken(lblock.GetBlockToken()) .SetStartOffset(0).SetLength(-1).SetVerifyChecksum(true).SetClientName("TestDataNodeVolumeFailure" ).SetDatanodeInfo(datanode).SetCachingStrategy(CachingStrategy.NewDefaultStrategy ()).SetClientCacheContext(ClientContext.GetFromConf(conf)).SetConfiguration(conf ).SetRemotePeerFactory(new _RemotePeerFactory_422(this)).Build(); blockReader.Close(); }
/// <summary>Get a BlockReader for the given block.</summary> /// <exception cref="System.IO.IOException"/> public static BlockReader GetBlockReader(MiniDFSCluster cluster, LocatedBlock testBlock , int offset, int lenToRead) { IPEndPoint targetAddr = null; ExtendedBlock block = testBlock.GetBlock(); DatanodeInfo[] nodes = testBlock.GetLocations(); targetAddr = NetUtils.CreateSocketAddr(nodes[0].GetXferAddr()); DistributedFileSystem fs = cluster.GetFileSystem(); return(new BlockReaderFactory(fs.GetClient().GetConf()).SetInetSocketAddress(targetAddr ).SetBlock(block).SetFileName(targetAddr.ToString() + ":" + block.GetBlockId()). SetBlockToken(testBlock.GetBlockToken()).SetStartOffset(offset).SetLength(lenToRead ).SetVerifyChecksum(true).SetClientName("BlockReaderTestUtil").SetDatanodeInfo(nodes [0]).SetClientCacheContext(ClientContext.GetFromConf(fs.GetConf())).SetCachingStrategy (CachingStrategy.NewDefaultStrategy()).SetConfiguration(fs.GetConf()).SetAllowShortCircuitLocalReads (true).SetRemotePeerFactory(new _RemotePeerFactory_196(fs)).Build()); }
public virtual void TestUpdateReplicaUnderRecovery() { MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); cluster.WaitActive(); string bpid = cluster.GetNamesystem().GetBlockPoolId(); //create a file DistributedFileSystem dfs = cluster.GetFileSystem(); string filestr = "/foo"; Path filepath = new Path(filestr); DFSTestUtil.CreateFile(dfs, filepath, 1024L, (short)3, 0L); //get block info LocatedBlock locatedblock = GetLastLocatedBlock(DFSClientAdapter.GetDFSClient(dfs ).GetNamenode(), filestr); DatanodeInfo[] datanodeinfo = locatedblock.GetLocations(); NUnit.Framework.Assert.IsTrue(datanodeinfo.Length > 0); //get DataNode and FSDataset objects DataNode datanode = cluster.GetDataNode(datanodeinfo[0].GetIpcPort()); NUnit.Framework.Assert.IsTrue(datanode != null); //initReplicaRecovery ExtendedBlock b = locatedblock.GetBlock(); long recoveryid = b.GetGenerationStamp() + 1; long newlength = b.GetNumBytes() - 1; FsDatasetSpi <object> fsdataset = DataNodeTestUtils.GetFSDataset(datanode); ReplicaRecoveryInfo rri = fsdataset.InitReplicaRecovery(new BlockRecoveryCommand.RecoveringBlock (b, null, recoveryid)); //check replica ReplicaInfo replica = FsDatasetTestUtil.FetchReplicaInfo(fsdataset, bpid, b.GetBlockId ()); NUnit.Framework.Assert.AreEqual(HdfsServerConstants.ReplicaState.Rur, replica.GetState ()); //check meta data before update FsDatasetImpl.CheckReplicaFiles(replica); { //case "THIS IS NOT SUPPOSED TO HAPPEN" //with (block length) != (stored replica's on disk length). //create a block with same id and gs but different length. ExtendedBlock tmp = new ExtendedBlock(b.GetBlockPoolId(), rri.GetBlockId(), rri.GetNumBytes () - 1, rri.GetGenerationStamp()); try { //update should fail fsdataset.UpdateReplicaUnderRecovery(tmp, recoveryid, tmp.GetBlockId(), newlength ); NUnit.Framework.Assert.Fail(); } catch (IOException ioe) { System.Console.Out.WriteLine("GOOD: getting " + ioe); } } //update string storageID = fsdataset.UpdateReplicaUnderRecovery(new ExtendedBlock(b.GetBlockPoolId (), rri), recoveryid, rri.GetBlockId(), newlength); NUnit.Framework.Assert.IsTrue(storageID != null); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> public virtual void RunBlockReaderLocalTest(TestBlockReaderLocal.BlockReaderLocalTest test, bool checksum, long readahead) { Assume.AssumeThat(DomainSocket.GetLoadingFailureReason(), CoreMatchers.EqualTo(null )); MiniDFSCluster cluster = null; HdfsConfiguration conf = new HdfsConfiguration(); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, !checksum ); conf.SetLong(DFSConfigKeys.DfsBytesPerChecksumKey, TestBlockReaderLocal.BlockReaderLocalTest .BytesPerChecksum); conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "CRC32C"); conf.SetLong(DFSConfigKeys.DfsClientCacheReadahead, readahead); test.SetConfiguration(conf); FileInputStream dataIn = null; FileInputStream metaIn = null; Path TestPath = new Path("/a"); long RandomSeed = 4567L; BlockReaderLocal blockReaderLocal = null; FSDataInputStream fsIn = null; byte[] original = new byte[TestBlockReaderLocal.BlockReaderLocalTest.TestLength]; FileSystem fs = null; ShortCircuitShm shm = null; RandomAccessFile raf = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestBlockReaderLocal.BlockReaderLocalTest.TestLength , (short)1, RandomSeed); try { DFSTestUtil.WaitReplication(fs, TestPath, (short)1); } catch (Exception e) { NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: " + e); } catch (TimeoutException e) { NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: " + e); } fsIn = fs.Open(TestPath); IOUtils.ReadFully(fsIn, original, 0, TestBlockReaderLocal.BlockReaderLocalTest.TestLength ); fsIn.Close(); fsIn = null; ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, TestPath); FilePath dataFile = cluster.GetBlockFile(0, block); FilePath metaFile = cluster.GetBlockMetadataFile(0, block); ShortCircuitCache shortCircuitCache = ClientContext.GetFromConf(conf).GetShortCircuitCache (); cluster.Shutdown(); cluster = null; test.Setup(dataFile, checksum); FileInputStream[] streams = new FileInputStream[] { new FileInputStream(dataFile) , new FileInputStream(metaFile) }; dataIn = streams[0]; metaIn = streams[1]; ExtendedBlockId key = new ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId ()); raf = new RandomAccessFile(new FilePath(sockDir.GetDir().GetAbsolutePath(), UUID. RandomUUID().ToString()), "rw"); raf.SetLength(8192); FileInputStream shmStream = new FileInputStream(raf.GetFD()); shm = new ShortCircuitShm(ShortCircuitShm.ShmId.CreateRandom(), shmStream); ShortCircuitReplica replica = new ShortCircuitReplica(key, dataIn, metaIn, shortCircuitCache , Time.Now(), shm.AllocAndRegisterSlot(ExtendedBlockId.FromExtendedBlock(block)) ); blockReaderLocal = new BlockReaderLocal.Builder(new DFSClient.Conf(conf)).SetFilename (TestPath.GetName()).SetBlock(block).SetShortCircuitReplica(replica).SetCachingStrategy (new CachingStrategy(false, readahead)).SetVerifyChecksum(checksum).Build(); dataIn = null; metaIn = null; test.DoTest(blockReaderLocal, original); // BlockReaderLocal should not alter the file position. NUnit.Framework.Assert.AreEqual(0, streams[0].GetChannel().Position()); NUnit.Framework.Assert.AreEqual(0, streams[1].GetChannel().Position()); } finally { if (fsIn != null) { fsIn.Close(); } if (fs != null) { fs.Close(); } if (cluster != null) { cluster.Shutdown(); } if (dataIn != null) { dataIn.Close(); } if (metaIn != null) { metaIn.Close(); } if (blockReaderLocal != null) { blockReaderLocal.Close(); } if (shm != null) { shm.Free(); } if (raf != null) { raf.Close(); } } }
public virtual void TestOpWrite() { int numDataNodes = 1; long BlockIdFudge = 128; Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes ).Build(); try { cluster.WaitActive(); string poolId = cluster.GetNamesystem().GetBlockPoolId(); datanode = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes()[0], poolId ); dnAddr = NetUtils.CreateSocketAddr(datanode.GetXferAddr()); FileSystem fileSys = cluster.GetFileSystem(); /* Test writing to finalized replicas */ Path file = new Path("dataprotocol.dat"); DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L); // get the first blockid for the file ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file); // test PIPELINE_SETUP_CREATE on a finalized block TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create an existing block" , true); // test PIPELINE_DATA_STREAMING on a finalized block TestWrite(firstBlock, BlockConstructionStage.DataStreaming, 0L, "Unexpected stage" , true); // test PIPELINE_SETUP_STREAMING_RECOVERY on an existing block long newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS , "Cannot recover data streaming to a finalized replica", true); // test PIPELINE_SETUP_APPEND on an existing block newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Append to a finalized replica" , false); firstBlock.SetGenerationStamp(newGS); // test PIPELINE_SETUP_APPEND_RECOVERY on an existing block file = new Path("dataprotocol1.dat"); DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L); firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file); newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Recover appending to a finalized replica", false); // test PIPELINE_CLOSE_RECOVERY on an existing block file = new Path("dataprotocol2.dat"); DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L); firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file); newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineCloseRecovery, newGS, "Recover failed close to a finalized replica" , false); firstBlock.SetGenerationStamp(newGS); // Test writing to a new block. Don't choose the next sequential // block ID to avoid conflicting with IDs chosen by the NN. long newBlockId = firstBlock.GetBlockId() + BlockIdFudge; ExtendedBlock newBlock = new ExtendedBlock(firstBlock.GetBlockPoolId(), newBlockId , 0, firstBlock.GetGenerationStamp()); // test PIPELINE_SETUP_CREATE on a new block TestWrite(newBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Create a new block" , false); // test PIPELINE_SETUP_STREAMING_RECOVERY on a new block newGS = newBlock.GetGenerationStamp() + 1; newBlock.SetBlockId(newBlock.GetBlockId() + 1); TestWrite(newBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS, "Recover a new block", true); // test PIPELINE_SETUP_APPEND on a new block newGS = newBlock.GetGenerationStamp() + 1; TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a new block" , true); // test PIPELINE_SETUP_APPEND_RECOVERY on a new block newBlock.SetBlockId(newBlock.GetBlockId() + 1); newGS = newBlock.GetGenerationStamp() + 1; TestWrite(newBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Cannot append to a new block" , true); /* Test writing to RBW replicas */ Path file1 = new Path("dataprotocol1.dat"); DFSTestUtil.CreateFile(fileSys, file1, 1L, (short)numDataNodes, 0L); DFSOutputStream @out = (DFSOutputStream)(fileSys.Append(file1).GetWrappedStream() ); @out.Write(1); @out.Hflush(); FSDataInputStream @in = fileSys.Open(file1); firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock(); firstBlock.SetNumBytes(2L); try { // test PIPELINE_SETUP_CREATE on a RBW block TestWrite(firstBlock, BlockConstructionStage.PipelineSetupCreate, 0L, "Cannot create a RBW block" , true); // test PIPELINE_SETUP_APPEND on an existing block newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppend, newGS, "Cannot append to a RBW replica" , true); // test PIPELINE_SETUP_APPEND on an existing block TestWrite(firstBlock, BlockConstructionStage.PipelineSetupAppendRecovery, newGS, "Recover append to a RBW replica", false); firstBlock.SetGenerationStamp(newGS); // test PIPELINE_SETUP_STREAMING_RECOVERY on a RBW block file = new Path("dataprotocol2.dat"); DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L); @out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream()); @out.Write(1); @out.Hflush(); @in = fileSys.Open(file); firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock(); firstBlock.SetNumBytes(2L); newGS = firstBlock.GetGenerationStamp() + 1; TestWrite(firstBlock, BlockConstructionStage.PipelineSetupStreamingRecovery, newGS , "Recover a RBW replica", false); } finally { IOUtils.CloseStream(@in); IOUtils.CloseStream(@out); } } finally { cluster.Shutdown(); } }
public virtual void TestGetNewStamp() { int numDataNodes = 1; Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes ).Build(); try { cluster.WaitActive(); FileSystem fileSys = cluster.GetFileSystem(); NamenodeProtocols namenode = cluster.GetNameNodeRpc(); /* Test writing to finalized replicas */ Path file = new Path("dataprotocol.dat"); DFSTestUtil.CreateFile(fileSys, file, 1L, (short)numDataNodes, 0L); // get the first blockid for the file ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fileSys, file); // test getNewStampAndToken on a finalized block try { namenode.UpdateBlockForPipeline(firstBlock, string.Empty); NUnit.Framework.Assert.Fail("Can not get a new GS from a finalized block"); } catch (IOException e) { NUnit.Framework.Assert.IsTrue(e.Message.Contains("is not under Construction")); } // test getNewStampAndToken on a non-existent block try { long newBlockId = firstBlock.GetBlockId() + 1; ExtendedBlock newBlock = new ExtendedBlock(firstBlock.GetBlockPoolId(), newBlockId , 0, firstBlock.GetGenerationStamp()); namenode.UpdateBlockForPipeline(newBlock, string.Empty); NUnit.Framework.Assert.Fail("Cannot get a new GS from a non-existent block"); } catch (IOException e) { NUnit.Framework.Assert.IsTrue(e.Message.Contains("does not exist")); } /* Test RBW replicas */ // change first block to a RBW DFSOutputStream @out = null; try { @out = (DFSOutputStream)(fileSys.Append(file).GetWrappedStream()); @out.Write(1); @out.Hflush(); FSDataInputStream @in = null; try { @in = fileSys.Open(file); firstBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock(); } finally { IOUtils.CloseStream(@in); } // test non-lease holder DFSClient dfs = ((DistributedFileSystem)fileSys).dfs; try { namenode.UpdateBlockForPipeline(firstBlock, "test" + dfs.clientName); NUnit.Framework.Assert.Fail("Cannot get a new GS for a non lease holder"); } catch (LeaseExpiredException e) { NUnit.Framework.Assert.IsTrue(e.Message.StartsWith("Lease mismatch")); } // test null lease holder try { namenode.UpdateBlockForPipeline(firstBlock, null); NUnit.Framework.Assert.Fail("Cannot get a new GS for a null lease holder"); } catch (LeaseExpiredException e) { NUnit.Framework.Assert.IsTrue(e.Message.StartsWith("Lease mismatch")); } // test getNewStampAndToken on a rbw block namenode.UpdateBlockForPipeline(firstBlock, dfs.clientName); } finally { IOUtils.CloseStream(@out); } } finally { cluster.Shutdown(); } }
public static Org.Apache.Hadoop.Hdfs.ExtendedBlockId FromExtendedBlock(ExtendedBlock block) { return(new Org.Apache.Hadoop.Hdfs.ExtendedBlockId(block.GetBlockId(), block.GetBlockPoolId ())); }