/// <exception cref="System.IO.IOException"/> private BlockReaderLocalLegacy(DFSClient.Conf conf, string hdfsfile, ExtendedBlock block, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token, long startOffset, long length, BlockLocalPathInfo pathinfo, FileInputStream dataIn) : this(conf, hdfsfile, block, token, startOffset, length, pathinfo, DataChecksum. NewDataChecksum(DataChecksum.Type.Null, 4), false, dataIn, startOffset, null) { }
/// <exception cref="System.Exception"/> public virtual void TestBlockReaderLocalLegacyWithAppend() { short ReplFactor = 1; HdfsConfiguration conf = GetConfiguration(null); conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreaderlocal, true); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); Path path = new Path("/testBlockReaderLocalLegacy"); DFSTestUtil.CreateFile(dfs, path, 10, ReplFactor, 0); DFSTestUtil.WaitReplication(dfs, path, ReplFactor); ClientDatanodeProtocol proxy; Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token; ExtendedBlock originalBlock; long originalGS; { LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString (), 0, 1).Get(0); proxy = DFSUtil.CreateClientDatanodeProtocolProxy(lb.GetLocations()[0], conf, 60000 , false); token = lb.GetBlockToken(); // get block and generation stamp ExtendedBlock blk = new ExtendedBlock(lb.GetBlock()); originalBlock = new ExtendedBlock(blk); originalGS = originalBlock.GetGenerationStamp(); // test getBlockLocalPathInfo BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(blk, token); NUnit.Framework.Assert.AreEqual(originalGS, info.GetBlock().GetGenerationStamp()); } { // append one byte FSDataOutputStream @out = dfs.Append(path); @out.Write(1); @out.Close(); } { // get new generation stamp LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString (), 0, 1).Get(0); long newGS = lb.GetBlock().GetGenerationStamp(); NUnit.Framework.Assert.IsTrue(newGS > originalGS); // getBlockLocalPathInfo using the original block. NUnit.Framework.Assert.AreEqual(originalGS, originalBlock.GetGenerationStamp()); BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(originalBlock, token); NUnit.Framework.Assert.AreEqual(newGS, info.GetBlock().GetGenerationStamp()); } cluster.Shutdown(); }
/// <summary>Test assumes that the file has a single block</summary> /// <exception cref="System.IO.IOException"/> private FilePath GetBlockForFile(Path path, bool exists) { LocatedBlocks blocks = nn.GetRpcServer().GetBlockLocations(path.ToString(), 0, long.MaxValue ); NUnit.Framework.Assert.AreEqual("The test helper functions assume that each file has a single block" , 1, blocks.GetLocatedBlocks().Count); ExtendedBlock block = blocks.GetLocatedBlocks()[0].GetBlock(); BlockLocalPathInfo bInfo = dn0.GetFSDataset().GetBlockLocalPathInfo(block); FilePath blockFile = new FilePath(bInfo.GetBlockPath()); NUnit.Framework.Assert.AreEqual(exists, blockFile.Exists()); return(blockFile); }
public virtual void TestBlockRecoveryWithLessMetafile() { Configuration conf = new Configuration(); conf.Set(DFSConfigKeys.DfsBlockLocalPathAccessUserKey, UserGroupInformation.GetCurrentUser ().GetShortUserName()); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); Path file = new Path("/testRecoveryFile"); DistributedFileSystem dfs = cluster.GetFileSystem(); FSDataOutputStream @out = dfs.Create(file); int count = 0; while (count < 2 * 1024 * 1024) { @out.WriteBytes("Data"); count += 4; } @out.Hsync(); // abort the original stream ((DFSOutputStream)@out.GetWrappedStream()).Abort(); LocatedBlocks locations = cluster.GetNameNodeRpc().GetBlockLocations(file.ToString (), 0, count); ExtendedBlock block = locations.Get(0).GetBlock(); DataNode dn = cluster.GetDataNodes()[0]; BlockLocalPathInfo localPathInfo = dn.GetBlockLocalPathInfo(block, null); FilePath metafile = new FilePath(localPathInfo.GetMetaPath()); NUnit.Framework.Assert.IsTrue(metafile.Exists()); // reduce the block meta file size RandomAccessFile raf = new RandomAccessFile(metafile, "rw"); raf.SetLength(metafile.Length() - 20); raf.Close(); // restart DN to make replica to RWR MiniDFSCluster.DataNodeProperties dnProp = cluster.StopDataNode(0); cluster.RestartDataNode(dnProp, true); // try to recover the lease DistributedFileSystem newdfs = (DistributedFileSystem)FileSystem.NewInstance(cluster .GetConfiguration(0)); count = 0; while (++count < 10 && !newdfs.RecoverLease(file)) { Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsTrue("File should be closed", newdfs.RecoverLease(file)); }
/// <exception cref="System.IO.IOException"/> private BlockReaderLocalLegacy(DFSClient.Conf conf, string hdfsfile, ExtendedBlock block, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token, long startOffset, long length, BlockLocalPathInfo pathinfo, DataChecksum checksum, bool verifyChecksum, FileInputStream dataIn, long firstChunkOffset, FileInputStream checksumIn) { this.filename = hdfsfile; this.checksum = checksum; this.verifyChecksum = verifyChecksum; this.startOffset = Math.Max(startOffset, 0); this.blockId = block.GetBlockId(); bytesPerChecksum = this.checksum.GetBytesPerChecksum(); checksumSize = this.checksum.GetChecksumSize(); this.dataIn = dataIn; this.checksumIn = checksumIn; this.offsetFromChunkBoundary = (int)(startOffset - firstChunkOffset); int chunksPerChecksumRead = GetSlowReadBufferNumChunks(conf.shortCircuitBufferSize , bytesPerChecksum); slowReadBuff = bufferPool.GetBuffer(bytesPerChecksum * chunksPerChecksumRead); checksumBuff = bufferPool.GetBuffer(checksumSize * chunksPerChecksumRead); // Initially the buffers have nothing to read. slowReadBuff.Flip(); checksumBuff.Flip(); bool success = false; try { // Skip both input streams to beginning of the chunk containing startOffset IOUtils.SkipFully(dataIn, firstChunkOffset); if (checksumIn != null) { long checkSumOffset = (firstChunkOffset / bytesPerChecksum) * checksumSize; IOUtils.SkipFully(checksumIn, checkSumOffset); } success = true; } finally { if (!success) { bufferPool.ReturnBuffer(slowReadBuff); bufferPool.ReturnBuffer(checksumBuff); } } }
/// <exception cref="System.IO.IOException"/> private static BlockLocalPathInfo GetBlockPathInfo(UserGroupInformation ugi, ExtendedBlock blk, DatanodeInfo node, Configuration conf, int timeout, Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token, bool connectToDnViaHostname, StorageType storageType ) { BlockReaderLocalLegacy.LocalDatanodeInfo localDatanodeInfo = GetLocalDatanodeInfo (node.GetIpcPort()); BlockLocalPathInfo pathinfo = null; ClientDatanodeProtocol proxy = localDatanodeInfo.GetDatanodeProxy(ugi, node, conf , timeout, connectToDnViaHostname); try { // make RPC to local datanode to find local pathnames of blocks pathinfo = proxy.GetBlockLocalPathInfo(blk, token); // We cannot cache the path information for a replica on transient storage. // If the replica gets evicted, then it moves to a different path. Then, // our next attempt to read from the cached path would fail to find the // file. Additionally, the failure would cause us to disable legacy // short-circuit read for all subsequent use in the ClientContext. Unlike // the newer short-circuit read implementation, we have no communication // channel for the DataNode to notify the client that the path has been // invalidated. Therefore, our only option is to skip caching. if (pathinfo != null && !storageType.IsTransient()) { if (Log.IsDebugEnabled()) { Log.Debug("Cached location of block " + blk + " as " + pathinfo); } localDatanodeInfo.SetBlockLocalPathInfo(blk, pathinfo); } } catch (IOException e) { localDatanodeInfo.ResetDatanodeProxy(); // Reset proxy on error throw; } return(pathinfo); }
// Multiple datanodes could be running on the local machine. Store proxies in // a map keyed by the ipc port of the datanode. // reader for the data file // reader for the checksum file /// <summary>The only way this object can be instantiated.</summary> /// <exception cref="System.IO.IOException"/> internal static BlockReaderLocalLegacy NewBlockReader(DFSClient.Conf conf, UserGroupInformation userGroupInformation, Configuration configuration, string file, ExtendedBlock blk , Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token, DatanodeInfo node, long startOffset, long length, StorageType storageType) { BlockReaderLocalLegacy.LocalDatanodeInfo localDatanodeInfo = GetLocalDatanodeInfo (node.GetIpcPort()); // check the cache first BlockLocalPathInfo pathinfo = localDatanodeInfo.GetBlockLocalPathInfo(blk); if (pathinfo == null) { if (userGroupInformation == null) { userGroupInformation = UserGroupInformation.GetCurrentUser(); } pathinfo = GetBlockPathInfo(userGroupInformation, blk, node, configuration, conf. socketTimeout, token, conf.connectToDnViaHostname, storageType); } // check to see if the file exists. It may so happen that the // HDFS file has been deleted and this block-lookup is occurring // on behalf of a new HDFS file. This time, the block file could // be residing in a different portion of the fs.data.dir directory. // In this case, we remove this entry from the cache. The next // call to this method will re-populate the cache. FileInputStream dataIn = null; FileInputStream checksumIn = null; BlockReaderLocalLegacy localBlockReader = null; bool skipChecksumCheck = conf.skipShortCircuitChecksums || storageType.IsTransient (); try { // get a local file system FilePath blkfile = new FilePath(pathinfo.GetBlockPath()); dataIn = new FileInputStream(blkfile); if (Log.IsDebugEnabled()) { Log.Debug("New BlockReaderLocalLegacy for file " + blkfile + " of size " + blkfile .Length() + " startOffset " + startOffset + " length " + length + " short circuit checksum " + !skipChecksumCheck); } if (!skipChecksumCheck) { // get the metadata file FilePath metafile = new FilePath(pathinfo.GetMetaPath()); checksumIn = new FileInputStream(metafile); DataChecksum checksum = BlockMetadataHeader.ReadDataChecksum(new DataInputStream( checksumIn), blk); long firstChunkOffset = startOffset - (startOffset % checksum.GetBytesPerChecksum ()); localBlockReader = new BlockReaderLocalLegacy(conf, file, blk, token, startOffset , length, pathinfo, checksum, true, dataIn, firstChunkOffset, checksumIn); } else { localBlockReader = new BlockReaderLocalLegacy(conf, file, blk, token, startOffset , length, pathinfo, dataIn); } } catch (IOException e) { // remove from cache localDatanodeInfo.RemoveBlockLocalPathInfo(blk); DFSClient.Log.Warn("BlockReaderLocalLegacy: Removing " + blk + " from cache because local file " + pathinfo.GetBlockPath() + " could not be opened."); throw; } finally { if (localBlockReader == null) { if (dataIn != null) { dataIn.Close(); } if (checksumIn != null) { checksumIn.Close(); } } } return(localBlockReader); }
private void SetBlockLocalPathInfo(ExtendedBlock b, BlockLocalPathInfo info) { cache[b] = info; }