/// <exception cref="System.IO.IOException"/> public override bool SeekToNewSource(long targetPos) { long sumsPos = GetChecksumFilePos(targetPos); fs.ReportChecksumFailure(file, datas, targetPos, sums, sumsPos); bool newDataSource = datas.SeekToNewSource(targetPos); return(sums.SeekToNewSource(sumsPos) || newDataSource); }
public virtual void TestRead() { MiniDFSCluster cluster = null; int numDataNodes = 2; Configuration conf = GetConf(numDataNodes); try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build(); cluster.WaitActive(); NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count); NameNode nn = cluster.GetNameNode(); NamenodeProtocols nnProto = nn.GetRpcServer(); BlockManager bm = nn.GetNamesystem().GetBlockManager(); BlockTokenSecretManager sm = bm.GetBlockTokenSecretManager(); // set a short token lifetime (1 second) initially SecurityTestUtil.SetBlockTokenLifetime(sm, 1000L); Path fileToRead = new Path(FileToRead); FileSystem fs = cluster.GetFileSystem(); CreateFile(fs, fileToRead); /* * setup for testing expiration handling of cached tokens */ // read using blockSeekTo(). Acquired tokens are cached in in1 FSDataInputStream in1 = fs.Open(fileToRead); NUnit.Framework.Assert.IsTrue(CheckFile1(in1)); // read using blockSeekTo(). Acquired tokens are cached in in2 FSDataInputStream in2 = fs.Open(fileToRead); NUnit.Framework.Assert.IsTrue(CheckFile1(in2)); // read using fetchBlockByteRange(). Acquired tokens are cached in in3 FSDataInputStream in3 = fs.Open(fileToRead); NUnit.Framework.Assert.IsTrue(CheckFile2(in3)); /* * testing READ interface on DN using a BlockReader */ DFSClient client = null; try { client = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()), conf ); } finally { if (client != null) { client.Close(); } } IList <LocatedBlock> locatedBlocks = nnProto.GetBlockLocations(FileToRead, 0, FileSize ).GetLocatedBlocks(); LocatedBlock lblock = locatedBlocks[0]; // first block Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> myToken = lblock.GetBlockToken (); // verify token is not expired NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(myToken)); // read with valid token, should succeed TryRead(conf, lblock, true); /* * wait till myToken and all cached tokens in in1, in2 and in3 expire */ while (!SecurityTestUtil.IsBlockTokenExpired(myToken)) { try { Sharpen.Thread.Sleep(10); } catch (Exception) { } } /* * continue testing READ interface on DN using a BlockReader */ // verify token is expired NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(myToken)); // read should fail TryRead(conf, lblock, false); // use a valid new token lblock.SetBlockToken(sm.GenerateToken(lblock.GetBlock(), EnumSet.Of(BlockTokenSecretManager.AccessMode .Read))); // read should succeed TryRead(conf, lblock, true); // use a token with wrong blockID ExtendedBlock wrongBlock = new ExtendedBlock(lblock.GetBlock().GetBlockPoolId(), lblock.GetBlock().GetBlockId() + 1); lblock.SetBlockToken(sm.GenerateToken(wrongBlock, EnumSet.Of(BlockTokenSecretManager.AccessMode .Read))); // read should fail TryRead(conf, lblock, false); // use a token with wrong access modes lblock.SetBlockToken(sm.GenerateToken(lblock.GetBlock(), EnumSet.Of(BlockTokenSecretManager.AccessMode .Write, BlockTokenSecretManager.AccessMode.Copy, BlockTokenSecretManager.AccessMode .Replace))); // read should fail TryRead(conf, lblock, false); // set a long token lifetime for future tokens SecurityTestUtil.SetBlockTokenLifetime(sm, 600 * 1000L); /* * testing that when cached tokens are expired, DFSClient will re-fetch * tokens transparently for READ. */ // confirm all tokens cached in in1 are expired by now IList <LocatedBlock> lblocks = DFSTestUtil.GetAllBlocks(in1); foreach (LocatedBlock blk in lblocks) { NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(blk.GetBlockToken ())); } // verify blockSeekTo() is able to re-fetch token transparently in1.Seek(0); NUnit.Framework.Assert.IsTrue(CheckFile1(in1)); // confirm all tokens cached in in2 are expired by now IList <LocatedBlock> lblocks2 = DFSTestUtil.GetAllBlocks(in2); foreach (LocatedBlock blk_1 in lblocks2) { NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(blk_1.GetBlockToken ())); } // verify blockSeekTo() is able to re-fetch token transparently (testing // via another interface method) NUnit.Framework.Assert.IsTrue(in2.SeekToNewSource(0)); NUnit.Framework.Assert.IsTrue(CheckFile1(in2)); // confirm all tokens cached in in3 are expired by now IList <LocatedBlock> lblocks3 = DFSTestUtil.GetAllBlocks(in3); foreach (LocatedBlock blk_2 in lblocks3) { NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(blk_2.GetBlockToken ())); } // verify fetchBlockByteRange() is able to re-fetch token transparently NUnit.Framework.Assert.IsTrue(CheckFile2(in3)); /* * testing that after datanodes are restarted on the same ports, cached * tokens should still work and there is no need to fetch new tokens from * namenode. This test should run while namenode is down (to make sure no * new tokens can be fetched from namenode). */ // restart datanodes on the same ports that they currently use NUnit.Framework.Assert.IsTrue(cluster.RestartDataNodes(true)); cluster.WaitActive(); NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count); cluster.ShutdownNameNode(0); // confirm tokens cached in in1 are still valid lblocks = DFSTestUtil.GetAllBlocks(in1); foreach (LocatedBlock blk_3 in lblocks) { NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(blk_3.GetBlockToken ())); } // verify blockSeekTo() still works (forced to use cached tokens) in1.Seek(0); NUnit.Framework.Assert.IsTrue(CheckFile1(in1)); // confirm tokens cached in in2 are still valid lblocks2 = DFSTestUtil.GetAllBlocks(in2); foreach (LocatedBlock blk_4 in lblocks2) { NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(blk_4.GetBlockToken ())); } // verify blockSeekTo() still works (forced to use cached tokens) in2.SeekToNewSource(0); NUnit.Framework.Assert.IsTrue(CheckFile1(in2)); // confirm tokens cached in in3 are still valid lblocks3 = DFSTestUtil.GetAllBlocks(in3); foreach (LocatedBlock blk_5 in lblocks3) { NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(blk_5.GetBlockToken ())); } // verify fetchBlockByteRange() still works (forced to use cached tokens) NUnit.Framework.Assert.IsTrue(CheckFile2(in3)); /* * testing that when namenode is restarted, cached tokens should still * work and there is no need to fetch new tokens from namenode. Like the * previous test, this test should also run while namenode is down. The * setup for this test depends on the previous test. */ // restart the namenode and then shut it down for test cluster.RestartNameNode(0); cluster.ShutdownNameNode(0); // verify blockSeekTo() still works (forced to use cached tokens) in1.Seek(0); NUnit.Framework.Assert.IsTrue(CheckFile1(in1)); // verify again blockSeekTo() still works (forced to use cached tokens) in2.SeekToNewSource(0); NUnit.Framework.Assert.IsTrue(CheckFile1(in2)); // verify fetchBlockByteRange() still works (forced to use cached tokens) NUnit.Framework.Assert.IsTrue(CheckFile2(in3)); /* * testing that after both namenode and datanodes got restarted (namenode * first, followed by datanodes), DFSClient can't access DN without * re-fetching tokens and is able to re-fetch tokens transparently. The * setup of this test depends on the previous test. */ // restore the cluster and restart the datanodes for test cluster.RestartNameNode(0); NUnit.Framework.Assert.IsTrue(cluster.RestartDataNodes(true)); cluster.WaitActive(); NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count); // shutdown namenode so that DFSClient can't get new tokens from namenode cluster.ShutdownNameNode(0); // verify blockSeekTo() fails (cached tokens become invalid) in1.Seek(0); NUnit.Framework.Assert.IsFalse(CheckFile1(in1)); // verify fetchBlockByteRange() fails (cached tokens become invalid) NUnit.Framework.Assert.IsFalse(CheckFile2(in3)); // restart the namenode to allow DFSClient to re-fetch tokens cluster.RestartNameNode(0); // verify blockSeekTo() works again (by transparently re-fetching // tokens from namenode) in1.Seek(0); NUnit.Framework.Assert.IsTrue(CheckFile1(in1)); in2.SeekToNewSource(0); NUnit.Framework.Assert.IsTrue(CheckFile1(in2)); // verify fetchBlockByteRange() works again (by transparently // re-fetching tokens from namenode) NUnit.Framework.Assert.IsTrue(CheckFile2(in3)); /* * testing that when datanodes are restarted on different ports, DFSClient * is able to re-fetch tokens transparently to connect to them */ // restart datanodes on newly assigned ports NUnit.Framework.Assert.IsTrue(cluster.RestartDataNodes(false)); cluster.WaitActive(); NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count); // verify blockSeekTo() is able to re-fetch token transparently in1.Seek(0); NUnit.Framework.Assert.IsTrue(CheckFile1(in1)); // verify blockSeekTo() is able to re-fetch token transparently in2.SeekToNewSource(0); NUnit.Framework.Assert.IsTrue(CheckFile1(in2)); // verify fetchBlockByteRange() is able to re-fetch token transparently NUnit.Framework.Assert.IsTrue(CheckFile2(in3)); } finally { if (cluster != null) { cluster.Shutdown(); } } }