public virtual void TestScheduleSameBlock() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build(); try { cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); string file = "/testScheduleSameBlock/file"; { FSDataOutputStream @out = dfs.Create(new Path(file)); @out.WriteChars("testScheduleSameBlock"); @out.Close(); } Org.Apache.Hadoop.Hdfs.Server.Mover.Mover mover = NewMover(conf); mover.Init(); Mover.Processor processor = new Mover.Processor(this); LocatedBlock lb = dfs.GetClient().GetLocatedBlocks(file, 0).Get(0); IList <Mover.MLocation> locations = Mover.MLocation.ToLocations(lb); Mover.MLocation ml = locations[0]; Dispatcher.DBlock db = mover.NewDBlock(lb.GetBlock().GetLocalBlock(), locations); IList <StorageType> storageTypes = new AList <StorageType>(Arrays.AsList(StorageType .Default, StorageType.Default)); NUnit.Framework.Assert.IsTrue(processor.ScheduleMoveReplica(db, ml, storageTypes) ); NUnit.Framework.Assert.IsFalse(processor.ScheduleMoveReplica(db, ml, storageTypes )); } finally { cluster.Shutdown(); } }
public LocatedBlock AddBlock(string srcFile) { var randomDataNodeID = _dataNodeRepository.GetRandomDataNodeId(); var dataNodeDescriptor = _dataNodeRepository.GetDataNodeDescriptorById(randomDataNodeID); var dataNodeID = new DataNodeId { HostName = dataNodeDescriptor.HostName, IPAddress = dataNodeDescriptor.IPAddress }; var dataNodeIds = new List <DataNodeId> { dataNodeID }; var blockId = Guid.NewGuid(); var block = new Block(blockId, 0, DateTime.Now); var blockInfo = new BlockInfo(block, dataNodeIds); var node = _fileSystem.GetFile(srcFile); node.AddBlock(blockInfo); var locatedBlock = new LocatedBlock { Block = block, Locations = dataNodeIds }; return(locatedBlock); }
/// <summary>Create a file with one block and corrupt some/all of the block replicas. /// </summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/> /// <exception cref="System.IO.FileNotFoundException"/> /// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/> /// <exception cref="System.Exception"/> /// <exception cref="Sharpen.TimeoutException"/> private void CreateAFileWithCorruptedBlockReplicas(Path filePath, short repl, int corruptBlockCount) { DFSTestUtil.CreateFile(dfs, filePath, BlockSize, repl, 0); DFSTestUtil.WaitReplication(dfs, filePath, repl); // Locate the file blocks by asking name node LocatedBlocks locatedblocks = dfs.dfs.GetNamenode().GetBlockLocations(filePath.ToString (), 0L, BlockSize); NUnit.Framework.Assert.AreEqual(repl, locatedblocks.Get(0).GetLocations().Length); // The file only has one block LocatedBlock lblock = locatedblocks.Get(0); DatanodeInfo[] datanodeinfos = lblock.GetLocations(); ExtendedBlock block = lblock.GetBlock(); // corrupt some /all of the block replicas for (int i = 0; i < corruptBlockCount; i++) { DatanodeInfo dninfo = datanodeinfos[i]; DataNode dn = cluster.GetDataNode(dninfo.GetIpcPort()); CorruptBlock(block, dn); Log.Debug("Corrupted block " + block.GetBlockName() + " on data node " + dninfo); } }
/// <exception cref="System.IO.IOException"/> internal static ClientDatanodeProtocolPB CreateClientDatanodeProtocolProxy(DatanodeID datanodeid, Configuration conf, int socketTimeout, bool connectToDnViaHostname, LocatedBlock locatedBlock) { string dnAddr = datanodeid.GetIpcAddr(connectToDnViaHostname); IPEndPoint addr = NetUtils.CreateSocketAddr(dnAddr); if (Log.IsDebugEnabled()) { Log.Debug("Connecting to datanode " + dnAddr + " addr=" + addr); } // Since we're creating a new UserGroupInformation here, we know that no // future RPC proxies will be able to re-use the same connection. And // usages of this proxy tend to be one-off calls. // // This is a temporary fix: callers should really achieve this by using // RPC.stopProxy() on the resulting object, but this is currently not // working in trunk. See the discussion on HDFS-1965. Configuration confWithNoIpcIdle = new Configuration(conf); confWithNoIpcIdle.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey , 0); UserGroupInformation ticket = UserGroupInformation.CreateRemoteUser(locatedBlock. GetBlock().GetLocalBlock().ToString()); ticket.AddToken(locatedBlock.GetBlockToken()); return(CreateClientDatanodeProtocolProxy(addr, ticket, confWithNoIpcIdle, NetUtils .GetDefaultSocketFactory(conf), socketTimeout)); }
public virtual void TestMissingBlock() { // Create a file with single block with two replicas Path file = GetTestPath("testMissingBlocks"); CreateFile(file, 100, (short)1); // Corrupt the only replica of the block to result in a missing block LocatedBlock block = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), file .ToString(), 0, 1).Get(0); cluster.GetNamesystem().WriteLock(); try { bm.FindAndMarkBlockAsCorrupt(block.GetBlock(), block.GetLocations()[0], "STORAGE_ID" , "TEST"); } finally { cluster.GetNamesystem().WriteUnlock(); } UpdateMetrics(); MetricsRecordBuilder rb = MetricsAsserts.GetMetrics(NsMetrics); MetricsAsserts.AssertGauge("UnderReplicatedBlocks", 1L, rb); MetricsAsserts.AssertGauge("MissingBlocks", 1L, rb); MetricsAsserts.AssertGauge("MissingReplOneBlocks", 1L, rb); fs.Delete(file, true); WaitForDnMetricValue(NsMetrics, "UnderReplicatedBlocks", 0L); }
internal Pipeline(LocatedBlock lb) { foreach (DatanodeInfo d in lb.GetLocations()) { datanodes.AddItem(d.GetName()); } }
/// <summary>Get a DataNode that serves our testBlock.</summary> public virtual DataNode GetDataNode(LocatedBlock testBlock) { DatanodeInfo[] nodes = testBlock.GetLocations(); int ipcport = nodes[0].GetIpcPort(); return(cluster.GetDataNode(ipcport)); }
public virtual void TestLocatedBlocks2Locations() { DatanodeInfo d = DFSTestUtil.GetLocalDatanodeInfo(); DatanodeInfo[] ds = new DatanodeInfo[1]; ds[0] = d; // ok ExtendedBlock b1 = new ExtendedBlock("bpid", 1, 1, 1); LocatedBlock l1 = new LocatedBlock(b1, ds, 0, false); // corrupt ExtendedBlock b2 = new ExtendedBlock("bpid", 2, 1, 1); LocatedBlock l2 = new LocatedBlock(b2, ds, 0, true); IList <LocatedBlock> ls = Arrays.AsList(l1, l2); LocatedBlocks lbs = new LocatedBlocks(10, false, ls, l2, true, null); BlockLocation[] bs = DFSUtil.LocatedBlocks2Locations(lbs); NUnit.Framework.Assert.IsTrue("expected 2 blocks but got " + bs.Length, bs.Length == 2); int corruptCount = 0; foreach (BlockLocation b in bs) { if (b.IsCorrupt()) { corruptCount++; } } NUnit.Framework.Assert.IsTrue("expected 1 corrupt files but got " + corruptCount, corruptCount == 1); // test an empty location bs = DFSUtil.LocatedBlocks2Locations(new LocatedBlocks()); NUnit.Framework.Assert.AreEqual(0, bs.Length); }
internal virtual bool ScheduleMoves4Block(Mover.StorageTypeDiff diff, LocatedBlock lb) { IList <Mover.MLocation> locations = Mover.MLocation.ToLocations(lb); Sharpen.Collections.Shuffle(locations); Dispatcher.DBlock db = this._enclosing.NewDBlock(lb.GetBlock().GetLocalBlock(), locations ); foreach (StorageType t in diff.existing) { foreach (Mover.MLocation ml in locations) { Dispatcher.Source source = this._enclosing.storages.GetSource(ml); if (ml.storageType == t && source != null) { // try to schedule one replica move. if (this.ScheduleMoveReplica(db, source, diff.expected)) { return(true); } } } } return(false); }
public virtual void TestAddBlockRetryShouldReturnBlockWithLocations() { string src = "/testAddBlockRetryShouldReturnBlockWithLocations"; NamenodeProtocols nameNodeRpc = cluster.GetNameNodeRpc(); // create file nameNodeRpc.Create(src, FsPermission.GetFileDefault(), "clientName", new EnumSetWritable <CreateFlag>(EnumSet.Of(CreateFlag.Create)), true, (short)3, 1024, null); // start first addBlock() Log.Info("Starting first addBlock for " + src); LocatedBlock lb1 = nameNodeRpc.AddBlock(src, "clientName", null, null, INodeId.GrandfatherInodeId , null); NUnit.Framework.Assert.IsTrue("Block locations should be present", lb1.GetLocations ().Length > 0); cluster.RestartNameNode(); nameNodeRpc = cluster.GetNameNodeRpc(); LocatedBlock lb2 = nameNodeRpc.AddBlock(src, "clientName", null, null, INodeId.GrandfatherInodeId , null); NUnit.Framework.Assert.AreEqual("Blocks are not equal", lb1.GetBlock(), lb2.GetBlock ()); NUnit.Framework.Assert.IsTrue("Wrong locations with retry", lb2.GetLocations().Length > 0); }
public _Thread_612(LocatedBlock block, DataNode dataNode, AtomicBoolean recoveryInitResult ) { this.block = block; this.dataNode = dataNode; this.recoveryInitResult = recoveryInitResult; }
public void Handle_WithPutCommand_CallsPutAPI() { const string testFilePath = "TestFilePath"; const string testSrcFile = "TestSrcFile"; // Arrange var mockClientProtocol = new Mock <IRestClientProtocol>(); var locatedBlock = new LocatedBlock() { Locations = new DataNodeId[] { new DataNodeId { IPAddress = "http://localhost" } } }; mockClientProtocol.Setup(x => x.AddBlock(It.IsAny <string>())).Returns(locatedBlock); var stubDataTransferProtocol = new Mock <IRestDataTransferProtocol>(); var sut = new PutCommandHandler(mockClientProtocol.Object, stubDataTransferProtocol.Object); var stubPutCommand = new PutCommand() { FilePath = testFilePath, SrcFile = testSrcFile }; // Act sut.Handle(stubPutCommand); // Assert mockClientProtocol.Verify(x => x.Create(testSrcFile, testFilePath)); }
/// <exception cref="Org.Apache.Hadoop.Hdfs.Server.Datanode.BPServiceActorActionException /// "/> public virtual void ReportTo(DatanodeProtocolClientSideTranslatorPB bpNamenode, DatanodeRegistration bpRegistration) { if (bpRegistration == null) { return; } DatanodeInfo[] dnArr = new DatanodeInfo[] { new DatanodeInfo(bpRegistration) }; string[] uuids = new string[] { storageUuid }; StorageType[] types = new StorageType[] { storageType }; LocatedBlock[] locatedBlock = new LocatedBlock[] { new LocatedBlock(block, dnArr, uuids, types) }; try { bpNamenode.ReportBadBlocks(locatedBlock); } catch (RemoteException re) { DataNode.Log.Info("reportBadBlock encountered RemoteException for " + "block: " + block, re); } catch (IOException) { throw new BPServiceActorActionException("Failed to report bad block " + block + " to namenode: " ); } }
public virtual void TestMoverFailedRetry() { // HDFS-8147 Configuration conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsMoverRetryMaxAttemptsKey, "2"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).StorageTypes (new StorageType[][] { new StorageType[] { StorageType.Disk, StorageType.Archive }, new StorageType[] { StorageType.Disk, StorageType.Archive }, new StorageType [] { StorageType.Disk, StorageType.Archive } }).Build(); try { cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); string file = "/testMoverFailedRetry"; // write to DISK FSDataOutputStream @out = dfs.Create(new Path(file), (short)2); @out.WriteChars("testMoverFailedRetry"); @out.Close(); // Delete block file so, block move will fail with FileNotFoundException LocatedBlock lb = dfs.GetClient().GetLocatedBlocks(file, 0).Get(0); cluster.CorruptBlockOnDataNodesByDeletingBlockFile(lb.GetBlock()); // move to ARCHIVE dfs.SetStoragePolicy(new Path(file), "COLD"); int rc = ToolRunner.Run(conf, new Mover.Cli(), new string[] { "-p", file.ToString () }); NUnit.Framework.Assert.AreEqual("Movement should fail after some retry", ExitStatus .IoException.GetExitCode(), rc); } finally { cluster.Shutdown(); } }
public virtual void TestAbandonBlock() { string src = FileNamePrefix + "foo"; // Start writing a file but do not close it FSDataOutputStream fout = fs.Create(new Path(src), true, 4096, (short)1, 512L); for (int i = 0; i < 1024; i++) { fout.Write(123); } fout.Hflush(); long fileId = ((DFSOutputStream)fout.GetWrappedStream()).GetFileId(); // Now abandon the last block DFSClient dfsclient = DFSClientAdapter.GetDFSClient(fs); LocatedBlocks blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue ); int orginalNumBlocks = blocks.LocatedBlockCount(); LocatedBlock b = blocks.GetLastLocatedBlock(); dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName ); // call abandonBlock again to make sure the operation is idempotent dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName ); // And close the file fout.Close(); // Close cluster and check the block has been abandoned after restart cluster.RestartNameNode(); blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue); NUnit.Framework.Assert.AreEqual("Blocks " + b + " has not been abandoned.", orginalNumBlocks , blocks.LocatedBlockCount() + 1); }
public virtual void TestBlockTokenRpcLeak() { Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); Assume.AssumeTrue(FdDir.Exists()); BlockTokenSecretManager sm = new BlockTokenSecretManager(blockKeyUpdateInterval, blockTokenLifetime, 0, "fake-pool", null); Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token = sm.GenerateToken (block3, EnumSet.AllOf <BlockTokenSecretManager.AccessMode>()); Server server = CreateMockDatanode(sm, token, conf); server.Start(); IPEndPoint addr = NetUtils.GetConnectAddress(server); DatanodeID fakeDnId = DFSTestUtil.GetLocalDatanodeID(addr.Port); ExtendedBlock b = new ExtendedBlock("fake-pool", new Org.Apache.Hadoop.Hdfs.Protocol.Block (12345L)); LocatedBlock fakeBlock = new LocatedBlock(b, new DatanodeInfo[0]); fakeBlock.SetBlockToken(token); // Create another RPC proxy with the same configuration - this will never // attempt to connect anywhere -- but it causes the refcount on the // RPC "Client" object to stay above 0 such that RPC.stopProxy doesn't // actually close the TCP connections to the real target DN. ClientDatanodeProtocol proxyToNoWhere = RPC.GetProxy <ClientDatanodeProtocol>(ClientDatanodeProtocol .versionID, new IPEndPoint("1.1.1.1", 1), UserGroupInformation.CreateRemoteUser( "junk"), conf, NetUtils.GetDefaultSocketFactory(conf)); ClientDatanodeProtocol proxy = null; int fdsAtStart = CountOpenFileDescriptors(); try { long endTime = Time.Now() + 3000; while (Time.Now() < endTime) { proxy = DFSUtil.CreateClientDatanodeProtocolProxy(fakeDnId, conf, 1000, false, fakeBlock ); NUnit.Framework.Assert.AreEqual(block3.GetBlockId(), proxy.GetReplicaVisibleLength (block3)); if (proxy != null) { RPC.StopProxy(proxy); } Log.Info("Num open fds:" + CountOpenFileDescriptors()); } int fdsAtEnd = CountOpenFileDescriptors(); if (fdsAtEnd - fdsAtStart > 50) { NUnit.Framework.Assert.Fail("Leaked " + (fdsAtEnd - fdsAtStart) + " fds!"); } } finally { server.Stop(); } RPC.StopProxy(proxyToNoWhere); }
/// <summary>Verify the first block of the file is corrupted (for all its replica).</summary> /// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/> /// <exception cref="System.IO.FileNotFoundException"/> /// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/> /// <exception cref="System.IO.IOException"/> private void VerifyFirstBlockCorrupted(Path filePath, bool isCorrupted) { LocatedBlocks locatedBlocks = dfs.dfs.GetNamenode().GetBlockLocations(filePath.ToUri ().GetPath(), 0, long.MaxValue); LocatedBlock firstLocatedBlock = locatedBlocks.Get(0); NUnit.Framework.Assert.AreEqual(isCorrupted, firstLocatedBlock.IsCorrupt()); }
public virtual void TestConvertLocatedBlockNoStorageMedia() { LocatedBlock lb = CreateLocatedBlockNoStorageMedia(); HdfsProtos.LocatedBlockProto lbProto = PBHelper.Convert(lb); LocatedBlock lb2 = PBHelper.Convert(lbProto); Compare(lb, lb2); }
public _Runnable_471(MiniDFSCluster cluster, LocatedBlock lblock, int TestFileLen , Semaphore sem, byte[] buf, AtomicBoolean testFailed) { this.cluster = cluster; this.lblock = lblock; this.TestFileLen = TestFileLen; this.sem = sem; this.buf = buf; this.testFailed = testFailed; }
public static void SetupCluster() { int ReplicationFactor = 1; util = new BlockReaderTestUtil(ReplicationFactor); util.WriteFile(TestFile, FileSizeK); IList <LocatedBlock> blkList = util.GetFileBlocks(TestFile, FileSizeK); testBlock = blkList[0]; }
/// <summary> /// Verify the number of corrupted block replicas by fetching the block /// location from name node. /// </summary> /// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/> /// <exception cref="System.IO.FileNotFoundException"/> /// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/> /// <exception cref="System.IO.IOException"/> private void VerifyCorruptedBlockCount(Path filePath, int expectedReplicas) { LocatedBlocks lBlocks = dfs.dfs.GetNamenode().GetBlockLocations(filePath.ToUri(). GetPath(), 0, long.MaxValue); // we expect only the first block of the file is used for this test LocatedBlock firstLocatedBlock = lBlocks.Get(0); NUnit.Framework.Assert.AreEqual(expectedReplicas, firstLocatedBlock.GetLocations( ).Length); }
public virtual void TestReplicationError() { // create a file of replication factor of 1 Path fileName = new Path("/test.txt"); int fileLen = 1; DFSTestUtil.CreateFile(fs, fileName, 1, (short)1, 1L); DFSTestUtil.WaitReplication(fs, fileName, (short)1); // get the block belonged to the created file LocatedBlocks blocks = NameNodeAdapter.GetBlockLocations(cluster.GetNameNode(), fileName .ToString(), 0, (long)fileLen); NUnit.Framework.Assert.AreEqual("Should only find 1 block", blocks.LocatedBlockCount (), 1); LocatedBlock block = blocks.Get(0); // bring up a second datanode cluster.StartDataNodes(conf, 1, true, null, null); cluster.WaitActive(); int sndNode = 1; DataNode datanode = cluster.GetDataNodes()[sndNode]; // replicate the block to the second datanode IPEndPoint target = datanode.GetXferAddress(); Socket s = Sharpen.Extensions.CreateSocket(target.Address, target.Port); // write the header. DataOutputStream @out = new DataOutputStream(s.GetOutputStream()); DataChecksum checksum = DataChecksum.NewDataChecksum(DataChecksum.Type.Crc32, 512 ); new Sender(@out).WriteBlock(block.GetBlock(), StorageType.Default, BlockTokenSecretManager .DummyToken, string.Empty, new DatanodeInfo[0], new StorageType[0], null, BlockConstructionStage .PipelineSetupCreate, 1, 0L, 0L, 0L, checksum, CachingStrategy.NewDefaultStrategy (), false, false, null); @out.Flush(); // close the connection before sending the content of the block @out.Close(); // the temporary block & meta files should be deleted string bpid = cluster.GetNamesystem().GetBlockPoolId(); FilePath storageDir = cluster.GetInstanceStorageDir(sndNode, 0); FilePath dir1 = MiniDFSCluster.GetRbwDir(storageDir, bpid); storageDir = cluster.GetInstanceStorageDir(sndNode, 1); FilePath dir2 = MiniDFSCluster.GetRbwDir(storageDir, bpid); while (dir1.ListFiles().Length != 0 || dir2.ListFiles().Length != 0) { Sharpen.Thread.Sleep(100); } // then increase the file's replication factor fs.SetReplication(fileName, (short)2); // replication should succeed DFSTestUtil.WaitReplication(fs, fileName, (short)1); // clean up the file fs.Delete(fileName, false); }
/// <exception cref="System.Exception"/> public virtual void TestBlockReaderLocalLegacyWithAppend() { short ReplFactor = 1; HdfsConfiguration conf = GetConfiguration(null); conf.SetBoolean(DFSConfigKeys.DfsClientUseLegacyBlockreaderlocal, true); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); Path path = new Path("/testBlockReaderLocalLegacy"); DFSTestUtil.CreateFile(dfs, path, 10, ReplFactor, 0); DFSTestUtil.WaitReplication(dfs, path, ReplFactor); ClientDatanodeProtocol proxy; Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> token; ExtendedBlock originalBlock; long originalGS; { LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString (), 0, 1).Get(0); proxy = DFSUtil.CreateClientDatanodeProtocolProxy(lb.GetLocations()[0], conf, 60000 , false); token = lb.GetBlockToken(); // get block and generation stamp ExtendedBlock blk = new ExtendedBlock(lb.GetBlock()); originalBlock = new ExtendedBlock(blk); originalGS = originalBlock.GetGenerationStamp(); // test getBlockLocalPathInfo BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(blk, token); NUnit.Framework.Assert.AreEqual(originalGS, info.GetBlock().GetGenerationStamp()); } { // append one byte FSDataOutputStream @out = dfs.Append(path); @out.Write(1); @out.Close(); } { // get new generation stamp LocatedBlock lb = cluster.GetNameNode().GetRpcServer().GetBlockLocations(path.ToString (), 0, 1).Get(0); long newGS = lb.GetBlock().GetGenerationStamp(); NUnit.Framework.Assert.IsTrue(newGS > originalGS); // getBlockLocalPathInfo using the original block. NUnit.Framework.Assert.AreEqual(originalGS, originalBlock.GetGenerationStamp()); BlockLocalPathInfo info = proxy.GetBlockLocalPathInfo(originalBlock, token); NUnit.Framework.Assert.AreEqual(newGS, info.GetBlock().GetGenerationStamp()); } cluster.Shutdown(); }
public virtual void TestLocatedBlockConstructorWithNullCachedLocs() { DatanodeInfo d = DFSTestUtil.GetLocalDatanodeInfo(); DatanodeInfo[] ds = new DatanodeInfo[1]; ds[0] = d; ExtendedBlock b1 = new ExtendedBlock("bpid", 1, 1, 1); LocatedBlock l1 = new LocatedBlock(b1, ds, null, null, 0, false, null); DatanodeInfo[] cachedLocs = l1.GetCachedLocations(); NUnit.Framework.Assert.IsTrue(cachedLocs.Length == 0); }
/// <exception cref="System.IO.IOException"/> private Block FindBlock(Path path, long size) { Block ret; IList <LocatedBlock> lbs = cluster.GetNameNodeRpc().GetBlockLocations(path.ToString (), FileStart, size).GetLocatedBlocks(); LocatedBlock lb = lbs[lbs.Count - 1]; // Get block from the first DN ret = cluster.GetDataNodes()[DnN0].data.GetStoredBlock(lb.GetBlock().GetBlockPoolId (), lb.GetBlock().GetBlockId()); return(ret); }
internal static IList <Mover.MLocation> ToLocations(LocatedBlock lb) { DatanodeInfo[] datanodeInfos = lb.GetLocations(); StorageType[] storageTypes = lb.GetStorageTypes(); long size = lb.GetBlockSize(); IList <Mover.MLocation> locations = new List <Mover.MLocation>(); for (int i = 0; i < datanodeInfos.Length; i++) { locations.AddItem(new Mover.MLocation(datanodeInfos[i], storageTypes[i], size)); } return(locations); }
/// <summary>Initialize the pipeline.</summary> public virtual Pipeline InitPipeline(LocatedBlock lb) { lock (this) { Pipeline pl = new Pipeline(lb); if (pipelines.Contains(pl)) { throw new InvalidOperationException("thepipeline != null"); } pipelines.AddItem(pl); return(pl); } }
public void Run() { try { IList <LocatedBlock> locatedBlocks = cluster.GetNameNode().GetRpcServer().GetBlockLocations (TestFile, 0, TestFileLen).GetLocatedBlocks(); LocatedBlock lblock = locatedBlocks[0]; BlockReader blockReader = null; try { blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen); NUnit.Framework.Assert.Fail("expected getBlockReader to fail the first time."); } catch (Exception t) { NUnit.Framework.Assert.IsTrue("expected to see 'TCP reads were disabled " + "for testing' in exception " + t, t.Message.Contains("TCP reads were disabled for testing")); } finally { if (blockReader != null) { blockReader.Close(); } } gotFailureLatch.CountDown(); shouldRetryLatch.Await(); try { blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen); } catch (Exception t) { TestBlockReaderFactory.Log.Error("error trying to retrieve a block reader " + "the second time." , t); throw; } finally { if (blockReader != null) { blockReader.Close(); } } } catch (Exception t) { TestBlockReaderFactory.Log.Error("getBlockReader failure", t); testFailed.Set(true); } }
/// <summary>Test to verify the race between finalizeBlock and Lease recovery</summary> /// <exception cref="System.Exception"/> public virtual void TestRaceBetweenReplicaRecoveryAndFinalizeBlock() { TearDown(); // Stop the Mocked DN started in startup() Configuration conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsDatanodeXceiverStopTimeoutMillisKey, "1000"); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); try { cluster.WaitClusterUp(); DistributedFileSystem fs = cluster.GetFileSystem(); Path path = new Path("/test"); FSDataOutputStream @out = fs.Create(path); @out.WriteBytes("data"); @out.Hsync(); IList <LocatedBlock> blocks = DFSTestUtil.GetAllBlocks(fs.Open(path)); LocatedBlock block = blocks[0]; DataNode dataNode = cluster.GetDataNodes()[0]; AtomicBoolean recoveryInitResult = new AtomicBoolean(true); Sharpen.Thread recoveryThread = new _Thread_612(block, dataNode, recoveryInitResult ); recoveryThread.Start(); try { @out.Close(); } catch (IOException e) { NUnit.Framework.Assert.IsTrue("Writing should fail", e.Message.Contains("are bad. Aborting..." )); } finally { recoveryThread.Join(); } NUnit.Framework.Assert.IsTrue("Recovery should be initiated successfully", recoveryInitResult .Get()); dataNode.UpdateReplicaUnderRecovery(block.GetBlock(), block.GetBlock().GetGenerationStamp () + 1, block.GetBlock().GetBlockId(), block.GetBlockSize()); } finally { if (null != cluster) { cluster.Shutdown(); cluster = null; } } }
// try reading a block using a BlockReader directly private static void TryRead(Configuration conf, LocatedBlock lblock, bool shouldSucceed ) { IPEndPoint targetAddr = null; IOException ioe = null; BlockReader blockReader = null; ExtendedBlock block = lblock.GetBlock(); try { DatanodeInfo[] nodes = lblock.GetLocations(); targetAddr = NetUtils.CreateSocketAddr(nodes[0].GetXferAddr()); blockReader = new BlockReaderFactory(new DFSClient.Conf(conf)).SetFileName(BlockReaderFactory .GetFileName(targetAddr, "test-blockpoolid", block.GetBlockId())).SetBlock(block ).SetBlockToken(lblock.GetBlockToken()).SetInetSocketAddress(targetAddr).SetStartOffset (0).SetLength(-1).SetVerifyChecksum(true).SetClientName("TestBlockTokenWithDFS") .SetDatanodeInfo(nodes[0]).SetCachingStrategy(CachingStrategy.NewDefaultStrategy ()).SetClientCacheContext(ClientContext.GetFromConf(conf)).SetConfiguration(conf ).SetRemotePeerFactory(new _RemotePeerFactory_162(conf)).Build(); } catch (IOException ex) { ioe = ex; } finally { if (blockReader != null) { try { blockReader.Close(); } catch (IOException e) { throw new RuntimeException(e); } } } if (shouldSucceed) { NUnit.Framework.Assert.IsNotNull("OP_READ_BLOCK: access token is invalid, " + "when it is expected to be valid" , blockReader); } else { NUnit.Framework.Assert.IsNotNull("OP_READ_BLOCK: access token is valid, " + "when it is expected to be invalid" , ioe); NUnit.Framework.Assert.IsTrue("OP_READ_BLOCK failed due to reasons other than access token: " , ioe is InvalidBlockTokenException); } }