public virtual void TestAddVolumeFailures() { StartDFSCluster(1, 1); string dataDir = cluster.GetDataDirectory(); DataNode dn = cluster.GetDataNodes()[0]; IList <string> newDirs = Lists.NewArrayList(); int NumNewDirs = 4; for (int i = 0; i < NumNewDirs; i++) { FilePath newVolume = new FilePath(dataDir, "new_vol" + i); newDirs.AddItem(newVolume.ToString()); if (i % 2 == 0) { // Make addVolume() fail. newVolume.CreateNewFile(); } } string newValue = dn.GetConf().Get(DFSConfigKeys.DfsDatanodeDataDirKey) + "," + Joiner .On(",").Join(newDirs); try { dn.ReconfigurePropertyImpl(DFSConfigKeys.DfsDatanodeDataDirKey, newValue); NUnit.Framework.Assert.Fail("Expect to throw IOException."); } catch (ReconfigurationException e) { string errorMessage = e.InnerException.Message; string[] messages = errorMessage.Split("\\r?\\n"); NUnit.Framework.Assert.AreEqual(2, messages.Length); Assert.AssertThat(messages[0], CoreMatchers.ContainsString("new_vol0")); Assert.AssertThat(messages[1], CoreMatchers.ContainsString("new_vol2")); } // Make sure that vol0 and vol2's metadata are not left in memory. FsDatasetSpi <object> dataset = dn.GetFSDataset(); foreach (FsVolumeSpi volume in dataset.GetVolumes()) { Assert.AssertThat(volume.GetBasePath(), IS.Is(CoreMatchers.Not(CoreMatchers.AnyOf (IS.Is(newDirs[0]), IS.Is(newDirs[2]))))); } DataStorage storage = dn.GetStorage(); for (int i_1 = 0; i_1 < storage.GetNumStorageDirs(); i_1++) { Storage.StorageDirectory sd = storage.GetStorageDir(i_1); Assert.AssertThat(sd.GetRoot().ToString(), IS.Is(CoreMatchers.Not(CoreMatchers.AnyOf (IS.Is(newDirs[0]), IS.Is(newDirs[2]))))); } // The newly effective conf does not have vol0 and vol2. string[] effectiveVolumes = dn.GetConf().Get(DFSConfigKeys.DfsDatanodeDataDirKey) .Split(","); NUnit.Framework.Assert.AreEqual(4, effectiveVolumes.Length); foreach (string ev in effectiveVolumes) { Assert.AssertThat(StorageLocation.Parse(ev).GetFile().GetCanonicalPath(), IS.Is(CoreMatchers.Not (CoreMatchers.AnyOf(IS.Is(newDirs[0]), IS.Is(newDirs[2]))))); } }
public virtual void TestBlockHasMultipleReplicasOnSameDN() { string filename = MakeFileName(GenericTestUtils.GetMethodName()); Path filePath = new Path(filename); // Write out a file with a few blocks. DFSTestUtil.CreateFile(fs, filePath, BlockSize, BlockSize * NumBlocks, BlockSize, NumDatanodes, seed); // Get the block list for the file with the block locations. LocatedBlocks locatedBlocks = client.GetLocatedBlocks(filePath.ToString(), 0, BlockSize * NumBlocks); // Generate a fake block report from one of the DataNodes, such // that it reports one copy of each block on either storage. DataNode dn = cluster.GetDataNodes()[0]; DatanodeRegistration dnReg = dn.GetDNRegistrationForBP(bpid); StorageBlockReport[] reports = new StorageBlockReport[cluster.GetStoragesPerDatanode ()]; AList <Replica> blocks = new AList <Replica>(); foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks()) { Block localBlock = locatedBlock.GetBlock().GetLocalBlock(); blocks.AddItem(new FinalizedReplica(localBlock, null, null)); } BlockListAsLongs bll = BlockListAsLongs.Encode(blocks); for (int i = 0; i < cluster.GetStoragesPerDatanode(); ++i) { FsVolumeSpi v = dn.GetFSDataset().GetVolumes()[i]; DatanodeStorage dns = new DatanodeStorage(v.GetStorageID()); reports[i] = new StorageBlockReport(dns, bll); } // Should not assert! cluster.GetNameNodeRpc().BlockReport(dnReg, bpid, reports, new BlockReportContext (1, 0, Runtime.NanoTime())); // Get the block locations once again. locatedBlocks = client.GetLocatedBlocks(filename, 0, BlockSize * NumBlocks); // Make sure that each block has two replicas, one on each DataNode. foreach (LocatedBlock locatedBlock_1 in locatedBlocks.GetLocatedBlocks()) { DatanodeInfo[] locations = locatedBlock_1.GetLocations(); Assert.AssertThat(locations.Length, IS.Is((int)NumDatanodes)); Assert.AssertThat(locations[0].GetDatanodeUuid(), CoreMatchers.Not(locations[1].GetDatanodeUuid ())); } }
public virtual void Setup() { conf = new HdfsConfiguration(); SimulatedFSDataset.SetFactory(conf); Configuration[] overlays = new Configuration[NumDatanodes]; for (int i = 0; i < overlays.Length; i++) { overlays[i] = new Configuration(); if (i == RoNodeIndex) { overlays[i].SetEnum(SimulatedFSDataset.ConfigPropertyState, i == RoNodeIndex ? DatanodeStorage.State .ReadOnlyShared : DatanodeStorage.State.Normal); } } cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDatanodes).DataNodeConfOverlays (overlays).Build(); fs = cluster.GetFileSystem(); blockManager = cluster.GetNameNode().GetNamesystem().GetBlockManager(); datanodeManager = blockManager.GetDatanodeManager(); client = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()), cluster .GetConfiguration(0)); for (int i_1 = 0; i_1 < NumDatanodes; i_1++) { DataNode dataNode = cluster.GetDataNodes()[i_1]; ValidateStorageState(BlockManagerTestUtil.GetStorageReportsForDatanode(datanodeManager .GetDatanode(dataNode.GetDatanodeId())), i_1 == RoNodeIndex ? DatanodeStorage.State .ReadOnlyShared : DatanodeStorage.State.Normal); } // Create a 1 block file DFSTestUtil.CreateFile(fs, Path, BlockSize, BlockSize, BlockSize, (short)1, seed); LocatedBlock locatedBlock = GetLocatedBlock(); extendedBlock = locatedBlock.GetBlock(); block = extendedBlock.GetLocalBlock(); Assert.AssertThat(locatedBlock.GetLocations().Length, CoreMatchers.Is(1)); normalDataNode = locatedBlock.GetLocations()[0]; readOnlyDataNode = datanodeManager.GetDatanode(cluster.GetDataNodes()[RoNodeIndex ].GetDatanodeId()); Assert.AssertThat(normalDataNode, CoreMatchers.Is(CoreMatchers.Not(readOnlyDataNode ))); ValidateNumberReplicas(1); // Inject the block into the datanode with READ_ONLY_SHARED storage cluster.InjectBlocks(0, RoNodeIndex, Collections.Singleton(block)); // There should now be 2 *locations* for the block // Must wait until the NameNode has processed the block report for the injected blocks WaitForLocations(2); }
public virtual void TestLocalhostReverseLookup() { // 127.0.0.1 -> localhost reverse resolution does not happen on Windows. Assume.AssumeTrue(!Shell.Windows); // Make sure when config FS_DEFAULT_NAME_KEY using IP address, // it will automatically convert it to hostname HdfsConfiguration conf = new HdfsConfiguration(); conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://127.0.0.1:8020"); ICollection <URI> uris = DFSUtil.GetNameServiceUris(conf); NUnit.Framework.Assert.AreEqual(1, uris.Count); foreach (URI uri in uris) { Assert.AssertThat(uri.GetHost(), CoreMatchers.Not("127.0.0.1")); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestGetReconfigureStatus() { ReconfigurationUtil ru = Org.Mockito.Mockito.Mock <ReconfigurationUtil>(); datanode.SetReconfigurationUtil(ru); IList <ReconfigurationUtil.PropertyChange> changes = new AList <ReconfigurationUtil.PropertyChange >(); FilePath newDir = new FilePath(cluster.GetDataDirectory(), "data_new"); newDir.Mkdirs(); changes.AddItem(new ReconfigurationUtil.PropertyChange(DFSConfigKeys.DfsDatanodeDataDirKey , newDir.ToString(), datanode.GetConf().Get(DFSConfigKeys.DfsDatanodeDataDirKey) )); changes.AddItem(new ReconfigurationUtil.PropertyChange("randomKey", "new123", "old456" )); Org.Mockito.Mockito.When(ru.ParseChangedProperties(Matchers.Any <Configuration>(), Matchers.Any <Configuration>())).ThenReturn(changes); int port = datanode.GetIpcPort(); string address = "localhost:" + port; Assert.AssertThat(admin.StartReconfiguration("datanode", address), CoreMatchers.Is (0)); IList <string> outputs = null; int count = 100; while (count > 0) { outputs = GetReconfigureStatus("datanode", address); if (!outputs.IsEmpty() && outputs[0].Contains("finished")) { break; } count--; Sharpen.Thread.Sleep(100); } NUnit.Framework.Assert.IsTrue(count > 0); Assert.AssertThat(outputs.Count, CoreMatchers.Is(8)); // 3 (SUCCESS) + 4 (FAILED) IList <StorageLocation> locations = DataNode.GetStorageLocations(datanode.GetConf( )); Assert.AssertThat(locations.Count, CoreMatchers.Is(1)); Assert.AssertThat(locations[0].GetFile(), CoreMatchers.Is(newDir)); // Verify the directory is appropriately formatted. NUnit.Framework.Assert.IsTrue(new FilePath(newDir, Storage.StorageDirCurrent).IsDirectory ()); int successOffset = outputs[1].StartsWith("SUCCESS:") ? 1 : 5; int failedOffset = outputs[1].StartsWith("FAILED:") ? 1 : 4; Assert.AssertThat(outputs[successOffset], CoreMatchers.ContainsString("Change property " + DFSConfigKeys.DfsDatanodeDataDirKey)); Assert.AssertThat(outputs[successOffset + 1], CoreMatchers.Is(CoreMatchers.AllOf( CoreMatchers.ContainsString("From:"), CoreMatchers.ContainsString("data1"), CoreMatchers.ContainsString ("data2")))); Assert.AssertThat(outputs[successOffset + 2], CoreMatchers.Is(CoreMatchers.Not(CoreMatchers.AnyOf (CoreMatchers.ContainsString("data1"), CoreMatchers.ContainsString("data2"))))); Assert.AssertThat(outputs[successOffset + 2], CoreMatchers.Is(CoreMatchers.AllOf( CoreMatchers.ContainsString("To"), CoreMatchers.ContainsString("data_new")))); Assert.AssertThat(outputs[failedOffset], CoreMatchers.ContainsString("Change property randomKey" )); Assert.AssertThat(outputs[failedOffset + 1], CoreMatchers.ContainsString("From: \"old456\"" )); Assert.AssertThat(outputs[failedOffset + 2], CoreMatchers.ContainsString("To: \"new123\"" )); }
/// <summary> /// Test that we cannot read a file beyond its snapshot length /// when accessing it via a snapshot path. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotfileLength() { hdfs.Mkdirs(sub); int bytesRead; byte[] buffer = new byte[Blocksize * 8]; int origLen = Blocksize + 1; int toAppend = Blocksize; FSDataInputStream fis = null; FileStatus fileStatus = null; // Create and write a file. Path file1 = new Path(sub, file1Name); DFSTestUtil.CreateFile(hdfs, file1, Blocksize, 0, Blocksize, Replication, Seed); DFSTestUtil.AppendFile(hdfs, file1, origLen); // Create a snapshot on the parent directory. hdfs.AllowSnapshot(sub); hdfs.CreateSnapshot(sub, snapshot1); Path file1snap1 = SnapshotTestHelper.GetSnapshotPath(sub, snapshot1, file1Name); FileChecksum snapChksum1 = hdfs.GetFileChecksum(file1snap1); Assert.AssertThat("file and snapshot file checksums are not equal", hdfs.GetFileChecksum (file1), CoreMatchers.Is(snapChksum1)); // Append to the file. FSDataOutputStream @out = hdfs.Append(file1); // Nothing has been appended yet. All checksums should still be equal. Assert.AssertThat("file and snapshot checksums (open for append) are not equal", hdfs.GetFileChecksum(file1), CoreMatchers.Is(snapChksum1)); Assert.AssertThat("snapshot checksum (post-open for append) has changed", hdfs.GetFileChecksum (file1snap1), CoreMatchers.Is(snapChksum1)); try { AppendTestUtil.Write(@out, 0, toAppend); // Test reading from snapshot of file that is open for append byte[] dataFromSnapshot = DFSTestUtil.ReadFileBuffer(hdfs, file1snap1); Assert.AssertThat("Wrong data size in snapshot.", dataFromSnapshot.Length, CoreMatchers.Is (origLen)); // Verify that checksum didn't change Assert.AssertThat("snapshot file checksum (pre-close) has changed", hdfs.GetFileChecksum (file1), CoreMatchers.Is(snapChksum1)); Assert.AssertThat("snapshot checksum (post-append) has changed", hdfs.GetFileChecksum (file1snap1), CoreMatchers.Is(snapChksum1)); } finally { @out.Close(); } Assert.AssertThat("file and snapshot file checksums (post-close) are equal", hdfs .GetFileChecksum(file1), CoreMatchers.Not(snapChksum1)); Assert.AssertThat("snapshot file checksum (post-close) has changed", hdfs.GetFileChecksum (file1snap1), CoreMatchers.Is(snapChksum1)); // Make sure we can read the entire file via its non-snapshot path. fileStatus = hdfs.GetFileStatus(file1); Assert.AssertThat(fileStatus.GetLen(), CoreMatchers.Is((long)origLen + toAppend)); fis = hdfs.Open(file1); bytesRead = fis.Read(0, buffer, 0, buffer.Length); Assert.AssertThat(bytesRead, CoreMatchers.Is(origLen + toAppend)); fis.Close(); // Try to open the file via its snapshot path. fis = hdfs.Open(file1snap1); fileStatus = hdfs.GetFileStatus(file1snap1); Assert.AssertThat(fileStatus.GetLen(), CoreMatchers.Is((long)origLen)); // Make sure we can only read up to the snapshot length. bytesRead = fis.Read(0, buffer, 0, buffer.Length); Assert.AssertThat(bytesRead, CoreMatchers.Is(origLen)); fis.Close(); byte[] dataFromSnapshot_1 = DFSTestUtil.ReadFileBuffer(hdfs, file1snap1); Assert.AssertThat("Wrong data size in snapshot.", dataFromSnapshot_1.Length, CoreMatchers.Is (origLen)); }