/// <summary> /// Support for layout version change with rolling upgrade was /// added by HDFS-6800 and HDFS-6981. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestWithLayoutChangeAndFinalize() { long seed = unchecked ((int)(0x600DF00D)); try { StartCluster(); Path[] paths = new Path[3]; FilePath[] blockFiles = new FilePath[3]; // Create two files in DFS. for (int i = 0; i < 2; ++i) { paths[i] = new Path("/" + GenericTestUtils.GetMethodName() + "." + i + ".dat"); DFSTestUtil.CreateFile(fs, paths[i], BlockSize, (short)2, seed); } StartRollingUpgrade(); // Delete the first file. The DN will save its block files in trash. blockFiles[0] = GetBlockForFile(paths[0], true); FilePath trashFile0 = GetTrashFileForBlock(blockFiles[0], false); DeleteAndEnsureInTrash(paths[0], blockFiles[0], trashFile0); // Restart the DN with a new layout version to trigger layout upgrade. Log.Info("Shutting down the Datanode"); MiniDFSCluster.DataNodeProperties dnprop = cluster.StopDataNode(0); DFSTestUtil.AddDataNodeLayoutVersion(DataNodeLayoutVersion.CurrentLayoutVersion - 1, "Test Layout for TestDataNodeRollingUpgrade"); Log.Info("Restarting the DataNode"); cluster.RestartDataNode(dnprop, true); cluster.WaitActive(); dn0 = cluster.GetDataNodes()[0]; Log.Info("The DN has been restarted"); NUnit.Framework.Assert.IsFalse(trashFile0.Exists()); NUnit.Framework.Assert.IsFalse(dn0.GetStorage().GetBPStorage(blockPoolId).IsTrashAllowed (blockFiles[0])); // Ensure that the block file for the first file was moved from 'trash' to 'previous'. NUnit.Framework.Assert.IsTrue(IsBlockFileInPrevious(blockFiles[0])); NUnit.Framework.Assert.IsFalse(IsTrashRootPresent()); // Delete the second file. Ensure that its block file is in previous. blockFiles[1] = GetBlockForFile(paths[1], true); fs.Delete(paths[1], false); NUnit.Framework.Assert.IsTrue(IsBlockFileInPrevious(blockFiles[1])); NUnit.Framework.Assert.IsFalse(IsTrashRootPresent()); // Finalize and ensure that neither block file exists in trash or previous. FinalizeRollingUpgrade(); NUnit.Framework.Assert.IsFalse(IsTrashRootPresent()); NUnit.Framework.Assert.IsFalse(IsBlockFileInPrevious(blockFiles[0])); NUnit.Framework.Assert.IsFalse(IsBlockFileInPrevious(blockFiles[1])); } finally { ShutdownCluster(); } }
/// <summary> /// Support for layout version change with rolling upgrade was /// added by HDFS-6800 and HDFS-6981. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestWithLayoutChangeAndRollback() { long seed = unchecked ((int)(0x600DF00D)); try { StartCluster(); Path[] paths = new Path[3]; FilePath[] blockFiles = new FilePath[3]; // Create two files in DFS. for (int i = 0; i < 2; ++i) { paths[i] = new Path("/" + GenericTestUtils.GetMethodName() + "." + i + ".dat"); DFSTestUtil.CreateFile(fs, paths[i], BlockSize, (short)1, seed); } StartRollingUpgrade(); // Delete the first file. The DN will save its block files in trash. blockFiles[0] = GetBlockForFile(paths[0], true); FilePath trashFile0 = GetTrashFileForBlock(blockFiles[0], false); DeleteAndEnsureInTrash(paths[0], blockFiles[0], trashFile0); // Restart the DN with a new layout version to trigger layout upgrade. Log.Info("Shutting down the Datanode"); MiniDFSCluster.DataNodeProperties dnprop = cluster.StopDataNode(0); DFSTestUtil.AddDataNodeLayoutVersion(DataNodeLayoutVersion.CurrentLayoutVersion - 1, "Test Layout for TestDataNodeRollingUpgrade"); Log.Info("Restarting the DataNode"); cluster.RestartDataNode(dnprop, true); cluster.WaitActive(); dn0 = cluster.GetDataNodes()[0]; Log.Info("The DN has been restarted"); NUnit.Framework.Assert.IsFalse(trashFile0.Exists()); NUnit.Framework.Assert.IsFalse(dn0.GetStorage().GetBPStorage(blockPoolId).IsTrashAllowed (blockFiles[0])); // Ensure that the block file for the first file was moved from 'trash' to 'previous'. NUnit.Framework.Assert.IsTrue(IsBlockFileInPrevious(blockFiles[0])); NUnit.Framework.Assert.IsFalse(IsTrashRootPresent()); // Delete the second file. Ensure that its block file is in previous. blockFiles[1] = GetBlockForFile(paths[1], true); fs.Delete(paths[1], false); NUnit.Framework.Assert.IsTrue(IsBlockFileInPrevious(blockFiles[1])); NUnit.Framework.Assert.IsFalse(IsTrashRootPresent()); // Create and delete a third file. Its block file should not be // in either trash or previous after deletion. paths[2] = new Path("/" + GenericTestUtils.GetMethodName() + ".2.dat"); DFSTestUtil.CreateFile(fs, paths[2], BlockSize, (short)1, seed); blockFiles[2] = GetBlockForFile(paths[2], true); fs.Delete(paths[2], false); NUnit.Framework.Assert.IsFalse(IsBlockFileInPrevious(blockFiles[2])); NUnit.Framework.Assert.IsFalse(IsTrashRootPresent()); // Rollback and ensure that the first two file contents were restored. RollbackRollingUpgrade(); for (int i_1 = 0; i_1 < 2; ++i_1) { byte[] actual = DFSTestUtil.ReadFileBuffer(fs, paths[i_1]); byte[] calculated = DFSTestUtil.CalculateFileContentsFromSeed(seed, BlockSize); Assert.AssertArrayEquals(actual, calculated); } // And none of the block files must be in previous or trash. NUnit.Framework.Assert.IsFalse(IsTrashRootPresent()); for (int i_2 = 0; i_2 < 3; ++i_2) { NUnit.Framework.Assert.IsFalse(IsBlockFileInPrevious(blockFiles[i_2])); } } finally { ShutdownCluster(); } }