/// <exception cref="System.IO.IOException"/> private void CheckNameSpace(Configuration conf) { NameNode namenode = new NameNode(conf); NamenodeProtocols nnRpc = namenode.GetRpcServer(); NUnit.Framework.Assert.IsTrue(nnRpc.GetFileInfo("/test").IsDir()); nnRpc.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter, false); nnRpc.SaveNamespace(); namenode.Stop(); namenode.Join(); }
public virtual void TestCompression() { Log.Info("Test compressing image."); Configuration conf = new Configuration(); FileSystem.SetDefaultUri(conf, "hdfs://localhost:0"); conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, "127.0.0.1:0"); FilePath base_dir = new FilePath(PathUtils.GetTestDir(GetType()), "dfs/"); conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, new FilePath(base_dir, "name").GetPath ()); conf.SetBoolean(DFSConfigKeys.DfsPermissionsEnabledKey, false); DFSTestUtil.FormatNameNode(conf); // create an uncompressed image Log.Info("Create an uncompressed fsimage"); NameNode namenode = new NameNode(conf); namenode.GetNamesystem().Mkdirs("/test", new PermissionStatus("hairong", null, FsPermission .GetDefault()), true); NamenodeProtocols nnRpc = namenode.GetRpcServer(); NUnit.Framework.Assert.IsTrue(nnRpc.GetFileInfo("/test").IsDir()); nnRpc.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter, false); nnRpc.SaveNamespace(); namenode.Stop(); namenode.Join(); // compress image using default codec Log.Info("Read an uncomressed image and store it compressed using default codec." ); conf.SetBoolean(DFSConfigKeys.DfsImageCompressKey, true); CheckNameSpace(conf); // read image compressed using the default and compress it using Gzip codec Log.Info("Read a compressed image and store it using a different codec."); conf.Set(DFSConfigKeys.DfsImageCompressionCodecKey, "org.apache.hadoop.io.compress.GzipCodec" ); CheckNameSpace(conf); // read an image compressed in Gzip and store it uncompressed Log.Info("Read a compressed image and store it as uncompressed."); conf.SetBoolean(DFSConfigKeys.DfsImageCompressKey, false); CheckNameSpace(conf); // read an uncomrpessed image and store it uncompressed Log.Info("Read an uncompressed image and store it as uncompressed."); CheckNameSpace(conf); }
// Root scratch directory on local filesystem // The singleton master storage directory for Namenode // A checksum of the contents in namenodeStorage directory // The namespaceId of the namenodeStorage directory // The clusterId of the namenodeStorage directory // The blockpoolId of the namenodeStorage directory // The fsscTime of the namenodeStorage directory // The singleton master storage directory for Datanode // A checksum of the contents in datanodeStorage directory // A checksum of the contents in blockpool storage directory // A checksum of the contents in blockpool finalize storage directory // A checksum of the contents in blockpool rbw storage directory /// <summary>Initialize the data structures used by this class.</summary> /// <remarks> /// Initialize the data structures used by this class. /// IMPORTANT NOTE: This method must be called once before calling /// any other public method on this class. /// <p> /// Creates a singleton master populated storage /// directory for a Namenode (contains edits, fsimage, /// version, and time files) and a Datanode (contains version and /// block files). This can be a lengthy operation. /// </remarks> /// <exception cref="System.Exception"/> public static void Initialize() { CreateEmptyDirs(new string[] { TestRootDir.ToString() }); Configuration config = new HdfsConfiguration(); config.Set(DFSConfigKeys.DfsNamenodeNameDirKey, namenodeStorage.ToString()); config.Set(DFSConfigKeys.DfsNamenodeEditsDirKey, namenodeStorage.ToString()); config.Set(DFSConfigKeys.DfsDatanodeDataDirKey, datanodeStorage.ToString()); MiniDFSCluster cluster = null; string bpid = null; try { // format data-node CreateEmptyDirs(new string[] { datanodeStorage.ToString() }); // format and start NameNode and start DataNode DFSTestUtil.FormatNameNode(config); cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).StartupOption(HdfsServerConstants.StartupOption .Regular).Format(false).ManageDataDfsDirs(false).ManageNameDfsDirs(false).Build( ); NamenodeProtocols namenode = cluster.GetNameNodeRpc(); namenodeStorageNamespaceID = namenode.VersionRequest().GetNamespaceID(); namenodeStorageFsscTime = namenode.VersionRequest().GetCTime(); namenodeStorageClusterID = namenode.VersionRequest().GetClusterID(); namenodeStorageBlockPoolID = namenode.VersionRequest().GetBlockPoolID(); FileSystem fs = FileSystem.Get(config); Path baseDir = new Path("/TestUpgrade"); fs.Mkdirs(baseDir); // write some files int bufferSize = 4096; byte[] buffer = new byte[bufferSize]; for (int i = 0; i < bufferSize; i++) { buffer[i] = unchecked ((byte)((byte)('0') + i % 50)); } WriteFile(fs, new Path(baseDir, "file1"), buffer, bufferSize); WriteFile(fs, new Path(baseDir, "file2"), buffer, bufferSize); // save image namenode.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter, false); namenode.SaveNamespace(); namenode.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave, false); // write more files WriteFile(fs, new Path(baseDir, "file3"), buffer, bufferSize); WriteFile(fs, new Path(baseDir, "file4"), buffer, bufferSize); bpid = cluster.GetNamesystem(0).GetBlockPoolId(); } finally { // shutdown if (cluster != null) { cluster.Shutdown(); } FileUtil.FullyDelete(new FilePath(namenodeStorage, "in_use.lock")); FileUtil.FullyDelete(new FilePath(datanodeStorage, "in_use.lock")); } namenodeStorageChecksum = ChecksumContents(HdfsServerConstants.NodeType.NameNode, new FilePath(namenodeStorage, "current"), false); FilePath dnCurDir = new FilePath(datanodeStorage, "current"); datanodeStorageChecksum = ChecksumContents(HdfsServerConstants.NodeType.DataNode, dnCurDir, false); FilePath bpCurDir = new FilePath(BlockPoolSliceStorage.GetBpRoot(bpid, dnCurDir), "current"); blockPoolStorageChecksum = ChecksumContents(HdfsServerConstants.NodeType.DataNode , bpCurDir, false); FilePath bpCurFinalizeDir = new FilePath(BlockPoolSliceStorage.GetBpRoot(bpid, dnCurDir ), "current/" + DataStorage.StorageDirFinalized); blockPoolFinalizedStorageChecksum = ChecksumContents(HdfsServerConstants.NodeType .DataNode, bpCurFinalizeDir, true); FilePath bpCurRbwDir = new FilePath(BlockPoolSliceStorage.GetBpRoot(bpid, dnCurDir ), "current/" + DataStorage.StorageDirRbw); blockPoolRbwStorageChecksum = ChecksumContents(HdfsServerConstants.NodeType.DataNode , bpCurRbwDir, false); }