public static void CreateOriginalFSImage() { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); conf.SetLong(DFSConfigKeys.DfsNamenodeDelegationTokenMaxLifetimeKey, 10000); conf.SetLong(DFSConfigKeys.DfsNamenodeDelegationTokenRenewIntervalKey, 5000); conf.SetBoolean(DFSConfigKeys.DfsNamenodeDelegationTokenAlwaysUseKey, true); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthToLocal, "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT"); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); DistributedFileSystem hdfs = cluster.GetFileSystem(); // Create a reasonable namespace for (int i = 0; i < NumDirs; i++) { Path dir = new Path("/dir" + i); hdfs.Mkdirs(dir); writtenFiles[dir.ToString()] = PathToFileEntry(hdfs, dir.ToString()); for (int j = 0; j < FilesPerDir; j++) { Path file = new Path(dir, "file" + j); FSDataOutputStream o = hdfs.Create(file); o.Write(23); o.Close(); writtenFiles[file.ToString()] = PathToFileEntry(hdfs, file.ToString()); } } // Create an empty directory Path emptydir = new Path("/emptydir"); hdfs.Mkdirs(emptydir); writtenFiles[emptydir.ToString()] = hdfs.GetFileStatus(emptydir); //Create a directory whose name should be escaped in XML Path invalidXMLDir = new Path("/dirContainingInvalidXMLChar\u0000here"); hdfs.Mkdirs(invalidXMLDir); // Get delegation tokens so we log the delegation token op Org.Apache.Hadoop.Security.Token.Token <object>[] delegationTokens = hdfs.AddDelegationTokens (TestRenewer, null); foreach (Org.Apache.Hadoop.Security.Token.Token <object> t in delegationTokens) { Log.Debug("got token " + t); } Path snapshot = new Path("/snapshot"); hdfs.Mkdirs(snapshot); hdfs.AllowSnapshot(snapshot); hdfs.Mkdirs(new Path("/snapshot/1")); hdfs.Delete(snapshot, true); // Set XAttrs so the fsimage contains XAttr ops Path xattr = new Path("/xattr"); hdfs.Mkdirs(xattr); hdfs.SetXAttr(xattr, "user.a1", new byte[] { unchecked ((int)(0x31)), unchecked ((int )(0x32)), unchecked ((int)(0x33)) }); hdfs.SetXAttr(xattr, "user.a2", new byte[] { unchecked ((int)(0x37)), unchecked ((int )(0x38)), unchecked ((int)(0x39)) }); // OIV should be able to handle empty value XAttrs hdfs.SetXAttr(xattr, "user.a3", null); writtenFiles[xattr.ToString()] = hdfs.GetFileStatus(xattr); // Write results to the fsimage file hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter, false); hdfs.SaveNamespace(); // Determine location of fsimage file originalFsimage = FSImageTestUtil.FindLatestImageFile(FSImageTestUtil.GetFSImage( cluster.GetNameNode()).GetStorage().GetStorageDir(0)); if (originalFsimage == null) { throw new RuntimeException("Didn't generate or can't find fsimage"); } Log.Debug("original FS image file is " + originalFsimage); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public static void CreateOriginalFSImage() { MiniDFSCluster cluster = null; try { Configuration conf = new Configuration(); conf.SetBoolean(DFSConfigKeys.DfsNamenodeAclsEnabledKey, true); cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); DistributedFileSystem hdfs = cluster.GetFileSystem(); // Create a reasonable namespace with ACLs Path dir = new Path("/dirWithNoAcl"); hdfs.Mkdirs(dir); writtenAcls[dir.ToString()] = hdfs.GetAclStatus(dir); dir = new Path("/dirWithDefaultAcl"); hdfs.Mkdirs(dir); hdfs.SetAcl(dir, Lists.NewArrayList(AclTestHelpers.AclEntry(AclEntryScope.Default , AclEntryType.User, FsAction.All), AclTestHelpers.AclEntry(AclEntryScope.Default , AclEntryType.User, "foo", FsAction.All), AclTestHelpers.AclEntry(AclEntryScope .Default, AclEntryType.Group, FsAction.ReadExecute), AclTestHelpers.AclEntry(AclEntryScope .Default, AclEntryType.Other, FsAction.None))); writtenAcls[dir.ToString()] = hdfs.GetAclStatus(dir); Path file = new Path("/noAcl"); FSDataOutputStream o = hdfs.Create(file); o.Write(23); o.Close(); writtenAcls[file.ToString()] = hdfs.GetAclStatus(file); file = new Path("/withAcl"); o = hdfs.Create(file); o.Write(23); o.Close(); hdfs.SetAcl(file, Lists.NewArrayList(AclTestHelpers.AclEntry(AclEntryScope.Access , AclEntryType.User, FsAction.ReadWrite), AclTestHelpers.AclEntry(AclEntryScope. Access, AclEntryType.User, "foo", FsAction.Read), AclTestHelpers.AclEntry(AclEntryScope .Access, AclEntryType.Group, FsAction.Read), AclTestHelpers.AclEntry(AclEntryScope .Access, AclEntryType.Other, FsAction.None))); writtenAcls[file.ToString()] = hdfs.GetAclStatus(file); file = new Path("/withSeveralAcls"); o = hdfs.Create(file); o.Write(23); o.Close(); hdfs.SetAcl(file, Lists.NewArrayList(AclTestHelpers.AclEntry(AclEntryScope.Access , AclEntryType.User, FsAction.ReadWrite), AclTestHelpers.AclEntry(AclEntryScope. Access, AclEntryType.User, "foo", FsAction.ReadWrite), AclTestHelpers.AclEntry(AclEntryScope .Access, AclEntryType.User, "bar", FsAction.Read), AclTestHelpers.AclEntry(AclEntryScope .Access, AclEntryType.Group, FsAction.Read), AclTestHelpers.AclEntry(AclEntryScope .Access, AclEntryType.Group, "group", FsAction.Read), AclTestHelpers.AclEntry(AclEntryScope .Access, AclEntryType.Other, FsAction.None))); writtenAcls[file.ToString()] = hdfs.GetAclStatus(file); // Write results to the fsimage file hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter, false); hdfs.SaveNamespace(); // Determine the location of the fsimage file originalFsimage = FSImageTestUtil.FindLatestImageFile(FSImageTestUtil.GetFSImage( cluster.GetNameNode()).GetStorage().GetStorageDir(0)); if (originalFsimage == null) { throw new RuntimeException("Didn't generate or can't find fsimage"); } Log.Debug("original FS image file is " + originalFsimage); } finally { if (cluster != null) { cluster.Shutdown(); } } }