/// <exception cref="System.Exception"/> public virtual void TestClientAccessPrivilegeForRemove() { // Configure ro access for nfs1 service config.Set("dfs.nfs.exports.allowed.hosts", "* ro"); // Start nfs Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 nfs = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 (config); nfs.StartServiceInternal(false); RpcProgramNfs3 nfsd = (RpcProgramNfs3)nfs.GetRpcProgram(); // Create a remove request HdfsFileStatus status = nn.GetRpcServer().GetFileInfo(testdir); long dirId = status.GetFileId(); XDR xdr_req = new XDR(); FileHandle handle = new FileHandle(dirId); handle.Serialize(xdr_req); xdr_req.WriteString("f1"); // Remove operation REMOVE3Response response = nfsd.Remove(xdr_req.AsReadOnlyWrap(), securityHandler, new IPEndPoint("localhost", 1234)); // Assert on return code NUnit.Framework.Assert.AreEqual("Incorrect return code", Nfs3Status.Nfs3errAcces, response.GetStatus()); }
public virtual void TestCreate() { string src = "/testNamenodeRetryCache/testCreate/file"; // Two retried calls succeed NewCall(); HdfsFileStatus status = nnRpc.Create(src, perm, "holder", new EnumSetWritable <CreateFlag >(EnumSet.Of(CreateFlag.Create)), true, (short)1, BlockSize, null); NUnit.Framework.Assert.AreEqual(status, nnRpc.Create(src, perm, "holder", new EnumSetWritable <CreateFlag>(EnumSet.Of(CreateFlag.Create)), true, (short)1, BlockSize, null)); NUnit.Framework.Assert.AreEqual(status, nnRpc.Create(src, perm, "holder", new EnumSetWritable <CreateFlag>(EnumSet.Of(CreateFlag.Create)), true, (short)1, BlockSize, null)); // A non-retried call fails NewCall(); try { nnRpc.Create(src, perm, "holder", new EnumSetWritable <CreateFlag>(EnumSet.Of(CreateFlag .Create)), true, (short)1, BlockSize, null); NUnit.Framework.Assert.Fail("testCreate - expected exception is not thrown"); } catch (IOException) { } }
/// <exception cref="System.Exception"/> public virtual void TestMkdir() { //FixME HdfsFileStatus status = nn.GetRpcServer().GetFileInfo(testdir); long dirId = status.GetFileId(); XDR xdr_req = new XDR(); FileHandle handle = new FileHandle(dirId); MKDIR3Request req = new MKDIR3Request(handle, "fubar1", new SetAttr3()); req.Serialize(xdr_req); // Attempt to mkdir by an unprivileged user should fail. MKDIR3Response response1 = nfsd.Mkdir(xdr_req.AsReadOnlyWrap(), securityHandlerUnpriviledged , new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.AreEqual("Incorrect return code:", Nfs3Status.Nfs3errAcces , response1.GetStatus()); XDR xdr_req2 = new XDR(); MKDIR3Request req2 = new MKDIR3Request(handle, "fubar2", new SetAttr3()); req2.Serialize(xdr_req2); // Attempt to mkdir by a privileged user should pass. MKDIR3Response response2 = nfsd.Mkdir(xdr_req2.AsReadOnlyWrap(), securityHandler, new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.AreEqual("Incorrect return code:", Nfs3Status.Nfs3Ok, response2 .GetStatus()); }
/// <exception cref="System.IO.IOException"/> public Void Run() { ClientProtocol nn = this._enclosing.CreateNameNodeProxy(); string path = ServletUtil.GetDecodedPath(request, "/data"); string encodedPath = ServletUtil.GetRawPath(request, "/data"); string delegationToken = request.GetParameter(JspHelper.DelegationParameterName); HdfsFileStatus info = nn.GetFileInfo(path); if (info != null && !info.IsDir()) { response.SendRedirect(this._enclosing.CreateRedirectURL(path, encodedPath, info, ugi, nn, request, delegationToken).ToString()); } else { if (info == null) { response.SendError(400, "File not found " + path); } else { response.SendError(400, path + ": is a directory"); } } return(null); }
internal static FSDirRenameOp.RenameOldResult RenameToInt(FSDirectory fsd, string srcArg, string dstArg, bool logRetryCache) { string src = srcArg; string dst = dstArg; if (NameNode.stateChangeLog.IsDebugEnabled()) { NameNode.stateChangeLog.Debug("DIR* NameSystem.renameTo: " + src + " to " + dst); } if (!DFSUtil.IsValidName(dst)) { throw new IOException("Invalid name: " + dst); } FSPermissionChecker pc = fsd.GetPermissionChecker(); byte[][] srcComponents = FSDirectory.GetPathComponentsForReservedPath(src); byte[][] dstComponents = FSDirectory.GetPathComponentsForReservedPath(dst); HdfsFileStatus resultingStat = null; src = fsd.ResolvePath(pc, src, srcComponents); dst = fsd.ResolvePath(pc, dst, dstComponents); bool status = RenameTo(fsd, pc, src, dst, logRetryCache); if (status) { INodesInPath dstIIP = fsd.GetINodesInPath(dst, false); resultingStat = fsd.GetAuditFileInfo(dstIIP); } return(new FSDirRenameOp.RenameOldResult(status, resultingStat)); }
/* fsid */ /// <exception cref="System.IO.IOException"/> public static Nfs3FileAttributes GetFileAttr(DFSClient client, string fileIdPath, IdMappingServiceProvider iug) { HdfsFileStatus fs = GetFileStatus(client, fileIdPath); return(fs == null ? null : GetNfs3FileAttrFromFileStatus(fs, iug)); }
internal static FileStatus ToFileStatus(HdfsFileStatus f, string parent) { return(new FileStatus(f.GetLen(), f.IsDir(), f.GetReplication(), f.GetBlockSize() , f.GetModificationTime(), f.GetAccessTime(), f.GetPermission(), f.GetOwner(), f .GetGroup(), f.IsSymlink() ? new Path(f.GetSymlink()) : null, new Path(f.GetFullName (parent)))); }
/// <summary>Create a redirection URL</summary> /// <exception cref="System.IO.IOException"/> private Uri CreateRedirectURL(string path, string encodedPath, HdfsFileStatus status , UserGroupInformation ugi, ClientProtocol nnproxy, HttpServletRequest request, string dt) { string scheme = request.GetScheme(); LocatedBlocks blks = nnproxy.GetBlockLocations(status.GetFullPath(new Path(path)) .ToUri().GetPath(), 0, 1); Configuration conf = NameNodeHttpServer.GetConfFromContext(GetServletContext()); DatanodeID host = PickSrcDatanode(blks, status, conf); string hostname; if (host is DatanodeInfo) { hostname = host.GetHostName(); } else { hostname = host.GetIpAddr(); } int port = "https".Equals(scheme) ? host.GetInfoSecurePort() : host.GetInfoPort(); string dtParam = string.Empty; if (dt != null) { dtParam = JspHelper.GetDelegationTokenUrlParam(dt); } // Add namenode address to the url params NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext()); string addr = nn.GetNameNodeAddressHostPortString(); string addrParam = JspHelper.GetUrlParam(JspHelper.NamenodeAddress, addr); return(new Uri(scheme, hostname, port, "/streamFile" + encodedPath + '?' + "ugi=" + ServletUtil.EncodeQueryValue(ugi.GetShortUserName()) + dtParam + addrParam)); }
/// <summary>Get a listing of all the snapshots of a snapshottable directory</summary> /// <exception cref="System.IO.IOException"/> private static DirectoryListing GetSnapshotsListing(FSDirectory fsd, string src, byte[] startAfter) { Preconditions.CheckState(fsd.HasReadLock()); Preconditions.CheckArgument(src.EndsWith(HdfsConstants.SeparatorDotSnapshotDir), "%s does not end with %s", src, HdfsConstants.SeparatorDotSnapshotDir); string dirPath = FSDirectory.NormalizePath(Sharpen.Runtime.Substring(src, 0, src. Length - HdfsConstants.DotSnapshotDir.Length)); INode node = fsd.GetINode(dirPath); INodeDirectory dirNode = INodeDirectory.ValueOf(node, dirPath); DirectorySnapshottableFeature sf = dirNode.GetDirectorySnapshottableFeature(); if (sf == null) { throw new SnapshotException("Directory is not a snapshottable directory: " + dirPath ); } ReadOnlyList <Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot> snapshots = sf.GetSnapshotList(); int skipSize = ReadOnlyList.Util.BinarySearch(snapshots, startAfter); skipSize = skipSize < 0 ? -skipSize - 1 : skipSize + 1; int numOfListing = Math.Min(snapshots.Size() - skipSize, fsd.GetLsLimit()); HdfsFileStatus[] listing = new HdfsFileStatus[numOfListing]; for (int i = 0; i < numOfListing; i++) { Snapshot.Root sRoot = snapshots.Get(i + skipSize).GetRoot(); listing[i] = CreateFileStatus(fsd, src, sRoot.GetLocalNameBytes(), sRoot, BlockStoragePolicySuite .IdUnspecified, Org.Apache.Hadoop.Hdfs.Server.Namenode.Snapshot.Snapshot.CurrentStateId , false, INodesInPath.FromINode(sRoot)); } return(new DirectoryListing(listing, snapshots.Size() - skipSize - numOfListing)); }
/// <summary>The new rename which has the POSIX semantic.</summary> /// <exception cref="System.IO.IOException"/> internal static KeyValuePair <INode.BlocksMapUpdateInfo, HdfsFileStatus> RenameToInt (FSDirectory fsd, string srcArg, string dstArg, bool logRetryCache, params Options.Rename [] options) { string src = srcArg; string dst = dstArg; if (NameNode.stateChangeLog.IsDebugEnabled()) { NameNode.stateChangeLog.Debug("DIR* NameSystem.renameTo: with options -" + " " + src + " to " + dst); } if (!DFSUtil.IsValidName(dst)) { throw new InvalidPathException("Invalid name: " + dst); } FSPermissionChecker pc = fsd.GetPermissionChecker(); byte[][] srcComponents = FSDirectory.GetPathComponentsForReservedPath(src); byte[][] dstComponents = FSDirectory.GetPathComponentsForReservedPath(dst); INode.BlocksMapUpdateInfo collectedBlocks = new INode.BlocksMapUpdateInfo(); src = fsd.ResolvePath(pc, src, srcComponents); dst = fsd.ResolvePath(pc, dst, dstComponents); RenameTo(fsd, pc, src, dst, collectedBlocks, logRetryCache, options); INodesInPath dstIIP = fsd.GetINodesInPath(dst, false); HdfsFileStatus resultingStat = fsd.GetAuditFileInfo(dstIIP); return(new AbstractMap.SimpleImmutableEntry <INode.BlocksMapUpdateInfo, HdfsFileStatus >(collectedBlocks, resultingStat)); }
/// <exception cref="System.Exception"/> public virtual void TestWrite() { HdfsFileStatus status = nn.GetRpcServer().GetFileInfo("/tmp/bar"); long dirId = status.GetFileId(); FileHandle handle = new FileHandle(dirId); byte[] buffer = new byte[10]; for (int i = 0; i < 10; i++) { buffer[i] = unchecked ((byte)i); } WRITE3Request writeReq = new WRITE3Request(handle, 0, 10, Nfs3Constant.WriteStableHow .DataSync, ByteBuffer.Wrap(buffer)); XDR xdr_req = new XDR(); writeReq.Serialize(xdr_req); // Attempt by an unpriviledged user should fail. WRITE3Response response1 = nfsd.Write(xdr_req.AsReadOnlyWrap(), null, 1, securityHandlerUnpriviledged , new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.AreEqual("Incorrect return code:", Nfs3Status.Nfs3errAcces , response1.GetStatus()); // Attempt by a priviledged user should pass. WRITE3Response response2 = nfsd.Write(xdr_req.AsReadOnlyWrap(), null, 1, securityHandler , new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.AreEqual("Incorrect response:", null, response2); }
/// <exception cref="System.Exception"/> public virtual void TestReadlink() { // Create a symlink first. HdfsFileStatus status = nn.GetRpcServer().GetFileInfo(testdir); long dirId = status.GetFileId(); XDR xdr_req = new XDR(); FileHandle handle = new FileHandle(dirId); SYMLINK3Request req = new SYMLINK3Request(handle, "fubar", new SetAttr3(), "bar"); req.Serialize(xdr_req); SYMLINK3Response response = nfsd.Symlink(xdr_req.AsReadOnlyWrap(), securityHandler , new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.AreEqual("Incorrect return code:", Nfs3Status.Nfs3Ok, response .GetStatus()); // Now perform readlink operations. FileHandle handle2 = response.GetObjFileHandle(); XDR xdr_req2 = new XDR(); READLINK3Request req2 = new READLINK3Request(handle2); req2.Serialize(xdr_req2); // Attempt by an unpriviledged user should fail. READLINK3Response response1 = nfsd.Readlink(xdr_req2.AsReadOnlyWrap(), securityHandlerUnpriviledged , new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.AreEqual("Incorrect return code:", Nfs3Status.Nfs3errAcces , response1.GetStatus()); // Attempt by a priviledged user should pass. READLINK3Response response2 = nfsd.Readlink(xdr_req2.AsReadOnlyWrap(), securityHandler , new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.AreEqual("Incorrect return code:", Nfs3Status.Nfs3Ok, response2 .GetStatus()); }
public virtual void TestReaddirPlus() { // Test readdirplus // Get inodeId of /tmp HdfsFileStatus status = nn.GetRpcServer().GetFileInfo(testdir); long dirId = status.GetFileId(); // Create related part of the XDR request XDR xdr_req = new XDR(); FileHandle handle = new FileHandle(dirId); handle.Serialize(xdr_req); xdr_req.WriteLongAsHyper(0); // cookie xdr_req.WriteLongAsHyper(0); // verifier xdr_req.WriteInt(100); // dirCount xdr_req.WriteInt(1000); // maxCount READDIRPLUS3Response responsePlus = nfsd.Readdirplus(xdr_req.AsReadOnlyWrap(), securityHandler , new IPEndPoint("localhost", 1234)); IList <READDIRPLUS3Response.EntryPlus3> direntPlus = responsePlus.GetDirListPlus() .GetEntries(); NUnit.Framework.Assert.IsTrue(direntPlus.Count == 5); // including dot, dotdot // Test start listing from f2 status = nn.GetRpcServer().GetFileInfo(testdir + "/f2"); long f2Id = status.GetFileId(); // Create related part of the XDR request xdr_req = new XDR(); handle = new FileHandle(dirId); handle.Serialize(xdr_req); xdr_req.WriteLongAsHyper(f2Id); // cookie xdr_req.WriteLongAsHyper(0); // verifier xdr_req.WriteInt(100); // dirCount xdr_req.WriteInt(1000); // maxCount responsePlus = nfsd.Readdirplus(xdr_req.AsReadOnlyWrap(), securityHandler, new IPEndPoint ("localhost", 1234)); direntPlus = responsePlus.GetDirListPlus().GetEntries(); NUnit.Framework.Assert.IsTrue(direntPlus.Count == 1); READDIRPLUS3Response.EntryPlus3 entryPlus = direntPlus[0]; NUnit.Framework.Assert.IsTrue(entryPlus.GetName().Equals("f3")); // When the cookie is deleted, list starts over no including dot, dotdot hdfs.Delete(new Path(testdir + "/f2"), false); responsePlus = nfsd.Readdirplus(xdr_req.AsReadOnlyWrap(), securityHandler, new IPEndPoint ("localhost", 1234)); direntPlus = responsePlus.GetDirListPlus().GetEntries(); NUnit.Framework.Assert.IsTrue(direntPlus.Count == 2); }
/// <summary>Select a datanode to service this request.</summary> /// <remarks> /// Select a datanode to service this request. /// Currently, this looks at no more than the first five blocks of a file, /// selecting a datanode randomly from the most represented. /// </remarks> /// <param name="conf"></param> /// <exception cref="System.IO.IOException"/> private DatanodeID PickSrcDatanode(LocatedBlocks blks, HdfsFileStatus i, Configuration conf) { if (i.GetLen() == 0 || blks.GetLocatedBlocks().Count <= 0) { // pick a random datanode NameNode nn = NameNodeHttpServer.GetNameNodeFromContext(GetServletContext()); return(NamenodeJspHelper.GetRandomDatanode(nn)); } return(JspHelper.BestNode(blks, conf)); }
public HdfsFileStatus HdfsFileStatus() { HdfsFileStatus status = new HdfsFileStatus ( Name: pathSuffix, Owner: owner, FileType: type.HdfsFileType(), Permission: permission ); return(status); }
public virtual void TestPolicyPropagation() { StartUpCluster(false, -1); string MethodName = GenericTestUtils.GetMethodName(); Path path = new Path("/" + MethodName + ".dat"); MakeTestFile(path, 0, true); // Stat the file and check that the lazyPersist flag is returned back. HdfsFileStatus status = client.GetFileInfo(path.ToString()); Assert.AssertThat(status.GetStoragePolicy(), IS.Is(LazyPersistPolicyId)); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.FS.UnresolvedLinkException"/> public override FileStatus GetFileStatus(Path f) { HdfsFileStatus fi = dfs.GetFileInfo(GetUriPath(f)); if (fi != null) { return(fi.MakeQualified(GetUri(), f)); } else { throw new FileNotFoundException("File does not exist: " + f.ToString()); } }
public virtual void TestPolicyNotSetByDefault() { StartUpCluster(false, -1); string MethodName = GenericTestUtils.GetMethodName(); Path path = new Path("/" + MethodName + ".dat"); MakeTestFile(path, 0, false); // Stat the file and check that the LAZY_PERSIST policy is not // returned back. HdfsFileStatus status = client.GetFileInfo(path.ToString()); Assert.AssertThat(status.GetStoragePolicy(), IsNot.Not(LazyPersistPolicyId)); }
/// <exception cref="System.IO.IOException"/> public static WccAttr GetWccAttr(DFSClient client, string fileIdPath) { HdfsFileStatus fstat = GetFileStatus(client, fileIdPath); if (fstat == null) { return(null); } long size = fstat.IsDir() ? GetDirSize(fstat.GetChildrenNum()) : fstat.GetLen(); return(new WccAttr(size, new NfsTime(fstat.GetModificationTime()), new NfsTime(fstat .GetModificationTime()))); }
public override XDR Mnt(XDR xdr, XDR @out, int xid, IPAddress client) { if (hostsMatcher == null) { return(MountResponse.WriteMNTResponse(Nfs3Status.Nfs3errAcces, @out, xid, null)); } AccessPrivilege accessPrivilege = hostsMatcher.GetAccessPrivilege(client); if (accessPrivilege == AccessPrivilege.None) { return(MountResponse.WriteMNTResponse(Nfs3Status.Nfs3errAcces, @out, xid, null)); } string path = xdr.ReadString(); if (Log.IsDebugEnabled()) { Log.Debug("MOUNT MNT path: " + path + " client: " + client); } string host = client.GetHostName(); if (Log.IsDebugEnabled()) { Log.Debug("Got host: " + host + " path: " + path); } if (!exports.Contains(path)) { Log.Info("Path " + path + " is not shared."); MountResponse.WriteMNTResponse(Nfs3Status.Nfs3errNoent, @out, xid, null); return(@out); } FileHandle handle = null; try { HdfsFileStatus exFileStatus = dfsClient.GetFileInfo(path); handle = new FileHandle(exFileStatus.GetFileId()); } catch (IOException e) { Log.Error("Can't get handle for export:" + path, e); MountResponse.WriteMNTResponse(Nfs3Status.Nfs3errNoent, @out, xid, null); return(@out); } System.Diagnostics.Debug.Assert((handle != null)); Log.Info("Giving handle (fileId:" + handle.GetFileId() + ") to client for export " + path); mounts.AddItem(new MountEntry(host, path)); MountResponse.WriteMNTResponse(Nfs3Status.Nfs3Ok, @out, xid, handle.GetContent()); return(@out); }
public static Nfs3FileAttributes GetNfs3FileAttrFromFileStatus(HdfsFileStatus fs, IdMappingServiceProvider iug) { NfsFileType fileType = fs.IsDir() ? NfsFileType.Nfsdir : NfsFileType.Nfsreg; fileType = fs.IsSymlink() ? NfsFileType.Nfslnk : fileType; int nlink = (fileType == NfsFileType.Nfsdir) ? fs.GetChildrenNum() + 2 : 1; long size = (fileType == NfsFileType.Nfsdir) ? GetDirSize(fs.GetChildrenNum()) : fs.GetLen(); return(new Nfs3FileAttributes(fileType, nlink, fs.GetPermission().ToShort(), iug. GetUidAllowingUnknown(fs.GetOwner()), iug.GetGidAllowingUnknown(fs.GetGroup()), size, 0, fs.GetFileId(), fs.GetModificationTime(), fs.GetAccessTime(), new Nfs3FileAttributes.Specdata3 ())); }
/// <exception cref="System.Exception"/> private void CreateFileUsingNfs(string fileName, byte[] buffer) { DFSTestUtil.CreateFile(hdfs, new Path(fileName), 0, (short)1, 0); HdfsFileStatus status = nn.GetRpcServer().GetFileInfo(fileName); long dirId = status.GetFileId(); FileHandle handle = new FileHandle(dirId); WRITE3Request writeReq = new WRITE3Request(handle, 0, buffer.Length, Nfs3Constant.WriteStableHow .DataSync, ByteBuffer.Wrap(buffer)); XDR xdr_req = new XDR(); writeReq.Serialize(xdr_req); WRITE3Response response = nfsd.Write(xdr_req.AsReadOnlyWrap(), null, 1, securityHandler , new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.AreEqual("Incorrect response: ", null, response); }
/// <exception cref="System.Exception"/> private void Commit(string fileName, int len) { HdfsFileStatus status = nn.GetRpcServer().GetFileInfo(fileName); long dirId = status.GetFileId(); FileHandle handle = new FileHandle(dirId); XDR xdr_req = new XDR(); COMMIT3Request req = new COMMIT3Request(handle, 0, len); req.Serialize(xdr_req); Org.Jboss.Netty.Channel.Channel ch = Org.Mockito.Mockito.Mock <Org.Jboss.Netty.Channel.Channel >(); COMMIT3Response response2 = nfsd.Commit(xdr_req.AsReadOnlyWrap(), ch, 1, securityHandler , new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.AreEqual("Incorrect COMMIT3Response:", null, response2); }
/// <summary>Convert a HdfsFileStatus object to a Json string.</summary> public static string ToJsonString(HdfsFileStatus status, bool includeType) { if (status == null) { return(null); } IDictionary <string, object> m = new SortedDictionary <string, object>(); m["pathSuffix"] = status.GetLocalName(); m["type"] = JsonUtil.PathType.ValueOf(status); if (status.IsSymlink()) { m["symlink"] = status.GetSymlink(); } m["length"] = status.GetLen(); m["owner"] = status.GetOwner(); m["group"] = status.GetGroup(); FsPermission perm = status.GetPermission(); m["permission"] = ToString(perm); if (perm.GetAclBit()) { m["aclBit"] = true; } if (perm.GetEncryptedBit()) { m["encBit"] = true; } m["accessTime"] = status.GetAccessTime(); m["modificationTime"] = status.GetModificationTime(); m["blockSize"] = status.GetBlockSize(); m["replication"] = status.GetReplication(); m["fileId"] = status.GetFileId(); m["childrenNum"] = status.GetChildrenNum(); m["storagePolicy"] = status.GetStoragePolicy(); ObjectMapper mapper = new ObjectMapper(); try { return(includeType ? ToJsonString(typeof(FileStatus), m) : mapper.WriteValueAsString (m)); } catch (IOException) { } return(null); }
public virtual void TestPolicyPersistenceInFsImage() { StartUpCluster(false, -1); string MethodName = GenericTestUtils.GetMethodName(); Path path = new Path("/" + MethodName + ".dat"); MakeTestFile(path, 0, true); // checkpoint fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); fs.SaveNamespace(); fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); cluster.RestartNameNode(true); // Stat the file and check that the lazyPersist flag is returned back. HdfsFileStatus status = client.GetFileInfo(path.ToString()); Assert.AssertThat(status.GetStoragePolicy(), IS.Is(LazyPersistPolicyId)); }
public void GetFileStatus_AddRandomFileWithRandomContent_ResultFileStatus() { //Assign WebHdfsClient client = GetWebHdfsClient(); string fileName = Guid.NewGuid().ToString(); String fileNamePath = $"/dewey/tests/{fileName}"; string text = $"{Guid.NewGuid()}"; Stream textStream = GenerateStreamFromString(text); bool resultWrite = client.WriteStream(textStream, fileNamePath).Result; //Act HdfsFileStatus fileStatus = client.GetFileStatus(fileNamePath).Result; //Assert Assert.IsTrue(resultWrite); Assert.AreEqual(HdfsFileType.FILE, fileStatus.FileType); }
/// <exception cref="System.Exception"/> private byte[] GetFileContentsUsingNfs(string fileName, int len) { HdfsFileStatus status = nn.GetRpcServer().GetFileInfo(fileName); long dirId = status.GetFileId(); FileHandle handle = new FileHandle(dirId); READ3Request readReq = new READ3Request(handle, 0, len); XDR xdr_req = new XDR(); readReq.Serialize(xdr_req); READ3Response response = nfsd.Read(xdr_req.AsReadOnlyWrap(), securityHandler, new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.AreEqual("Incorrect return code: ", Nfs3Status.Nfs3Ok, response .GetStatus()); NUnit.Framework.Assert.IsTrue("expected full read", response.IsEof()); return((byte[])response.GetData().Array()); }
/// <exception cref="System.IO.IOException"/> public virtual int Run(Configuration conf, IList <string> args) { string path = StringUtils.PopOptionWithArgument("-path", args); if (path == null) { System.Console.Error.WriteLine("Please specify the path with -path.\nUsage:" + GetLongUsage ()); return(1); } DistributedFileSystem dfs = AdminHelper.GetDFS(conf); try { HdfsFileStatus status = dfs.GetClient().GetFileInfo(path); if (status == null) { System.Console.Error.WriteLine("File/Directory does not exist: " + path); return(2); } byte storagePolicyId = status.GetStoragePolicy(); if (storagePolicyId == BlockStoragePolicySuite.IdUnspecified) { System.Console.Out.WriteLine("The storage policy of " + path + " is unspecified"); return(0); } BlockStoragePolicy[] policies = dfs.GetStoragePolicies(); foreach (BlockStoragePolicy p in policies) { if (p.GetId() == storagePolicyId) { System.Console.Out.WriteLine("The storage policy of " + path + ":\n" + p); return(0); } } } catch (Exception e) { System.Console.Error.WriteLine(AdminHelper.PrettifyException(e)); return(2); } System.Console.Error.WriteLine("Cannot identify the storage policy for " + path); return(2); }
/// <returns>whether the migration requires next round</returns> private bool ProcessRecursively(string parent, HdfsFileStatus status) { string fullPath = status.GetFullName(parent); bool hasRemaining = false; if (status.IsDir()) { if (!fullPath.EndsWith(Path.Separator)) { fullPath = fullPath + Path.Separator; } hasRemaining = this.ProcessPath(fullPath); // process snapshots if this is a snapshottable directory if (this.snapshottableDirs.Contains(fullPath)) { string dirSnapshot = fullPath + HdfsConstants.DotSnapshotDir; hasRemaining |= this.ProcessPath(dirSnapshot); } } else { if (!status.IsSymlink()) { // file try { if (!this.IsSnapshotPathInCurrent(fullPath)) { // the full path is a snapshot path but it is also included in the // current directory tree, thus ignore it. hasRemaining = this.ProcessFile(fullPath, (HdfsLocatedFileStatus)status); } } catch (IOException e) { Org.Apache.Hadoop.Hdfs.Server.Mover.Mover.Log.Warn("Failed to check the status of " + parent + ". Ignore it and continue.", e); return(false); } } } return(hasRemaining); }
public void MakeDirectory_MakerRandomDirectory_ResultListStatusDirectory() { //Assign WebHdfsClient client = GetWebHdfsClient(); string directoryName = Guid.NewGuid().ToString(); string rootPath = "/dewey/tests"; String directoryNamePath = $"{rootPath}/{directoryName}"; //Act bool resultMakeDirectory = client.MakeDirectory(directoryNamePath).Result; //Assert Assert.IsTrue(resultMakeDirectory); IEnumerable <HdfsFileStatus> fileStatus = client.ListStatus(rootPath).Result; HdfsFileStatus findStatus = fileStatus.SingleOrDefault(a => a.FileType == HdfsFileType.DIRECTORY && a.Name == directoryName); Assert.IsNotNull(findStatus); }