/// <exception cref="System.IO.IOException"/> protected internal void MakeTestFile(Path path, long length, bool isLazyPersist) { EnumSet <CreateFlag> createFlags = EnumSet.Of(CreateFlag.Create); if (isLazyPersist) { createFlags.AddItem(CreateFlag.LazyPersist); } FSDataOutputStream fos = null; try { fos = fs.Create(path, FsPermission.GetFileDefault(), createFlags, BufferLength, ReplFactor , BlockSize, null); // Allocate a block. byte[] buffer = new byte[BufferLength]; for (int bytesWritten = 0; bytesWritten < length;) { fos.Write(buffer, 0, buffer.Length); bytesWritten += buffer.Length; } if (length > 0) { fos.Hsync(); } } finally { IOUtils.CloseQuietly(fos); } }
public virtual void ConstructorBlank() { FileStatus fileStatus = new FileStatus(); ValidateAccessors(fileStatus, 0, false, 0, 0, 0, 0, FsPermission.GetFileDefault() , string.Empty, string.Empty, null, null); }
/// <summary>Constructor</summary> /// <param name="length">the number of bytes the file has</param> /// <param name="isdir">if the path is a directory</param> /// <param name="block_replication">the replication factor</param> /// <param name="blocksize">the block size</param> /// <param name="modification_time">modification time</param> /// <param name="access_time">access time</param> /// <param name="permission">permission</param> /// <param name="owner">the owner of the path</param> /// <param name="group">the group of the path</param> /// <param name="path">the local name in java UTF8 encoding the same as that in-memory /// </param> /// <param name="fileId">the file id</param> /// <param name="feInfo">the file's encryption info</param> public HdfsFileStatus(long length, bool isdir, int block_replication, long blocksize , long modification_time, long access_time, FsPermission permission, string owner , string group, byte[] symlink, byte[] path, long fileId, int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy) { // local name of the inode that's encoded in java UTF8 // symlink target encoded in java UTF8 or null // Used by dir, not including dot and dotdot. Always zero for a regular file. this.length = length; this.isdir = isdir; this.block_replication = (short)block_replication; this.blocksize = blocksize; this.modification_time = modification_time; this.access_time = access_time; this.permission = (permission == null) ? ((isdir || symlink != null) ? FsPermission .GetDefault() : FsPermission.GetFileDefault()) : permission; this.owner = (owner == null) ? string.Empty : owner; this.group = (group == null) ? string.Empty : group; this.symlink = symlink; this.path = path; this.fileId = fileId; this.childrenNum = childrenNum; this.feInfo = feInfo; this.storagePolicy = storagePolicy; }
public virtual void TestAddBlockRetryShouldReturnBlockWithLocations() { string src = "/testAddBlockRetryShouldReturnBlockWithLocations"; NamenodeProtocols nameNodeRpc = cluster.GetNameNodeRpc(); // create file nameNodeRpc.Create(src, FsPermission.GetFileDefault(), "clientName", new EnumSetWritable <CreateFlag>(EnumSet.Of(CreateFlag.Create)), true, (short)3, 1024, null); // start first addBlock() Log.Info("Starting first addBlock for " + src); LocatedBlock lb1 = nameNodeRpc.AddBlock(src, "clientName", null, null, INodeId.GrandfatherInodeId , null); NUnit.Framework.Assert.IsTrue("Block locations should be present", lb1.GetLocations ().Length > 0); cluster.RestartNameNode(); nameNodeRpc = cluster.GetNameNodeRpc(); LocatedBlock lb2 = nameNodeRpc.AddBlock(src, "clientName", null, null, INodeId.GrandfatherInodeId , null); NUnit.Framework.Assert.AreEqual("Blocks are not equal", lb1.GetBlock(), lb2.GetBlock ()); NUnit.Framework.Assert.IsTrue("Wrong locations with retry", lb2.GetLocations().Length > 0); }
public virtual void TestRetryAddBlockWhileInChooseTarget() { string src = "/testRetryAddBlockWhileInChooseTarget"; FSNamesystem ns = cluster.GetNamesystem(); NamenodeProtocols nn = cluster.GetNameNodeRpc(); // create file nn.Create(src, FsPermission.GetFileDefault(), "clientName", new EnumSetWritable <CreateFlag >(EnumSet.Of(CreateFlag.Create)), true, (short)3, 1024, null); // start first addBlock() Log.Info("Starting first addBlock for " + src); LocatedBlock[] onRetryBlock = new LocatedBlock[1]; DatanodeStorageInfo[] targets = ns.GetNewBlockTargets(src, INodeId.GrandfatherInodeId , "clientName", null, null, null, onRetryBlock); NUnit.Framework.Assert.IsNotNull("Targets must be generated", targets); // run second addBlock() Log.Info("Starting second addBlock for " + src); nn.AddBlock(src, "clientName", null, null, INodeId.GrandfatherInodeId, null); NUnit.Framework.Assert.IsTrue("Penultimate block must be complete", CheckFileProgress (src, false)); LocatedBlocks lbs = nn.GetBlockLocations(src, 0, long.MaxValue); NUnit.Framework.Assert.AreEqual("Must be one block", 1, lbs.GetLocatedBlocks().Count ); LocatedBlock lb2 = lbs.Get(0); NUnit.Framework.Assert.AreEqual("Wrong replication", Replication, lb2.GetLocations ().Length); // continue first addBlock() LocatedBlock newBlock = ns.StoreAllocatedBlock(src, INodeId.GrandfatherInodeId, "clientName" , null, targets); NUnit.Framework.Assert.AreEqual("Blocks are not equal", lb2.GetBlock(), newBlock. GetBlock()); // check locations lbs = nn.GetBlockLocations(src, 0, long.MaxValue); NUnit.Framework.Assert.AreEqual("Must be one block", 1, lbs.GetLocatedBlocks().Count ); LocatedBlock lb1 = lbs.Get(0); NUnit.Framework.Assert.AreEqual("Wrong replication", Replication, lb1.GetLocations ().Length); NUnit.Framework.Assert.AreEqual("Blocks are not equal", lb1.GetBlock(), lb2.GetBlock ()); }
public FileStatus(long length, bool isdir, int block_replication, long blocksize, long modification_time, long access_time, FsPermission permission, string owner, string group, Path symlink, Path path) { //We should deprecate this soon? this.length = length; this.isdir = isdir; this.block_replication = (short)block_replication; this.blocksize = blocksize; this.modification_time = modification_time; this.access_time = access_time; if (permission != null) { this.permission = permission; } else { if (isdir) { this.permission = FsPermission.GetDirDefault(); } else { if (symlink != null) { this.permission = FsPermission.GetDefault(); } else { this.permission = FsPermission.GetFileDefault(); } } } this.owner = (owner == null) ? string.Empty : owner; this.group = (group == null) ? string.Empty : group; this.symlink = symlink; this.path = path; // The variables isdir and symlink indicate the type: // 1. isdir implies directory, in which case symlink must be null. // 2. !isdir implies a file or symlink, symlink != null implies a // symlink, otherwise it's a file. System.Diagnostics.Debug.Assert((isdir && symlink == null) || !isdir); }
/// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/> /// <exception cref="Org.Apache.Hadoop.FS.FileAlreadyExistsException"/> /// <exception cref="System.IO.FileNotFoundException"/> /// <exception cref="Org.Apache.Hadoop.FS.ParentNotDirectoryException"/> /// <exception cref="Org.Apache.Hadoop.FS.UnsupportedFileSystemException"/> /// <exception cref="System.IO.IOException"/> public override FSDataOutputStream Create(Path f, EnumSet <CreateFlag> createFlag, params Options.CreateOpts[] opts) { // Need to translate the FileContext-style options into FileSystem-style // Permissions with umask Options.CreateOpts.Perms permOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.Perms >(opts); FsPermission umask = FsPermission.GetUMask(fs.GetConf()); FsPermission permission = (permOpt != null) ? permOpt.GetValue() : FsPermission.GetFileDefault ().ApplyUMask(umask); permission = permission.ApplyUMask(umask); // Overwrite bool overwrite = createFlag.Contains(CreateFlag.Overwrite); // bufferSize int bufferSize = fs.GetConf().GetInt(CommonConfigurationKeysPublic.IoFileBufferSizeKey , CommonConfigurationKeysPublic.IoFileBufferSizeDefault); Options.CreateOpts.BufferSize bufOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.BufferSize >(opts); bufferSize = (bufOpt != null) ? bufOpt.GetValue() : bufferSize; // replication short replication = fs.GetDefaultReplication(f); Options.CreateOpts.ReplicationFactor repOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.ReplicationFactor >(opts); replication = (repOpt != null) ? repOpt.GetValue() : replication; // blockSize long blockSize = fs.GetDefaultBlockSize(f); Options.CreateOpts.BlockSize blockOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.BlockSize >(opts); blockSize = (blockOpt != null) ? blockOpt.GetValue() : blockSize; // Progressable Progressable progress = null; Options.CreateOpts.Progress progressOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.Progress >(opts); progress = (progressOpt != null) ? progressOpt.GetValue() : progress; return(fs.Create(f, permission, overwrite, bufferSize, replication, blockSize, progress )); }
/// <summary>Test LocalFileSystem.setPermission</summary> /// <exception cref="System.IO.IOException"/> public virtual void TestLocalFSsetPermission() { if (Path.Windows) { System.Console.Out.WriteLine("Cannot run test for Windows"); return; } Configuration conf = new Configuration(); LocalFileSystem localfs = FileSystem.GetLocal(conf); string filename = "foo"; Path f = WriteFile(localfs, filename); try { FsPermission initialPermission = GetPermission(localfs, f); System.Console.Out.WriteLine(filename + ": " + initialPermission); Assert.Equal(FsPermission.GetFileDefault().ApplyUMask(FsPermission .GetUMask(conf)), initialPermission); } catch (Exception e) { System.Console.Out.WriteLine(StringUtils.StringifyException(e)); System.Console.Out.WriteLine("Cannot run test"); return; } try { // create files and manipulate them. FsPermission all = new FsPermission((short)0x1ff); FsPermission none = new FsPermission((short)0); localfs.SetPermission(f, none); Assert.Equal(none, GetPermission(localfs, f)); localfs.SetPermission(f, all); Assert.Equal(all, GetPermission(localfs, f)); } finally { CleanupFile(localfs, f); } }
// just in case copyBytes didn't // tag created files as temp files /// <exception cref="System.IO.IOException"/> internal virtual FSDataOutputStream Create(PathData item, bool lazyPersist) { try { if (lazyPersist) { EnumSet <CreateFlag> createFlags = EnumSet.Of(CreateFlag.Create, CreateFlag.LazyPersist ); return(Create(item.path, FsPermission.GetFileDefault().ApplyUMask(FsPermission.GetUMask (GetConf())), createFlags, GetConf().GetInt("io.file.buffer.size", 4096), lazyPersist ? 1 : GetDefaultReplication(item.path), GetDefaultBlockSize(), null, null)); } else { return(Create(item.path, true)); } } finally { // might have been created but stream was interrupted DeleteOnExit(item.path); } }
/* These are provided so that these values could be loaded lazily * by a filesystem (e.g. local file system). */ /// <summary>Sets permission.</summary> /// <param name="permission">if permission is null, default value is set</param> protected internal virtual void SetPermission(FsPermission permission) { this.permission = (permission == null) ? FsPermission.GetFileDefault() : permission; }