/// <exception cref="System.IO.IOException"/>
protected internal void MakeTestFile(Path path, long length, bool isLazyPersist)
{
EnumSet <CreateFlag> createFlags = EnumSet.Of(CreateFlag.Create);
if (isLazyPersist)
{
createFlags.AddItem(CreateFlag.LazyPersist);
}
FSDataOutputStream fos = null;
try
{
fos = fs.Create(path, FsPermission.GetFileDefault(), createFlags, BufferLength, ReplFactor
, BlockSize, null);
// Allocate a block.
byte[] buffer = new byte[BufferLength];
for (int bytesWritten = 0; bytesWritten < length;)
{
fos.Write(buffer, 0, buffer.Length);
bytesWritten += buffer.Length;
}
if (length > 0)
{
fos.Hsync();
}
}
finally
{
IOUtils.CloseQuietly(fos);
}
}
public virtual void ConstructorBlank()
{
FileStatus fileStatus = new FileStatus();
ValidateAccessors(fileStatus, 0, false, 0, 0, 0, 0, FsPermission.GetFileDefault()
, string.Empty, string.Empty, null, null);
}
/// <summary>Constructor</summary>
/// <param name="length">the number of bytes the file has</param>
/// <param name="isdir">if the path is a directory</param>
/// <param name="block_replication">the replication factor</param>
/// <param name="blocksize">the block size</param>
/// <param name="modification_time">modification time</param>
/// <param name="access_time">access time</param>
/// <param name="permission">permission</param>
/// <param name="owner">the owner of the path</param>
/// <param name="group">the group of the path</param>
/// <param name="path">the local name in java UTF8 encoding the same as that in-memory
/// </param>
/// <param name="fileId">the file id</param>
/// <param name="feInfo">the file's encryption info</param>
public HdfsFileStatus(long length, bool isdir, int block_replication, long blocksize
, long modification_time, long access_time, FsPermission permission, string owner
, string group, byte[] symlink, byte[] path, long fileId, int childrenNum, FileEncryptionInfo
feInfo, byte storagePolicy)
{
// local name of the inode that's encoded in java UTF8
// symlink target encoded in java UTF8 or null
// Used by dir, not including dot and dotdot. Always zero for a regular file.
this.length = length;
this.isdir = isdir;
this.block_replication = (short)block_replication;
this.blocksize = blocksize;
this.modification_time = modification_time;
this.access_time = access_time;
this.permission = (permission == null) ? ((isdir || symlink != null) ? FsPermission
.GetDefault() : FsPermission.GetFileDefault()) : permission;
this.owner = (owner == null) ? string.Empty : owner;
this.group = (group == null) ? string.Empty : group;
this.symlink = symlink;
this.path = path;
this.fileId = fileId;
this.childrenNum = childrenNum;
this.feInfo = feInfo;
this.storagePolicy = storagePolicy;
}
public virtual void TestAddBlockRetryShouldReturnBlockWithLocations()
{
string src = "/testAddBlockRetryShouldReturnBlockWithLocations";
NamenodeProtocols nameNodeRpc = cluster.GetNameNodeRpc();
// create file
nameNodeRpc.Create(src, FsPermission.GetFileDefault(), "clientName", new EnumSetWritable
<CreateFlag>(EnumSet.Of(CreateFlag.Create)), true, (short)3, 1024, null);
// start first addBlock()
Log.Info("Starting first addBlock for " + src);
LocatedBlock lb1 = nameNodeRpc.AddBlock(src, "clientName", null, null, INodeId.GrandfatherInodeId
, null);
NUnit.Framework.Assert.IsTrue("Block locations should be present", lb1.GetLocations
().Length > 0);
cluster.RestartNameNode();
nameNodeRpc = cluster.GetNameNodeRpc();
LocatedBlock lb2 = nameNodeRpc.AddBlock(src, "clientName", null, null, INodeId.GrandfatherInodeId
, null);
NUnit.Framework.Assert.AreEqual("Blocks are not equal", lb1.GetBlock(), lb2.GetBlock
());
NUnit.Framework.Assert.IsTrue("Wrong locations with retry", lb2.GetLocations().Length
> 0);
}
public virtual void TestRetryAddBlockWhileInChooseTarget()
{
string src = "/testRetryAddBlockWhileInChooseTarget";
FSNamesystem ns = cluster.GetNamesystem();
NamenodeProtocols nn = cluster.GetNameNodeRpc();
// create file
nn.Create(src, FsPermission.GetFileDefault(), "clientName", new EnumSetWritable <CreateFlag
>(EnumSet.Of(CreateFlag.Create)), true, (short)3, 1024, null);
// start first addBlock()
Log.Info("Starting first addBlock for " + src);
LocatedBlock[] onRetryBlock = new LocatedBlock[1];
DatanodeStorageInfo[] targets = ns.GetNewBlockTargets(src, INodeId.GrandfatherInodeId
, "clientName", null, null, null, onRetryBlock);
NUnit.Framework.Assert.IsNotNull("Targets must be generated", targets);
// run second addBlock()
Log.Info("Starting second addBlock for " + src);
nn.AddBlock(src, "clientName", null, null, INodeId.GrandfatherInodeId, null);
NUnit.Framework.Assert.IsTrue("Penultimate block must be complete", CheckFileProgress
(src, false));
LocatedBlocks lbs = nn.GetBlockLocations(src, 0, long.MaxValue);
NUnit.Framework.Assert.AreEqual("Must be one block", 1, lbs.GetLocatedBlocks().Count
);
LocatedBlock lb2 = lbs.Get(0);
NUnit.Framework.Assert.AreEqual("Wrong replication", Replication, lb2.GetLocations
().Length);
// continue first addBlock()
LocatedBlock newBlock = ns.StoreAllocatedBlock(src, INodeId.GrandfatherInodeId, "clientName"
, null, targets);
NUnit.Framework.Assert.AreEqual("Blocks are not equal", lb2.GetBlock(), newBlock.
GetBlock());
// check locations
lbs = nn.GetBlockLocations(src, 0, long.MaxValue);
NUnit.Framework.Assert.AreEqual("Must be one block", 1, lbs.GetLocatedBlocks().Count
);
LocatedBlock lb1 = lbs.Get(0);
NUnit.Framework.Assert.AreEqual("Wrong replication", Replication, lb1.GetLocations
().Length);
NUnit.Framework.Assert.AreEqual("Blocks are not equal", lb1.GetBlock(), lb2.GetBlock
());
}
public FileStatus(long length, bool isdir, int block_replication, long blocksize,
long modification_time, long access_time, FsPermission permission, string owner,
string group, Path symlink, Path path)
{
//We should deprecate this soon?
this.length = length;
this.isdir = isdir;
this.block_replication = (short)block_replication;
this.blocksize = blocksize;
this.modification_time = modification_time;
this.access_time = access_time;
if (permission != null)
{
this.permission = permission;
}
else
{
if (isdir)
{
this.permission = FsPermission.GetDirDefault();
}
else
{
if (symlink != null)
{
this.permission = FsPermission.GetDefault();
}
else
{
this.permission = FsPermission.GetFileDefault();
}
}
}
this.owner = (owner == null) ? string.Empty : owner;
this.group = (group == null) ? string.Empty : group;
this.symlink = symlink;
this.path = path;
// The variables isdir and symlink indicate the type:
// 1. isdir implies directory, in which case symlink must be null.
// 2. !isdir implies a file or symlink, symlink != null implies a
// symlink, otherwise it's a file.
System.Diagnostics.Debug.Assert((isdir && symlink == null) || !isdir);
}
/// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/>
/// <exception cref="Org.Apache.Hadoop.FS.FileAlreadyExistsException"/>
/// <exception cref="System.IO.FileNotFoundException"/>
/// <exception cref="Org.Apache.Hadoop.FS.ParentNotDirectoryException"/>
/// <exception cref="Org.Apache.Hadoop.FS.UnsupportedFileSystemException"/>
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream Create(Path f, EnumSet <CreateFlag> createFlag,
params Options.CreateOpts[] opts)
{
// Need to translate the FileContext-style options into FileSystem-style
// Permissions with umask
Options.CreateOpts.Perms permOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.Perms
>(opts);
FsPermission umask = FsPermission.GetUMask(fs.GetConf());
FsPermission permission = (permOpt != null) ? permOpt.GetValue() : FsPermission.GetFileDefault
().ApplyUMask(umask);
permission = permission.ApplyUMask(umask);
// Overwrite
bool overwrite = createFlag.Contains(CreateFlag.Overwrite);
// bufferSize
int bufferSize = fs.GetConf().GetInt(CommonConfigurationKeysPublic.IoFileBufferSizeKey
, CommonConfigurationKeysPublic.IoFileBufferSizeDefault);
Options.CreateOpts.BufferSize bufOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.BufferSize
>(opts);
bufferSize = (bufOpt != null) ? bufOpt.GetValue() : bufferSize;
// replication
short replication = fs.GetDefaultReplication(f);
Options.CreateOpts.ReplicationFactor repOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.ReplicationFactor
>(opts);
replication = (repOpt != null) ? repOpt.GetValue() : replication;
// blockSize
long blockSize = fs.GetDefaultBlockSize(f);
Options.CreateOpts.BlockSize blockOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.BlockSize
>(opts);
blockSize = (blockOpt != null) ? blockOpt.GetValue() : blockSize;
// Progressable
Progressable progress = null;
Options.CreateOpts.Progress progressOpt = Options.CreateOpts.GetOpt <Options.CreateOpts.Progress
>(opts);
progress = (progressOpt != null) ? progressOpt.GetValue() : progress;
return(fs.Create(f, permission, overwrite, bufferSize, replication, blockSize, progress
));
}
/// <summary>Test LocalFileSystem.setPermission</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestLocalFSsetPermission()
{
if (Path.Windows)
{
System.Console.Out.WriteLine("Cannot run test for Windows");
return;
}
Configuration conf = new Configuration();
LocalFileSystem localfs = FileSystem.GetLocal(conf);
string filename = "foo";
Path f = WriteFile(localfs, filename);
try
{
FsPermission initialPermission = GetPermission(localfs, f);
System.Console.Out.WriteLine(filename + ": " + initialPermission);
Assert.Equal(FsPermission.GetFileDefault().ApplyUMask(FsPermission
.GetUMask(conf)), initialPermission);
}
catch (Exception e)
{
System.Console.Out.WriteLine(StringUtils.StringifyException(e));
System.Console.Out.WriteLine("Cannot run test");
return;
}
try
{
// create files and manipulate them.
FsPermission all = new FsPermission((short)0x1ff);
FsPermission none = new FsPermission((short)0);
localfs.SetPermission(f, none);
Assert.Equal(none, GetPermission(localfs, f));
localfs.SetPermission(f, all);
Assert.Equal(all, GetPermission(localfs, f));
}
finally
{
CleanupFile(localfs, f);
}
}
// just in case copyBytes didn't
// tag created files as temp files
/// <exception cref="System.IO.IOException"/>
internal virtual FSDataOutputStream Create(PathData item, bool lazyPersist)
{
try
{
if (lazyPersist)
{
EnumSet <CreateFlag> createFlags = EnumSet.Of(CreateFlag.Create, CreateFlag.LazyPersist
);
return(Create(item.path, FsPermission.GetFileDefault().ApplyUMask(FsPermission.GetUMask
(GetConf())), createFlags, GetConf().GetInt("io.file.buffer.size", 4096), lazyPersist
? 1 : GetDefaultReplication(item.path), GetDefaultBlockSize(), null, null));
}
else
{
return(Create(item.path, true));
}
}
finally
{
// might have been created but stream was interrupted
DeleteOnExit(item.path);
}
}
/* These are provided so that these values could be loaded lazily
* by a filesystem (e.g. local file system).
*/
/// <summary>Sets permission.</summary>
/// <param name="permission">if permission is null, default value is set</param>
protected internal virtual void SetPermission(FsPermission permission)
{
this.permission = (permission == null) ? FsPermission.GetFileDefault() : permission;
}
