/// <summary>
/// Convenience method, so that we don't open a new connection when using this
/// method from within another method.
/// </summary>
/// <remarks>
/// Convenience method, so that we don't open a new connection when using this
/// method from within another method. Otherwise every API invocation incurs
/// the overhead of opening/closing a TCP connection.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
private bool Mkdirs(FTPClient client, Path file, FsPermission permission)
{
bool created = true;
Path workDir = new Path(client.PrintWorkingDirectory());
Path absolute = MakeAbsolute(workDir, file);
string pathName = absolute.GetName();
if (!Exists(client, absolute))
{
Path parent = absolute.GetParent();
created = (parent == null || Mkdirs(client, parent, FsPermission.GetDirDefault())
);
if (created)
{
string parentDir = parent.ToUri().GetPath();
client.ChangeWorkingDirectory(parentDir);
created = created && client.MakeDirectory(pathName);
}
}
else
{
if (IsFile(client, absolute))
{
throw new ParentNotDirectoryException(string.Format("Can't make directory for path %s since it is a file."
, absolute));
}
}
return(created);
}
/// <exception cref="System.IO.IOException"/>
private Path CreatePath(FileContext fc, Path root, string year, string month, string
day, string id)
{
Path path = new Path(root, year + Path.Separator + month + Path.Separator + day +
Path.Separator + id);
fc.Mkdir(path, FsPermission.GetDirDefault(), true);
return(path);
}
/// <summary>
/// Function to run a log handler with directories failing the getFileStatus
/// call.
/// </summary>
/// <remarks>
/// Function to run a log handler with directories failing the getFileStatus
/// call. The function accepts the log handler, setup the mocks to fail with
/// specific exceptions and ensures the deletion service has the correct calls.
/// </remarks>
/// <param name="logHandler">the logHandler implementation to test</param>
/// <param name="appId">
/// the application id that we wish when sending events to the log
/// handler
/// </param>
/// <param name="user">the user name to use</param>
/// <param name="mockDelService">
/// a mock of the DeletionService which we will verify
/// the delete calls against
/// </param>
/// <param name="dirsHandler">
/// a spy or mock on the LocalDirsHandler service used to
/// when creating the logHandler. It needs to be a spy so that we can intercept
/// the getAllLogDirs() call.
/// </param>
/// <param name="conf">the configuration used</param>
/// <param name="spylfs">a spy on the AbstractFileSystem object used when creating lfs
/// </param>
/// <param name="lfs">
/// the FileContext object to be used to mock the getFileStatus()
/// calls
/// </param>
/// <param name="localLogDirs">
/// list of the log dirs to run the test against, must have
/// at least 7 entries
/// </param>
/// <exception cref="System.Exception"/>
public static void RunMockedFailedDirs(LogHandler logHandler, ApplicationId appId
, string user, DeletionService mockDelService, LocalDirsHandlerService dirsHandler
, Configuration conf, AbstractFileSystem spylfs, FileContext lfs, FilePath[] localLogDirs
)
{
IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType
, string>();
if (localLogDirs.Length < 7)
{
throw new ArgumentException("Argument localLogDirs must be at least of length 7");
}
Path[] localAppLogDirPaths = new Path[localLogDirs.Length];
for (int i = 0; i < localAppLogDirPaths.Length; i++)
{
localAppLogDirPaths[i] = new Path(localLogDirs[i].GetAbsolutePath(), appId.ToString
());
}
IList <string> localLogDirPaths = new AList <string>(localLogDirs.Length);
for (int i_1 = 0; i_1 < localLogDirs.Length; i_1++)
{
localLogDirPaths.AddItem(localLogDirs[i_1].GetAbsolutePath());
}
// setup mocks
FsPermission defaultPermission = FsPermission.GetDirDefault().ApplyUMask(lfs.GetUMask
());
FileStatus fs = new FileStatus(0, true, 1, 0, Runtime.CurrentTimeMillis(), 0, defaultPermission
, string.Empty, string.Empty, new Path(localLogDirs[0].GetAbsolutePath()));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.IsA <Path>());
Org.Mockito.Mockito.DoReturn(localLogDirPaths).When(dirsHandler).GetLogDirsForCleanup
();
logHandler.Handle(new LogHandlerAppStartedEvent(appId, user, null, ContainerLogsRetentionPolicy
.AllContainers, appAcls));
// test case where some dirs have the log dir to delete
// mock some dirs throwing various exceptions
// verify deletion happens only on the others
Org.Mockito.Mockito.DoThrow(new FileNotFoundException()).When(spylfs).GetFileStatus
(Matchers.Eq(localAppLogDirPaths[0]));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths
[1]));
Org.Mockito.Mockito.DoThrow(new AccessControlException()).When(spylfs).GetFileStatus
(Matchers.Eq(localAppLogDirPaths[2]));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths
[3]));
Org.Mockito.Mockito.DoThrow(new IOException()).When(spylfs).GetFileStatus(Matchers.Eq
(localAppLogDirPaths[4]));
Org.Mockito.Mockito.DoThrow(new UnsupportedFileSystemException("test")).When(spylfs
).GetFileStatus(Matchers.Eq(localAppLogDirPaths[5]));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths
[6]));
logHandler.Handle(new LogHandlerAppFinishedEvent(appId));
TestDeletionServiceCall(mockDelService, user, 5000, localAppLogDirPaths[1], localAppLogDirPaths
[3], localAppLogDirPaths[6]);
return;
}
/// <exception cref="System.Exception"/>
public virtual void TestCreateEZPopulatesEDEKCache()
{
Path zonePath = new Path("/TestEncryptionZone");
fsWrapper.Mkdir(zonePath, FsPermission.GetDirDefault(), false);
dfsAdmin.CreateEncryptionZone(zonePath, TestKey);
NUnit.Framework.Assert.IsTrue(((KMSClientProvider)fs.GetClient().GetKeyProvider()
).GetEncKeyQueueSize(TestKey) > 0);
}
/// <summary>
/// A stream obtained via this call must be closed before using other APIs of
/// this class or else the invocation will block.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public override FSDataOutputStream Create(Path file, FsPermission permission, bool
overwrite, int bufferSize, short replication, long blockSize, Progressable progress
)
{
FTPClient client = Connect();
Path workDir = new Path(client.PrintWorkingDirectory());
Path absolute = MakeAbsolute(workDir, file);
FileStatus status;
try
{
status = GetFileStatus(client, file);
}
catch (FileNotFoundException)
{
status = null;
}
if (status != null)
{
if (overwrite && !status.IsDirectory())
{
Delete(client, file, false);
}
else
{
Disconnect(client);
throw new FileAlreadyExistsException("File already exists: " + file);
}
}
Path parent = absolute.GetParent();
if (parent == null || !Mkdirs(client, parent, FsPermission.GetDirDefault()))
{
parent = (parent == null) ? new Path("/") : parent;
Disconnect(client);
throw new IOException("create(): Mkdirs failed to create: " + parent);
}
client.Allocate(bufferSize);
// Change to parent directory on the server. Only then can we write to the
// file on the server by opening up an OutputStream. As a side effect the
// working directory on the server is changed to the parent directory of the
// file. The FTP client connection is closed when close() is called on the
// FSDataOutputStream.
client.ChangeWorkingDirectory(parent.ToUri().GetPath());
FSDataOutputStream fos = new _FSDataOutputStream_263(this, client, client.StoreFileStream
(file.GetName()), statistics);
if (!FTPReply.IsPositivePreliminary(client.GetReplyCode()))
{
// The ftpClient is an inconsistent state. Must close the stream
// which in turn will logout and disconnect from FTP server
fos.Close();
throw new IOException("Unable to create file: " + file + ", Aborting");
}
return(fos);
}
/// <exception cref="System.IO.IOException"/>
private void ClearDir(FileContext fc, Path p)
{
try
{
fc.Delete(p, true);
}
catch (FileNotFoundException)
{
}
// ignore
fc.Mkdir(p, FsPermission.GetDirDefault(), false);
}
public virtual void TestLogDeletion()
{
FilePath[] localLogDirs = GetLocalLogDirFiles(this.GetType().FullName, 2);
string localLogDirsString = localLogDirs[0].GetAbsolutePath() + "," + localLogDirs
[1].GetAbsolutePath();
conf.Set(YarnConfiguration.NmLogDirs, localLogDirsString);
conf.SetBoolean(YarnConfiguration.LogAggregationEnabled, false);
conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 0l);
dirsHandler.Init(conf);
NonAggregatingLogHandler rawLogHandler = new NonAggregatingLogHandler(dispatcher,
mockDelService, dirsHandler, new NMNullStateStoreService());
NonAggregatingLogHandler logHandler = Org.Mockito.Mockito.Spy(rawLogHandler);
AbstractFileSystem spylfs = Org.Mockito.Mockito.Spy(FileContext.GetLocalFSFileContext
().GetDefaultFileSystem());
FileContext lfs = FileContext.GetFileContext(spylfs, conf);
Org.Mockito.Mockito.DoReturn(lfs).When(logHandler).GetLocalFileContext(Matchers.IsA
<Configuration>());
FsPermission defaultPermission = FsPermission.GetDirDefault().ApplyUMask(lfs.GetUMask
());
FileStatus fs = new FileStatus(0, true, 1, 0, Runtime.CurrentTimeMillis(), 0, defaultPermission
, string.Empty, string.Empty, new Path(localLogDirs[0].GetAbsolutePath()));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.IsA <Path>());
logHandler.Init(conf);
logHandler.Start();
logHandler.Handle(new LogHandlerAppStartedEvent(appId, user, null, ContainerLogsRetentionPolicy
.AllContainers, null));
logHandler.Handle(new LogHandlerContainerFinishedEvent(container11, 0));
logHandler.Handle(new LogHandlerAppFinishedEvent(appId));
Path[] localAppLogDirs = new Path[2];
localAppLogDirs[0] = new Path(localLogDirs[0].GetAbsolutePath(), appId.ToString()
);
localAppLogDirs[1] = new Path(localLogDirs[1].GetAbsolutePath(), appId.ToString()
);
TestDeletionServiceCall(mockDelService, user, 5000, localAppLogDirs);
logHandler.Close();
for (int i = 0; i < localLogDirs.Length; i++)
{
FileUtils.DeleteDirectory(localLogDirs[i]);
}
}
public FileStatus(long length, bool isdir, int block_replication, long blocksize,
long modification_time, long access_time, FsPermission permission, string owner,
string group, Path symlink, Path path)
{
//We should deprecate this soon?
this.length = length;
this.isdir = isdir;
this.block_replication = (short)block_replication;
this.blocksize = blocksize;
this.modification_time = modification_time;
this.access_time = access_time;
if (permission != null)
{
this.permission = permission;
}
else
{
if (isdir)
{
this.permission = FsPermission.GetDirDefault();
}
else
{
if (symlink != null)
{
this.permission = FsPermission.GetDefault();
}
else
{
this.permission = FsPermission.GetFileDefault();
}
}
}
this.owner = (owner == null) ? string.Empty : owner;
this.group = (group == null) ? string.Empty : group;
this.symlink = symlink;
this.path = path;
// The variables isdir and symlink indicate the type:
// 1. isdir implies directory, in which case symlink must be null.
// 2. !isdir implies a file or symlink, symlink != null implies a
// symlink, otherwise it's a file.
System.Diagnostics.Debug.Assert((isdir && symlink == null) || !isdir);
}
public virtual void Setup()
{
try
{
lfs = FileContext.GetLocalFSFileContext();
workDir = new Path("/tmp/temp-" + Runtime.CurrentTimeMillis());
workSpace = new FilePath(workDir.ToUri().GetPath());
lfs.Mkdir(workDir, FsPermission.GetDirDefault(), true);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
Configuration conf = new Configuration();
yarnImage = "yarnImage";
long time = Runtime.CurrentTimeMillis();
conf.Set(YarnConfiguration.NmLocalDirs, "/tmp/nm-local-dir" + time);
conf.Set(YarnConfiguration.NmLogDirs, "/tmp/userlogs" + time);
dockerUrl = Runtime.GetProperty("docker-service-url");
Log.Info("dockerUrl: " + dockerUrl);
if (Strings.IsNullOrEmpty(dockerUrl))
{
return;
}
dockerUrl = " -H " + dockerUrl;
dockerExec = "docker " + dockerUrl;
conf.Set(YarnConfiguration.NmDockerContainerExecutorImageName, yarnImage);
conf.Set(YarnConfiguration.NmDockerContainerExecutorExecName, dockerExec);
exec = new DockerContainerExecutor();
dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
exec.SetConf(conf);
appSubmitter = Runtime.GetProperty("application.submitter");
if (appSubmitter == null || appSubmitter.IsEmpty())
{
appSubmitter = "nobody";
}
ShellExec(dockerExec + " pull " + testImage);
}
public virtual void Setup()
{
Assume.AssumeTrue(Shell.Linux);
FilePath f = new FilePath("./src/test/resources/mock-container-executor");
if (!FileUtil.CanExecute(f))
{
FileUtil.SetExecutable(f, true);
}
string executorPath = f.GetAbsolutePath();
Configuration conf = new Configuration();
yarnImage = "yarnImage";
long time = Runtime.CurrentTimeMillis();
conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath);
conf.Set(YarnConfiguration.NmLocalDirs, "/tmp/nm-local-dir" + time);
conf.Set(YarnConfiguration.NmLogDirs, "/tmp/userlogs" + time);
conf.Set(YarnConfiguration.NmDockerContainerExecutorImageName, yarnImage);
conf.Set(YarnConfiguration.NmDockerContainerExecutorExecName, DockerLaunchCommand
);
dockerContainerExecutor = new DockerContainerExecutor();
dirsHandler = new LocalDirsHandlerService();
dirsHandler.Init(conf);
dockerContainerExecutor.SetConf(conf);
lfs = null;
try
{
lfs = FileContext.GetLocalFSFileContext();
workDir = new Path("/tmp/temp-" + Runtime.CurrentTimeMillis());
lfs.Mkdir(workDir, FsPermission.GetDirDefault(), true);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
/// <summary>Test that encryption zones are properly tracked by the standby.</summary>
/// <exception cref="System.Exception"/>
public virtual void TestEncryptionZonesTrackedOnStandby()
{
int len = 8196;
Path dir = new Path("/enc");
Path dirChild = new Path(dir, "child");
Path dirFile = new Path(dir, "file");
fs.Mkdir(dir, FsPermission.GetDirDefault());
dfsAdmin0.CreateEncryptionZone(dir, TestKey);
fs.Mkdir(dirChild, FsPermission.GetDirDefault());
DFSTestUtil.CreateFile(fs, dirFile, len, (short)1, unchecked ((int)(0xFEED)));
string contents = DFSTestUtil.ReadFile(fs, dirFile);
// Failover the current standby to active.
HATestUtil.WaitForStandbyToCatchUp(nn0, nn1);
cluster.ShutdownNameNode(0);
cluster.TransitionToActive(1);
NUnit.Framework.Assert.AreEqual("Got unexpected ez path", dir.ToString(), dfsAdmin1
.GetEncryptionZoneForPath(dir).GetPath().ToString());
NUnit.Framework.Assert.AreEqual("Got unexpected ez path", dir.ToString(), dfsAdmin1
.GetEncryptionZoneForPath(dirChild).GetPath().ToString());
NUnit.Framework.Assert.AreEqual("File contents after failover were changed", contents
, DFSTestUtil.ReadFile(fs, dirFile));
}
public virtual void ConstructorNoOwner()
{
bool isdir = true;
FileStatus fileStatus = new FileStatus(Length, isdir, Replication, Blksize, Mtime
, Path);
ValidateAccessors(fileStatus, Length, isdir, Replication, Blksize, Mtime, 0, FsPermission
.GetDirDefault(), string.Empty, string.Empty, null, Path);
}
