// nothing to do or verify here
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void StartLocalizer(Path nmPrivateContainerTokensPath, IPEndPoint
nmAddr, string user, string appId, string locId, LocalDirsHandlerService dirsHandler
)
{
IList <string> localDirs = dirsHandler.GetLocalDirs();
IList <string> logDirs = dirsHandler.GetLogDirs();
CreateUserLocalDirs(localDirs, user);
CreateUserCacheDirs(localDirs, user);
CreateAppDirs(localDirs, user, appId);
CreateAppLogDirs(appId, logDirs, user);
// randomly choose the local directory
Path appStorageDir = GetWorkingDir(localDirs, user, appId);
string tokenFn = string.Format(ContainerLocalizer.TokenFileNameFmt, locId);
Path tokenDst = new Path(appStorageDir, tokenFn);
CopyFile(nmPrivateContainerTokensPath, tokenDst, user);
Log.Info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
FileContext localizerFc = FileContext.GetFileContext(lfs.GetDefaultFileSystem(),
GetConf());
localizerFc.SetUMask(lfs.GetUMask());
localizerFc.SetWorkingDirectory(appStorageDir);
Log.Info("Localizer CWD set to " + appStorageDir + " = " + localizerFc.GetWorkingDirectory
());
ContainerLocalizer localizer = new ContainerLocalizer(localizerFc, user, appId, locId
, GetPaths(localDirs), RecordFactoryProvider.GetRecordFactory(GetConf()));
// TODO: DO it over RPC for maintaining similarity?
localizer.RunLocalization(nmAddr);
}
/// <exception cref="System.IO.IOException"/>
private static void CreateDir(FileContext lfs, Path dirPath, FsPermission perms,
bool createParent)
{
lfs.Mkdir(dirPath, perms, createParent);
if (!perms.Equals(perms.ApplyUMask(lfs.GetUMask())))
{
lfs.SetPermission(dirPath, perms);
}
}
/// <summary>
/// Function to run a log handler with directories failing the getFileStatus
/// call.
/// </summary>
/// <remarks>
/// Function to run a log handler with directories failing the getFileStatus
/// call. The function accepts the log handler, setup the mocks to fail with
/// specific exceptions and ensures the deletion service has the correct calls.
/// </remarks>
/// <param name="logHandler">the logHandler implementation to test</param>
/// <param name="appId">
/// the application id that we wish when sending events to the log
/// handler
/// </param>
/// <param name="user">the user name to use</param>
/// <param name="mockDelService">
/// a mock of the DeletionService which we will verify
/// the delete calls against
/// </param>
/// <param name="dirsHandler">
/// a spy or mock on the LocalDirsHandler service used to
/// when creating the logHandler. It needs to be a spy so that we can intercept
/// the getAllLogDirs() call.
/// </param>
/// <param name="conf">the configuration used</param>
/// <param name="spylfs">a spy on the AbstractFileSystem object used when creating lfs
/// </param>
/// <param name="lfs">
/// the FileContext object to be used to mock the getFileStatus()
/// calls
/// </param>
/// <param name="localLogDirs">
/// list of the log dirs to run the test against, must have
/// at least 7 entries
/// </param>
/// <exception cref="System.Exception"/>
public static void RunMockedFailedDirs(LogHandler logHandler, ApplicationId appId
, string user, DeletionService mockDelService, LocalDirsHandlerService dirsHandler
, Configuration conf, AbstractFileSystem spylfs, FileContext lfs, FilePath[] localLogDirs
)
{
IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType
, string>();
if (localLogDirs.Length < 7)
{
throw new ArgumentException("Argument localLogDirs must be at least of length 7");
}
Path[] localAppLogDirPaths = new Path[localLogDirs.Length];
for (int i = 0; i < localAppLogDirPaths.Length; i++)
{
localAppLogDirPaths[i] = new Path(localLogDirs[i].GetAbsolutePath(), appId.ToString
());
}
IList <string> localLogDirPaths = new AList <string>(localLogDirs.Length);
for (int i_1 = 0; i_1 < localLogDirs.Length; i_1++)
{
localLogDirPaths.AddItem(localLogDirs[i_1].GetAbsolutePath());
}
// setup mocks
FsPermission defaultPermission = FsPermission.GetDirDefault().ApplyUMask(lfs.GetUMask
());
FileStatus fs = new FileStatus(0, true, 1, 0, Runtime.CurrentTimeMillis(), 0, defaultPermission
, string.Empty, string.Empty, new Path(localLogDirs[0].GetAbsolutePath()));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.IsA <Path>());
Org.Mockito.Mockito.DoReturn(localLogDirPaths).When(dirsHandler).GetLogDirsForCleanup
();
logHandler.Handle(new LogHandlerAppStartedEvent(appId, user, null, ContainerLogsRetentionPolicy
.AllContainers, appAcls));
// test case where some dirs have the log dir to delete
// mock some dirs throwing various exceptions
// verify deletion happens only on the others
Org.Mockito.Mockito.DoThrow(new FileNotFoundException()).When(spylfs).GetFileStatus
(Matchers.Eq(localAppLogDirPaths[0]));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths
[1]));
Org.Mockito.Mockito.DoThrow(new AccessControlException()).When(spylfs).GetFileStatus
(Matchers.Eq(localAppLogDirPaths[2]));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths
[3]));
Org.Mockito.Mockito.DoThrow(new IOException()).When(spylfs).GetFileStatus(Matchers.Eq
(localAppLogDirPaths[4]));
Org.Mockito.Mockito.DoThrow(new UnsupportedFileSystemException("test")).When(spylfs
).GetFileStatus(Matchers.Eq(localAppLogDirPaths[5]));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths
[6]));
logHandler.Handle(new LogHandlerAppFinishedEvent(appId));
TestDeletionServiceCall(mockDelService, user, 5000, localAppLogDirPaths[1], localAppLogDirPaths
[3], localAppLogDirPaths[6]);
return;
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual void CreateDir(Path dirPath, FsPermission perms, bool
createParent, string user)
{
lfs.Mkdir(dirPath, perms, createParent);
if (!perms.Equals(perms.ApplyUMask(lfs.GetUMask())))
{
lfs.SetPermission(dirPath, perms);
}
}
public virtual void TestCreatePermission()
{
if (Path.Windows)
{
System.Console.Out.WriteLine("Cannot run test for Windows");
return;
}
string filename = "foo";
Path f = fileContextTestHelper.GetTestRootPath(fc, filename);
fileContextTestHelper.CreateFile(fc, filename);
DoFilePermissionCheck(FileContext.FileDefaultPerm.ApplyUMask(fc.GetUMask()), fc.GetFileStatus
(f).GetPermission());
}
public virtual void TestLogDeletion()
{
FilePath[] localLogDirs = GetLocalLogDirFiles(this.GetType().FullName, 2);
string localLogDirsString = localLogDirs[0].GetAbsolutePath() + "," + localLogDirs
[1].GetAbsolutePath();
conf.Set(YarnConfiguration.NmLogDirs, localLogDirsString);
conf.SetBoolean(YarnConfiguration.LogAggregationEnabled, false);
conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 0l);
dirsHandler.Init(conf);
NonAggregatingLogHandler rawLogHandler = new NonAggregatingLogHandler(dispatcher,
mockDelService, dirsHandler, new NMNullStateStoreService());
NonAggregatingLogHandler logHandler = Org.Mockito.Mockito.Spy(rawLogHandler);
AbstractFileSystem spylfs = Org.Mockito.Mockito.Spy(FileContext.GetLocalFSFileContext
().GetDefaultFileSystem());
FileContext lfs = FileContext.GetFileContext(spylfs, conf);
Org.Mockito.Mockito.DoReturn(lfs).When(logHandler).GetLocalFileContext(Matchers.IsA
<Configuration>());
FsPermission defaultPermission = FsPermission.GetDirDefault().ApplyUMask(lfs.GetUMask
());
FileStatus fs = new FileStatus(0, true, 1, 0, Runtime.CurrentTimeMillis(), 0, defaultPermission
, string.Empty, string.Empty, new Path(localLogDirs[0].GetAbsolutePath()));
Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.IsA <Path>());
logHandler.Init(conf);
logHandler.Start();
logHandler.Handle(new LogHandlerAppStartedEvent(appId, user, null, ContainerLogsRetentionPolicy
.AllContainers, null));
logHandler.Handle(new LogHandlerContainerFinishedEvent(container11, 0));
logHandler.Handle(new LogHandlerAppFinishedEvent(appId));
Path[] localAppLogDirs = new Path[2];
localAppLogDirs[0] = new Path(localLogDirs[0].GetAbsolutePath(), appId.ToString()
);
localAppLogDirs[1] = new Path(localLogDirs[1].GetAbsolutePath(), appId.ToString()
);
TestDeletionServiceCall(mockDelService, user, 5000, localAppLogDirs);
logHandler.Close();
for (int i = 0; i < localLogDirs.Length; i++)
{
FileUtils.DeleteDirectory(localLogDirs[i]);
}
}
/// <exception cref="System.IO.IOException"/>
private void CreateDir(FileContext localFs, Path dir, FsPermission perm)
{
if (dir == null)
{
return;
}
try
{
localFs.GetFileStatus(dir);
}
catch (FileNotFoundException)
{
CreateDir(localFs, dir.GetParent(), perm);
localFs.Mkdir(dir, perm, false);
if (!perm.Equals(perm.ApplyUMask(localFs.GetUMask())))
{
localFs.SetPermission(dir, perm);
}
}
}
