public virtual void TestDirectoryStateChangeFromFullToNonFull()
{
YarnConfiguration conf = new YarnConfiguration();
conf.Set(YarnConfiguration.NmLocalCacheMaxFilesPerDirectory, "40");
LocalCacheDirectoryManager dir = new LocalCacheDirectoryManager(conf);
// checking for first four paths
string rootPath = string.Empty;
string firstSubDir = "0";
for (int i = 0; i < 4; i++)
{
NUnit.Framework.Assert.AreEqual(rootPath, dir.GetRelativePathForLocalization());
}
// Releasing two files from the root directory.
dir.DecrementFileCountForPath(rootPath);
dir.DecrementFileCountForPath(rootPath);
// Space for two files should be available in root directory.
NUnit.Framework.Assert.AreEqual(rootPath, dir.GetRelativePathForLocalization());
NUnit.Framework.Assert.AreEqual(rootPath, dir.GetRelativePathForLocalization());
// As no space is now available in root directory so it should be from
// first sub directory
NUnit.Framework.Assert.AreEqual(firstSubDir, dir.GetRelativePathForLocalization()
);
}
public virtual void TestHierarchicalSubDirectoryCreation()
{
// setting per directory file limit to 1.
YarnConfiguration conf = new YarnConfiguration();
conf.Set(YarnConfiguration.NmLocalCacheMaxFilesPerDirectory, "37");
LocalCacheDirectoryManager hDir = new LocalCacheDirectoryManager(conf);
// Test root directory path = ""
NUnit.Framework.Assert.IsTrue(hDir.GetRelativePathForLocalization().IsEmpty());
// Testing path generation from "0" to "0/0/z/z"
for (int i = 1; i <= 37 * 36 * 36; i++)
{
StringBuilder sb = new StringBuilder();
string num = Sharpen.Extensions.ToString(i - 1, 36);
if (num.Length == 1)
{
sb.Append(num[0]);
}
else
{
sb.Append(Sharpen.Extensions.ToString(System.Convert.ToInt32(Sharpen.Runtime.Substring
(num, 0, 1), 36) - 1, 36));
}
for (int j = 1; j < num.Length; j++)
{
sb.Append(Path.Separator).Append(num[j]);
}
NUnit.Framework.Assert.AreEqual(sb.ToString(), hDir.GetRelativePathForLocalization
());
}
string testPath1 = "4";
string testPath2 = "2";
/*
* Making sure directory "4" and "2" becomes non-full so that they are
* reused for future getRelativePathForLocalization() calls in the order
* they are freed.
*/
hDir.DecrementFileCountForPath(testPath1);
hDir.DecrementFileCountForPath(testPath2);
// After below call directory "4" should become full.
NUnit.Framework.Assert.AreEqual(testPath1, hDir.GetRelativePathForLocalization());
NUnit.Framework.Assert.AreEqual(testPath2, hDir.GetRelativePathForLocalization());
}
/// <returns>
///
/// <see cref="Org.Apache.Hadoop.FS.Path"/>
/// absolute path for localization which includes local
/// directory path and the relative hierarchical path (if use local
/// cache directory manager is enabled)
/// </returns>
/// <?/>
/// <?/>
/// <?/>
public virtual Path GetPathForLocalization(LocalResourceRequest req, Path localDirPath
, DeletionService delService)
{
Path rPath = localDirPath;
if (useLocalCacheDirectoryManager && localDirPath != null)
{
if (!directoryManagers.Contains(localDirPath))
{
directoryManagers.PutIfAbsent(localDirPath, new LocalCacheDirectoryManager(conf));
}
LocalCacheDirectoryManager dir = directoryManagers[localDirPath];
rPath = localDirPath;
string hierarchicalPath = dir.GetRelativePathForLocalization();
// For most of the scenarios we will get root path only which
// is an empty string
if (!hierarchicalPath.IsEmpty())
{
rPath = new Path(localDirPath, hierarchicalPath);
}
inProgressLocalResourcesMap[req] = rPath;
}
while (true)
{
Path uniquePath = new Path(rPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FilePath file = new FilePath(uniquePath.ToUri().GetRawPath());
if (!file.Exists())
{
rPath = uniquePath;
break;
}
// If the directory already exists, delete it and move to next one.
Log.Warn("Directory " + uniquePath + " already exists, " + "try next one.");
if (delService != null)
{
delService.Delete(GetUser(), uniquePath);
}
}
Path localPath = new Path(rPath, req.GetPath().GetName());
LocalizedResource rsrc = localrsrc[req];
rsrc.SetLocalPath(localPath);
LocalResource lr = LocalResource.NewInstance(req.GetResource(), req.GetType(), req
.GetVisibility(), req.GetSize(), req.GetTimestamp());
try
{
stateStore.StartResourceLocalization(user, appId, ((LocalResourcePBImpl)lr).GetProto
(), localPath);
}
catch (IOException e)
{
Log.Error("Unable to record localization start for " + rsrc, e);
}
return(rPath);
}
public virtual void TestIncrementFileCountForPath()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.NmLocalCacheMaxFilesPerDirectory, LocalCacheDirectoryManager
.DirectoriesPerLevel + 2);
LocalCacheDirectoryManager mgr = new LocalCacheDirectoryManager(conf);
string rootPath = string.Empty;
mgr.IncrementFileCountForPath(rootPath);
NUnit.Framework.Assert.AreEqual(rootPath, mgr.GetRelativePathForLocalization());
NUnit.Framework.Assert.IsFalse("root dir should be full", rootPath.Equals(mgr.GetRelativePathForLocalization
()));
// finish filling the other directory
mgr.GetRelativePathForLocalization();
// free up space in the root dir
mgr.DecrementFileCountForPath(rootPath);
mgr.DecrementFileCountForPath(rootPath);
NUnit.Framework.Assert.AreEqual(rootPath, mgr.GetRelativePathForLocalization());
NUnit.Framework.Assert.AreEqual(rootPath, mgr.GetRelativePathForLocalization());
string otherDir = mgr.GetRelativePathForLocalization();
NUnit.Framework.Assert.IsFalse("root dir should be full", otherDir.Equals(rootPath
));
string deepDir0 = "d/e/e/p/0";
string deepDir1 = "d/e/e/p/1";
string deepDir2 = "d/e/e/p/2";
string deepDir3 = "d/e/e/p/3";
mgr.IncrementFileCountForPath(deepDir0);
NUnit.Framework.Assert.AreEqual(otherDir, mgr.GetRelativePathForLocalization());
NUnit.Framework.Assert.AreEqual(deepDir0, mgr.GetRelativePathForLocalization());
NUnit.Framework.Assert.AreEqual("total dir count incorrect after increment", deepDir1
, mgr.GetRelativePathForLocalization());
mgr.IncrementFileCountForPath(deepDir2);
mgr.IncrementFileCountForPath(deepDir1);
mgr.IncrementFileCountForPath(deepDir2);
NUnit.Framework.Assert.AreEqual(deepDir3, mgr.GetRelativePathForLocalization());
}