public virtual void IncrementFileCountForLocalCacheDirectory(Path cacheDir)
{
if (useLocalCacheDirectoryManager)
{
Path cacheRoot = LocalCacheDirectoryManager.GetCacheDirectoryRoot(cacheDir);
if (cacheRoot != null)
{
LocalCacheDirectoryManager dir = directoryManagers[cacheRoot];
if (dir == null)
{
dir = new LocalCacheDirectoryManager(conf);
LocalCacheDirectoryManager otherDir = directoryManagers.PutIfAbsent(cacheRoot, dir
);
if (otherDir != null)
{
dir = otherDir;
}
}
if (cacheDir.Equals(cacheRoot))
{
dir.IncrementFileCountForPath(string.Empty);
}
else
{
string dirStr = cacheDir.ToUri().GetRawPath();
string rootStr = cacheRoot.ToUri().GetRawPath();
dir.IncrementFileCountForPath(Sharpen.Runtime.Substring(dirStr, rootStr.Length +
1));
}
}
}
}
public virtual void TestDirectoryStateChangeFromFullToNonFull()
{
YarnConfiguration conf = new YarnConfiguration();
conf.Set(YarnConfiguration.NmLocalCacheMaxFilesPerDirectory, "40");
LocalCacheDirectoryManager dir = new LocalCacheDirectoryManager(conf);
// checking for first four paths
string rootPath = string.Empty;
string firstSubDir = "0";
for (int i = 0; i < 4; i++)
{
NUnit.Framework.Assert.AreEqual(rootPath, dir.GetRelativePathForLocalization());
}
// Releasing two files from the root directory.
dir.DecrementFileCountForPath(rootPath);
dir.DecrementFileCountForPath(rootPath);
// Space for two files should be available in root directory.
NUnit.Framework.Assert.AreEqual(rootPath, dir.GetRelativePathForLocalization());
NUnit.Framework.Assert.AreEqual(rootPath, dir.GetRelativePathForLocalization());
// As no space is now available in root directory so it should be from
// first sub directory
NUnit.Framework.Assert.AreEqual(firstSubDir, dir.GetRelativePathForLocalization()
);
}
/// <returns>
///
/// <see cref="Org.Apache.Hadoop.FS.Path"/>
/// absolute path for localization which includes local
/// directory path and the relative hierarchical path (if use local
/// cache directory manager is enabled)
/// </returns>
/// <?/>
/// <?/>
/// <?/>
public virtual Path GetPathForLocalization(LocalResourceRequest req, Path localDirPath
, DeletionService delService)
{
Path rPath = localDirPath;
if (useLocalCacheDirectoryManager && localDirPath != null)
{
if (!directoryManagers.Contains(localDirPath))
{
directoryManagers.PutIfAbsent(localDirPath, new LocalCacheDirectoryManager(conf));
}
LocalCacheDirectoryManager dir = directoryManagers[localDirPath];
rPath = localDirPath;
string hierarchicalPath = dir.GetRelativePathForLocalization();
// For most of the scenarios we will get root path only which
// is an empty string
if (!hierarchicalPath.IsEmpty())
{
rPath = new Path(localDirPath, hierarchicalPath);
}
inProgressLocalResourcesMap[req] = rPath;
}
while (true)
{
Path uniquePath = new Path(rPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
()));
FilePath file = new FilePath(uniquePath.ToUri().GetRawPath());
if (!file.Exists())
{
rPath = uniquePath;
break;
}
// If the directory already exists, delete it and move to next one.
Log.Warn("Directory " + uniquePath + " already exists, " + "try next one.");
if (delService != null)
{
delService.Delete(GetUser(), uniquePath);
}
}
Path localPath = new Path(rPath, req.GetPath().GetName());
LocalizedResource rsrc = localrsrc[req];
rsrc.SetLocalPath(localPath);
LocalResource lr = LocalResource.NewInstance(req.GetResource(), req.GetType(), req
.GetVisibility(), req.GetSize(), req.GetTimestamp());
try
{
stateStore.StartResourceLocalization(user, appId, ((LocalResourcePBImpl)lr).GetProto
(), localPath);
}
catch (IOException e)
{
Log.Error("Unable to record localization start for " + rsrc, e);
}
return(rPath);
}
internal virtual LocalCacheDirectoryManager GetDirectoryManager(Path localDirPath
)
{
LocalCacheDirectoryManager mgr = null;
if (useLocalCacheDirectoryManager)
{
mgr = directoryManagers[localDirPath];
}
return(mgr);
}
/*
* Update the file-count statistics for a local cache-directory.
* This will retrieve the localized path for the resource from
* 1) inProgressRsrcMap if the resource was under localization and it
* failed.
* 2) LocalizedResource if the resource is already localized.
* From this path it will identify the local directory under which the
* resource was localized. Then rest of the path will be used to decrement
* file count for the HierarchicalSubDirectory pointing to this relative
* path.
*/
private void DecrementFileCountForLocalCacheDirectory(LocalResourceRequest req, LocalizedResource
rsrc)
{
if (useLocalCacheDirectoryManager)
{
Path rsrcPath = null;
if (inProgressLocalResourcesMap.Contains(req))
{
// This happens when localization of a resource fails.
rsrcPath = Sharpen.Collections.Remove(inProgressLocalResourcesMap, req);
}
else
{
if (rsrc != null && rsrc.GetLocalPath() != null)
{
rsrcPath = rsrc.GetLocalPath().GetParent().GetParent();
}
}
if (rsrcPath != null)
{
Path parentPath = new Path(rsrcPath.ToUri().GetRawPath());
while (!directoryManagers.Contains(parentPath))
{
parentPath = parentPath.GetParent();
if (parentPath == null)
{
return;
}
}
if (parentPath != null)
{
string parentDir = parentPath.ToUri().GetRawPath().ToString();
LocalCacheDirectoryManager dir = directoryManagers[parentPath];
string rsrcDir = rsrcPath.ToUri().GetRawPath();
if (rsrcDir.Equals(parentDir))
{
dir.DecrementFileCountForPath(string.Empty);
}
else
{
dir.DecrementFileCountForPath(Sharpen.Runtime.Substring(rsrcDir, parentDir.Length
+ 1));
}
}
}
}
}
public virtual void TestHierarchicalSubDirectoryCreation()
{
// setting per directory file limit to 1.
YarnConfiguration conf = new YarnConfiguration();
conf.Set(YarnConfiguration.NmLocalCacheMaxFilesPerDirectory, "37");
LocalCacheDirectoryManager hDir = new LocalCacheDirectoryManager(conf);
// Test root directory path = ""
NUnit.Framework.Assert.IsTrue(hDir.GetRelativePathForLocalization().IsEmpty());
// Testing path generation from "0" to "0/0/z/z"
for (int i = 1; i <= 37 * 36 * 36; i++)
{
StringBuilder sb = new StringBuilder();
string num = Sharpen.Extensions.ToString(i - 1, 36);
if (num.Length == 1)
{
sb.Append(num[0]);
}
else
{
sb.Append(Sharpen.Extensions.ToString(System.Convert.ToInt32(Sharpen.Runtime.Substring
(num, 0, 1), 36) - 1, 36));
}
for (int j = 1; j < num.Length; j++)
{
sb.Append(Path.Separator).Append(num[j]);
}
NUnit.Framework.Assert.AreEqual(sb.ToString(), hDir.GetRelativePathForLocalization
());
}
string testPath1 = "4";
string testPath2 = "2";
/*
* Making sure directory "4" and "2" becomes non-full so that they are
* reused for future getRelativePathForLocalization() calls in the order
* they are freed.
*/
hDir.DecrementFileCountForPath(testPath1);
hDir.DecrementFileCountForPath(testPath2);
// After below call directory "4" should become full.
NUnit.Framework.Assert.AreEqual(testPath1, hDir.GetRelativePathForLocalization());
NUnit.Framework.Assert.AreEqual(testPath2, hDir.GetRelativePathForLocalization());
}
public virtual void TestIncrementFileCountForPath()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.NmLocalCacheMaxFilesPerDirectory, LocalCacheDirectoryManager
.DirectoriesPerLevel + 2);
LocalCacheDirectoryManager mgr = new LocalCacheDirectoryManager(conf);
string rootPath = string.Empty;
mgr.IncrementFileCountForPath(rootPath);
NUnit.Framework.Assert.AreEqual(rootPath, mgr.GetRelativePathForLocalization());
NUnit.Framework.Assert.IsFalse("root dir should be full", rootPath.Equals(mgr.GetRelativePathForLocalization
()));
// finish filling the other directory
mgr.GetRelativePathForLocalization();
// free up space in the root dir
mgr.DecrementFileCountForPath(rootPath);
mgr.DecrementFileCountForPath(rootPath);
NUnit.Framework.Assert.AreEqual(rootPath, mgr.GetRelativePathForLocalization());
NUnit.Framework.Assert.AreEqual(rootPath, mgr.GetRelativePathForLocalization());
string otherDir = mgr.GetRelativePathForLocalization();
NUnit.Framework.Assert.IsFalse("root dir should be full", otherDir.Equals(rootPath
));
string deepDir0 = "d/e/e/p/0";
string deepDir1 = "d/e/e/p/1";
string deepDir2 = "d/e/e/p/2";
string deepDir3 = "d/e/e/p/3";
mgr.IncrementFileCountForPath(deepDir0);
NUnit.Framework.Assert.AreEqual(otherDir, mgr.GetRelativePathForLocalization());
NUnit.Framework.Assert.AreEqual(deepDir0, mgr.GetRelativePathForLocalization());
NUnit.Framework.Assert.AreEqual("total dir count incorrect after increment", deepDir1
, mgr.GetRelativePathForLocalization());
mgr.IncrementFileCountForPath(deepDir2);
mgr.IncrementFileCountForPath(deepDir1);
mgr.IncrementFileCountForPath(deepDir2);
NUnit.Framework.Assert.AreEqual(deepDir3, mgr.GetRelativePathForLocalization());
}