public virtual void IncrementFileCountForLocalCacheDirectory(Path cacheDir) { if (useLocalCacheDirectoryManager) { Path cacheRoot = LocalCacheDirectoryManager.GetCacheDirectoryRoot(cacheDir); if (cacheRoot != null) { LocalCacheDirectoryManager dir = directoryManagers[cacheRoot]; if (dir == null) { dir = new LocalCacheDirectoryManager(conf); LocalCacheDirectoryManager otherDir = directoryManagers.PutIfAbsent(cacheRoot, dir ); if (otherDir != null) { dir = otherDir; } } if (cacheDir.Equals(cacheRoot)) { dir.IncrementFileCountForPath(string.Empty); } else { string dirStr = cacheDir.ToUri().GetRawPath(); string rootStr = cacheRoot.ToUri().GetRawPath(); dir.IncrementFileCountForPath(Sharpen.Runtime.Substring(dirStr, rootStr.Length + 1)); } } } }
public virtual void TestDirectoryStateChangeFromFullToNonFull() { YarnConfiguration conf = new YarnConfiguration(); conf.Set(YarnConfiguration.NmLocalCacheMaxFilesPerDirectory, "40"); LocalCacheDirectoryManager dir = new LocalCacheDirectoryManager(conf); // checking for first four paths string rootPath = string.Empty; string firstSubDir = "0"; for (int i = 0; i < 4; i++) { NUnit.Framework.Assert.AreEqual(rootPath, dir.GetRelativePathForLocalization()); } // Releasing two files from the root directory. dir.DecrementFileCountForPath(rootPath); dir.DecrementFileCountForPath(rootPath); // Space for two files should be available in root directory. NUnit.Framework.Assert.AreEqual(rootPath, dir.GetRelativePathForLocalization()); NUnit.Framework.Assert.AreEqual(rootPath, dir.GetRelativePathForLocalization()); // As no space is now available in root directory so it should be from // first sub directory NUnit.Framework.Assert.AreEqual(firstSubDir, dir.GetRelativePathForLocalization() ); }
/// <returns> /// /// <see cref="Org.Apache.Hadoop.FS.Path"/> /// absolute path for localization which includes local /// directory path and the relative hierarchical path (if use local /// cache directory manager is enabled) /// </returns> /// <?/> /// <?/> /// <?/> public virtual Path GetPathForLocalization(LocalResourceRequest req, Path localDirPath , DeletionService delService) { Path rPath = localDirPath; if (useLocalCacheDirectoryManager && localDirPath != null) { if (!directoryManagers.Contains(localDirPath)) { directoryManagers.PutIfAbsent(localDirPath, new LocalCacheDirectoryManager(conf)); } LocalCacheDirectoryManager dir = directoryManagers[localDirPath]; rPath = localDirPath; string hierarchicalPath = dir.GetRelativePathForLocalization(); // For most of the scenarios we will get root path only which // is an empty string if (!hierarchicalPath.IsEmpty()) { rPath = new Path(localDirPath, hierarchicalPath); } inProgressLocalResourcesMap[req] = rPath; } while (true) { Path uniquePath = new Path(rPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); FilePath file = new FilePath(uniquePath.ToUri().GetRawPath()); if (!file.Exists()) { rPath = uniquePath; break; } // If the directory already exists, delete it and move to next one. Log.Warn("Directory " + uniquePath + " already exists, " + "try next one."); if (delService != null) { delService.Delete(GetUser(), uniquePath); } } Path localPath = new Path(rPath, req.GetPath().GetName()); LocalizedResource rsrc = localrsrc[req]; rsrc.SetLocalPath(localPath); LocalResource lr = LocalResource.NewInstance(req.GetResource(), req.GetType(), req .GetVisibility(), req.GetSize(), req.GetTimestamp()); try { stateStore.StartResourceLocalization(user, appId, ((LocalResourcePBImpl)lr).GetProto (), localPath); } catch (IOException e) { Log.Error("Unable to record localization start for " + rsrc, e); } return(rPath); }
internal virtual LocalCacheDirectoryManager GetDirectoryManager(Path localDirPath ) { LocalCacheDirectoryManager mgr = null; if (useLocalCacheDirectoryManager) { mgr = directoryManagers[localDirPath]; } return(mgr); }
/* * Update the file-count statistics for a local cache-directory. * This will retrieve the localized path for the resource from * 1) inProgressRsrcMap if the resource was under localization and it * failed. * 2) LocalizedResource if the resource is already localized. * From this path it will identify the local directory under which the * resource was localized. Then rest of the path will be used to decrement * file count for the HierarchicalSubDirectory pointing to this relative * path. */ private void DecrementFileCountForLocalCacheDirectory(LocalResourceRequest req, LocalizedResource rsrc) { if (useLocalCacheDirectoryManager) { Path rsrcPath = null; if (inProgressLocalResourcesMap.Contains(req)) { // This happens when localization of a resource fails. rsrcPath = Sharpen.Collections.Remove(inProgressLocalResourcesMap, req); } else { if (rsrc != null && rsrc.GetLocalPath() != null) { rsrcPath = rsrc.GetLocalPath().GetParent().GetParent(); } } if (rsrcPath != null) { Path parentPath = new Path(rsrcPath.ToUri().GetRawPath()); while (!directoryManagers.Contains(parentPath)) { parentPath = parentPath.GetParent(); if (parentPath == null) { return; } } if (parentPath != null) { string parentDir = parentPath.ToUri().GetRawPath().ToString(); LocalCacheDirectoryManager dir = directoryManagers[parentPath]; string rsrcDir = rsrcPath.ToUri().GetRawPath(); if (rsrcDir.Equals(parentDir)) { dir.DecrementFileCountForPath(string.Empty); } else { dir.DecrementFileCountForPath(Sharpen.Runtime.Substring(rsrcDir, parentDir.Length + 1)); } } } } }
public virtual void TestHierarchicalSubDirectoryCreation() { // setting per directory file limit to 1. YarnConfiguration conf = new YarnConfiguration(); conf.Set(YarnConfiguration.NmLocalCacheMaxFilesPerDirectory, "37"); LocalCacheDirectoryManager hDir = new LocalCacheDirectoryManager(conf); // Test root directory path = "" NUnit.Framework.Assert.IsTrue(hDir.GetRelativePathForLocalization().IsEmpty()); // Testing path generation from "0" to "0/0/z/z" for (int i = 1; i <= 37 * 36 * 36; i++) { StringBuilder sb = new StringBuilder(); string num = Sharpen.Extensions.ToString(i - 1, 36); if (num.Length == 1) { sb.Append(num[0]); } else { sb.Append(Sharpen.Extensions.ToString(System.Convert.ToInt32(Sharpen.Runtime.Substring (num, 0, 1), 36) - 1, 36)); } for (int j = 1; j < num.Length; j++) { sb.Append(Path.Separator).Append(num[j]); } NUnit.Framework.Assert.AreEqual(sb.ToString(), hDir.GetRelativePathForLocalization ()); } string testPath1 = "4"; string testPath2 = "2"; /* * Making sure directory "4" and "2" becomes non-full so that they are * reused for future getRelativePathForLocalization() calls in the order * they are freed. */ hDir.DecrementFileCountForPath(testPath1); hDir.DecrementFileCountForPath(testPath2); // After below call directory "4" should become full. NUnit.Framework.Assert.AreEqual(testPath1, hDir.GetRelativePathForLocalization()); NUnit.Framework.Assert.AreEqual(testPath2, hDir.GetRelativePathForLocalization()); }
public virtual void TestIncrementFileCountForPath() { YarnConfiguration conf = new YarnConfiguration(); conf.SetInt(YarnConfiguration.NmLocalCacheMaxFilesPerDirectory, LocalCacheDirectoryManager .DirectoriesPerLevel + 2); LocalCacheDirectoryManager mgr = new LocalCacheDirectoryManager(conf); string rootPath = string.Empty; mgr.IncrementFileCountForPath(rootPath); NUnit.Framework.Assert.AreEqual(rootPath, mgr.GetRelativePathForLocalization()); NUnit.Framework.Assert.IsFalse("root dir should be full", rootPath.Equals(mgr.GetRelativePathForLocalization ())); // finish filling the other directory mgr.GetRelativePathForLocalization(); // free up space in the root dir mgr.DecrementFileCountForPath(rootPath); mgr.DecrementFileCountForPath(rootPath); NUnit.Framework.Assert.AreEqual(rootPath, mgr.GetRelativePathForLocalization()); NUnit.Framework.Assert.AreEqual(rootPath, mgr.GetRelativePathForLocalization()); string otherDir = mgr.GetRelativePathForLocalization(); NUnit.Framework.Assert.IsFalse("root dir should be full", otherDir.Equals(rootPath )); string deepDir0 = "d/e/e/p/0"; string deepDir1 = "d/e/e/p/1"; string deepDir2 = "d/e/e/p/2"; string deepDir3 = "d/e/e/p/3"; mgr.IncrementFileCountForPath(deepDir0); NUnit.Framework.Assert.AreEqual(otherDir, mgr.GetRelativePathForLocalization()); NUnit.Framework.Assert.AreEqual(deepDir0, mgr.GetRelativePathForLocalization()); NUnit.Framework.Assert.AreEqual("total dir count incorrect after increment", deepDir1 , mgr.GetRelativePathForLocalization()); mgr.IncrementFileCountForPath(deepDir2); mgr.IncrementFileCountForPath(deepDir1); mgr.IncrementFileCountForPath(deepDir2); NUnit.Framework.Assert.AreEqual(deepDir3, mgr.GetRelativePathForLocalization()); }