Пример #1
0
 public virtual void IncrementFileCountForLocalCacheDirectory(Path cacheDir)
 {
     if (useLocalCacheDirectoryManager)
     {
         Path cacheRoot = LocalCacheDirectoryManager.GetCacheDirectoryRoot(cacheDir);
         if (cacheRoot != null)
         {
             LocalCacheDirectoryManager dir = directoryManagers[cacheRoot];
             if (dir == null)
             {
                 dir = new LocalCacheDirectoryManager(conf);
                 LocalCacheDirectoryManager otherDir = directoryManagers.PutIfAbsent(cacheRoot, dir
                                                                                     );
                 if (otherDir != null)
                 {
                     dir = otherDir;
                 }
             }
             if (cacheDir.Equals(cacheRoot))
             {
                 dir.IncrementFileCountForPath(string.Empty);
             }
             else
             {
                 string dirStr  = cacheDir.ToUri().GetRawPath();
                 string rootStr = cacheRoot.ToUri().GetRawPath();
                 dir.IncrementFileCountForPath(Sharpen.Runtime.Substring(dirStr, rootStr.Length +
                                                                         1));
             }
         }
     }
 }
Пример #2
0
        public virtual void TestDirectoryStateChangeFromFullToNonFull()
        {
            YarnConfiguration conf = new YarnConfiguration();

            conf.Set(YarnConfiguration.NmLocalCacheMaxFilesPerDirectory, "40");
            LocalCacheDirectoryManager dir = new LocalCacheDirectoryManager(conf);
            // checking for first four paths
            string rootPath    = string.Empty;
            string firstSubDir = "0";

            for (int i = 0; i < 4; i++)
            {
                NUnit.Framework.Assert.AreEqual(rootPath, dir.GetRelativePathForLocalization());
            }
            // Releasing two files from the root directory.
            dir.DecrementFileCountForPath(rootPath);
            dir.DecrementFileCountForPath(rootPath);
            // Space for two files should be available in root directory.
            NUnit.Framework.Assert.AreEqual(rootPath, dir.GetRelativePathForLocalization());
            NUnit.Framework.Assert.AreEqual(rootPath, dir.GetRelativePathForLocalization());
            // As no space is now available in root directory so it should be from
            // first sub directory
            NUnit.Framework.Assert.AreEqual(firstSubDir, dir.GetRelativePathForLocalization()
                                            );
        }
Пример #3
0
        /// <returns>
        ///
        /// <see cref="Org.Apache.Hadoop.FS.Path"/>
        /// absolute path for localization which includes local
        /// directory path and the relative hierarchical path (if use local
        /// cache directory manager is enabled)
        /// </returns>
        /// <?/>
        /// <?/>
        /// <?/>
        public virtual Path GetPathForLocalization(LocalResourceRequest req, Path localDirPath
                                                   , DeletionService delService)
        {
            Path rPath = localDirPath;

            if (useLocalCacheDirectoryManager && localDirPath != null)
            {
                if (!directoryManagers.Contains(localDirPath))
                {
                    directoryManagers.PutIfAbsent(localDirPath, new LocalCacheDirectoryManager(conf));
                }
                LocalCacheDirectoryManager dir = directoryManagers[localDirPath];
                rPath = localDirPath;
                string hierarchicalPath = dir.GetRelativePathForLocalization();
                // For most of the scenarios we will get root path only which
                // is an empty string
                if (!hierarchicalPath.IsEmpty())
                {
                    rPath = new Path(localDirPath, hierarchicalPath);
                }
                inProgressLocalResourcesMap[req] = rPath;
            }
            while (true)
            {
                Path uniquePath = new Path(rPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet
                                                                              ()));
                FilePath file = new FilePath(uniquePath.ToUri().GetRawPath());
                if (!file.Exists())
                {
                    rPath = uniquePath;
                    break;
                }
                // If the directory already exists, delete it and move to next one.
                Log.Warn("Directory " + uniquePath + " already exists, " + "try next one.");
                if (delService != null)
                {
                    delService.Delete(GetUser(), uniquePath);
                }
            }
            Path localPath         = new Path(rPath, req.GetPath().GetName());
            LocalizedResource rsrc = localrsrc[req];

            rsrc.SetLocalPath(localPath);
            LocalResource lr = LocalResource.NewInstance(req.GetResource(), req.GetType(), req
                                                         .GetVisibility(), req.GetSize(), req.GetTimestamp());

            try
            {
                stateStore.StartResourceLocalization(user, appId, ((LocalResourcePBImpl)lr).GetProto
                                                         (), localPath);
            }
            catch (IOException e)
            {
                Log.Error("Unable to record localization start for " + rsrc, e);
            }
            return(rPath);
        }
Пример #4
0
        internal virtual LocalCacheDirectoryManager GetDirectoryManager(Path localDirPath
                                                                        )
        {
            LocalCacheDirectoryManager mgr = null;

            if (useLocalCacheDirectoryManager)
            {
                mgr = directoryManagers[localDirPath];
            }
            return(mgr);
        }
Пример #5
0
 /*
  * Update the file-count statistics for a local cache-directory.
  * This will retrieve the localized path for the resource from
  * 1) inProgressRsrcMap if the resource was under localization and it
  * failed.
  * 2) LocalizedResource if the resource is already localized.
  * From this path it will identify the local directory under which the
  * resource was localized. Then rest of the path will be used to decrement
  * file count for the HierarchicalSubDirectory pointing to this relative
  * path.
  */
 private void DecrementFileCountForLocalCacheDirectory(LocalResourceRequest req, LocalizedResource
                                                       rsrc)
 {
     if (useLocalCacheDirectoryManager)
     {
         Path rsrcPath = null;
         if (inProgressLocalResourcesMap.Contains(req))
         {
             // This happens when localization of a resource fails.
             rsrcPath = Sharpen.Collections.Remove(inProgressLocalResourcesMap, req);
         }
         else
         {
             if (rsrc != null && rsrc.GetLocalPath() != null)
             {
                 rsrcPath = rsrc.GetLocalPath().GetParent().GetParent();
             }
         }
         if (rsrcPath != null)
         {
             Path parentPath = new Path(rsrcPath.ToUri().GetRawPath());
             while (!directoryManagers.Contains(parentPath))
             {
                 parentPath = parentPath.GetParent();
                 if (parentPath == null)
                 {
                     return;
                 }
             }
             if (parentPath != null)
             {
                 string parentDir = parentPath.ToUri().GetRawPath().ToString();
                 LocalCacheDirectoryManager dir = directoryManagers[parentPath];
                 string rsrcDir = rsrcPath.ToUri().GetRawPath();
                 if (rsrcDir.Equals(parentDir))
                 {
                     dir.DecrementFileCountForPath(string.Empty);
                 }
                 else
                 {
                     dir.DecrementFileCountForPath(Sharpen.Runtime.Substring(rsrcDir, parentDir.Length
                                                                             + 1));
                 }
             }
         }
     }
 }
Пример #6
0
        public virtual void TestHierarchicalSubDirectoryCreation()
        {
            // setting per directory file limit to 1.
            YarnConfiguration conf = new YarnConfiguration();

            conf.Set(YarnConfiguration.NmLocalCacheMaxFilesPerDirectory, "37");
            LocalCacheDirectoryManager hDir = new LocalCacheDirectoryManager(conf);

            // Test root directory path = ""
            NUnit.Framework.Assert.IsTrue(hDir.GetRelativePathForLocalization().IsEmpty());
            // Testing path generation from "0" to "0/0/z/z"
            for (int i = 1; i <= 37 * 36 * 36; i++)
            {
                StringBuilder sb  = new StringBuilder();
                string        num = Sharpen.Extensions.ToString(i - 1, 36);
                if (num.Length == 1)
                {
                    sb.Append(num[0]);
                }
                else
                {
                    sb.Append(Sharpen.Extensions.ToString(System.Convert.ToInt32(Sharpen.Runtime.Substring
                                                                                     (num, 0, 1), 36) - 1, 36));
                }
                for (int j = 1; j < num.Length; j++)
                {
                    sb.Append(Path.Separator).Append(num[j]);
                }
                NUnit.Framework.Assert.AreEqual(sb.ToString(), hDir.GetRelativePathForLocalization
                                                    ());
            }
            string testPath1 = "4";
            string testPath2 = "2";

            /*
             * Making sure directory "4" and "2" becomes non-full so that they are
             * reused for future getRelativePathForLocalization() calls in the order
             * they are freed.
             */
            hDir.DecrementFileCountForPath(testPath1);
            hDir.DecrementFileCountForPath(testPath2);
            // After below call directory "4" should become full.
            NUnit.Framework.Assert.AreEqual(testPath1, hDir.GetRelativePathForLocalization());
            NUnit.Framework.Assert.AreEqual(testPath2, hDir.GetRelativePathForLocalization());
        }
Пример #7
0
        public virtual void TestIncrementFileCountForPath()
        {
            YarnConfiguration conf = new YarnConfiguration();

            conf.SetInt(YarnConfiguration.NmLocalCacheMaxFilesPerDirectory, LocalCacheDirectoryManager
                        .DirectoriesPerLevel + 2);
            LocalCacheDirectoryManager mgr = new LocalCacheDirectoryManager(conf);
            string rootPath = string.Empty;

            mgr.IncrementFileCountForPath(rootPath);
            NUnit.Framework.Assert.AreEqual(rootPath, mgr.GetRelativePathForLocalization());
            NUnit.Framework.Assert.IsFalse("root dir should be full", rootPath.Equals(mgr.GetRelativePathForLocalization
                                                                                          ()));
            // finish filling the other directory
            mgr.GetRelativePathForLocalization();
            // free up space in the root dir
            mgr.DecrementFileCountForPath(rootPath);
            mgr.DecrementFileCountForPath(rootPath);
            NUnit.Framework.Assert.AreEqual(rootPath, mgr.GetRelativePathForLocalization());
            NUnit.Framework.Assert.AreEqual(rootPath, mgr.GetRelativePathForLocalization());
            string otherDir = mgr.GetRelativePathForLocalization();

            NUnit.Framework.Assert.IsFalse("root dir should be full", otherDir.Equals(rootPath
                                                                                      ));
            string deepDir0 = "d/e/e/p/0";
            string deepDir1 = "d/e/e/p/1";
            string deepDir2 = "d/e/e/p/2";
            string deepDir3 = "d/e/e/p/3";

            mgr.IncrementFileCountForPath(deepDir0);
            NUnit.Framework.Assert.AreEqual(otherDir, mgr.GetRelativePathForLocalization());
            NUnit.Framework.Assert.AreEqual(deepDir0, mgr.GetRelativePathForLocalization());
            NUnit.Framework.Assert.AreEqual("total dir count incorrect after increment", deepDir1
                                            , mgr.GetRelativePathForLocalization());
            mgr.IncrementFileCountForPath(deepDir2);
            mgr.IncrementFileCountForPath(deepDir1);
            mgr.IncrementFileCountForPath(deepDir2);
            NUnit.Framework.Assert.AreEqual(deepDir3, mgr.GetRelativePathForLocalization());
        }