/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { this.clientBindAddress = GetBindAddress(conf); this.cacheDepth = SharedCacheUtil.GetCacheDepth(conf); this.cacheRoot = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration.DefaultSharedCacheRoot ); base.ServiceInit(conf); }
/// <summary> /// Sweeps and processes the shared cache area to clean up stale and orphaned /// files. /// </summary> internal virtual void Process() { // mark the beginning of the run in the metrics metrics.ReportCleaningStart(); try { // now traverse individual directories and process them // the directory structure is specified by the nested level parameter // (e.g. 9/c/d/<checksum>) string pattern = SharedCacheUtil.GetCacheEntryGlobPattern(nestedLevel); FileStatus[] resources = fs.GlobStatus(new Path(root, pattern)); int numResources = resources == null ? 0 : resources.Length; Log.Info("Processing " + numResources + " resources in the shared cache"); long beginMs = Runtime.CurrentTimeMillis(); if (resources != null) { foreach (FileStatus resource in resources) { // check for interruption so it can abort in a timely manner in case // of shutdown if (Sharpen.Thread.CurrentThread().IsInterrupted()) { Log.Warn("The cleaner task was interrupted. Aborting."); break; } if (resource.IsDirectory()) { ProcessSingleResource(resource); } else { Log.Warn("Invalid file at path " + resource.GetPath().ToString() + " when a directory was expected" ); } // add sleep time between cleaning each directory if it is non-zero if (sleepTime > 0) { Sharpen.Thread.Sleep(sleepTime); } } } long endMs = Runtime.CurrentTimeMillis(); long durationMs = endMs - beginMs; Log.Info("Processed " + numResources + " resource(s) in " + durationMs + " ms."); } catch (IOException e1) { Log.Error("Unable to complete the cleaner task", e1); } catch (Exception) { Sharpen.Thread.CurrentThread().Interrupt(); } }
/// <param name="resource">the local resource that contains the original remote path</param> /// <param name="localPath"> /// the path in the local filesystem where the resource is /// localized /// </param> /// <param name="fs">the filesystem of the shared cache</param> /// <param name="localFs">the local filesystem</param> public SharedCacheUploader(LocalResource resource, Path localPath, string user, Configuration conf, SCMUploaderProtocol scmClient, FileSystem fs, FileSystem localFs) { this.resource = resource; this.localPath = localPath; this.user = user; this.conf = conf; this.scmClient = scmClient; this.fs = fs; this.sharedCacheRootDir = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration .DefaultSharedCacheRoot); this.nestedLevel = SharedCacheUtil.GetCacheDepth(conf); this.checksum = SharedCacheChecksumFactory.GetChecksum(conf); this.localFs = localFs; this.recordFactory = RecordFactoryProvider.GetRecordFactory(null); }
/// <summary>Creates a cleaner task based on the configuration.</summary> /// <remarks> /// Creates a cleaner task based on the configuration. This is provided for /// convenience. /// </remarks> /// <param name="conf"/> /// <param name="store"/> /// <param name="metrics"/> /// <param name="cleanerTaskLock"> /// lock that ensures a serial execution of cleaner /// task /// </param> /// <returns>an instance of a CleanerTask</returns> public static Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.CleanerTask Create (Configuration conf, SCMStore store, CleanerMetrics metrics, Lock cleanerTaskLock ) { try { // get the root directory for the shared cache string location = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration.DefaultSharedCacheRoot ); long sleepTime = conf.GetLong(YarnConfiguration.ScmCleanerResourceSleepMs, YarnConfiguration .DefaultScmCleanerResourceSleepMs); int nestedLevel = SharedCacheUtil.GetCacheDepth(conf); FileSystem fs = FileSystem.Get(conf); return(new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.CleanerTask(location, sleepTime, nestedLevel, fs, store, metrics, cleanerTaskLock)); } catch (IOException e) { Log.Error("Unable to obtain the filesystem for the cleaner service", e); throw new ExceptionInInitializerError(e); } }
private string GetCacheEntryFilePath(string checksum, string filename) { return(SharedCacheUtil.GetCacheEntryPath(this.cacheDepth, this.cacheRoot, checksum ) + Path.SeparatorChar + filename); }
internal virtual IDictionary <string, string> GetInitialCachedResources(FileSystem fs, Configuration conf) { // get the root directory for the shared cache string location = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration.DefaultSharedCacheRoot ); Path root = new Path(location); if (!fs.Exists(root)) { string message = "The shared cache root directory " + location + " was not found"; Log.Error(message); throw new IOException(message); } int nestedLevel = SharedCacheUtil.GetCacheDepth(conf); // now traverse individual directories and process them // the directory structure is specified by the nested level parameter // (e.g. 9/c/d/<checksum>/file) string pattern = SharedCacheUtil.GetCacheEntryGlobPattern(nestedLevel + 1); Log.Info("Querying for all individual cached resource files"); FileStatus[] entries = fs.GlobStatus(new Path(root, pattern)); int numEntries = entries == null ? 0 : entries.Length; Log.Info("Found " + numEntries + " files: processing for one resource per " + "key" ); IDictionary <string, string> initialCachedEntries = new Dictionary <string, string> (); if (entries != null) { foreach (FileStatus entry in entries) { Path file = entry.GetPath(); string fileName = file.GetName(); if (entry.IsFile()) { // get the parent to get the checksum Path parent = file.GetParent(); if (parent != null) { // the name of the immediate parent directory is the checksum string key = parent.GetName(); // make sure we insert only one file per checksum whichever comes // first if (initialCachedEntries.Contains(key)) { Log.Warn("Key " + key + " is already mapped to file " + initialCachedEntries[key] + "; file " + fileName + " will not be added"); } else { initialCachedEntries[key] = fileName; } } } } } Log.Info("A total of " + initialCachedEntries.Count + " files are now mapped"); return(initialCachedEntries); }
/// <summary> /// Uploads the file under the shared cache, and notifies the shared cache /// manager. /// </summary> /// <remarks> /// Uploads the file under the shared cache, and notifies the shared cache /// manager. If it is unable to upload the file because it already exists, it /// returns false. /// </remarks> /// <exception cref="System.Exception"/> public virtual bool Call() { Path tempPath = null; try { if (!VerifyAccess()) { Log.Warn("User " + user + " is not authorized to upload file " + localPath.GetName ()); return(false); } // first determine the actual local path that will be used for upload Path actualPath = GetActualPath(); // compute the checksum string checksumVal = ComputeChecksum(actualPath); // create the directory (if it doesn't exist) Path directoryPath = new Path(SharedCacheUtil.GetCacheEntryPath(nestedLevel, sharedCacheRootDir , checksumVal)); // let's not check if the directory already exists: in the vast majority // of the cases, the directory does not exist; as long as mkdirs does not // error out if it exists, we should be fine fs.Mkdirs(directoryPath, DirectoryPermission); // create the temporary file tempPath = new Path(directoryPath, GetTemporaryFileName(actualPath)); if (!UploadFile(actualPath, tempPath)) { Log.Warn("Could not copy the file to the shared cache at " + tempPath); return(false); } // set the permission so that it is readable but not writable fs.SetPermission(tempPath, FilePermission); // rename it to the final filename Path finalPath = new Path(directoryPath, actualPath.GetName()); if (!fs.Rename(tempPath, finalPath)) { Log.Warn("The file already exists under " + finalPath + ". Ignoring this attempt." ); DeleteTempFile(tempPath); return(false); } // notify the SCM if (!NotifySharedCacheManager(checksumVal, actualPath.GetName())) { // the shared cache manager rejected the upload (as it is likely // uploaded under a different name // clean up this file and exit fs.Delete(finalPath, false); return(false); } // set the replication factor short replication = (short)conf.GetInt(YarnConfiguration.SharedCacheNmUploaderReplicationFactor , YarnConfiguration.DefaultSharedCacheNmUploaderReplicationFactor); fs.SetReplication(finalPath, replication); Log.Info("File " + actualPath.GetName() + " was uploaded to the shared cache at " + finalPath); return(true); } catch (IOException e) { Log.Warn("Exception while uploading the file " + localPath.GetName(), e); // in case an exception is thrown, delete the temp file DeleteTempFile(tempPath); throw; } }