/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
this.clientBindAddress = GetBindAddress(conf);
this.cacheDepth = SharedCacheUtil.GetCacheDepth(conf);
this.cacheRoot = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration.DefaultSharedCacheRoot
);
base.ServiceInit(conf);
}
/// <summary>
/// Sweeps and processes the shared cache area to clean up stale and orphaned
/// files.
/// </summary>
internal virtual void Process()
{
// mark the beginning of the run in the metrics
metrics.ReportCleaningStart();
try
{
// now traverse individual directories and process them
// the directory structure is specified by the nested level parameter
// (e.g. 9/c/d/<checksum>)
string pattern = SharedCacheUtil.GetCacheEntryGlobPattern(nestedLevel);
FileStatus[] resources = fs.GlobStatus(new Path(root, pattern));
int numResources = resources == null ? 0 : resources.Length;
Log.Info("Processing " + numResources + " resources in the shared cache");
long beginMs = Runtime.CurrentTimeMillis();
if (resources != null)
{
foreach (FileStatus resource in resources)
{
// check for interruption so it can abort in a timely manner in case
// of shutdown
if (Sharpen.Thread.CurrentThread().IsInterrupted())
{
Log.Warn("The cleaner task was interrupted. Aborting.");
break;
}
if (resource.IsDirectory())
{
ProcessSingleResource(resource);
}
else
{
Log.Warn("Invalid file at path " + resource.GetPath().ToString() + " when a directory was expected"
);
}
// add sleep time between cleaning each directory if it is non-zero
if (sleepTime > 0)
{
Sharpen.Thread.Sleep(sleepTime);
}
}
}
long endMs = Runtime.CurrentTimeMillis();
long durationMs = endMs - beginMs;
Log.Info("Processed " + numResources + " resource(s) in " + durationMs + " ms.");
}
catch (IOException e1)
{
Log.Error("Unable to complete the cleaner task", e1);
}
catch (Exception)
{
Sharpen.Thread.CurrentThread().Interrupt();
}
}
/// <param name="resource">the local resource that contains the original remote path</param>
/// <param name="localPath">
/// the path in the local filesystem where the resource is
/// localized
/// </param>
/// <param name="fs">the filesystem of the shared cache</param>
/// <param name="localFs">the local filesystem</param>
public SharedCacheUploader(LocalResource resource, Path localPath, string user, Configuration
conf, SCMUploaderProtocol scmClient, FileSystem fs, FileSystem localFs)
{
this.resource = resource;
this.localPath = localPath;
this.user = user;
this.conf = conf;
this.scmClient = scmClient;
this.fs = fs;
this.sharedCacheRootDir = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration
.DefaultSharedCacheRoot);
this.nestedLevel = SharedCacheUtil.GetCacheDepth(conf);
this.checksum = SharedCacheChecksumFactory.GetChecksum(conf);
this.localFs = localFs;
this.recordFactory = RecordFactoryProvider.GetRecordFactory(null);
}
/// <summary>Creates a cleaner task based on the configuration.</summary>
/// <remarks>
/// Creates a cleaner task based on the configuration. This is provided for
/// convenience.
/// </remarks>
/// <param name="conf"/>
/// <param name="store"/>
/// <param name="metrics"/>
/// <param name="cleanerTaskLock">
/// lock that ensures a serial execution of cleaner
/// task
/// </param>
/// <returns>an instance of a CleanerTask</returns>
public static Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.CleanerTask Create
(Configuration conf, SCMStore store, CleanerMetrics metrics, Lock cleanerTaskLock
)
{
try
{
// get the root directory for the shared cache
string location = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration.DefaultSharedCacheRoot
);
long sleepTime = conf.GetLong(YarnConfiguration.ScmCleanerResourceSleepMs, YarnConfiguration
.DefaultScmCleanerResourceSleepMs);
int nestedLevel = SharedCacheUtil.GetCacheDepth(conf);
FileSystem fs = FileSystem.Get(conf);
return(new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.CleanerTask(location,
sleepTime, nestedLevel, fs, store, metrics, cleanerTaskLock));
}
catch (IOException e)
{
Log.Error("Unable to obtain the filesystem for the cleaner service", e);
throw new ExceptionInInitializerError(e);
}
}
private string GetCacheEntryFilePath(string checksum, string filename)
{
return(SharedCacheUtil.GetCacheEntryPath(this.cacheDepth, this.cacheRoot, checksum
) + Path.SeparatorChar + filename);
}
internal virtual IDictionary <string, string> GetInitialCachedResources(FileSystem
fs, Configuration conf)
{
// get the root directory for the shared cache
string location = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration.DefaultSharedCacheRoot
);
Path root = new Path(location);
if (!fs.Exists(root))
{
string message = "The shared cache root directory " + location + " was not found";
Log.Error(message);
throw new IOException(message);
}
int nestedLevel = SharedCacheUtil.GetCacheDepth(conf);
// now traverse individual directories and process them
// the directory structure is specified by the nested level parameter
// (e.g. 9/c/d/<checksum>/file)
string pattern = SharedCacheUtil.GetCacheEntryGlobPattern(nestedLevel + 1);
Log.Info("Querying for all individual cached resource files");
FileStatus[] entries = fs.GlobStatus(new Path(root, pattern));
int numEntries = entries == null ? 0 : entries.Length;
Log.Info("Found " + numEntries + " files: processing for one resource per " + "key"
);
IDictionary <string, string> initialCachedEntries = new Dictionary <string, string>
();
if (entries != null)
{
foreach (FileStatus entry in entries)
{
Path file = entry.GetPath();
string fileName = file.GetName();
if (entry.IsFile())
{
// get the parent to get the checksum
Path parent = file.GetParent();
if (parent != null)
{
// the name of the immediate parent directory is the checksum
string key = parent.GetName();
// make sure we insert only one file per checksum whichever comes
// first
if (initialCachedEntries.Contains(key))
{
Log.Warn("Key " + key + " is already mapped to file " + initialCachedEntries[key]
+ "; file " + fileName + " will not be added");
}
else
{
initialCachedEntries[key] = fileName;
}
}
}
}
}
Log.Info("A total of " + initialCachedEntries.Count + " files are now mapped");
return(initialCachedEntries);
}
/// <summary>
/// Uploads the file under the shared cache, and notifies the shared cache
/// manager.
/// </summary>
/// <remarks>
/// Uploads the file under the shared cache, and notifies the shared cache
/// manager. If it is unable to upload the file because it already exists, it
/// returns false.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual bool Call()
{
Path tempPath = null;
try
{
if (!VerifyAccess())
{
Log.Warn("User " + user + " is not authorized to upload file " + localPath.GetName
());
return(false);
}
// first determine the actual local path that will be used for upload
Path actualPath = GetActualPath();
// compute the checksum
string checksumVal = ComputeChecksum(actualPath);
// create the directory (if it doesn't exist)
Path directoryPath = new Path(SharedCacheUtil.GetCacheEntryPath(nestedLevel, sharedCacheRootDir
, checksumVal));
// let's not check if the directory already exists: in the vast majority
// of the cases, the directory does not exist; as long as mkdirs does not
// error out if it exists, we should be fine
fs.Mkdirs(directoryPath, DirectoryPermission);
// create the temporary file
tempPath = new Path(directoryPath, GetTemporaryFileName(actualPath));
if (!UploadFile(actualPath, tempPath))
{
Log.Warn("Could not copy the file to the shared cache at " + tempPath);
return(false);
}
// set the permission so that it is readable but not writable
fs.SetPermission(tempPath, FilePermission);
// rename it to the final filename
Path finalPath = new Path(directoryPath, actualPath.GetName());
if (!fs.Rename(tempPath, finalPath))
{
Log.Warn("The file already exists under " + finalPath + ". Ignoring this attempt."
);
DeleteTempFile(tempPath);
return(false);
}
// notify the SCM
if (!NotifySharedCacheManager(checksumVal, actualPath.GetName()))
{
// the shared cache manager rejected the upload (as it is likely
// uploaded under a different name
// clean up this file and exit
fs.Delete(finalPath, false);
return(false);
}
// set the replication factor
short replication = (short)conf.GetInt(YarnConfiguration.SharedCacheNmUploaderReplicationFactor
, YarnConfiguration.DefaultSharedCacheNmUploaderReplicationFactor);
fs.SetReplication(finalPath, replication);
Log.Info("File " + actualPath.GetName() + " was uploaded to the shared cache at "
+ finalPath);
return(true);
}
catch (IOException e)
{
Log.Warn("Exception while uploading the file " + localPath.GetName(), e);
// in case an exception is thrown, delete the temp file
DeleteTempFile(tempPath);
throw;
}
}
