/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
this.clientBindAddress = GetBindAddress(conf);
this.cacheDepth = SharedCacheUtil.GetCacheDepth(conf);
this.cacheRoot = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration.DefaultSharedCacheRoot
);
base.ServiceInit(conf);
}
/// <param name="resource">the local resource that contains the original remote path</param>
/// <param name="localPath">
/// the path in the local filesystem where the resource is
/// localized
/// </param>
/// <param name="fs">the filesystem of the shared cache</param>
/// <param name="localFs">the local filesystem</param>
public SharedCacheUploader(LocalResource resource, Path localPath, string user, Configuration
conf, SCMUploaderProtocol scmClient, FileSystem fs, FileSystem localFs)
{
this.resource = resource;
this.localPath = localPath;
this.user = user;
this.conf = conf;
this.scmClient = scmClient;
this.fs = fs;
this.sharedCacheRootDir = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration
.DefaultSharedCacheRoot);
this.nestedLevel = SharedCacheUtil.GetCacheDepth(conf);
this.checksum = SharedCacheChecksumFactory.GetChecksum(conf);
this.localFs = localFs;
this.recordFactory = RecordFactoryProvider.GetRecordFactory(null);
}
/// <summary>Creates a cleaner task based on the configuration.</summary>
/// <remarks>
/// Creates a cleaner task based on the configuration. This is provided for
/// convenience.
/// </remarks>
/// <param name="conf"/>
/// <param name="store"/>
/// <param name="metrics"/>
/// <param name="cleanerTaskLock">
/// lock that ensures a serial execution of cleaner
/// task
/// </param>
/// <returns>an instance of a CleanerTask</returns>
public static Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.CleanerTask Create
(Configuration conf, SCMStore store, CleanerMetrics metrics, Lock cleanerTaskLock
)
{
try
{
// get the root directory for the shared cache
string location = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration.DefaultSharedCacheRoot
);
long sleepTime = conf.GetLong(YarnConfiguration.ScmCleanerResourceSleepMs, YarnConfiguration
.DefaultScmCleanerResourceSleepMs);
int nestedLevel = SharedCacheUtil.GetCacheDepth(conf);
FileSystem fs = FileSystem.Get(conf);
return(new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.CleanerTask(location,
sleepTime, nestedLevel, fs, store, metrics, cleanerTaskLock));
}
catch (IOException e)
{
Log.Error("Unable to obtain the filesystem for the cleaner service", e);
throw new ExceptionInInitializerError(e);
}
}
internal virtual IDictionary <string, string> GetInitialCachedResources(FileSystem
fs, Configuration conf)
{
// get the root directory for the shared cache
string location = conf.Get(YarnConfiguration.SharedCacheRoot, YarnConfiguration.DefaultSharedCacheRoot
);
Path root = new Path(location);
if (!fs.Exists(root))
{
string message = "The shared cache root directory " + location + " was not found";
Log.Error(message);
throw new IOException(message);
}
int nestedLevel = SharedCacheUtil.GetCacheDepth(conf);
// now traverse individual directories and process them
// the directory structure is specified by the nested level parameter
// (e.g. 9/c/d/<checksum>/file)
string pattern = SharedCacheUtil.GetCacheEntryGlobPattern(nestedLevel + 1);
Log.Info("Querying for all individual cached resource files");
FileStatus[] entries = fs.GlobStatus(new Path(root, pattern));
int numEntries = entries == null ? 0 : entries.Length;
Log.Info("Found " + numEntries + " files: processing for one resource per " + "key"
);
IDictionary <string, string> initialCachedEntries = new Dictionary <string, string>
();
if (entries != null)
{
foreach (FileStatus entry in entries)
{
Path file = entry.GetPath();
string fileName = file.GetName();
if (entry.IsFile())
{
// get the parent to get the checksum
Path parent = file.GetParent();
if (parent != null)
{
// the name of the immediate parent directory is the checksum
string key = parent.GetName();
// make sure we insert only one file per checksum whichever comes
// first
if (initialCachedEntries.Contains(key))
{
Log.Warn("Key " + key + " is already mapped to file " + initialCachedEntries[key]
+ "; file " + fileName + " will not be added");
}
else
{
initialCachedEntries[key] = fileName;
}
}
}
}
}
Log.Info("A total of " + initialCachedEntries.Count + " files are now mapped");
return(initialCachedEntries);
}
