protected internal virtual ThreadPoolExecutor InitializeCacheExecutor(FilePath parent
)
{
if (storageType.IsTransient())
{
return(null);
}
if (dataset.datanode == null)
{
// FsVolumeImpl is used in test.
return(null);
}
int maxNumThreads = dataset.datanode.GetConf().GetInt(DFSConfigKeys.DfsDatanodeFsdatasetcacheMaxThreadsPerVolumeKey
, DFSConfigKeys.DfsDatanodeFsdatasetcacheMaxThreadsPerVolumeDefault);
ThreadFactory workerFactory = new ThreadFactoryBuilder().SetDaemon(true).SetNameFormat
("FsVolumeImplWorker-" + parent.ToString() + "-%d").Build();
ThreadPoolExecutor executor = new ThreadPoolExecutor(1, maxNumThreads, 60, TimeUnit
.Seconds, new LinkedBlockingQueue <Runnable>(), workerFactory);
executor.AllowCoreThreadTimeOut(true);
return(executor);
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
ThreadFactoryBuilder tfBuilder = new ThreadFactoryBuilder().SetNameFormat("CommitterEvent Processor #%d"
);
if (jobClassLoader != null)
{
// if the job classloader is enabled, we need to use the job classloader
// as the thread context classloader (TCCL) of these threads in case the
// committer needs to load another class via TCCL
ThreadFactory backingTf = new _ThreadFactory_125(this);
tfBuilder.SetThreadFactory(backingTf);
}
ThreadFactory tf = tfBuilder.Build();
launcherPool = new ThreadPoolExecutor(5, 5, 1, TimeUnit.Hours, new LinkedBlockingQueue
<Runnable>(), tf);
eventHandlingThread = new Sharpen.Thread(new _Runnable_138(this));
// the events from the queue are handled in parallel
// using a thread pool
eventHandlingThread.SetName("CommitterEvent Handler");
eventHandlingThread.Start();
base.ServiceStart();
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("DeletionService #%d"
).Build();
if (conf != null)
{
sched = new DeletionService.DelServiceSchedThreadPoolExecutor(conf.GetInt(YarnConfiguration
.NmDeleteThreadCount, YarnConfiguration.DefaultNmDeleteThreadCount), tf);
debugDelay = conf.GetInt(YarnConfiguration.DebugNmDeleteDelaySec, 0);
}
else
{
sched = new DeletionService.DelServiceSchedThreadPoolExecutor(YarnConfiguration.DefaultNmDeleteThreadCount
, tf);
}
sched.SetExecuteExistingDelayedTasksAfterShutdownPolicy(false);
sched.SetKeepAliveTime(60L, TimeUnit.Seconds);
if (stateStore.CanRecover())
{
Recover(stateStore.LoadDeletionServiceState());
}
base.ServiceInit(conf);
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("ContainerLauncher #%d"
).SetDaemon(true).Build();
// Start with a default core-pool size of 10 and change it dynamically.
launcherPool = new ThreadPoolExecutor(initialPoolSize, int.MaxValue, 1, TimeUnit.
Hours, new LinkedBlockingQueue <Runnable>(), tf);
eventHandlingThread = new _Thread_269(this);
// See if we need up the pool size only if haven't reached the
// maximum limit yet.
// nodes where containers will run at *this* point of time. This is
// *not* the cluster size and doesn't need to be.
// Bump up the pool size to idealPoolSize+initialPoolSize, the
// later is just a buffer so we are not always increasing the
// pool-size
// the events from the queue are handled in parallel
// using a thread pool
// TODO: Group launching of multiple containers to a single
// NodeManager into a single connection
eventHandlingThread.SetName("ContainerLauncher Event Handler");
eventHandlingThread.Start();
base.ServiceStart();
}
/// <summary>Creates the executor service used to run map tasks.</summary>
/// <returns>an ExecutorService instance that handles map tasks</returns>
protected internal virtual ExecutorService CreateMapExecutor()
{
lock (this)
{
// Determine the size of the thread pool to use
int maxMapThreads = this.job.GetInt(LocalJobRunner.LocalMaxMaps, 1);
if (maxMapThreads < 1)
{
throw new ArgumentException("Configured " + LocalJobRunner.LocalMaxMaps + " must be >= 1"
);
}
maxMapThreads = Math.Min(maxMapThreads, this.numMapTasks);
maxMapThreads = Math.Max(maxMapThreads, 1);
// In case of no tasks.
LocalJobRunner.Log.Debug("Starting mapper thread pool executor.");
LocalJobRunner.Log.Debug("Max local threads: " + maxMapThreads);
LocalJobRunner.Log.Debug("Map tasks to process: " + this.numMapTasks);
// Create a new executor service to drain the work queue.
ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("LocalJobRunner Map Task Executor #%d"
).Build();
ExecutorService executor = Executors.NewFixedThreadPool(maxMapThreads, tf);
return(executor);
}
}
/// <summary>
/// Set up the distributed cache by localizing the resources, and updating
/// the configuration with references to the localized resources.
/// </summary>
/// <param name="conf"/>
/// <exception cref="System.IO.IOException"/>
public virtual void Setup(JobConf conf)
{
FilePath workDir = new FilePath(Runtime.GetProperty("user.dir"));
// Generate YARN local resources objects corresponding to the distributed
// cache configuration
IDictionary <string, LocalResource> localResources = new LinkedHashMap <string, LocalResource
>();
MRApps.SetupDistributedCache(conf, localResources);
// Generating unique numbers for FSDownload.
AtomicLong uniqueNumberGenerator = new AtomicLong(Runtime.CurrentTimeMillis());
// Find which resources are to be put on the local classpath
IDictionary <string, Path> classpaths = new Dictionary <string, Path>();
Path[] archiveClassPaths = DistributedCache.GetArchiveClassPaths(conf);
if (archiveClassPaths != null)
{
foreach (Path p in archiveClassPaths)
{
classpaths[p.ToUri().GetPath().ToString()] = p;
}
}
Path[] fileClassPaths = DistributedCache.GetFileClassPaths(conf);
if (fileClassPaths != null)
{
foreach (Path p in fileClassPaths)
{
classpaths[p.ToUri().GetPath().ToString()] = p;
}
}
// Localize the resources
LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LocalDir);
FileContext localFSFileContext = FileContext.GetLocalFSFileContext();
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
ExecutorService exec = null;
try
{
ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("LocalDistributedCacheManager Downloader #%d"
).Build();
exec = Executors.NewCachedThreadPool(tf);
Path destPath = localDirAllocator.GetLocalPathForWrite(".", conf);
IDictionary <LocalResource, Future <Path> > resourcesToPaths = Maps.NewHashMap();
foreach (LocalResource resource in localResources.Values)
{
Callable <Path> download = new FSDownload(localFSFileContext, ugi, conf, new Path(
destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet())), resource
);
Future <Path> future = exec.Submit(download);
resourcesToPaths[resource] = future;
}
foreach (KeyValuePair <string, LocalResource> entry in localResources)
{
LocalResource resource_1 = entry.Value;
Path path;
try
{
path = resourcesToPaths[resource_1].Get();
}
catch (Exception e)
{
throw new IOException(e);
}
catch (ExecutionException e)
{
throw new IOException(e);
}
string pathString = path.ToUri().ToString();
string link = entry.Key;
string target = new FilePath(path.ToUri()).GetPath();
Symlink(workDir, target, link);
if (resource_1.GetType() == LocalResourceType.Archive)
{
localArchives.AddItem(pathString);
}
else
{
if (resource_1.GetType() == LocalResourceType.File)
{
localFiles.AddItem(pathString);
}
else
{
if (resource_1.GetType() == LocalResourceType.Pattern)
{
//PATTERN is not currently used in local mode
throw new ArgumentException("Resource type PATTERN is not " + "implemented yet. "
+ resource_1.GetResource());
}
}
}
Path resourcePath;
try
{
resourcePath = ConverterUtils.GetPathFromYarnURL(resource_1.GetResource());
}
catch (URISyntaxException e)
{
throw new IOException(e);
}
Log.Info(string.Format("Localized %s as %s", resourcePath, path));
string cp = resourcePath.ToUri().GetPath();
if (classpaths.Keys.Contains(cp))
{
localClasspaths.AddItem(path.ToUri().GetPath().ToString());
}
}
}
finally
{
if (exec != null)
{
exec.Shutdown();
}
}
// Update the configuration object with localized data.
if (!localArchives.IsEmpty())
{
conf.Set(MRJobConfig.CacheLocalarchives, StringUtils.ArrayToString(Sharpen.Collections.ToArray
(localArchives, new string[localArchives.Count])));
}
if (!localFiles.IsEmpty())
{
conf.Set(MRJobConfig.CacheLocalfiles, StringUtils.ArrayToString(Sharpen.Collections.ToArray
(localFiles, new string[localArchives.Count])));
}
setupCalled = true;
}
