protected internal virtual ThreadPoolExecutor InitializeCacheExecutor(FilePath parent ) { if (storageType.IsTransient()) { return(null); } if (dataset.datanode == null) { // FsVolumeImpl is used in test. return(null); } int maxNumThreads = dataset.datanode.GetConf().GetInt(DFSConfigKeys.DfsDatanodeFsdatasetcacheMaxThreadsPerVolumeKey , DFSConfigKeys.DfsDatanodeFsdatasetcacheMaxThreadsPerVolumeDefault); ThreadFactory workerFactory = new ThreadFactoryBuilder().SetDaemon(true).SetNameFormat ("FsVolumeImplWorker-" + parent.ToString() + "-%d").Build(); ThreadPoolExecutor executor = new ThreadPoolExecutor(1, maxNumThreads, 60, TimeUnit .Seconds, new LinkedBlockingQueue <Runnable>(), workerFactory); executor.AllowCoreThreadTimeOut(true); return(executor); }
/// <exception cref="System.Exception"/> protected override void ServiceStart() { ThreadFactoryBuilder tfBuilder = new ThreadFactoryBuilder().SetNameFormat("CommitterEvent Processor #%d" ); if (jobClassLoader != null) { // if the job classloader is enabled, we need to use the job classloader // as the thread context classloader (TCCL) of these threads in case the // committer needs to load another class via TCCL ThreadFactory backingTf = new _ThreadFactory_125(this); tfBuilder.SetThreadFactory(backingTf); } ThreadFactory tf = tfBuilder.Build(); launcherPool = new ThreadPoolExecutor(5, 5, 1, TimeUnit.Hours, new LinkedBlockingQueue <Runnable>(), tf); eventHandlingThread = new Sharpen.Thread(new _Runnable_138(this)); // the events from the queue are handled in parallel // using a thread pool eventHandlingThread.SetName("CommitterEvent Handler"); eventHandlingThread.Start(); base.ServiceStart(); }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("DeletionService #%d" ).Build(); if (conf != null) { sched = new DeletionService.DelServiceSchedThreadPoolExecutor(conf.GetInt(YarnConfiguration .NmDeleteThreadCount, YarnConfiguration.DefaultNmDeleteThreadCount), tf); debugDelay = conf.GetInt(YarnConfiguration.DebugNmDeleteDelaySec, 0); } else { sched = new DeletionService.DelServiceSchedThreadPoolExecutor(YarnConfiguration.DefaultNmDeleteThreadCount , tf); } sched.SetExecuteExistingDelayedTasksAfterShutdownPolicy(false); sched.SetKeepAliveTime(60L, TimeUnit.Seconds); if (stateStore.CanRecover()) { Recover(stateStore.LoadDeletionServiceState()); } base.ServiceInit(conf); }
/// <exception cref="System.Exception"/> protected override void ServiceStart() { ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("ContainerLauncher #%d" ).SetDaemon(true).Build(); // Start with a default core-pool size of 10 and change it dynamically. launcherPool = new ThreadPoolExecutor(initialPoolSize, int.MaxValue, 1, TimeUnit. Hours, new LinkedBlockingQueue <Runnable>(), tf); eventHandlingThread = new _Thread_269(this); // See if we need up the pool size only if haven't reached the // maximum limit yet. // nodes where containers will run at *this* point of time. This is // *not* the cluster size and doesn't need to be. // Bump up the pool size to idealPoolSize+initialPoolSize, the // later is just a buffer so we are not always increasing the // pool-size // the events from the queue are handled in parallel // using a thread pool // TODO: Group launching of multiple containers to a single // NodeManager into a single connection eventHandlingThread.SetName("ContainerLauncher Event Handler"); eventHandlingThread.Start(); base.ServiceStart(); }
/// <summary>Creates the executor service used to run map tasks.</summary> /// <returns>an ExecutorService instance that handles map tasks</returns> protected internal virtual ExecutorService CreateMapExecutor() { lock (this) { // Determine the size of the thread pool to use int maxMapThreads = this.job.GetInt(LocalJobRunner.LocalMaxMaps, 1); if (maxMapThreads < 1) { throw new ArgumentException("Configured " + LocalJobRunner.LocalMaxMaps + " must be >= 1" ); } maxMapThreads = Math.Min(maxMapThreads, this.numMapTasks); maxMapThreads = Math.Max(maxMapThreads, 1); // In case of no tasks. LocalJobRunner.Log.Debug("Starting mapper thread pool executor."); LocalJobRunner.Log.Debug("Max local threads: " + maxMapThreads); LocalJobRunner.Log.Debug("Map tasks to process: " + this.numMapTasks); // Create a new executor service to drain the work queue. ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("LocalJobRunner Map Task Executor #%d" ).Build(); ExecutorService executor = Executors.NewFixedThreadPool(maxMapThreads, tf); return(executor); } }
/// <summary> /// Set up the distributed cache by localizing the resources, and updating /// the configuration with references to the localized resources. /// </summary> /// <param name="conf"/> /// <exception cref="System.IO.IOException"/> public virtual void Setup(JobConf conf) { FilePath workDir = new FilePath(Runtime.GetProperty("user.dir")); // Generate YARN local resources objects corresponding to the distributed // cache configuration IDictionary <string, LocalResource> localResources = new LinkedHashMap <string, LocalResource >(); MRApps.SetupDistributedCache(conf, localResources); // Generating unique numbers for FSDownload. AtomicLong uniqueNumberGenerator = new AtomicLong(Runtime.CurrentTimeMillis()); // Find which resources are to be put on the local classpath IDictionary <string, Path> classpaths = new Dictionary <string, Path>(); Path[] archiveClassPaths = DistributedCache.GetArchiveClassPaths(conf); if (archiveClassPaths != null) { foreach (Path p in archiveClassPaths) { classpaths[p.ToUri().GetPath().ToString()] = p; } } Path[] fileClassPaths = DistributedCache.GetFileClassPaths(conf); if (fileClassPaths != null) { foreach (Path p in fileClassPaths) { classpaths[p.ToUri().GetPath().ToString()] = p; } } // Localize the resources LocalDirAllocator localDirAllocator = new LocalDirAllocator(MRConfig.LocalDir); FileContext localFSFileContext = FileContext.GetLocalFSFileContext(); UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); ExecutorService exec = null; try { ThreadFactory tf = new ThreadFactoryBuilder().SetNameFormat("LocalDistributedCacheManager Downloader #%d" ).Build(); exec = Executors.NewCachedThreadPool(tf); Path destPath = localDirAllocator.GetLocalPathForWrite(".", conf); IDictionary <LocalResource, Future <Path> > resourcesToPaths = Maps.NewHashMap(); foreach (LocalResource resource in localResources.Values) { Callable <Path> download = new FSDownload(localFSFileContext, ugi, conf, new Path( destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet())), resource ); Future <Path> future = exec.Submit(download); resourcesToPaths[resource] = future; } foreach (KeyValuePair <string, LocalResource> entry in localResources) { LocalResource resource_1 = entry.Value; Path path; try { path = resourcesToPaths[resource_1].Get(); } catch (Exception e) { throw new IOException(e); } catch (ExecutionException e) { throw new IOException(e); } string pathString = path.ToUri().ToString(); string link = entry.Key; string target = new FilePath(path.ToUri()).GetPath(); Symlink(workDir, target, link); if (resource_1.GetType() == LocalResourceType.Archive) { localArchives.AddItem(pathString); } else { if (resource_1.GetType() == LocalResourceType.File) { localFiles.AddItem(pathString); } else { if (resource_1.GetType() == LocalResourceType.Pattern) { //PATTERN is not currently used in local mode throw new ArgumentException("Resource type PATTERN is not " + "implemented yet. " + resource_1.GetResource()); } } } Path resourcePath; try { resourcePath = ConverterUtils.GetPathFromYarnURL(resource_1.GetResource()); } catch (URISyntaxException e) { throw new IOException(e); } Log.Info(string.Format("Localized %s as %s", resourcePath, path)); string cp = resourcePath.ToUri().GetPath(); if (classpaths.Keys.Contains(cp)) { localClasspaths.AddItem(path.ToUri().GetPath().ToString()); } } } finally { if (exec != null) { exec.Shutdown(); } } // Update the configuration object with localized data. if (!localArchives.IsEmpty()) { conf.Set(MRJobConfig.CacheLocalarchives, StringUtils.ArrayToString(Sharpen.Collections.ToArray (localArchives, new string[localArchives.Count]))); } if (!localFiles.IsEmpty()) { conf.Set(MRJobConfig.CacheLocalfiles, StringUtils.ArrayToString(Sharpen.Collections.ToArray (localFiles, new string[localArchives.Count]))); } setupCalled = true; }