/// <summary>Constructor.</summary>
        /// <param name="numThreads">
        /// -- if less than or equal to 0, then automatically determine the number
        /// of threads. Otherwise, the size of the underlying threadpool.
        /// </param>
        /// <param name="processor"/>
        /// <param name="orderResults">
        /// -- If true, return results in the order submitted. Otherwise, return results
        /// as they become available.
        /// </param>
        public MulticoreWrapper(int numThreads, IThreadsafeProcessor <I, O> processor, bool orderResults)
        {
            // Which id was the last id returned.  Only meaningful in the case
            // of a queue where output order matters.
            //  private final ExecutorCompletionService<Integer> queue;
            nThreads          = numThreads <= 0 ? Runtime.GetRuntime().AvailableProcessors() : numThreads;
            this.orderResults = orderResults;
            outputQueue       = new ConcurrentHashMap <int, O>(2 * nThreads);
            threadPool        = BuildThreadPool(nThreads);
            //    queue = new ExecutorCompletionService<Integer>(threadPool);
            idleProcessors = new ArrayBlockingQueue <int>(nThreads, false);
            callback       = null;
            // Sanity check: Fixed thread pool so prevent timeouts.
            // Default should be false
            threadPool.AllowCoreThreadTimeOut(false);
            threadPool.PrestartAllCoreThreads();
            // Setup the processors, one per thread
            IList <IThreadsafeProcessor <I, O> > procList = new List <IThreadsafeProcessor <I, O> >(nThreads);

            procList.Add(processor);
            idleProcessors.Add(0);
            for (int i = 1; i < nThreads; ++i)
            {
                procList.Add(processor.NewInstance());
                idleProcessors.Add(i);
            }
            processorList = Java.Util.Collections.UnmodifiableList(procList);
        }
Example #2
0
 /// <summary>
 /// Create a AsyncDiskServices with a set of volumes (specified by their
 /// root directories).
 /// </summary>
 /// <remarks>
 /// Create a AsyncDiskServices with a set of volumes (specified by their
 /// root directories).
 /// The AsyncDiskServices uses one ThreadPool per volume to do the async
 /// disk operations.
 /// </remarks>
 /// <param name="volumes">The roots of the file system volumes.</param>
 public AsyncDiskService(string[] volumes)
 {
     /*
      * This class is a container of multiple thread pools, each for a volume,
      * so that we can schedule async disk operations easily.
      *
      * Examples of async disk operations are deletion of files.
      * We can move the files to a "TO_BE_DELETED" folder before asychronously
      * deleting it, to make sure the caller can run it faster.
      */
     // ThreadPool core pool size
     // ThreadPool maximum pool size
     // ThreadPool keep-alive time for threads over core pool size
     threadFactory = new _ThreadFactory_73(this);
     // Create one ThreadPool per volume
     for (int v = 0; v < volumes.Length; v++)
     {
         ThreadPoolExecutor executor = new ThreadPoolExecutor(CoreThreadsPerVolume, MaximumThreadsPerVolume
                                                              , ThreadsKeepAliveSeconds, TimeUnit.Seconds, new LinkedBlockingQueue <Runnable>()
                                                              , threadFactory);
         // This can reduce the number of running threads
         executor.AllowCoreThreadTimeOut(true);
         executors[volumes[v]] = executor;
     }
 }
        private void AddExecutorForVolume(FilePath volume)
        {
            ThreadFactory      threadFactory = new _ThreadFactory_71(this, volume);
            ThreadPoolExecutor executor      = new ThreadPoolExecutor(CoreThreadsPerVolume, MaximumThreadsPerVolume
                                                                      , ThreadsKeepAliveSeconds, TimeUnit.Seconds, new LinkedBlockingQueue <Runnable>()
                                                                      , threadFactory);

            // This can reduce the number of running threads
            executor.AllowCoreThreadTimeOut(true);
            executors[volume] = executor;
        }
Example #4
0
 public AsyncDataService()
 {
     // ThreadPool core pool size
     // ThreadPool maximum pool size
     // ThreadPool keep-alive time for threads over core pool size
     threadFactory = new _ThreadFactory_47(this);
     executor      = new ThreadPoolExecutor(CoreThreadsPerVolume, MaximumThreadsPerVolume,
                                            ThreadsKeepAliveSeconds, TimeUnit.Seconds, new LinkedBlockingQueue <Runnable>(),
                                            threadFactory);
     // This can reduce the number of running threads
     executor.AllowCoreThreadTimeOut(true);
 }
Example #5
0
        protected internal virtual ThreadPoolExecutor InitializeCacheExecutor(FilePath parent
                                                                              )
        {
            if (storageType.IsTransient())
            {
                return(null);
            }
            if (dataset.datanode == null)
            {
                // FsVolumeImpl is used in test.
                return(null);
            }
            int maxNumThreads = dataset.datanode.GetConf().GetInt(DFSConfigKeys.DfsDatanodeFsdatasetcacheMaxThreadsPerVolumeKey
                                                                  , DFSConfigKeys.DfsDatanodeFsdatasetcacheMaxThreadsPerVolumeDefault);
            ThreadFactory workerFactory = new ThreadFactoryBuilder().SetDaemon(true).SetNameFormat
                                              ("FsVolumeImplWorker-" + parent.ToString() + "-%d").Build();
            ThreadPoolExecutor executor = new ThreadPoolExecutor(1, maxNumThreads, 60, TimeUnit
                                                                 .Seconds, new LinkedBlockingQueue <Runnable>(), workerFactory);

            executor.AllowCoreThreadTimeOut(true);
            return(executor);
        }