コード例 #1
0
        public virtual void TestDeleteOnExit()
        {
            // Create deleteOnExit entries
            Path file1 = helper.GetTestRootPath(fc, "file1");

            FileContextTestHelper.CreateFile(fc, file1, numBlocks, blockSize);
            fc.DeleteOnExit(file1);
            CheckDeleteOnExitData(1, fc, file1);
            // Ensure shutdown hook is added
            Assert.True(ShutdownHookManager.Get().HasShutdownHook(FileContext
                                                                  .Finalizer));
            Path file2 = helper.GetTestRootPath(fc, "dir1/file2");

            FileContextTestHelper.CreateFile(fc, file2, numBlocks, blockSize);
            fc.DeleteOnExit(file2);
            CheckDeleteOnExitData(1, fc, file1, file2);
            Path dir = helper.GetTestRootPath(fc, "dir3/dir4/dir5/dir6");

            FileContextTestHelper.CreateFile(fc, dir, numBlocks, blockSize);
            fc.DeleteOnExit(dir);
            CheckDeleteOnExitData(1, fc, file1, file2, dir);
            // trigger deleteOnExit and ensure the registered
            // paths are cleaned up
            FileContext.Finalizer.Run();
            CheckDeleteOnExitData(0, fc, new Path[0]);
            NUnit.Framework.Assert.IsFalse(FileContextTestHelper.Exists(fc, file1));
            NUnit.Framework.Assert.IsFalse(FileContextTestHelper.Exists(fc, file2));
            NUnit.Framework.Assert.IsFalse(FileContextTestHelper.Exists(fc, dir));
        }
コード例 #2
0
 LaunchAppHistoryServer(string[] args)
 {
     Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
                                                           ());
     StringUtils.StartupShutdownMessage(typeof(Org.Apache.Hadoop.Yarn.Server.Applicationhistoryservice.ApplicationHistoryServer
                                               ), args, Log);
     Org.Apache.Hadoop.Yarn.Server.Applicationhistoryservice.ApplicationHistoryServer
         appHistoryServer = null;
     try
     {
         appHistoryServer = new Org.Apache.Hadoop.Yarn.Server.Applicationhistoryservice.ApplicationHistoryServer
                                ();
         ShutdownHookManager.Get().AddShutdownHook(new CompositeService.CompositeServiceShutdownHook
                                                       (appHistoryServer), ShutdownHookPriority);
         YarnConfiguration conf = new YarnConfiguration();
         new GenericOptionsParser(conf, args);
         appHistoryServer.Init(conf);
         appHistoryServer.Start();
     }
     catch (Exception t)
     {
         Log.Fatal("Error starting ApplicationHistoryServer", t);
         ExitUtil.Terminate(-1, "Error starting ApplicationHistoryServer");
     }
     return(appHistoryServer);
 }
コード例 #3
0
ファイル: TaskLog.cs プロジェクト: orf53975/hadoop.net
        public static ScheduledExecutorService CreateLogSyncer()
        {
            ScheduledExecutorService scheduler = Executors.NewSingleThreadScheduledExecutor(new
                                                                                            _ThreadFactory_331());

            ShutdownHookManager.Get().AddShutdownHook(new _Runnable_340(scheduler), 50);
            scheduler.ScheduleWithFixedDelay(new _Runnable_347(), 0L, 5L, TimeUnit.Seconds);
            return(scheduler);
        }
コード例 #4
0
 StartServer(Configuration configuration)
 {
     Org.Apache.Hadoop.Yarn.Server.Webproxy.WebAppProxyServer proxy = new Org.Apache.Hadoop.Yarn.Server.Webproxy.WebAppProxyServer
                                                                          ();
     ShutdownHookManager.Get().AddShutdownHook(new CompositeService.CompositeServiceShutdownHook
                                                   (proxy), ShutdownHookPriority);
     proxy.Init(configuration);
     proxy.Start();
     return(proxy);
 }
コード例 #5
0
ファイル: DFSClientCache.cs プロジェクト: orf53975/hadoop.net
 internal DFSClientCache(NfsConfiguration config, int clientCache)
 {
     this.config      = config;
     this.clientCache = CacheBuilder.NewBuilder().MaximumSize(clientCache).RemovalListener
                            (ClientRemovalListener()).Build(ClientLoader());
     this.inputstreamCache = CacheBuilder.NewBuilder().MaximumSize(DefaultDfsInputstreamCacheSize
                                                                   ).ExpireAfterAccess(DefaultDfsInputstreamCacheTtl, TimeUnit.Seconds).RemovalListener
                                 (InputStreamRemovalListener()).Build(InputStreamLoader());
     ShutdownHookManager.Get().AddShutdownHook(new DFSClientCache.CacheFinalizer(this)
                                               , ShutdownHookPriority);
 }
コード例 #6
0
 public static Org.Apache.Hadoop.Tracing.SpanReceiverHost Get(Configuration conf,
                                                              string confPrefix)
 {
     lock (typeof(Org.Apache.Hadoop.Tracing.SpanReceiverHost))
     {
         Org.Apache.Hadoop.Tracing.SpanReceiverHost host = hosts[confPrefix];
         if (host != null)
         {
             return(host);
         }
         Org.Apache.Hadoop.Tracing.SpanReceiverHost newHost = new Org.Apache.Hadoop.Tracing.SpanReceiverHost
                                                                  (confPrefix);
         newHost.LoadSpanReceivers(conf);
         ShutdownHookManager.Get().AddShutdownHook(new _Runnable_79(newHost), 0);
         hosts[confPrefix] = newHost;
         return(newHost);
     }
 }
コード例 #7
0
 public virtual void UncaughtException(Sharpen.Thread t, Exception e)
 {
     if (ShutdownHookManager.Get().IsShutdownInProgress())
     {
         Log.Error("Thread " + t + " threw an Throwable, but we are shutting " + "down, so ignoring this"
                   , e);
     }
     else
     {
         if (e is Error)
         {
             try
             {
                 Log.Fatal("Thread " + t + " threw an Error.  Shutting down now...", e);
             }
             catch
             {
             }
             //We don't want to not exit because of an issue with logging
             if (e is OutOfMemoryException)
             {
                 //After catching an OOM java says it is undefined behavior, so don't
                 //even try to clean up or we can get stuck on shutdown.
                 try
                 {
                     System.Console.Error.WriteLine("Halting due to Out Of Memory Error...");
                 }
                 catch
                 {
                 }
                 //Again we done want to exit because of logging issues.
                 ExitUtil.Halt(-1);
             }
             else
             {
                 ExitUtil.Terminate(-1);
             }
         }
         else
         {
             Log.Error("Thread " + t + " threw an Exception.", e);
         }
     }
 }
コード例 #8
0
ファイル: Nfs3Base.cs プロジェクト: orf53975/hadoop.net
 public virtual void Start(bool register)
 {
     StartTCPServer();
     // Start TCP server
     if (register)
     {
         ShutdownHookManager.Get().AddShutdownHook(new Nfs3Base.NfsShutdownHook(this), ShutdownHookPriority
                                                   );
         try
         {
             rpcProgram.Register(PortmapMapping.TransportTcp, nfsBoundPort);
         }
         catch (Exception e)
         {
             Log.Fatal("Failed to register the NFSv3 service.", e);
             ExitUtil.Terminate(1, e);
         }
     }
 }
コード例 #9
0
ファイル: NodeManager.cs プロジェクト: orf53975/hadoop.net
 public override void Run()
 {
     try
     {
         this._enclosing.Stop();
     }
     catch (Exception t)
     {
         Org.Apache.Hadoop.Yarn.Server.Nodemanager.NodeManager.Log.Error("Error while shutting down NodeManager"
                                                                         , t);
     }
     finally
     {
         if (this._enclosing.shouldExitOnShutdownEvent && !ShutdownHookManager.Get().IsShutdownInProgress
                 ())
         {
             ExitUtil.Terminate(-1);
         }
     }
 }
コード例 #10
0
 public virtual void Start(bool register)
 {
     StartUDPServer();
     StartTCPServer();
     if (register)
     {
         ShutdownHookManager.Get().AddShutdownHook(new MountdBase.Unregister(this), ShutdownHookPriority
                                                   );
         try
         {
             rpcProgram.Register(PortmapMapping.TransportUdp, udpBoundPort);
             rpcProgram.Register(PortmapMapping.TransportTcp, tcpBoundPort);
         }
         catch (Exception e)
         {
             Log.Fatal("Failed to register the MOUNT service.", e);
             ExitUtil.Terminate(1, e);
         }
     }
 }
コード例 #11
0
 public static void Main(string[] args)
 {
     Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
                                                           ());
     StringUtils.StartupShutdownMessage(typeof(Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.SharedCacheManager
                                               ), args, Log);
     try
     {
         Configuration conf = new YarnConfiguration();
         Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.SharedCacheManager sharedCacheManager
             = new Org.Apache.Hadoop.Yarn.Server.Sharedcachemanager.SharedCacheManager();
         ShutdownHookManager.Get().AddShutdownHook(new CompositeService.CompositeServiceShutdownHook
                                                       (sharedCacheManager), ShutdownHookPriority);
         sharedCacheManager.Init(conf);
         sharedCacheManager.Start();
     }
     catch (Exception t)
     {
         Log.Fatal("Error starting SharedCacheManager", t);
         System.Environment.Exit(-1);
     }
 }
コード例 #12
0
        protected internal virtual void Dispatch(Org.Apache.Hadoop.Yarn.Event.Event @event
                                                 )
        {
            //all events go thru this loop
            if (Log.IsDebugEnabled())
            {
                Log.Debug("Dispatching the event " + @event.GetType().FullName + "." + @event.ToString
                              ());
            }
            Type type = @event.GetType().GetDeclaringClass();

            try
            {
                EventHandler handler = eventDispatchers[type];
                if (handler != null)
                {
                    handler.Handle(@event);
                }
                else
                {
                    throw new Exception("No handler for registered for " + type);
                }
            }
            catch (Exception t)
            {
                //TODO Maybe log the state of the queue
                Log.Fatal("Error in dispatcher thread", t);
                // If serviceStop is called, we should exit this thread gracefully.
                if (exitOnDispatchException && (ShutdownHookManager.Get().IsShutdownInProgress())
                    == false && stopped == false)
                {
                    Sharpen.Thread shutDownThread = new Sharpen.Thread(CreateShutDownThread());
                    shutDownThread.SetName("AsyncDispatcher ShutDown handler");
                    shutDownThread.Start();
                }
            }
        }
コード例 #13
0
ファイル: NodeManager.cs プロジェクト: orf53975/hadoop.net
 private void InitAndStartNodeManager(Configuration conf, bool hasToReboot)
 {
     try
     {
         // Remove the old hook if we are rebooting.
         if (hasToReboot && null != nodeManagerShutdownHook)
         {
             ShutdownHookManager.Get().RemoveShutdownHook(nodeManagerShutdownHook);
         }
         nodeManagerShutdownHook = new CompositeService.CompositeServiceShutdownHook(this);
         ShutdownHookManager.Get().AddShutdownHook(nodeManagerShutdownHook, ShutdownHookPriority
                                                   );
         // System exit should be called only when NodeManager is instantiated from
         // main() funtion
         this.shouldExitOnShutdownEvent = true;
         this.Init(conf);
         this.Start();
     }
     catch (Exception t)
     {
         Log.Fatal("Error starting NodeManager", t);
         System.Environment.Exit(-1);
     }
 }
コード例 #14
0
        internal static JobHistoryServer LaunchJobHistoryServer(string[] args)
        {
            Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
                                                                  ());
            StringUtils.StartupShutdownMessage(typeof(JobHistoryServer), args, Log);
            JobHistoryServer jobHistoryServer = null;

            try
            {
                jobHistoryServer = new JobHistoryServer();
                ShutdownHookManager.Get().AddShutdownHook(new CompositeService.CompositeServiceShutdownHook
                                                              (jobHistoryServer), ShutdownHookPriority);
                YarnConfiguration conf = new YarnConfiguration(new JobConf());
                new GenericOptionsParser(conf, args);
                jobHistoryServer.Init(conf);
                jobHistoryServer.Start();
            }
            catch (Exception t)
            {
                Log.Fatal("Error starting JobHistoryServer", t);
                ExitUtil.Terminate(-1, "Error starting JobHistoryServer");
            }
            return(jobHistoryServer);
        }
コード例 #15
0
            /// <exception cref="Sharpen.RuntimeException"/>
            /// <exception cref="System.IO.IOException"/>
            private void RunSubtask(Task task, TaskType taskType, TaskAttemptId attemptID, int
                                    numMapTasks, bool renameOutputs, IDictionary <TaskAttemptID, MapOutputFile> localMapFiles
                                    )
            {
                TaskAttemptID classicAttemptID = TypeConverter.FromYarn(attemptID);

                try
                {
                    JobConf conf = new JobConf(this._enclosing.GetConfig());
                    conf.Set(JobContext.TaskId, task.GetTaskID().ToString());
                    conf.Set(JobContext.TaskAttemptId, classicAttemptID.ToString());
                    conf.SetBoolean(JobContext.TaskIsmap, (taskType == TaskType.Map));
                    conf.SetInt(JobContext.TaskPartition, task.GetPartition());
                    conf.Set(JobContext.Id, task.GetJobID().ToString());
                    // Use the AM's local dir env to generate the intermediate step
                    // output files
                    string[] localSysDirs = StringUtils.GetTrimmedStrings(Runtime.Getenv(ApplicationConstants.Environment
                                                                                         .LocalDirs.ToString()));
                    conf.SetStrings(MRConfig.LocalDir, localSysDirs);
                    LocalContainerLauncher.Log.Info(MRConfig.LocalDir + " for uber task: " + conf.Get
                                                        (MRConfig.LocalDir));
                    // mark this as an uberized subtask so it can set task counter
                    // (longer-term/FIXME:  could redefine as job counter and send
                    // "JobCounterEvent" to JobImpl on [successful] completion of subtask;
                    // will need new Job state-machine transition and JobImpl jobCounters
                    // map to handle)
                    conf.SetBoolean("mapreduce.task.uberized", true);
                    // Check and handle Encrypted spill key
                    task.SetEncryptedSpillKey(this._enclosing.encryptedSpillKey);
                    YarnChild.SetEncryptedSpillKeyIfRequired(task);
                    // META-FIXME: do we want the extra sanity-checking (doneWithMaps,
                    // etc.), or just assume/hope the state machine(s) and uber-AM work
                    // as expected?
                    if (taskType == TaskType.Map)
                    {
                        if (this.doneWithMaps)
                        {
                            LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a map task ("
                                                             + attemptID + "), but should be finished with maps");
                            throw new RuntimeException();
                        }
                        MapTask map = (MapTask)task;
                        map.SetConf(conf);
                        map.Run(conf, this._enclosing.umbilical);
                        if (renameOutputs)
                        {
                            MapOutputFile renamed = LocalContainerLauncher.RenameMapOutputForReduce(conf, attemptID
                                                                                                    , map.GetMapOutputFile());
                            localMapFiles[classicAttemptID] = renamed;
                        }
                        this.Relocalize();
                        if (++this.finishedSubMaps == numMapTasks)
                        {
                            this.doneWithMaps = true;
                        }
                    }
                    else
                    {
                        /* TaskType.REDUCE */
                        if (!this.doneWithMaps)
                        {
                            // check if event-queue empty?  whole idea of counting maps vs.
                            // checking event queue is a tad wacky...but could enforce ordering
                            // (assuming no "lost events") at LocalMRAppMaster [CURRENT BUG(?):
                            // doesn't send reduce event until maps all done]
                            LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a reduce task ("
                                                             + attemptID + "), but not yet finished with maps");
                            throw new RuntimeException();
                        }
                        // a.k.a. "mapreduce.jobtracker.address" in LocalJobRunner:
                        // set framework name to local to make task local
                        conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName);
                        conf.Set(MRConfig.MasterAddress, "local");
                        // bypass shuffle
                        ReduceTask reduce = (ReduceTask)task;
                        reduce.SetLocalMapFiles(localMapFiles);
                        reduce.SetConf(conf);
                        reduce.Run(conf, this._enclosing.umbilical);
                        this.Relocalize();
                    }
                }
                catch (FSError e)
                {
                    LocalContainerLauncher.Log.Fatal("FSError from child", e);
                    // umbilical:  MRAppMaster creates (taskAttemptListener), passes to us
                    if (!ShutdownHookManager.Get().IsShutdownInProgress())
                    {
                        this._enclosing.umbilical.FsError(classicAttemptID, e.Message);
                    }
                    throw new RuntimeException();
                }
                catch (Exception exception)
                {
                    LocalContainerLauncher.Log.Warn("Exception running local (uberized) 'child' : " +
                                                    StringUtils.StringifyException(exception));
                    try
                    {
                        if (task != null)
                        {
                            // do cleanup for the task
                            task.TaskCleanup(this._enclosing.umbilical);
                        }
                    }
                    catch (Exception e)
                    {
                        LocalContainerLauncher.Log.Info("Exception cleaning up: " + StringUtils.StringifyException
                                                            (e));
                    }
                    // Report back any failures, for diagnostic purposes
                    this._enclosing.umbilical.ReportDiagnosticInfo(classicAttemptID, StringUtils.StringifyException
                                                                       (exception));
                    throw new RuntimeException();
                }
                catch (Exception throwable)
                {
                    LocalContainerLauncher.Log.Fatal("Error running local (uberized) 'child' : " + StringUtils
                                                     .StringifyException(throwable));
                    if (!ShutdownHookManager.Get().IsShutdownInProgress())
                    {
                        Exception tCause = throwable.InnerException;
                        string    cause  = (tCause == null) ? throwable.Message : StringUtils.StringifyException
                                               (tCause);
                        this._enclosing.umbilical.FatalError(classicAttemptID, cause);
                    }
                    throw new RuntimeException();
                }
            }
コード例 #16
0
        /// <exception cref="System.Exception"/>
        public static void Main(string[] args)
        {
            Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler
                                                                  ());
            Log.Debug("Child starting");
            JobConf job = new JobConf(MRJobConfig.JobConfFile);

            // Initing with our JobConf allows us to avoid loading confs twice
            Limits.Init(job);
            UserGroupInformation.SetConfiguration(job);
            string        host        = args[0];
            int           port        = System.Convert.ToInt32(args[1]);
            IPEndPoint    address     = NetUtils.CreateSocketAddrForHost(host, port);
            TaskAttemptID firstTaskid = ((TaskAttemptID)TaskAttemptID.ForName(args[2]));
            long          jvmIdLong   = long.Parse(args[3]);
            JVMId         jvmId       = new JVMId(((JobID)firstTaskid.GetJobID()), firstTaskid.GetTaskType(
                                                      ) == TaskType.Map, jvmIdLong);

            // initialize metrics
            DefaultMetricsSystem.Initialize(StringUtils.Camelize(firstTaskid.GetTaskType().ToString
                                                                     ()) + "Task");
            // Security framework already loaded the tokens into current ugi
            Credentials credentials = UserGroupInformation.GetCurrentUser().GetCredentials();

            Log.Info("Executing with tokens:");
            foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in credentials.GetAllTokens
                         ())
            {
                Log.Info(token);
            }
            // Create TaskUmbilicalProtocol as actual task owner.
            UserGroupInformation taskOwner = UserGroupInformation.CreateRemoteUser(((JobID)firstTaskid
                                                                                    .GetJobID()).ToString());

            Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt = TokenCache.GetJobToken
                                                                                 (credentials);
            SecurityUtil.SetTokenService(jt, address);
            taskOwner.AddToken(jt);
            TaskUmbilicalProtocol umbilical = taskOwner.DoAs(new _PrivilegedExceptionAction_108
                                                                 (address, job));
            // report non-pid to application master
            JvmContext context = new JvmContext(jvmId, "-1000");

            Log.Debug("PID: " + Sharpen.Runtime.GetEnv()["JVM_PID"]);
            Task task = null;
            UserGroupInformation     childUGI  = null;
            ScheduledExecutorService logSyncer = null;

            try
            {
                int     idleLoopCount = 0;
                JvmTask myTask        = null;
                // poll for new task
                for (int idle = 0; null == myTask; ++idle)
                {
                    long sleepTimeMilliSecs = Math.Min(idle * 500, 1500);
                    Log.Info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now."
                             );
                    TimeUnit.Milliseconds.Sleep(sleepTimeMilliSecs);
                    myTask = umbilical.GetTask(context);
                }
                if (myTask.ShouldDie())
                {
                    return;
                }
                task             = myTask.GetTask();
                YarnChild.taskid = task.GetTaskID();
                // Create the job-conf and set credentials
                ConfigureTask(job, task, credentials, jt);
                // Initiate Java VM metrics
                JvmMetrics.InitSingleton(jvmId.ToString(), job.GetSessionId());
                childUGI = UserGroupInformation.CreateRemoteUser(Runtime.Getenv(ApplicationConstants.Environment
                                                                                .User.ToString()));
                // Add tokens to new user so that it may execute its task correctly.
                childUGI.AddCredentials(credentials);
                // set job classloader if configured before invoking the task
                MRApps.SetJobClassLoader(job);
                logSyncer = TaskLog.CreateLogSyncer();
                // Create a final reference to the task for the doAs block
                Task taskFinal = task;
                childUGI.DoAs(new _PrivilegedExceptionAction_158(taskFinal, job, umbilical));
            }
            catch (FSError e)
            {
                // use job-specified working directory
                // run the task
                Log.Fatal("FSError from child", e);
                if (!ShutdownHookManager.Get().IsShutdownInProgress())
                {
                    umbilical.FsError(taskid, e.Message);
                }
            }
            catch (Exception exception)
            {
                Log.Warn("Exception running child : " + StringUtils.StringifyException(exception)
                         );
                try
                {
                    if (task != null)
                    {
                        // do cleanup for the task
                        if (childUGI == null)
                        {
                            // no need to job into doAs block
                            task.TaskCleanup(umbilical);
                        }
                        else
                        {
                            Task taskFinal = task;
                            childUGI.DoAs(new _PrivilegedExceptionAction_183(taskFinal, umbilical));
                        }
                    }
                }
                catch (Exception e)
                {
                    Log.Info("Exception cleaning up: " + StringUtils.StringifyException(e));
                }
                // Report back any failures, for diagnostic purposes
                if (taskid != null)
                {
                    if (!ShutdownHookManager.Get().IsShutdownInProgress())
                    {
                        umbilical.FatalError(taskid, StringUtils.StringifyException(exception));
                    }
                }
            }
            catch (Exception throwable)
            {
                Log.Fatal("Error running child : " + StringUtils.StringifyException(throwable));
                if (taskid != null)
                {
                    if (!ShutdownHookManager.Get().IsShutdownInProgress())
                    {
                        Exception tCause = throwable.InnerException;
                        string    cause  = tCause == null ? throwable.Message : StringUtils.StringifyException
                                               (tCause);
                        umbilical.FatalError(taskid, cause);
                    }
                }
            }
            finally
            {
                RPC.StopProxy(umbilical);
                DefaultMetricsSystem.Shutdown();
                TaskLog.SyncLogsShutdown(logSyncer);
            }
        }
コード例 #17
0
ファイル: BlockPoolSlice.cs プロジェクト: orf53975/hadoop.net
        /// <summary>Create a blook pool slice</summary>
        /// <param name="bpid">Block pool Id</param>
        /// <param name="volume">
        ///
        /// <see cref="FsVolumeImpl"/>
        /// to which this BlockPool belongs to
        /// </param>
        /// <param name="bpDir">directory corresponding to the BlockPool</param>
        /// <param name="conf">configuration</param>
        /// <exception cref="System.IO.IOException"/>
        internal BlockPoolSlice(string bpid, FsVolumeImpl volume, FilePath bpDir, Configuration
                                conf)
        {
            // volume to which this BlockPool belongs to
            // StorageDirectory/current/bpid/current
            // directory where finalized replicas are stored
            // directory store RBW replica
            // directory store Temporary replica
            // TODO:FEDERATION scalability issue - a thread per DU is needed
            this.bpid           = bpid;
            this.volume         = volume;
            this.currentDir     = new FilePath(bpDir, DataStorage.StorageDirCurrent);
            this.finalizedDir   = new FilePath(currentDir, DataStorage.StorageDirFinalized);
            this.lazypersistDir = new FilePath(currentDir, DataStorage.StorageDirLazyPersist);
            if (!this.finalizedDir.Exists())
            {
                if (!this.finalizedDir.Mkdirs())
                {
                    throw new IOException("Failed to mkdirs " + this.finalizedDir);
                }
            }
            this.deleteDuplicateReplicas = conf.GetBoolean(DFSConfigKeys.DfsDatanodeDuplicateReplicaDeletion
                                                           , DFSConfigKeys.DfsDatanodeDuplicateReplicaDeletionDefault);
            // Files that were being written when the datanode was last shutdown
            // are now moved back to the data directory. It is possible that
            // in the future, we might want to do some sort of datanode-local
            // recovery for these blocks. For example, crc validation.
            //
            this.tmpDir = new FilePath(bpDir, DataStorage.StorageDirTmp);
            if (tmpDir.Exists())
            {
                FileUtil.FullyDelete(tmpDir);
            }
            this.rbwDir = new FilePath(currentDir, DataStorage.StorageDirRbw);
            bool supportAppends = conf.GetBoolean(DFSConfigKeys.DfsSupportAppendKey, DFSConfigKeys
                                                  .DfsSupportAppendDefault);

            if (rbwDir.Exists() && !supportAppends)
            {
                FileUtil.FullyDelete(rbwDir);
            }
            if (!rbwDir.Mkdirs())
            {
                // create rbw directory if not exist
                if (!rbwDir.IsDirectory())
                {
                    throw new IOException("Mkdirs failed to create " + rbwDir.ToString());
                }
            }
            if (!tmpDir.Mkdirs())
            {
                if (!tmpDir.IsDirectory())
                {
                    throw new IOException("Mkdirs failed to create " + tmpDir.ToString());
                }
            }
            // Use cached value initially if available. Or the following call will
            // block until the initial du command completes.
            this.dfsUsage = new DU(bpDir, conf, LoadDfsUsed());
            this.dfsUsage.Start();
            // Make the dfs usage to be saved during shutdown.
            ShutdownHookManager.Get().AddShutdownHook(new _Runnable_145(this), ShutdownHookPriority
                                                      );
        }