// Time interval for the move thread. //History job cleaner interval /// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { Log.Info("JobHistory Init"); this.conf = conf; this.appID = ApplicationId.NewInstance(0, 0); this.appAttemptID = RecordFactoryProvider.GetRecordFactory(conf).NewRecordInstance <ApplicationAttemptId>(); moveThreadInterval = conf.GetLong(JHAdminConfig.MrHistoryMoveIntervalMs, JHAdminConfig .DefaultMrHistoryMoveIntervalMs); hsManager = CreateHistoryFileManager(); hsManager.Init(conf); try { hsManager.InitExisting(); } catch (IOException e) { throw new YarnRuntimeException("Failed to intialize existing directories", e); } storage = CreateHistoryStorage(); if (storage is Org.Apache.Hadoop.Service.Service) { ((Org.Apache.Hadoop.Service.Service)storage).Init(conf); } storage.SetHistoryFileManager(hsManager); base.ServiceInit(conf); }
/// <summary>test clean old history files.</summary> /// <remarks> /// test clean old history files. Files should be deleted after 1 week by /// default. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestDeleteFileInfo() { Log.Info("STARTING testDeleteFileInfo"); try { Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeysPublic.NetTopologyNodeSwitchMappingImplKey, typeof(TestJobHistoryParsing.MyResolver), typeof(DNSToSwitchMapping)); RackResolver.Init(conf); MRApp app = new TestJobHistoryEvents.MRAppWithHistory(1, 1, true, this.GetType(). FullName, true); app.Submit(conf); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.GetContext().GetAllJobs().Values .GetEnumerator().Next(); JobId jobId = job.GetID(); app.WaitForState(job, JobState.Succeeded); // make sure all events are flushed app.WaitForState(Service.STATE.Stopped); HistoryFileManager hfm = new HistoryFileManager(); hfm.Init(conf); HistoryFileManager.HistoryFileInfo fileInfo = hfm.GetFileInfo(jobId); hfm.InitExisting(); // wait for move files form the done_intermediate directory to the gone // directory while (fileInfo.IsMovePending()) { Sharpen.Thread.Sleep(300); } NUnit.Framework.Assert.IsNotNull(hfm.jobListCache.Values()); // try to remove fileInfo hfm.Clean(); // check that fileInfo does not deleted NUnit.Framework.Assert.IsFalse(fileInfo.IsDeleted()); // correct live time hfm.SetMaxHistoryAge(-1); hfm.Clean(); hfm.Stop(); NUnit.Framework.Assert.IsTrue("Thread pool shutdown", hfm.moveToDoneExecutor.IsTerminated ()); // should be deleted ! NUnit.Framework.Assert.IsTrue("file should be deleted ", fileInfo.IsDeleted()); } finally { Log.Info("FINISHED testDeleteFileInfo"); } }