public virtual void TestGetFullDirs() { Configuration conf = new YarnConfiguration(); conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077"); FileContext localFs = FileContext.GetLocalFSFileContext(conf); string localDir1 = new FilePath(testDir, "localDir1").GetPath(); string localDir2 = new FilePath(testDir, "localDir2").GetPath(); string logDir1 = new FilePath(testDir, "logDir1").GetPath(); string logDir2 = new FilePath(testDir, "logDir2").GetPath(); Path localDir1Path = new Path(localDir1); Path logDir1Path = new Path(logDir1); FsPermission dirPermissions = new FsPermission((short)0x108); localFs.Mkdir(localDir1Path, dirPermissions, true); localFs.Mkdir(logDir1Path, dirPermissions, true); conf.Set(YarnConfiguration.NmLocalDirs, localDir1 + "," + localDir2); conf.Set(YarnConfiguration.NmLogDirs, logDir1 + "," + logDir2); conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f); LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(); dirSvc.Init(conf); NUnit.Framework.Assert.AreEqual(0, dirSvc.GetLocalDirs().Count); NUnit.Framework.Assert.AreEqual(0, dirSvc.GetLogDirs().Count); NUnit.Framework.Assert.AreEqual(1, dirSvc.GetDiskFullLocalDirs().Count); NUnit.Framework.Assert.AreEqual(1, dirSvc.GetDiskFullLogDirs().Count); FileUtils.DeleteDirectory(new FilePath(localDir1)); FileUtils.DeleteDirectory(new FilePath(localDir2)); FileUtils.DeleteDirectory(new FilePath(logDir1)); FileUtils.DeleteDirectory(new FilePath(logDir1)); dirSvc.Close(); }
/// <exception cref="System.IO.IOException"/> internal virtual byte[] CreateTmpFile(Path dst, Random r, int len) { // use unmodified local context FileContext lfs = FileContext.GetLocalFSFileContext(); dst = lfs.MakeQualified(dst); lfs.Mkdir(dst.GetParent(), null, true); byte[] bytes = new byte[len]; FSDataOutputStream @out = null; try { @out = lfs.Create(dst, EnumSet.Of(CreateFlag.Create, CreateFlag.Overwrite)); r.NextBytes(bytes); @out.Write(bytes); } finally { if (@out != null) { @out.Close(); } } return(bytes); }
public virtual void TestFcResolveAfs() { Configuration conf = new Configuration(); FileContext fcLocal = FileContext.GetLocalFSFileContext(); FileContext fcHdfs = FileContext.GetFileContext(cluster.GetFileSystem().GetUri()); string localTestRoot = helper.GetAbsoluteTestRootDir(fcLocal); Path alphaLocalPath = new Path(fcLocal.GetDefaultFileSystem().GetUri().ToString() , new FilePath(localTestRoot, "alpha").GetAbsolutePath()); DFSTestUtil.CreateFile(FileSystem.GetLocal(conf), alphaLocalPath, 16, (short)1, 2 ); Path linkTarget = new Path(fcLocal.GetDefaultFileSystem().GetUri().ToString(), localTestRoot ); Path hdfsLink = new Path(fcHdfs.GetDefaultFileSystem().GetUri().ToString(), "/tmp/link" ); fcHdfs.CreateSymlink(linkTarget, hdfsLink, true); Path alphaHdfsPathViaLink = new Path(fcHdfs.GetDefaultFileSystem().GetUri().ToString () + "/tmp/link/alpha"); ICollection <AbstractFileSystem> afsList = fcHdfs.ResolveAbstractFileSystems(alphaHdfsPathViaLink ); NUnit.Framework.Assert.AreEqual(2, afsList.Count); foreach (AbstractFileSystem afs in afsList) { if ((!afs.Equals(fcHdfs.GetDefaultFileSystem())) && (!afs.Equals(fcLocal.GetDefaultFileSystem ()))) { NUnit.Framework.Assert.Fail("Failed to resolve AFS correctly"); } } }
public MRApp(ApplicationAttemptId appAttemptId, ContainerId amContainerId, int maps , int reduces, bool autoComplete, string testName, bool cleanOnStart, int startCount , Clock clock, bool unregistered, string assignedQueue) : base(appAttemptId, amContainerId, NmHost, NmPort, NmHttpPort, clock, Runtime.CurrentTimeMillis ()) { this.testWorkDir = new FilePath("target", testName); testAbsPath = new Path(testWorkDir.GetAbsolutePath()); Log.Info("PathUsed: " + testAbsPath); if (cleanOnStart) { testAbsPath = new Path(testWorkDir.GetAbsolutePath()); try { FileContext.GetLocalFSFileContext().Delete(testAbsPath, true); } catch (Exception e) { Log.Warn("COULD NOT CLEANUP: " + testAbsPath, e); throw new YarnRuntimeException("could not cleanup test dir", e); } } this.maps = maps; this.reduces = reduces; this.autoComplete = autoComplete; // If safeToReportTerminationToUser is set to true, we can verify whether // the job can reaches the final state when MRAppMaster shuts down. this.successfullyUnregistered.Set(unregistered); this.assignedQueue = assignedQueue; }
public virtual void TestContainerLocalizerClosesFilesystems() { // mocked generics // verify filesystems are closed when localizer doesn't fail FileContext fs = FileContext.GetLocalFSFileContext(); spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem()); ContainerLocalizer localizer = SetupContainerLocalizerForTest(); Org.Mockito.Mockito.DoNothing().When(localizer).LocalizeFiles(Matchers.Any <LocalizationProtocol >(), Matchers.Any <CompletionService>(), Matchers.Any <UserGroupInformation>()); Org.Mockito.Mockito.Verify(localizer, Org.Mockito.Mockito.Never()).CloseFileSystems (Matchers.Any <UserGroupInformation>()); localizer.RunLocalization(nmAddr); Org.Mockito.Mockito.Verify(localizer).CloseFileSystems(Matchers.Any <UserGroupInformation >()); spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem()); // verify filesystems are closed when localizer fails localizer = SetupContainerLocalizerForTest(); Org.Mockito.Mockito.DoThrow(new YarnRuntimeException("Forced Failure")).When(localizer ).LocalizeFiles(Matchers.Any <LocalizationProtocol>(), Matchers.Any <CompletionService >(), Matchers.Any <UserGroupInformation>()); Org.Mockito.Mockito.Verify(localizer, Org.Mockito.Mockito.Never()).CloseFileSystems (Matchers.Any <UserGroupInformation>()); localizer.RunLocalization(nmAddr); Org.Mockito.Mockito.Verify(localizer).CloseFileSystems(Matchers.Any <UserGroupInformation >()); }
/// <summary>delete the test directory in the target local fs</summary> /// <exception cref="System.Exception"/> public static void TearDownForViewFsLocalFs(FileContextTestHelper helper) { FileContext fclocal = FileContext.GetLocalFSFileContext(); Path targetOfTests = helper.GetTestRootPath(fclocal); fclocal.Delete(targetOfTests, true); }
/// <exception cref="System.Exception"/> public virtual void TestUniqueDestinationPath() { Configuration conf = new Configuration(); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); ExecutorService singleThreadedExec = Executors.NewSingleThreadExecutor(); LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName); Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf); destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); Path p = new Path(basedir, "dir" + 0 + ".jar"); LocalResourceVisibility vis = LocalResourceVisibility.Private; LocalResource rsrc = CreateJar(files, p, vis); FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf , destPath, rsrc); Future <Path> rPath = singleThreadedExec.Submit(fsd); singleThreadedExec.Shutdown(); while (!singleThreadedExec.AwaitTermination(1000, TimeUnit.Milliseconds)) { } NUnit.Framework.Assert.IsTrue(rPath.IsDone()); // Now FSDownload will not create a random directory to localize the // resource. Therefore the final localizedPath for the resource should be // destination directory (passed as an argument) + file name. NUnit.Framework.Assert.AreEqual(destPath, rPath.Get().GetParent()); }
/* * return the ViewFS File context to be used for tests */ /// <exception cref="System.Exception"/> public static FileContext SetupForViewFsLocalFs(FileContextTestHelper helper) { FileContext fsTarget = FileContext.GetLocalFSFileContext(); Path targetOfTests = helper.GetTestRootPath(fsTarget); // In case previous test was killed before cleanup fsTarget.Delete(targetOfTests, true); fsTarget.Mkdir(targetOfTests, FileContext.DefaultPerm, true); Configuration conf = new Configuration(); // Set up viewfs link for test dir as described above string testDir = helper.GetTestRootPath(fsTarget).ToUri().GetPath(); LinkUpFirstComponents(conf, testDir, fsTarget, "test dir"); // Set up viewfs link for home dir as described above SetUpHomeDir(conf, fsTarget); // the test path may be relative to working dir - we need to make that work: // Set up viewfs link for wd as described above string wdDir = fsTarget.GetWorkingDirectory().ToUri().GetPath(); LinkUpFirstComponents(conf, wdDir, fsTarget, "working dir"); FileContext fc = FileContext.GetFileContext(FsConstants.ViewfsUri, conf); fc.SetWorkingDirectory(new Path(wdDir)); // in case testdir relative to wd. Org.Mortbay.Log.Log.Info("Working dir is: " + fc.GetWorkingDirectory()); //System.out.println("SRCOfTests = "+ getTestRootPath(fc, "test")); //System.out.println("TargetOfTests = "+ targetOfTests.toUri()); return(fc); }
public virtual void TestRenameMapOutputForReduce() { JobConf conf = new JobConf(); MROutputFiles mrOutputFiles = new MROutputFiles(); mrOutputFiles.SetConf(conf); // make sure both dirs are distinct // conf.Set(MRConfig.LocalDir, localDirs[0].ToString()); Path mapOut = mrOutputFiles.GetOutputFileForWrite(1); conf.Set(MRConfig.LocalDir, localDirs[1].ToString()); Path mapOutIdx = mrOutputFiles.GetOutputIndexFileForWrite(1); Assert.AssertNotEquals("Paths must be different!", mapOut.GetParent(), mapOutIdx. GetParent()); // make both dirs part of LOCAL_DIR conf.SetStrings(MRConfig.LocalDir, localDirs); FileContext lfc = FileContext.GetLocalFSFileContext(conf); lfc.Create(mapOut, EnumSet.Of(CreateFlag.Create)).Close(); lfc.Create(mapOutIdx, EnumSet.Of(CreateFlag.Create)).Close(); JobId jobId = MRBuilderUtils.NewJobId(12345L, 1, 2); TaskId tid = MRBuilderUtils.NewTaskId(jobId, 0, TaskType.Map); TaskAttemptId taid = MRBuilderUtils.NewTaskAttemptId(tid, 0); LocalContainerLauncher.RenameMapOutputForReduce(conf, taid, mrOutputFiles); }
public virtual void TestFailedDirLogDeletion() { FilePath[] localLogDirs = GetLocalLogDirFiles(this.GetType().FullName, 7); IList <string> localLogDirPaths = new AList <string>(localLogDirs.Length); for (int i = 0; i < localLogDirs.Length; i++) { localLogDirPaths.AddItem(localLogDirs[i].GetAbsolutePath()); } string localLogDirsString = StringUtils.Join(localLogDirPaths, ","); conf.Set(YarnConfiguration.NmLogDirs, localLogDirsString); conf.SetBoolean(YarnConfiguration.LogAggregationEnabled, false); conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 0l); LocalDirsHandlerService mockDirsHandler = Org.Mockito.Mockito.Mock <LocalDirsHandlerService >(); NonAggregatingLogHandler rawLogHandler = new NonAggregatingLogHandler(dispatcher, mockDelService, mockDirsHandler, new NMNullStateStoreService()); NonAggregatingLogHandler logHandler = Org.Mockito.Mockito.Spy(rawLogHandler); AbstractFileSystem spylfs = Org.Mockito.Mockito.Spy(FileContext.GetLocalFSFileContext ().GetDefaultFileSystem()); FileContext lfs = FileContext.GetFileContext(spylfs, conf); Org.Mockito.Mockito.DoReturn(lfs).When(logHandler).GetLocalFileContext(Matchers.IsA <Configuration>()); logHandler.Init(conf); logHandler.Start(); RunMockedFailedDirs(logHandler, appId, user, mockDelService, mockDirsHandler, conf , spylfs, lfs, localLogDirs); logHandler.Close(); }
/// <summary> /// Tests that an edits log created using CreateEditsLog is valid and can be /// loaded successfully by a namenode. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestCanLoadCreatedEditsLog() { // Format namenode. HdfsConfiguration conf = new HdfsConfiguration(); FilePath nameDir = new FilePath(HdfsDir, "name"); conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, Util.FileAsURI(nameDir).ToString()); DFSTestUtil.FormatNameNode(conf); // Call CreateEditsLog and move the resulting edits to the name dir. CreateEditsLog.Main(new string[] { "-f", "1000", "0", "1", "-d", TestDir.GetAbsolutePath () }); Path editsWildcard = new Path(TestDir.GetAbsolutePath(), "*"); FileContext localFc = FileContext.GetLocalFSFileContext(); foreach (FileStatus edits in localFc.Util().GlobStatus(editsWildcard)) { Path src = edits.GetPath(); Path dst = new Path(new FilePath(nameDir, "current").GetAbsolutePath(), src.GetName ()); localFc.Rename(src, dst); } // Start a namenode to try to load the edits. cluster = new MiniDFSCluster.Builder(conf).Format(false).ManageNameDfsDirs(false) .WaitSafeMode(false).Build(); cluster.WaitClusterUp(); }
public virtual void SetupForTests() { conf = new Configuration(); localFs = FileContext.GetLocalFSFileContext(conf); testDir.Mkdirs(); testFile.CreateNewFile(); }
/// <exception cref="System.Exception"/> protected override void ServiceStop() { lock (this) { if (this._enclosing.resourceManagers[this.index] != null) { this.WaitForAppMastersToFinish(5000); this._enclosing.resourceManagers[this.index].Stop(); } if (Shell.Windows) { // On Windows, clean up the short temporary symlink that was created to // work around path length limitation. string testWorkDirPath = this._enclosing.testWorkDir.GetAbsolutePath(); try { FileContext.GetLocalFSFileContext().Delete(new Path(testWorkDirPath), true); } catch (IOException) { MiniYARNCluster.Log.Warn("could not cleanup symlink: " + this._enclosing.testWorkDir .GetAbsolutePath()); } } base.ServiceStop(); } }
public virtual void TearDown() { if (testRootDir.Exists()) { FileContext.GetLocalFSFileContext().Delete(new Path(testRootDir.GetAbsolutePath() ), true); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> /// <exception cref="System.Exception"/> /// <exception cref="Sharpen.ExecutionException"/> public virtual void TestDownloadPublicWithStatCache() { Configuration conf = new Configuration(); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); // if test directory doesn't have ancestor permission, skip this test FileSystem f = basedir.GetFileSystem(conf); Assume.AssumeTrue(FSDownload.AncestorsHaveExecutePermissions(f, basedir, null)); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); int size = 512; ConcurrentMap <Path, AtomicInteger> counts = new ConcurrentHashMap <Path, AtomicInteger >(); CacheLoader <Path, Future <FileStatus> > loader = FSDownload.CreateStatusCacheLoader (conf); LoadingCache <Path, Future <FileStatus> > statCache = CacheBuilder.NewBuilder().Build (new _CacheLoader_328(counts, loader)); // increment the count // use the default loader // test FSDownload.isPublic() concurrently int fileCount = 3; IList <Callable <bool> > tasks = new AList <Callable <bool> >(); for (int i = 0; i < fileCount; i++) { Random rand = new Random(); long sharedSeed = rand.NextLong(); rand.SetSeed(sharedSeed); System.Console.Out.WriteLine("SEED: " + sharedSeed); Path path = new Path(basedir, "test-file-" + i); CreateFile(files, path, size, rand); FileSystem fs = path.GetFileSystem(conf); FileStatus sStat = fs.GetFileStatus(path); tasks.AddItem(new _Callable_358(fs, path, sStat, statCache)); } ExecutorService exec = Executors.NewFixedThreadPool(fileCount); try { IList <Future <bool> > futures = exec.InvokeAll(tasks); // files should be public foreach (Future <bool> future in futures) { NUnit.Framework.Assert.IsTrue(future.Get()); } // for each path exactly one file status call should be made foreach (AtomicInteger count in counts.Values) { NUnit.Framework.Assert.AreSame(count.Get(), 1); } } finally { exec.Shutdown(); } }
public virtual void Setup() { localFS = FileContext.GetLocalFSFileContext(); tmpDir.Mkdirs(); logsDir.Mkdirs(); remoteLogsDir.Mkdirs(); nmLocalDir.Mkdirs(); syncBarrier = new CyclicBarrier(2); }
public virtual void Setup() { localFS = FileContext.GetLocalFSFileContext(); tmpDir.Mkdirs(); logsDir.Mkdirs(); remoteLogsDir.Mkdirs(); nmLocalDir.Mkdirs(); // Construct the Container-id cId = CreateContainerId(); }
public static void Setup() { //Do not error out if metrics are inited multiple times DefaultMetricsSystem.SetMiniClusterMode(true); FilePath dir = new FilePath(stagingDir); stagingDir = dir.GetAbsolutePath(); localFS = FileContext.GetLocalFSFileContext(); localFS.Delete(new Path(testDir.GetAbsolutePath()), true); testDir.Mkdir(); }
public DefaultContainerExecutor() { try { this.lfs = FileContext.GetLocalFSFileContext(); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> /// <exception cref="System.Exception"/> public virtual void TestDownloadBadPublic() { Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077"); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource , LocalResourceVisibility>(); Random rand = new Random(); long sharedSeed = rand.NextLong(); rand.SetSeed(sharedSeed); System.Console.Out.WriteLine("SEED: " + sharedSeed); IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource, Future <Path> >(); ExecutorService exec = Executors.NewSingleThreadExecutor(); LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName); int size = 512; LocalResourceVisibility vis = LocalResourceVisibility.Public; Path path = new Path(basedir, "test-file"); LocalResource rsrc = CreateFile(files, path, size, rand, vis); rsrcVis[rsrc] = vis; Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), size, conf); destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf , destPath, rsrc); pending[rsrc] = exec.Submit(fsd); exec.Shutdown(); while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds)) { } NUnit.Framework.Assert.IsTrue(pending[rsrc].IsDone()); try { foreach (KeyValuePair <LocalResource, Future <Path> > p in pending) { p.Value.Get(); NUnit.Framework.Assert.Fail("We localized a file that is not public."); } } catch (ExecutionException e) { NUnit.Framework.Assert.IsTrue(e.InnerException is IOException); } }
internal virtual FileContext GetLocalFileContext(Configuration conf) { try { return(FileContext.GetLocalFSFileContext(conf)); } catch (IOException) { throw new YarnRuntimeException("Failed to access local fs"); } }
internal static FileContext GetLfs() { try { return(FileContext.GetLocalFSFileContext()); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } }
public override void SetUp() { // create the test root on local_fs fcTarget = FileContext.GetLocalFSFileContext(); base.SetUp(); // this sets up conf (and fcView which we replace) // Now create a viewfs using a mount table called "default" // hence viewfs://default/ schemeWithAuthority = new URI(FsConstants.ViewfsScheme, "default", "/", null, null ); fcView = FileContext.GetFileContext(schemeWithAuthority, conf); }
public virtual void Setup() { fsPath = new FilePath("target", GetType().Name + "-tmpDir").GetAbsoluteFile(); fsContext = FileContext.GetLocalFSFileContext(); fsContext.Delete(new Path(fsPath.GetAbsolutePath()), true); conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.TimelineServiceRecoveryEnabled, true); conf.SetClass(YarnConfiguration.TimelineServiceStateStoreClass, typeof(LeveldbTimelineStateStore ), typeof(TimelineStateStore)); conf.Set(YarnConfiguration.TimelineServiceLeveldbStateStorePath, fsPath.GetAbsolutePath ()); }
public static void DeleteTmpFiles() { FileContext lfs = FileContext.GetLocalFSFileContext(); try { lfs.Delete(BaseTmpPath, true); } catch (FileNotFoundException) { } }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { try { //TODO Is this required? FileContext.GetLocalFSFileContext(conf); } catch (UnsupportedFileSystemException e) { throw new YarnRuntimeException("Failed to start ContainersLauncher", e); } base.ServiceInit(conf); }
public virtual void TestLocalizerTokenIsGettingRemoved() { FileContext fs = FileContext.GetLocalFSFileContext(); spylfs = Org.Mockito.Mockito.Spy(fs.GetDefaultFileSystem()); ContainerLocalizer localizer = SetupContainerLocalizerForTest(); Org.Mockito.Mockito.DoNothing().When(localizer).LocalizeFiles(Matchers.Any <LocalizationProtocol >(), Matchers.Any <CompletionService>(), Matchers.Any <UserGroupInformation>()); localizer.RunLocalization(nmAddr); Org.Mockito.Mockito.Verify(spylfs, Org.Mockito.Mockito.Times(1)).Delete(tokenPath , false); }
public DockerContainerExecutor() { // This validates that the image is a proper docker image and would not crash docker. try { this.lfs = FileContext.GetLocalFSFileContext(); this.dockerImagePattern = Sharpen.Pattern.Compile(DockerImagePattern); } catch (UnsupportedFileSystemException e) { throw new RuntimeException(e); } }
/// <exception cref="System.Exception"/> protected internal virtual void SetupInternal(int numNodeManager) { Log.Info("Starting up YARN cluster"); conf = new YarnConfiguration(); conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128); conf.Set("yarn.log.dir", "target"); conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true); conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName); conf.SetBoolean(YarnConfiguration.NodeLabelsEnabled, true); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(typeof(TestDistributedShell).Name, 1, numNodeManager , 1, 1); yarnCluster.Init(conf); yarnCluster.Start(); conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname () + ":" + yarnCluster.GetApplicationHistoryServer().GetPort()); WaitForNMsToRegister(); Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml" ); if (url == null) { throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath" ); } Configuration yarnClusterConfig = yarnCluster.GetConfig(); yarnClusterConfig.Set("yarn.application.classpath", new FilePath(url.AbsolutePath ).GetParent()); //write the document to a buffer (not directly to the file, as that //can cause the file being written to get read -which will then fail. ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); yarnClusterConfig.WriteXml(bytesOut); bytesOut.Close(); //write the bytes to the file in the classpath OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath)); os.Write(bytesOut.ToByteArray()); os.Close(); } FileContext fsContext = FileContext.GetLocalFSFileContext(); fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path" )), true); try { Sharpen.Thread.Sleep(2000); } catch (Exception e) { Log.Info("setup thread sleep interrupted. message=" + e.Message); } }
public virtual void SetUp() { // create the test root on local_fs fcTarget = FileContext.GetLocalFSFileContext(); chrootedTo = fileContextTestHelper.GetAbsoluteTestRootPath(fcTarget); // In case previous test was killed before cleanup fcTarget.Delete(chrootedTo, true); fcTarget.Mkdir(chrootedTo, FileContext.DefaultPerm, true); Configuration conf = new Configuration(); // ChRoot to the root of the testDirectory fc = FileContext.GetFileContext(new ChRootedFs(fcTarget.GetDefaultFileSystem(), chrootedTo ), conf); }