Beispiel #1
0
 public virtual void TestCreateDelete()
 {
     // Create file
     fileContextTestHelper.CreateFileNonRecursive(fc, "/foo");
     Assert.True(FileContextTestHelper.IsFile(fc, new Path("/foo")));
     Assert.True(FileContextTestHelper.IsFile(fcTarget, new Path(chrootedTo
                                                                 , "foo")));
     // Create file with recursive dir
     fileContextTestHelper.CreateFile(fc, "/newDir/foo");
     Assert.True(FileContextTestHelper.IsFile(fc, new Path("/newDir/foo"
                                                           )));
     Assert.True(FileContextTestHelper.IsFile(fcTarget, new Path(chrootedTo
                                                                 , "newDir/foo")));
     // Delete the created file
     Assert.True(fc.Delete(new Path("/newDir/foo"), false));
     NUnit.Framework.Assert.IsFalse(FileContextTestHelper.Exists(fc, new Path("/newDir/foo"
                                                                              )));
     NUnit.Framework.Assert.IsFalse(FileContextTestHelper.Exists(fcTarget, new Path(chrootedTo
                                                                                    , "newDir/foo")));
     // Create file with a 2 component dirs recursively
     fileContextTestHelper.CreateFile(fc, "/newDir/newDir2/foo");
     Assert.True(FileContextTestHelper.IsFile(fc, new Path("/newDir/newDir2/foo"
                                                           )));
     Assert.True(FileContextTestHelper.IsFile(fcTarget, new Path(chrootedTo
                                                                 , "newDir/newDir2/foo")));
     // Delete the created file
     Assert.True(fc.Delete(new Path("/newDir/newDir2/foo"), false));
     NUnit.Framework.Assert.IsFalse(FileContextTestHelper.Exists(fc, new Path("/newDir/newDir2/foo"
                                                                              )));
     NUnit.Framework.Assert.IsFalse(FileContextTestHelper.Exists(fcTarget, new Path(chrootedTo
                                                                                    , "newDir/newDir2/foo")));
 }
 /// <exception cref="System.IO.IOException"/>
 /// <exception cref="System.Exception"/>
 public override void DeleteAsUser(string user, Path subDir, params Path[] baseDirs
                                   )
 {
     if (baseDirs == null || baseDirs.Length == 0)
     {
         Log.Info("Deleting absolute path : " + subDir);
         if (!lfs.Delete(subDir, true))
         {
             //Maybe retry
             Log.Warn("delete returned false for path: [" + subDir + "]");
         }
         return;
     }
     foreach (Path baseDir in baseDirs)
     {
         Path del = subDir == null ? baseDir : new Path(baseDir, subDir);
         Log.Info("Deleting path : " + del);
         try
         {
             if (!lfs.Delete(del, true))
             {
                 Log.Warn("delete returned false for path: [" + del + "]");
             }
         }
         catch (FileNotFoundException)
         {
             continue;
         }
     }
 }
Beispiel #3
0
        public virtual void Setup()
        {
            localFS.Delete(new Path(localDir.GetAbsolutePath()), true);
            localFS.Delete(new Path(tmpDir.GetAbsolutePath()), true);
            localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true);
            localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true);
            localDir.Mkdir();
            tmpDir.Mkdir();
            localLogDir.Mkdir();
            remoteLogDir.Mkdir();
            Log.Info("Created localDir in " + localDir.GetAbsolutePath());
            Log.Info("Created tmpDir in " + tmpDir.GetAbsolutePath());
            string bindAddress = "0.0.0.0:12345";

            conf.Set(YarnConfiguration.NmAddress, bindAddress);
            conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath());
            conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath());
            conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath());
            conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 1);
            // Default delSrvc
            delSrvc = CreateDeletionService();
            delSrvc.Init(conf);
            exec = CreateContainerExecutor();
            nodeHealthChecker = new NodeHealthCheckerService();
            nodeHealthChecker.Init(conf);
            dirsHandler      = nodeHealthChecker.GetDiskHandler();
            containerManager = CreateContainerManager(delSrvc);
            ((NodeManager.NMContext)context).SetContainerManager(containerManager);
            nodeStatusUpdater.Init(conf);
            containerManager.Init(conf);
            nodeStatusUpdater.Start();
        }
Beispiel #4
0
 // Also try viewfs://default/    - note authority is name of mount table
 /// <exception cref="System.IO.IOException"/>
 internal virtual void InitializeTargetTestRoot()
 {
     targetTestRoot = fileContextTestHelper.GetAbsoluteTestRootPath(fcTarget);
     // In case previous test was killed before cleanup
     fcTarget.Delete(targetTestRoot, true);
     fcTarget.Mkdir(targetTestRoot, FileContext.DefaultPerm, true);
 }
        public HttpResponseMessage DeleteFile(Guid fileId)
        {
            try
            {
                HttpResponseMessage response = new HttpResponseMessage();

                CFile file = _fileContext.GetByFileId(fileId);

                // if file exist - delete it, if not continue to delete DB record cuz user wanted to delete real file anyway
                if (File.Exists(file.Path + file.Name))
                {
                    File.Delete(file.Path + file.Name);
                }

                // delete record from DB
                if (_fileContext.Delete(fileId) > 0)
                {
                    Request.CreateResponse(HttpStatusCode.OK, $"File {file.Name} has been deleted!");
                    return(response);
                }

                Request.CreateResponse(HttpStatusCode.BadRequest, $"Error: File {file.Name} has not been deleted!");
                return(response);
            }
            catch (ContextException e)
            {
                HttpContext.Current.Response.StatusCode = (Int32)HttpStatusCode.BadRequest;
                throw new ContextException(e.Message, e);
            }
        }
Beispiel #6
0
        /// <summary>delete the test directory in the target local fs</summary>
        /// <exception cref="System.Exception"/>
        public static void TearDownForViewFsLocalFs(FileContextTestHelper helper)
        {
            FileContext fclocal       = FileContext.GetLocalFSFileContext();
            Path        targetOfTests = helper.GetTestRootPath(fclocal);

            fclocal.Delete(targetOfTests, true);
        }
Beispiel #7
0
        /*
         * return the ViewFS File context to be used for tests
         */
        /// <exception cref="System.Exception"/>
        public static FileContext SetupForViewFsLocalFs(FileContextTestHelper helper)
        {
            FileContext fsTarget      = FileContext.GetLocalFSFileContext();
            Path        targetOfTests = helper.GetTestRootPath(fsTarget);

            // In case previous test was killed before cleanup
            fsTarget.Delete(targetOfTests, true);
            fsTarget.Mkdir(targetOfTests, FileContext.DefaultPerm, true);
            Configuration conf = new Configuration();
            // Set up viewfs link for test dir as described above
            string testDir = helper.GetTestRootPath(fsTarget).ToUri().GetPath();

            LinkUpFirstComponents(conf, testDir, fsTarget, "test dir");
            // Set up viewfs link for home dir as described above
            SetUpHomeDir(conf, fsTarget);
            // the test path may be relative to working dir - we need to make that work:
            // Set up viewfs link for wd as described above
            string wdDir = fsTarget.GetWorkingDirectory().ToUri().GetPath();

            LinkUpFirstComponents(conf, wdDir, fsTarget, "working dir");
            FileContext fc = FileContext.GetFileContext(FsConstants.ViewfsUri, conf);

            fc.SetWorkingDirectory(new Path(wdDir));
            // in case testdir relative to wd.
            Org.Mortbay.Log.Log.Info("Working dir is: " + fc.GetWorkingDirectory());
            //System.out.println("SRCOfTests = "+ getTestRootPath(fc, "test"));
            //System.out.println("TargetOfTests = "+ targetOfTests.toUri());
            return(fc);
        }
Beispiel #8
0
        public virtual void TestLinkTargetNonSymlink()
        {
            FileContext fc         = null;
            Path        notSymlink = new Path("/notasymlink");

            try
            {
                fc = FileContext.GetFileContext(cluster.GetFileSystem().GetUri());
                fc.Create(notSymlink, EnumSet.Of(CreateFlag.Create));
                DFSClient client = new DFSClient(cluster.GetFileSystem().GetUri(), cluster.GetConfiguration
                                                     (0));
                try
                {
                    client.GetLinkTarget(notSymlink.ToString());
                    NUnit.Framework.Assert.Fail("Expected exception for resolving non-symlink");
                }
                catch (IOException e)
                {
                    GenericTestUtils.AssertExceptionContains("is not a symbolic link", e);
                }
            }
            finally
            {
                if (fc != null)
                {
                    fc.Delete(notSymlink, false);
                }
            }
        }
Beispiel #9
0
            /// <exception cref="System.IO.IOException"/>
            public object Run()
            {
                FileContext myfc = FileContext.GetFileContext(TestPermissionSymlinks.conf);

                myfc.Delete(TestPermissionSymlinks.link, false);
                return(null);
            }
        public virtual void TestStatistics()
        {
            URI fsUri = GetFsUri();

            FileSystem.Statistics stats = FileContext.GetStatistics(fsUri);
            Assert.Equal(0, stats.GetBytesRead());
            Path filePath = fileContextTestHelper.GetTestRootPath(fc, "file1");

            FileContextTestHelper.CreateFile(fc, filePath, numBlocks, blockSize);
            Assert.Equal(0, stats.GetBytesRead());
            VerifyWrittenBytes(stats);
            FSDataInputStream fstr = fc.Open(filePath);

            byte[] buf       = new byte[blockSize];
            int    bytesRead = fstr.Read(buf, 0, blockSize);

            fstr.Read(0, buf, 0, blockSize);
            Assert.Equal(blockSize, bytesRead);
            VerifyReadBytes(stats);
            VerifyWrittenBytes(stats);
            VerifyReadBytes(FileContext.GetStatistics(GetFsUri()));
            IDictionary <URI, FileSystem.Statistics> statsMap = FileContext.GetAllStatistics();
            URI exactUri = GetSchemeAuthorityUri();

            VerifyWrittenBytes(statsMap[exactUri]);
            fc.Delete(filePath, true);
        }
        /// <summary>
        /// Override this so that we don't set the targetTestRoot to any path under the
        /// root of the FS, and so that we don't try to delete the test dir, but rather
        /// only its contents.
        /// </summary>
        /// <exception cref="System.IO.IOException"/>
        internal override void InitializeTargetTestRoot()
        {
            targetTestRoot = fc.MakeQualified(new Path("/"));
            RemoteIterator <FileStatus> dirContents = fc.ListStatus(targetTestRoot);

            while (dirContents.HasNext())
            {
                fc.Delete(dirContents.Next().GetPath(), true);
            }
        }
Beispiel #12
0
		public static void Setup()
		{
			//Do not error out if metrics are inited multiple times
			DefaultMetricsSystem.SetMiniClusterMode(true);
			FilePath dir = new FilePath(stagingDir);
			stagingDir = dir.GetAbsolutePath();
			localFS = FileContext.GetLocalFSFileContext();
			localFS.Delete(new Path(testDir.GetAbsolutePath()), true);
			testDir.Mkdir();
		}
Beispiel #13
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void Close()
        {
            foreach (FilePath symlink in symlinksCreated)
            {
                if (!symlink.Delete())
                {
                    Log.Warn("Failed to delete symlink created by the local job runner: " + symlink);
                }
            }
            FileContext localFSFileContext = FileContext.GetLocalFSFileContext();

            foreach (string archive in localArchives)
            {
                localFSFileContext.Delete(new Path(archive), true);
            }
            foreach (string file in localFiles)
            {
                localFSFileContext.Delete(new Path(file), true);
            }
        }
Beispiel #14
0
 public virtual void Teardown()
 {
     if (fc != null)
     {
         fc.Delete(GetTestRootPath(), true);
     }
     if (cluster != null)
     {
         cluster.Shutdown();
     }
 }
 public virtual void TearDown()
 {
     if (store != null)
     {
         store.Stop();
     }
     if (fsContext != null)
     {
         fsContext.Delete(new Path(fsPath.GetAbsolutePath()), true);
     }
 }
 public virtual void TearDown()
 {
     try
     {
         lfs.Delete(workDir, true);
     }
     catch (IOException e)
     {
         throw new RuntimeException(e);
     }
 }
Beispiel #17
0
 /// <exception cref="System.IO.IOException"/>
 private void ClearDir(FileContext fc, Path p)
 {
     try
     {
         fc.Delete(p, true);
     }
     catch (FileNotFoundException)
     {
     }
     // ignore
     fc.Mkdir(p, FsPermission.GetDirDefault(), false);
 }
Beispiel #18
0
        public static void DeleteTmpFiles()
        {
            FileContext lfs = FileContext.GetLocalFSFileContext();

            try
            {
                lfs.Delete(BaseTmpPath, true);
            }
            catch (FileNotFoundException)
            {
            }
        }
 public virtual void Setup()
 {
     fsPath    = new FilePath("target", GetType().Name + "-tmpDir").GetAbsoluteFile();
     fsContext = FileContext.GetLocalFSFileContext();
     fsContext.Delete(new Path(fsPath.GetAbsolutePath()), true);
     conf = new YarnConfiguration();
     conf.SetBoolean(YarnConfiguration.TimelineServiceRecoveryEnabled, true);
     conf.SetClass(YarnConfiguration.TimelineServiceStateStoreClass, typeof(LeveldbTimelineStateStore
                                                                            ), typeof(TimelineStateStore));
     conf.Set(YarnConfiguration.TimelineServiceLeveldbStateStorePath, fsPath.GetAbsolutePath
                  ());
 }
        /// <exception cref="System.Exception"/>
        protected internal virtual void SetupInternal(int numNodeManager)
        {
            Log.Info("Starting up YARN cluster");
            conf = new YarnConfiguration();
            conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128);
            conf.Set("yarn.log.dir", "target");
            conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
            conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName);
            conf.SetBoolean(YarnConfiguration.NodeLabelsEnabled, true);
            if (yarnCluster == null)
            {
                yarnCluster = new MiniYARNCluster(typeof(TestDistributedShell).Name, 1, numNodeManager
                                                  , 1, 1);
                yarnCluster.Init(conf);
                yarnCluster.Start();
                conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
                             () + ":" + yarnCluster.GetApplicationHistoryServer().GetPort());
                WaitForNMsToRegister();
                Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml"
                                                                                             );
                if (url == null)
                {
                    throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath"
                                               );
                }
                Configuration yarnClusterConfig = yarnCluster.GetConfig();
                yarnClusterConfig.Set("yarn.application.classpath", new FilePath(url.AbsolutePath
                                                                                 ).GetParent());
                //write the document to a buffer (not directly to the file, as that
                //can cause the file being written to get read -which will then fail.
                ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
                yarnClusterConfig.WriteXml(bytesOut);
                bytesOut.Close();
                //write the bytes to the file in the classpath
                OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath));
                os.Write(bytesOut.ToByteArray());
                os.Close();
            }
            FileContext fsContext = FileContext.GetLocalFSFileContext();

            fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path"
                                               )), true);
            try
            {
                Sharpen.Thread.Sleep(2000);
            }
            catch (Exception e)
            {
                Log.Info("setup thread sleep interrupted. message=" + e.Message);
            }
        }
Beispiel #21
0
        public virtual void SetUp()
        {
            // create the test root on local_fs
            fcTarget   = FileContext.GetLocalFSFileContext();
            chrootedTo = fileContextTestHelper.GetAbsoluteTestRootPath(fcTarget);
            // In case previous test was killed before cleanup
            fcTarget.Delete(chrootedTo, true);
            fcTarget.Mkdir(chrootedTo, FileContext.DefaultPerm, true);
            Configuration conf = new Configuration();

            // ChRoot to the root of the testDirectory
            fc = FileContext.GetFileContext(new ChRootedFs(fcTarget.GetDefaultFileSystem(), chrootedTo
                                                           ), conf);
        }
 public virtual void SetUp()
 {
     fcTarget        = fc;
     fcTarget2       = fc2;
     targetTestRoot  = fileContextTestHelper.GetAbsoluteTestRootPath(fc);
     targetTestRoot2 = fileContextTestHelper.GetAbsoluteTestRootPath(fc2);
     fcTarget.Delete(targetTestRoot, true);
     fcTarget2.Delete(targetTestRoot2, true);
     fcTarget.Mkdir(targetTestRoot, new FsPermission((short)0x1e8), true);
     fcTarget2.Mkdir(targetTestRoot2, new FsPermission((short)0x1e8), true);
     fsViewConf = ViewFileSystemTestSetup.CreateConfig();
     SetupMountPoints();
     fcView = FileContext.GetFileContext(FsConstants.ViewfsUri, fsViewConf);
 }
Beispiel #23
0
 public virtual void Setup()
 {
     fsContext = FileContext.GetLocalFSFileContext();
     fsPath    = new FilePath("target", this.GetType().Name + "-tmpDir").GetAbsoluteFile(
         );
     fsContext.Delete(new Path(fsPath.GetAbsolutePath()), true);
     config.Set(YarnConfiguration.TimelineServiceLeveldbPath, fsPath.GetAbsolutePath()
                );
     config.SetBoolean(YarnConfiguration.TimelineServiceTtlEnable, false);
     store = new LeveldbTimelineStore();
     store.Init(config);
     store.Start();
     LoadTestEntityData();
     LoadVerificationEntityData();
     LoadTestDomainData();
 }
        public virtual void TearDown()
        {
            if (yarnCluster != null)
            {
                try
                {
                    yarnCluster.Stop();
                }
                finally
                {
                    yarnCluster = null;
                }
            }
            FileContext fsContext = FileContext.GetLocalFSFileContext();

            fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path"
                                               )), true);
        }
        public virtual void Setup()
        {
            fsPath = new FilePath("target", this.GetType().Name + "-tmpDir").GetAbsoluteFile(
                );
            fsContext = FileContext.GetLocalFSFileContext();
            fsContext.Delete(new Path(fsPath.GetAbsolutePath()), true);
            Configuration conf = new YarnConfiguration();

            conf.Set(YarnConfiguration.TimelineServiceLeveldbPath, fsPath.GetAbsolutePath());
            conf.SetBoolean(YarnConfiguration.TimelineServiceTtlEnable, false);
            store = new LeveldbTimelineStore();
            store.Init(conf);
            store.Start();
            LoadTestEntityData();
            LoadVerificationEntityData();
            LoadTestDomainData();
            conf.SetBoolean(YarnConfiguration.YarnAclEnable, false);
            aclsManager = new TimelineACLsManager(conf);
            dataManaer  = new TimelineDataManager(store, aclsManager);
            conf.SetBoolean(YarnConfiguration.YarnAclEnable, true);
            conf.Set(YarnConfiguration.YarnAdminAcl, "admin");
            adminACLsManager = new AdminACLsManager(conf);
        }
Beispiel #26
0
        /// <summary>Cleanup the container.</summary>
        /// <remarks>
        /// Cleanup the container.
        /// Cancels the launch if launch has not started yet or signals
        /// the executor to not execute the process if not already done so.
        /// Also, sends a SIGTERM followed by a SIGKILL to the process if
        /// the process id is available.
        /// </remarks>
        /// <exception cref="System.IO.IOException"/>
        public virtual void CleanupContainer()
        {
            // dispatcher not typed
            ContainerId containerId    = container.GetContainerId();
            string      containerIdStr = ConverterUtils.ToString(containerId);

            Log.Info("Cleaning up container " + containerIdStr);
            try
            {
                context.GetNMStateStore().StoreContainerKilled(containerId);
            }
            catch (IOException e)
            {
                Log.Error("Unable to mark container " + containerId + " killed in store", e);
            }
            // launch flag will be set to true if process already launched
            bool alreadyLaunched = !shouldLaunchContainer.CompareAndSet(false, true);

            if (!alreadyLaunched)
            {
                Log.Info("Container " + containerIdStr + " not launched." + " No cleanup needed to be done"
                         );
                return;
            }
            Log.Debug("Marking container " + containerIdStr + " as inactive");
            // this should ensure that if the container process has not launched
            // by this time, it will never be launched
            exec.DeactivateContainer(containerId);
            if (Log.IsDebugEnabled())
            {
                Log.Debug("Getting pid for container " + containerIdStr + " to kill" + " from pid file "
                          + (pidFilePath != null ? pidFilePath.ToString() : "null"));
            }
            // however the container process may have already started
            try
            {
                // get process id from pid file if available
                // else if shell is still active, get it from the shell
                string processId = null;
                if (pidFilePath != null)
                {
                    processId = GetContainerPid(pidFilePath);
                }
                // kill process
                if (processId != null)
                {
                    string user = container.GetUser();
                    Log.Debug("Sending signal to pid " + processId + " as user " + user + " for container "
                              + containerIdStr);
                    ContainerExecutor.Signal signal = sleepDelayBeforeSigKill > 0 ? ContainerExecutor.Signal
                                                      .Term : ContainerExecutor.Signal.Kill;
                    bool result = exec.SignalContainer(user, processId, signal);
                    Log.Debug("Sent signal " + signal + " to pid " + processId + " as user " + user +
                              " for container " + containerIdStr + ", result=" + (result ? "success" : "failed"
                                                                                  ));
                    if (sleepDelayBeforeSigKill > 0)
                    {
                        new ContainerExecutor.DelayedProcessKiller(container, user, processId, sleepDelayBeforeSigKill
                                                                   , ContainerExecutor.Signal.Kill, exec).Start();
                    }
                }
            }
            catch (Exception e)
            {
                string message = "Exception when trying to cleanup container " + containerIdStr +
                                 ": " + StringUtils.StringifyException(e);
                Log.Warn(message);
                dispatcher.GetEventHandler().Handle(new ContainerDiagnosticsUpdateEvent(containerId
                                                                                        , message));
            }
            finally
            {
                // cleanup pid file if present
                if (pidFilePath != null)
                {
                    FileContext lfs = FileContext.GetLocalFSFileContext();
                    lfs.Delete(pidFilePath, false);
                    lfs.Delete(pidFilePath.Suffix(ExitCodeFileSuffix), false);
                }
            }
        }
Beispiel #27
0
        /// <exception cref="System.Exception"/>
        protected override void ServiceInit(Configuration conf)
        {
            conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
            if (conf.Get(MRJobConfig.MrAmStagingDir) == null)
            {
                conf.Set(MRJobConfig.MrAmStagingDir, new FilePath(GetTestWorkDir(), "apps_staging_dir/"
                                                                  ).GetAbsolutePath());
            }
            // By default, VMEM monitoring disabled, PMEM monitoring enabled.
            if (!conf.GetBoolean(MRConfig.MapreduceMiniclusterControlResourceMonitoring, MRConfig
                                 .DefaultMapreduceMiniclusterControlResourceMonitoring))
            {
                conf.SetBoolean(YarnConfiguration.NmPmemCheckEnabled, false);
                conf.SetBoolean(YarnConfiguration.NmVmemCheckEnabled, false);
            }
            conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "000");
            try
            {
                Path stagingPath = FileContext.GetFileContext(conf).MakeQualified(new Path(conf.Get
                                                                                               (MRJobConfig.MrAmStagingDir)));

                /*
                 * Re-configure the staging path on Windows if the file system is localFs.
                 * We need to use a absolute path that contains the drive letter. The unit
                 * test could run on a different drive than the AM. We can run into the
                 * issue that job files are localized to the drive where the test runs on,
                 * while the AM starts on a different drive and fails to find the job
                 * metafiles. Using absolute path can avoid this ambiguity.
                 */
                if (Path.Windows)
                {
                    if (typeof(LocalFileSystem).IsInstanceOfType(stagingPath.GetFileSystem(conf)))
                    {
                        conf.Set(MRJobConfig.MrAmStagingDir, new FilePath(conf.Get(MRJobConfig.MrAmStagingDir
                                                                                   )).GetAbsolutePath());
                    }
                }
                FileContext fc = FileContext.GetFileContext(stagingPath.ToUri(), conf);
                if (fc.Util().Exists(stagingPath))
                {
                    Log.Info(stagingPath + " exists! deleting...");
                    fc.Delete(stagingPath, true);
                }
                Log.Info("mkdir: " + stagingPath);
                //mkdir the staging directory so that right permissions are set while running as proxy user
                fc.Mkdir(stagingPath, null, true);
                //mkdir done directory as well
                string doneDir     = JobHistoryUtils.GetConfiguredHistoryServerDoneDirPrefix(conf);
                Path   doneDirPath = fc.MakeQualified(new Path(doneDir));
                fc.Mkdir(doneDirPath, null, true);
            }
            catch (IOException e)
            {
                throw new YarnRuntimeException("Could not create staging directory. ", e);
            }
            conf.Set(MRConfig.MasterAddress, "test");
            // The default is local because of
            // which shuffle doesn't happen
            //configure the shuffle service in NM
            conf.SetStrings(YarnConfiguration.NmAuxServices, new string[] { ShuffleHandler.MapreduceShuffleServiceid });
            conf.SetClass(string.Format(YarnConfiguration.NmAuxServiceFmt, ShuffleHandler.MapreduceShuffleServiceid
                                        ), typeof(ShuffleHandler), typeof(Org.Apache.Hadoop.Service.Service));
            // Non-standard shuffle port
            conf.SetInt(ShuffleHandler.ShufflePortConfigKey, 0);
            conf.SetClass(YarnConfiguration.NmContainerExecutor, typeof(DefaultContainerExecutor
                                                                        ), typeof(ContainerExecutor));
            // TestMRJobs is for testing non-uberized operation only; see TestUberAM
            // for corresponding uberized tests.
            conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false);
            base.ServiceInit(conf);
        }
Beispiel #28
0
 public virtual void TearDown()
 {
     localFS.Delete(new Path(basedir.GetPath()), true);
     assertionFailedInThread.Set(false);
 }
 public virtual void TearDown()
 {
     fc.Delete(fileContextTestHelper.GetTestRootPath(fc), true);
 }
Beispiel #30
0
 public virtual void TearDown()
 {
     fcTarget.Delete(chrootedTo, true);
 }