public virtual void TestCreateDelete() { // Create file fileContextTestHelper.CreateFileNonRecursive(fc, "/foo"); Assert.True(FileContextTestHelper.IsFile(fc, new Path("/foo"))); Assert.True(FileContextTestHelper.IsFile(fcTarget, new Path(chrootedTo , "foo"))); // Create file with recursive dir fileContextTestHelper.CreateFile(fc, "/newDir/foo"); Assert.True(FileContextTestHelper.IsFile(fc, new Path("/newDir/foo" ))); Assert.True(FileContextTestHelper.IsFile(fcTarget, new Path(chrootedTo , "newDir/foo"))); // Delete the created file Assert.True(fc.Delete(new Path("/newDir/foo"), false)); NUnit.Framework.Assert.IsFalse(FileContextTestHelper.Exists(fc, new Path("/newDir/foo" ))); NUnit.Framework.Assert.IsFalse(FileContextTestHelper.Exists(fcTarget, new Path(chrootedTo , "newDir/foo"))); // Create file with a 2 component dirs recursively fileContextTestHelper.CreateFile(fc, "/newDir/newDir2/foo"); Assert.True(FileContextTestHelper.IsFile(fc, new Path("/newDir/newDir2/foo" ))); Assert.True(FileContextTestHelper.IsFile(fcTarget, new Path(chrootedTo , "newDir/newDir2/foo"))); // Delete the created file Assert.True(fc.Delete(new Path("/newDir/newDir2/foo"), false)); NUnit.Framework.Assert.IsFalse(FileContextTestHelper.Exists(fc, new Path("/newDir/newDir2/foo" ))); NUnit.Framework.Assert.IsFalse(FileContextTestHelper.Exists(fcTarget, new Path(chrootedTo , "newDir/newDir2/foo"))); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override void DeleteAsUser(string user, Path subDir, params Path[] baseDirs ) { if (baseDirs == null || baseDirs.Length == 0) { Log.Info("Deleting absolute path : " + subDir); if (!lfs.Delete(subDir, true)) { //Maybe retry Log.Warn("delete returned false for path: [" + subDir + "]"); } return; } foreach (Path baseDir in baseDirs) { Path del = subDir == null ? baseDir : new Path(baseDir, subDir); Log.Info("Deleting path : " + del); try { if (!lfs.Delete(del, true)) { Log.Warn("delete returned false for path: [" + del + "]"); } } catch (FileNotFoundException) { continue; } } }
public virtual void Setup() { localFS.Delete(new Path(localDir.GetAbsolutePath()), true); localFS.Delete(new Path(tmpDir.GetAbsolutePath()), true); localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true); localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true); localDir.Mkdir(); tmpDir.Mkdir(); localLogDir.Mkdir(); remoteLogDir.Mkdir(); Log.Info("Created localDir in " + localDir.GetAbsolutePath()); Log.Info("Created tmpDir in " + tmpDir.GetAbsolutePath()); string bindAddress = "0.0.0.0:12345"; conf.Set(YarnConfiguration.NmAddress, bindAddress); conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath()); conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 1); // Default delSrvc delSrvc = CreateDeletionService(); delSrvc.Init(conf); exec = CreateContainerExecutor(); nodeHealthChecker = new NodeHealthCheckerService(); nodeHealthChecker.Init(conf); dirsHandler = nodeHealthChecker.GetDiskHandler(); containerManager = CreateContainerManager(delSrvc); ((NodeManager.NMContext)context).SetContainerManager(containerManager); nodeStatusUpdater.Init(conf); containerManager.Init(conf); nodeStatusUpdater.Start(); }
// Also try viewfs://default/ - note authority is name of mount table /// <exception cref="System.IO.IOException"/> internal virtual void InitializeTargetTestRoot() { targetTestRoot = fileContextTestHelper.GetAbsoluteTestRootPath(fcTarget); // In case previous test was killed before cleanup fcTarget.Delete(targetTestRoot, true); fcTarget.Mkdir(targetTestRoot, FileContext.DefaultPerm, true); }
public HttpResponseMessage DeleteFile(Guid fileId) { try { HttpResponseMessage response = new HttpResponseMessage(); CFile file = _fileContext.GetByFileId(fileId); // if file exist - delete it, if not continue to delete DB record cuz user wanted to delete real file anyway if (File.Exists(file.Path + file.Name)) { File.Delete(file.Path + file.Name); } // delete record from DB if (_fileContext.Delete(fileId) > 0) { Request.CreateResponse(HttpStatusCode.OK, $"File {file.Name} has been deleted!"); return(response); } Request.CreateResponse(HttpStatusCode.BadRequest, $"Error: File {file.Name} has not been deleted!"); return(response); } catch (ContextException e) { HttpContext.Current.Response.StatusCode = (Int32)HttpStatusCode.BadRequest; throw new ContextException(e.Message, e); } }
/// <summary>delete the test directory in the target local fs</summary> /// <exception cref="System.Exception"/> public static void TearDownForViewFsLocalFs(FileContextTestHelper helper) { FileContext fclocal = FileContext.GetLocalFSFileContext(); Path targetOfTests = helper.GetTestRootPath(fclocal); fclocal.Delete(targetOfTests, true); }
/* * return the ViewFS File context to be used for tests */ /// <exception cref="System.Exception"/> public static FileContext SetupForViewFsLocalFs(FileContextTestHelper helper) { FileContext fsTarget = FileContext.GetLocalFSFileContext(); Path targetOfTests = helper.GetTestRootPath(fsTarget); // In case previous test was killed before cleanup fsTarget.Delete(targetOfTests, true); fsTarget.Mkdir(targetOfTests, FileContext.DefaultPerm, true); Configuration conf = new Configuration(); // Set up viewfs link for test dir as described above string testDir = helper.GetTestRootPath(fsTarget).ToUri().GetPath(); LinkUpFirstComponents(conf, testDir, fsTarget, "test dir"); // Set up viewfs link for home dir as described above SetUpHomeDir(conf, fsTarget); // the test path may be relative to working dir - we need to make that work: // Set up viewfs link for wd as described above string wdDir = fsTarget.GetWorkingDirectory().ToUri().GetPath(); LinkUpFirstComponents(conf, wdDir, fsTarget, "working dir"); FileContext fc = FileContext.GetFileContext(FsConstants.ViewfsUri, conf); fc.SetWorkingDirectory(new Path(wdDir)); // in case testdir relative to wd. Org.Mortbay.Log.Log.Info("Working dir is: " + fc.GetWorkingDirectory()); //System.out.println("SRCOfTests = "+ getTestRootPath(fc, "test")); //System.out.println("TargetOfTests = "+ targetOfTests.toUri()); return(fc); }
public virtual void TestLinkTargetNonSymlink() { FileContext fc = null; Path notSymlink = new Path("/notasymlink"); try { fc = FileContext.GetFileContext(cluster.GetFileSystem().GetUri()); fc.Create(notSymlink, EnumSet.Of(CreateFlag.Create)); DFSClient client = new DFSClient(cluster.GetFileSystem().GetUri(), cluster.GetConfiguration (0)); try { client.GetLinkTarget(notSymlink.ToString()); NUnit.Framework.Assert.Fail("Expected exception for resolving non-symlink"); } catch (IOException e) { GenericTestUtils.AssertExceptionContains("is not a symbolic link", e); } } finally { if (fc != null) { fc.Delete(notSymlink, false); } } }
/// <exception cref="System.IO.IOException"/> public object Run() { FileContext myfc = FileContext.GetFileContext(TestPermissionSymlinks.conf); myfc.Delete(TestPermissionSymlinks.link, false); return(null); }
public virtual void TestStatistics() { URI fsUri = GetFsUri(); FileSystem.Statistics stats = FileContext.GetStatistics(fsUri); Assert.Equal(0, stats.GetBytesRead()); Path filePath = fileContextTestHelper.GetTestRootPath(fc, "file1"); FileContextTestHelper.CreateFile(fc, filePath, numBlocks, blockSize); Assert.Equal(0, stats.GetBytesRead()); VerifyWrittenBytes(stats); FSDataInputStream fstr = fc.Open(filePath); byte[] buf = new byte[blockSize]; int bytesRead = fstr.Read(buf, 0, blockSize); fstr.Read(0, buf, 0, blockSize); Assert.Equal(blockSize, bytesRead); VerifyReadBytes(stats); VerifyWrittenBytes(stats); VerifyReadBytes(FileContext.GetStatistics(GetFsUri())); IDictionary <URI, FileSystem.Statistics> statsMap = FileContext.GetAllStatistics(); URI exactUri = GetSchemeAuthorityUri(); VerifyWrittenBytes(statsMap[exactUri]); fc.Delete(filePath, true); }
/// <summary> /// Override this so that we don't set the targetTestRoot to any path under the /// root of the FS, and so that we don't try to delete the test dir, but rather /// only its contents. /// </summary> /// <exception cref="System.IO.IOException"/> internal override void InitializeTargetTestRoot() { targetTestRoot = fc.MakeQualified(new Path("/")); RemoteIterator <FileStatus> dirContents = fc.ListStatus(targetTestRoot); while (dirContents.HasNext()) { fc.Delete(dirContents.Next().GetPath(), true); } }
public static void Setup() { //Do not error out if metrics are inited multiple times DefaultMetricsSystem.SetMiniClusterMode(true); FilePath dir = new FilePath(stagingDir); stagingDir = dir.GetAbsolutePath(); localFS = FileContext.GetLocalFSFileContext(); localFS.Delete(new Path(testDir.GetAbsolutePath()), true); testDir.Mkdir(); }
/// <exception cref="System.IO.IOException"/> public virtual void Close() { foreach (FilePath symlink in symlinksCreated) { if (!symlink.Delete()) { Log.Warn("Failed to delete symlink created by the local job runner: " + symlink); } } FileContext localFSFileContext = FileContext.GetLocalFSFileContext(); foreach (string archive in localArchives) { localFSFileContext.Delete(new Path(archive), true); } foreach (string file in localFiles) { localFSFileContext.Delete(new Path(file), true); } }
public virtual void Teardown() { if (fc != null) { fc.Delete(GetTestRootPath(), true); } if (cluster != null) { cluster.Shutdown(); } }
public virtual void TearDown() { if (store != null) { store.Stop(); } if (fsContext != null) { fsContext.Delete(new Path(fsPath.GetAbsolutePath()), true); } }
public virtual void TearDown() { try { lfs.Delete(workDir, true); } catch (IOException e) { throw new RuntimeException(e); } }
/// <exception cref="System.IO.IOException"/> private void ClearDir(FileContext fc, Path p) { try { fc.Delete(p, true); } catch (FileNotFoundException) { } // ignore fc.Mkdir(p, FsPermission.GetDirDefault(), false); }
public static void DeleteTmpFiles() { FileContext lfs = FileContext.GetLocalFSFileContext(); try { lfs.Delete(BaseTmpPath, true); } catch (FileNotFoundException) { } }
public virtual void Setup() { fsPath = new FilePath("target", GetType().Name + "-tmpDir").GetAbsoluteFile(); fsContext = FileContext.GetLocalFSFileContext(); fsContext.Delete(new Path(fsPath.GetAbsolutePath()), true); conf = new YarnConfiguration(); conf.SetBoolean(YarnConfiguration.TimelineServiceRecoveryEnabled, true); conf.SetClass(YarnConfiguration.TimelineServiceStateStoreClass, typeof(LeveldbTimelineStateStore ), typeof(TimelineStateStore)); conf.Set(YarnConfiguration.TimelineServiceLeveldbStateStorePath, fsPath.GetAbsolutePath ()); }
/// <exception cref="System.Exception"/> protected internal virtual void SetupInternal(int numNodeManager) { Log.Info("Starting up YARN cluster"); conf = new YarnConfiguration(); conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 128); conf.Set("yarn.log.dir", "target"); conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true); conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName); conf.SetBoolean(YarnConfiguration.NodeLabelsEnabled, true); if (yarnCluster == null) { yarnCluster = new MiniYARNCluster(typeof(TestDistributedShell).Name, 1, numNodeManager , 1, 1); yarnCluster.Init(conf); yarnCluster.Start(); conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname () + ":" + yarnCluster.GetApplicationHistoryServer().GetPort()); WaitForNMsToRegister(); Uri url = Sharpen.Thread.CurrentThread().GetContextClassLoader().GetResource("yarn-site.xml" ); if (url == null) { throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath" ); } Configuration yarnClusterConfig = yarnCluster.GetConfig(); yarnClusterConfig.Set("yarn.application.classpath", new FilePath(url.AbsolutePath ).GetParent()); //write the document to a buffer (not directly to the file, as that //can cause the file being written to get read -which will then fail. ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); yarnClusterConfig.WriteXml(bytesOut); bytesOut.Close(); //write the bytes to the file in the classpath OutputStream os = new FileOutputStream(new FilePath(url.AbsolutePath)); os.Write(bytesOut.ToByteArray()); os.Close(); } FileContext fsContext = FileContext.GetLocalFSFileContext(); fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path" )), true); try { Sharpen.Thread.Sleep(2000); } catch (Exception e) { Log.Info("setup thread sleep interrupted. message=" + e.Message); } }
public virtual void SetUp() { // create the test root on local_fs fcTarget = FileContext.GetLocalFSFileContext(); chrootedTo = fileContextTestHelper.GetAbsoluteTestRootPath(fcTarget); // In case previous test was killed before cleanup fcTarget.Delete(chrootedTo, true); fcTarget.Mkdir(chrootedTo, FileContext.DefaultPerm, true); Configuration conf = new Configuration(); // ChRoot to the root of the testDirectory fc = FileContext.GetFileContext(new ChRootedFs(fcTarget.GetDefaultFileSystem(), chrootedTo ), conf); }
public virtual void SetUp() { fcTarget = fc; fcTarget2 = fc2; targetTestRoot = fileContextTestHelper.GetAbsoluteTestRootPath(fc); targetTestRoot2 = fileContextTestHelper.GetAbsoluteTestRootPath(fc2); fcTarget.Delete(targetTestRoot, true); fcTarget2.Delete(targetTestRoot2, true); fcTarget.Mkdir(targetTestRoot, new FsPermission((short)0x1e8), true); fcTarget2.Mkdir(targetTestRoot2, new FsPermission((short)0x1e8), true); fsViewConf = ViewFileSystemTestSetup.CreateConfig(); SetupMountPoints(); fcView = FileContext.GetFileContext(FsConstants.ViewfsUri, fsViewConf); }
public virtual void Setup() { fsContext = FileContext.GetLocalFSFileContext(); fsPath = new FilePath("target", this.GetType().Name + "-tmpDir").GetAbsoluteFile( ); fsContext.Delete(new Path(fsPath.GetAbsolutePath()), true); config.Set(YarnConfiguration.TimelineServiceLeveldbPath, fsPath.GetAbsolutePath() ); config.SetBoolean(YarnConfiguration.TimelineServiceTtlEnable, false); store = new LeveldbTimelineStore(); store.Init(config); store.Start(); LoadTestEntityData(); LoadVerificationEntityData(); LoadTestDomainData(); }
public virtual void TearDown() { if (yarnCluster != null) { try { yarnCluster.Stop(); } finally { yarnCluster = null; } } FileContext fsContext = FileContext.GetLocalFSFileContext(); fsContext.Delete(new Path(conf.Get("yarn.timeline-service.leveldb-timeline-store.path" )), true); }
public virtual void Setup() { fsPath = new FilePath("target", this.GetType().Name + "-tmpDir").GetAbsoluteFile( ); fsContext = FileContext.GetLocalFSFileContext(); fsContext.Delete(new Path(fsPath.GetAbsolutePath()), true); Configuration conf = new YarnConfiguration(); conf.Set(YarnConfiguration.TimelineServiceLeveldbPath, fsPath.GetAbsolutePath()); conf.SetBoolean(YarnConfiguration.TimelineServiceTtlEnable, false); store = new LeveldbTimelineStore(); store.Init(conf); store.Start(); LoadTestEntityData(); LoadVerificationEntityData(); LoadTestDomainData(); conf.SetBoolean(YarnConfiguration.YarnAclEnable, false); aclsManager = new TimelineACLsManager(conf); dataManaer = new TimelineDataManager(store, aclsManager); conf.SetBoolean(YarnConfiguration.YarnAclEnable, true); conf.Set(YarnConfiguration.YarnAdminAcl, "admin"); adminACLsManager = new AdminACLsManager(conf); }
/// <summary>Cleanup the container.</summary> /// <remarks> /// Cleanup the container. /// Cancels the launch if launch has not started yet or signals /// the executor to not execute the process if not already done so. /// Also, sends a SIGTERM followed by a SIGKILL to the process if /// the process id is available. /// </remarks> /// <exception cref="System.IO.IOException"/> public virtual void CleanupContainer() { // dispatcher not typed ContainerId containerId = container.GetContainerId(); string containerIdStr = ConverterUtils.ToString(containerId); Log.Info("Cleaning up container " + containerIdStr); try { context.GetNMStateStore().StoreContainerKilled(containerId); } catch (IOException e) { Log.Error("Unable to mark container " + containerId + " killed in store", e); } // launch flag will be set to true if process already launched bool alreadyLaunched = !shouldLaunchContainer.CompareAndSet(false, true); if (!alreadyLaunched) { Log.Info("Container " + containerIdStr + " not launched." + " No cleanup needed to be done" ); return; } Log.Debug("Marking container " + containerIdStr + " as inactive"); // this should ensure that if the container process has not launched // by this time, it will never be launched exec.DeactivateContainer(containerId); if (Log.IsDebugEnabled()) { Log.Debug("Getting pid for container " + containerIdStr + " to kill" + " from pid file " + (pidFilePath != null ? pidFilePath.ToString() : "null")); } // however the container process may have already started try { // get process id from pid file if available // else if shell is still active, get it from the shell string processId = null; if (pidFilePath != null) { processId = GetContainerPid(pidFilePath); } // kill process if (processId != null) { string user = container.GetUser(); Log.Debug("Sending signal to pid " + processId + " as user " + user + " for container " + containerIdStr); ContainerExecutor.Signal signal = sleepDelayBeforeSigKill > 0 ? ContainerExecutor.Signal .Term : ContainerExecutor.Signal.Kill; bool result = exec.SignalContainer(user, processId, signal); Log.Debug("Sent signal " + signal + " to pid " + processId + " as user " + user + " for container " + containerIdStr + ", result=" + (result ? "success" : "failed" )); if (sleepDelayBeforeSigKill > 0) { new ContainerExecutor.DelayedProcessKiller(container, user, processId, sleepDelayBeforeSigKill , ContainerExecutor.Signal.Kill, exec).Start(); } } } catch (Exception e) { string message = "Exception when trying to cleanup container " + containerIdStr + ": " + StringUtils.StringifyException(e); Log.Warn(message); dispatcher.GetEventHandler().Handle(new ContainerDiagnosticsUpdateEvent(containerId , message)); } finally { // cleanup pid file if present if (pidFilePath != null) { FileContext lfs = FileContext.GetLocalFSFileContext(); lfs.Delete(pidFilePath, false); lfs.Delete(pidFilePath.Suffix(ExitCodeFileSuffix), false); } } }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); if (conf.Get(MRJobConfig.MrAmStagingDir) == null) { conf.Set(MRJobConfig.MrAmStagingDir, new FilePath(GetTestWorkDir(), "apps_staging_dir/" ).GetAbsolutePath()); } // By default, VMEM monitoring disabled, PMEM monitoring enabled. if (!conf.GetBoolean(MRConfig.MapreduceMiniclusterControlResourceMonitoring, MRConfig .DefaultMapreduceMiniclusterControlResourceMonitoring)) { conf.SetBoolean(YarnConfiguration.NmPmemCheckEnabled, false); conf.SetBoolean(YarnConfiguration.NmVmemCheckEnabled, false); } conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "000"); try { Path stagingPath = FileContext.GetFileContext(conf).MakeQualified(new Path(conf.Get (MRJobConfig.MrAmStagingDir))); /* * Re-configure the staging path on Windows if the file system is localFs. * We need to use a absolute path that contains the drive letter. The unit * test could run on a different drive than the AM. We can run into the * issue that job files are localized to the drive where the test runs on, * while the AM starts on a different drive and fails to find the job * metafiles. Using absolute path can avoid this ambiguity. */ if (Path.Windows) { if (typeof(LocalFileSystem).IsInstanceOfType(stagingPath.GetFileSystem(conf))) { conf.Set(MRJobConfig.MrAmStagingDir, new FilePath(conf.Get(MRJobConfig.MrAmStagingDir )).GetAbsolutePath()); } } FileContext fc = FileContext.GetFileContext(stagingPath.ToUri(), conf); if (fc.Util().Exists(stagingPath)) { Log.Info(stagingPath + " exists! deleting..."); fc.Delete(stagingPath, true); } Log.Info("mkdir: " + stagingPath); //mkdir the staging directory so that right permissions are set while running as proxy user fc.Mkdir(stagingPath, null, true); //mkdir done directory as well string doneDir = JobHistoryUtils.GetConfiguredHistoryServerDoneDirPrefix(conf); Path doneDirPath = fc.MakeQualified(new Path(doneDir)); fc.Mkdir(doneDirPath, null, true); } catch (IOException e) { throw new YarnRuntimeException("Could not create staging directory. ", e); } conf.Set(MRConfig.MasterAddress, "test"); // The default is local because of // which shuffle doesn't happen //configure the shuffle service in NM conf.SetStrings(YarnConfiguration.NmAuxServices, new string[] { ShuffleHandler.MapreduceShuffleServiceid }); conf.SetClass(string.Format(YarnConfiguration.NmAuxServiceFmt, ShuffleHandler.MapreduceShuffleServiceid ), typeof(ShuffleHandler), typeof(Org.Apache.Hadoop.Service.Service)); // Non-standard shuffle port conf.SetInt(ShuffleHandler.ShufflePortConfigKey, 0); conf.SetClass(YarnConfiguration.NmContainerExecutor, typeof(DefaultContainerExecutor ), typeof(ContainerExecutor)); // TestMRJobs is for testing non-uberized operation only; see TestUberAM // for corresponding uberized tests. conf.SetBoolean(MRJobConfig.JobUbertaskEnable, false); base.ServiceInit(conf); }
public virtual void TearDown() { localFS.Delete(new Path(basedir.GetPath()), true); assertionFailedInThread.Set(false); }
public virtual void TearDown() { fc.Delete(fileContextTestHelper.GetTestRootPath(fc), true); }
public virtual void TearDown() { fcTarget.Delete(chrootedTo, true); }