/// <summary> /// Convenience method, so that we don't open a new connection when using this /// method from within another method. /// </summary> /// <remarks> /// Convenience method, so that we don't open a new connection when using this /// method from within another method. Otherwise every API invocation incurs /// the overhead of opening/closing a TCP connection. /// </remarks> /// <exception cref="System.IO.IOException"/> private bool Mkdirs(FTPClient client, Path file, FsPermission permission) { bool created = true; Path workDir = new Path(client.PrintWorkingDirectory()); Path absolute = MakeAbsolute(workDir, file); string pathName = absolute.GetName(); if (!Exists(client, absolute)) { Path parent = absolute.GetParent(); created = (parent == null || Mkdirs(client, parent, FsPermission.GetDirDefault()) ); if (created) { string parentDir = parent.ToUri().GetPath(); client.ChangeWorkingDirectory(parentDir); created = created && client.MakeDirectory(pathName); } } else { if (IsFile(client, absolute)) { throw new ParentNotDirectoryException(string.Format("Can't make directory for path %s since it is a file." , absolute)); } } return(created); }
/// <exception cref="System.IO.IOException"/> private Path CreatePath(FileContext fc, Path root, string year, string month, string day, string id) { Path path = new Path(root, year + Path.Separator + month + Path.Separator + day + Path.Separator + id); fc.Mkdir(path, FsPermission.GetDirDefault(), true); return(path); }
/// <summary> /// Function to run a log handler with directories failing the getFileStatus /// call. /// </summary> /// <remarks> /// Function to run a log handler with directories failing the getFileStatus /// call. The function accepts the log handler, setup the mocks to fail with /// specific exceptions and ensures the deletion service has the correct calls. /// </remarks> /// <param name="logHandler">the logHandler implementation to test</param> /// <param name="appId"> /// the application id that we wish when sending events to the log /// handler /// </param> /// <param name="user">the user name to use</param> /// <param name="mockDelService"> /// a mock of the DeletionService which we will verify /// the delete calls against /// </param> /// <param name="dirsHandler"> /// a spy or mock on the LocalDirsHandler service used to /// when creating the logHandler. It needs to be a spy so that we can intercept /// the getAllLogDirs() call. /// </param> /// <param name="conf">the configuration used</param> /// <param name="spylfs">a spy on the AbstractFileSystem object used when creating lfs /// </param> /// <param name="lfs"> /// the FileContext object to be used to mock the getFileStatus() /// calls /// </param> /// <param name="localLogDirs"> /// list of the log dirs to run the test against, must have /// at least 7 entries /// </param> /// <exception cref="System.Exception"/> public static void RunMockedFailedDirs(LogHandler logHandler, ApplicationId appId , string user, DeletionService mockDelService, LocalDirsHandlerService dirsHandler , Configuration conf, AbstractFileSystem spylfs, FileContext lfs, FilePath[] localLogDirs ) { IDictionary <ApplicationAccessType, string> appAcls = new Dictionary <ApplicationAccessType , string>(); if (localLogDirs.Length < 7) { throw new ArgumentException("Argument localLogDirs must be at least of length 7"); } Path[] localAppLogDirPaths = new Path[localLogDirs.Length]; for (int i = 0; i < localAppLogDirPaths.Length; i++) { localAppLogDirPaths[i] = new Path(localLogDirs[i].GetAbsolutePath(), appId.ToString ()); } IList <string> localLogDirPaths = new AList <string>(localLogDirs.Length); for (int i_1 = 0; i_1 < localLogDirs.Length; i_1++) { localLogDirPaths.AddItem(localLogDirs[i_1].GetAbsolutePath()); } // setup mocks FsPermission defaultPermission = FsPermission.GetDirDefault().ApplyUMask(lfs.GetUMask ()); FileStatus fs = new FileStatus(0, true, 1, 0, Runtime.CurrentTimeMillis(), 0, defaultPermission , string.Empty, string.Empty, new Path(localLogDirs[0].GetAbsolutePath())); Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.IsA <Path>()); Org.Mockito.Mockito.DoReturn(localLogDirPaths).When(dirsHandler).GetLogDirsForCleanup (); logHandler.Handle(new LogHandlerAppStartedEvent(appId, user, null, ContainerLogsRetentionPolicy .AllContainers, appAcls)); // test case where some dirs have the log dir to delete // mock some dirs throwing various exceptions // verify deletion happens only on the others Org.Mockito.Mockito.DoThrow(new FileNotFoundException()).When(spylfs).GetFileStatus (Matchers.Eq(localAppLogDirPaths[0])); Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths [1])); Org.Mockito.Mockito.DoThrow(new AccessControlException()).When(spylfs).GetFileStatus (Matchers.Eq(localAppLogDirPaths[2])); Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths [3])); Org.Mockito.Mockito.DoThrow(new IOException()).When(spylfs).GetFileStatus(Matchers.Eq (localAppLogDirPaths[4])); Org.Mockito.Mockito.DoThrow(new UnsupportedFileSystemException("test")).When(spylfs ).GetFileStatus(Matchers.Eq(localAppLogDirPaths[5])); Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.Eq(localAppLogDirPaths [6])); logHandler.Handle(new LogHandlerAppFinishedEvent(appId)); TestDeletionServiceCall(mockDelService, user, 5000, localAppLogDirPaths[1], localAppLogDirPaths [3], localAppLogDirPaths[6]); return; }
/// <exception cref="System.Exception"/> public virtual void TestCreateEZPopulatesEDEKCache() { Path zonePath = new Path("/TestEncryptionZone"); fsWrapper.Mkdir(zonePath, FsPermission.GetDirDefault(), false); dfsAdmin.CreateEncryptionZone(zonePath, TestKey); NUnit.Framework.Assert.IsTrue(((KMSClientProvider)fs.GetClient().GetKeyProvider() ).GetEncKeyQueueSize(TestKey) > 0); }
/// <summary> /// A stream obtained via this call must be closed before using other APIs of /// this class or else the invocation will block. /// </summary> /// <exception cref="System.IO.IOException"/> public override FSDataOutputStream Create(Path file, FsPermission permission, bool overwrite, int bufferSize, short replication, long blockSize, Progressable progress ) { FTPClient client = Connect(); Path workDir = new Path(client.PrintWorkingDirectory()); Path absolute = MakeAbsolute(workDir, file); FileStatus status; try { status = GetFileStatus(client, file); } catch (FileNotFoundException) { status = null; } if (status != null) { if (overwrite && !status.IsDirectory()) { Delete(client, file, false); } else { Disconnect(client); throw new FileAlreadyExistsException("File already exists: " + file); } } Path parent = absolute.GetParent(); if (parent == null || !Mkdirs(client, parent, FsPermission.GetDirDefault())) { parent = (parent == null) ? new Path("/") : parent; Disconnect(client); throw new IOException("create(): Mkdirs failed to create: " + parent); } client.Allocate(bufferSize); // Change to parent directory on the server. Only then can we write to the // file on the server by opening up an OutputStream. As a side effect the // working directory on the server is changed to the parent directory of the // file. The FTP client connection is closed when close() is called on the // FSDataOutputStream. client.ChangeWorkingDirectory(parent.ToUri().GetPath()); FSDataOutputStream fos = new _FSDataOutputStream_263(this, client, client.StoreFileStream (file.GetName()), statistics); if (!FTPReply.IsPositivePreliminary(client.GetReplyCode())) { // The ftpClient is an inconsistent state. Must close the stream // which in turn will logout and disconnect from FTP server fos.Close(); throw new IOException("Unable to create file: " + file + ", Aborting"); } return(fos); }
/// <exception cref="System.IO.IOException"/> private void ClearDir(FileContext fc, Path p) { try { fc.Delete(p, true); } catch (FileNotFoundException) { } // ignore fc.Mkdir(p, FsPermission.GetDirDefault(), false); }
public virtual void TestLogDeletion() { FilePath[] localLogDirs = GetLocalLogDirFiles(this.GetType().FullName, 2); string localLogDirsString = localLogDirs[0].GetAbsolutePath() + "," + localLogDirs [1].GetAbsolutePath(); conf.Set(YarnConfiguration.NmLogDirs, localLogDirsString); conf.SetBoolean(YarnConfiguration.LogAggregationEnabled, false); conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 0l); dirsHandler.Init(conf); NonAggregatingLogHandler rawLogHandler = new NonAggregatingLogHandler(dispatcher, mockDelService, dirsHandler, new NMNullStateStoreService()); NonAggregatingLogHandler logHandler = Org.Mockito.Mockito.Spy(rawLogHandler); AbstractFileSystem spylfs = Org.Mockito.Mockito.Spy(FileContext.GetLocalFSFileContext ().GetDefaultFileSystem()); FileContext lfs = FileContext.GetFileContext(spylfs, conf); Org.Mockito.Mockito.DoReturn(lfs).When(logHandler).GetLocalFileContext(Matchers.IsA <Configuration>()); FsPermission defaultPermission = FsPermission.GetDirDefault().ApplyUMask(lfs.GetUMask ()); FileStatus fs = new FileStatus(0, true, 1, 0, Runtime.CurrentTimeMillis(), 0, defaultPermission , string.Empty, string.Empty, new Path(localLogDirs[0].GetAbsolutePath())); Org.Mockito.Mockito.DoReturn(fs).When(spylfs).GetFileStatus(Matchers.IsA <Path>()); logHandler.Init(conf); logHandler.Start(); logHandler.Handle(new LogHandlerAppStartedEvent(appId, user, null, ContainerLogsRetentionPolicy .AllContainers, null)); logHandler.Handle(new LogHandlerContainerFinishedEvent(container11, 0)); logHandler.Handle(new LogHandlerAppFinishedEvent(appId)); Path[] localAppLogDirs = new Path[2]; localAppLogDirs[0] = new Path(localLogDirs[0].GetAbsolutePath(), appId.ToString() ); localAppLogDirs[1] = new Path(localLogDirs[1].GetAbsolutePath(), appId.ToString() ); TestDeletionServiceCall(mockDelService, user, 5000, localAppLogDirs); logHandler.Close(); for (int i = 0; i < localLogDirs.Length; i++) { FileUtils.DeleteDirectory(localLogDirs[i]); } }
public FileStatus(long length, bool isdir, int block_replication, long blocksize, long modification_time, long access_time, FsPermission permission, string owner, string group, Path symlink, Path path) { //We should deprecate this soon? this.length = length; this.isdir = isdir; this.block_replication = (short)block_replication; this.blocksize = blocksize; this.modification_time = modification_time; this.access_time = access_time; if (permission != null) { this.permission = permission; } else { if (isdir) { this.permission = FsPermission.GetDirDefault(); } else { if (symlink != null) { this.permission = FsPermission.GetDefault(); } else { this.permission = FsPermission.GetFileDefault(); } } } this.owner = (owner == null) ? string.Empty : owner; this.group = (group == null) ? string.Empty : group; this.symlink = symlink; this.path = path; // The variables isdir and symlink indicate the type: // 1. isdir implies directory, in which case symlink must be null. // 2. !isdir implies a file or symlink, symlink != null implies a // symlink, otherwise it's a file. System.Diagnostics.Debug.Assert((isdir && symlink == null) || !isdir); }
public virtual void Setup() { try { lfs = FileContext.GetLocalFSFileContext(); workDir = new Path("/tmp/temp-" + Runtime.CurrentTimeMillis()); workSpace = new FilePath(workDir.ToUri().GetPath()); lfs.Mkdir(workDir, FsPermission.GetDirDefault(), true); } catch (IOException e) { throw new RuntimeException(e); } Configuration conf = new Configuration(); yarnImage = "yarnImage"; long time = Runtime.CurrentTimeMillis(); conf.Set(YarnConfiguration.NmLocalDirs, "/tmp/nm-local-dir" + time); conf.Set(YarnConfiguration.NmLogDirs, "/tmp/userlogs" + time); dockerUrl = Runtime.GetProperty("docker-service-url"); Log.Info("dockerUrl: " + dockerUrl); if (Strings.IsNullOrEmpty(dockerUrl)) { return; } dockerUrl = " -H " + dockerUrl; dockerExec = "docker " + dockerUrl; conf.Set(YarnConfiguration.NmDockerContainerExecutorImageName, yarnImage); conf.Set(YarnConfiguration.NmDockerContainerExecutorExecName, dockerExec); exec = new DockerContainerExecutor(); dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); exec.SetConf(conf); appSubmitter = Runtime.GetProperty("application.submitter"); if (appSubmitter == null || appSubmitter.IsEmpty()) { appSubmitter = "nobody"; } ShellExec(dockerExec + " pull " + testImage); }
public virtual void Setup() { Assume.AssumeTrue(Shell.Linux); FilePath f = new FilePath("./src/test/resources/mock-container-executor"); if (!FileUtil.CanExecute(f)) { FileUtil.SetExecutable(f, true); } string executorPath = f.GetAbsolutePath(); Configuration conf = new Configuration(); yarnImage = "yarnImage"; long time = Runtime.CurrentTimeMillis(); conf.Set(YarnConfiguration.NmLinuxContainerExecutorPath, executorPath); conf.Set(YarnConfiguration.NmLocalDirs, "/tmp/nm-local-dir" + time); conf.Set(YarnConfiguration.NmLogDirs, "/tmp/userlogs" + time); conf.Set(YarnConfiguration.NmDockerContainerExecutorImageName, yarnImage); conf.Set(YarnConfiguration.NmDockerContainerExecutorExecName, DockerLaunchCommand ); dockerContainerExecutor = new DockerContainerExecutor(); dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); dockerContainerExecutor.SetConf(conf); lfs = null; try { lfs = FileContext.GetLocalFSFileContext(); workDir = new Path("/tmp/temp-" + Runtime.CurrentTimeMillis()); lfs.Mkdir(workDir, FsPermission.GetDirDefault(), true); } catch (IOException e) { throw new RuntimeException(e); } }
/// <summary>Test that encryption zones are properly tracked by the standby.</summary> /// <exception cref="System.Exception"/> public virtual void TestEncryptionZonesTrackedOnStandby() { int len = 8196; Path dir = new Path("/enc"); Path dirChild = new Path(dir, "child"); Path dirFile = new Path(dir, "file"); fs.Mkdir(dir, FsPermission.GetDirDefault()); dfsAdmin0.CreateEncryptionZone(dir, TestKey); fs.Mkdir(dirChild, FsPermission.GetDirDefault()); DFSTestUtil.CreateFile(fs, dirFile, len, (short)1, unchecked ((int)(0xFEED))); string contents = DFSTestUtil.ReadFile(fs, dirFile); // Failover the current standby to active. HATestUtil.WaitForStandbyToCatchUp(nn0, nn1); cluster.ShutdownNameNode(0); cluster.TransitionToActive(1); NUnit.Framework.Assert.AreEqual("Got unexpected ez path", dir.ToString(), dfsAdmin1 .GetEncryptionZoneForPath(dir).GetPath().ToString()); NUnit.Framework.Assert.AreEqual("Got unexpected ez path", dir.ToString(), dfsAdmin1 .GetEncryptionZoneForPath(dirChild).GetPath().ToString()); NUnit.Framework.Assert.AreEqual("File contents after failover were changed", contents , DFSTestUtil.ReadFile(fs, dirFile)); }
public virtual void ConstructorNoOwner() { bool isdir = true; FileStatus fileStatus = new FileStatus(Length, isdir, Replication, Blksize, Mtime , Path); ValidateAccessors(fileStatus, Length, isdir, Replication, Blksize, Mtime, 0, FsPermission .GetDirDefault(), string.Empty, string.Empty, null, Path); }