public virtual void TestCreateDirectories() { conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077"); string dirA = new FilePath(testDir, "dirA").GetPath(); string dirB = new FilePath(dirA, "dirB").GetPath(); string dirC = new FilePath(testDir, "dirC").GetPath(); Path pathC = new Path(dirC); FsPermission permDirC = new FsPermission((short)0x1c8); localFs.Mkdir(pathC, null, true); localFs.SetPermission(pathC, permDirC); string[] dirs = new string[] { dirA, dirB, dirC }; DirectoryCollection dc = new DirectoryCollection(dirs, conf.GetFloat(YarnConfiguration .NmMaxPerDiskUtilizationPercentage, YarnConfiguration.DefaultNmMaxPerDiskUtilizationPercentage )); FsPermission defaultPerm = FsPermission.GetDefault().ApplyUMask(new FsPermission( (short)FsPermission.DefaultUmask)); bool createResult = dc.CreateNonExistentDirs(localFs, defaultPerm); NUnit.Framework.Assert.IsTrue(createResult); FileStatus status = localFs.GetFileStatus(new Path(dirA)); NUnit.Framework.Assert.AreEqual("local dir parent not created with proper permissions" , defaultPerm, status.GetPermission()); status = localFs.GetFileStatus(new Path(dirB)); NUnit.Framework.Assert.AreEqual("local dir not created with proper permissions", defaultPerm, status.GetPermission()); status = localFs.GetFileStatus(pathC); NUnit.Framework.Assert.AreEqual("existing local directory permissions modified", permDirC, status.GetPermission()); }
/// <exception cref="System.IO.IOException"/> public virtual void TestMkdirWithExistingDir(FsPermission umask, FsPermission expectedPerms ) { Path f = fileContextTestHelper.GetTestRootPath(fc, "aDir"); fc.SetUMask(umask); fc.Mkdir(f, FileContext.DefaultPerm, true); NUnit.Framework.Assert.IsTrue(FileContextTestHelper.IsDir(fc, f)); NUnit.Framework.Assert.AreEqual("permissions on directory are wrong", expectedPerms , fc.GetFileStatus(f).GetPermission()); }
/// <exception cref="System.IO.IOException"/> public override bool IsFile(Path p) { try { return(fc.GetFileStatus(p).IsFile()); } catch (FileNotFoundException) { return(false); } }
public virtual void TestCreatePermission() { if (Path.Windows) { System.Console.Out.WriteLine("Cannot run test for Windows"); return; } string filename = "foo"; Path f = fileContextTestHelper.GetTestRootPath(fc, filename); fileContextTestHelper.CreateFile(fc, filename); DoFilePermissionCheck(FileContext.FileDefaultPerm.ApplyUMask(fc.GetUMask()), fc.GetFileStatus (f).GetPermission()); }
public virtual void TestModificationTime() { string testFile = "file1"; long fc2ModificationTime; long fc1ModificationTime; Path testPath = QualifiedPath(testFile, fc2); // Create a file on fc2's file system using fc1 FileContextTestHelper.CreateFile(fc1, testPath); // Get modification time using fc2 and fc1 fc1ModificationTime = fc1.GetFileStatus(testPath).GetModificationTime(); fc2ModificationTime = fc2.GetFileStatus(testPath).GetModificationTime(); // Ensure fc1 and fc2 reports same modification time Assert.Equal(fc1ModificationTime, fc2ModificationTime); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> internal static LocalResource CreateZipFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) { byte[] bytes = new byte[len]; r.NextBytes(bytes); FilePath archiveFile = new FilePath(p.ToUri().GetPath() + ".ZIP"); archiveFile.CreateNewFile(); ZipOutputStream @out = new ZipOutputStream(new FileOutputStream(archiveFile)); @out.PutNextEntry(new ZipEntry(p.GetName())); @out.Write(bytes); @out.CloseEntry(); @out.Close(); LocalResource ret = recordFactory.NewRecordInstance <LocalResource>(); ret.SetResource(ConverterUtils.GetYarnUrlFromPath(new Path(p.ToString() + ".ZIP") )); ret.SetSize(len); ret.SetType(LocalResourceType.Archive); ret.SetVisibility(vis); ret.SetTimestamp(files.GetFileStatus(new Path(p.ToString() + ".ZIP")).GetModificationTime ()); return(ret); }
public virtual void TestList() { FileStatus fs = fc.GetFileStatus(new Path("/")); Assert.True(fs.IsDirectory()); // should return the full path not the chrooted path Assert.Equal(fs.GetPath(), chrootedTo); // list on Slash FileStatus[] dirPaths = fc.Util().ListStatus(new Path("/")); Assert.Equal(0, dirPaths.Length); fileContextTestHelper.CreateFileNonRecursive(fc, "/foo"); fileContextTestHelper.CreateFileNonRecursive(fc, "/bar"); fc.Mkdir(new Path("/dirX"), FileContext.DefaultPerm, false); fc.Mkdir(fileContextTestHelper.GetTestRootPath(fc, "/dirY"), FileContext.DefaultPerm , false); fc.Mkdir(new Path("/dirX/dirXX"), FileContext.DefaultPerm, false); dirPaths = fc.Util().ListStatus(new Path("/")); Assert.Equal(4, dirPaths.Length); // Note the the file status paths are the full paths on target fs = fileContextTestHelper.ContainsPath(fcTarget, "foo", dirPaths); NUnit.Framework.Assert.IsNotNull(fs); Assert.True(fs.IsFile()); fs = fileContextTestHelper.ContainsPath(fcTarget, "bar", dirPaths); NUnit.Framework.Assert.IsNotNull(fs); Assert.True(fs.IsFile()); fs = fileContextTestHelper.ContainsPath(fcTarget, "dirX", dirPaths); NUnit.Framework.Assert.IsNotNull(fs); Assert.True(fs.IsDirectory()); fs = fileContextTestHelper.ContainsPath(fcTarget, "dirY", dirPaths); NUnit.Framework.Assert.IsNotNull(fs); Assert.True(fs.IsDirectory()); }
private void DoAppLogAggregationPostCleanUp() { // Remove the local app-log-dirs IList <Path> localAppLogDirs = new AList <Path>(); foreach (string rootLogDir in dirsHandler.GetLogDirsForCleanup()) { Path logPath = new Path(rootLogDir, applicationId); try { // check if log dir exists lfs.GetFileStatus(logPath); localAppLogDirs.AddItem(logPath); } catch (UnsupportedFileSystemException ue) { Log.Warn("Log dir " + rootLogDir + "is an unsupported file system", ue); continue; } catch (IOException) { continue; } } if (localAppLogDirs.Count > 0) { this.delService.Delete(this.userUgi.GetShortUserName(), null, Sharpen.Collections.ToArray (localAppLogDirs, new Path[localAppLogDirs.Count])); } }
/// <exception cref="System.IO.IOException"/> public static void CheckFileStatus(FileContext aFc, string path, FileContextTestHelper.FileType expectedType) { FileStatus s = aFc.GetFileStatus(new Path(path)); NUnit.Framework.Assert.IsNotNull(s); if (expectedType == FileContextTestHelper.FileType.isDir) { Assert.True(s.IsDirectory()); } else { if (expectedType == FileContextTestHelper.FileType.isFile) { Assert.True(s.IsFile()); } else { if (expectedType == FileContextTestHelper.FileType.isSymlink) { Assert.True(s.IsSymlink()); } } } Assert.Equal(aFc.MakeQualified(new Path(path)), s.GetPath()); }
// length of a file (path name) from NN. /// <exception cref="System.IO.IOException"/> private long GetFileLengthFromNN(Path path) { FileStatus fileStatus = useFCOption ? mfc.GetFileStatus(path) : mfs.GetFileStatus (path); return(fileStatus.GetLen()); }
/// <summary> /// Construct given a /// <see cref="FileContext"/> /// and a /// <see cref="Path"/> /// . /// </summary> /// <exception cref="System.IO.IOException"/> public AvroFSInput(FileContext fc, Path p) { FileStatus status = fc.GetFileStatus(p); this.len = status.GetLen(); this.stream = fc.Open(p); }
/// <exception cref="System.IO.IOException"/> public LogReader(Configuration conf, Path remoteAppLogFile) { FileContext fileContext = FileContext.GetFileContext(conf); this.fsDataIStream = fileContext.Open(remoteAppLogFile); reader = new TFile.Reader(this.fsDataIStream, fileContext.GetFileStatus(remoteAppLogFile ).GetLen(), conf); this.scanner = reader.CreateScanner(); }
/// <exception cref="System.IO.IOException"/> public static bool IsDir(FileContext fc, Path p) { try { return(fc.GetFileStatus(p).IsDirectory()); } catch (FileNotFoundException) { return(false); } }
public virtual void TestFileStatusOnMountLink() { Assert.True("Slash should appear as dir", fcView.GetFileStatus( new Path("/")).IsDirectory()); FileContextTestHelper.CheckFileStatus(fcView, "/", FileContextTestHelper.FileType .isDir); FileContextTestHelper.CheckFileStatus(fcView, "/user", FileContextTestHelper.FileType .isDir); FileContextTestHelper.CheckFileStatus(fcView, "/data", FileContextTestHelper.FileType .isDir); FileContextTestHelper.CheckFileStatus(fcView, "/internalDir", FileContextTestHelper.FileType .isDir); FileContextTestHelper.CheckFileStatus(fcView, "/internalDir/linkToDir2", FileContextTestHelper.FileType .isDir); FileContextTestHelper.CheckFileStatus(fcView, "/internalDir/internalDir2/linkToDir3" , FileContextTestHelper.FileType.isDir); FileContextTestHelper.CheckFileStatus(fcView, "/linkToAFile", FileContextTestHelper.FileType .isFile); try { fcView.GetFileStatus(new Path("/danglingLink")); NUnit.Framework.Assert.Fail("Excepted a not found exception here"); } catch (FileNotFoundException) { } }
/// <exception cref="System.IO.IOException"/> internal static LocalResource CreateFile(FileContext files, Path p, int len, Random r, LocalResourceVisibility vis) { CreateFile(files, p, len, r); LocalResource ret = recordFactory.NewRecordInstance <LocalResource>(); ret.SetResource(ConverterUtils.GetYarnUrlFromPath(p)); ret.SetSize(len); ret.SetType(LocalResourceType.File); ret.SetVisibility(vis); ret.SetTimestamp(files.GetFileStatus(p).GetModificationTime()); return(ret); }
/// <exception cref="System.IO.IOException"/> private LocalResource CreateApplicationResource(FileContext fs, Path p, LocalResourceType type) { LocalResource rsrc = recordFactory.NewRecordInstance <LocalResource>(); FileStatus rsrcStat = fs.GetFileStatus(p); rsrc.SetResource(ConverterUtils.GetYarnUrlFromPath(fs.GetDefaultFileSystem().ResolvePath (rsrcStat.GetPath()))); rsrc.SetSize(rsrcStat.GetLen()); rsrc.SetTimestamp(rsrcStat.GetModificationTime()); rsrc.SetType(type); rsrc.SetVisibility(LocalResourceVisibility.Application); return(rsrc); }
internal virtual bool StopFileCreated() { try { fc.GetFileStatus(flagFile); } catch (FileNotFoundException) { return(false); } catch (IOException e) { Log.Error("Got error when checking if file exists:" + flagFile, e); } Log.Info("Flag file was created. Stopping the test."); return(true); }
/// <exception cref="System.IO.IOException"/> private FileStatus GetFileStatus(Path path) { try { if (fs != null) { return(fs.GetFileStatus(path)); } else { return(fc.GetFileStatus(path)); } } catch (FileNotFoundException) { return(null); } }
public virtual void Run() { IList <Path> localAppLogDirs = new AList <Path>(); FileContext lfs = this._enclosing.GetLocalFileContext(this._enclosing.GetConfig() ); foreach (string rootLogDir in this._enclosing.dirsHandler.GetLogDirsForCleanup()) { Path logDir = new Path(rootLogDir, this.applicationId.ToString()); try { lfs.GetFileStatus(logDir); localAppLogDirs.AddItem(logDir); } catch (UnsupportedFileSystemException ue) { NonAggregatingLogHandler.Log.Warn("Unsupported file system used for log dir " + logDir , ue); continue; } catch (IOException) { continue; } } // Inform the application before the actual delete itself, so that links // to logs will no longer be there on NM web-UI. this._enclosing.dispatcher.GetEventHandler().Handle(new ApplicationEvent(this.applicationId , ApplicationEventType.ApplicationLogHandlingFinished)); if (localAppLogDirs.Count > 0) { this._enclosing.delService.Delete(this.user, null, (Path[])Sharpen.Collections.ToArray (localAppLogDirs, new Path[localAppLogDirs.Count])); } try { this._enclosing.stateStore.RemoveLogDeleter(this.applicationId); } catch (IOException e) { NonAggregatingLogHandler.Log.Error("Error removing log deletion state", e); } }
/// <exception cref="System.IO.IOException"/> private void CreateDir(FileContext localFs, Path dir, FsPermission perm) { if (dir == null) { return; } try { localFs.GetFileStatus(dir); } catch (FileNotFoundException) { CreateDir(localFs, dir.GetParent(), perm); localFs.Mkdir(dir, perm, false); if (!perm.Equals(perm.ApplyUMask(localFs.GetUMask()))) { localFs.SetPermission(dir, perm); } } }
/// <exception cref="System.IO.IOException"/> private void VerifyPermsRecursively(FileSystem fs, FileContext files, Path p, LocalResourceVisibility vis) { FileStatus status = files.GetFileStatus(p); if (status.IsDirectory()) { if (vis == LocalResourceVisibility.Public) { NUnit.Framework.Assert.IsTrue(status.GetPermission().ToShort() == FSDownload.PublicDirPerms .ToShort()); } else { NUnit.Framework.Assert.IsTrue(status.GetPermission().ToShort() == FSDownload.PrivateDirPerms .ToShort()); } if (!status.IsSymlink()) { FileStatus[] statuses = fs.ListStatus(p); foreach (FileStatus stat in statuses) { VerifyPermsRecursively(fs, files, stat.GetPath(), vis); } } } else { if (vis == LocalResourceVisibility.Public) { NUnit.Framework.Assert.IsTrue(status.GetPermission().ToShort() == FSDownload.PublicFilePerms .ToShort()); } else { NUnit.Framework.Assert.IsTrue(status.GetPermission().ToShort() == FSDownload.PrivateFilePerms .ToShort()); } } }
/// <exception cref="System.IO.IOException"/> internal static LocalResource CreateJar(FileContext files, Path p, LocalResourceVisibility vis) { Log.Info("Create jar file " + p); FilePath jarFile = new FilePath((files.MakeQualified(p)).ToUri()); FileOutputStream stream = new FileOutputStream(jarFile); Log.Info("Create jar out stream "); JarOutputStream @out = new JarOutputStream(stream, new Manifest()); Log.Info("Done writing jar stream "); @out.Close(); LocalResource ret = recordFactory.NewRecordInstance <LocalResource>(); ret.SetResource(ConverterUtils.GetYarnUrlFromPath(p)); FileStatus status = files.GetFileStatus(p); ret.SetSize(status.GetLen()); ret.SetTimestamp(status.GetModificationTime()); ret.SetType(LocalResourceType.Pattern); ret.SetVisibility(vis); ret.SetPattern("classes/.*"); return(ret); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Sharpen.URISyntaxException"/> /// <exception cref="System.Exception"/> public virtual void TestDownload() { Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077"); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource , LocalResourceVisibility>(); Random rand = new Random(); long sharedSeed = rand.NextLong(); rand.SetSeed(sharedSeed); System.Console.Out.WriteLine("SEED: " + sharedSeed); IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource, Future <Path> >(); ExecutorService exec = Executors.NewSingleThreadExecutor(); LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName); int[] sizes = new int[10]; for (int i = 0; i < 10; ++i) { sizes[i] = rand.Next(512) + 512; LocalResourceVisibility vis = LocalResourceVisibility.Private; if (i % 2 == 1) { vis = LocalResourceVisibility.Application; } Path p = new Path(basedir, string.Empty + i); LocalResource rsrc = CreateFile(files, p, sizes[i], rand, vis); rsrcVis[rsrc] = vis; Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), sizes[i], conf); destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf , destPath, rsrc); pending[rsrc] = exec.Submit(fsd); } exec.Shutdown(); while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds)) { } foreach (Future <Path> path in pending.Values) { NUnit.Framework.Assert.IsTrue(path.IsDone()); } try { foreach (KeyValuePair <LocalResource, Future <Path> > p in pending) { Path localized = p.Value.Get(); NUnit.Framework.Assert.AreEqual(sizes[Sharpen.Extensions.ValueOf(localized.GetName ())], p.Key.GetSize()); FileStatus status = files.GetFileStatus(localized.GetParent()); FsPermission perm = status.GetPermission(); NUnit.Framework.Assert.AreEqual("Cache directory permissions are incorrect", new FsPermission((short)0x1ed), perm); status = files.GetFileStatus(localized); perm = status.GetPermission(); System.Console.Out.WriteLine("File permission " + perm + " for rsrc vis " + p.Key .GetVisibility().ToString()); System.Diagnostics.Debug.Assert((rsrcVis.Contains(p.Key))); NUnit.Framework.Assert.IsTrue("Private file should be 500", perm.ToShort() == FSDownload .PrivateFilePerms.ToShort()); } } catch (ExecutionException e) { throw new IOException("Failed exec", e); } }
public virtual void TestDirPermissions() { DeleteTmpFiles(); string user = "******"; string appId = "app_12345_123"; FsPermission userCachePerm = new FsPermission(DefaultContainerExecutor.UserPerm); FsPermission appCachePerm = new FsPermission(DefaultContainerExecutor.AppcachePerm ); FsPermission fileCachePerm = new FsPermission(DefaultContainerExecutor.FilecachePerm ); FsPermission appDirPerm = new FsPermission(DefaultContainerExecutor.AppdirPerm); FsPermission logDirPerm = new FsPermission(DefaultContainerExecutor.LogdirPerm); IList <string> localDirs = new AList <string>(); localDirs.AddItem(new Path(BaseTmpPath, "localDirA").ToString()); localDirs.AddItem(new Path(BaseTmpPath, "localDirB").ToString()); IList <string> logDirs = new AList <string>(); logDirs.AddItem(new Path(BaseTmpPath, "logDirA").ToString()); logDirs.AddItem(new Path(BaseTmpPath, "logDirB").ToString()); Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077"); FileContext lfs = FileContext.GetLocalFSFileContext(conf); DefaultContainerExecutor executor = new DefaultContainerExecutor(lfs); executor.Init(); try { executor.CreateUserLocalDirs(localDirs, user); executor.CreateUserCacheDirs(localDirs, user); executor.CreateAppDirs(localDirs, user, appId); foreach (string dir in localDirs) { FileStatus stats = lfs.GetFileStatus(new Path(new Path(dir, ContainerLocalizer.Usercache ), user)); NUnit.Framework.Assert.AreEqual(userCachePerm, stats.GetPermission()); } foreach (string dir_1 in localDirs) { Path userCachePath = new Path(new Path(dir_1, ContainerLocalizer.Usercache), user ); Path appCachePath = new Path(userCachePath, ContainerLocalizer.Appcache); FileStatus stats = lfs.GetFileStatus(appCachePath); NUnit.Framework.Assert.AreEqual(appCachePerm, stats.GetPermission()); stats = lfs.GetFileStatus(new Path(userCachePath, ContainerLocalizer.Filecache)); NUnit.Framework.Assert.AreEqual(fileCachePerm, stats.GetPermission()); stats = lfs.GetFileStatus(new Path(appCachePath, appId)); NUnit.Framework.Assert.AreEqual(appDirPerm, stats.GetPermission()); } executor.CreateAppLogDirs(appId, logDirs, user); foreach (string dir_2 in logDirs) { FileStatus stats = lfs.GetFileStatus(new Path(dir_2, appId)); NUnit.Framework.Assert.AreEqual(logDirPerm, stats.GetPermission()); } } finally { DeleteTmpFiles(); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void TestDirDownload() { Configuration conf = new Configuration(); FileContext files = FileContext.GetLocalFSFileContext(conf); Path basedir = files.MakeQualified(new Path("target", typeof(TestFSDownload).Name )); files.Mkdir(basedir, null, true); conf.SetStrings(typeof(TestFSDownload).FullName, basedir.ToString()); IDictionary <LocalResource, LocalResourceVisibility> rsrcVis = new Dictionary <LocalResource , LocalResourceVisibility>(); Random rand = new Random(); long sharedSeed = rand.NextLong(); rand.SetSeed(sharedSeed); System.Console.Out.WriteLine("SEED: " + sharedSeed); IDictionary <LocalResource, Future <Path> > pending = new Dictionary <LocalResource, Future <Path> >(); ExecutorService exec = Executors.NewSingleThreadExecutor(); LocalDirAllocator dirs = new LocalDirAllocator(typeof(TestFSDownload).FullName); for (int i = 0; i < 5; ++i) { LocalResourceVisibility vis = LocalResourceVisibility.Private; if (i % 2 == 1) { vis = LocalResourceVisibility.Application; } Path p = new Path(basedir, "dir" + i + ".jar"); LocalResource rsrc = CreateJar(files, p, vis); rsrcVis[rsrc] = vis; Path destPath = dirs.GetLocalPathForWrite(basedir.ToString(), conf); destPath = new Path(destPath, System.Convert.ToString(uniqueNumberGenerator.IncrementAndGet ())); FSDownload fsd = new FSDownload(files, UserGroupInformation.GetCurrentUser(), conf , destPath, rsrc); pending[rsrc] = exec.Submit(fsd); } exec.Shutdown(); while (!exec.AwaitTermination(1000, TimeUnit.Milliseconds)) { } foreach (Future <Path> path in pending.Values) { NUnit.Framework.Assert.IsTrue(path.IsDone()); } try { foreach (KeyValuePair <LocalResource, Future <Path> > p in pending) { Path localized = p.Value.Get(); FileStatus status = files.GetFileStatus(localized); System.Console.Out.WriteLine("Testing path " + localized); System.Diagnostics.Debug.Assert((status.IsDirectory())); System.Diagnostics.Debug.Assert((rsrcVis.Contains(p.Key))); VerifyPermsRecursively(localized.GetFileSystem(conf), files, localized, rsrcVis[p .Key]); } } catch (ExecutionException e) { throw new IOException("Failed exec", e); } }