static void CopyLength(Stream fs, string dstFilePath, long fileLength, IEnumerable <byte> expectedHash) { const int bufSz = 65536; var remain = fileLength; var buffer = new byte[bufSz]; using (var md5 = MD5.Create()) using (var fout = NativeIO.OpenFileStream(new PathInfo(dstFilePath), FileAccess.Write, FileMode.CreateNew)) { int len; while (remain > bufSz) { len = fs.Read(buffer, 0, bufSz); if (len != bufSz) { throw new Exception("Malformed file: data truncated"); } md5.TransformBlock(buffer, 0, len, null, 0); fout.Write(buffer, 0, bufSz); remain -= bufSz; } if (remain != 0) { len = fs.Read(buffer, 0, (int)remain); if (len != remain) { throw new Exception("Malformed file: data truncated at end"); } md5.TransformBlock(buffer, 0, (int)remain, null, 0); fout.Write(buffer, 0, (int)remain); } md5.TransformFinalBlock(new byte[0], 0, 0); if (!HashesEqual(expectedHash, md5.Hash)) { throw new Exception("Damaged archive: File at " + dstFilePath + " failed a checksum"); } } }
/// <exception cref="System.IO.IOException"/> private void RenameSelf(string newSuffix) { FilePath src = file; FilePath dst = new FilePath(src.GetParent(), src.GetName() + newSuffix); // renameTo fails on Windows if the destination file already exists. try { if (dst.Exists()) { if (!dst.Delete()) { throw new IOException("Couldn't delete " + dst); } } NativeIO.RenameTo(src, dst); } catch (IOException e) { throw new IOException("Couldn't rename log " + src + " to " + dst, e); } file = dst; }
/// <exception cref="System.IO.IOException"/> protected internal virtual bool MkOneDirWithMode(Path p, FilePath p2f, FsPermission permission) { if (permission == null) { return(p2f.Mkdir()); } else { if (Shell.Windows && NativeIO.IsAvailable()) { try { NativeIO.Windows.CreateDirectoryWithMode(p2f, permission.ToShort()); return(true); } catch (IOException e) { if (Log.IsDebugEnabled()) { Log.Debug(string.Format("NativeIO.createDirectoryWithMode error, path = %s, mode = %o" , p2f, permission.ToShort()), e); } return(false); } } else { bool b = p2f.Mkdir(); if (b) { SetPermission(p, permission); } return(b); } } }
public static void files_can_be_created_and_written_and_read_and_copied_in_very_long_paths() { // Create a >255 length path const string path = TempRoot + "\\QIO\\Pseudopseudohypoparathyroidism\\Pneumonoultramicroscopicsilicovolcanoconiosis\\Floccinaucinihilipilification\\Antidisestablishmentarianism\\Honorificabilitudinitatibus\\Donaudampfschiffahrtselektrizitätenhauptbetriebswerkbauunterbeamtengesellschaft"; NativeIO.CreateDirectory(new PathInfo(path), recursive: true); var sampleData = new byte[] { 1, 2, 3, 4, 5, 6, 7, 8 }; var srcFile = new PathInfo(path + "\\example.file.txt"); var dstFile = new PathInfo(path + "\\example.copy.txt"); // write a file using (var fs = NativeIO.OpenFileStream(srcFile, FileAccess.Write, FileMode.Create, FileShare.None)) { fs.Write(sampleData, 4, 4); fs.Write(sampleData, 0, 4); fs.Flush(); } // copy the file elsewhere Assert.True(NativeIO.Exists(srcFile), "Source file can't be found (didn't write correctly?)"); Assert.False(NativeIO.SymbolicLink.IsSymLink(srcFile), "File was a sym-link"); Assert.True(NativeIO.CopyFile(srcFile, dstFile), "Failed to copy file"); Assert.True(NativeIO.Exists(dstFile), "Target file can't be found"); // Check the contents using (var fs = NativeIO.OpenFileStream(srcFile, FileAccess.Read)) { var buf = new byte[8]; var length = fs.Read(buf, 0, 8); Assert.That(length, Is.EqualTo(8)); Assert.That(buf, Is.EquivalentTo(new byte[] { 5, 6, 7, 8, 1, 2, 3, 4 })); } // cleanup NativeIO.DeleteDirectory(new DirectoryDetail(TempRoot + "\\QIO"), recursive: true); }
static SecureIOUtils() { bool shouldBeSecure = UserGroupInformation.IsSecurityEnabled(); bool canBeSecure = NativeIO.IsAvailable(); if (!canBeSecure && shouldBeSecure) { throw new RuntimeException("Secure IO is not possible without native code extensions." ); } // Pre-cache an instance of the raw FileSystem since we sometimes // do secure IO in a shutdown hook, where this call could fail. try { rawFilesystem = FileSystem.GetLocal(new Configuration()).GetRaw(); } catch (IOException) { throw new RuntimeException("Couldn't obtain an instance of RawLocalFileSystem."); } // SecureIO just skips security checks in the case that security is // disabled skipSecurity = !canBeSecure; }
/// <exception cref="System.IO.IOException"/> private LocalFSFileOutputStream(RawLocalFileSystem _enclosing, Path f, bool append , FsPermission permission) { this._enclosing = _enclosing; FilePath file = this._enclosing.PathToFile(f); if (permission == null) { this.fos = new FileOutputStream(file, append); } else { if (Shell.Windows && NativeIO.IsAvailable()) { this.fos = NativeIO.Windows.CreateFileOutputStreamWithMode(file, append, permission .ToShort()); } else { this.fos = new FileOutputStream(file, append); bool success = false; try { this._enclosing.SetPermission(f, permission); success = true; } finally { if (!success) { IOUtils.Cleanup(FileSystem.Log, this.fos); } } } } }
public FileStream(String path, FileMode mode, FileAccess access, FileShare share, int bufferSize) { // This will perform validation on path _fileName = Path.GetFullPath(path); // make sure mode, access, and share are within range if (mode < FileMode.CreateNew || mode > FileMode.Append || access < FileAccess.Read || access > FileAccess.ReadWrite || share < FileShare.None || share > FileShare.ReadWrite) { throw new ArgumentOutOfRangeException(); } // Get wantsRead and wantsWrite from access, note that they cannot both be false bool wantsRead = (access & FileAccess.Read) == FileAccess.Read; bool wantsWrite = (access & FileAccess.Write) == FileAccess.Write; // You can't open for readonly access (wantsWrite == false) when // mode is CreateNew, Create, Truncate or Append (when it's not Open or OpenOrCreate) if (mode != FileMode.Open && mode != FileMode.OpenOrCreate && !wantsWrite) { throw new ArgumentException(); } // We need to register the share information prior to the actual file open call (the NativeFileStream ctor) // so subsequent file operation on the same file will behave correctly _fileRecord = FileSystemManager.AddToOpenList(_fileName, (int)access, (int)share); try { uint attributes = NativeIO.GetAttributes(_fileName); bool exists = (attributes != 0xFFFFFFFF); bool isReadOnly = (exists) ? (((FileAttributes)attributes) & FileAttributes.ReadOnly) == FileAttributes.ReadOnly : false; // If the path specified is an existing directory, fail if (exists && ((((FileAttributes)attributes) & FileAttributes.Directory) == FileAttributes.Directory)) { throw new IOException("", (int)IOException.IOExceptionErrorCode.UnauthorizedAccess); } // The seek limit is 0 (the beginning of the file) for all modes except Append _seekLimit = 0; switch (mode) { case FileMode.CreateNew: // if the file exists, IOException is thrown if (exists) { throw new IOException("", (int)IOException.IOExceptionErrorCode.PathAlreadyExists); } _nativeFileStream = new NativeFileStream(_fileName, bufferSize); break; case FileMode.Create: // if the file exists, it should be overwritten _nativeFileStream = new NativeFileStream(_fileName, bufferSize); if (exists) { _nativeFileStream.SetLength(0); } break; case FileMode.Open: // if the file does not exist, IOException/FileNotFound is thrown if (!exists) { throw new IOException("", (int)IOException.IOExceptionErrorCode.FileNotFound); } _nativeFileStream = new NativeFileStream(_fileName, bufferSize); break; case FileMode.OpenOrCreate: // if the file does not exist, it is created _nativeFileStream = new NativeFileStream(_fileName, bufferSize); break; case FileMode.Truncate: // the file would be overwritten. if the file does not exist, IOException/FileNotFound is thrown if (!exists) { throw new IOException("", (int)IOException.IOExceptionErrorCode.FileNotFound); } _nativeFileStream = new NativeFileStream(_fileName, bufferSize); _nativeFileStream.SetLength(0); break; case FileMode.Append: // Opens the file if it exists and seeks to the end of the file. Append can only be used in conjunction with FileAccess.Write // Attempting to seek to a position before the end of the file will throw an IOException and any attempt to read fails and throws an NotSupportedException if (access != FileAccess.Write) { throw new ArgumentException(); } _nativeFileStream = new NativeFileStream(_fileName, bufferSize); _seekLimit = _nativeFileStream.Seek(0, (uint)SeekOrigin.End); break; // We've already checked the mode value previously, so no need for default //default: // throw new ArgumentOutOfRangeException(); } // Now that we have a valid NativeFileStream, we add it to the FileRecord, so it could gets clean up // in case an eject or force format _fileRecord.NativeFileStream = _nativeFileStream; // Retrive the filesystem capabilities _nativeFileStream.GetStreamProperties(out _canRead, out _canWrite, out _canSeek); // If the file is readonly, regardless of the filesystem capability, we'll turn off write if (isReadOnly) { _canWrite = false; } // Make sure the requests (wantsRead / wantsWrite) matches the filesystem capabilities (canRead / canWrite) if ((wantsRead && !_canRead) || (wantsWrite && !_canWrite)) { throw new IOException("", (int)IOException.IOExceptionErrorCode.UnauthorizedAccess); } // finally, adjust the _canRead / _canWrite to match the requests if (!wantsWrite) { _canWrite = false; } else if (!wantsRead) { _canRead = false; } } catch { // something went wrong, clean up and re-throw the exception if (_nativeFileStream != null) { _nativeFileStream.Close(); } FileSystemManager.RemoveFromOpenList(_fileRecord); throw; } }
/// <summary> /// Move replicas in the lazy persist directory to their corresponding locations /// in the finalized directory. /// </summary> /// <returns>number of replicas recovered.</returns> /// <exception cref="System.IO.IOException"/> private int MoveLazyPersistReplicasToFinalized(FilePath source) { FilePath[] files = FileUtil.ListFiles(source); int numRecovered = 0; foreach (FilePath file in files) { if (file.IsDirectory()) { numRecovered += MoveLazyPersistReplicasToFinalized(file); } if (Block.IsMetaFilename(file.GetName())) { FilePath metaFile = file; FilePath blockFile = Block.MetaToBlockFile(metaFile); long blockId = Block.Filename2id(blockFile.GetName()); FilePath targetDir = DatanodeUtil.IdToBlockDir(finalizedDir, blockId); if (blockFile.Exists()) { if (!targetDir.Exists() && !targetDir.Mkdirs()) { Log.Warn("Failed to mkdirs " + targetDir); continue; } FilePath targetMetaFile = new FilePath(targetDir, metaFile.GetName()); try { NativeIO.RenameTo(metaFile, targetMetaFile); } catch (IOException e) { Log.Warn("Failed to move meta file from " + metaFile + " to " + targetMetaFile, e ); continue; } FilePath targetBlockFile = new FilePath(targetDir, blockFile.GetName()); try { NativeIO.RenameTo(blockFile, targetBlockFile); } catch (IOException e) { Log.Warn("Failed to move block file from " + blockFile + " to " + targetBlockFile , e); continue; } if (targetBlockFile.Exists() && targetMetaFile.Exists()) { ++numRecovered; } else { // Failure should be rare. Log.Warn("Failed to move " + blockFile + " to " + targetDir); } } } } FileUtil.FullyDelete(source); return(numRecovered); }
/// <summary> /// Read all files under the source path into a single destination file. /// </summary> /// <param name="srcPath">Full path of source directory</param> /// <param name="dstFilePath">Full path of destination file</param> /// <param name="signingCertPath">PFX file containing </param> /// <param name="certPassword">password securing the pfx file</param> public static void FolderToFile(string srcPath, string dstFilePath, string signingCertPath = null, string certPassword = null) { // list out name+hash -> [path] // write this to a single file // gzip that file var tmpPath = dstFilePath+".tmp"; var filePaths = new Dictionary<string, PathList>(); var symLinks = new Dictionary<string, string>(); // link path -> target path // find distinct files var files = NativeIO.EnumerateFiles(new PathInfo(srcPath).FullNameUnc, (symPath, targetPath)=>{ if (IsSubpath(srcPath, targetPath)) { symLinks.Add(symPath, targetPath); return false; } return true; }, searchOption: SearchOption.AllDirectories); foreach (var file in files) { var hash = HashOf(file); Add(hash, file, filePaths); } // pack everything into a temp file if (File.Exists(tmpPath)) File.Delete(tmpPath); using (var fs = File.OpenWrite(tmpPath)) { // Write data files foreach (var fileKey in filePaths.Keys) { var pathList = filePaths[fileKey]; var catPaths = Encoding.UTF8.GetBytes(string.Join("|", Filter(pathList.Paths, srcPath))); // Write <MD5:16 bytes> fs.Write(pathList.HashData, 0, 16); // Write <length:8 bytes><paths:utf8 str> WriteLength(catPaths.Length, fs); fs.Write(catPaths, 0, catPaths.Length); var info = NativeIO.ReadFileDetails(new PathInfo(pathList.Paths[0])); // Write <length:8 bytes><data:byte array> WriteLength((long)info.Length, fs); using (var inf = NativeIO.OpenFileStream(info.PathInfo, FileAccess.Read)) inf.CopyTo(fs); fs.Flush(); } // Write symbolic links foreach (var linkSrc in symLinks.Keys) { var linkTarget = symLinks[linkSrc]; var linkData = Encoding.UTF8.GetBytes(string.Join("|", Filter(new[] { linkSrc, linkTarget }, srcPath))); // Write <zeros:16 bytes> WriteLength(0, fs); WriteLength(0, fs); // Write <length:8 bytes><path pair:utf8 str>, path pair is 'src|target' WriteLength(linkData.Length, fs); fs.Write(linkData, 0, linkData.Length); // Write <length:8 bytes>, always zero (there is not file content in a link) WriteLength(0, fs); } fs.Flush(); } // If cert, write to *another* temp file with the signing header in place if ( ! string.IsNullOrWhiteSpace(signingCertPath)) { var tmpSignPath = tmpPath + ".signed"; try { using (var cat = File.OpenRead(tmpPath)) { var signingBytes = Crypto.BuildSigningHeader(cat,signingCertPath, certPassword); cat.Seek(0, SeekOrigin.Begin); using (var final = File.Open(tmpSignPath, FileMode.Create, FileAccess.Write)) { final.Write(signingBytes, 0, signingBytes.Length); cat.CopyTo(final); final.Flush(); } } File.Delete(tmpPath); // wipe the old one File.Move(tmpSignPath, tmpPath); // use the new one for compression } catch (Exception ex) { Console.WriteLine("Signing failed: "+ex); throw; } } // Compress the file if (File.Exists(dstFilePath)) File.Delete(dstFilePath); using (var compressing = new GZipStream(File.OpenWrite(dstFilePath), CompressionLevel.Optimal)) using (var cat = File.OpenRead(tmpPath)) { cat.CopyTo(compressing, 65536); compressing.Flush(); } // Kill the temp file File.Delete(tmpPath); }
/// <exception cref="System.IO.IOException"/> public virtual void TestContainerLogsFileAccess() { // This test will run only if NativeIO is enabled as SecureIOUtils // require it to be enabled. Assume.AssumeTrue(NativeIO.IsAvailable()); Configuration conf = new Configuration(); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); FilePath workDir = new FilePath(testWorkDir, "testContainerLogsFileAccess1"); Path remoteAppLogFile = new Path(workDir.GetAbsolutePath(), "aggregatedLogFile"); Path srcFileRoot = new Path(workDir.GetAbsolutePath(), "srcFiles"); string data = "Log File content for container : "; // Creating files for container1. Log aggregator will try to read log files // with illegal user. ApplicationId applicationId = ApplicationId.NewInstance(1, 1); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId , 1); ContainerId testContainerId1 = ContainerId.NewContainerId(applicationAttemptId, 1 ); Path appDir = new Path(srcFileRoot, testContainerId1.GetApplicationAttemptId().GetApplicationId ().ToString()); Path srcFilePath1 = new Path(appDir, testContainerId1.ToString()); string stdout = "stdout"; string stderr = "stderr"; WriteSrcFile(srcFilePath1, stdout, data + testContainerId1.ToString() + stdout); WriteSrcFile(srcFilePath1, stderr, data + testContainerId1.ToString() + stderr); UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); AggregatedLogFormat.LogWriter logWriter = new AggregatedLogFormat.LogWriter(conf, remoteAppLogFile, ugi); AggregatedLogFormat.LogKey logKey = new AggregatedLogFormat.LogKey(testContainerId1 ); string randomUser = "******"; AggregatedLogFormat.LogValue logValue = Org.Mockito.Mockito.Spy(new AggregatedLogFormat.LogValue (Collections.SingletonList(srcFileRoot.ToString()), testContainerId1, randomUser )); // It is trying simulate a situation where first log file is owned by // different user (probably symlink) and second one by the user itself. // The first file should not be aggregated. Because this log file has the invalid // user name. Org.Mockito.Mockito.When(logValue.GetUser()).ThenReturn(randomUser).ThenReturn(ugi .GetShortUserName()); logWriter.Append(logKey, logValue); logWriter.Close(); BufferedReader @in = new BufferedReader(new FileReader(new FilePath(remoteAppLogFile .ToUri().GetRawPath()))); string line; StringBuilder sb = new StringBuilder(string.Empty); while ((line = @in.ReadLine()) != null) { Log.Info(line); sb.Append(line); } line = sb.ToString(); string expectedOwner = ugi.GetShortUserName(); if (Path.Windows) { string adminsGroupString = "Administrators"; if (Arrays.AsList(ugi.GetGroupNames()).Contains(adminsGroupString)) { expectedOwner = adminsGroupString; } } // This file: stderr should not be aggregated. // And we will not aggregate the log message. string stdoutFile1 = StringUtils.Join(FilePath.separator, Arrays.AsList(new string [] { workDir.GetAbsolutePath(), "srcFiles", testContainerId1.GetApplicationAttemptId ().GetApplicationId().ToString(), testContainerId1.ToString(), stderr })); // The file: stdout is expected to be aggregated. string stdoutFile2 = StringUtils.Join(FilePath.separator, Arrays.AsList(new string [] { workDir.GetAbsolutePath(), "srcFiles", testContainerId1.GetApplicationAttemptId ().GetApplicationId().ToString(), testContainerId1.ToString(), stdout })); string message2 = "Owner '" + expectedOwner + "' for path " + stdoutFile2 + " did not match expected owner '" + ugi.GetShortUserName() + "'"; NUnit.Framework.Assert.IsFalse(line.Contains(message2)); NUnit.Framework.Assert.IsFalse(line.Contains(data + testContainerId1.ToString() + stderr)); NUnit.Framework.Assert.IsTrue(line.Contains(data + testContainerId1.ToString() + stdout)); }
/// <exception cref="System.IO.IOException"/> public virtual void TestContainerLogPageAccess() { // SecureIOUtils require Native IO to be enabled. This test will run // only if it is enabled. Assume.AssumeTrue(NativeIO.IsAvailable()); string user = "******" + Runtime.CurrentTimeMillis(); FilePath absLogDir = null; FilePath appDir = null; FilePath containerDir = null; FilePath syslog = null; try { // target log directory absLogDir = new FilePath("target", typeof(TestContainerLogsPage).Name + "LogDir") .GetAbsoluteFile(); absLogDir.Mkdir(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLogDirs, absLogDir.ToURI().ToString()); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.Init(conf); LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler(); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf); long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp , 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); // Making sure that application returns a random user. This is required // for SecureIOUtils' file owner check. Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId , 0); // Testing secure read access for log files // Creating application and container directory and syslog file. appDir = new FilePath(absLogDir, appId.ToString()); appDir.Mkdir(); containerDir = new FilePath(appDir, container1.ToString()); containerDir.Mkdir(); syslog = new FilePath(containerDir, "syslog"); syslog.CreateNewFile(); BufferedOutputStream @out = new BufferedOutputStream(new FileOutputStream(syslog) ); @out.Write(Sharpen.Runtime.GetBytesForString("Log file Content")); @out.Close(); Context context = Org.Mockito.Mockito.Mock <Context>(); ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application > appMap = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); appMap[appId] = app; Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(appMap); ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container > containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers); Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(dirsHandler); MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.SetState(ContainerState.Running); context.GetContainers()[container1] = container; ContainerLogsPage.ContainersLogsBlock cLogsBlock = new ContainerLogsPage.ContainersLogsBlock (context); IDictionary <string, string> @params = new Dictionary <string, string>(); @params[YarnWebParams.ContainerId] = container1.ToString(); @params[YarnWebParams.ContainerLogType] = "syslog"; Injector injector = WebAppTests.TestPage <ContainerLogsPage.ContainersLogsBlock>(typeof( ContainerLogsPage), cLogsBlock, @params, (Module[])null); PrintWriter spyPw = WebAppTests.GetPrintWriter(injector); Org.Mockito.Mockito.Verify(spyPw).Write("Exception reading log file. Application submitted by '" + user + "' doesn't own requested log file : syslog"); } finally { if (syslog != null) { syslog.Delete(); } if (containerDir != null) { containerDir.Delete(); } if (appDir != null) { appDir.Delete(); } if (absLogDir != null) { absLogDir.Delete(); } } }
/// <summary> /// Create new instance of <see cref="FileDetail"/> /// </summary> public FileDetail(PathInfo pathInfo) : this(pathInfo, NativeIO.GetFindDataFromPath(pathInfo)) { }
/// <summary> /// Create new instance of <see cref="DirectoryDetail"/> /// </summary> public DirectoryDetail(PathInfo pathInfo) : this(pathInfo, pathInfo.IsRoot ? null : NativeIO.GetFindDataFromPath(pathInfo)) { }