public virtual void TestDigest() { Configuration conf = new Configuration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); DistributedFileSystem fs = cluster.GetFileSystem(); fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); fs.SaveNamespace(); fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); FilePath currentDir = FSImageTestUtil.GetNameNodeCurrentDirs(cluster, 0)[0]; FilePath fsimage = FSImageTestUtil.FindNewestImageFile(currentDir.GetAbsolutePath ()); NUnit.Framework.Assert.AreEqual(MD5FileUtils.ReadStoredMd5ForFile(fsimage), MD5FileUtils .ComputeMd5ForFile(fsimage)); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary>Set headers for content length, and, if available, md5.</summary> /// <exception cref="System.IO.IOException"></exception> public static void SetVerificationHeadersForGet(HttpServletResponse response, FilePath file) { response.SetHeader(TransferFsImage.ContentLength, file.Length().ToString()); MD5Hash hash = MD5FileUtils.ReadStoredMd5ForFile(file); if (hash != null) { response.SetHeader(TransferFsImage.Md5Header, hash.ToString()); } }
/// <summary>Set headers for image length and if available, md5.</summary> /// <exception cref="System.IO.IOException"/> internal static void SetVerificationHeadersForPut(HttpURLConnection connection, FilePath file) { connection.SetRequestProperty(TransferFsImage.ContentLength, file.Length().ToString ()); MD5Hash hash = MD5FileUtils.ReadStoredMd5ForFile(file); if (hash != null) { connection.SetRequestProperty(TransferFsImage.Md5Header, hash.ToString()); } }
/// <summary> /// Run `hdfs dfsadmin -fetchImage ...' and verify that the downloaded image is /// correct. /// </summary> /// <exception cref="System.Exception"/> private static void RunFetchImage(DFSAdmin dfsAdmin, MiniDFSCluster cluster) { int retVal = dfsAdmin.Run(new string[] { "-fetchImage", FetchedImageFile.GetPath( ) }); NUnit.Framework.Assert.AreEqual(0, retVal); FilePath highestImageOnNn = GetHighestFsImageOnCluster(cluster); MD5Hash expected = MD5FileUtils.ComputeMd5ForFile(highestImageOnNn); MD5Hash actual = MD5FileUtils.ComputeMd5ForFile(new FilePath(FetchedImageFile, highestImageOnNn .GetName())); NUnit.Framework.Assert.AreEqual(expected, actual); }
/// <exception cref="System.IO.IOException"/> internal void Load(FilePath file) { long start = Time.MonotonicNow(); imgDigest = MD5FileUtils.ComputeMd5ForFile(file); RandomAccessFile raFile = new RandomAccessFile(file, "r"); FileInputStream fin = new FileInputStream(file); try { LoadInternal(raFile, fin); long end = Time.MonotonicNow(); Log.Info("Loaded FSImage in " + (end - start) / 1000 + " seconds."); } finally { fin.Close(); raFile.Close(); } }
/// <summary>Corrupts the MD5 sum of the fsimage.</summary> /// <param name="corruptAll"> /// whether to corrupt one or all of the MD5 sums in the configured /// namedirs /// </param> /// <exception cref="System.IO.IOException"/> private void CorruptFSImageMD5(bool corruptAll) { IList <URI> nameDirs = (IList <URI>)FSNamesystem.GetNamespaceDirs(config); // Corrupt the md5 files in all the namedirs foreach (URI uri in nameDirs) { // Directory layout looks like: // test/data/dfs/nameN/current/{fsimage,edits,...} FilePath nameDir = new FilePath(uri.GetPath()); FilePath dfsDir = nameDir.GetParentFile(); NUnit.Framework.Assert.AreEqual(dfsDir.GetName(), "dfs"); // make sure we got right dir // Set the md5 file to all zeros FilePath imageFile = new FilePath(nameDir, Storage.StorageDirCurrent + "/" + NNStorage .GetImageFileName(0)); MD5FileUtils.SaveMD5File(imageFile, new MD5Hash(new byte[16])); // Only need to corrupt one if !corruptAll if (!corruptAll) { break; } } }
public virtual void PurgeImage(FSImageStorageInspector.FSImageFile image) { Log.Info("Purging old image " + image); DeleteOrWarn(image.GetFile()); DeleteOrWarn(MD5FileUtils.GetDigestFileForFile(image.GetFile())); }
/// <summary>This function returns a md5 hash of a file.</summary> /// <param name="file">input file</param> /// <returns>The md5 string</returns> /// <exception cref="System.IO.IOException"/> public static string GetFileMD5(FilePath file) { return(MD5FileUtils.ComputeMd5ForFile(file).ToString()); }