public virtual void TestDigest()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
try
{
cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build();
DistributedFileSystem fs = cluster.GetFileSystem();
fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter);
fs.SaveNamespace();
fs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave);
FilePath currentDir = FSImageTestUtil.GetNameNodeCurrentDirs(cluster, 0)[0];
FilePath fsimage = FSImageTestUtil.FindNewestImageFile(currentDir.GetAbsolutePath
());
NUnit.Framework.Assert.AreEqual(MD5FileUtils.ReadStoredMd5ForFile(fsimage), MD5FileUtils
.ComputeMd5ForFile(fsimage));
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
/// <summary>Set headers for content length, and, if available, md5.</summary>
/// <exception cref="System.IO.IOException"></exception>
public static void SetVerificationHeadersForGet(HttpServletResponse response, FilePath
file)
{
response.SetHeader(TransferFsImage.ContentLength, file.Length().ToString());
MD5Hash hash = MD5FileUtils.ReadStoredMd5ForFile(file);
if (hash != null)
{
response.SetHeader(TransferFsImage.Md5Header, hash.ToString());
}
}
/// <summary>Set headers for image length and if available, md5.</summary>
/// <exception cref="System.IO.IOException"/>
internal static void SetVerificationHeadersForPut(HttpURLConnection connection, FilePath
file)
{
connection.SetRequestProperty(TransferFsImage.ContentLength, file.Length().ToString
());
MD5Hash hash = MD5FileUtils.ReadStoredMd5ForFile(file);
if (hash != null)
{
connection.SetRequestProperty(TransferFsImage.Md5Header, hash.ToString());
}
}
/// <summary>
/// Run `hdfs dfsadmin -fetchImage ...' and verify that the downloaded image is
/// correct.
/// </summary>
/// <exception cref="System.Exception"/>
private static void RunFetchImage(DFSAdmin dfsAdmin, MiniDFSCluster cluster)
{
int retVal = dfsAdmin.Run(new string[] { "-fetchImage", FetchedImageFile.GetPath(
) });
NUnit.Framework.Assert.AreEqual(0, retVal);
FilePath highestImageOnNn = GetHighestFsImageOnCluster(cluster);
MD5Hash expected = MD5FileUtils.ComputeMd5ForFile(highestImageOnNn);
MD5Hash actual = MD5FileUtils.ComputeMd5ForFile(new FilePath(FetchedImageFile, highestImageOnNn
.GetName()));
NUnit.Framework.Assert.AreEqual(expected, actual);
}
/// <exception cref="System.IO.IOException"/>
internal void Load(FilePath file)
{
long start = Time.MonotonicNow();
imgDigest = MD5FileUtils.ComputeMd5ForFile(file);
RandomAccessFile raFile = new RandomAccessFile(file, "r");
FileInputStream fin = new FileInputStream(file);
try
{
LoadInternal(raFile, fin);
long end = Time.MonotonicNow();
Log.Info("Loaded FSImage in " + (end - start) / 1000 + " seconds.");
}
finally
{
fin.Close();
raFile.Close();
}
}
/// <summary>Corrupts the MD5 sum of the fsimage.</summary>
/// <param name="corruptAll">
/// whether to corrupt one or all of the MD5 sums in the configured
/// namedirs
/// </param>
/// <exception cref="System.IO.IOException"/>
private void CorruptFSImageMD5(bool corruptAll)
{
IList <URI> nameDirs = (IList <URI>)FSNamesystem.GetNamespaceDirs(config);
// Corrupt the md5 files in all the namedirs
foreach (URI uri in nameDirs)
{
// Directory layout looks like:
// test/data/dfs/nameN/current/{fsimage,edits,...}
FilePath nameDir = new FilePath(uri.GetPath());
FilePath dfsDir = nameDir.GetParentFile();
NUnit.Framework.Assert.AreEqual(dfsDir.GetName(), "dfs");
// make sure we got right dir
// Set the md5 file to all zeros
FilePath imageFile = new FilePath(nameDir, Storage.StorageDirCurrent + "/" + NNStorage
.GetImageFileName(0));
MD5FileUtils.SaveMD5File(imageFile, new MD5Hash(new byte[16]));
// Only need to corrupt one if !corruptAll
if (!corruptAll)
{
break;
}
}
}
public virtual void PurgeImage(FSImageStorageInspector.FSImageFile image)
{
Log.Info("Purging old image " + image);
DeleteOrWarn(image.GetFile());
DeleteOrWarn(MD5FileUtils.GetDigestFileForFile(image.GetFile()));
}
/// <summary>This function returns a md5 hash of a file.</summary>
/// <param name="file">input file</param>
/// <returns>The md5 string</returns>
/// <exception cref="System.IO.IOException"/>
public static string GetFileMD5(FilePath file)
{
return(MD5FileUtils.ComputeMd5ForFile(file).ToString());
}
