/// <summary>Read dataFile and compute its MD5 checksum.</summary> /// <exception cref="System.IO.IOException"/> public static MD5Hash ComputeMd5ForFile(FilePath dataFile) { InputStream @in = new FileInputStream(dataFile); try { MessageDigest digester = MD5Hash.GetDigester(); DigestInputStream dis = new DigestInputStream(@in, digester); IOUtils.CopyBytes(dis, new IOUtils.NullOutputStream(), 128 * 1024); return(new MD5Hash(digester.Digest())); } finally { IOUtils.CloseStream(@in); } }
/// <exception cref="System.IO.IOException"/> private static MD5Hash ReceiveFile(string url, IList <FilePath> localPaths, Storage dstStorage, bool getChecksum, long advertisedSize, MD5Hash advertisedDigest, string fsImageName, InputStream stream, DataTransferThrottler throttler) { long startTime = Time.MonotonicNow(); if (localPaths != null) { // If the local paths refer to directories, use the server-provided header // as the filename within that directory IList <FilePath> newLocalPaths = new AList <FilePath>(); foreach (FilePath localPath in localPaths) { if (localPath.IsDirectory()) { if (fsImageName == null) { throw new IOException("No filename header provided by server"); } newLocalPaths.AddItem(new FilePath(localPath, fsImageName)); } else { newLocalPaths.AddItem(localPath); } } localPaths = newLocalPaths; } long received = 0; MessageDigest digester = null; if (getChecksum) { digester = MD5Hash.GetDigester(); stream = new DigestInputStream(stream, digester); } bool finishedReceiving = false; IList <FileOutputStream> outputStreams = Lists.NewArrayList(); try { if (localPaths != null) { foreach (FilePath f in localPaths) { try { if (f.Exists()) { Log.Warn("Overwriting existing file " + f + " with file downloaded from " + url); } outputStreams.AddItem(new FileOutputStream(f)); } catch (IOException ioe) { Log.Warn("Unable to download file " + f, ioe); // This will be null if we're downloading the fsimage to a file // outside of an NNStorage directory. if (dstStorage != null && (dstStorage is StorageErrorReporter)) { ((StorageErrorReporter)dstStorage).ReportErrorOnFile(f); } } } if (outputStreams.IsEmpty()) { throw new IOException("Unable to download to any storage directory"); } } int num = 1; byte[] buf = new byte[HdfsConstants.IoFileBufferSize]; while (num > 0) { num = stream.Read(buf); if (num > 0) { received += num; foreach (FileOutputStream fos in outputStreams) { fos.Write(buf, 0, num); } if (throttler != null) { throttler.Throttle(num); } } } finishedReceiving = true; } finally { stream.Close(); foreach (FileOutputStream fos in outputStreams) { fos.GetChannel().Force(true); fos.Close(); } // Something went wrong and did not finish reading. // Remove the temporary files. if (!finishedReceiving) { DeleteTmpFiles(localPaths); } if (finishedReceiving && received != advertisedSize) { // only throw this exception if we think we read all of it on our end // -- otherwise a client-side IOException would be masked by this // exception that makes it look like a server-side problem! DeleteTmpFiles(localPaths); throw new IOException("File " + url + " received length " + received + " is not of the advertised size " + advertisedSize); } } double xferSec = Math.Max(((float)(Time.MonotonicNow() - startTime)) / 1000.0, 0.001 ); long xferKb = received / 1024; Log.Info(string.Format("Transfer took %.2fs at %.2f KB/s", xferSec, xferKb / xferSec )); if (digester != null) { MD5Hash computedDigest = new MD5Hash(digester.Digest()); if (advertisedDigest != null && !computedDigest.Equals(advertisedDigest)) { DeleteTmpFiles(localPaths); throw new IOException("File " + url + " computed digest " + computedDigest + " does not match advertised digest " + advertisedDigest); } return(computedDigest); } else { return(null); } }