/// <exception cref="System.IO.IOException"/>
private void SaveInternal(FileOutputStream fout, FSImageCompression compression,
string filePath)
{
StartupProgress prog = NameNode.GetStartupProgress();
MessageDigest digester = MD5Hash.GetDigester();
underlyingOutputStream = new DigestOutputStream(new BufferedOutputStream(fout), digester
);
underlyingOutputStream.Write(FSImageUtil.MagicHeader);
fileChannel = fout.GetChannel();
FsImageProto.FileSummary.Builder b = FsImageProto.FileSummary.NewBuilder().SetOndiskVersion
(FSImageUtil.FileVersion).SetLayoutVersion(NameNodeLayoutVersion.CurrentLayoutVersion
);
codec = compression.GetImageCodec();
if (codec != null)
{
b.SetCodec(codec.GetType().GetCanonicalName());
sectionOutputStream = codec.CreateOutputStream(underlyingOutputStream);
}
else
{
sectionOutputStream = underlyingOutputStream;
}
SaveNameSystemSection(b);
// Check for cancellation right after serializing the name system section.
// Some unit tests, such as TestSaveNamespace#testCancelSaveNameSpace
// depends on this behavior.
context.CheckCancelled();
Step step = new Step(StepType.Inodes, filePath);
prog.BeginStep(Phase.SavingCheckpoint, step);
SaveInodes(b);
SaveSnapshots(b);
prog.EndStep(Phase.SavingCheckpoint, step);
step = new Step(StepType.DelegationTokens, filePath);
prog.BeginStep(Phase.SavingCheckpoint, step);
SaveSecretManagerSection(b);
prog.EndStep(Phase.SavingCheckpoint, step);
step = new Step(StepType.CachePools, filePath);
prog.BeginStep(Phase.SavingCheckpoint, step);
SaveCacheManagerSection(b);
prog.EndStep(Phase.SavingCheckpoint, step);
SaveStringTableSection(b);
// We use the underlyingOutputStream to write the header. Therefore flush
// the buffered stream (which is potentially compressed) first.
FlushSectionOutputStream();
FsImageProto.FileSummary summary = ((FsImageProto.FileSummary)b.Build());
SaveFileSummary(underlyingOutputStream, summary);
underlyingOutputStream.Close();
savedDigest = new MD5Hash(digester.Digest());
}
/// <summary>Read dataFile and compute its MD5 checksum.</summary>
/// <exception cref="System.IO.IOException"/>
public static MD5Hash ComputeMd5ForFile(FilePath dataFile)
{
InputStream @in = new FileInputStream(dataFile);
try
{
MessageDigest digester = MD5Hash.GetDigester();
DigestInputStream dis = new DigestInputStream(@in, digester);
IOUtils.CopyBytes(dis, new IOUtils.NullOutputStream(), 128 * 1024);
return(new MD5Hash(digester.Digest()));
}
finally
{
IOUtils.CloseStream(@in);
}
}
/// <exception cref="System.IO.IOException"/>
private static MD5Hash ReceiveFile(string url, IList <FilePath> localPaths, Storage
dstStorage, bool getChecksum, long advertisedSize, MD5Hash advertisedDigest, string
fsImageName, InputStream stream, DataTransferThrottler throttler)
{
long startTime = Time.MonotonicNow();
if (localPaths != null)
{
// If the local paths refer to directories, use the server-provided header
// as the filename within that directory
IList <FilePath> newLocalPaths = new AList <FilePath>();
foreach (FilePath localPath in localPaths)
{
if (localPath.IsDirectory())
{
if (fsImageName == null)
{
throw new IOException("No filename header provided by server");
}
newLocalPaths.AddItem(new FilePath(localPath, fsImageName));
}
else
{
newLocalPaths.AddItem(localPath);
}
}
localPaths = newLocalPaths;
}
long received = 0;
MessageDigest digester = null;
if (getChecksum)
{
digester = MD5Hash.GetDigester();
stream = new DigestInputStream(stream, digester);
}
bool finishedReceiving = false;
IList <FileOutputStream> outputStreams = Lists.NewArrayList();
try
{
if (localPaths != null)
{
foreach (FilePath f in localPaths)
{
try
{
if (f.Exists())
{
Log.Warn("Overwriting existing file " + f + " with file downloaded from " + url);
}
outputStreams.AddItem(new FileOutputStream(f));
}
catch (IOException ioe)
{
Log.Warn("Unable to download file " + f, ioe);
// This will be null if we're downloading the fsimage to a file
// outside of an NNStorage directory.
if (dstStorage != null && (dstStorage is StorageErrorReporter))
{
((StorageErrorReporter)dstStorage).ReportErrorOnFile(f);
}
}
}
if (outputStreams.IsEmpty())
{
throw new IOException("Unable to download to any storage directory");
}
}
int num = 1;
byte[] buf = new byte[HdfsConstants.IoFileBufferSize];
while (num > 0)
{
num = stream.Read(buf);
if (num > 0)
{
received += num;
foreach (FileOutputStream fos in outputStreams)
{
fos.Write(buf, 0, num);
}
if (throttler != null)
{
throttler.Throttle(num);
}
}
}
finishedReceiving = true;
}
finally
{
stream.Close();
foreach (FileOutputStream fos in outputStreams)
{
fos.GetChannel().Force(true);
fos.Close();
}
// Something went wrong and did not finish reading.
// Remove the temporary files.
if (!finishedReceiving)
{
DeleteTmpFiles(localPaths);
}
if (finishedReceiving && received != advertisedSize)
{
// only throw this exception if we think we read all of it on our end
// -- otherwise a client-side IOException would be masked by this
// exception that makes it look like a server-side problem!
DeleteTmpFiles(localPaths);
throw new IOException("File " + url + " received length " + received + " is not of the advertised size "
+ advertisedSize);
}
}
double xferSec = Math.Max(((float)(Time.MonotonicNow() - startTime)) / 1000.0, 0.001
);
long xferKb = received / 1024;
Log.Info(string.Format("Transfer took %.2fs at %.2f KB/s", xferSec, xferKb / xferSec
));
if (digester != null)
{
MD5Hash computedDigest = new MD5Hash(digester.Digest());
if (advertisedDigest != null && !computedDigest.Equals(advertisedDigest))
{
DeleteTmpFiles(localPaths);
throw new IOException("File " + url + " computed digest " + computedDigest + " does not match advertised digest "
+ advertisedDigest);
}
return(computedDigest);
}
else
{
return(null);
}
}
