/// <summary>Create empty edits logs file.</summary>
/// <exception cref="System.IO.IOException"/>
public override void Create(int layoutVersion)
{
fc.Truncate(0);
fc.Position(0);
WriteHeader(layoutVersion, doubleBuf.GetCurrentBuf());
SetReadyToFlush();
Flush();
}
/// <summary>
/// Return the length of bytes in the given file after subtracting
/// the trailer of 0xFF (OP_INVALID)s.
/// </summary>
/// <remarks>
/// Return the length of bytes in the given file after subtracting
/// the trailer of 0xFF (OP_INVALID)s.
/// This seeks to the end of the file and reads chunks backwards until
/// it finds a non-0xFF byte.
/// </remarks>
/// <exception cref="System.IO.IOException">if the file cannot be read</exception>
private static long GetNonTrailerLength(FilePath f)
{
int chunkSizeToRead = 256 * 1024;
FileInputStream fis = new FileInputStream(f);
try
{
byte[] buf = new byte[chunkSizeToRead];
FileChannel fc = fis.GetChannel();
long size = fc.Size();
long pos = size - (size % chunkSizeToRead);
while (pos >= 0)
{
fc.Position(pos);
int readLen = (int)Math.Min(size - pos, chunkSizeToRead);
IOUtils.ReadFully(fis, buf, 0, readLen);
for (int i = readLen - 1; i >= 0; i--)
{
if (buf[i] != FSEditLogOpCodes.OpInvalid.GetOpCode())
{
return(pos + i + 1);
}
}
// + 1 since we count this byte!
pos -= chunkSizeToRead;
}
return(0);
}
finally
{
fis.Close();
}
}
/// <exception cref="System.IO.IOException"/>
public void CommitSection(FsImageProto.FileSummary.Builder summary, FSImageFormatProtobuf.SectionName
name)
{
long oldOffset = currentOffset;
FlushSectionOutputStream();
if (codec != null)
{
sectionOutputStream = codec.CreateOutputStream(underlyingOutputStream);
}
else
{
sectionOutputStream = underlyingOutputStream;
}
long length = fileChannel.Position() - oldOffset;
summary.AddSections(FsImageProto.FileSummary.Section.NewBuilder().SetName(name.name
).SetLength(length).SetOffset(currentOffset));
currentOffset += length;
}
/// <summary>Creates output buffers and file object.</summary>
/// <param name="conf">Configuration object</param>
/// <param name="name">File name to store edit log</param>
/// <param name="size">Size of flush buffer</param>
/// <exception cref="System.IO.IOException"/>
public EditLogFileOutputStream(Configuration conf, FilePath name, int size)
: base()
{
shouldSyncWritesAndSkipFsync = conf.GetBoolean(DFSConfigKeys.DfsNamenodeEditsNoeditlogchannelflush
, DFSConfigKeys.DfsNamenodeEditsNoeditlogchannelflushDefault);
file = name;
doubleBuf = new EditsDoubleBuffer(size);
RandomAccessFile rp;
if (shouldSyncWritesAndSkipFsync)
{
rp = new RandomAccessFile(name, "rws");
}
else
{
rp = new RandomAccessFile(name, "rw");
}
fp = new FileOutputStream(rp.GetFD());
// open for append
fc = rp.GetChannel();
fc.Position(fc.Size());
}
/// <exception cref="System.IO.IOException"/>
private void LoadInternal(RandomAccessFile raFile, FileInputStream fin)
{
if (!FSImageUtil.CheckFileFormat(raFile))
{
throw new IOException("Unrecognized file format");
}
FsImageProto.FileSummary summary = FSImageUtil.LoadSummary(raFile);
if (requireSameLayoutVersion && summary.GetLayoutVersion() != HdfsConstants.NamenodeLayoutVersion)
{
throw new IOException("Image version " + summary.GetLayoutVersion() + " is not equal to the software version "
+ HdfsConstants.NamenodeLayoutVersion);
}
FileChannel channel = fin.GetChannel();
FSImageFormatPBINode.Loader inodeLoader = new FSImageFormatPBINode.Loader(fsn, this
);
FSImageFormatPBSnapshot.Loader snapshotLoader = new FSImageFormatPBSnapshot.Loader
(fsn, this);
AList <FsImageProto.FileSummary.Section> sections = Lists.NewArrayList(summary.GetSectionsList
());
sections.Sort(new _IComparer_210());
StartupProgress prog = NameNode.GetStartupProgress();
Step currentStep = null;
foreach (FsImageProto.FileSummary.Section s in sections)
{
channel.Position(s.GetOffset());
InputStream @in = new BufferedInputStream(new LimitInputStream(fin, s.GetLength()
));
@in = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec(), @in);
string n = s.GetName();
switch (FSImageFormatProtobuf.SectionName.FromString(n))
{
case FSImageFormatProtobuf.SectionName.NsInfo:
{
LoadNameSystemSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.StringTable:
{
LoadStringTableSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.Inode:
{
currentStep = new Step(StepType.Inodes);
prog.BeginStep(Phase.LoadingFsimage, currentStep);
inodeLoader.LoadINodeSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.InodeReference:
{
snapshotLoader.LoadINodeReferenceSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.InodeDir:
{
inodeLoader.LoadINodeDirectorySection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.FilesUnderconstruction:
{
inodeLoader.LoadFilesUnderConstructionSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.Snapshot:
{
snapshotLoader.LoadSnapshotSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.SnapshotDiff:
{
snapshotLoader.LoadSnapshotDiffSection(@in);
break;
}
case FSImageFormatProtobuf.SectionName.SecretManager:
{
prog.EndStep(Phase.LoadingFsimage, currentStep);
Step step = new Step(StepType.DelegationTokens);
prog.BeginStep(Phase.LoadingFsimage, step);
LoadSecretManagerSection(@in);
prog.EndStep(Phase.LoadingFsimage, step);
break;
}
case FSImageFormatProtobuf.SectionName.CacheManager:
{
Step step = new Step(StepType.CachePools);
prog.BeginStep(Phase.LoadingFsimage, step);
LoadCacheManagerSection(@in);
prog.EndStep(Phase.LoadingFsimage, step);
break;
}
default:
{
Log.Warn("Unrecognized section " + n);
break;
}
}
}
}
/// <exception cref="System.IO.IOException"/>
internal override int Run(IList <string> args)
{
if (args.Count == 0)
{
System.Console.Out.WriteLine(this.usageText);
System.Console.Out.WriteLine(this.helpText + "\n");
return(1);
}
string blockFile = StringUtils.PopOptionWithArgument("-block", args);
string metaFile = StringUtils.PopOptionWithArgument("-meta", args);
if (metaFile == null)
{
System.Console.Error.WriteLine("You must specify a meta file with -meta");
return(1);
}
FileInputStream metaStream = null;
FileInputStream dataStream = null;
FileChannel metaChannel = null;
FileChannel dataChannel = null;
DataInputStream checksumStream = null;
try
{
BlockMetadataHeader header;
try
{
metaStream = new FileInputStream(metaFile);
checksumStream = new DataInputStream(metaStream);
header = BlockMetadataHeader.ReadHeader(checksumStream);
metaChannel = metaStream.GetChannel();
metaChannel.Position(DebugAdmin.HeaderLen);
}
catch (RuntimeException e)
{
System.Console.Error.WriteLine("Failed to read HDFS metadata file header for " +
metaFile + ": " + StringUtils.StringifyException(e));
return(1);
}
catch (IOException e)
{
System.Console.Error.WriteLine("Failed to read HDFS metadata file header for " +
metaFile + ": " + StringUtils.StringifyException(e));
return(1);
}
DataChecksum checksum = header.GetChecksum();
System.Console.Out.WriteLine("Checksum type: " + checksum.ToString());
if (blockFile == null)
{
return(0);
}
ByteBuffer metaBuf;
ByteBuffer dataBuf;
try
{
dataStream = new FileInputStream(blockFile);
dataChannel = dataStream.GetChannel();
int ChecksumsPerBuf = 1024 * 32;
metaBuf = ByteBuffer.Allocate(checksum.GetChecksumSize() * ChecksumsPerBuf);
dataBuf = ByteBuffer.Allocate(checksum.GetBytesPerChecksum() * ChecksumsPerBuf);
}
catch (IOException e)
{
System.Console.Error.WriteLine("Failed to open HDFS block file for " + blockFile
+ ": " + StringUtils.StringifyException(e));
return(1);
}
long offset = 0;
while (true)
{
dataBuf.Clear();
int dataRead = -1;
try
{
dataRead = dataChannel.Read(dataBuf);
if (dataRead < 0)
{
break;
}
}
catch (IOException e)
{
System.Console.Error.WriteLine("Got I/O error reading block file " + blockFile +
"from disk at offset " + dataChannel.Position() + ": " + StringUtils.StringifyException
(e));
return(1);
}
try
{
int csumToRead = (((checksum.GetBytesPerChecksum() - 1) + dataRead) / checksum.GetBytesPerChecksum
()) * checksum.GetChecksumSize();
metaBuf.Clear();
metaBuf.Limit(csumToRead);
metaChannel.Read(metaBuf);
dataBuf.Flip();
metaBuf.Flip();
}
catch (IOException e)
{
System.Console.Error.WriteLine("Got I/O error reading metadata file " + metaFile
+ "from disk at offset " + metaChannel.Position() + ": " + StringUtils.StringifyException
(e));
return(1);
}
try
{
checksum.VerifyChunkedSums(dataBuf, metaBuf, blockFile, offset);
}
catch (IOException e)
{
System.Console.Out.WriteLine("verifyChunkedSums error: " + StringUtils.StringifyException
(e));
return(1);
}
offset += dataRead;
}
System.Console.Out.WriteLine("Checksum verification succeeded on block file " + blockFile
);
return(0);
}
finally
{
IOUtils.Cleanup(null, metaStream, dataStream, checksumStream);
}
}
/// <exception cref="System.IO.IOException"/>
private long DoSendBlock(DataOutputStream @out, OutputStream baseStream, DataTransferThrottler
throttler)
{
if (@out == null)
{
throw new IOException("out stream is null");
}
initialOffset = offset;
long totalRead = 0;
OutputStream streamForSendChunks = @out;
lastCacheDropOffset = initialOffset;
if (IsLongRead() && blockInFd != null)
{
// Advise that this file descriptor will be accessed sequentially.
NativeIO.POSIX.GetCacheManipulator().PosixFadviseIfPossible(block.GetBlockName(),
blockInFd, 0, 0, NativeIO.POSIX.PosixFadvSequential);
}
// Trigger readahead of beginning of file if configured.
ManageOsCache();
long startTime = ClientTraceLog.IsDebugEnabled() ? Runtime.NanoTime() : 0;
try
{
int maxChunksPerPacket;
int pktBufSize = PacketHeader.PktMaxHeaderLen;
bool transferTo = transferToAllowed && !verifyChecksum && baseStream is SocketOutputStream &&
blockIn is FileInputStream;
if (transferTo)
{
FileChannel fileChannel = ((FileInputStream)blockIn).GetChannel();
blockInPosition = fileChannel.Position();
streamForSendChunks = baseStream;
maxChunksPerPacket = NumberOfChunks(TransfertoBufferSize);
// Smaller packet size to only hold checksum when doing transferTo
pktBufSize += checksumSize * maxChunksPerPacket;
}
else
{
maxChunksPerPacket = Math.Max(1, NumberOfChunks(HdfsConstants.IoFileBufferSize));
// Packet size includes both checksum and data
pktBufSize += (chunkSize + checksumSize) * maxChunksPerPacket;
}
ByteBuffer pktBuf = ByteBuffer.Allocate(pktBufSize);
while (endOffset > offset && !Sharpen.Thread.CurrentThread().IsInterrupted())
{
ManageOsCache();
long len = SendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, transferTo
, throttler);
offset += len;
totalRead += len + (NumberOfChunks(len) * checksumSize);
seqno++;
}
// If this thread was interrupted, then it did not send the full block.
if (!Sharpen.Thread.CurrentThread().IsInterrupted())
{
try
{
// send an empty packet to mark the end of the block
SendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, transferTo, throttler
);
@out.Flush();
}
catch (IOException e)
{
//socket error
throw IoeToSocketException(e);
}
sentEntireByteRange = true;
}
}
finally
{
if ((clientTraceFmt != null) && ClientTraceLog.IsDebugEnabled())
{
long endTime = Runtime.NanoTime();
ClientTraceLog.Debug(string.Format(clientTraceFmt, totalRead, initialOffset, endTime
- startTime));
}
Close();
}
return(totalRead);
}