/// <summary>Create empty edits logs file.</summary> /// <exception cref="System.IO.IOException"/> public override void Create(int layoutVersion) { fc.Truncate(0); fc.Position(0); WriteHeader(layoutVersion, doubleBuf.GetCurrentBuf()); SetReadyToFlush(); Flush(); }
/// <summary> /// Return the length of bytes in the given file after subtracting /// the trailer of 0xFF (OP_INVALID)s. /// </summary> /// <remarks> /// Return the length of bytes in the given file after subtracting /// the trailer of 0xFF (OP_INVALID)s. /// This seeks to the end of the file and reads chunks backwards until /// it finds a non-0xFF byte. /// </remarks> /// <exception cref="System.IO.IOException">if the file cannot be read</exception> private static long GetNonTrailerLength(FilePath f) { int chunkSizeToRead = 256 * 1024; FileInputStream fis = new FileInputStream(f); try { byte[] buf = new byte[chunkSizeToRead]; FileChannel fc = fis.GetChannel(); long size = fc.Size(); long pos = size - (size % chunkSizeToRead); while (pos >= 0) { fc.Position(pos); int readLen = (int)Math.Min(size - pos, chunkSizeToRead); IOUtils.ReadFully(fis, buf, 0, readLen); for (int i = readLen - 1; i >= 0; i--) { if (buf[i] != FSEditLogOpCodes.OpInvalid.GetOpCode()) { return(pos + i + 1); } } // + 1 since we count this byte! pos -= chunkSizeToRead; } return(0); } finally { fis.Close(); } }
/// <exception cref="System.IO.IOException"/> public void CommitSection(FsImageProto.FileSummary.Builder summary, FSImageFormatProtobuf.SectionName name) { long oldOffset = currentOffset; FlushSectionOutputStream(); if (codec != null) { sectionOutputStream = codec.CreateOutputStream(underlyingOutputStream); } else { sectionOutputStream = underlyingOutputStream; } long length = fileChannel.Position() - oldOffset; summary.AddSections(FsImageProto.FileSummary.Section.NewBuilder().SetName(name.name ).SetLength(length).SetOffset(currentOffset)); currentOffset += length; }
/// <summary>Creates output buffers and file object.</summary> /// <param name="conf">Configuration object</param> /// <param name="name">File name to store edit log</param> /// <param name="size">Size of flush buffer</param> /// <exception cref="System.IO.IOException"/> public EditLogFileOutputStream(Configuration conf, FilePath name, int size) : base() { shouldSyncWritesAndSkipFsync = conf.GetBoolean(DFSConfigKeys.DfsNamenodeEditsNoeditlogchannelflush , DFSConfigKeys.DfsNamenodeEditsNoeditlogchannelflushDefault); file = name; doubleBuf = new EditsDoubleBuffer(size); RandomAccessFile rp; if (shouldSyncWritesAndSkipFsync) { rp = new RandomAccessFile(name, "rws"); } else { rp = new RandomAccessFile(name, "rw"); } fp = new FileOutputStream(rp.GetFD()); // open for append fc = rp.GetChannel(); fc.Position(fc.Size()); }
/// <exception cref="System.IO.IOException"/> private void LoadInternal(RandomAccessFile raFile, FileInputStream fin) { if (!FSImageUtil.CheckFileFormat(raFile)) { throw new IOException("Unrecognized file format"); } FsImageProto.FileSummary summary = FSImageUtil.LoadSummary(raFile); if (requireSameLayoutVersion && summary.GetLayoutVersion() != HdfsConstants.NamenodeLayoutVersion) { throw new IOException("Image version " + summary.GetLayoutVersion() + " is not equal to the software version " + HdfsConstants.NamenodeLayoutVersion); } FileChannel channel = fin.GetChannel(); FSImageFormatPBINode.Loader inodeLoader = new FSImageFormatPBINode.Loader(fsn, this ); FSImageFormatPBSnapshot.Loader snapshotLoader = new FSImageFormatPBSnapshot.Loader (fsn, this); AList <FsImageProto.FileSummary.Section> sections = Lists.NewArrayList(summary.GetSectionsList ()); sections.Sort(new _IComparer_210()); StartupProgress prog = NameNode.GetStartupProgress(); Step currentStep = null; foreach (FsImageProto.FileSummary.Section s in sections) { channel.Position(s.GetOffset()); InputStream @in = new BufferedInputStream(new LimitInputStream(fin, s.GetLength() )); @in = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec(), @in); string n = s.GetName(); switch (FSImageFormatProtobuf.SectionName.FromString(n)) { case FSImageFormatProtobuf.SectionName.NsInfo: { LoadNameSystemSection(@in); break; } case FSImageFormatProtobuf.SectionName.StringTable: { LoadStringTableSection(@in); break; } case FSImageFormatProtobuf.SectionName.Inode: { currentStep = new Step(StepType.Inodes); prog.BeginStep(Phase.LoadingFsimage, currentStep); inodeLoader.LoadINodeSection(@in); break; } case FSImageFormatProtobuf.SectionName.InodeReference: { snapshotLoader.LoadINodeReferenceSection(@in); break; } case FSImageFormatProtobuf.SectionName.InodeDir: { inodeLoader.LoadINodeDirectorySection(@in); break; } case FSImageFormatProtobuf.SectionName.FilesUnderconstruction: { inodeLoader.LoadFilesUnderConstructionSection(@in); break; } case FSImageFormatProtobuf.SectionName.Snapshot: { snapshotLoader.LoadSnapshotSection(@in); break; } case FSImageFormatProtobuf.SectionName.SnapshotDiff: { snapshotLoader.LoadSnapshotDiffSection(@in); break; } case FSImageFormatProtobuf.SectionName.SecretManager: { prog.EndStep(Phase.LoadingFsimage, currentStep); Step step = new Step(StepType.DelegationTokens); prog.BeginStep(Phase.LoadingFsimage, step); LoadSecretManagerSection(@in); prog.EndStep(Phase.LoadingFsimage, step); break; } case FSImageFormatProtobuf.SectionName.CacheManager: { Step step = new Step(StepType.CachePools); prog.BeginStep(Phase.LoadingFsimage, step); LoadCacheManagerSection(@in); prog.EndStep(Phase.LoadingFsimage, step); break; } default: { Log.Warn("Unrecognized section " + n); break; } } } }
/// <exception cref="System.IO.IOException"/> internal override int Run(IList <string> args) { if (args.Count == 0) { System.Console.Out.WriteLine(this.usageText); System.Console.Out.WriteLine(this.helpText + "\n"); return(1); } string blockFile = StringUtils.PopOptionWithArgument("-block", args); string metaFile = StringUtils.PopOptionWithArgument("-meta", args); if (metaFile == null) { System.Console.Error.WriteLine("You must specify a meta file with -meta"); return(1); } FileInputStream metaStream = null; FileInputStream dataStream = null; FileChannel metaChannel = null; FileChannel dataChannel = null; DataInputStream checksumStream = null; try { BlockMetadataHeader header; try { metaStream = new FileInputStream(metaFile); checksumStream = new DataInputStream(metaStream); header = BlockMetadataHeader.ReadHeader(checksumStream); metaChannel = metaStream.GetChannel(); metaChannel.Position(DebugAdmin.HeaderLen); } catch (RuntimeException e) { System.Console.Error.WriteLine("Failed to read HDFS metadata file header for " + metaFile + ": " + StringUtils.StringifyException(e)); return(1); } catch (IOException e) { System.Console.Error.WriteLine("Failed to read HDFS metadata file header for " + metaFile + ": " + StringUtils.StringifyException(e)); return(1); } DataChecksum checksum = header.GetChecksum(); System.Console.Out.WriteLine("Checksum type: " + checksum.ToString()); if (blockFile == null) { return(0); } ByteBuffer metaBuf; ByteBuffer dataBuf; try { dataStream = new FileInputStream(blockFile); dataChannel = dataStream.GetChannel(); int ChecksumsPerBuf = 1024 * 32; metaBuf = ByteBuffer.Allocate(checksum.GetChecksumSize() * ChecksumsPerBuf); dataBuf = ByteBuffer.Allocate(checksum.GetBytesPerChecksum() * ChecksumsPerBuf); } catch (IOException e) { System.Console.Error.WriteLine("Failed to open HDFS block file for " + blockFile + ": " + StringUtils.StringifyException(e)); return(1); } long offset = 0; while (true) { dataBuf.Clear(); int dataRead = -1; try { dataRead = dataChannel.Read(dataBuf); if (dataRead < 0) { break; } } catch (IOException e) { System.Console.Error.WriteLine("Got I/O error reading block file " + blockFile + "from disk at offset " + dataChannel.Position() + ": " + StringUtils.StringifyException (e)); return(1); } try { int csumToRead = (((checksum.GetBytesPerChecksum() - 1) + dataRead) / checksum.GetBytesPerChecksum ()) * checksum.GetChecksumSize(); metaBuf.Clear(); metaBuf.Limit(csumToRead); metaChannel.Read(metaBuf); dataBuf.Flip(); metaBuf.Flip(); } catch (IOException e) { System.Console.Error.WriteLine("Got I/O error reading metadata file " + metaFile + "from disk at offset " + metaChannel.Position() + ": " + StringUtils.StringifyException (e)); return(1); } try { checksum.VerifyChunkedSums(dataBuf, metaBuf, blockFile, offset); } catch (IOException e) { System.Console.Out.WriteLine("verifyChunkedSums error: " + StringUtils.StringifyException (e)); return(1); } offset += dataRead; } System.Console.Out.WriteLine("Checksum verification succeeded on block file " + blockFile ); return(0); } finally { IOUtils.Cleanup(null, metaStream, dataStream, checksumStream); } }
/// <exception cref="System.IO.IOException"/> private long DoSendBlock(DataOutputStream @out, OutputStream baseStream, DataTransferThrottler throttler) { if (@out == null) { throw new IOException("out stream is null"); } initialOffset = offset; long totalRead = 0; OutputStream streamForSendChunks = @out; lastCacheDropOffset = initialOffset; if (IsLongRead() && blockInFd != null) { // Advise that this file descriptor will be accessed sequentially. NativeIO.POSIX.GetCacheManipulator().PosixFadviseIfPossible(block.GetBlockName(), blockInFd, 0, 0, NativeIO.POSIX.PosixFadvSequential); } // Trigger readahead of beginning of file if configured. ManageOsCache(); long startTime = ClientTraceLog.IsDebugEnabled() ? Runtime.NanoTime() : 0; try { int maxChunksPerPacket; int pktBufSize = PacketHeader.PktMaxHeaderLen; bool transferTo = transferToAllowed && !verifyChecksum && baseStream is SocketOutputStream && blockIn is FileInputStream; if (transferTo) { FileChannel fileChannel = ((FileInputStream)blockIn).GetChannel(); blockInPosition = fileChannel.Position(); streamForSendChunks = baseStream; maxChunksPerPacket = NumberOfChunks(TransfertoBufferSize); // Smaller packet size to only hold checksum when doing transferTo pktBufSize += checksumSize * maxChunksPerPacket; } else { maxChunksPerPacket = Math.Max(1, NumberOfChunks(HdfsConstants.IoFileBufferSize)); // Packet size includes both checksum and data pktBufSize += (chunkSize + checksumSize) * maxChunksPerPacket; } ByteBuffer pktBuf = ByteBuffer.Allocate(pktBufSize); while (endOffset > offset && !Sharpen.Thread.CurrentThread().IsInterrupted()) { ManageOsCache(); long len = SendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, transferTo , throttler); offset += len; totalRead += len + (NumberOfChunks(len) * checksumSize); seqno++; } // If this thread was interrupted, then it did not send the full block. if (!Sharpen.Thread.CurrentThread().IsInterrupted()) { try { // send an empty packet to mark the end of the block SendPacket(pktBuf, maxChunksPerPacket, streamForSendChunks, transferTo, throttler ); @out.Flush(); } catch (IOException e) { //socket error throw IoeToSocketException(e); } sentEntireByteRange = true; } } finally { if ((clientTraceFmt != null) && ClientTraceLog.IsDebugEnabled()) { long endTime = Runtime.NanoTime(); ClientTraceLog.Debug(string.Format(clientTraceFmt, totalRead, initialOffset, endTime - startTime)); } Close(); } return(totalRead); }