예제 #1
0
        /// <summary>Reads bytes into a buffer until EOF or the buffer's limit is reached</summary>
        /// <exception cref="System.IO.IOException"/>
        private int FillBuffer(FileInputStream stream, ByteBuffer buf)
        {
            TraceScope scope = Trace.StartSpan("BlockReaderLocalLegacy#fillBuffer(" + blockId
                                               + ")", Sampler.Never);

            try
            {
                int bytesRead = stream.GetChannel().Read(buf);
                if (bytesRead < 0)
                {
                    //EOF
                    return(bytesRead);
                }
                while (buf.Remaining() > 0)
                {
                    int n = stream.GetChannel().Read(buf);
                    if (n < 0)
                    {
                        //EOF
                        return(bytesRead);
                    }
                    bytesRead += n;
                }
                return(bytesRead);
            }
            finally
            {
                scope.Close();
            }
        }
예제 #2
0
        /// <exception cref="System.Exception"/>
        public virtual void TestMlock()
        {
            Assume.AssumeTrue(NativeIO.IsAvailable());
            FilePath TestFile = new FilePath(new FilePath(Runtime.GetProperty("test.build.data"
                                                                              , "build/test/data")), "testMlockFile");
            int BufLen = 12289;

            byte[] buf    = new byte[BufLen];
            int    bufSum = 0;

            for (int i = 0; i < buf.Length; i++)
            {
                buf[i]  = unchecked ((byte)(i % 60));
                bufSum += buf[i];
            }
            FileOutputStream fos = new FileOutputStream(TestFile);

            try
            {
                fos.Write(buf);
                fos.GetChannel().Force(true);
            }
            finally
            {
                fos.Close();
            }
            FileInputStream fis     = null;
            FileChannel     channel = null;

            try
            {
                // Map file into memory
                fis     = new FileInputStream(TestFile);
                channel = fis.GetChannel();
                long             fileSize = channel.Size();
                MappedByteBuffer mapbuf   = channel.Map(FileChannel.MapMode.ReadOnly, 0, fileSize);
                // mlock the buffer
                NativeIO.POSIX.Mlock(mapbuf, fileSize);
                // Read the buffer
                int sum = 0;
                for (int i_1 = 0; i_1 < fileSize; i_1++)
                {
                    sum += mapbuf.Get(i_1);
                }
                Assert.Equal("Expected sums to be equal", bufSum, sum);
                // munmap the buffer, which also implicitly unlocks it
                NativeIO.POSIX.Munmap(mapbuf);
            }
            finally
            {
                if (channel != null)
                {
                    channel.Close();
                }
                if (fis != null)
                {
                    fis.Close();
                }
            }
        }
예제 #3
0
        /// <summary>
        /// Return the length of bytes in the given file after subtracting
        /// the trailer of 0xFF (OP_INVALID)s.
        /// </summary>
        /// <remarks>
        /// Return the length of bytes in the given file after subtracting
        /// the trailer of 0xFF (OP_INVALID)s.
        /// This seeks to the end of the file and reads chunks backwards until
        /// it finds a non-0xFF byte.
        /// </remarks>
        /// <exception cref="System.IO.IOException">if the file cannot be read</exception>
        private static long GetNonTrailerLength(FilePath f)
        {
            int             chunkSizeToRead = 256 * 1024;
            FileInputStream fis             = new FileInputStream(f);

            try
            {
                byte[]      buf  = new byte[chunkSizeToRead];
                FileChannel fc   = fis.GetChannel();
                long        size = fc.Size();
                long        pos  = size - (size % chunkSizeToRead);
                while (pos >= 0)
                {
                    fc.Position(pos);
                    int readLen = (int)Math.Min(size - pos, chunkSizeToRead);
                    IOUtils.ReadFully(fis, buf, 0, readLen);
                    for (int i = readLen - 1; i >= 0; i--)
                    {
                        if (buf[i] != FSEditLogOpCodes.OpInvalid.GetOpCode())
                        {
                            return(pos + i + 1);
                        }
                    }
                    // + 1 since we count this byte!
                    pos -= chunkSizeToRead;
                }
                return(0);
            }
            finally
            {
                fis.Close();
            }
        }
예제 #4
0
파일: IOUtil.cs 프로젝트: ashmind/ngit
        /// <summary>Read an entire local file into memory as a byte array.</summary>
        /// <remarks>Read an entire local file into memory as a byte array.</remarks>
        /// <param name="path">location of the file to read.</param>
        /// <param name="max">
        /// maximum number of bytes to read, if the file is larger than
        /// this limit an IOException is thrown.
        /// </param>
        /// <returns>complete contents of the requested local file.</returns>
        /// <exception cref="System.IO.FileNotFoundException">the file does not exist.</exception>
        /// <exception cref="System.IO.IOException">the file exists, but its contents cannot be read.
        ///     </exception>
        public static byte[] ReadFully(FilePath path, int max)
        {
            FileInputStream @in = new FileInputStream(path);

            try
            {
                long sz = @in.GetChannel().Size();
                if (sz > max)
                {
                    throw new IOException(MessageFormat.Format(JGitText.Get().fileIsTooLarge, path));
                }
                byte[] buf = new byte[(int)sz];
                IOUtil.ReadFully(@in, buf, 0, buf.Length);
                return(buf);
            }
            finally
            {
                try
                {
                    @in.Close();
                }
                catch (IOException)
                {
                }
            }
        }
예제 #5
0
        Load(long length, FileInputStream blockIn, FileInputStream metaIn, string blockFileName
             )
        {
            Org.Apache.Hadoop.Hdfs.Server.Datanode.Fsdataset.Impl.MappableBlock mappableBlock
                = null;
            MappedByteBuffer mmap         = null;
            FileChannel      blockChannel = null;

            try
            {
                blockChannel = blockIn.GetChannel();
                if (blockChannel == null)
                {
                    throw new IOException("Block InputStream has no FileChannel.");
                }
                mmap = blockChannel.Map(FileChannel.MapMode.ReadOnly, 0, length);
                NativeIO.POSIX.GetCacheManipulator().Mlock(blockFileName, mmap, length);
                VerifyChecksum(length, metaIn, blockChannel, blockFileName);
                mappableBlock = new Org.Apache.Hadoop.Hdfs.Server.Datanode.Fsdataset.Impl.MappableBlock
                                    (mmap, length);
            }
            finally
            {
                IOUtils.CloseQuietly(blockChannel);
                if (mappableBlock == null)
                {
                    if (mmap != null)
                    {
                        NativeIO.POSIX.Munmap(mmap);
                    }
                }
            }
            // unmapping also unlocks
            return(mappableBlock);
        }
예제 #6
0
                /// <exception cref="NGit.Errors.MissingObjectException"></exception>
                /// <exception cref="System.IO.IOException"></exception>
                public override ObjectStream OpenStream()
                {
                    FileInputStream     @in  = new FileInputStream(p);
                    long                sz   = @in.GetChannel().Size();
                    int                 type = this.GetType();
                    BufferedInputStream b    = new BufferedInputStream(@in);

                    return(new ObjectStream.Filter(type, sz, b));
                }
예제 #7
0
파일: LockFile.cs 프로젝트: shoff/ngit
 /// <summary>Copy the current file content into the temporary file.</summary>
 /// <remarks>
 /// Copy the current file content into the temporary file.
 /// <p>
 /// This method saves the current file content by inserting it into the
 /// temporary file, so that the caller can safely append rather than replace
 /// the primary file.
 /// <p>
 /// This method does nothing if the current file does not exist, or exists
 /// but is empty.
 /// </remarks>
 /// <exception cref="System.IO.IOException">
 /// the temporary file could not be written, or a read error
 /// occurred while reading from the current file. The lock is
 /// released before throwing the underlying IO exception to the
 /// caller.
 /// </exception>
 /// <exception cref="Sharpen.RuntimeException">
 /// the temporary file could not be written. The lock is released
 /// before throwing the underlying exception to the caller.
 /// </exception>
 public virtual void CopyCurrentContent()
 {
     RequireLock();
     try
     {
         FileInputStream fis = new FileInputStream(@ref);
         try
         {
             if (fsync)
             {
                 FileChannel @in = fis.GetChannel();
                 long        pos = 0;
                 long        cnt = @in.Size();
                 while (0 < cnt)
                 {
                     long r = os.GetChannel().TransferFrom(@in, pos, cnt);
                     pos += r;
                     cnt -= r;
                 }
             }
             else
             {
                 byte[] buf = new byte[2048];
                 int    r;
                 while ((r = fis.Read(buf)) >= 0)
                 {
                     os.Write(buf, 0, r);
                 }
             }
         }
         finally
         {
             fis.Close();
         }
     }
     catch (FileNotFoundException)
     {
     }
     catch (IOException ioe)
     {
         // Don't worry about a file that doesn't exist yet, it
         // conceptually has no current content to copy.
         //
         Unlock();
         throw;
     }
     catch (RuntimeException ioe)
     {
         Unlock();
         throw;
     }
     catch (Error ioe)
     {
         Unlock();
         throw;
     }
 }
예제 #8
0
        /// <summary>Calculate the usable size of a shared memory segment.</summary>
        /// <remarks>
        /// Calculate the usable size of a shared memory segment.
        /// We round down to a multiple of the slot size and do some validation.
        /// </remarks>
        /// <param name="stream">The stream we're using.</param>
        /// <returns>The usable size of the shared memory segment.</returns>
        /// <exception cref="System.IO.IOException"/>
        private static int GetUsableLength(FileInputStream stream)
        {
            int intSize = Ints.CheckedCast(stream.GetChannel().Size());
            int slots   = intSize / BytesPerSlot;

            if (slots == 0)
            {
                throw new IOException("size of shared memory segment was " + intSize + ", but that is not enough to hold even one slot."
                                      );
            }
            return(slots * BytesPerSlot);
        }
예제 #9
0
        /// <summary>Verifies the block's checksum.</summary>
        /// <remarks>Verifies the block's checksum. This is an I/O intensive operation.</remarks>
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.FS.ChecksumException"/>
        private static void VerifyChecksum(long length, FileInputStream metaIn, FileChannel
                                           blockChannel, string blockFileName)
        {
            // Verify the checksum from the block's meta file
            // Get the DataChecksum from the meta file header
            BlockMetadataHeader header = BlockMetadataHeader.ReadHeader(new DataInputStream(new
                                                                                            BufferedInputStream(metaIn, BlockMetadataHeader.GetHeaderSize())));
            FileChannel metaChannel = null;

            try
            {
                metaChannel = metaIn.GetChannel();
                if (metaChannel == null)
                {
                    throw new IOException("Block InputStream meta file has no FileChannel.");
                }
                DataChecksum checksum         = header.GetChecksum();
                int          bytesPerChecksum = checksum.GetBytesPerChecksum();
                int          checksumSize     = checksum.GetChecksumSize();
                int          numChunks        = (8 * 1024 * 1024) / bytesPerChecksum;
                ByteBuffer   blockBuf         = ByteBuffer.Allocate(numChunks * bytesPerChecksum);
                ByteBuffer   checksumBuf      = ByteBuffer.Allocate(numChunks * checksumSize);
                // Verify the checksum
                int bytesVerified = 0;
                while (bytesVerified < length)
                {
                    Preconditions.CheckState(bytesVerified % bytesPerChecksum == 0, "Unexpected partial chunk before EOF"
                                             );
                    System.Diagnostics.Debug.Assert(bytesVerified % bytesPerChecksum == 0);
                    int bytesRead = FillBuffer(blockChannel, blockBuf);
                    if (bytesRead == -1)
                    {
                        throw new IOException("checksum verification failed: premature EOF");
                    }
                    blockBuf.Flip();
                    // Number of read chunks, including partial chunk at end
                    int chunks = (bytesRead + bytesPerChecksum - 1) / bytesPerChecksum;
                    checksumBuf.Limit(chunks * checksumSize);
                    FillBuffer(metaChannel, checksumBuf);
                    checksumBuf.Flip();
                    checksum.VerifyChunkedSums(blockBuf, checksumBuf, blockFileName, bytesVerified);
                    // Success
                    bytesVerified += bytesRead;
                    blockBuf.Clear();
                    checksumBuf.Clear();
                }
            }
            finally
            {
                IOUtils.CloseQuietly(metaChannel);
            }
        }
예제 #10
0
        /// <exception cref="System.Exception"/>
        public virtual void TestReadAndWrite()
        {
            FilePath path = new FilePath(TestBase, "testReadAndWrite");

            path.Mkdirs();
            SharedFileDescriptorFactory factory = SharedFileDescriptorFactory.Create("woot_",
                                                                                     new string[] { path.GetAbsolutePath() });
            FileInputStream  inStream  = factory.CreateDescriptor("testReadAndWrite", 4096);
            FileOutputStream outStream = new FileOutputStream(inStream.GetFD());

            outStream.Write(101);
            inStream.GetChannel().Position(0);
            Assert.Equal(101, inStream.Read());
            inStream.Close();
            outStream.Close();
            FileUtil.FullyDelete(path);
        }
예제 #11
0
 /// <exception cref="System.IO.IOException"/>
 public ShortCircuitReplica(ExtendedBlockId key, FileInputStream dataStream, FileInputStream
                            metaStream, ShortCircuitCache cache, long creationTimeMs, ShortCircuitShm.Slot
                            slot)
 {
     this.key        = key;
     this.dataStream = dataStream;
     this.metaStream = metaStream;
     this.metaHeader = BlockMetadataHeader.PreadHeader(metaStream.GetChannel());
     if (metaHeader.GetVersion() != 1)
     {
         throw new IOException("invalid metadata header version " + metaHeader.GetVersion(
                                   ) + ".  Can only handle version 1.");
     }
     this.cache          = cache;
     this.creationTimeMs = creationTimeMs;
     this.slot           = slot;
 }
예제 #12
0
        /// <exception cref="System.IO.IOException"/>
        private void CopyPartOfFile(FilePath src, FilePath dest)
        {
            FileInputStream  @in  = null;
            FileOutputStream @out = null;
            int MaxBytes          = 700;

            try
            {
                @in  = new FileInputStream(src);
                @out = new FileOutputStream(dest);
                @in.GetChannel().TransferTo(0, MaxBytes, @out.GetChannel());
            }
            finally
            {
                IOUtils.Cleanup(null, @in);
                IOUtils.Cleanup(null, @out);
            }
        }
예제 #13
0
        /// <exception cref="System.IO.IOException"/>
        private void LoadINodeDirSection(FileInputStream fin, IList <FsImageProto.FileSummary.Section
                                                                     > sections, FsImageProto.FileSummary summary, Configuration conf)
        {
            Log.Info("Loading INode directory section.");
            long startTime = Time.MonotonicNow();

            foreach (FsImageProto.FileSummary.Section section in sections)
            {
                if (FSImageFormatProtobuf.SectionName.FromString(section.GetName()) == FSImageFormatProtobuf.SectionName
                    .InodeDir)
                {
                    fin.GetChannel().Position(section.GetOffset());
                    InputStream @is = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec
                                                                                    (), new BufferedInputStream(new LimitInputStream(fin, section.GetLength())));
                    BuildNamespace(@is);
                }
            }
            long timeTaken = Time.MonotonicNow() - startTime;

            Log.Info("Finished loading INode directory section in {}ms", timeTaken);
        }
예제 #14
0
        /// <exception cref="System.IO.IOException"/>
        private void Output(Configuration conf, FsImageProto.FileSummary summary, FileInputStream
                            fin, AList <FsImageProto.FileSummary.Section> sections)
        {
            InputStream @is;
            long        startTime = Time.MonotonicNow();

            foreach (FsImageProto.FileSummary.Section section in sections)
            {
                if (FSImageFormatProtobuf.SectionName.FromString(section.GetName()) == FSImageFormatProtobuf.SectionName
                    .Inode)
                {
                    fin.GetChannel().Position(section.GetOffset());
                    @is = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec(), new BufferedInputStream
                                                                        (new LimitInputStream(fin, section.GetLength())));
                    OutputINodes(@is);
                }
            }
            long timeTaken = Time.MonotonicNow() - startTime;

            Log.Debug("Time to output inodes: {}ms", timeTaken);
        }
예제 #15
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void Visit(RandomAccessFile file)
        {
            Configuration conf = new Configuration();

            if (!FSImageUtil.CheckFileFormat(file))
            {
                throw new IOException("Unrecognized FSImage");
            }
            FsImageProto.FileSummary summary = FSImageUtil.LoadSummary(file);
            using (FileInputStream fin = new FileInputStream(file.GetFD()))
            {
                InputStream @is;
                AList <FsImageProto.FileSummary.Section> sections = Lists.NewArrayList(summary.GetSectionsList
                                                                                           ());
                sections.Sort(new _IComparer_427());
                foreach (FsImageProto.FileSummary.Section section in sections)
                {
                    fin.GetChannel().Position(section.GetOffset());
                    @is = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec(), new BufferedInputStream
                                                                        (new LimitInputStream(fin, section.GetLength())));
                    switch (FSImageFormatProtobuf.SectionName.FromString(section.GetName()))
                    {
                    case FSImageFormatProtobuf.SectionName.StringTable:
                    {
                        stringTable = FSImageLoader.LoadStringTable(@is);
                        break;
                    }

                    default:
                    {
                        break;
                    }
                    }
                }
                LoadDirectories(fin, sections, summary, conf);
                LoadINodeDirSection(fin, sections, summary, conf);
                metadataMap.Sync();
                Output(conf, summary, fin, sections);
            }
        }
예제 #16
0
 /// <summary>
 /// Unbuffered file copy from src to dst without tainting OS buffer cache
 /// In POSIX platform:
 /// It uses FileChannel#transferTo() which internally attempts
 /// unbuffered IO on OS with native sendfile64() support and falls back to
 /// buffered IO otherwise.
 /// </summary>
 /// <remarks>
 /// Unbuffered file copy from src to dst without tainting OS buffer cache
 /// In POSIX platform:
 /// It uses FileChannel#transferTo() which internally attempts
 /// unbuffered IO on OS with native sendfile64() support and falls back to
 /// buffered IO otherwise.
 /// It minimizes the number of FileChannel#transferTo call by passing the the
 /// src file size directly instead of a smaller size as the 3rd parameter.
 /// This saves the number of sendfile64() system call when native sendfile64()
 /// is supported. In the two fall back cases where sendfile is not supported,
 /// FileChannle#transferTo already has its own batching of size 8 MB and 8 KB,
 /// respectively.
 /// In Windows Platform:
 /// It uses its own native wrapper of CopyFileEx with COPY_FILE_NO_BUFFERING
 /// flag, which is supported on Windows Server 2008 and above.
 /// Ideally, we should use FileChannel#transferTo() across both POSIX and Windows
 /// platform. Unfortunately, the wrapper(Java_sun_nio_ch_FileChannelImpl_transferTo0)
 /// used by FileChannel#transferTo for unbuffered IO is not implemented on Windows.
 /// Based on OpenJDK 6/7/8 source code, Java_sun_nio_ch_FileChannelImpl_transferTo0
 /// on Windows simply returns IOS_UNSUPPORTED.
 /// Note: This simple native wrapper does minimal parameter checking before copy and
 /// consistency check (e.g., size) after copy.
 /// It is recommended to use wrapper function like
 /// the Storage#nativeCopyFileUnbuffered() function in hadoop-hdfs with pre/post copy
 /// checks.
 /// </remarks>
 /// <param name="src">The source path</param>
 /// <param name="dst">The destination path</param>
 /// <exception cref="System.IO.IOException"/>
 public static void CopyFileUnbuffered(FilePath src, FilePath dst)
 {
     if (nativeLoaded && Shell.Windows)
     {
         CopyFileUnbuffered0(src.GetAbsolutePath(), dst.GetAbsolutePath());
     }
     else
     {
         FileInputStream  fis    = null;
         FileOutputStream fos    = null;
         FileChannel      input  = null;
         FileChannel      output = null;
         try
         {
             fis    = new FileInputStream(src);
             fos    = new FileOutputStream(dst);
             input  = fis.GetChannel();
             output = fos.GetChannel();
             long remaining   = input.Size();
             long position    = 0;
             long transferred = 0;
             while (remaining > 0)
             {
                 transferred = input.TransferTo(position, remaining, output);
                 remaining  -= transferred;
                 position   += transferred;
             }
         }
         finally
         {
             IOUtils.Cleanup(Log, output);
             IOUtils.Cleanup(Log, fos);
             IOUtils.Cleanup(Log, input);
             IOUtils.Cleanup(Log, fis);
         }
     }
 }
 /// <exception cref="System.IO.IOException"/>
 internal void Visit(RandomAccessFile file)
 {
     if (!FSImageUtil.CheckFileFormat(file))
     {
         throw new IOException("Unrecognized FSImage");
     }
     FsImageProto.FileSummary summary = FSImageUtil.LoadSummary(file);
     using (FileInputStream @in = new FileInputStream(file.GetFD()))
     {
         foreach (FsImageProto.FileSummary.Section s in summary.GetSectionsList())
         {
             if (FSImageFormatProtobuf.SectionName.FromString(s.GetName()) != FSImageFormatProtobuf.SectionName
                 .Inode)
             {
                 continue;
             }
             @in.GetChannel().Position(s.GetOffset());
             InputStream @is = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec
                                                                             (), new BufferedInputStream(new LimitInputStream(@in, s.GetLength())));
             Run(@is);
             Output();
         }
     }
 }
예제 #18
0
 /// <exception cref="System.Exception"/>
 private static long[] GetBlockSizes(HdfsBlockLocation[] locs)
 {
     long[] sizes = new long[locs.Length];
     for (int i = 0; i < locs.Length; i++)
     {
         HdfsBlockLocation loc              = locs[i];
         string            bpid             = loc.GetLocatedBlock().GetBlock().GetBlockPoolId();
         Block             block            = loc.GetLocatedBlock().GetBlock().GetLocalBlock();
         ExtendedBlock     extBlock         = new ExtendedBlock(bpid, block);
         FileInputStream   blockInputStream = null;
         FileChannel       blockChannel     = null;
         try
         {
             blockInputStream = (FileInputStream)fsd.GetBlockInputStream(extBlock, 0);
             blockChannel     = blockInputStream.GetChannel();
             sizes[i]         = blockChannel.Size();
         }
         finally
         {
             IOUtils.Cleanup(Log, blockChannel, blockInputStream);
         }
     }
     return(sizes);
 }
예제 #19
0
 internal virtual MappedByteBuffer LoadMmapInternal()
 {
     try
     {
         FileChannel      channel = dataStream.GetChannel();
         MappedByteBuffer mmap    = channel.Map(FileChannel.MapMode.ReadOnly, 0, Math.Min(int.MaxValue
                                                                                          , channel.Size()));
         if (Log.IsTraceEnabled())
         {
             Log.Trace(this + ": created mmap of size " + channel.Size());
         }
         return(mmap);
     }
     catch (IOException e)
     {
         Log.Warn(this + ": mmap error", e);
         return(null);
     }
     catch (RuntimeException e)
     {
         Log.Warn(this + ": mmap error", e);
         return(null);
     }
 }
예제 #20
0
            /// <exception cref="System.IO.IOException"/>
            internal override int Run(IList <string> args)
            {
                if (args.Count == 0)
                {
                    System.Console.Out.WriteLine(this.usageText);
                    System.Console.Out.WriteLine(this.helpText + "\n");
                    return(1);
                }
                string blockFile = StringUtils.PopOptionWithArgument("-block", args);
                string metaFile  = StringUtils.PopOptionWithArgument("-meta", args);

                if (metaFile == null)
                {
                    System.Console.Error.WriteLine("You must specify a meta file with -meta");
                    return(1);
                }
                FileInputStream metaStream     = null;
                FileInputStream dataStream     = null;
                FileChannel     metaChannel    = null;
                FileChannel     dataChannel    = null;
                DataInputStream checksumStream = null;

                try
                {
                    BlockMetadataHeader header;
                    try
                    {
                        metaStream     = new FileInputStream(metaFile);
                        checksumStream = new DataInputStream(metaStream);
                        header         = BlockMetadataHeader.ReadHeader(checksumStream);
                        metaChannel    = metaStream.GetChannel();
                        metaChannel.Position(DebugAdmin.HeaderLen);
                    }
                    catch (RuntimeException e)
                    {
                        System.Console.Error.WriteLine("Failed to read HDFS metadata file header for " +
                                                       metaFile + ": " + StringUtils.StringifyException(e));
                        return(1);
                    }
                    catch (IOException e)
                    {
                        System.Console.Error.WriteLine("Failed to read HDFS metadata file header for " +
                                                       metaFile + ": " + StringUtils.StringifyException(e));
                        return(1);
                    }
                    DataChecksum checksum = header.GetChecksum();
                    System.Console.Out.WriteLine("Checksum type: " + checksum.ToString());
                    if (blockFile == null)
                    {
                        return(0);
                    }
                    ByteBuffer metaBuf;
                    ByteBuffer dataBuf;
                    try
                    {
                        dataStream  = new FileInputStream(blockFile);
                        dataChannel = dataStream.GetChannel();
                        int ChecksumsPerBuf = 1024 * 32;
                        metaBuf = ByteBuffer.Allocate(checksum.GetChecksumSize() * ChecksumsPerBuf);
                        dataBuf = ByteBuffer.Allocate(checksum.GetBytesPerChecksum() * ChecksumsPerBuf);
                    }
                    catch (IOException e)
                    {
                        System.Console.Error.WriteLine("Failed to open HDFS block file for " + blockFile
                                                       + ": " + StringUtils.StringifyException(e));
                        return(1);
                    }
                    long offset = 0;
                    while (true)
                    {
                        dataBuf.Clear();
                        int dataRead = -1;
                        try
                        {
                            dataRead = dataChannel.Read(dataBuf);
                            if (dataRead < 0)
                            {
                                break;
                            }
                        }
                        catch (IOException e)
                        {
                            System.Console.Error.WriteLine("Got I/O error reading block file " + blockFile +
                                                           "from disk at offset " + dataChannel.Position() + ": " + StringUtils.StringifyException
                                                               (e));
                            return(1);
                        }
                        try
                        {
                            int csumToRead = (((checksum.GetBytesPerChecksum() - 1) + dataRead) / checksum.GetBytesPerChecksum
                                                  ()) * checksum.GetChecksumSize();
                            metaBuf.Clear();
                            metaBuf.Limit(csumToRead);
                            metaChannel.Read(metaBuf);
                            dataBuf.Flip();
                            metaBuf.Flip();
                        }
                        catch (IOException e)
                        {
                            System.Console.Error.WriteLine("Got I/O error reading metadata file " + metaFile
                                                           + "from disk at offset " + metaChannel.Position() + ": " + StringUtils.StringifyException
                                                               (e));
                            return(1);
                        }
                        try
                        {
                            checksum.VerifyChunkedSums(dataBuf, metaBuf, blockFile, offset);
                        }
                        catch (IOException e)
                        {
                            System.Console.Out.WriteLine("verifyChunkedSums error: " + StringUtils.StringifyException
                                                             (e));
                            return(1);
                        }
                        offset += dataRead;
                    }
                    System.Console.Out.WriteLine("Checksum verification succeeded on block file " + blockFile
                                                 );
                    return(0);
                }
                finally
                {
                    IOUtils.Cleanup(null, metaStream, dataStream, checksumStream);
                }
            }
예제 #21
0
        /// <summary>Load fsimage into the memory.</summary>
        /// <param name="inputFile">the filepath of the fsimage to load.</param>
        /// <returns>FSImageLoader</returns>
        /// <exception cref="System.IO.IOException">if failed to load fsimage.</exception>
        internal static Org.Apache.Hadoop.Hdfs.Tools.OfflineImageViewer.FSImageLoader Load
            (string inputFile)
        {
            Configuration    conf = new Configuration();
            RandomAccessFile file = new RandomAccessFile(inputFile, "r");

            if (!FSImageUtil.CheckFileFormat(file))
            {
                throw new IOException("Unrecognized FSImage");
            }
            FsImageProto.FileSummary summary = FSImageUtil.LoadSummary(file);
            using (FileInputStream fin = new FileInputStream(file.GetFD()))
            {
                // Map to record INodeReference to the referred id
                ImmutableList <long>       refIdList              = null;
                string[]                   stringTable            = null;
                byte[][]                   inodes                 = null;
                IDictionary <long, long[]> dirmap                 = null;
                AList <FsImageProto.FileSummary.Section> sections = Lists.NewArrayList(summary.GetSectionsList
                                                                                           ());
                sections.Sort(new _IComparer_126());
                foreach (FsImageProto.FileSummary.Section s in sections)
                {
                    fin.GetChannel().Position(s.GetOffset());
                    InputStream @is = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec
                                                                                    (), new BufferedInputStream(new LimitInputStream(fin, s.GetLength())));
                    Log.Debug("Loading section " + s.GetName() + " length: " + s.GetLength());
                    switch (FSImageFormatProtobuf.SectionName.FromString(s.GetName()))
                    {
                    case FSImageFormatProtobuf.SectionName.StringTable:
                    {
                        stringTable = LoadStringTable(@is);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.Inode:
                    {
                        inodes = LoadINodeSection(@is);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.InodeReference:
                    {
                        refIdList = LoadINodeReferenceSection(@is);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.InodeDir:
                    {
                        dirmap = LoadINodeDirectorySection(@is, refIdList);
                        break;
                    }

                    default:
                    {
                        break;
                    }
                    }
                }
                return(new Org.Apache.Hadoop.Hdfs.Tools.OfflineImageViewer.FSImageLoader(stringTable
                                                                                         , inodes, dirmap));
            }
        }
예제 #22
0
        /// <exception cref="System.IO.IOException"/>
        public void Visit(RandomAccessFile file)
        {
            if (!FSImageUtil.CheckFileFormat(file))
            {
                throw new IOException("Unrecognized FSImage");
            }
            FsImageProto.FileSummary summary = FSImageUtil.LoadSummary(file);
            using (FileInputStream fin = new FileInputStream(file.GetFD()))
            {
                @out.Write("<?xml version=\"1.0\"?>\n<fsimage>");
                AList <FsImageProto.FileSummary.Section> sections = Lists.NewArrayList(summary.GetSectionsList
                                                                                           ());
                sections.Sort(new _IComparer_83());
                foreach (FsImageProto.FileSummary.Section s in sections)
                {
                    fin.GetChannel().Position(s.GetOffset());
                    InputStream @is = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec
                                                                                    (), new BufferedInputStream(new LimitInputStream(fin, s.GetLength())));
                    switch (FSImageFormatProtobuf.SectionName.FromString(s.GetName()))
                    {
                    case FSImageFormatProtobuf.SectionName.NsInfo:
                    {
                        DumpNameSection(@is);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.StringTable:
                    {
                        LoadStringTable(@is);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.Inode:
                    {
                        DumpINodeSection(@is);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.InodeReference:
                    {
                        DumpINodeReferenceSection(@is);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.InodeDir:
                    {
                        DumpINodeDirectorySection(@is);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.FilesUnderconstruction:
                    {
                        DumpFileUnderConstructionSection(@is);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.Snapshot:
                    {
                        DumpSnapshotSection(@is);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.SnapshotDiff:
                    {
                        DumpSnapshotDiffSection(@is);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.SecretManager:
                    {
                        DumpSecretManagerSection(@is);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.CacheManager:
                    {
                        DumpCacheManagerSection(@is);
                        break;
                    }

                    default:
                    {
                        break;
                    }
                    }
                }
                @out.Write("</fsimage>\n");
            }
        }
예제 #23
0
            /// <exception cref="System.IO.IOException"/>
            private void LoadInternal(RandomAccessFile raFile, FileInputStream fin)
            {
                if (!FSImageUtil.CheckFileFormat(raFile))
                {
                    throw new IOException("Unrecognized file format");
                }
                FsImageProto.FileSummary summary = FSImageUtil.LoadSummary(raFile);
                if (requireSameLayoutVersion && summary.GetLayoutVersion() != HdfsConstants.NamenodeLayoutVersion)
                {
                    throw new IOException("Image version " + summary.GetLayoutVersion() + " is not equal to the software version "
                                          + HdfsConstants.NamenodeLayoutVersion);
                }
                FileChannel channel = fin.GetChannel();

                FSImageFormatPBINode.Loader inodeLoader = new FSImageFormatPBINode.Loader(fsn, this
                                                                                          );
                FSImageFormatPBSnapshot.Loader snapshotLoader = new FSImageFormatPBSnapshot.Loader
                                                                    (fsn, this);
                AList <FsImageProto.FileSummary.Section> sections = Lists.NewArrayList(summary.GetSectionsList
                                                                                           ());

                sections.Sort(new _IComparer_210());
                StartupProgress prog        = NameNode.GetStartupProgress();
                Step            currentStep = null;

                foreach (FsImageProto.FileSummary.Section s in sections)
                {
                    channel.Position(s.GetOffset());
                    InputStream @in = new BufferedInputStream(new LimitInputStream(fin, s.GetLength()
                                                                                   ));
                    @in = FSImageUtil.WrapInputStreamForCompression(conf, summary.GetCodec(), @in);
                    string n = s.GetName();
                    switch (FSImageFormatProtobuf.SectionName.FromString(n))
                    {
                    case FSImageFormatProtobuf.SectionName.NsInfo:
                    {
                        LoadNameSystemSection(@in);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.StringTable:
                    {
                        LoadStringTableSection(@in);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.Inode:
                    {
                        currentStep = new Step(StepType.Inodes);
                        prog.BeginStep(Phase.LoadingFsimage, currentStep);
                        inodeLoader.LoadINodeSection(@in);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.InodeReference:
                    {
                        snapshotLoader.LoadINodeReferenceSection(@in);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.InodeDir:
                    {
                        inodeLoader.LoadINodeDirectorySection(@in);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.FilesUnderconstruction:
                    {
                        inodeLoader.LoadFilesUnderConstructionSection(@in);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.Snapshot:
                    {
                        snapshotLoader.LoadSnapshotSection(@in);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.SnapshotDiff:
                    {
                        snapshotLoader.LoadSnapshotDiffSection(@in);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.SecretManager:
                    {
                        prog.EndStep(Phase.LoadingFsimage, currentStep);
                        Step step = new Step(StepType.DelegationTokens);
                        prog.BeginStep(Phase.LoadingFsimage, step);
                        LoadSecretManagerSection(@in);
                        prog.EndStep(Phase.LoadingFsimage, step);
                        break;
                    }

                    case FSImageFormatProtobuf.SectionName.CacheManager:
                    {
                        Step step = new Step(StepType.CachePools);
                        prog.BeginStep(Phase.LoadingFsimage, step);
                        LoadCacheManagerSection(@in);
                        prog.EndStep(Phase.LoadingFsimage, step);
                        break;
                    }

                    default:
                    {
                        Log.Warn("Unrecognized section " + n);
                        break;
                    }
                    }
                }
            }