Ejemplo n.º 1
0
        /// <exception cref="System.IO.IOException"/>
        public SpillRecord(Path indexFileName, JobConf job, Checksum crc, string expectedIndexOwner
                           )
        {
            FileSystem        rfs = FileSystem.GetLocal(job).GetRaw();
            FSDataInputStream @in = SecureIOUtils.OpenFSDataInputStream(new FilePath(indexFileName
                                                                                     .ToUri().GetRawPath()), expectedIndexOwner, null);

            try
            {
                long length     = rfs.GetFileStatus(indexFileName).GetLen();
                int  partitions = (int)length / MapTask.MapOutputIndexRecordLength;
                int  size       = partitions * MapTask.MapOutputIndexRecordLength;
                buf = ByteBuffer.Allocate(size);
                if (crc != null)
                {
                    crc.Reset();
                    CheckedInputStream chk = new CheckedInputStream(@in, crc);
                    IOUtils.ReadFully(chk, ((byte[])buf.Array()), 0, size);
                    if (chk.GetChecksum().GetValue() != @in.ReadLong())
                    {
                        throw new ChecksumException("Checksum error reading spill index: " + indexFileName
                                                    , -1);
                    }
                }
                else
                {
                    IOUtils.ReadFully(@in, ((byte[])buf.Array()), 0, size);
                }
                entries = buf.AsLongBuffer();
            }
            finally
            {
                @in.Close();
            }
        }
Ejemplo n.º 2
0
            /// <summary>Constructor</summary>
            /// <param name="fin">FS input stream.</param>
            /// <param name="fileLength">Length of the corresponding file</param>
            /// <exception cref="System.IO.IOException"/>
            public Reader(FSDataInputStream fin, long fileLength, Configuration conf)
            {
                this.@in  = fin;
                this.conf = conf;
                // move the cursor to the beginning of the tail, containing: offset to the
                // meta block index, version and magic
                fin.Seek(fileLength - BCFile.Magic.Size() - Utils.Version.Size() - long.Size / byte
                         .Size);
                long offsetIndexMeta = fin.ReadLong();

                version = new Utils.Version(fin);
                BCFile.Magic.ReadAndVerify(fin);
                if (!version.CompatibleWith(BCFile.ApiVersion))
                {
                    throw new RuntimeException("Incompatible BCFile fileBCFileVersion.");
                }
                // read meta index
                fin.Seek(offsetIndexMeta);
                metaIndex = new BCFile.MetaIndex(fin);
                // read data:BCFile.index, the data block index
                BCFile.Reader.BlockReader blockR = GetMetaBlock(BCFile.DataIndex.BlockName);
                try
                {
                    dataIndex = new BCFile.DataIndex(blockR);
                }
                finally
                {
                    blockR.Close();
                }
            }