/// <exception cref="System.IO.IOException"/>
public SpillRecord(Path indexFileName, JobConf job, Checksum crc, string expectedIndexOwner
)
{
FileSystem rfs = FileSystem.GetLocal(job).GetRaw();
FSDataInputStream @in = SecureIOUtils.OpenFSDataInputStream(new FilePath(indexFileName
.ToUri().GetRawPath()), expectedIndexOwner, null);
try
{
long length = rfs.GetFileStatus(indexFileName).GetLen();
int partitions = (int)length / MapTask.MapOutputIndexRecordLength;
int size = partitions * MapTask.MapOutputIndexRecordLength;
buf = ByteBuffer.Allocate(size);
if (crc != null)
{
crc.Reset();
CheckedInputStream chk = new CheckedInputStream(@in, crc);
IOUtils.ReadFully(chk, ((byte[])buf.Array()), 0, size);
if (chk.GetChecksum().GetValue() != @in.ReadLong())
{
throw new ChecksumException("Checksum error reading spill index: " + indexFileName
, -1);
}
}
else
{
IOUtils.ReadFully(@in, ((byte[])buf.Array()), 0, size);
}
entries = buf.AsLongBuffer();
}
finally
{
@in.Close();
}
}
/// <summary>Constructor</summary>
/// <param name="fin">FS input stream.</param>
/// <param name="fileLength">Length of the corresponding file</param>
/// <exception cref="System.IO.IOException"/>
public Reader(FSDataInputStream fin, long fileLength, Configuration conf)
{
this.@in = fin;
this.conf = conf;
// move the cursor to the beginning of the tail, containing: offset to the
// meta block index, version and magic
fin.Seek(fileLength - BCFile.Magic.Size() - Utils.Version.Size() - long.Size / byte
.Size);
long offsetIndexMeta = fin.ReadLong();
version = new Utils.Version(fin);
BCFile.Magic.ReadAndVerify(fin);
if (!version.CompatibleWith(BCFile.ApiVersion))
{
throw new RuntimeException("Incompatible BCFile fileBCFileVersion.");
}
// read meta index
fin.Seek(offsetIndexMeta);
metaIndex = new BCFile.MetaIndex(fin);
// read data:BCFile.index, the data block index
BCFile.Reader.BlockReader blockR = GetMetaBlock(BCFile.DataIndex.BlockName);
try
{
dataIndex = new BCFile.DataIndex(blockR);
}
finally
{
blockR.Close();
}
}
