/// <summary>Construct an IFile Reader.</summary>
/// <param name="conf">Configuration File</param>
/// <param name="in">The input stream</param>
/// <param name="length">
/// Length of the data in the stream, including the checksum
/// bytes.
/// </param>
/// <param name="codec">codec</param>
/// <param name="readsCounter">Counter for records read from disk</param>
/// <exception cref="System.IO.IOException"/>
public Reader(Configuration conf, FSDataInputStream @in, long length, CompressionCodec
codec, Counters.Counter readsCounter)
{
// Count records read from disk
// Possibly decompressed stream that we read
readRecordsCounter = readsCounter;
checksumIn = new IFileInputStream(@in, length, conf);
if (codec != null)
{
decompressor = CodecPool.GetDecompressor(codec);
if (decompressor != null)
{
this.@in = codec.CreateInputStream(checksumIn, decompressor);
}
else
{
Log.Warn("Could not obtain decompressor from CodecPool");
this.@in = checksumIn;
}
}
else
{
this.@in = checksumIn;
}
this.dataIn = new DataInputStream(this.@in);
this.fileLength = length;
if (conf != null)
{
bufferSize = conf.GetInt("io.file.buffer.size", DefaultBufferSize);
}
}
/// <summary>
/// Unwrap a compressed input stream by wrapping it with a decompressor based
/// on this codec.
/// </summary>
/// <remarks>
/// Unwrap a compressed input stream by wrapping it with a decompressor based
/// on this codec. If this instance represents no compression, simply adds
/// buffering to the input stream.
/// </remarks>
/// <returns>a buffered stream that provides uncompressed data</returns>
/// <exception cref="System.IO.IOException">
/// If the decompressor cannot be instantiated or an IO
/// error occurs.
/// </exception>
internal virtual DataInputStream UnwrapInputStream(InputStream @is)
{
if (imageCodec != null)
{
return(new DataInputStream(imageCodec.CreateInputStream(@is)));
}
else
{
return(new DataInputStream(new BufferedInputStream(@is)));
}
}
/// <exception cref="System.IO.IOException"/>
public static InputStream WrapInputStreamForCompression(Configuration conf, string
codec, InputStream @in)
{
if (codec.IsEmpty())
{
return(@in);
}
FSImageCompression compression = FSImageCompression.CreateCompression(conf, codec
);
CompressionCodec imageCodec = compression.GetImageCodec();
return(imageCodec.CreateInputStream(@in));
}
/// <exception cref="System.IO.IOException"/>
public LineRecordReader(Configuration job, FileSplit split, byte[] recordDelimiter
)
{
this.maxLineLength = job.GetInt(LineRecordReader.MaxLineLength, int.MaxValue);
start = split.GetStart();
end = start + split.GetLength();
Path file = split.GetPath();
compressionCodecs = new CompressionCodecFactory(job);
codec = compressionCodecs.GetCodec(file);
// open the file and seek to the start of the split
FileSystem fs = file.GetFileSystem(job);
fileIn = fs.Open(file);
if (IsCompressedInput())
{
decompressor = CodecPool.GetDecompressor(codec);
if (codec is SplittableCompressionCodec)
{
SplitCompressionInputStream cIn = ((SplittableCompressionCodec)codec).CreateInputStream
(fileIn, decompressor, start, end, SplittableCompressionCodec.READ_MODE.Byblock);
@in = new CompressedSplitLineReader(cIn, job, recordDelimiter);
start = cIn.GetAdjustedStart();
end = cIn.GetAdjustedEnd();
filePosition = cIn;
}
else
{
// take pos from compressed stream
@in = new SplitLineReader(codec.CreateInputStream(fileIn, decompressor), job, recordDelimiter
);
filePosition = fileIn;
}
}
else
{
fileIn.Seek(start);
@in = new UncompressedSplitLineReader(fileIn, job, recordDelimiter, split.GetLength
());
filePosition = fileIn;
}
// If this is not the first split, we always throw away first record
// because we always (except the last split) read one extra line in
// next() method.
if (start != 0)
{
start += @in.ReadLine(new Text(), 0, MaxBytesToConsume(start));
}
this.pos = start;
}
/// <exception cref="System.IO.IOException"/>
public override void Shuffle(MapHost host, InputStream input, long compressedLength
, long decompressedLength, ShuffleClientMetrics metrics, Reporter reporter)
{
IFileInputStream checksumIn = new IFileInputStream(input, compressedLength, conf);
input = checksumIn;
// Are map-outputs compressed?
if (codec != null)
{
decompressor.Reset();
input = codec.CreateInputStream(input, decompressor);
}
try
{
IOUtils.ReadFully(input, memory, 0, memory.Length);
metrics.InputBytes(memory.Length);
reporter.Progress();
Log.Info("Read " + memory.Length + " bytes from map-output for " + GetMapId());
if (input.Read() >= 0)
{
throw new IOException("Unexpected extra bytes from input stream for " + GetMapId(
));
}
}
catch (IOException ioe)
{
// Close the streams
IOUtils.Cleanup(Log, input);
// Re-throw
throw;
}
finally
{
CodecPool.ReturnDecompressor(decompressor);
}
}
/* (non-Javadoc)
* @see ImageLoader#processImage(java.io.DataInputStream, ImageVisitor, boolean)
*/
/// <exception cref="System.IO.IOException"/>
public override void LoadImage(DataInputStream @in, ImageVisitor v, bool skipBlocks
)
{
bool done = false;
try
{
v.Start();
v.VisitEnclosingElement(ImageVisitor.ImageElement.FsImage);
imageVersion = @in.ReadInt();
if (!CanLoadVersion(imageVersion))
{
throw new IOException("Cannot process fslayout version " + imageVersion);
}
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.AddLayoutFlags, imageVersion
))
{
LayoutFlags.Read(@in);
}
v.Visit(ImageVisitor.ImageElement.ImageVersion, imageVersion);
v.Visit(ImageVisitor.ImageElement.NamespaceId, @in.ReadInt());
long numInodes = @in.ReadLong();
v.Visit(ImageVisitor.ImageElement.GenerationStamp, @in.ReadLong());
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.SequentialBlockId, imageVersion
))
{
v.Visit(ImageVisitor.ImageElement.GenerationStampV2, @in.ReadLong());
v.Visit(ImageVisitor.ImageElement.GenerationStampV1Limit, @in.ReadLong());
v.Visit(ImageVisitor.ImageElement.LastAllocatedBlockId, @in.ReadLong());
}
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.StoredTxids, imageVersion
))
{
v.Visit(ImageVisitor.ImageElement.TransactionId, @in.ReadLong());
}
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.AddInodeId, imageVersion
))
{
v.Visit(ImageVisitor.ImageElement.LastInodeId, @in.ReadLong());
}
bool supportSnapshot = NameNodeLayoutVersion.Supports(LayoutVersion.Feature.Snapshot
, imageVersion);
if (supportSnapshot)
{
v.Visit(ImageVisitor.ImageElement.SnapshotCounter, @in.ReadInt());
int numSnapshots = @in.ReadInt();
v.Visit(ImageVisitor.ImageElement.NumSnapshotsTotal, numSnapshots);
for (int i = 0; i < numSnapshots; i++)
{
ProcessSnapshot(@in, v);
}
}
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.FsimageCompression, imageVersion
))
{
bool isCompressed = @in.ReadBoolean();
v.Visit(ImageVisitor.ImageElement.IsCompressed, isCompressed.ToString());
if (isCompressed)
{
string codecClassName = Text.ReadString(@in);
v.Visit(ImageVisitor.ImageElement.CompressCodec, codecClassName);
CompressionCodecFactory codecFac = new CompressionCodecFactory(new Configuration(
));
CompressionCodec codec = codecFac.GetCodecByClassName(codecClassName);
if (codec == null)
{
throw new IOException("Image compression codec not supported: " + codecClassName);
}
@in = new DataInputStream(codec.CreateInputStream(@in));
}
}
ProcessINodes(@in, v, numInodes, skipBlocks, supportSnapshot);
subtreeMap.Clear();
dirNodeMap.Clear();
ProcessINodesUC(@in, v, skipBlocks);
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.DelegationToken, imageVersion
))
{
ProcessDelegationTokens(@in, v);
}
if (NameNodeLayoutVersion.Supports(LayoutVersion.Feature.Caching, imageVersion))
{
ProcessCacheManagerState(@in, v);
}
v.LeaveEnclosingElement();
// FSImage
done = true;
}
finally
{
if (done)
{
v.Finish();
}
else
{
v.FinishAbnormally();
}
}
}
/// <exception cref="System.IO.IOException"/>
protected internal override InputStream GetInputStream(PathData item)
{
FSDataInputStream i = (FSDataInputStream)base.GetInputStream(item);
// Handle 0 and 1-byte files
short leadBytes;
try
{
leadBytes = i.ReadShort();
}
catch (EOFException)
{
i.Seek(0);
return(i);
}
switch (leadBytes)
{
case unchecked ((int)(0x1f8b)):
{
// Check type of stream first
// RFC 1952
// Must be gzip
i.Seek(0);
return(new GZIPInputStream(i));
}
case unchecked ((int)(0x5345)):
{
// 'S' 'E'
// Might be a SequenceFile
if (i.ReadByte() == 'Q')
{
i.Close();
return(new Display.TextRecordInputStream(this, item.stat));
}
goto default;
}
default:
{
// Check the type of compression instead, depending on Codec class's
// own detection methods, based on the provided path.
CompressionCodecFactory cf = new CompressionCodecFactory(GetConf());
CompressionCodec codec = cf.GetCodec(item.path);
if (codec != null)
{
i.Seek(0);
return(codec.CreateInputStream(i));
}
break;
}
case unchecked ((int)(0x4f62)):
{
// 'O' 'b'
if (i.ReadByte() == 'j')
{
i.Close();
return(new Display.AvroFileInputStream(item.stat));
}
break;
}
}
// File is non-compressed, or not a file container we know.
i.Seek(0);
return(i);
}
