public virtual void TestIFileReaderWithCodec()
{
Configuration conf = new Configuration();
FileSystem localFs = FileSystem.GetLocal(conf);
FileSystem rfs = ((LocalFileSystem)localFs).GetRaw();
Path path = new Path(new Path("build/test.ifile"), "data");
DefaultCodec codec = new GzipCodec();
codec.SetConf(conf);
FSDataOutputStream @out = rfs.Create(path);
IFile.Writer <Text, Text> writer = new IFile.Writer <Text, Text>(conf, @out, typeof(
Text), typeof(Text), codec, null);
writer.Close();
FSDataInputStream @in = rfs.Open(path);
IFile.Reader <Text, Text> reader = new IFile.Reader <Text, Text>(conf, @in, rfs.GetFileStatus
(path).GetLen(), codec, null);
reader.Close();
// test check sum
byte[] ab = new byte[100];
int readed = reader.checksumIn.ReadWithChecksum(ab, 0, ab.Length);
NUnit.Framework.Assert.AreEqual(readed, reader.checksumIn.GetChecksum().Length);
}
/// <exception cref="System.IO.IOException"/>
internal virtual void CloseReader()
{
if (reader != null)
{
reader.Close();
reader = null;
}
}
public Segment(IFile.Reader <K, V> reader, bool preserve, Counters.Counter mapOutputsCounter
)
{
this.reader = reader;
this.preserve = preserve;
this.segmentLength = reader.GetLength();
this.mapOutputsCounter = mapOutputsCounter;
}
/// <exception cref="System.IO.IOException"/>
internal virtual void Init(Counters.Counter readsCounter)
{
if (reader == null)
{
FSDataInputStream @in = fs.Open(file);
@in.Seek(segmentOffset);
@in = CryptoUtils.WrapIfNecessary(conf, @in);
reader = new IFile.Reader <K, V>(conf, @in, segmentLength - CryptoUtils.CryptoPadding
(conf), codec, readsCounter);
}
if (mapOutputsCounter != null)
{
mapOutputsCounter.Increment(1);
}
}
public Segment(IFile.Reader <K, V> reader, bool preserve, long rawDataLength)
: this(reader, preserve, null)
{
this.rawDataLength = rawDataLength;
}
public Segment(IFile.Reader <K, V> reader, bool preserve)
: this(reader, preserve, null)
{
}