public virtual void TestIFileReaderWithCodec() { Configuration conf = new Configuration(); FileSystem localFs = FileSystem.GetLocal(conf); FileSystem rfs = ((LocalFileSystem)localFs).GetRaw(); Path path = new Path(new Path("build/test.ifile"), "data"); DefaultCodec codec = new GzipCodec(); codec.SetConf(conf); FSDataOutputStream @out = rfs.Create(path); IFile.Writer <Text, Text> writer = new IFile.Writer <Text, Text>(conf, @out, typeof( Text), typeof(Text), codec, null); writer.Close(); FSDataInputStream @in = rfs.Open(path); IFile.Reader <Text, Text> reader = new IFile.Reader <Text, Text>(conf, @in, rfs.GetFileStatus (path).GetLen(), codec, null); reader.Close(); // test check sum byte[] ab = new byte[100]; int readed = reader.checksumIn.ReadWithChecksum(ab, 0, ab.Length); NUnit.Framework.Assert.AreEqual(readed, reader.checksumIn.GetChecksum().Length); }
/// <exception cref="System.IO.IOException"/> internal virtual void CloseReader() { if (reader != null) { reader.Close(); reader = null; } }
public Segment(IFile.Reader <K, V> reader, bool preserve, Counters.Counter mapOutputsCounter ) { this.reader = reader; this.preserve = preserve; this.segmentLength = reader.GetLength(); this.mapOutputsCounter = mapOutputsCounter; }
/// <exception cref="System.IO.IOException"/> internal virtual void Init(Counters.Counter readsCounter) { if (reader == null) { FSDataInputStream @in = fs.Open(file); @in.Seek(segmentOffset); @in = CryptoUtils.WrapIfNecessary(conf, @in); reader = new IFile.Reader <K, V>(conf, @in, segmentLength - CryptoUtils.CryptoPadding (conf), codec, readsCounter); } if (mapOutputsCounter != null) { mapOutputsCounter.Increment(1); } }
public Segment(IFile.Reader <K, V> reader, bool preserve, long rawDataLength) : this(reader, preserve, null) { this.rawDataLength = rawDataLength; }
public Segment(IFile.Reader <K, V> reader, bool preserve) : this(reader, preserve, null) { }