/// <exception cref="System.IO.IOException"/> public SequenceFileAsBinaryRecordReader(Configuration conf, FileSplit split) { Path path = split.GetPath(); FileSystem fs = path.GetFileSystem(conf); this.@in = new SequenceFile.Reader(fs, path, conf); this.end = split.GetStart() + split.GetLength(); if (split.GetStart() > @in.GetPosition()) { @in.Sync(split.GetStart()); } // sync to start this.start = @in.GetPosition(); vbytes = @in.CreateValueBytes(); done = start >= end; }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public override void Initialize(InputSplit split, TaskAttemptContext context) { Path path = ((FileSplit)split).GetPath(); Configuration conf = context.GetConfiguration(); FileSystem fs = path.GetFileSystem(conf); this.@in = new SequenceFile.Reader(fs, path, conf); this.end = ((FileSplit)split).GetStart() + split.GetLength(); if (((FileSplit)split).GetStart() > @in.GetPosition()) { @in.Sync(((FileSplit)split).GetStart()); } // sync to start this.start = @in.GetPosition(); vbytes = @in.CreateValueBytes(); done = start >= end; }
/// <exception cref="System.IO.IOException"/> private void ReadTest(FileSystem fs, int count, int seed, Path file) { Log.Debug("reading " + count + " records"); SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf); RandomDatum.Generator generator = new RandomDatum.Generator(seed); RandomDatum k = new RandomDatum(); RandomDatum v = new RandomDatum(); DataOutputBuffer rawKey = new DataOutputBuffer(); SequenceFile.ValueBytes rawValue = reader.CreateValueBytes(); for (int i = 0; i < count; i++) { generator.Next(); RandomDatum key = generator.GetKey(); RandomDatum value = generator.GetValue(); try { if ((i % 5) == 0) { // Testing 'raw' apis rawKey.Reset(); reader.NextRaw(rawKey, rawValue); } else { // Testing 'non-raw' apis if ((i % 2) == 0) { reader.Next(k); reader.GetCurrentValue(v); } else { reader.Next(k, v); } // Check if (!k.Equals(key)) { throw new RuntimeException("wrong key at " + i); } if (!v.Equals(value)) { throw new RuntimeException("wrong value at " + i); } } } catch (IOException ioe) { Log.Info("Problem on row " + i); Log.Info("Expected key = " + key); Log.Info("Expected len = " + key.GetLength()); Log.Info("Actual key = " + k); Log.Info("Actual len = " + k.GetLength()); Log.Info("Expected value = " + value); Log.Info("Expected len = " + value.GetLength()); Log.Info("Actual value = " + v); Log.Info("Actual len = " + v.GetLength()); Log.Info("Key equals: " + k.Equals(key)); Log.Info("value equals: " + v.Equals(value)); throw; } } reader.Close(); }