/// <exception cref="System.IO.IOException"/>
public SequenceFileAsBinaryRecordReader(Configuration conf, FileSplit split)
{
Path path = split.GetPath();
FileSystem fs = path.GetFileSystem(conf);
this.@in = new SequenceFile.Reader(fs, path, conf);
this.end = split.GetStart() + split.GetLength();
if (split.GetStart() > @in.GetPosition())
{
@in.Sync(split.GetStart());
}
// sync to start
this.start = @in.GetPosition();
vbytes = @in.CreateValueBytes();
done = start >= end;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public override void Initialize(InputSplit split, TaskAttemptContext context)
{
Path path = ((FileSplit)split).GetPath();
Configuration conf = context.GetConfiguration();
FileSystem fs = path.GetFileSystem(conf);
this.@in = new SequenceFile.Reader(fs, path, conf);
this.end = ((FileSplit)split).GetStart() + split.GetLength();
if (((FileSplit)split).GetStart() > @in.GetPosition())
{
@in.Sync(((FileSplit)split).GetStart());
}
// sync to start
this.start = @in.GetPosition();
vbytes = @in.CreateValueBytes();
done = start >= end;
}
/// <exception cref="System.IO.IOException"/>
private void ReadTest(FileSystem fs, int count, int seed, Path file)
{
Log.Debug("reading " + count + " records");
SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);
RandomDatum.Generator generator = new RandomDatum.Generator(seed);
RandomDatum k = new RandomDatum();
RandomDatum v = new RandomDatum();
DataOutputBuffer rawKey = new DataOutputBuffer();
SequenceFile.ValueBytes rawValue = reader.CreateValueBytes();
for (int i = 0; i < count; i++)
{
generator.Next();
RandomDatum key = generator.GetKey();
RandomDatum value = generator.GetValue();
try
{
if ((i % 5) == 0)
{
// Testing 'raw' apis
rawKey.Reset();
reader.NextRaw(rawKey, rawValue);
}
else
{
// Testing 'non-raw' apis
if ((i % 2) == 0)
{
reader.Next(k);
reader.GetCurrentValue(v);
}
else
{
reader.Next(k, v);
}
// Check
if (!k.Equals(key))
{
throw new RuntimeException("wrong key at " + i);
}
if (!v.Equals(value))
{
throw new RuntimeException("wrong value at " + i);
}
}
}
catch (IOException ioe)
{
Log.Info("Problem on row " + i);
Log.Info("Expected key = " + key);
Log.Info("Expected len = " + key.GetLength());
Log.Info("Actual key = " + k);
Log.Info("Actual len = " + k.GetLength());
Log.Info("Expected value = " + value);
Log.Info("Expected len = " + value.GetLength());
Log.Info("Actual value = " + v);
Log.Info("Actual len = " + v.GetLength());
Log.Info("Key equals: " + k.Equals(key));
Log.Info("value equals: " + v.Equals(value));
throw;
}
}
reader.Close();
}