Ejemplo n.º 1
0
            /// <exception cref="System.IO.IOException"/>
            public SequenceFileAsBinaryRecordReader(Configuration conf, FileSplit split)
            {
                Path       path = split.GetPath();
                FileSystem fs   = path.GetFileSystem(conf);

                this.@in = new SequenceFile.Reader(fs, path, conf);
                this.end = split.GetStart() + split.GetLength();
                if (split.GetStart() > @in.GetPosition())
                {
                    @in.Sync(split.GetStart());
                }
                // sync to start
                this.start = @in.GetPosition();
                vbytes     = @in.CreateValueBytes();
                done       = start >= end;
            }
Ejemplo n.º 2
0
            /// <exception cref="System.IO.IOException"/>
            /// <exception cref="System.Exception"/>
            public override void Initialize(InputSplit split, TaskAttemptContext context)
            {
                Path          path = ((FileSplit)split).GetPath();
                Configuration conf = context.GetConfiguration();
                FileSystem    fs   = path.GetFileSystem(conf);

                this.@in = new SequenceFile.Reader(fs, path, conf);
                this.end = ((FileSplit)split).GetStart() + split.GetLength();
                if (((FileSplit)split).GetStart() > @in.GetPosition())
                {
                    @in.Sync(((FileSplit)split).GetStart());
                }
                // sync to start
                this.start = @in.GetPosition();
                vbytes     = @in.CreateValueBytes();
                done       = start >= end;
            }
Ejemplo n.º 3
0
        /// <exception cref="System.IO.IOException"/>
        private void ReadTest(FileSystem fs, int count, int seed, Path file)
        {
            Log.Debug("reading " + count + " records");
            SequenceFile.Reader   reader    = new SequenceFile.Reader(fs, file, conf);
            RandomDatum.Generator generator = new RandomDatum.Generator(seed);
            RandomDatum           k         = new RandomDatum();
            RandomDatum           v         = new RandomDatum();
            DataOutputBuffer      rawKey    = new DataOutputBuffer();

            SequenceFile.ValueBytes rawValue = reader.CreateValueBytes();
            for (int i = 0; i < count; i++)
            {
                generator.Next();
                RandomDatum key   = generator.GetKey();
                RandomDatum value = generator.GetValue();
                try
                {
                    if ((i % 5) == 0)
                    {
                        // Testing 'raw' apis
                        rawKey.Reset();
                        reader.NextRaw(rawKey, rawValue);
                    }
                    else
                    {
                        // Testing 'non-raw' apis
                        if ((i % 2) == 0)
                        {
                            reader.Next(k);
                            reader.GetCurrentValue(v);
                        }
                        else
                        {
                            reader.Next(k, v);
                        }
                        // Check
                        if (!k.Equals(key))
                        {
                            throw new RuntimeException("wrong key at " + i);
                        }
                        if (!v.Equals(value))
                        {
                            throw new RuntimeException("wrong value at " + i);
                        }
                    }
                }
                catch (IOException ioe)
                {
                    Log.Info("Problem on row " + i);
                    Log.Info("Expected key = " + key);
                    Log.Info("Expected len = " + key.GetLength());
                    Log.Info("Actual key = " + k);
                    Log.Info("Actual len = " + k.GetLength());
                    Log.Info("Expected value = " + value);
                    Log.Info("Expected len = " + value.GetLength());
                    Log.Info("Actual value = " + v);
                    Log.Info("Actual len = " + v.GetLength());
                    Log.Info("Key equals: " + k.Equals(key));
                    Log.Info("value equals: " + v.Equals(value));
                    throw;
                }
            }
            reader.Close();
        }