Exemplo n.º 1
0
        public virtual void TestIFileReaderWithCodec()
        {
            Configuration conf    = new Configuration();
            FileSystem    localFs = FileSystem.GetLocal(conf);
            FileSystem    rfs     = ((LocalFileSystem)localFs).GetRaw();
            Path          path    = new Path(new Path("build/test.ifile"), "data");
            DefaultCodec  codec   = new GzipCodec();

            codec.SetConf(conf);
            FSDataOutputStream @out = rfs.Create(path);

            IFile.Writer <Text, Text> writer = new IFile.Writer <Text, Text>(conf, @out, typeof(
                                                                                 Text), typeof(Text), codec, null);
            writer.Close();
            FSDataInputStream @in = rfs.Open(path);

            IFile.Reader <Text, Text> reader = new IFile.Reader <Text, Text>(conf, @in, rfs.GetFileStatus
                                                                                 (path).GetLen(), codec, null);
            reader.Close();
            // test check sum
            byte[] ab     = new byte[100];
            int    readed = reader.checksumIn.ReadWithChecksum(ab, 0, ab.Length);

            NUnit.Framework.Assert.AreEqual(readed, reader.checksumIn.GetChecksum().Length);
        }
Exemplo n.º 2
0
 /// <exception cref="System.IO.IOException"/>
 internal virtual void CloseReader()
 {
     if (reader != null)
     {
         reader.Close();
         reader = null;
     }
 }
Exemplo n.º 3
0
 public Segment(IFile.Reader <K, V> reader, bool preserve, Counters.Counter mapOutputsCounter
                )
 {
     this.reader            = reader;
     this.preserve          = preserve;
     this.segmentLength     = reader.GetLength();
     this.mapOutputsCounter = mapOutputsCounter;
 }
Exemplo n.º 4
0
 /// <exception cref="System.IO.IOException"/>
 internal virtual void Init(Counters.Counter readsCounter)
 {
     if (reader == null)
     {
         FSDataInputStream @in = fs.Open(file);
         @in.Seek(segmentOffset);
         @in    = CryptoUtils.WrapIfNecessary(conf, @in);
         reader = new IFile.Reader <K, V>(conf, @in, segmentLength - CryptoUtils.CryptoPadding
                                              (conf), codec, readsCounter);
     }
     if (mapOutputsCounter != null)
     {
         mapOutputsCounter.Increment(1);
     }
 }
Exemplo n.º 5
0
 public Segment(IFile.Reader <K, V> reader, bool preserve, long rawDataLength)
     : this(reader, preserve, null)
 {
     this.rawDataLength = rawDataLength;
 }
Exemplo n.º 6
0
 public Segment(IFile.Reader <K, V> reader, bool preserve)
     : this(reader, preserve, null)
 {
 }