public virtual void TestInitNextRecordReader()
        {
            JobConf conf = new JobConf();

            Path[]       paths      = new Path[3];
            long[]       fileLength = new long[3];
            FilePath[]   files      = new FilePath[3];
            LongWritable key        = new LongWritable(1);
            Text         value      = new Text();

            try
            {
                for (int i = 0; i < 3; i++)
                {
                    fileLength[i] = i;
                    FilePath dir = new FilePath(outDir.ToString());
                    dir.Mkdir();
                    files[i] = new FilePath(dir, "testfile" + i);
                    FileWriter fileWriter = new FileWriter(files[i]);
                    fileWriter.Close();
                    paths[i] = new Path(outDir + "/testfile" + i);
                }
                CombineFileSplit        combineFileSplit = new CombineFileSplit(conf, paths, fileLength);
                Reporter                reporter         = Org.Mockito.Mockito.Mock <Reporter>();
                CombineFileRecordReader cfrr             = new CombineFileRecordReader(conf, combineFileSplit
                                                                                       , reporter, typeof(TestCombineFileRecordReader.TextRecordReaderWrapper));
                Org.Mockito.Mockito.Verify(reporter).Progress();
                NUnit.Framework.Assert.IsFalse(cfrr.Next(key, value));
                Org.Mockito.Mockito.Verify(reporter, Org.Mockito.Mockito.Times(3)).Progress();
            }
            finally
            {
                FileUtil.FullyDelete(new FilePath(outDir.ToString()));
            }
        }
        /// <exception cref="System.IO.IOException"/>
        protected internal CombineFileRecordReaderWrapper(FileInputFormat <K, V> inputFormat
                                                          , CombineFileSplit split, Configuration conf, Reporter reporter, int idx)
        {
            FileSplit fileSplit = new FileSplit(split.GetPath(idx), split.GetOffset(idx), split
                                                .GetLength(idx), split.GetLocations());

            delegate_ = inputFormat.GetRecordReader(fileSplit, (JobConf)conf, reporter);
        }
Exemplo n.º 3
0
        /// <exception cref="System.IO.IOException"/>
        public virtual InputSplit[] GetSplits(JobConf job, int numSplits)
        {
            IList <InputSplit> newStyleSplits = base.GetSplits(Job.GetInstance(job));

            InputSplit[] ret = new InputSplit[newStyleSplits.Count];
            for (int pos = 0; pos < newStyleSplits.Count; ++pos)
            {
                CombineFileSplit newStyleSplit = (CombineFileSplit)newStyleSplits[pos];
                ret[pos] = new CombineFileSplit(job, newStyleSplit.GetPaths(), newStyleSplit.GetStartOffsets
                                                    (), newStyleSplit.GetLengths(), newStyleSplit.GetLocations());
            }
            return(ret);
        }
 /// <summary>
 /// A generic RecordReader that can hand out different recordReaders
 /// for each chunk in the CombineFileSplit.
 /// </summary>
 /// <exception cref="System.IO.IOException"/>
 public CombineFileRecordReader(JobConf job, CombineFileSplit split, Reporter reporter
                                , Type rrClass)
 {
     this.split     = split;
     this.jc        = job;
     this.reporter  = reporter;
     this.idx       = 0;
     this.curReader = null;
     this.progress  = 0;
     try
     {
         rrConstructor = rrClass.GetDeclaredConstructor(constructorSignature);
     }
     catch (Exception e)
     {
         throw new RuntimeException(rrClass.FullName + " does not have valid constructor",
                                    e);
     }
     InitNextRecordReader();
 }
Exemplo n.º 5
0
 /// <exception cref="System.IO.IOException"/>
 public TextRecordReaderWrapper(CombineFileSplit split, Configuration conf, Reporter
                                reporter, int idx)
     : base(new TextInputFormat(), split, conf, reporter, idx)
 {
 }
 /// <exception cref="System.IO.IOException"/>
 public SequenceFileRecordReaderWrapper(CombineFileSplit split, Configuration conf
                                        , Reporter reporter, int idx)
     : base(new SequenceFileInputFormat <K, V>(), split, conf, reporter, idx)
 {
 }