/// <exception cref="System.IO.IOException"/> public virtual void Map(WritableComparable key, Writable value, OutputCollector <BytesWritable , BytesWritable> output, Reporter reporter) { int itemCount = 0; while (numBytesToWrite > 0) { int keyLength = minKeySize + (keySizeRange != 0 ? random.Next(keySizeRange) : 0); randomKey.SetSize(keyLength); RandomizeBytes(randomKey.GetBytes(), 0, randomKey.GetLength()); int valueLength = minValueSize + (valueSizeRange != 0 ? random.Next(valueSizeRange ) : 0); randomValue.SetSize(valueLength); RandomizeBytes(randomValue.GetBytes(), 0, randomValue.GetLength()); output.Collect(randomKey, randomValue); numBytesToWrite -= keyLength + valueLength; reporter.IncrCounter(ThreadedMapBenchmark.Counters.BytesWritten, 1); reporter.IncrCounter(ThreadedMapBenchmark.Counters.RecordsWritten, 1); if (++itemCount % 200 == 0) { reporter.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left." ); } } reporter.SetStatus("done with " + itemCount + " records."); }
/// <summary>Create a record reader for the given split</summary> /// <param name="split">file split</param> /// <param name="job">job configuration</param> /// <param name="reporter">reporter who sends report to task tracker</param> /// <returns>RecordReader</returns> /// <exception cref="System.IO.IOException"/> public override RecordReader <K, V> GetRecordReader(InputSplit split, JobConf job, Reporter reporter) { reporter.SetStatus(split.ToString()); return(new SequenceFileInputFilter.FilterRecordReader <K, V>(job, (FileSplit)split )); }
/// <exception cref="System.IO.IOException"/> public override RecordReader <LongWritable, BytesWritable> GetRecordReader(InputSplit genericSplit, JobConf job, Reporter reporter) { reporter.SetStatus(genericSplit.ToString()); int recordLength = GetRecordLength(job); if (recordLength <= 0) { throw new IOException("Fixed record length " + recordLength + " is invalid. It should be set to a value greater than zero" ); } return(new FixedLengthRecordReader(job, (FileSplit)genericSplit, recordLength)); }
/// <exception cref="System.IO.IOException"/> public override RecordReader <LongWritable, Text> GetRecordReader(InputSplit genericSplit , JobConf job, Reporter reporter) { reporter.SetStatus(genericSplit.ToString()); string delimiter = job.Get("textinputformat.record.delimiter"); byte[] recordDelimiterBytes = null; if (null != delimiter) { recordDelimiterBytes = Sharpen.Runtime.GetBytesForString(delimiter, Charsets.Utf8 ); } return(new LineRecordReader(job, (FileSplit)genericSplit, recordDelimiterBytes)); }
/// <exception cref="System.IO.IOException"/> public virtual void Map(Org.Apache.Hadoop.IO.Text key, Org.Apache.Hadoop.IO.Text val, OutputCollector <Org.Apache.Hadoop.IO.Text, Org.Apache.Hadoop.IO.Text> output , Reporter reporter) { long acc = 0L; long recs = 0; int keydiff = keymax - keymin; int valdiff = valmax - valmin; for (long i = 0L; acc < bytesToWrite; ++i) { int recacc = 0; recacc += GenerateSentence(key, keymin + (0 == keydiff ? 0 : r.Next(keydiff))); recacc += GenerateSentence(val, valmin + (0 == valdiff ? 0 : r.Next(valdiff))); output.Collect(key, val); ++recs; acc += recacc; reporter.IncrCounter(GenericMRLoadGenerator.Counters.BytesWritten, recacc); reporter.IncrCounter(GenericMRLoadGenerator.Counters.RecordsWritten, 1); reporter.SetStatus(acc + "/" + (bytesToWrite - acc) + " bytes"); } reporter.SetStatus("Wrote " + recs + " records"); }
/// <summary>Set the current status of the task to the given string.</summary> public override void SetStatus(string status) { SetStatusString(status); reporter.SetStatus(status); }
/// <exception cref="System.IO.IOException"/> public override RecordReader <Text, Text> GetRecordReader(InputSplit genericSplit, JobConf job, Reporter reporter) { reporter.SetStatus(genericSplit.ToString()); return(new KeyValueLineRecordReader(job, (FileSplit)genericSplit)); }