/// <exception cref="System.IO.IOException"/>
public virtual void Map(WritableComparable key, Writable value, OutputCollector <BytesWritable
, BytesWritable> output, Reporter reporter)
{
int itemCount = 0;
while (numBytesToWrite > 0)
{
int keyLength = minKeySize + (keySizeRange != 0 ? random.Next(keySizeRange) : 0);
randomKey.SetSize(keyLength);
RandomizeBytes(randomKey.GetBytes(), 0, randomKey.GetLength());
int valueLength = minValueSize + (valueSizeRange != 0 ? random.Next(valueSizeRange
) : 0);
randomValue.SetSize(valueLength);
RandomizeBytes(randomValue.GetBytes(), 0, randomValue.GetLength());
output.Collect(randomKey, randomValue);
numBytesToWrite -= keyLength + valueLength;
reporter.IncrCounter(ThreadedMapBenchmark.Counters.BytesWritten, 1);
reporter.IncrCounter(ThreadedMapBenchmark.Counters.RecordsWritten, 1);
if (++itemCount % 200 == 0)
{
reporter.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left."
);
}
}
reporter.SetStatus("done with " + itemCount + " records.");
}
/// <summary>Create a record reader for the given split</summary>
/// <param name="split">file split</param>
/// <param name="job">job configuration</param>
/// <param name="reporter">reporter who sends report to task tracker</param>
/// <returns>RecordReader</returns>
/// <exception cref="System.IO.IOException"/>
public override RecordReader <K, V> GetRecordReader(InputSplit split, JobConf job,
Reporter reporter)
{
reporter.SetStatus(split.ToString());
return(new SequenceFileInputFilter.FilterRecordReader <K, V>(job, (FileSplit)split
));
}
/// <exception cref="System.IO.IOException"/>
public override RecordReader <LongWritable, BytesWritable> GetRecordReader(InputSplit
genericSplit, JobConf job, Reporter reporter)
{
reporter.SetStatus(genericSplit.ToString());
int recordLength = GetRecordLength(job);
if (recordLength <= 0)
{
throw new IOException("Fixed record length " + recordLength + " is invalid. It should be set to a value greater than zero"
);
}
return(new FixedLengthRecordReader(job, (FileSplit)genericSplit, recordLength));
}
/// <exception cref="System.IO.IOException"/>
public override RecordReader <LongWritable, Text> GetRecordReader(InputSplit genericSplit
, JobConf job, Reporter reporter)
{
reporter.SetStatus(genericSplit.ToString());
string delimiter = job.Get("textinputformat.record.delimiter");
byte[] recordDelimiterBytes = null;
if (null != delimiter)
{
recordDelimiterBytes = Sharpen.Runtime.GetBytesForString(delimiter, Charsets.Utf8
);
}
return(new LineRecordReader(job, (FileSplit)genericSplit, recordDelimiterBytes));
}
/// <exception cref="System.IO.IOException"/>
public virtual void Map(Org.Apache.Hadoop.IO.Text key, Org.Apache.Hadoop.IO.Text
val, OutputCollector <Org.Apache.Hadoop.IO.Text, Org.Apache.Hadoop.IO.Text> output
, Reporter reporter)
{
long acc = 0L;
long recs = 0;
int keydiff = keymax - keymin;
int valdiff = valmax - valmin;
for (long i = 0L; acc < bytesToWrite; ++i)
{
int recacc = 0;
recacc += GenerateSentence(key, keymin + (0 == keydiff ? 0 : r.Next(keydiff)));
recacc += GenerateSentence(val, valmin + (0 == valdiff ? 0 : r.Next(valdiff)));
output.Collect(key, val);
++recs;
acc += recacc;
reporter.IncrCounter(GenericMRLoadGenerator.Counters.BytesWritten, recacc);
reporter.IncrCounter(GenericMRLoadGenerator.Counters.RecordsWritten, 1);
reporter.SetStatus(acc + "/" + (bytesToWrite - acc) + " bytes");
}
reporter.SetStatus("Wrote " + recs + " records");
}
/// <summary>Set the current status of the task to the given string.</summary>
public override void SetStatus(string status)
{
SetStatusString(status);
reporter.SetStatus(status);
}
/// <exception cref="System.IO.IOException"/>
public override RecordReader <Text, Text> GetRecordReader(InputSplit genericSplit,
JobConf job, Reporter reporter)
{
reporter.SetStatus(genericSplit.ToString());
return(new KeyValueLineRecordReader(job, (FileSplit)genericSplit));
}
