protected internal override void Setup(Mapper.Context context)
{
Configuration conf = context.GetConfiguration();
pattern = Sharpen.Pattern.Compile(conf.Get(Pattern));
group = conf.GetInt(Group, 0);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(object key, Text value, Mapper.Context context)
{
// Make one mapper slower for speculative execution
TaskAttemptID taid = context.GetTaskAttemptID();
long sleepTime = 100;
Configuration conf = context.GetConfiguration();
bool test_speculate_map = conf.GetBoolean(MRJobConfig.MapSpeculative, false);
// IF TESTING MAPPER SPECULATIVE EXECUTION:
// Make the "*_m_000000_0" attempt take much longer than the others.
// When speculative execution is enabled, this should cause the attempt
// to be killed and restarted. At that point, the attempt ID will be
// "*_m_000000_1", so sleepTime will still remain 100ms.
if ((taid.GetTaskType() == TaskType.Map) && test_speculate_map && (taid.GetTaskID
().GetId() == 0) && (taid.GetId() == 0))
{
sleepTime = 10000;
}
try
{
Sharpen.Thread.Sleep(sleepTime);
}
catch (Exception)
{
}
// Ignore
context.Write(value, new IntWritable(1));
}
/// <summary>Given an output filename, write a bunch of random records to it.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(Text key, Text value, Mapper.Context context)
{
int itemCount = 0;
while (numBytesToWrite > 0)
{
// Generate the key/value
int noWordsKey = minWordsInKey + (wordsInKeyRange != 0 ? random.Next(wordsInKeyRange
) : 0);
int noWordsValue = minWordsInValue + (wordsInValueRange != 0 ? random.Next(wordsInValueRange
) : 0);
Text keyWords = GenerateSentence(noWordsKey);
Text valueWords = GenerateSentence(noWordsValue);
// Write the sentence
context.Write(keyWords, valueWords);
numBytesToWrite -= (keyWords.GetLength() + valueWords.GetLength());
// Update counters, progress etc.
context.GetCounter(RandomTextWriter.Counters.BytesWritten).Increment(keyWords.GetLength
() + valueWords.GetLength());
context.GetCounter(RandomTextWriter.Counters.RecordsWritten).Increment(1);
if (++itemCount % 200 == 0)
{
context.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left."
);
}
}
context.SetStatus("done with " + itemCount + " records.");
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Setup(Mapper.Context context)
{
Configuration conf = context.GetConfiguration();
NUnit.Framework.Assert.AreEqual(prop, conf.Get("a"));
WriteFlag(conf, "map.setup." + name);
}
/// <summary>Map method.</summary>
/// <param name="offset">samples starting from the (offset+1)th sample.</param>
/// <param name="size">the number of samples for this map</param>
/// <param name="context">output {ture->numInside, false->numOutside}</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable offset, LongWritable size, Mapper.Context
context)
{
QuasiMonteCarlo.HaltonSequence haltonsequence = new QuasiMonteCarlo.HaltonSequence
(offset.Get());
long numInside = 0L;
long numOutside = 0L;
for (long i = 0; i < size.Get();)
{
//generate points in a unit square
double[] point = haltonsequence.NextPoint();
//count points inside/outside of the inscribed circle of the square
double x = point[0] - 0.5;
double y = point[1] - 0.5;
if (x * x + y * y > 0.25)
{
numOutside++;
}
else
{
numInside++;
}
//report status
i++;
if (i % 1000 == 0)
{
context.SetStatus("Generated " + i + " samples.");
}
}
//output map results
context.Write(new BooleanWritable(true), new LongWritable(numInside));
context.Write(new BooleanWritable(false), new LongWritable(numOutside));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(Text key, Text value, Mapper.Context context)
{
if (lastKey == null)
{
FileSplit fs = (FileSplit)context.GetInputSplit();
filename = GetFilename(fs);
context.Write(new Text(filename + ":begin"), key);
lastKey = new Text();
}
else
{
if (key.CompareTo(lastKey) < 0)
{
context.Write(Error, new Text("misorder in " + filename + " between " + TextifyBytes
(lastKey) + " and " + TextifyBytes(key)));
}
}
// compute the crc of the key and value and add it to the sum
crc32.Reset();
crc32.Update(key.GetBytes(), 0, key.GetLength());
crc32.Update(value.GetBytes(), 0, value.GetLength());
tmp.Set(crc32.GetValue());
checksum.Add(tmp);
lastKey.Set(key);
}
protected override void Cleanup(Mapper.Context context)
{
if (checksumCounter != null)
{
checksumCounter.Increment(total.GetLow8());
}
}
// this map throws IOException for input value "b"
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text value, Mapper.Context context)
{
if (value.ToString().Equals("b"))
{
throw new IOException();
}
}
/// <summary>Given an output filename, write a bunch of random records to it.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(WritableComparable key, Writable value, Mapper.Context
context)
{
int itemCount = 0;
while (numBytesToWrite > 0)
{
int keyLength = minKeySize + (keySizeRange != 0 ? random.Next(keySizeRange) : 0);
randomKey.SetSize(keyLength);
RandomizeBytes(randomKey.GetBytes(), 0, randomKey.GetLength());
int valueLength = minValueSize + (valueSizeRange != 0 ? random.Next(valueSizeRange
) : 0);
randomValue.SetSize(valueLength);
RandomizeBytes(randomValue.GetBytes(), 0, randomValue.GetLength());
context.Write(randomKey, randomValue);
numBytesToWrite -= keyLength + valueLength;
context.GetCounter(RandomWriter.Counters.BytesWritten).Increment(keyLength + valueLength
);
context.GetCounter(RandomWriter.Counters.RecordsWritten).Increment(1);
if (++itemCount % 200 == 0)
{
context.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left."
);
}
}
context.SetStatus("done with " + itemCount + " records.");
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, DBCountPageView.AccessRecord value,
Mapper.Context context)
{
Text oKey = new Text(value.url);
context.Write(oKey, One);
}
/// <exception cref="System.IO.IOException"/>
protected override void Map(Text key, Text value, Mapper.Context context)
{
crc32.Reset();
crc32.Update(key.GetBytes(), 0, key.GetLength());
crc32.Update(value.GetBytes(), 0, value.GetLength());
checksum.Set(crc32.GetValue());
sum.Add(checksum);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Cleanup(Mapper.Context context)
{
if (lastKey != null)
{
context.Write(new Text(filename + ":end"), lastKey);
context.Write(Checksum, new Text(checksum.ToString()));
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text value, Mapper.Context context)
{
context.Write(key.Get(), value.ToString());
if (value.ToString().Equals("a"))
{
mos.Write(Text, key.Get(), Text);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Setup(Mapper.Context context)
{
// Find the Mapper from the TaggedInputSplit.
TaggedInputSplit inputSplit = (TaggedInputSplit)context.GetInputSplit();
mapper = (Mapper <K1, V1, K2, V2>)ReflectionUtils.NewInstance(inputSplit.GetMapperClass
(), context.GetConfiguration());
}
/// <summary>The identify function.</summary>
/// <remarks>The identify function. Input key/value pair is written directly to output.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Map(K key, V val, Mapper.Context context)
{
FieldSelectionHelper helper = new FieldSelectionHelper(FieldSelectionHelper.emptyText
, FieldSelectionHelper.emptyText);
helper.ExtractOutputKeyValue(key.ToString(), val.ToString(), fieldSeparator, mapOutputKeyFieldList
, mapOutputValueFieldList, allMapValueFieldsFrom, ignoreInputKey, true);
context.Write(helper.GetKey(), helper.GetValue());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public MapRunner(Chain _enclosing, Mapper <KEYIN, VALUEIN, KEYOUT, VALUEOUT> mapper
, Mapper.Context mapperContext, RecordReader <KEYIN, VALUEIN> rr, RecordWriter <KEYOUT
, VALUEOUT> rw)
{
this._enclosing = _enclosing;
this.mapper = mapper;
this.rr = rr;
this.rw = rw;
this.chainContext = mapperContext;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(object key, Text value, Mapper.Context context)
{
StringTokenizer itr = new StringTokenizer(value.ToString());
while (itr.HasMoreTokens())
{
word.Set(itr.NextToken());
context.Write(word, one);
}
}
/// <summary>Save the configuration value that we need to write the data.</summary>
protected override void Setup(Mapper.Context context)
{
Configuration conf = context.GetConfiguration();
numBytesToWrite = conf.GetLong(BytesPerMap, 1 * 1024 * 1024 * 1024);
minWordsInKey = conf.GetInt(MinKey, 5);
wordsInKeyRange = (conf.GetInt(MaxKey, 10) - minWordsInKey);
minWordsInValue = conf.GetInt(MinValue, 10);
wordsInValueRange = (conf.GetInt(MaxValue, 100) - minWordsInValue);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Map(K key, Text value, Mapper.Context context)
{
string text = value.ToString();
Matcher matcher = pattern.Matcher(text);
while (matcher.Find())
{
context.Write(new Text(matcher.Group(group)), new LongWritable(1));
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text value, Mapper.Context context)
{
context.Write(key, value);
if (value.ToString().Equals("a"))
{
mos.Write(Text, key, new Text(Text));
mos.Write(Sequence, new IntWritable(1), new Text(Sequence), (Sequence + "_A"));
mos.Write(Sequence, new IntWritable(2), new Text(Sequence), (Sequence + "_B"));
}
}
/// <summary>
/// Save the values out of the configuaration that we need to write
/// the data.
/// </summary>
protected override void Setup(Mapper.Context context)
{
Configuration conf = context.GetConfiguration();
numBytesToWrite = conf.GetLong(BytesPerMap, 1 * 1024 * 1024 * 1024);
minKeySize = conf.GetInt(MinKey, 10);
keySizeRange = conf.GetInt(MaxKey, 1000) - minKeySize;
minValueSize = conf.GetInt(MinValue, 0);
valueSizeRange = conf.GetInt(MaxValue, 20000) - minValueSize;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
internal virtual void RunMapper(TaskInputOutputContext context, int index)
{
Mapper mapper = mappers[index];
RecordReader rr = new Chain.ChainRecordReader(context);
RecordWriter rw = new Chain.ChainRecordWriter(context);
Mapper.Context mapperContext = CreateMapContext(rr, rw, context, GetConf(index));
mapper.Run(mapperContext);
rr.Close();
rw.Close(context);
}
/// <summary>Emits a key-value pair for counting the word.</summary>
/// <remarks>
/// Emits a key-value pair for counting the word. Outputs are (IntWritable,
/// IntWritable).
/// </remarks>
/// <param name="value">This will be a line of text coming in from our input file.</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(object key, Text value, Mapper.Context context)
{
StringTokenizer itr = new StringTokenizer(value.ToString());
while (itr.HasMoreTokens())
{
string @string = itr.NextToken();
length.Set(@string.Length);
context.Write(length, One);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(object key, Text value, Mapper.Context context)
{
context.GetCounter("MyCounterGroup", "MAP_INPUT_RECORDS").Increment(1);
StringTokenizer iter = new StringTokenizer(value.ToString());
while (iter.HasMoreTokens())
{
word.Set(iter.NextToken());
context.Write(word, one);
context.GetCounter("MyCounterGroup", "MAP_OUTPUT_RECORDS").Increment(1);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text value, Mapper.Context context)
{
if (ioEx)
{
throw new IOException();
}
if (rtEx)
{
throw new RuntimeException();
}
base.Map(key, value, context);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Setup(Mapper.Context context)
{
base.Setup(context);
Configuration conf = context.GetConfiguration();
string ioSortMb = conf.Get(MRJobConfig.IoSortMb);
if (!TestIoSortMb.Equals(ioSortMb))
{
throw new IOException("io.sort.mb expected: " + TestIoSortMb + ", actual: " + ioSortMb
);
}
}
protected override void Setup(Mapper.Context context)
{
this.context = context;
Configuration conf = context.GetConfiguration();
depth = conf.GetInt(Pentomino.Depth, PentDepth);
width = conf.GetInt(Pentomino.Width, PentWidth);
height = conf.GetInt(Pentomino.Height, PentHeight);
pent = (Pentomino)ReflectionUtils.NewInstance(conf.GetClass(Pentomino.Class, typeof(
OneSidedPentomino)), conf);
pent.Initialize(width, height);
pent.SetPrinter(new DistributedPentomino.PentMap.SolutionCatcher(this));
}
/// <summary>
/// Emits 3 key-value pairs for counting the word, its length, and the
/// squares of its length.
/// </summary>
/// <remarks>
/// Emits 3 key-value pairs for counting the word, its length, and the
/// squares of its length. Outputs are (Text, LongWritable).
/// </remarks>
/// <param name="value">This will be a line of text coming in from our input file.</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(object key, Text value, Mapper.Context context)
{
StringTokenizer itr = new StringTokenizer(value.ToString());
while (itr.HasMoreTokens())
{
string @string = itr.NextToken();
this.wordLen.Set(@string.Length);
// the square of an integer is an integer...
this.wordLenSq.Set((long)Math.Pow(@string.Length, 2.0));
context.Write(Length, this.wordLen);
context.Write(Square, this.wordLenSq);
context.Write(Count, One);
}
}
/// <summary>
/// Add mapper(the first mapper) that reads input from the input
/// context and writes to queue
/// </summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
internal virtual void AddMapper(TaskInputOutputContext inputContext, Chain.ChainBlockingQueue
<Chain.KeyValuePair <object, object> > output, int index)
{
Configuration conf = GetConf(index);
Type keyOutClass = conf.GetClass(MapperOutputKeyClass, typeof(object));
Type valueOutClass = conf.GetClass(MapperOutputValueClass, typeof(object));
RecordReader rr = new Chain.ChainRecordReader(inputContext);
RecordWriter rw = new Chain.ChainRecordWriter(keyOutClass, valueOutClass, output,
conf);
Mapper.Context mapperContext = CreateMapContext(rr, rw, (MapContext)inputContext,
GetConf(index));
Chain.MapRunner runner = new Chain.MapRunner(this, mappers[index], mapperContext,
rr, rw);
threads.AddItem(runner);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
internal MapRunner(MultithreadedMapper <K1, V1, K2, V2> _enclosing, Mapper.Context
context)
{
this._enclosing = _enclosing;
reader = new MultithreadedMapper.SubMapRecordReader(this);
this.mapper = ReflectionUtils.NewInstance(this._enclosing.mapClass, context.GetConfiguration
());
MapContext <K1, V1, K2, V2> mapContext = new MapContextImpl <K1, V1, K2, V2>(this._enclosing
.outer.GetConfiguration(), this._enclosing.outer.GetTaskAttemptID(), this.reader
, new MultithreadedMapper.SubMapRecordWriter(this), context.GetOutputCommitter()
, new MultithreadedMapper.SubMapStatusReporter(this), this._enclosing.outer.GetInputSplit
());
this.subcontext = new WrappedMapper <K1, V1, K2, V2>().GetMapContext(mapContext);
this.reader.Initialize(context.GetInputSplit(), context);
}
