/// <summary>Map method.</summary>
/// <param name="offset">samples starting from the (offset+1)th sample.</param>
/// <param name="size">the number of samples for this map</param>
/// <param name="context">output {ture->numInside, false->numOutside}</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable offset, LongWritable size, Mapper.Context
context)
{
QuasiMonteCarlo.HaltonSequence haltonsequence = new QuasiMonteCarlo.HaltonSequence
(offset.Get());
long numInside = 0L;
long numOutside = 0L;
for (long i = 0; i < size.Get();)
{
//generate points in a unit square
double[] point = haltonsequence.NextPoint();
//count points inside/outside of the inscribed circle of the square
double x = point[0] - 0.5;
double y = point[1] - 0.5;
if (x * x + y * y > 0.25)
{
numOutside++;
}
else
{
numInside++;
}
//report status
i++;
if (i % 1000 == 0)
{
context.SetStatus("Generated " + i + " samples.");
}
}
//output map results
context.Write(new BooleanWritable(true), new LongWritable(numInside));
context.Write(new BooleanWritable(false), new LongWritable(numOutside));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(Text key, Text value, Mapper.Context context)
{
if (lastKey == null)
{
FileSplit fs = (FileSplit)context.GetInputSplit();
filename = GetFilename(fs);
context.Write(new Text(filename + ":begin"), key);
lastKey = new Text();
}
else
{
if (key.CompareTo(lastKey) < 0)
{
context.Write(Error, new Text("misorder in " + filename + " between " + TextifyBytes
(lastKey) + " and " + TextifyBytes(key)));
}
}
// compute the crc of the key and value and add it to the sum
crc32.Reset();
crc32.Update(key.GetBytes(), 0, key.GetLength());
crc32.Update(value.GetBytes(), 0, value.GetLength());
tmp.Set(crc32.GetValue());
checksum.Add(tmp);
lastKey.Set(key);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Cleanup(Mapper.Context context)
{
if (lastKey != null)
{
context.Write(new Text(filename + ":end"), lastKey);
context.Write(Checksum, new Text(checksum.ToString()));
}
}
/// <summary>Emits 2 key-value pairs for counting the word and its length.</summary>
/// <remarks>
/// Emits 2 key-value pairs for counting the word and its length. Outputs are
/// (Text, LongWritable).
/// </remarks>
/// <param name="value">This will be a line of text coming in from our input file.</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(object key, Text value, Mapper.Context context)
{
StringTokenizer itr = new StringTokenizer(value.ToString());
while (itr.HasMoreTokens())
{
string @string = itr.NextToken();
this.wordLen.Set(@string.Length);
context.Write(Length, this.wordLen);
context.Write(Count, One);
}
}
/// <summary>
/// Emits 3 key-value pairs for counting the word, its length, and the
/// squares of its length.
/// </summary>
/// <remarks>
/// Emits 3 key-value pairs for counting the word, its length, and the
/// squares of its length. Outputs are (Text, LongWritable).
/// </remarks>
/// <param name="value">This will be a line of text coming in from our input file.</param>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(object key, Text value, Mapper.Context context)
{
StringTokenizer itr = new StringTokenizer(value.ToString());
while (itr.HasMoreTokens())
{
string @string = itr.NextToken();
this.wordLen.Set(@string.Length);
// the square of an integer is an integer...
this.wordLenSq.Set((long)Math.Pow(@string.Length, 2.0));
context.Write(Length, this.wordLen);
context.Write(Square, this.wordLenSq);
context.Write(Count, One);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, DBCountPageView.AccessRecord value,
Mapper.Context context)
{
Text oKey = new Text(value.url);
context.Write(oKey, One);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(object key, Text value, Mapper.Context context)
{
// Make one mapper slower for speculative execution
TaskAttemptID taid = context.GetTaskAttemptID();
long sleepTime = 100;
Configuration conf = context.GetConfiguration();
bool test_speculate_map = conf.GetBoolean(MRJobConfig.MapSpeculative, false);
// IF TESTING MAPPER SPECULATIVE EXECUTION:
// Make the "*_m_000000_0" attempt take much longer than the others.
// When speculative execution is enabled, this should cause the attempt
// to be killed and restarted. At that point, the attempt ID will be
// "*_m_000000_1", so sleepTime will still remain 100ms.
if ((taid.GetTaskType() == TaskType.Map) && test_speculate_map && (taid.GetTaskID
().GetId() == 0) && (taid.GetId() == 0))
{
sleepTime = 10000;
}
try
{
Sharpen.Thread.Sleep(sleepTime);
}
catch (Exception)
{
}
// Ignore
context.Write(value, new IntWritable(1));
}
/// <summary>Given an output filename, write a bunch of random records to it.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(Text key, Text value, Mapper.Context context)
{
int itemCount = 0;
while (numBytesToWrite > 0)
{
// Generate the key/value
int noWordsKey = minWordsInKey + (wordsInKeyRange != 0 ? random.Next(wordsInKeyRange
) : 0);
int noWordsValue = minWordsInValue + (wordsInValueRange != 0 ? random.Next(wordsInValueRange
) : 0);
Text keyWords = GenerateSentence(noWordsKey);
Text valueWords = GenerateSentence(noWordsValue);
// Write the sentence
context.Write(keyWords, valueWords);
numBytesToWrite -= (keyWords.GetLength() + valueWords.GetLength());
// Update counters, progress etc.
context.GetCounter(RandomTextWriter.Counters.BytesWritten).Increment(keyWords.GetLength
() + valueWords.GetLength());
context.GetCounter(RandomTextWriter.Counters.RecordsWritten).Increment(1);
if (++itemCount % 200 == 0)
{
context.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left."
);
}
}
context.SetStatus("done with " + itemCount + " records.");
}
/// <summary>Given an output filename, write a bunch of random records to it.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(WritableComparable key, Writable value, Mapper.Context
context)
{
int itemCount = 0;
while (numBytesToWrite > 0)
{
int keyLength = minKeySize + (keySizeRange != 0 ? random.Next(keySizeRange) : 0);
randomKey.SetSize(keyLength);
RandomizeBytes(randomKey.GetBytes(), 0, randomKey.GetLength());
int valueLength = minValueSize + (valueSizeRange != 0 ? random.Next(valueSizeRange
) : 0);
randomValue.SetSize(valueLength);
RandomizeBytes(randomValue.GetBytes(), 0, randomValue.GetLength());
context.Write(randomKey, randomValue);
numBytesToWrite -= keyLength + valueLength;
context.GetCounter(RandomWriter.Counters.BytesWritten).Increment(keyLength + valueLength
);
context.GetCounter(RandomWriter.Counters.RecordsWritten).Increment(1);
if (++itemCount % 200 == 0)
{
context.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left."
);
}
}
context.SetStatus("done with " + itemCount + " records.");
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text value, Mapper.Context context)
{
context.Write(key.Get(), value.ToString());
if (value.ToString().Equals("a"))
{
mos.Write(Text, key.Get(), Text);
}
}
/// <summary>The identify function.</summary>
/// <remarks>The identify function. Input key/value pair is written directly to output.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Map(K key, V val, Mapper.Context context)
{
FieldSelectionHelper helper = new FieldSelectionHelper(FieldSelectionHelper.emptyText
, FieldSelectionHelper.emptyText);
helper.ExtractOutputKeyValue(key.ToString(), val.ToString(), fieldSeparator, mapOutputKeyFieldList
, mapOutputValueFieldList, allMapValueFieldsFrom, ignoreInputKey, true);
context.Write(helper.GetKey(), helper.GetValue());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Map(K key, Text value, Mapper.Context context)
{
string text = value.ToString();
Matcher matcher = pattern.Matcher(text);
while (matcher.Find())
{
context.Write(new Text(matcher.Group(group)), new LongWritable(1));
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text value, Mapper.Context context)
{
context.Write(key, value);
if (value.ToString().Equals("a"))
{
mos.Write(Text, key, new Text(Text));
mos.Write(Sequence, new IntWritable(1), new Text(Sequence), (Sequence + "_A"));
mos.Write(Sequence, new IntWritable(2), new Text(Sequence), (Sequence + "_B"));
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(object key, Text value, Mapper.Context context)
{
StringTokenizer itr = new StringTokenizer(value.ToString());
while (itr.HasMoreTokens())
{
word.Set(itr.NextToken());
context.Write(word, one);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(object key, Text value, Mapper.Context context)
{
context.GetCounter("MyCounterGroup", "MAP_INPUT_RECORDS").Increment(1);
StringTokenizer iter = new StringTokenizer(value.ToString());
while (iter.HasMoreTokens())
{
word.Set(iter.NextToken());
context.Write(word, one);
context.GetCounter("MyCounterGroup", "MAP_OUTPUT_RECORDS").Increment(1);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(MultiFileWordCount.WordOffset key, Text value, Mapper.Context
context)
{
string line = value.ToString();
StringTokenizer itr = new StringTokenizer(line);
while (itr.HasMoreTokens())
{
word.Set(itr.NextToken());
context.Write(word, one);
}
}
/// <summary>Partitions sigma into parts</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(NullWritable nw, SummationWritable sigma, Mapper.Context
context)
{
Configuration conf = context.GetConfiguration();
int nParts = conf.GetInt(NParts, 0);
Summation[] parts = sigma.GetElement().Partition(nParts);
for (int i = 0; i < parts.Length; ++i)
{
context.Write(new IntWritable(i), new SummationWritable(parts[i]));
Log.Info("parts[" + i + "] = " + parts[i]);
}
}
//custom job properties
/// <summary>Compute the (offset+1)th to (offset+length)th digits.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable offset, IntWritable length, Mapper.Context
context)
{
Log.Info("offset=" + offset + ", length=" + length);
// compute digits
byte[] bytes = new byte[length.Get() >> 1];
long d = offset.Get();
for (int i = 0; i < bytes.Length; d += 4)
{
long digits = HexDigits(d);
bytes[i++] = unchecked ((byte)(digits >> 8));
bytes[i++] = unchecked ((byte)digits);
}
// output map results
context.Write(offset, new BytesWritable(bytes));
}
/// <summary>the map function.</summary>
/// <remarks>
/// the map function. It iterates through the value aggregator descriptor
/// list to generate aggregation id/value pairs and emit them.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Map(K1 key, V1 value, Mapper.Context context)
{
IEnumerator <object> iter = ValueAggregatorJobBase.aggregatorDescriptorList.GetEnumerator
();
while (iter.HasNext())
{
ValueAggregatorDescriptor ad = (ValueAggregatorDescriptor)iter.Next();
IEnumerator <KeyValuePair <Text, Text> > ens = ad.GenerateKeyValPairs(key, value).GetEnumerator
();
while (ens.HasNext())
{
KeyValuePair <Text, Text> en = ens.Next();
context.Write(en.Key, en.Value);
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable inKey, Text inValue, Mapper.Context context
)
{
StringTokenizer itr = new StringTokenizer(inValue.ToString());
int left = 0;
int right = 0;
if (itr.HasMoreTokens())
{
left = System.Convert.ToInt32(itr.NextToken());
if (itr.HasMoreTokens())
{
right = System.Convert.ToInt32(itr.NextToken());
}
key.Set(left, right);
value.Set(right);
context.Write(key, value);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable row, NullWritable ignored, Mapper.Context
context)
{
if (rand == null)
{
rowId = new Unsigned16(row.Get());
rand = Random16.SkipAhead(rowId);
checksumCounter = context.GetCounter(TeraGen.Counters.Checksum);
}
Random16.NextRand(rand);
GenSort.GenerateRecord(buffer, rand, rowId);
key.Set(buffer, 0, TeraInputFormat.KeyLength);
value.Set(buffer, TeraInputFormat.KeyLength, TeraInputFormat.ValueLength);
context.Write(key, value);
crc32.Reset();
crc32.Update(buffer, 0, TeraInputFormat.KeyLength + TeraInputFormat.ValueLength);
checksum.Set(crc32.GetValue());
total.Add(checksum);
rowId.Add(One);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(IntWritable key, TupleWritable val, Mapper.Context context
)
{
int k = key.Get();
string kvstr = "Unexpected tuple: " + Stringify(key, val);
NUnit.Framework.Assert.IsTrue(kvstr, 0 == k % (srcs * srcs));
for (int i = 0; i < val.Size(); ++i)
{
int vali = ((IntWritable)val.Get(i)).Get();
NUnit.Framework.Assert.IsTrue(kvstr, (vali - i) * srcs == 10 * k);
}
context.Write(key, one);
// If the user modifies the key or any of the values in the tuple, it
// should not affect the rest of the join.
key.Set(-1);
if (val.Has(0))
{
((IntWritable)val.Get(0)).Set(0);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(IntWritable key, IntWritable val, Mapper.Context context
)
{
int k = key.Get();
int vali = val.Get();
string kvstr = "Unexpected tuple: " + Stringify(key, val);
if (0 == k % (srcs * srcs))
{
NUnit.Framework.Assert.IsTrue(kvstr, vali == k * 10 / srcs + srcs - 1);
}
else
{
int i = k % srcs;
NUnit.Framework.Assert.IsTrue(kvstr, srcs * (vali - i) == 10 * (k - i));
}
context.Write(key, one);
//If the user modifies the key or any of the values in the tuple, it
// should not affect the rest of the join.
key.Set(-1);
val.Set(0);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text value, Mapper.Context context)
{
WriteFlag(context.GetConfiguration(), "map." + name + ".value." + value);
context.Write(key, new Text(value + name));
}
// these 3 classes do a reduce side join with 2 different mappers
// receives "a", "b", "c" as values
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text value, Mapper.Context ctx)
{
ctx.Write(value, blah);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(object k, object v, Mapper.Context c)
{
c.Write(v, NullWritable.Get());
}
/// <summary>The inverse function.</summary>
/// <remarks>The inverse function. Input keys and values are swapped.</remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal override void Map(K key, V value, Mapper.Context context)
{
context.Write(value, key);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Cleanup(Mapper.Context context)
{
context.Write(NullWritable.Get(), sum);
}
// receives "a", "b", "c" as keys
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(Text key, Text value, Mapper.Context ctx)
{
ctx.Write(key, blah);
}