/// <summary>Map method.</summary> /// <param name="offset">samples starting from the (offset+1)th sample.</param> /// <param name="size">the number of samples for this map</param> /// <param name="context">output {ture->numInside, false->numOutside}</param> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable offset, LongWritable size, Mapper.Context context) { QuasiMonteCarlo.HaltonSequence haltonsequence = new QuasiMonteCarlo.HaltonSequence (offset.Get()); long numInside = 0L; long numOutside = 0L; for (long i = 0; i < size.Get();) { //generate points in a unit square double[] point = haltonsequence.NextPoint(); //count points inside/outside of the inscribed circle of the square double x = point[0] - 0.5; double y = point[1] - 0.5; if (x * x + y * y > 0.25) { numOutside++; } else { numInside++; } //report status i++; if (i % 1000 == 0) { context.SetStatus("Generated " + i + " samples."); } } //output map results context.Write(new BooleanWritable(true), new LongWritable(numInside)); context.Write(new BooleanWritable(false), new LongWritable(numOutside)); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(Text key, Text value, Mapper.Context context) { if (lastKey == null) { FileSplit fs = (FileSplit)context.GetInputSplit(); filename = GetFilename(fs); context.Write(new Text(filename + ":begin"), key); lastKey = new Text(); } else { if (key.CompareTo(lastKey) < 0) { context.Write(Error, new Text("misorder in " + filename + " between " + TextifyBytes (lastKey) + " and " + TextifyBytes(key))); } } // compute the crc of the key and value and add it to the sum crc32.Reset(); crc32.Update(key.GetBytes(), 0, key.GetLength()); crc32.Update(value.GetBytes(), 0, value.GetLength()); tmp.Set(crc32.GetValue()); checksum.Add(tmp); lastKey.Set(key); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Cleanup(Mapper.Context context) { if (lastKey != null) { context.Write(new Text(filename + ":end"), lastKey); context.Write(Checksum, new Text(checksum.ToString())); } }
/// <summary>Emits 2 key-value pairs for counting the word and its length.</summary> /// <remarks> /// Emits 2 key-value pairs for counting the word and its length. Outputs are /// (Text, LongWritable). /// </remarks> /// <param name="value">This will be a line of text coming in from our input file.</param> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(object key, Text value, Mapper.Context context) { StringTokenizer itr = new StringTokenizer(value.ToString()); while (itr.HasMoreTokens()) { string @string = itr.NextToken(); this.wordLen.Set(@string.Length); context.Write(Length, this.wordLen); context.Write(Count, One); } }
/// <summary> /// Emits 3 key-value pairs for counting the word, its length, and the /// squares of its length. /// </summary> /// <remarks> /// Emits 3 key-value pairs for counting the word, its length, and the /// squares of its length. Outputs are (Text, LongWritable). /// </remarks> /// <param name="value">This will be a line of text coming in from our input file.</param> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(object key, Text value, Mapper.Context context) { StringTokenizer itr = new StringTokenizer(value.ToString()); while (itr.HasMoreTokens()) { string @string = itr.NextToken(); this.wordLen.Set(@string.Length); // the square of an integer is an integer... this.wordLenSq.Set((long)Math.Pow(@string.Length, 2.0)); context.Write(Length, this.wordLen); context.Write(Square, this.wordLenSq); context.Write(Count, One); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable key, DBCountPageView.AccessRecord value, Mapper.Context context) { Text oKey = new Text(value.url); context.Write(oKey, One); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(object key, Text value, Mapper.Context context) { // Make one mapper slower for speculative execution TaskAttemptID taid = context.GetTaskAttemptID(); long sleepTime = 100; Configuration conf = context.GetConfiguration(); bool test_speculate_map = conf.GetBoolean(MRJobConfig.MapSpeculative, false); // IF TESTING MAPPER SPECULATIVE EXECUTION: // Make the "*_m_000000_0" attempt take much longer than the others. // When speculative execution is enabled, this should cause the attempt // to be killed and restarted. At that point, the attempt ID will be // "*_m_000000_1", so sleepTime will still remain 100ms. if ((taid.GetTaskType() == TaskType.Map) && test_speculate_map && (taid.GetTaskID ().GetId() == 0) && (taid.GetId() == 0)) { sleepTime = 10000; } try { Sharpen.Thread.Sleep(sleepTime); } catch (Exception) { } // Ignore context.Write(value, new IntWritable(1)); }
/// <summary>Given an output filename, write a bunch of random records to it.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(Text key, Text value, Mapper.Context context) { int itemCount = 0; while (numBytesToWrite > 0) { // Generate the key/value int noWordsKey = minWordsInKey + (wordsInKeyRange != 0 ? random.Next(wordsInKeyRange ) : 0); int noWordsValue = minWordsInValue + (wordsInValueRange != 0 ? random.Next(wordsInValueRange ) : 0); Text keyWords = GenerateSentence(noWordsKey); Text valueWords = GenerateSentence(noWordsValue); // Write the sentence context.Write(keyWords, valueWords); numBytesToWrite -= (keyWords.GetLength() + valueWords.GetLength()); // Update counters, progress etc. context.GetCounter(RandomTextWriter.Counters.BytesWritten).Increment(keyWords.GetLength () + valueWords.GetLength()); context.GetCounter(RandomTextWriter.Counters.RecordsWritten).Increment(1); if (++itemCount % 200 == 0) { context.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left." ); } } context.SetStatus("done with " + itemCount + " records."); }
/// <summary>Given an output filename, write a bunch of random records to it.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(WritableComparable key, Writable value, Mapper.Context context) { int itemCount = 0; while (numBytesToWrite > 0) { int keyLength = minKeySize + (keySizeRange != 0 ? random.Next(keySizeRange) : 0); randomKey.SetSize(keyLength); RandomizeBytes(randomKey.GetBytes(), 0, randomKey.GetLength()); int valueLength = minValueSize + (valueSizeRange != 0 ? random.Next(valueSizeRange ) : 0); randomValue.SetSize(valueLength); RandomizeBytes(randomValue.GetBytes(), 0, randomValue.GetLength()); context.Write(randomKey, randomValue); numBytesToWrite -= keyLength + valueLength; context.GetCounter(RandomWriter.Counters.BytesWritten).Increment(keyLength + valueLength ); context.GetCounter(RandomWriter.Counters.RecordsWritten).Increment(1); if (++itemCount % 200 == 0) { context.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left." ); } } context.SetStatus("done with " + itemCount + " records."); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable key, Text value, Mapper.Context context) { context.Write(key.Get(), value.ToString()); if (value.ToString().Equals("a")) { mos.Write(Text, key.Get(), Text); } }
/// <summary>The identify function.</summary> /// <remarks>The identify function. Input key/value pair is written directly to output. /// </remarks> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected internal override void Map(K key, V val, Mapper.Context context) { FieldSelectionHelper helper = new FieldSelectionHelper(FieldSelectionHelper.emptyText , FieldSelectionHelper.emptyText); helper.ExtractOutputKeyValue(key.ToString(), val.ToString(), fieldSeparator, mapOutputKeyFieldList , mapOutputValueFieldList, allMapValueFieldsFrom, ignoreInputKey, true); context.Write(helper.GetKey(), helper.GetValue()); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected internal override void Map(K key, Text value, Mapper.Context context) { string text = value.ToString(); Matcher matcher = pattern.Matcher(text); while (matcher.Find()) { context.Write(new Text(matcher.Group(group)), new LongWritable(1)); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable key, Text value, Mapper.Context context) { context.Write(key, value); if (value.ToString().Equals("a")) { mos.Write(Text, key, new Text(Text)); mos.Write(Sequence, new IntWritable(1), new Text(Sequence), (Sequence + "_A")); mos.Write(Sequence, new IntWritable(2), new Text(Sequence), (Sequence + "_B")); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(object key, Text value, Mapper.Context context) { StringTokenizer itr = new StringTokenizer(value.ToString()); while (itr.HasMoreTokens()) { word.Set(itr.NextToken()); context.Write(word, one); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(object key, Text value, Mapper.Context context) { context.GetCounter("MyCounterGroup", "MAP_INPUT_RECORDS").Increment(1); StringTokenizer iter = new StringTokenizer(value.ToString()); while (iter.HasMoreTokens()) { word.Set(iter.NextToken()); context.Write(word, one); context.GetCounter("MyCounterGroup", "MAP_OUTPUT_RECORDS").Increment(1); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(MultiFileWordCount.WordOffset key, Text value, Mapper.Context context) { string line = value.ToString(); StringTokenizer itr = new StringTokenizer(line); while (itr.HasMoreTokens()) { word.Set(itr.NextToken()); context.Write(word, one); } }
/// <summary>Partitions sigma into parts</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(NullWritable nw, SummationWritable sigma, Mapper.Context context) { Configuration conf = context.GetConfiguration(); int nParts = conf.GetInt(NParts, 0); Summation[] parts = sigma.GetElement().Partition(nParts); for (int i = 0; i < parts.Length; ++i) { context.Write(new IntWritable(i), new SummationWritable(parts[i])); Log.Info("parts[" + i + "] = " + parts[i]); } }
//custom job properties /// <summary>Compute the (offset+1)th to (offset+length)th digits.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable offset, IntWritable length, Mapper.Context context) { Log.Info("offset=" + offset + ", length=" + length); // compute digits byte[] bytes = new byte[length.Get() >> 1]; long d = offset.Get(); for (int i = 0; i < bytes.Length; d += 4) { long digits = HexDigits(d); bytes[i++] = unchecked ((byte)(digits >> 8)); bytes[i++] = unchecked ((byte)digits); } // output map results context.Write(offset, new BytesWritable(bytes)); }
/// <summary>the map function.</summary> /// <remarks> /// the map function. It iterates through the value aggregator descriptor /// list to generate aggregation id/value pairs and emit them. /// </remarks> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected internal override void Map(K1 key, V1 value, Mapper.Context context) { IEnumerator <object> iter = ValueAggregatorJobBase.aggregatorDescriptorList.GetEnumerator (); while (iter.HasNext()) { ValueAggregatorDescriptor ad = (ValueAggregatorDescriptor)iter.Next(); IEnumerator <KeyValuePair <Text, Text> > ens = ad.GenerateKeyValPairs(key, value).GetEnumerator (); while (ens.HasNext()) { KeyValuePair <Text, Text> en = ens.Next(); context.Write(en.Key, en.Value); } } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable inKey, Text inValue, Mapper.Context context ) { StringTokenizer itr = new StringTokenizer(inValue.ToString()); int left = 0; int right = 0; if (itr.HasMoreTokens()) { left = System.Convert.ToInt32(itr.NextToken()); if (itr.HasMoreTokens()) { right = System.Convert.ToInt32(itr.NextToken()); } key.Set(left, right); value.Set(right); context.Write(key, value); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable row, NullWritable ignored, Mapper.Context context) { if (rand == null) { rowId = new Unsigned16(row.Get()); rand = Random16.SkipAhead(rowId); checksumCounter = context.GetCounter(TeraGen.Counters.Checksum); } Random16.NextRand(rand); GenSort.GenerateRecord(buffer, rand, rowId); key.Set(buffer, 0, TeraInputFormat.KeyLength); value.Set(buffer, TeraInputFormat.KeyLength, TeraInputFormat.ValueLength); context.Write(key, value); crc32.Reset(); crc32.Update(buffer, 0, TeraInputFormat.KeyLength + TeraInputFormat.ValueLength); checksum.Set(crc32.GetValue()); total.Add(checksum); rowId.Add(One); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(IntWritable key, TupleWritable val, Mapper.Context context ) { int k = key.Get(); string kvstr = "Unexpected tuple: " + Stringify(key, val); NUnit.Framework.Assert.IsTrue(kvstr, 0 == k % (srcs * srcs)); for (int i = 0; i < val.Size(); ++i) { int vali = ((IntWritable)val.Get(i)).Get(); NUnit.Framework.Assert.IsTrue(kvstr, (vali - i) * srcs == 10 * k); } context.Write(key, one); // If the user modifies the key or any of the values in the tuple, it // should not affect the rest of the join. key.Set(-1); if (val.Has(0)) { ((IntWritable)val.Get(0)).Set(0); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(IntWritable key, IntWritable val, Mapper.Context context ) { int k = key.Get(); int vali = val.Get(); string kvstr = "Unexpected tuple: " + Stringify(key, val); if (0 == k % (srcs * srcs)) { NUnit.Framework.Assert.IsTrue(kvstr, vali == k * 10 / srcs + srcs - 1); } else { int i = k % srcs; NUnit.Framework.Assert.IsTrue(kvstr, srcs * (vali - i) == 10 * (k - i)); } context.Write(key, one); //If the user modifies the key or any of the values in the tuple, it // should not affect the rest of the join. key.Set(-1); val.Set(0); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable key, Text value, Mapper.Context context) { WriteFlag(context.GetConfiguration(), "map." + name + ".value." + value); context.Write(key, new Text(value + name)); }
// these 3 classes do a reduce side join with 2 different mappers // receives "a", "b", "c" as values /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable key, Text value, Mapper.Context ctx) { ctx.Write(value, blah); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(object k, object v, Mapper.Context c) { c.Write(v, NullWritable.Get()); }
/// <summary>The inverse function.</summary> /// <remarks>The inverse function. Input keys and values are swapped.</remarks> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected internal override void Map(K key, V value, Mapper.Context context) { context.Write(value, key); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Cleanup(Mapper.Context context) { context.Write(NullWritable.Get(), sum); }
// receives "a", "b", "c" as keys /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(Text key, Text value, Mapper.Context ctx) { ctx.Write(key, blah); }