protected internal override void Setup(Mapper.Context context) { Configuration conf = context.GetConfiguration(); pattern = Sharpen.Pattern.Compile(conf.Get(Pattern)); group = conf.GetInt(Group, 0); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(object key, Text value, Mapper.Context context) { // Make one mapper slower for speculative execution TaskAttemptID taid = context.GetTaskAttemptID(); long sleepTime = 100; Configuration conf = context.GetConfiguration(); bool test_speculate_map = conf.GetBoolean(MRJobConfig.MapSpeculative, false); // IF TESTING MAPPER SPECULATIVE EXECUTION: // Make the "*_m_000000_0" attempt take much longer than the others. // When speculative execution is enabled, this should cause the attempt // to be killed and restarted. At that point, the attempt ID will be // "*_m_000000_1", so sleepTime will still remain 100ms. if ((taid.GetTaskType() == TaskType.Map) && test_speculate_map && (taid.GetTaskID ().GetId() == 0) && (taid.GetId() == 0)) { sleepTime = 10000; } try { Sharpen.Thread.Sleep(sleepTime); } catch (Exception) { } // Ignore context.Write(value, new IntWritable(1)); }
/// <summary>Given an output filename, write a bunch of random records to it.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(Text key, Text value, Mapper.Context context) { int itemCount = 0; while (numBytesToWrite > 0) { // Generate the key/value int noWordsKey = minWordsInKey + (wordsInKeyRange != 0 ? random.Next(wordsInKeyRange ) : 0); int noWordsValue = minWordsInValue + (wordsInValueRange != 0 ? random.Next(wordsInValueRange ) : 0); Text keyWords = GenerateSentence(noWordsKey); Text valueWords = GenerateSentence(noWordsValue); // Write the sentence context.Write(keyWords, valueWords); numBytesToWrite -= (keyWords.GetLength() + valueWords.GetLength()); // Update counters, progress etc. context.GetCounter(RandomTextWriter.Counters.BytesWritten).Increment(keyWords.GetLength () + valueWords.GetLength()); context.GetCounter(RandomTextWriter.Counters.RecordsWritten).Increment(1); if (++itemCount % 200 == 0) { context.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left." ); } } context.SetStatus("done with " + itemCount + " records."); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Setup(Mapper.Context context) { Configuration conf = context.GetConfiguration(); NUnit.Framework.Assert.AreEqual(prop, conf.Get("a")); WriteFlag(conf, "map.setup." + name); }
/// <summary>Map method.</summary> /// <param name="offset">samples starting from the (offset+1)th sample.</param> /// <param name="size">the number of samples for this map</param> /// <param name="context">output {ture->numInside, false->numOutside}</param> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable offset, LongWritable size, Mapper.Context context) { QuasiMonteCarlo.HaltonSequence haltonsequence = new QuasiMonteCarlo.HaltonSequence (offset.Get()); long numInside = 0L; long numOutside = 0L; for (long i = 0; i < size.Get();) { //generate points in a unit square double[] point = haltonsequence.NextPoint(); //count points inside/outside of the inscribed circle of the square double x = point[0] - 0.5; double y = point[1] - 0.5; if (x * x + y * y > 0.25) { numOutside++; } else { numInside++; } //report status i++; if (i % 1000 == 0) { context.SetStatus("Generated " + i + " samples."); } } //output map results context.Write(new BooleanWritable(true), new LongWritable(numInside)); context.Write(new BooleanWritable(false), new LongWritable(numOutside)); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(Text key, Text value, Mapper.Context context) { if (lastKey == null) { FileSplit fs = (FileSplit)context.GetInputSplit(); filename = GetFilename(fs); context.Write(new Text(filename + ":begin"), key); lastKey = new Text(); } else { if (key.CompareTo(lastKey) < 0) { context.Write(Error, new Text("misorder in " + filename + " between " + TextifyBytes (lastKey) + " and " + TextifyBytes(key))); } } // compute the crc of the key and value and add it to the sum crc32.Reset(); crc32.Update(key.GetBytes(), 0, key.GetLength()); crc32.Update(value.GetBytes(), 0, value.GetLength()); tmp.Set(crc32.GetValue()); checksum.Add(tmp); lastKey.Set(key); }
protected override void Cleanup(Mapper.Context context) { if (checksumCounter != null) { checksumCounter.Increment(total.GetLow8()); } }
// this map throws IOException for input value "b" /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable key, Text value, Mapper.Context context) { if (value.ToString().Equals("b")) { throw new IOException(); } }
/// <summary>Given an output filename, write a bunch of random records to it.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(WritableComparable key, Writable value, Mapper.Context context) { int itemCount = 0; while (numBytesToWrite > 0) { int keyLength = minKeySize + (keySizeRange != 0 ? random.Next(keySizeRange) : 0); randomKey.SetSize(keyLength); RandomizeBytes(randomKey.GetBytes(), 0, randomKey.GetLength()); int valueLength = minValueSize + (valueSizeRange != 0 ? random.Next(valueSizeRange ) : 0); randomValue.SetSize(valueLength); RandomizeBytes(randomValue.GetBytes(), 0, randomValue.GetLength()); context.Write(randomKey, randomValue); numBytesToWrite -= keyLength + valueLength; context.GetCounter(RandomWriter.Counters.BytesWritten).Increment(keyLength + valueLength ); context.GetCounter(RandomWriter.Counters.RecordsWritten).Increment(1); if (++itemCount % 200 == 0) { context.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left." ); } } context.SetStatus("done with " + itemCount + " records."); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable key, DBCountPageView.AccessRecord value, Mapper.Context context) { Text oKey = new Text(value.url); context.Write(oKey, One); }
/// <exception cref="System.IO.IOException"/> protected override void Map(Text key, Text value, Mapper.Context context) { crc32.Reset(); crc32.Update(key.GetBytes(), 0, key.GetLength()); crc32.Update(value.GetBytes(), 0, value.GetLength()); checksum.Set(crc32.GetValue()); sum.Add(checksum); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Cleanup(Mapper.Context context) { if (lastKey != null) { context.Write(new Text(filename + ":end"), lastKey); context.Write(Checksum, new Text(checksum.ToString())); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable key, Text value, Mapper.Context context) { context.Write(key.Get(), value.ToString()); if (value.ToString().Equals("a")) { mos.Write(Text, key.Get(), Text); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected internal override void Setup(Mapper.Context context) { // Find the Mapper from the TaggedInputSplit. TaggedInputSplit inputSplit = (TaggedInputSplit)context.GetInputSplit(); mapper = (Mapper <K1, V1, K2, V2>)ReflectionUtils.NewInstance(inputSplit.GetMapperClass (), context.GetConfiguration()); }
/// <summary>The identify function.</summary> /// <remarks>The identify function. Input key/value pair is written directly to output. /// </remarks> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected internal override void Map(K key, V val, Mapper.Context context) { FieldSelectionHelper helper = new FieldSelectionHelper(FieldSelectionHelper.emptyText , FieldSelectionHelper.emptyText); helper.ExtractOutputKeyValue(key.ToString(), val.ToString(), fieldSeparator, mapOutputKeyFieldList , mapOutputValueFieldList, allMapValueFieldsFrom, ignoreInputKey, true); context.Write(helper.GetKey(), helper.GetValue()); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public MapRunner(Chain _enclosing, Mapper <KEYIN, VALUEIN, KEYOUT, VALUEOUT> mapper , Mapper.Context mapperContext, RecordReader <KEYIN, VALUEIN> rr, RecordWriter <KEYOUT , VALUEOUT> rw) { this._enclosing = _enclosing; this.mapper = mapper; this.rr = rr; this.rw = rw; this.chainContext = mapperContext; }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(object key, Text value, Mapper.Context context) { StringTokenizer itr = new StringTokenizer(value.ToString()); while (itr.HasMoreTokens()) { word.Set(itr.NextToken()); context.Write(word, one); } }
/// <summary>Save the configuration value that we need to write the data.</summary> protected override void Setup(Mapper.Context context) { Configuration conf = context.GetConfiguration(); numBytesToWrite = conf.GetLong(BytesPerMap, 1 * 1024 * 1024 * 1024); minWordsInKey = conf.GetInt(MinKey, 5); wordsInKeyRange = (conf.GetInt(MaxKey, 10) - minWordsInKey); minWordsInValue = conf.GetInt(MinValue, 10); wordsInValueRange = (conf.GetInt(MaxValue, 100) - minWordsInValue); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected internal override void Map(K key, Text value, Mapper.Context context) { string text = value.ToString(); Matcher matcher = pattern.Matcher(text); while (matcher.Find()) { context.Write(new Text(matcher.Group(group)), new LongWritable(1)); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable key, Text value, Mapper.Context context) { context.Write(key, value); if (value.ToString().Equals("a")) { mos.Write(Text, key, new Text(Text)); mos.Write(Sequence, new IntWritable(1), new Text(Sequence), (Sequence + "_A")); mos.Write(Sequence, new IntWritable(2), new Text(Sequence), (Sequence + "_B")); } }
/// <summary> /// Save the values out of the configuaration that we need to write /// the data. /// </summary> protected override void Setup(Mapper.Context context) { Configuration conf = context.GetConfiguration(); numBytesToWrite = conf.GetLong(BytesPerMap, 1 * 1024 * 1024 * 1024); minKeySize = conf.GetInt(MinKey, 10); keySizeRange = conf.GetInt(MaxKey, 1000) - minKeySize; minValueSize = conf.GetInt(MinValue, 0); valueSizeRange = conf.GetInt(MaxValue, 20000) - minValueSize; }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> internal virtual void RunMapper(TaskInputOutputContext context, int index) { Mapper mapper = mappers[index]; RecordReader rr = new Chain.ChainRecordReader(context); RecordWriter rw = new Chain.ChainRecordWriter(context); Mapper.Context mapperContext = CreateMapContext(rr, rw, context, GetConf(index)); mapper.Run(mapperContext); rr.Close(); rw.Close(context); }
/// <summary>Emits a key-value pair for counting the word.</summary> /// <remarks> /// Emits a key-value pair for counting the word. Outputs are (IntWritable, /// IntWritable). /// </remarks> /// <param name="value">This will be a line of text coming in from our input file.</param> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(object key, Text value, Mapper.Context context) { StringTokenizer itr = new StringTokenizer(value.ToString()); while (itr.HasMoreTokens()) { string @string = itr.NextToken(); length.Set(@string.Length); context.Write(length, One); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(object key, Text value, Mapper.Context context) { context.GetCounter("MyCounterGroup", "MAP_INPUT_RECORDS").Increment(1); StringTokenizer iter = new StringTokenizer(value.ToString()); while (iter.HasMoreTokens()) { word.Set(iter.NextToken()); context.Write(word, one); context.GetCounter("MyCounterGroup", "MAP_OUTPUT_RECORDS").Increment(1); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable key, Text value, Mapper.Context context) { if (ioEx) { throw new IOException(); } if (rtEx) { throw new RuntimeException(); } base.Map(key, value, context); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Setup(Mapper.Context context) { base.Setup(context); Configuration conf = context.GetConfiguration(); string ioSortMb = conf.Get(MRJobConfig.IoSortMb); if (!TestIoSortMb.Equals(ioSortMb)) { throw new IOException("io.sort.mb expected: " + TestIoSortMb + ", actual: " + ioSortMb ); } }
protected override void Setup(Mapper.Context context) { this.context = context; Configuration conf = context.GetConfiguration(); depth = conf.GetInt(Pentomino.Depth, PentDepth); width = conf.GetInt(Pentomino.Width, PentWidth); height = conf.GetInt(Pentomino.Height, PentHeight); pent = (Pentomino)ReflectionUtils.NewInstance(conf.GetClass(Pentomino.Class, typeof( OneSidedPentomino)), conf); pent.Initialize(width, height); pent.SetPrinter(new DistributedPentomino.PentMap.SolutionCatcher(this)); }
/// <summary> /// Emits 3 key-value pairs for counting the word, its length, and the /// squares of its length. /// </summary> /// <remarks> /// Emits 3 key-value pairs for counting the word, its length, and the /// squares of its length. Outputs are (Text, LongWritable). /// </remarks> /// <param name="value">This will be a line of text coming in from our input file.</param> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(object key, Text value, Mapper.Context context) { StringTokenizer itr = new StringTokenizer(value.ToString()); while (itr.HasMoreTokens()) { string @string = itr.NextToken(); this.wordLen.Set(@string.Length); // the square of an integer is an integer... this.wordLenSq.Set((long)Math.Pow(@string.Length, 2.0)); context.Write(Length, this.wordLen); context.Write(Square, this.wordLenSq); context.Write(Count, One); } }
/// <summary> /// Add mapper(the first mapper) that reads input from the input /// context and writes to queue /// </summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> internal virtual void AddMapper(TaskInputOutputContext inputContext, Chain.ChainBlockingQueue <Chain.KeyValuePair <object, object> > output, int index) { Configuration conf = GetConf(index); Type keyOutClass = conf.GetClass(MapperOutputKeyClass, typeof(object)); Type valueOutClass = conf.GetClass(MapperOutputValueClass, typeof(object)); RecordReader rr = new Chain.ChainRecordReader(inputContext); RecordWriter rw = new Chain.ChainRecordWriter(keyOutClass, valueOutClass, output, conf); Mapper.Context mapperContext = CreateMapContext(rr, rw, (MapContext)inputContext, GetConf(index)); Chain.MapRunner runner = new Chain.MapRunner(this, mappers[index], mapperContext, rr, rw); threads.AddItem(runner); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> internal MapRunner(MultithreadedMapper <K1, V1, K2, V2> _enclosing, Mapper.Context context) { this._enclosing = _enclosing; reader = new MultithreadedMapper.SubMapRecordReader(this); this.mapper = ReflectionUtils.NewInstance(this._enclosing.mapClass, context.GetConfiguration ()); MapContext <K1, V1, K2, V2> mapContext = new MapContextImpl <K1, V1, K2, V2>(this._enclosing .outer.GetConfiguration(), this._enclosing.outer.GetTaskAttemptID(), this.reader , new MultithreadedMapper.SubMapRecordWriter(this), context.GetOutputCommitter() , new MultithreadedMapper.SubMapStatusReporter(this), this._enclosing.outer.GetInputSplit ()); this.subcontext = new WrappedMapper <K1, V1, K2, V2>().GetMapContext(mapContext); this.reader.Initialize(context.GetInputSplit(), context); }