/// <summary>Given an output filename, write a bunch of random records to it.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(Text key, Text value, Mapper.Context context) { int itemCount = 0; while (numBytesToWrite > 0) { // Generate the key/value int noWordsKey = minWordsInKey + (wordsInKeyRange != 0 ? random.Next(wordsInKeyRange ) : 0); int noWordsValue = minWordsInValue + (wordsInValueRange != 0 ? random.Next(wordsInValueRange ) : 0); Text keyWords = GenerateSentence(noWordsKey); Text valueWords = GenerateSentence(noWordsValue); // Write the sentence context.Write(keyWords, valueWords); numBytesToWrite -= (keyWords.GetLength() + valueWords.GetLength()); // Update counters, progress etc. context.GetCounter(RandomTextWriter.Counters.BytesWritten).Increment(keyWords.GetLength () + valueWords.GetLength()); context.GetCounter(RandomTextWriter.Counters.RecordsWritten).Increment(1); if (++itemCount % 200 == 0) { context.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left." ); } } context.SetStatus("done with " + itemCount + " records."); }
/// <summary>Given an output filename, write a bunch of random records to it.</summary> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(WritableComparable key, Writable value, Mapper.Context context) { int itemCount = 0; while (numBytesToWrite > 0) { int keyLength = minKeySize + (keySizeRange != 0 ? random.Next(keySizeRange) : 0); randomKey.SetSize(keyLength); RandomizeBytes(randomKey.GetBytes(), 0, randomKey.GetLength()); int valueLength = minValueSize + (valueSizeRange != 0 ? random.Next(valueSizeRange ) : 0); randomValue.SetSize(valueLength); RandomizeBytes(randomValue.GetBytes(), 0, randomValue.GetLength()); context.Write(randomKey, randomValue); numBytesToWrite -= keyLength + valueLength; context.GetCounter(RandomWriter.Counters.BytesWritten).Increment(keyLength + valueLength ); context.GetCounter(RandomWriter.Counters.RecordsWritten).Increment(1); if (++itemCount % 200 == 0) { context.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left." ); } } context.SetStatus("done with " + itemCount + " records."); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(object key, Text value, Mapper.Context context) { context.GetCounter("MyCounterGroup", "MAP_INPUT_RECORDS").Increment(1); StringTokenizer iter = new StringTokenizer(value.ToString()); while (iter.HasMoreTokens()) { word.Set(iter.NextToken()); context.Write(word, one); context.GetCounter("MyCounterGroup", "MAP_OUTPUT_RECORDS").Increment(1); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected override void Map(LongWritable row, NullWritable ignored, Mapper.Context context) { if (rand == null) { rowId = new Unsigned16(row.Get()); rand = Random16.SkipAhead(rowId); checksumCounter = context.GetCounter(TeraGen.Counters.Checksum); } Random16.NextRand(rand); GenSort.GenerateRecord(buffer, rand, rowId); key.Set(buffer, 0, TeraInputFormat.KeyLength); value.Set(buffer, TeraInputFormat.KeyLength, TeraInputFormat.ValueLength); context.Write(key, value); crc32.Reset(); crc32.Update(buffer, 0, TeraInputFormat.KeyLength + TeraInputFormat.ValueLength); checksum.Set(crc32.GetValue()); total.Add(checksum); rowId.Add(One); }