/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Setup(Mapper.Context context)
{
Configuration conf = context.GetConfiguration();
bool oldConfigs = conf.GetBoolean(OldConfigs, false);
if (oldConfigs)
{
string javaOpts = conf.Get(JobConf.MapredTaskJavaOpts);
NUnit.Framework.Assert.IsNotNull(JobConf.MapredTaskJavaOpts + " is null!", javaOpts
);
NUnit.Framework.Assert.AreEqual(JobConf.MapredTaskJavaOpts + " has value of: " +
javaOpts, javaOpts, TaskOptsVal);
}
else
{
string mapJavaOpts = conf.Get(JobConf.MapredMapTaskJavaOpts);
NUnit.Framework.Assert.IsNotNull(JobConf.MapredMapTaskJavaOpts + " is null!", mapJavaOpts
);
NUnit.Framework.Assert.AreEqual(JobConf.MapredMapTaskJavaOpts + " has value of: "
+ mapJavaOpts, mapJavaOpts, MapOptsVal);
}
Level logLevel = Level.ToLevel(conf.Get(JobConf.MapredMapTaskLogLevel, Level.Info
.ToString()));
NUnit.Framework.Assert.AreEqual(JobConf.MapredMapTaskLogLevel + "has value of " +
logLevel, logLevel, Level.Off);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(IntWritable key, IntWritable value, Mapper.Context context
)
{
//it is expected that every map processes mapSleepCount number of records.
try
{
context.SetStatus("Sleeping... (" + (mapSleepDuration * (mapSleepCount - count))
+ ") ms left");
Sharpen.Thread.Sleep(mapSleepDuration);
}
catch (Exception ex)
{
throw (IOException)Sharpen.Extensions.InitCause(new IOException("Interrupted while sleeping"
), ex);
}
++count;
// output reduceSleepCount * numReduce number of random values, so that
// each reducer will get reduceSleepCount number of keys.
int k = key.Get();
for (int i = 0; i < value.Get(); ++i)
{
context.Write(new IntWritable(k + i), NullWritable.Get());
}
}
/// <summary>Given an output filename, write a bunch of random records to it.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(Text key, Text value, Mapper.Context context)
{
int itemCount = 0;
while (numBytesToWrite > 0)
{
// Generate the key/value
int noWordsKey = minWordsInKey + (wordsInKeyRange != 0 ? random.Next(wordsInKeyRange
) : 0);
int noWordsValue = minWordsInValue + (wordsInValueRange != 0 ? random.Next(wordsInValueRange
) : 0);
Text keyWords = GenerateSentence(noWordsKey);
Text valueWords = GenerateSentence(noWordsValue);
// Write the sentence
context.Write(keyWords, valueWords);
numBytesToWrite -= (keyWords.GetLength() + valueWords.GetLength());
// Update counters, progress etc.
context.GetCounter(RandomTextWriter.Counters.BytesWritten).Increment(keyWords.GetLength
() + valueWords.GetLength());
context.GetCounter(RandomTextWriter.Counters.RecordsWritten).Increment(1);
if (++itemCount % 200 == 0)
{
context.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left."
);
}
}
context.SetStatus("done with " + itemCount + " records.");
}
/// <summary>Given an output filename, write a bunch of random records to it.</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(WritableComparable key, Writable value, Mapper.Context
context)
{
int itemCount = 0;
while (numBytesToWrite > 0)
{
int keyLength = minKeySize + (keySizeRange != 0 ? random.Next(keySizeRange) : 0);
randomKey.SetSize(keyLength);
RandomizeBytes(randomKey.GetBytes(), 0, randomKey.GetLength());
int valueLength = minValueSize + (valueSizeRange != 0 ? random.Next(valueSizeRange
) : 0);
randomValue.SetSize(valueLength);
RandomizeBytes(randomValue.GetBytes(), 0, randomValue.GetLength());
context.Write(randomKey, randomValue);
numBytesToWrite -= keyLength + valueLength;
context.GetCounter(LargeSorter.Counters.BytesWritten).Increment(keyLength + valueLength
);
context.GetCounter(LargeSorter.Counters.RecordsWritten).Increment(1);
if (++itemCount % 200 == 0)
{
context.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left."
);
}
}
context.SetStatus("done with " + itemCount + " records.");
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Setup(Mapper.Context context)
{
Configuration conf = context.GetConfiguration();
this.mapSleepCount = conf.GetInt(MapSleepCount, mapSleepCount);
this.mapSleepDuration = mapSleepCount == 0 ? 0 : conf.GetLong(MapSleepTime, 100)
/ mapSleepCount;
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text value, Mapper.Context context)
{
if (context.GetConfiguration().GetBoolean(FailMap, true))
{
throw new RuntimeException("Intentional map failure");
}
context.Write(key, NullWritable.Get());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(IntWritable key, IntWritable val, Mapper.Context context
)
{
int keyint = key.Get();
int valint = val.Get();
context.Write(new IntWritable(keyint), new IntWritable(valint));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text value, Mapper.Context context)
{
string v = value.ToString();
string k = Sharpen.Runtime.Substring(v, 0, v.IndexOf(","));
v = Sharpen.Runtime.Substring(v, v.IndexOf(",") + 1);
context.Write(new Text(k), new LongWritable(long.Parse(v)));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected internal virtual void Emit(K key, V val, Mapper.Context context)
{
++total;
while ((float)kept / total < keep)
{
++kept;
context.Write(key, val);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable k, Text v, Mapper.Context c)
{
int max = Sharpen.Extensions.ValueOf(v.ToString());
for (int i = 0; i < max; i++)
{
c.Write(new Text(string.Empty + i), NullWritable.Get());
}
}
/// <summary>
/// Save the values out of the configuaration that we need to write
/// the data.
/// </summary>
protected override void Setup(Mapper.Context context)
{
Configuration conf = context.GetConfiguration();
numBytesToWrite = conf.GetLong(BytesPerMap, 1 * 1024 * 1024 * 1024);
minKeySize = conf.GetInt(MinKey, 10);
keySizeRange = conf.GetInt(MaxKey, 1000) - minKeySize;
minValueSize = conf.GetInt(MinValue, 0);
valueSizeRange = conf.GetInt(MaxValue, 20000) - minValueSize;
}
/// <summary>Save the configuration value that we need to write the data.</summary>
protected override void Setup(Mapper.Context context)
{
Configuration conf = context.GetConfiguration();
numBytesToWrite = conf.GetLong(BytesPerMap, 1 * 1024 * 1024 * 1024);
minWordsInKey = conf.GetInt(MinKey, 5);
wordsInKeyRange = (conf.GetInt(MaxKey, 10) - minWordsInKey);
minWordsInValue = conf.GetInt(MinValue, 10);
wordsInValueRange = (conf.GetInt(MaxValue, 100) - minWordsInValue);
}
protected override void Setup(Mapper.Context context)
{
Configuration conf = new Configuration();
bytesToWrite = conf.GetLong(RandomTextWriter.BytesPerMap, 1 * 1024 * 1024 * 1024);
keymin = conf.GetInt(RandomTextWriter.MinKey, 5);
keymax = conf.GetInt(RandomTextWriter.MaxKey, 10);
valmin = conf.GetInt(RandomTextWriter.MinValue, 5);
valmax = conf.GetInt(RandomTextWriter.MaxValue, 10);
}
// Different map tasks operate at different speeds.
// We define behavior for 6 threads.
// Used to ensure that the compiler doesn't optimize away
// some code.
protected override void Setup(Mapper.Context context)
{
// Get the thread num from the file number.
FileSplit split = (FileSplit)context.GetInputSplit();
Path filePath = split.GetPath();
string name = filePath.GetName();
this.threadId = Sharpen.Extensions.ValueOf(name);
Log.Info("Thread " + threadId + " : " + context.GetInputSplit());
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(object key, Text value, Mapper.Context context)
{
StringTokenizer itr = new StringTokenizer(value.ToString());
while (itr.HasMoreTokens())
{
word.Set(itr.NextToken());
context.Write(word, one);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text value, Mapper.Context context)
{
string id = context.GetTaskAttemptID().ToString();
// Mapper 0 does not output anything
if (!id.EndsWith("0_0"))
{
context.Write(key, value);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(IntWritable key, IntWritable val, Mapper.Context context
)
{
int randomVal = key.Get();
int randomCount = val.Get();
for (int i = 0; i < randomCount; i++)
{
context.Write(new IntWritable(Math.Abs(r.Next())), new IntWritable(randomVal));
}
}
/// <summary>Map method with different behavior based on the thread id</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text val, Mapper.Context c)
{
// Write many values quickly.
for (int i = 0; i < OutputSizes[threadId]; i++)
{
c.Write(new LongWritable(0), val);
if (i % SleepIntervals[threadId] == 1)
{
Sharpen.Thread.Sleep(1);
}
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text value, Mapper.Context context)
{
// get the map phase progress
float mapPhaseProgress = ((float)++recordCount) / InputLines;
// get the weighted map phase progress
float weightedMapProgress = progressRange * mapPhaseProgress;
// check the map progress
NUnit.Framework.Assert.AreEqual("Invalid progress in map", weightedMapProgress, context
.GetProgress(), 0f);
context.Write(new Text(value.ToString() + recordCount), value);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text value, Mapper.Context context)
{
IntWritable outKey = new IntWritable();
IntWritable outValue = new IntWritable();
for (int j = 0; j < NumTests; j++)
{
for (int i = 0; i < NumValues; i++)
{
outKey.Set(j);
outValue.Set(i);
context.Write(outKey, outValue);
}
}
}
// check map task reports
// TODO fix testcase
// Disabling checks for now to get builds to run
/*
* reports = job.getTaskReports(TaskType.MAP);
* assertEquals(numMaps, reports.length);
* assertEquals("map > sort", reports[0].getState());
*
* // check reduce task reports
* reports = job.getTaskReports(TaskType.REDUCE);
* assertEquals(numReduces, reports.length);
* assertEquals("reduce > reduce", reports[0].getState());
*/
// an input with 4 lines
/// <exception cref="System.IO.IOException"/>
protected override void Setup(Mapper.Context context)
{
// check if the map task attempt progress is 0
NUnit.Framework.Assert.AreEqual("Invalid progress in map setup", 0.0f, context.GetProgress
(), 0f);
// define the progress boundaries
if (context.GetNumReduceTasks() == 0)
{
progressRange = 1f;
}
else
{
progressRange = 0.667f;
}
}
// NOTHING
/// <summary>
/// Expert users can override this method for more complete control over the
/// execution of the Mapper.
/// </summary>
/// <param name="context"/>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void Run(Mapper.Context context)
{
Setup(context);
try
{
while (context.NextKeyValue())
{
Map(context.GetCurrentKey(), context.GetCurrentValue(), context);
}
}
finally
{
Cleanup(context);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text val, Mapper.Context c)
{
// Create a whole bunch of objects.
IList <int> lst = new AList <int>();
for (int i = 0; i < 20000; i++)
{
lst.AddItem(i);
}
// Actually use this list, to ensure that it isn't just optimized away.
int sum = 0;
foreach (int x in lst)
{
sum += x;
}
// throw away the list and run a GC.
lst = null;
System.GC.Collect();
c.Write(new LongWritable(sum), val);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(Org.Apache.Hadoop.IO.Text key, Org.Apache.Hadoop.IO.Text
val, Mapper.Context context)
{
long acc = 0L;
long recs = 0;
int keydiff = keymax - keymin;
int valdiff = valmax - valmin;
for (long i = 0L; acc < bytesToWrite; ++i)
{
int recacc = 0;
recacc += GenerateSentence(key, keymin + (0 == keydiff ? 0 : r.Next(keydiff)));
recacc += GenerateSentence(val, valmin + (0 == valdiff ? 0 : r.Next(valdiff)));
context.Write(key, val);
++recs;
acc += recacc;
context.GetCounter(GenericMRLoadGenerator.Counters.BytesWritten).Increment(recacc
);
context.GetCounter(GenericMRLoadGenerator.Counters.RecordsWritten).Increment(1);
context.SetStatus(acc + "/" + (bytesToWrite - acc) + " bytes");
}
context.SetStatus("Wrote " + recs + " records");
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Cleanup(Mapper.Context context)
{
mapCleanup = true;
base.Cleanup(context);
}
/// <summary>Map method with different behavior based on the thread id</summary>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(LongWritable key, Text val, Mapper.Context c)
{
throw new IOException("TestMapperReducerCleanup");
}
/// <exception cref="System.IO.IOException"/>
protected override void Setup(Mapper.Context context)
{
context.SetStatus(myStatus);
NUnit.Framework.Assert.AreEqual(myStatus, context.GetStatus());
}
protected override void Cleanup(Mapper.Context context)
{
// Output this here, to ensure that the incrementing done in map()
// cannot be optimized away.
Log.Debug("Busy loop counter: " + this.exposedState);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map <_T0>(WritableComparable <_T0> key, Text val, Mapper.Context
context)
{
context.Write(new IntWritable(System.Convert.ToInt32(val.ToString().Trim())), new
IntWritable(1));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
protected override void Map(IntWritable key, IntWritable value, Mapper.Context context
)
{
context.Write(key, value);
}
