/// <exception cref="System.IO.IOException"/>
public virtual void Map(WritableComparable key, Writable val, OutputCollector <WritableComparable
, Writable> output, Reporter reporter)
{
NUnit.Framework.Assert.IsNotNull("Mapper not configured!", loader);
// load the memory
loader.Load();
// work as identity mapper
output.Collect(key, val);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Map(WritableComparable key, Writable value, OutputCollector <BytesWritable
, BytesWritable> output, Reporter reporter)
{
int itemCount = 0;
while (numBytesToWrite > 0)
{
int keyLength = minKeySize + (keySizeRange != 0 ? random.Next(keySizeRange) : 0);
randomKey.SetSize(keyLength);
RandomizeBytes(randomKey.GetBytes(), 0, randomKey.GetLength());
int valueLength = minValueSize + (valueSizeRange != 0 ? random.Next(valueSizeRange
) : 0);
randomValue.SetSize(valueLength);
RandomizeBytes(randomValue.GetBytes(), 0, randomValue.GetLength());
output.Collect(randomKey, randomValue);
numBytesToWrite -= keyLength + valueLength;
reporter.IncrCounter(ThreadedMapBenchmark.Counters.BytesWritten, 1);
reporter.IncrCounter(ThreadedMapBenchmark.Counters.RecordsWritten, 1);
if (++itemCount % 200 == 0)
{
reporter.SetStatus("wrote record " + itemCount + ". " + numBytesToWrite + " bytes left."
);
}
}
reporter.SetStatus("done with " + itemCount + " records.");
}
/// <exception cref="System.IO.IOException"/>
public virtual void Map(BytesWritable key, BytesWritable value, OutputCollector <BytesWritable
, IntWritable> output, Reporter reporter)
{
// newKey = (key, value)
BytesWritable keyValue = new BytesWritable(Pair(key, value));
// output (newKey, value)
output.Collect(keyValue, this.value);
}
/// <exception cref="System.IO.IOException"/>
public override RecordReader <K, V> GetRecordReader(InputSplit split, JobConf job,
Reporter reporter)
{
return(null);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Map(WritableComparable key, Writable value, OutputCollector <IntWritable
, SortValidator.RecordStatsChecker.RecordStatsWritable> output, Reporter reporter
)
{
// Set up rawKey and rawValue on the first call to 'map'
if (recordId == -1)
{
rawKey = CreateRaw(key.GetType());
rawValue = CreateRaw(value.GetType());
}
++recordId;
if (this.key == sortOutput)
{
// Check if keys are 'sorted' if this
// record is from sort's output
if (prevKey == null)
{
prevKey = key;
keyClass = prevKey.GetType();
}
else
{
// Sanity check
if (keyClass != key.GetType())
{
throw new IOException("Type mismatch in key: expected " + keyClass.FullName + ", received "
+ key.GetType().FullName);
}
// Check if they were sorted correctly
if (prevKey.CompareTo(key) > 0)
{
throw new IOException("The 'map-reduce' framework wrongly" + " classifed (" + prevKey
+ ") > (" + key + ") " + "for record# " + recordId);
}
prevKey = key;
}
// Check if the sorted output is 'partitioned' right
int keyPartition = partitioner.GetPartition(key, value, noSortReducers);
if (partition != keyPartition)
{
throw new IOException("Partitions do not match for record# " + recordId + " ! - '"
+ partition + "' v/s '" + keyPartition + "'");
}
}
// Construct the record-stats and output (this.key, record-stats)
byte[] keyBytes = rawKey.GetRawBytes(key);
int keyBytesLen = rawKey.GetRawBytesLength(key);
byte[] valueBytes = rawValue.GetRawBytes(value);
int valueBytesLen = rawValue.GetRawBytesLength(value);
int keyValueChecksum = (WritableComparator.HashBytes(keyBytes, keyBytesLen) ^ WritableComparator
.HashBytes(valueBytes, valueBytesLen));
output.Collect(this.key, new SortValidator.RecordStatsChecker.RecordStatsWritable
((keyBytesLen + valueBytesLen), 1, keyValueChecksum));
}
/// <exception cref="System.IO.IOException"/>
public virtual void Reduce(IntWritable key, IEnumerator <SortValidator.RecordStatsChecker.RecordStatsWritable
> values, OutputCollector <IntWritable, SortValidator.RecordStatsChecker.RecordStatsWritable
> output, Reporter reporter)
{
long bytes = 0;
long records = 0;
int xor = 0;
while (values.HasNext())
{
SortValidator.RecordStatsChecker.RecordStatsWritable stats = values.Next();
bytes += stats.GetBytes();
records += stats.GetRecords();
xor ^= stats.GetChecksum();
}
output.Collect(key, new SortValidator.RecordStatsChecker.RecordStatsWritable(bytes
, records, xor));
}
/// <exception cref="System.IO.IOException"/>
public virtual void Reduce(Text key, IEnumerator <Text> values, OutputCollector <Text
, Text> @out, Reporter reporter)
{
@out.Collect(new Text("Test"), new Text("Me"));
}
/// <exception cref="System.IO.IOException"/>
public virtual void Map(WritableComparable key, Text value, OutputCollector<UTF8,
UTF8> output, Reporter reporter)
{
string line = value.ToString();
output.Collect(new UTF8(Process(line)), new UTF8(string.Empty));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestFailAbort()
{
JobConf job = new JobConf();
job.Set(FileSystem.FsDefaultNameKey, "faildel:///");
job.SetClass("fs.faildel.impl", typeof(TestMRCJCFileOutputCommitter.FakeFileSystem
), typeof(FileSystem));
SetConfForFileOutputCommitter(job);
JobContext jContext = new JobContextImpl(job, ((JobID)taskID.GetJobID()));
TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID);
FileOutputCommitter committer = new FileOutputCommitter();
FileOutputFormat.SetWorkOutputPath(job, committer.GetTaskAttemptPath(tContext));
// do setup
committer.SetupJob(jContext);
committer.SetupTask(tContext);
string file = "test.txt";
FilePath jobTmpDir = new FilePath(committer.GetJobAttemptPath(jContext).ToUri().GetPath
());
FilePath taskTmpDir = new FilePath(committer.GetTaskAttemptPath(tContext).ToUri()
.GetPath());
FilePath expectedFile = new FilePath(taskTmpDir, file);
// A reporter that does nothing
Reporter reporter = Reporter.Null;
// write output
FileSystem localFs = new TestMRCJCFileOutputCommitter.FakeFileSystem();
TextOutputFormat theOutputFormat = new TextOutputFormat();
RecordWriter theRecordWriter = theOutputFormat.GetRecordWriter(localFs, job, expectedFile
.GetAbsolutePath(), reporter);
WriteOutput(theRecordWriter, reporter);
// do abort
Exception th = null;
try
{
committer.AbortTask(tContext);
}
catch (IOException ie)
{
th = ie;
}
NUnit.Framework.Assert.IsNotNull(th);
NUnit.Framework.Assert.IsTrue(th is IOException);
NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
NUnit.Framework.Assert.IsTrue(expectedFile + " does not exists", expectedFile.Exists
());
th = null;
try
{
committer.AbortJob(jContext, JobStatus.State.Failed);
}
catch (IOException ie)
{
th = ie;
}
NUnit.Framework.Assert.IsNotNull(th);
NUnit.Framework.Assert.IsTrue(th is IOException);
NUnit.Framework.Assert.IsTrue(th.Message.Contains("fake delete failed"));
NUnit.Framework.Assert.IsTrue("job temp dir does not exists", jobTmpDir.Exists());
}
/// <exception cref="System.IO.IOException"/>
public virtual void Map(WritableComparable key, Writable val, OutputCollector <Text
, Text> @out, Reporter reporter)
{
@out.Collect(new Text("Hello"), new Text("World"));
}
/// <exception cref="System.IO.IOException"/>
public abstract RecordReader <K, V> GetRecordReader(InputSplit split, JobConf job,
Reporter reporter);
// Mapper that sleeps for a long time.
// Used for running a job that will be killed
/// <exception cref="System.IO.IOException"/>
public virtual void Map(WritableComparable key, Writable value, OutputCollector <WritableComparable
, Writable> @out, Reporter reporter)
{
try
{
Sharpen.Thread.Sleep(1000000);
}
catch (Exception)
{
}
}
// Mapper that fails
/// <exception cref="System.IO.IOException"/>
public virtual void Map(WritableComparable key, Writable value, OutputCollector <WritableComparable
, Writable> @out, Reporter reporter)
{
//NOTE- the next line is required for the TestDebugScript test to succeed
System.Console.Error.WriteLine("failing map");
throw new RuntimeException("failing map");
}
GetRecordReader(InputSplit split, JobConf job, Reporter reporter)
{
return(new UtilsForTests.RandomInputFormat.RandomRecordReader(((FileSplit)split).
GetPath()));
}
/// <summary>The waiting function.</summary>
/// <remarks>
/// The waiting function. The map exits once it gets a signal. Here the
/// signal is the file existence.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void Map(WritableComparable key, Writable val, OutputCollector <WritableComparable
, Writable> output, Reporter reporter)
{
if (ShouldWait(id))
{
if (fs != null)
{
while (!fs.Exists(GetSignalFile(id)))
{
try
{
reporter.Progress();
lock (this)
{
Sharpen.Runtime.Wait(this, 1000);
}
}
catch (Exception)
{
// wait for 1 sec
System.Console.Out.WriteLine("Interrupted while the map was waiting for " + " the signal."
);
break;
}
}
}
else
{
throw new IOException("Could not get the DFS!!");
}
}
}