/// <exception cref="System.Exception"/>
private void MrRun()
{
FileSystem fs = FileSystem.Get(GetJobConf());
Path inputDir = new Path("input");
fs.Mkdirs(inputDir);
TextWriter writer = new OutputStreamWriter(fs.Create(new Path(inputDir, "data.txt"
)));
writer.Write("hello");
writer.Close();
Path outputDir = new Path("output", "output");
JobConf jobConf = new JobConf(GetJobConf());
jobConf.SetInt("mapred.map.tasks", 1);
jobConf.SetInt("mapred.map.max.attempts", 1);
jobConf.SetInt("mapred.reduce.max.attempts", 1);
jobConf.Set("mapred.input.dir", inputDir.ToString());
jobConf.Set("mapred.output.dir", outputDir.ToString());
JobClient jobClient = new JobClient(jobConf);
RunningJob runJob = jobClient.SubmitJob(jobConf);
runJob.WaitForCompletion();
NUnit.Framework.Assert.IsTrue(runJob.IsComplete());
NUnit.Framework.Assert.IsTrue(runJob.IsSuccessful());
}
// Start a job with the specified input and return its RunningJob object
/// <exception cref="System.IO.IOException"/>
internal static RunningJob RunJob(JobConf conf, Path inDir, Path outDir, int numMaps
, int numReds, string input)
{
FileSystem fs = FileSystem.Get(conf);
if (fs.Exists(outDir))
{
fs.Delete(outDir, true);
}
if (!fs.Exists(inDir))
{
fs.Mkdirs(inDir);
}
for (int i = 0; i < numMaps; ++i)
{
DataOutputStream file = fs.Create(new Path(inDir, "part-" + i));
file.WriteBytes(input);
file.Close();
}
conf.SetInputFormat(typeof(TextInputFormat));
conf.SetOutputKeyClass(typeof(LongWritable));
conf.SetOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text));
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
conf.SetNumMapTasks(numMaps);
conf.SetNumReduceTasks(numReds);
JobClient jobClient = new JobClient(conf);
RunningJob job = jobClient.SubmitJob(conf);
return(job);
}
// Run a job that will be failed and wait until it completes
/// <exception cref="System.IO.IOException"/>
public static RunningJob RunJobFail(JobConf conf, Path inDir, Path outDir)
{
conf.SetJobName("test-job-fail");
conf.SetMapperClass(typeof(UtilsForTests.FailMapper));
conf.SetReducerClass(typeof(IdentityReducer));
conf.SetMaxMapAttempts(1);
RunningJob job = UtilsForTests.RunJob(conf, inDir, outDir);
long sleepCount = 0;
while (!job.IsComplete())
{
try
{
if (sleepCount > 300)
{
// 30 seconds
throw new IOException("Job didn't finish in 30 seconds");
}
Sharpen.Thread.Sleep(100);
sleepCount++;
}
catch (Exception)
{
break;
}
}
return(job);
}
public virtual void TestOldCounterC()
{
JobConf conf = CreateConfiguration();
CreateWordsFile(inFiles[3], conf);
CreateWordsFile(inFiles[4], conf);
long inputSize = 0;
inputSize += GetFileSize(inFiles[0]);
inputSize += GetFileSize(inFiles[1]);
inputSize += GetFileSize(inFiles[2]);
inputSize += GetFileSize(inFiles[3]);
inputSize += GetFileSize(inFiles[4]);
conf.SetNumMapTasks(4);
conf.SetInt(JobContext.IoSortFactor, 3);
FileInputFormat.SetInputPaths(conf, InDir);
FileOutputFormat.SetOutputPath(conf, new Path(OutDir, "outputO2"));
RunningJob myJob = JobClient.RunJob(conf);
Counters c1 = myJob.GetCounters();
// As above, each map spills 2^14 records, so 5 maps spill 81920
// 1st merge: read + write = 6 * 8192
// final merge: unmerged = 2 * 8192
// Total reduce: 45056
// 5 files, 5120 = 5 * 1024 rec/file = 15360 input records
// 4 records/line = 102400 output records
ValidateCounters(c1, 122880, 25600, 102400);
ValidateFileCounters(c1, inputSize, 0, 0, 0);
}
/// <exception cref="System.Exception"/>
public static Counters RunJob(JobConf conf)
{
conf.SetMapperClass(typeof(TestReduceFetchFromPartialMem.MapMB));
conf.SetReducerClass(typeof(TestReduceFetchFromPartialMem.MBValidate));
conf.SetOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text));
conf.SetOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text));
conf.SetNumReduceTasks(1);
conf.SetInputFormat(typeof(TestReduceFetchFromPartialMem.FakeIF));
conf.SetNumTasksToExecutePerJvm(1);
conf.SetInt(JobContext.MapMaxAttempts, 0);
conf.SetInt(JobContext.ReduceMaxAttempts, 0);
FileInputFormat.SetInputPaths(conf, new Path("/in"));
Path outp = new Path("/out");
FileOutputFormat.SetOutputPath(conf, outp);
RunningJob job = null;
try
{
job = JobClient.RunJob(conf);
NUnit.Framework.Assert.IsTrue(job.IsSuccessful());
}
finally
{
FileSystem fs = dfsCluster.GetFileSystem();
if (fs.Exists(outp))
{
fs.Delete(outp, true);
}
}
return(job.GetCounters());
}
/// <exception cref="System.Exception"/>
private void RunMapReduce(JobConf conf, IList <string> mapperBadRecords, IList <string
> redBadRecords)
{
CreateInput();
conf.SetJobName("mr");
conf.SetNumMapTasks(1);
conf.SetNumReduceTasks(1);
conf.SetInt(JobContext.TaskTimeout, 30 * 1000);
SkipBadRecords.SetMapperMaxSkipRecords(conf, long.MaxValue);
SkipBadRecords.SetReducerMaxSkipGroups(conf, long.MaxValue);
SkipBadRecords.SetAttemptsToStartSkipping(conf, 0);
//the no of attempts to successfully complete the task depends
//on the no of bad records.
conf.SetMaxMapAttempts(SkipBadRecords.GetAttemptsToStartSkipping(conf) + 1 + mapperBadRecords
.Count);
conf.SetMaxReduceAttempts(SkipBadRecords.GetAttemptsToStartSkipping(conf) + 1 + redBadRecords
.Count);
FileInputFormat.SetInputPaths(conf, GetInputDir());
FileOutputFormat.SetOutputPath(conf, GetOutputDir());
conf.SetInputFormat(typeof(TextInputFormat));
conf.SetMapOutputKeyClass(typeof(LongWritable));
conf.SetMapOutputValueClass(typeof(Text));
conf.SetOutputFormat(typeof(TextOutputFormat));
conf.SetOutputKeyClass(typeof(LongWritable));
conf.SetOutputValueClass(typeof(Text));
RunningJob runningJob = JobClient.RunJob(conf);
ValidateOutput(conf, runningJob, mapperBadRecords, redBadRecords);
}
// run a job with 1 map and let it run to completion
/// <exception cref="System.IO.IOException"/>
private void TestSuccessfulJob(string filename, Type committer, string[] exclude)
{
JobConf jc = mr.CreateJobConf();
Path outDir = GetNewOutputDir();
ConfigureJob(jc, "job with cleanup()", 1, 0, outDir);
jc.SetOutputCommitter(committer);
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.SubmitJob(jc);
JobID id = job.GetID();
job.WaitForCompletion();
Log.Info("Job finished : " + job.IsComplete());
Path testFile = new Path(outDir, filename);
NUnit.Framework.Assert.IsTrue("Done file \"" + testFile + "\" missing for job " +
id, fileSys.Exists(testFile));
// check if the files from the missing set exists
foreach (string ex in exclude)
{
Path file = new Path(outDir, ex);
NUnit.Framework.Assert.IsFalse("File " + file + " should not be present for successful job "
+ id, fileSys.Exists(file));
}
}
/// <exception cref="System.IO.IOException"/>
public static void VerifyCounters(RunningJob runningJob, int expected)
{
NUnit.Framework.Assert.AreEqual(expected, runningJob.GetCounters().GetCounter(TestUserDefinedCounters.EnumCounter
.MapRecords));
NUnit.Framework.Assert.AreEqual(expected, runningJob.GetCounters().GetGroup("StringCounter"
).GetCounter("MapRecords"));
}
static void Main()
{
var rj = new RunningJob();
rj.Property = new Child1();
rj.Property = new Child2();
}
// run a job for which all the attempts simply fail.
/// <exception cref="System.IO.IOException"/>
private void TestFailedJob(string fileName, Type committer, string[] exclude)
{
JobConf jc = mr.CreateJobConf();
Path outDir = GetNewOutputDir();
ConfigureJob(jc, "fail job with abort()", 1, 0, outDir);
jc.SetMaxMapAttempts(1);
// set the job to fail
jc.SetMapperClass(typeof(UtilsForTests.FailMapper));
jc.SetOutputCommitter(committer);
JobClient jobClient = new JobClient(jc);
RunningJob job = jobClient.SubmitJob(jc);
JobID id = job.GetID();
job.WaitForCompletion();
NUnit.Framework.Assert.AreEqual("Job did not fail", JobStatus.Failed, job.GetJobState
());
if (fileName != null)
{
Path testFile = new Path(outDir, fileName);
NUnit.Framework.Assert.IsTrue("File " + testFile + " missing for failed job " + id
, fileSys.Exists(testFile));
}
// check if the files from the missing set exists
foreach (string ex in exclude)
{
Path file = new Path(outDir, ex);
NUnit.Framework.Assert.IsFalse("File " + file + " should not be present for failed job "
+ id, fileSys.Exists(file));
}
}
public virtual void TestOldCounterB()
{
JobConf conf = CreateConfiguration();
CreateWordsFile(inFiles[3], conf);
RemoveWordsFile(inFiles[4], conf);
long inputSize = 0;
inputSize += GetFileSize(inFiles[0]);
inputSize += GetFileSize(inFiles[1]);
inputSize += GetFileSize(inFiles[2]);
inputSize += GetFileSize(inFiles[3]);
conf.SetNumMapTasks(4);
conf.SetInt(JobContext.IoSortFactor, 2);
FileInputFormat.SetInputPaths(conf, InDir);
FileOutputFormat.SetOutputPath(conf, new Path(OutDir, "outputO1"));
RunningJob myJob = JobClient.RunJob(conf);
Counters c1 = myJob.GetCounters();
// As above, each map spills 2^14 records, so 4 maps spill 2^16 records
// In the reduce, there are two intermediate merges before the reduce.
// 1st merge: read + write = 8192 * 4
// 2nd merge: read + write = 8192 * 4
// final merge: 0
// Total reduce: 32768
// Total: map + reduce = 2^16 + 2^15 = 98304
// 4 files, 5120 = 5 * 1024 rec/file = 15360 input records
// 4 records/line = 81920 output records
ValidateCounters(c1, 98304, 20480, 81920);
ValidateFileCounters(c1, inputSize, 0, 0, 0);
}
public void Schedule <TJob>(IScheduleJobManager scheduleJobManager, int siteId, Action <TriggerBuilder> configureTrigger) where TJob : JobBase
{
string groupName = GetJobGroup <TJob>();
string jobKey = GetJobKey <TJob>(siteId);
bool isRunning = IsRunning <TJob>(siteId);
if (!isRunning)
{
scheduleJobManager.ScheduleAsync <TJob, int>(
job =>
{
job.WithIdentity(jobKey, groupName);
},
configureTrigger,
siteId
);
}
var runningJob = new RunningJob
{
JobKey = jobKey,
JobGroup = groupName,
SiteId = siteId,
LastScheduleTime = DateTime.UtcNow
};
_runningJobCache.AddOrUpdate(jobKey, runningJob, (k, job) => { return(runningJob); });
}
/// <exception cref="System.Exception"/>
internal static bool RunJob(JobConf conf)
{
JobClient jobClient = new JobClient(conf);
RunningJob job = jobClient.SubmitJob(conf);
return(jobClient.MonitorAndPrintJob(conf, job));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
internal static bool RunJob(JobConf conf, Path inDir, Path outDir, int numMaps, int
numReds)
{
FileSystem fs = FileSystem.Get(conf);
if (fs.Exists(outDir))
{
fs.Delete(outDir, true);
}
if (!fs.Exists(inDir))
{
fs.Mkdirs(inDir);
}
string input = "The quick brown fox\n" + "has many silly\n" + "red fox sox\n";
for (int i = 0; i < numMaps; ++i)
{
DataOutputStream file = fs.Create(new Path(inDir, "part-" + i));
file.WriteBytes(input);
file.Close();
}
DistributedCache.AddFileToClassPath(TestMRJobs.AppJar, conf, fs);
conf.SetOutputCommitter(typeof(CustomOutputCommitter));
conf.SetInputFormat(typeof(TextInputFormat));
conf.SetOutputKeyClass(typeof(LongWritable));
conf.SetOutputValueClass(typeof(Text));
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
conf.SetNumMapTasks(numMaps);
conf.SetNumReduceTasks(numReds);
JobClient jobClient = new JobClient(conf);
RunningJob job = jobClient.SubmitJob(conf);
return(jobClient.MonitorAndPrintJob(conf, job));
}
// runs a sample job as a user (ugi)
/// <exception cref="System.Exception"/>
internal virtual void RunJobAsUser(JobConf job, UserGroupInformation ugi)
{
RunningJob rj = ugi.DoAs(new _PrivilegedExceptionAction_66(job));
rj.WaitForCompletion();
NUnit.Framework.Assert.AreEqual("SUCCEEDED", JobStatus.GetJobRunState(rj.GetJobState
()));
}
/// <exception cref="System.Exception"/>
private void CheckCompression(bool compressMapOutputs, SequenceFile.CompressionType
redCompression, bool includeCombine)
{
JobConf conf = new JobConf(typeof(TestMapRed));
Path testdir = new Path(TestDir.GetAbsolutePath());
Path inDir = new Path(testdir, "in");
Path outDir = new Path(testdir, "out");
FileSystem fs = FileSystem.Get(conf);
fs.Delete(testdir, true);
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
conf.SetMapperClass(typeof(TestMapRed.MyMap));
conf.SetReducerClass(typeof(TestMapRed.MyReduce));
conf.SetOutputKeyClass(typeof(Text));
conf.SetOutputValueClass(typeof(Text));
conf.SetOutputFormat(typeof(SequenceFileOutputFormat));
conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName);
if (includeCombine)
{
conf.SetCombinerClass(typeof(IdentityReducer));
}
conf.SetCompressMapOutput(compressMapOutputs);
SequenceFileOutputFormat.SetOutputCompressionType(conf, redCompression);
try
{
if (!fs.Mkdirs(testdir))
{
throw new IOException("Mkdirs failed to create " + testdir.ToString());
}
if (!fs.Mkdirs(inDir))
{
throw new IOException("Mkdirs failed to create " + inDir.ToString());
}
Path inFile = new Path(inDir, "part0");
DataOutputStream f = fs.Create(inFile);
f.WriteBytes("Owen was here\n");
f.WriteBytes("Hadoop is fun\n");
f.WriteBytes("Is this done, yet?\n");
f.Close();
RunningJob rj = JobClient.RunJob(conf);
NUnit.Framework.Assert.IsTrue("job was complete", rj.IsComplete());
NUnit.Framework.Assert.IsTrue("job was successful", rj.IsSuccessful());
Path output = new Path(outDir, Task.GetOutputName(0));
NUnit.Framework.Assert.IsTrue("reduce output exists " + output, fs.Exists(output)
);
SequenceFile.Reader rdr = new SequenceFile.Reader(fs, output, conf);
NUnit.Framework.Assert.AreEqual("is reduce output compressed " + output, redCompression
!= SequenceFile.CompressionType.None, rdr.IsCompressed());
rdr.Close();
}
finally
{
fs.Delete(testdir, true);
}
}
static void RunJob(Job job, DateTime time, CancellationToken cancellationToken, TaskFactory taskFactory, ICollection <RunningJob> runningJobs, Action <JobStartedEventArgs <T> > jobStarted)
{
var task = taskFactory.StartNew(() => job.Runner(job.UserObject, cancellationToken), cancellationToken).Unwrap();
var runningJob = new RunningJob(job, time, task);
runningJobs.Add(runningJob);
var args = new JobStartedEventArgs <T>(job.UserObject, runningJob.Task, runningJob.StartTime, cancellationToken);
jobStarted(args);
}
public virtual void TestMRTimelineEventHandling()
{
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, true);
MiniMRYarnCluster cluster = null;
try
{
cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1);
cluster.Init(conf);
cluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + cluster.GetApplicationHistoryServer().GetPort());
TimelineStore ts = cluster.GetApplicationHistoryServer().GetTimelineStore();
Path inDir = new Path("input");
Path outDir = new Path("output");
RunningJob job = UtilsForTests.RunJobSucceed(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Succeeded, job.GetJobStatus().GetState(
).GetValue());
TimelineEntities entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null
, null, null, null, null, null);
NUnit.Framework.Assert.AreEqual(1, entities.GetEntities().Count);
TimelineEntity tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFinished.ToString(), tEntity.GetEvents
()[0].GetEventType());
job = UtilsForTests.RunJobFail(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Failed, job.GetJobStatus().GetState().GetValue
());
entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null
, null, null);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFailed.ToString(), tEntity.GetEvents
()[0].GetEventType());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
}
/// <exception cref="System.Exception"/>
private void RunMergeTest(JobConf job, FileSystem fileSystem, int numMappers, int
numReducers, int numLines, bool isUber)
{
fileSystem.Delete(Output, true);
job.SetJobName("Test");
JobClient client = new JobClient(job);
RunningJob submittedJob = null;
FileInputFormat.SetInputPaths(job, InputDir);
FileOutputFormat.SetOutputPath(job, Output);
job.Set("mapreduce.output.textoutputformat.separator", " ");
job.SetInputFormat(typeof(TextInputFormat));
job.SetMapOutputKeyClass(typeof(Text));
job.SetMapOutputValueClass(typeof(Text));
job.SetOutputKeyClass(typeof(Text));
job.SetOutputValueClass(typeof(Text));
job.SetMapperClass(typeof(TestMRIntermediateDataEncryption.MyMapper));
job.SetPartitionerClass(typeof(TestMRIntermediateDataEncryption.MyPartitioner));
job.SetOutputFormat(typeof(TextOutputFormat));
job.SetNumReduceTasks(numReducers);
job.SetInt("mapreduce.map.maxattempts", 1);
job.SetInt("mapreduce.reduce.maxattempts", 1);
job.SetInt("mapred.test.num_lines", numLines);
if (isUber)
{
job.SetBoolean("mapreduce.job.ubertask.enable", true);
}
job.SetBoolean(MRJobConfig.MrEncryptedIntermediateData, true);
try
{
submittedJob = client.SubmitJob(job);
try
{
if (!client.MonitorAndPrintJob(job, submittedJob))
{
throw new IOException("Job failed!");
}
}
catch (Exception)
{
Sharpen.Thread.CurrentThread().Interrupt();
}
}
catch (IOException ioe)
{
System.Console.Error.WriteLine("Job failed with: " + ioe);
}
finally
{
VerifyOutput(submittedJob, fileSystem, numMappers, numLines);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestCommitFail()
{
Path inDir = new Path(rootDir, "./input");
Path outDir = new Path(rootDir, "./output");
JobConf jobConf = CreateJobConf();
jobConf.SetMaxMapAttempts(1);
jobConf.SetOutputCommitter(typeof(TestTaskCommit.CommitterWithCommitFail));
RunningJob rJob = UtilsForTests.RunJob(jobConf, inDir, outDir, 1, 0);
rJob.WaitForCompletion();
NUnit.Framework.Assert.AreEqual(JobStatus.Failed, rJob.GetJobState());
}
/// <exception cref="System.IO.IOException"/>
public static bool LaunchJob(URI fileSys, JobConf conf, int numMaps, int numReduces
)
{
Path inDir = new Path("/testing/input");
Path outDir = new Path("/testing/output");
FileSystem fs = FileSystem.Get(fileSys, conf);
fs.Delete(outDir, true);
if (!fs.Mkdirs(inDir))
{
Log.Warn("Can't create " + inDir);
return(false);
}
// generate an input file
DataOutputStream file = fs.Create(new Path(inDir, "part-0"));
file.WriteBytes("foo foo2 foo3");
file.Close();
// use WordCount example
FileSystem.SetDefaultUri(conf, fileSys);
conf.SetJobName("foo");
conf.SetInputFormat(typeof(TextInputFormat));
conf.SetOutputFormat(typeof(TestSpecialCharactersInOutputPath.SpecialTextOutputFormat
));
conf.SetOutputKeyClass(typeof(LongWritable));
conf.SetOutputValueClass(typeof(Text));
conf.SetMapperClass(typeof(IdentityMapper));
conf.SetReducerClass(typeof(IdentityReducer));
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
conf.SetNumMapTasks(numMaps);
conf.SetNumReduceTasks(numReduces);
// run job and wait for completion
RunningJob runningJob = JobClient.RunJob(conf);
try
{
NUnit.Framework.Assert.IsTrue(runningJob.IsComplete());
NUnit.Framework.Assert.IsTrue(runningJob.IsSuccessful());
NUnit.Framework.Assert.IsTrue("Output folder not found!", fs.Exists(new Path("/testing/output/"
+ OutputFilename)));
}
catch (ArgumentNullException)
{
// This NPE should no more happens
Fail("A NPE should not have happened.");
}
// return job result
Log.Info("job is complete: " + runningJob.IsSuccessful());
return(runningJob.IsSuccessful());
}
/// <exception cref="System.Exception"/>
private void VerifyOutput(RunningJob submittedJob, FileSystem fileSystem)
{
FSDataInputStream dis = null;
long numValidRecords = 0;
long numInvalidRecords = 0;
long numMappersLaunched = NumMappers;
string prevKeyValue = "000000000";
Path[] fileList = FileUtil.Stat2Paths(fileSystem.ListStatus(Output, new Utils.OutputFileUtils.OutputFilesFilter
()));
foreach (Path outFile in fileList)
{
try
{
dis = fileSystem.Open(outFile);
string record;
while ((record = dis.ReadLine()) != null)
{
// Split the line into key and value.
int blankPos = record.IndexOf(" ");
string keyString = Sharpen.Runtime.Substring(record, 0, blankPos);
string valueString = Sharpen.Runtime.Substring(record, blankPos + 1);
// Check for sorted output and correctness of record.
if (string.CompareOrdinal(keyString, prevKeyValue) >= 0 && keyString.Equals(valueString
))
{
prevKeyValue = keyString;
numValidRecords++;
}
else
{
numInvalidRecords++;
}
}
}
finally
{
if (dis != null)
{
dis.Close();
dis = null;
}
}
}
// Make sure we got all input records in the output in sorted order.
NUnit.Framework.Assert.AreEqual((long)(NumMappers * NumLines), numValidRecords);
// Make sure there is no extraneous invalid record.
NUnit.Framework.Assert.AreEqual(0, numInvalidRecords);
}
public virtual void TestDefaultMRComparator()
{
conf.SetMapperClass(typeof(TestComparators.IdentityMapper));
conf.SetReducerClass(typeof(TestComparators.AscendingKeysReducer));
RunningJob r_job = jc.SubmitJob(conf);
while (!r_job.IsComplete())
{
Sharpen.Thread.Sleep(1000);
}
if (!r_job.IsSuccessful())
{
NUnit.Framework.Assert.Fail("Oops! The job broke due to an unexpected error");
}
}
public virtual void TestValueMismatch()
{
conf.SetMapOutputKeyClass(typeof(Text));
conf.SetMapOutputValueClass(typeof(IntWritable));
RunningJob r_job = jc.SubmitJob(conf);
while (!r_job.IsComplete())
{
Sharpen.Thread.Sleep(1000);
}
if (r_job.IsSuccessful())
{
NUnit.Framework.Assert.Fail("Oops! The job was supposed to break due to an exception"
);
}
}
public virtual void TestNoMismatch()
{
// Set good MapOutputKeyClass and MapOutputValueClass
conf.SetMapOutputKeyClass(typeof(Text));
conf.SetMapOutputValueClass(typeof(Text));
RunningJob r_job = jc.SubmitJob(conf);
while (!r_job.IsComplete())
{
Sharpen.Thread.Sleep(1000);
}
if (!r_job.IsSuccessful())
{
NUnit.Framework.Assert.Fail("Oops! The job broke due to an unexpected error");
}
}
public virtual void TestReporterProgressForMapOnlyJob()
{
Path test = new Path(testRootTempDir, "testReporterProgressForMapOnlyJob");
JobConf conf = new JobConf();
conf.SetMapperClass(typeof(TestReporter.ProgressTesterMapper));
conf.SetMapOutputKeyClass(typeof(Org.Apache.Hadoop.IO.Text));
// fail early
conf.SetMaxMapAttempts(1);
conf.SetMaxReduceAttempts(0);
RunningJob job = UtilsForTests.RunJob(conf, new Path(test, "in"), new Path(test,
"out"), 1, 0, Input);
job.WaitForCompletion();
NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
}
public void SucceedsToStartRunningJob()
{
//Arrange
var jobApp = new JobAppointment(
"Job 1",
"My job #1",
DateTime.UtcNow,
TimeSpan.Zero,
RepeatType.NoRepeat,
null,
"https://localhost:4444");
var job = new RunningJob(jobApp, _logger);
//Act
job.Start(default);
/// <exception cref="System.Exception"/>
private void RunMergeTest(JobConf job, FileSystem fileSystem)
{
// Delete any existing output.
fileSystem.Delete(Output, true);
job.SetJobName("MergeTest");
JobClient client = new JobClient(job);
RunningJob submittedJob = null;
FileInputFormat.SetInputPaths(job, InputDir);
FileOutputFormat.SetOutputPath(job, Output);
job.Set("mapreduce.output.textoutputformat.separator", " ");
job.SetInputFormat(typeof(TextInputFormat));
job.SetMapOutputKeyClass(typeof(Text));
job.SetMapOutputValueClass(typeof(Text));
job.SetOutputKeyClass(typeof(Text));
job.SetOutputValueClass(typeof(Text));
job.SetMapperClass(typeof(TestMerge.MyMapper));
job.SetPartitionerClass(typeof(TestMerge.MyPartitioner));
job.SetOutputFormat(typeof(TextOutputFormat));
job.SetNumReduceTasks(NumReducers);
job.Set(JobContext.MapOutputCollectorClassAttr, typeof(TestMerge.MapOutputCopier)
.FullName);
try
{
submittedJob = client.SubmitJob(job);
try
{
if (!client.MonitorAndPrintJob(job, submittedJob))
{
throw new IOException("Job failed!");
}
}
catch (Exception)
{
Sharpen.Thread.CurrentThread().Interrupt();
}
}
catch (IOException ioe)
{
System.Console.Error.WriteLine("Job failed with: " + ioe);
}
finally
{
VerifyOutput(submittedJob, fileSystem);
}
}
public virtual void TestUserValueGroupingComparator()
{
conf.SetMapperClass(typeof(TestComparators.RandomGenMapper));
conf.SetReducerClass(typeof(TestComparators.AscendingGroupReducer));
conf.SetOutputValueGroupingComparator(typeof(TestComparators.CompositeIntGroupFn)
);
RunningJob r_job = jc.SubmitJob(conf);
while (!r_job.IsComplete())
{
Sharpen.Thread.Sleep(1000);
}
if (!r_job.IsSuccessful())
{
NUnit.Framework.Assert.Fail("Oops! The job broke due to an unexpected error");
}
}
// Run a job that will be killed and wait until it completes
/// <exception cref="System.IO.IOException"/>
public static RunningJob RunJobKill(JobConf conf, Path inDir, Path outDir)
{
conf.SetJobName("test-job-kill");
conf.SetMapperClass(typeof(UtilsForTests.KillMapper));
conf.SetReducerClass(typeof(IdentityReducer));
RunningJob job = UtilsForTests.RunJob(conf, inDir, outDir);
long sleepCount = 0;
while (job.GetJobState() != JobStatus.Running)
{
try
{
if (sleepCount > 300)
{
// 30 seconds
throw new IOException("Job didn't finish in 30 seconds");
}
Sharpen.Thread.Sleep(100);
sleepCount++;
}
catch (Exception)
{
break;
}
}
job.KillJob();
sleepCount = 0;
while (job.CleanupProgress() == 0.0f)
{
try
{
if (sleepCount > 2000)
{
// 20 seconds
throw new IOException("Job cleanup didn't start in 20 seconds");
}
Sharpen.Thread.Sleep(10);
sleepCount++;
}
catch (Exception)
{
break;
}
}
return(job);
}
static void setStatus(RunningJob job, StatusType status)
{
int left = Console.CursorLeft;
int top = Console.CursorTop;
Console.CursorLeft = 5;
Console.CursorTop = (int)job;
Console.ResetColor();
switch (status)
{
case StatusType.Done:
Console.ForegroundColor = ConsoleColor.Green;
Console.Write("=");
break;
case StatusType.Failed:
Console.ForegroundColor = ConsoleColor.Red;
Console.Write("X");
break;
case StatusType.Running:
Console.ForegroundColor = ConsoleColor.Yellow;
Console.Write("►");
break;
default:
Console.Write("?");
break;
}
Console.ResetColor();
Console.CursorTop = top;
Console.CursorLeft = left;
}
