public virtual void TestMaxTaskFailuresPerTracker()
{
JobConf jobConf = new JobConf(true);
NUnit.Framework.Assert.IsTrue("By default JobContext.MAX_TASK_FAILURES_PER_TRACKER was "
+ "not less than JobContext.MAP_MAX_ATTEMPTS and REDUCE_MAX_ATTEMPTS", jobConf.
GetMaxTaskFailuresPerTracker() < jobConf.GetMaxMapAttempts() && jobConf.GetMaxTaskFailuresPerTracker
() < jobConf.GetMaxReduceAttempts());
}
/// <exception cref="System.Exception"/>
protected override void TearDown()
{
Path rootDir = GetDir(RootDir);
JobConf conf = CreateJobConf();
FileSystem fs = FileSystem.Get(conf);
fs.Delete(rootDir, true);
base.TearDown();
}
public virtual void TestProfileParamsDefaults()
{
JobConf configuration = new JobConf();
string result = configuration.GetProfileParams();
NUnit.Framework.Assert.IsNotNull(result);
NUnit.Framework.Assert.IsTrue(result.Contains("file=%s"));
NUnit.Framework.Assert.IsTrue(result.StartsWith("-agentlib:hprof"));
}
/// <summary>
/// Add a
/// <see cref="Org.Apache.Hadoop.FS.Path"/>
/// to the list of inputs for the map-reduce job.
/// </summary>
/// <param name="conf">The configuration of the job</param>
/// <param name="path">
///
/// <see cref="Org.Apache.Hadoop.FS.Path"/>
/// to be added to the list of inputs for
/// the map-reduce job.
/// </param>
public static void AddInputPath(JobConf conf, Path path)
{
path = new Path(conf.GetWorkingDirectory(), path);
string dirStr = StringUtils.EscapeString(path.ToString());
string dirs = conf.Get(FileInputFormat.InputDir);
conf.Set(FileInputFormat.InputDir, dirs == null ? dirStr : dirs + StringUtils.CommaStr
+ dirStr);
}
/// <exception cref="System.IO.IOException"/>
private IList <FileStatus> SingleThreadedListStatus(JobConf job, Path[] dirs, PathFilter
inputFilter, bool recursive)
{
IList <FileStatus> result = new AList <FileStatus>();
IList <IOException> errors = new AList <IOException>();
foreach (Path p in dirs)
{
FileSystem fs = p.GetFileSystem(job);
FileStatus[] matches = fs.GlobStatus(p, inputFilter);
if (matches == null)
{
errors.AddItem(new IOException("Input path does not exist: " + p));
}
else
{
if (matches.Length == 0)
{
errors.AddItem(new IOException("Input Pattern " + p + " matches 0 files"));
}
else
{
foreach (FileStatus globStat in matches)
{
if (globStat.IsDirectory())
{
RemoteIterator <LocatedFileStatus> iter = fs.ListLocatedStatus(globStat.GetPath());
while (iter.HasNext())
{
LocatedFileStatus stat = iter.Next();
if (inputFilter.Accept(stat.GetPath()))
{
if (recursive && stat.IsDirectory())
{
AddInputPathRecursively(result, fs, stat.GetPath(), inputFilter);
}
else
{
result.AddItem(stat);
}
}
}
}
else
{
result.AddItem(globStat);
}
}
}
}
}
if (!errors.IsEmpty())
{
throw new InvalidInputException(errors);
}
return(result);
}
private static string GetChildEnv(JobConf jobConf, bool isMap)
{
if (isMap)
{
return(jobConf.Get(JobConf.MapredMapTaskEnv, jobConf.Get(JobConf.MapredTaskEnv)));
}
return(jobConf.Get(JobConf.MapredReduceTaskEnv, jobConf.Get(JobConf.MapredTaskEnv
)));
}
private static DBConfiguration SetOutput(JobConf job, string tableName)
{
job.SetOutputFormat(typeof(DBOutputFormat));
job.SetReduceSpeculativeExecution(false);
DBConfiguration dbConf = new DBConfiguration(job);
dbConf.SetOutputTableName(tableName);
return(dbConf);
}
/// <exception cref="System.IO.IOException"/>
private void GetBaseOutputFormat(JobConf job)
{
baseOut = ReflectionUtils.NewInstance(job.GetClass <OutputFormat>("mapreduce.output.lazyoutputformat.outputformat"
, null), job);
if (baseOut == null)
{
throw new IOException("Ouput format not set for LazyOutputFormat");
}
}
/// <exception cref="System.IO.IOException"/>
protected internal override RecordWriter <K, V> GetBaseRecordWriter(FileSystem fs,
JobConf job, string name, Progressable arg3)
{
if (theSequenceFileOutputFormat == null)
{
theSequenceFileOutputFormat = new SequenceFileOutputFormat <K, V>();
}
return(theSequenceFileOutputFormat.GetRecordWriter(fs, job, name, arg3));
}
public virtual void TestFormatWithCustomSeparator()
{
JobConf job = new JobConf();
string separator = "\u0001";
job.Set("mapreduce.output.textoutputformat.separator", separator);
job.Set(JobContext.TaskAttemptId, attempt);
FileOutputFormat.SetOutputPath(job, workDir.GetParent().GetParent());
FileOutputFormat.SetWorkOutputPath(job, workDir);
FileSystem fs = workDir.GetFileSystem(job);
if (!fs.Mkdirs(workDir))
{
NUnit.Framework.Assert.Fail("Failed to create output directory");
}
string file = "test_custom.txt";
// A reporter that does nothing
Reporter reporter = Reporter.Null;
TextOutputFormat <object, object> theOutputFormat = new TextOutputFormat <object, object
>();
RecordWriter <object, object> theRecordWriter = theOutputFormat.GetRecordWriter(localFs
, job, file, reporter);
Org.Apache.Hadoop.IO.Text key1 = new Org.Apache.Hadoop.IO.Text("key1");
Org.Apache.Hadoop.IO.Text key2 = new Org.Apache.Hadoop.IO.Text("key2");
Org.Apache.Hadoop.IO.Text val1 = new Org.Apache.Hadoop.IO.Text("val1");
Org.Apache.Hadoop.IO.Text val2 = new Org.Apache.Hadoop.IO.Text("val2");
NullWritable nullWritable = NullWritable.Get();
try
{
theRecordWriter.Write(key1, val1);
theRecordWriter.Write(null, nullWritable);
theRecordWriter.Write(null, val1);
theRecordWriter.Write(nullWritable, val2);
theRecordWriter.Write(key2, nullWritable);
theRecordWriter.Write(key1, null);
theRecordWriter.Write(null, null);
theRecordWriter.Write(key2, val2);
}
finally
{
theRecordWriter.Close(reporter);
}
FilePath expectedFile = new FilePath(new Path(workDir, file).ToString());
StringBuilder expectedOutput = new StringBuilder();
expectedOutput.Append(key1).Append(separator).Append(val1).Append("\n");
expectedOutput.Append(val1).Append("\n");
expectedOutput.Append(val2).Append("\n");
expectedOutput.Append(key2).Append("\n");
expectedOutput.Append(key1).Append("\n");
expectedOutput.Append(key2).Append(separator).Append(val2).Append("\n");
string output = UtilsForTests.Slurp(expectedFile);
NUnit.Framework.Assert.AreEqual(expectedOutput.ToString(), output);
}
public virtual void TestNegativeValuesForMemoryParams()
{
JobConf configuration = new JobConf();
configuration.Set(MRJobConfig.MapMemoryMb, "-5");
configuration.Set(MRJobConfig.ReduceMemoryMb, "-6");
NUnit.Framework.Assert.AreEqual(-5, configuration.GetMemoryForMapTask());
NUnit.Framework.Assert.AreEqual(-6, configuration.GetMemoryForReduceTask());
}
public FakeFetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl <K, V
> scheduler, MergeManagerImpl <K, V> merger, Reporter reporter, ShuffleClientMetrics
metrics, ExceptionReporter exceptionReporter, SecretKey jobTokenSecret, HttpURLConnection
connection, int id)
: base(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, jobTokenSecret
, id)
{
this.connection = connection;
}
private MiniMRYarnCluster InitAndStartMiniMRYarnCluster(JobConf jobConf)
{
MiniMRYarnCluster miniMRYarnCluster;
miniMRYarnCluster = new MiniMRYarnCluster(this.GetType().FullName, 1);
miniMRYarnCluster.Init(jobConf);
miniMRYarnCluster.Start();
return(miniMRYarnCluster);
}
protected internal override JobConf CreateJobConf()
{
JobConf conf = base.CreateJobConf();
conf.SetJobEndNotificationURI(GetNotificationUrlTemplate());
conf.SetInt(JobContext.MrJobEndRetryAttempts, 3);
conf.SetInt(JobContext.MrJobEndRetryInterval, 200);
return(conf);
}
/// <exception cref="System.IO.IOException"/>
public virtual InputSplit[] GetSplits(JobConf job, int numSplits)
{
InputSplit[] splits = new InputSplit[numSplits];
for (int i = 0; i < splits.Length; ++i)
{
splits[i] = new DummyInputFormat.EmptySplit();
}
return(splits);
}
public void JobConductorShouldCreateJobConfWithJobDirectorySet()
{
string name = MethodBase.GetCurrentMethod().Name;
JobManagerService foreman = GetTestJobConductor(name);
JobConf conf = foreman.CreateJobConf(name);
Expect.AreEqual(name, conf.Name);
Expect.IsTrue(conf.JobDirectory.StartsWith(foreman.JobsDirectory), "conf directory wasn't set correctly");
}
/// <exception cref="System.IO.IOException"/>
public override RecordWriter <K, V> GetRecordWriter(FileSystem ignored, JobConf job
, string name, Progressable progress)
{
if (baseOut == null)
{
GetBaseOutputFormat(job);
}
return(new LazyOutputFormat.LazyRecordWriter <K, V>(job, baseOut, name, progress));
}
/// <summary>Adds a Mapper class to the chain job's JobConf.</summary>
/// <remarks>
/// Adds a Mapper class to the chain job's JobConf.
/// <p>
/// It has to be specified how key and values are passed from one element of
/// the chain to the next, by value or by reference. If a Mapper leverages the
/// assumed semantics that the key and values are not modified by the collector
/// 'by value' must be used. If the Mapper does not expect this semantics, as
/// an optimization to avoid serialization and deserialization 'by reference'
/// can be used.
/// <p>
/// For the added Mapper the configuration given for it,
/// <code>mapperConf</code>, have precedence over the job's JobConf. This
/// precedence is in effect when the task is running.
/// <p>
/// IMPORTANT: There is no need to specify the output key/value classes for the
/// ChainMapper, this is done by the addMapper for the last mapper in the chain
/// .
/// </remarks>
/// <param name="job">chain job's JobConf to add the Mapper class.</param>
/// <param name="klass">the Mapper class to add.</param>
/// <param name="inputKeyClass">mapper input key class.</param>
/// <param name="inputValueClass">mapper input value class.</param>
/// <param name="outputKeyClass">mapper output key class.</param>
/// <param name="outputValueClass">mapper output value class.</param>
/// <param name="byValue">
/// indicates if key/values should be passed by value
/// to the next Mapper in the chain, if any.
/// </param>
/// <param name="mapperConf">
/// a JobConf with the configuration for the Mapper
/// class. It is recommended to use a JobConf without default values using the
/// <code>JobConf(boolean loadDefaults)</code> constructor with FALSE.
/// </param>
public static void AddMapper <K1, V1, K2, V2>(JobConf job, Type klass, Type inputKeyClass
, Type inputValueClass, Type outputKeyClass, Type outputValueClass, bool byValue
, JobConf mapperConf)
{
job.SetOutputKeyClass(outputKeyClass);
job.SetOutputValueClass(outputValueClass);
Chain.AddMapper(false, job, klass, inputKeyClass, inputValueClass, outputKeyClass
, outputValueClass, byValue, mapperConf);
}
/// <exception cref="System.Exception"/>
private void CheckCompression(bool compressMapOutputs, SequenceFile.CompressionType
redCompression, bool includeCombine)
{
JobConf conf = new JobConf(typeof(TestMapRed));
Path testdir = new Path(TestDir.GetAbsolutePath());
Path inDir = new Path(testdir, "in");
Path outDir = new Path(testdir, "out");
FileSystem fs = FileSystem.Get(conf);
fs.Delete(testdir, true);
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
conf.SetMapperClass(typeof(TestMapRed.MyMap));
conf.SetReducerClass(typeof(TestMapRed.MyReduce));
conf.SetOutputKeyClass(typeof(Text));
conf.SetOutputValueClass(typeof(Text));
conf.SetOutputFormat(typeof(SequenceFileOutputFormat));
conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName);
if (includeCombine)
{
conf.SetCombinerClass(typeof(IdentityReducer));
}
conf.SetCompressMapOutput(compressMapOutputs);
SequenceFileOutputFormat.SetOutputCompressionType(conf, redCompression);
try
{
if (!fs.Mkdirs(testdir))
{
throw new IOException("Mkdirs failed to create " + testdir.ToString());
}
if (!fs.Mkdirs(inDir))
{
throw new IOException("Mkdirs failed to create " + inDir.ToString());
}
Path inFile = new Path(inDir, "part0");
DataOutputStream f = fs.Create(inFile);
f.WriteBytes("Owen was here\n");
f.WriteBytes("Hadoop is fun\n");
f.WriteBytes("Is this done, yet?\n");
f.Close();
RunningJob rj = JobClient.RunJob(conf);
NUnit.Framework.Assert.IsTrue("job was complete", rj.IsComplete());
NUnit.Framework.Assert.IsTrue("job was successful", rj.IsSuccessful());
Path output = new Path(outDir, Task.GetOutputName(0));
NUnit.Framework.Assert.IsTrue("reduce output exists " + output, fs.Exists(output)
);
SequenceFile.Reader rdr = new SequenceFile.Reader(fs, output, conf);
NUnit.Framework.Assert.AreEqual("is reduce output compressed " + output, redCompression
!= SequenceFile.CompressionType.None, rdr.IsCompressed());
rdr.Close();
}
finally
{
fs.Delete(testdir, true);
}
}
/// <summary>
/// Randomize the split order, then take the specified number of keys from
/// each split sampled, where each key is selected with the specified
/// probability and possibly replaced by a subsequently selected key when
/// the quota of keys from that split is satisfied.
/// </summary>
/// <exception cref="System.IO.IOException"/>
public virtual K[] GetSample(InputFormat <K, V> inf, JobConf job)
{
// ArrayList::toArray doesn't preserve type
InputSplit[] splits = inf.GetSplits(job, job.GetNumMapTasks());
AList <K> samples = new AList <K>(numSamples);
int splitsToSample = Math.Min(maxSplitsSampled, splits.Length);
Random r = new Random();
long seed = r.NextLong();
r.SetSeed(seed);
Log.Debug("seed: " + seed);
// shuffle splits
for (int i = 0; i < splits.Length; ++i)
{
InputSplit tmp = splits[i];
int j = r.Next(splits.Length);
splits[i] = splits[j];
splits[j] = tmp;
}
// our target rate is in terms of the maximum number of sample splits,
// but we accept the possibility of sampling additional splits to hit
// the target sample keyset
for (int i_1 = 0; i_1 < splitsToSample || (i_1 < splits.Length && samples.Count <
numSamples); ++i_1)
{
RecordReader <K, V> reader = inf.GetRecordReader(splits[i_1], job, Reporter.Null);
K key = reader.CreateKey();
V value = reader.CreateValue();
while (reader.Next(key, value))
{
if (r.NextDouble() <= freq)
{
if (samples.Count < numSamples)
{
samples.AddItem(key);
}
else
{
// When exceeding the maximum number of samples, replace a
// random element with this one, then adjust the frequency
// to reflect the possibility of existing elements being
// pushed out
int ind = r.Next(numSamples);
if (ind != numSamples)
{
samples.Set(ind, key);
}
freq *= (numSamples - 1) / (double)numSamples;
}
key = reader.CreateKey();
}
}
reader.Close();
}
return((K[])Sharpen.Collections.ToArray(samples));
}
public virtual void TestSucceedAndFailedCopyMap <K, V>()
{
JobConf job = new JobConf();
job.SetNumMapTasks(2);
//mock creation
TaskUmbilicalProtocol mockUmbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol
>();
Reporter mockReporter = Org.Mockito.Mockito.Mock <Reporter>();
FileSystem mockFileSystem = Org.Mockito.Mockito.Mock <FileSystem>();
Type combinerClass = job.GetCombinerClass();
Task.CombineOutputCollector <K, V> mockCombineOutputCollector = (Task.CombineOutputCollector
<K, V>)Org.Mockito.Mockito.Mock <Task.CombineOutputCollector>();
// needed for mock with generic
TaskAttemptID mockTaskAttemptID = Org.Mockito.Mockito.Mock <TaskAttemptID>();
LocalDirAllocator mockLocalDirAllocator = Org.Mockito.Mockito.Mock <LocalDirAllocator
>();
CompressionCodec mockCompressionCodec = Org.Mockito.Mockito.Mock <CompressionCodec
>();
Counters.Counter mockCounter = Org.Mockito.Mockito.Mock <Counters.Counter>();
TaskStatus mockTaskStatus = Org.Mockito.Mockito.Mock <TaskStatus>();
Progress mockProgress = Org.Mockito.Mockito.Mock <Progress>();
MapOutputFile mockMapOutputFile = Org.Mockito.Mockito.Mock <MapOutputFile>();
Org.Apache.Hadoop.Mapred.Task mockTask = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Mapred.Task
>();
MapOutput <K, V> output = Org.Mockito.Mockito.Mock <MapOutput>();
ShuffleConsumerPlugin.Context <K, V> context = new ShuffleConsumerPlugin.Context <K
, V>(mockTaskAttemptID, job, mockFileSystem, mockUmbilical, mockLocalDirAllocator
, mockReporter, mockCompressionCodec, combinerClass, mockCombineOutputCollector,
mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockCounter, mockTaskStatus
, mockProgress, mockProgress, mockTask, mockMapOutputFile, null);
TaskStatus status = new _TaskStatus_251();
Progress progress = new Progress();
ShuffleSchedulerImpl <K, V> scheduler = new ShuffleSchedulerImpl <K, V>(job, status
, null, null, progress, context.GetShuffledMapsCounter(), context.GetReduceShuffleBytes
(), context.GetFailedShuffleCounter());
MapHost host1 = new MapHost("host1", null);
TaskAttemptID failedAttemptID = new TaskAttemptID(new TaskID(new JobID("test", 0)
, TaskType.Map, 0), 0);
TaskAttemptID succeedAttemptID = new TaskAttemptID(new TaskID(new JobID("test", 0
), TaskType.Map, 1), 1);
// handle output fetch failure for failedAttemptID, part I
scheduler.HostFailed(host1.GetHostName());
// handle output fetch succeed for succeedAttemptID
long bytes = (long)500 * 1024 * 1024;
scheduler.CopySucceeded(succeedAttemptID, host1, bytes, 0, 500000, output);
// handle output fetch failure for failedAttemptID, part II
// for MAPREDUCE-6361: verify no NPE exception get thrown out
scheduler.CopyFailed(failedAttemptID, host1, true, false);
}
public virtual void TestSocketFactory()
{
// Create a standard mini-cluster
Configuration sconf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(sconf).NumDataNodes(1).Build(
);
int nameNodePort = cluster.GetNameNodePort();
// Get a reference to its DFS directly
FileSystem fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue(fs is DistributedFileSystem);
DistributedFileSystem directDfs = (DistributedFileSystem)fs;
Configuration cconf = GetCustomSocketConfigs(nameNodePort);
fs = FileSystem.Get(cconf);
NUnit.Framework.Assert.IsTrue(fs is DistributedFileSystem);
DistributedFileSystem dfs = (DistributedFileSystem)fs;
JobClient client = null;
MiniMRYarnCluster miniMRYarnCluster = null;
try
{
// This will test RPC to the NameNode only.
// could we test Client-DataNode connections?
Path filePath = new Path("/dir");
NUnit.Framework.Assert.IsFalse(directDfs.Exists(filePath));
NUnit.Framework.Assert.IsFalse(dfs.Exists(filePath));
directDfs.Mkdirs(filePath);
NUnit.Framework.Assert.IsTrue(directDfs.Exists(filePath));
NUnit.Framework.Assert.IsTrue(dfs.Exists(filePath));
// This will test RPC to a Resource Manager
fs = FileSystem.Get(sconf);
JobConf jobConf = new JobConf();
FileSystem.SetDefaultUri(jobConf, fs.GetUri().ToString());
miniMRYarnCluster = InitAndStartMiniMRYarnCluster(jobConf);
JobConf jconf = new JobConf(miniMRYarnCluster.GetConfig());
jconf.Set("hadoop.rpc.socket.factory.class.default", "org.apache.hadoop.ipc.DummySocketFactory"
);
jconf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
string rmAddress = jconf.Get("yarn.resourcemanager.address");
string[] split = rmAddress.Split(":");
jconf.Set("yarn.resourcemanager.address", split[0] + ':' + (System.Convert.ToInt32
(split[1]) + 10));
client = new JobClient(jconf);
JobStatus[] jobs = client.JobsToComplete();
NUnit.Framework.Assert.IsTrue(jobs.Length == 0);
}
finally
{
CloseClient(client);
CloseDfs(dfs);
CloseDfs(directDfs);
StopMiniMRYarnCluster(miniMRYarnCluster);
ShutdownDFSCluster(cluster);
}
}
public void JobConductorShouldCreaetJobConf()
{
JobConductorService.Default.JobsDirectory = new DirectoryInfo(MethodBase.GetCurrentMethod().Name).FullName;
JobConductorService fm = JobConductorService.Default;
string name = "JobConfTest_".RandomLetters(4);
JobConf conf = fm.CreateJob(name);
string path = Path.Combine(fm.JobsDirectory, conf.Name, conf.Name + ".job");
Expect.IsTrue(File.Exists(path));
}
private static JobConf CreateTestJobConf(Configuration conf, int retryAttempts, string
notificationUri)
{
JobConf jobConf = new JobConf(conf);
jobConf.SetInt("job.end.retry.attempts", retryAttempts);
jobConf.Set("job.end.retry.interval", "0");
jobConf.SetJobEndNotificationURI(notificationUri);
return(jobConf);
}
/// <summary>
/// Create a composite record writer that can write key/value data to different
/// output files
/// </summary>
/// <param name="fs">the file system to use</param>
/// <param name="job">the job conf for the job</param>
/// <param name="name">the leaf file name for the output file (such as part-00000")</param>
/// <param name="arg3">a progressable for reporting progress.</param>
/// <returns>a composite record writer</returns>
/// <exception cref="System.IO.IOException"/>
public override RecordWriter <K, V> GetRecordWriter(FileSystem fs, JobConf job, string
name, Progressable arg3)
{
FileSystem myFS = fs;
string myName = GenerateLeafFileName(name);
JobConf myJob = job;
Progressable myProgressable = arg3;
return(new _RecordWriter_82(this, myName, myJob, myFS, myProgressable));
}
// Dummy Input format to send 1 record - number of spits is numMapTasks
public virtual InputSplit[] GetSplits(JobConf conf, int numSplits)
{
numSplits = conf.GetInt("LG.numMapTasks", 1);
InputSplit[] ret = new InputSplit[numSplits];
for (int i = 0; i < numSplits; ++i)
{
ret[i] = new LoadGeneratorMR.EmptySplit();
}
return(ret);
}
/// <summary>Initializes the map-part of the job with the appropriate input settings.
/// </summary>
/// <param name="job">The job</param>
/// <param name="inputClass">
/// the class object implementing DBWritable, which is the
/// Java object holding tuple fields.
/// </param>
/// <param name="inputQuery">
/// the input query to select fields. Example :
/// "SELECT f1, f2, f3 FROM Mytable ORDER BY f1"
/// </param>
/// <param name="inputCountQuery">
/// the input query that returns the number of records in
/// the table.
/// Example : "SELECT COUNT(f1) FROM Mytable"
/// </param>
/// <seealso cref="DBInputFormat{T}.SetInput(Org.Apache.Hadoop.Mapred.JobConf, System.Type{T}, string, string, string, string[])
/// "/>
public static void SetInput(JobConf job, Type inputClass, string inputQuery, string
inputCountQuery)
{
job.SetInputFormat(typeof(DBInputFormat));
DBConfiguration dbConf = new DBConfiguration(job);
dbConf.SetInputClass(inputClass);
dbConf.SetInputQuery(inputQuery);
dbConf.SetInputCountQuery(inputCountQuery);
}
/// <exception cref="System.IO.IOException"/>
public override void AbortJob(JobContext context, int state)
{
JobConf conf = context.GetJobConf();
Path outputPath = FileOutputFormat.GetOutputPath(conf);
FileSystem fs = outputPath.GetFileSystem(conf);
string fileName = (state == JobStatus.Failed) ? TestJobCleanup.AbortFailedFileName
: TestJobCleanup.AbortKilledFileName;
fs.Create(new Path(outputPath, fileName)).Close();
}
public RR_ClassLoaderChecker(JobConf job)
{
NUnit.Framework.Assert.IsTrue("The class loader has not been inherited from " + typeof(
CompositeRecordReader).Name, job.GetClassLoader() is TestWrappedRecordReaderClassloader.Fake_ClassLoader
);
keyclass = (Type)job.GetClass <WritableComparable>("test.fakeif.keyclass", typeof(
NullWritable));
valclass = (Type)job.GetClass <WritableComparable>("test.fakeif.valclass", typeof(
NullWritable));
}
public ReduceTaskImpl(JobId jobId, int partition, EventHandler eventHandler, Path
jobFile, JobConf conf, int numMapTasks, TaskAttemptListener taskAttemptListener
, Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jobToken, Credentials
credentials, Clock clock, int appAttemptId, MRAppMetrics metrics, AppContext appContext
)
: base(jobId, TaskType.Reduce, partition, eventHandler, jobFile, conf, taskAttemptListener
, jobToken, credentials, clock, appAttemptId, metrics, appContext)
{
this.numMapTasks = numMapTasks;
}
// Before
public void openFileSystem()
{
conf = new JobConf();
// all columns
conf.set("columns", "userid,string1,subtype,decimal1,ts");
conf.set("columns.types", "bigint,string,double,decimal,timestamp");
// needed columns
conf.set(ColumnProjectionUtils.READ_ALL_COLUMNS, "false");
conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, "0,2");
conf.set(ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR, "userid,subtype");
fs = FileSystem.getLocal(conf);
testFilePath = new Path(workDir, "TestOrcFile." +
testCaseName.getMethodName() + ".orc");
fs.delete(testFilePath, false);
}
public void NextEvent()
{
int cmd = WritableUtils.ReadVInt(down);
if (cmd < 0)
{
down.Close();
return;
}
BinaryMessageTypes message = (BinaryMessageTypes)cmd;
byte[] val;
switch (message)
{
case BinaryMessageTypes.START_MESSAGE:
handler.Start(WritableUtils.ReadVInt(down));
break;
case BinaryMessageTypes.SET_JOB_CONF:
int entries = WritableUtils.ReadVInt(down);
JobConf conf = new JobConf();
for (int i = 0; i < entries; ++i)
{
// TODO Fix
WritableUtils.ReadString(down);
}
//handler.SetJobConf(conf);
break;
case BinaryMessageTypes.SET_INPUT_TYPES:
string keyType = WritableUtils.ReadString(down);
string valueType = WritableUtils.ReadString(down);
handler.SetInputTypes(keyType, valueType);
break;
case BinaryMessageTypes.RUN_MAP:
string split = WritableUtils.ReadString(down);
int numReduces = WritableUtils.ReadVInt(down);
bool piped = WritableUtils.ReadVInt(down) == 1;
handler.RunMap(split, numReduces, piped);
break;
case BinaryMessageTypes.MAP_ITEM:
byte[] key = WritableUtils.ReadBytes(down);
val = WritableUtils.ReadBytes(down);
handler.MapItem(key, val);
break;
case BinaryMessageTypes.RUN_REDUCE:
handler.RunReduce(WritableUtils.ReadVInt(down), WritableUtils.ReadVInt(down) == 1);
break;
case BinaryMessageTypes.REDUCE_KEY:
key = WritableUtils.ReadBytes(down);
handler.ReduceKey(key);
break;
case BinaryMessageTypes.REDUCE_VALUE:
val = WritableUtils.ReadBytes(down);
handler.ReduceValue(val);
break;
case BinaryMessageTypes.CLOSE:
handler.Close();
break;
case BinaryMessageTypes.ABORT:
handler.Abort();
break;
default:
throw new ApplicationException("Unknown binary command: " + cmd);
}
}
