public static string RunRootCmd(string command, char split)
{
Process process = null;
var result = "false";
try
{
process = Runtime.GetRuntime().Exec("su");
var outputStream = process.OutputStream;
var dataOutputStream = new DataOutputStream(outputStream);
var cmds = command.Split(split);
var temp = cmds.Aggregate("", (current, t) => current + (t + "\n"));
dataOutputStream.WriteBytes(temp);
dataOutputStream.Flush();
dataOutputStream.WriteBytes("exit\n");
dataOutputStream.Flush();
process.WaitFor();
result = InputStreamToString(new InputStreamAdapter(process.InputStream));
}
catch
{
return(result);
}
finally
{
process?.Destroy();
}
return(result);
}
/// <summary>
/// 应用程序运行命令获取 Root权限，设备必须已破解(获得ROOT权限)
/// @param command 命令：String apkRoot = "chmod 777 " + getPackageCodePath(); RootCommand(apkRoot);
/// @return 应用程序是/否获取Root权限
/// </summary>
public static bool RootCommand(string command)
{
Process process = null;
try
{
process = Runtime.GetRuntime().Exec("su");
using (var os = new DataOutputStream(process.OutputStream))
{
os.WriteBytes(command + "\n");
os.WriteBytes("exit\n");
os.Flush();
}
process.WaitFor();
}
catch (Exception e)
{
return(false);
}
finally
{
if (process != null)
{
process.Destroy();
}
}
return(true);
}
/// <exception cref="System.Exception"/>
private void CheckCompression(bool compressMapOutputs, SequenceFile.CompressionType
redCompression, bool includeCombine)
{
JobConf conf = new JobConf(typeof(TestMapRed));
Path testdir = new Path(TestDir.GetAbsolutePath());
Path inDir = new Path(testdir, "in");
Path outDir = new Path(testdir, "out");
FileSystem fs = FileSystem.Get(conf);
fs.Delete(testdir, true);
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
conf.SetMapperClass(typeof(TestMapRed.MyMap));
conf.SetReducerClass(typeof(TestMapRed.MyReduce));
conf.SetOutputKeyClass(typeof(Text));
conf.SetOutputValueClass(typeof(Text));
conf.SetOutputFormat(typeof(SequenceFileOutputFormat));
conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName);
if (includeCombine)
{
conf.SetCombinerClass(typeof(IdentityReducer));
}
conf.SetCompressMapOutput(compressMapOutputs);
SequenceFileOutputFormat.SetOutputCompressionType(conf, redCompression);
try
{
if (!fs.Mkdirs(testdir))
{
throw new IOException("Mkdirs failed to create " + testdir.ToString());
}
if (!fs.Mkdirs(inDir))
{
throw new IOException("Mkdirs failed to create " + inDir.ToString());
}
Path inFile = new Path(inDir, "part0");
DataOutputStream f = fs.Create(inFile);
f.WriteBytes("Owen was here\n");
f.WriteBytes("Hadoop is fun\n");
f.WriteBytes("Is this done, yet?\n");
f.Close();
RunningJob rj = JobClient.RunJob(conf);
NUnit.Framework.Assert.IsTrue("job was complete", rj.IsComplete());
NUnit.Framework.Assert.IsTrue("job was successful", rj.IsSuccessful());
Path output = new Path(outDir, Task.GetOutputName(0));
NUnit.Framework.Assert.IsTrue("reduce output exists " + output, fs.Exists(output)
);
SequenceFile.Reader rdr = new SequenceFile.Reader(fs, output, conf);
NUnit.Framework.Assert.AreEqual("is reduce output compressed " + output, redCompression
!= SequenceFile.CompressionType.None, rdr.IsCompressed());
rdr.Close();
}
finally
{
fs.Delete(testdir, true);
}
}
public override void Configure(JobConf jconf)
{
conf = jconf;
try
{
Path[] localArchives = DistributedCache.GetLocalCacheArchives(conf);
Path[] localFiles = DistributedCache.GetLocalCacheFiles(conf);
// read the cached files (unzipped, unjarred and text)
// and put it into a single file TEST_ROOT_DIR/test.txt
string TestRootDir = jconf.Get("test.build.data", "/tmp");
Path file = new Path("file:///", TestRootDir);
FileSystem fs = FileSystem.GetLocal(conf);
if (!fs.Mkdirs(file))
{
throw new IOException("Mkdirs failed to create " + file.ToString());
}
Path fileOut = new Path(file, "test.txt");
fs.Delete(fileOut, true);
DataOutputStream @out = fs.Create(fileOut);
for (int i = 0; i < localArchives.Length; i++)
{
// read out the files from these archives
FilePath f = new FilePath(localArchives[i].ToString());
FilePath txt = new FilePath(f, "test.txt");
FileInputStream fin = new FileInputStream(txt);
DataInputStream din = new DataInputStream(fin);
string str = din.ReadLine();
din.Close();
@out.WriteBytes(str);
@out.WriteBytes("\n");
}
for (int i_1 = 0; i_1 < localFiles.Length; i_1++)
{
// read out the files from these archives
FilePath txt = new FilePath(localFiles[i_1].ToString());
FileInputStream fin = new FileInputStream(txt);
DataInputStream din = new DataInputStream(fin);
string str = din.ReadLine();
@out.WriteBytes(str);
@out.WriteBytes("\n");
}
@out.Close();
}
catch (IOException ie)
{
System.Console.Out.WriteLine(StringUtils.StringifyException(ie));
}
}
/// <exception cref="System.IO.IOException"/>
private void WriteFile(FileSystem fileSys, Path name)
{
DataOutputStream stm = fileSys.Create(name);
stm.WriteBytes("oom");
stm.Close();
}
/// <exception cref="System.IO.IOException"/>
internal static void ConfigureWordCount(FileSystem fs, JobConf conf, string input
, int numMaps, int numReduces, Path inDir, Path outDir)
{
fs.Delete(outDir, true);
if (!fs.Mkdirs(inDir))
{
throw new IOException("Mkdirs failed to create " + inDir.ToString());
}
DataOutputStream file = fs.Create(new Path(inDir, "part-0"));
file.WriteBytes(input);
file.Close();
FileSystem.SetDefaultUri(conf, fs.GetUri());
conf.Set(JTConfig.FrameworkName, JTConfig.YarnFrameworkName);
conf.SetJobName("wordcount");
conf.SetInputFormat(typeof(TextInputFormat));
// the keys are words (strings)
conf.SetOutputKeyClass(typeof(Text));
// the values are counts (ints)
conf.SetOutputValueClass(typeof(IntWritable));
conf.Set("mapred.mapper.class", "testjar.ClassWordCount$MapClass");
conf.Set("mapred.combine.class", "testjar.ClassWordCount$Reduce");
conf.Set("mapred.reducer.class", "testjar.ClassWordCount$Reduce");
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
conf.SetNumMapTasks(numMaps);
conf.SetNumReduceTasks(numReduces);
//set the tests jar file
conf.SetJarByClass(typeof(TestMiniMRClasspath));
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
internal static bool RunJob(JobConf conf, Path inDir, Path outDir, int numMaps, int
numReds)
{
FileSystem fs = FileSystem.Get(conf);
if (fs.Exists(outDir))
{
fs.Delete(outDir, true);
}
if (!fs.Exists(inDir))
{
fs.Mkdirs(inDir);
}
string input = "The quick brown fox\n" + "has many silly\n" + "red fox sox\n";
for (int i = 0; i < numMaps; ++i)
{
DataOutputStream file = fs.Create(new Path(inDir, "part-" + i));
file.WriteBytes(input);
file.Close();
}
DistributedCache.AddFileToClassPath(TestMRJobs.AppJar, conf, fs);
conf.SetOutputCommitter(typeof(CustomOutputCommitter));
conf.SetInputFormat(typeof(TextInputFormat));
conf.SetOutputKeyClass(typeof(LongWritable));
conf.SetOutputValueClass(typeof(Text));
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
conf.SetNumMapTasks(numMaps);
conf.SetNumReduceTasks(numReds);
JobClient jobClient = new JobClient(conf);
RunningJob job = jobClient.SubmitJob(conf);
return(jobClient.MonitorAndPrintJob(conf, job));
}
/// <summary>Writes the Manifest to the specified OutputStream.</summary>
/// <remarks>
/// Writes the Manifest to the specified OutputStream.
/// Attributes.Name.MANIFEST_VERSION must be set in
/// MainAttributes prior to invoking this method.
/// </remarks>
/// <param name="out">the output stream</param>
/// <exception>
/// IOException
/// if an I/O error has occurred
/// </exception>
/// <seealso cref="GetMainAttributes()"/>
/// <exception cref="IOException"/>
public virtual void Write(OutputStream @out)
{
DataOutputStream dos = new DataOutputStream(@out);
// Write out the main attributes for the manifest
attr.WriteMain(dos);
// Now write out the pre-entry attributes
IEnumerator <KeyValuePair <string, Attributes> > it = entries.GetEnumerator();
while (it.MoveNext())
{
KeyValuePair <string, Attributes> e = it.Current;
StringBuilder buffer = new StringBuilder("Name: ");
string value = e.Key;
if (value != null)
{
byte[] vb = Sharpen.Runtime.GetBytesForString(value, "UTF8");
value = Encoding.UTF8.GetString(vb);
}
buffer.Append(value);
buffer.Append("\r\n");
Make72Safe(buffer);
dos.WriteBytes(buffer.ToString());
e.Value.Write(dos);
}
dos.Flush();
}
/// <summary>
/// Writes the Manifest to the specified OutputStream.
/// Attributes.Name.MANIFEST_VERSION must be set in
/// MainAttributes prior to invoking this method.
/// </summary>
/// <param name="out"> the output stream </param>
/// <exception cref="IOException"> if an I/O error has occurred </exception>
/// <seealso cref= #getMainAttributes </seealso>
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void write(java.io.OutputStream out) throws java.io.IOException
public virtual void Write(OutputStream @out)
{
DataOutputStream dos = new DataOutputStream(@out);
// Write out the main attributes for the manifest
Attr.WriteMain(dos);
// Now write out the pre-entry attributes
IEnumerator <java.util.Map_Entry <String, Attributes> > it = Entries_Renamed.GetEnumerator();
while (it.MoveNext())
{
java.util.Map_Entry <String, Attributes> e = it.Current;
StringBuffer buffer = new StringBuffer("Name: ");
String value = e.Key;
if (value != null)
{
sbyte[] vb = value.GetBytes("UTF8");
value = StringHelperClass.NewString(vb, 0, 0, vb.Length);
}
buffer.Append(value);
buffer.Append("\r\n");
Make72Safe(buffer);
dos.WriteBytes(buffer.ToString());
e.Value.Write(dos);
}
dos.Flush();
}
public virtual void TestLocalFileCorruption()
{
Configuration conf = new HdfsConfiguration();
Path file = new Path(PathUtils.GetTestDirName(GetType()), "corruptFile");
FileSystem fs = FileSystem.GetLocal(conf);
DataOutputStream dos = fs.Create(file);
dos.WriteBytes("original bytes");
dos.Close();
// Now deliberately corrupt the file
dos = new DataOutputStream(new FileOutputStream(file.ToString()));
dos.WriteBytes("corruption");
dos.Close();
// Now attempt to read the file
DataInputStream dis = fs.Open(file, 512);
try
{
System.Console.Out.WriteLine("A ChecksumException is expected to be logged.");
dis.ReadByte();
}
catch (ChecksumException)
{
}
//expect this exception but let any NPE get thrown
fs.Delete(file, true);
}
/// <exception cref="System.IO.IOException"/>
public static Job CreateJob(Configuration conf, Path inDir, Path outDir, int numInputFiles
, int numReds, string input)
{
Job job = Job.GetInstance(conf);
FileSystem fs = FileSystem.Get(conf);
if (fs.Exists(outDir))
{
fs.Delete(outDir, true);
}
if (fs.Exists(inDir))
{
fs.Delete(inDir, true);
}
fs.Mkdirs(inDir);
for (int i = 0; i < numInputFiles; ++i)
{
DataOutputStream file = fs.Create(new Path(inDir, "part-" + i));
file.WriteBytes(input);
file.Close();
}
FileInputFormat.SetInputPaths(job, inDir);
FileOutputFormat.SetOutputPath(job, outDir);
job.SetNumReduceTasks(numReds);
return(job);
}
// configure a job
/// <exception cref="System.IO.IOException"/>
private void Configure(JobConf conf, Path inDir, Path outDir, string input, Type
map, Type reduce)
{
// set up the input file system and write input text.
FileSystem inFs = inDir.GetFileSystem(conf);
FileSystem outFs = outDir.GetFileSystem(conf);
outFs.Delete(outDir, true);
if (!inFs.Mkdirs(inDir))
{
throw new IOException("Mkdirs failed to create " + inDir.ToString());
}
{
// write input into input file
DataOutputStream file = inFs.Create(new Path(inDir, "part-0"));
file.WriteBytes(input);
file.Close();
}
// configure the mapred Job which creates a tempfile in map.
conf.SetJobName("testmap");
conf.SetMapperClass(map);
conf.SetReducerClass(reduce);
conf.SetNumMapTasks(1);
conf.SetNumReduceTasks(0);
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
string TestRootDir = new Path(Runtime.GetProperty("test.build.data", "/tmp")).ToString
().Replace(' ', '+');
conf.Set("test.build.data", TestRootDir);
}
// Start a job with the specified input and return its RunningJob object
/// <exception cref="System.IO.IOException"/>
internal static RunningJob RunJob(JobConf conf, Path inDir, Path outDir, int numMaps
, int numReds, string input)
{
FileSystem fs = FileSystem.Get(conf);
if (fs.Exists(outDir))
{
fs.Delete(outDir, true);
}
if (!fs.Exists(inDir))
{
fs.Mkdirs(inDir);
}
for (int i = 0; i < numMaps; ++i)
{
DataOutputStream file = fs.Create(new Path(inDir, "part-" + i));
file.WriteBytes(input);
file.Close();
}
conf.SetInputFormat(typeof(TextInputFormat));
conf.SetOutputKeyClass(typeof(LongWritable));
conf.SetOutputValueClass(typeof(Org.Apache.Hadoop.IO.Text));
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
conf.SetNumMapTasks(numMaps);
conf.SetNumReduceTasks(numReds);
JobClient jobClient = new JobClient(conf);
RunningJob job = jobClient.SubmitJob(conf);
return(job);
}
public virtual void TestLease()
{
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
try
{
FileSystem fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue(fs.Mkdirs(dir));
Path a = new Path(dir, "a");
Path b = new Path(dir, "b");
DataOutputStream a_out = fs.Create(a);
a_out.WriteBytes("something");
NUnit.Framework.Assert.IsTrue(HasLease(cluster, a));
NUnit.Framework.Assert.IsTrue(!HasLease(cluster, b));
DataOutputStream b_out = fs.Create(b);
b_out.WriteBytes("something");
NUnit.Framework.Assert.IsTrue(HasLease(cluster, a));
NUnit.Framework.Assert.IsTrue(HasLease(cluster, b));
a_out.Close();
b_out.Close();
NUnit.Framework.Assert.IsTrue(!HasLease(cluster, a));
NUnit.Framework.Assert.IsTrue(!HasLease(cluster, b));
fs.Delete(dir, true);
}
finally
{
if (cluster != null)
{
cluster.Shutdown();
}
}
}
public virtual void TestStatusLimit()
{
Path test = new Path(testRootTempDir, "testStatusLimit");
Configuration conf = new Configuration();
Path inDir = new Path(test, "in");
Path outDir = new Path(test, "out");
FileSystem fs = FileSystem.Get(conf);
if (fs.Exists(inDir))
{
fs.Delete(inDir, true);
}
fs.Mkdirs(inDir);
DataOutputStream file = fs.Create(new Path(inDir, "part-" + 0));
file.WriteBytes("testStatusLimit");
file.Close();
if (fs.Exists(outDir))
{
fs.Delete(outDir, true);
}
Job job = Job.GetInstance(conf, "testStatusLimit");
job.SetMapperClass(typeof(TestReporter.StatusLimitMapper));
job.SetNumReduceTasks(0);
FileInputFormat.AddInputPath(job, inDir);
FileOutputFormat.SetOutputPath(job, outDir);
job.WaitForCompletion(true);
NUnit.Framework.Assert.IsTrue("Job failed", job.IsSuccessful());
}
/// <exception cref="System.IO.IOException"/>
internal static void CreateFile(FileSystem fs, Path f)
{
DataOutputStream a_out = fs.Create(f);
a_out.WriteBytes("something");
a_out.Close();
}
/*
* Writes the current attributes to the specified data output stream,
* make sure to write out the MANIFEST_VERSION or SIGNATURE_VERSION
* attributes first.
*
* XXX Need to handle UTF8 values and break up lines longer than 72 bytes
*/
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: void writeMain(java.io.DataOutputStream out) throws java.io.IOException
internal virtual void WriteMain(DataOutputStream @out)
{
// write out the *-Version header first, if it exists
String vername = Name.MANIFEST_VERSION.ToString();
String version = GetValue(vername);
if (version == java.util.Map_Fields.Null)
{
vername = Name.SIGNATURE_VERSION.ToString();
version = GetValue(vername);
}
if (version != java.util.Map_Fields.Null)
{
@out.WriteBytes(vername + ": " + version + "\r\n");
}
// write out all attributes except for the version
// we wrote out earlier
IEnumerator <java.util.Map_Entry <Object, Object> > it = EntrySet().Iterator();
while (it.MoveNext())
{
java.util.Map_Entry <Object, Object> e = it.Current;
String name = ((Name)e.Key).ToString();
if ((version != java.util.Map_Fields.Null) && !(name.EqualsIgnoreCase(vername)))
{
StringBuffer buffer = new StringBuffer(name);
buffer.Append(": ");
String value = (String)e.Value;
if (value != java.util.Map_Fields.Null)
{
sbyte[] vb = value.GetBytes("UTF8");
value = StringHelperClass.NewString(vb, 0, 0, vb.Length);
}
buffer.Append(value);
buffer.Append("\r\n");
Manifest.Make72Safe(buffer);
@out.WriteBytes(buffer.ToString());
}
}
@out.WriteBytes("\r\n");
}
/// <summary>This creates a file in the dfs</summary>
/// <param name="dfs">FileSystem Local File System where file needs to be picked</param>
/// <param name="Uripath">Path dfs path where file needs to be copied</param>
/// <param name="permission">FsPermission File permission</param>
/// <returns>returns the DataOutputStream</returns>
/// <exception cref="System.Exception"/>
public static DataOutputStream CreateTmpFileDFS(FileSystem dfs, Path Uripath, FsPermission
permission, string input)
{
//Creating the path with the file
DataOutputStream file = FileSystem.Create(dfs, Uripath, permission);
file.WriteBytes(input);
file.Close();
return(file);
}
public override void Configure(JobConf jconf)
{
conf = jconf;
try
{
// read the cached files (unzipped, unjarred and text)
// and put it into a single file TEST_ROOT_DIR/test.txt
string TestRootDir = jconf.Get("test.build.data", "/tmp");
Path file = new Path("file:///", TestRootDir);
FileSystem fs = FileSystem.GetLocal(conf);
if (!fs.Mkdirs(file))
{
throw new IOException("Mkdirs failed to create " + file.ToString());
}
Path fileOut = new Path(file, "test.txt");
fs.Delete(fileOut, true);
DataOutputStream @out = fs.Create(fileOut);
string[] symlinks = new string[6];
symlinks[0] = ".";
symlinks[1] = "testjar";
symlinks[2] = "testzip";
symlinks[3] = "testtgz";
symlinks[4] = "testtargz";
symlinks[5] = "testtar";
for (int i = 0; i < symlinks.Length; i++)
{
// read out the files from these archives
FilePath f = new FilePath(symlinks[i]);
FilePath txt = new FilePath(f, "test.txt");
FileInputStream fin = new FileInputStream(txt);
BufferedReader reader = new BufferedReader(new InputStreamReader(fin));
string str = reader.ReadLine();
reader.Close();
@out.WriteBytes(str);
@out.WriteBytes("\n");
}
@out.Close();
}
catch (IOException ie)
{
System.Console.Out.WriteLine(StringUtils.StringifyException(ie));
}
}
/// <exception cref="System.IO.IOException"/>
internal static string LaunchExternal(URI uri, JobConf conf, string input, int numMaps
, int numReduces)
{
Path inDir = new Path("/testing/ext/input");
Path outDir = new Path("/testing/ext/output");
FileSystem fs = FileSystem.Get(uri, conf);
fs.Delete(outDir, true);
if (!fs.Mkdirs(inDir))
{
throw new IOException("Mkdirs failed to create " + inDir.ToString());
}
{
DataOutputStream file = fs.Create(new Path(inDir, "part-0"));
file.WriteBytes(input);
file.Close();
}
FileSystem.SetDefaultUri(conf, uri);
conf.Set(JTConfig.FrameworkName, JTConfig.YarnFrameworkName);
conf.SetJobName("wordcount");
conf.SetInputFormat(typeof(TextInputFormat));
// the keys are counts
conf.SetOutputValueClass(typeof(IntWritable));
// the values are the messages
conf.Set(JobContext.OutputKeyClass, "testjar.ExternalWritable");
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
conf.SetNumMapTasks(numMaps);
conf.SetNumReduceTasks(numReduces);
conf.Set("mapred.mapper.class", "testjar.ExternalMapperReducer");
conf.Set("mapred.reducer.class", "testjar.ExternalMapperReducer");
// set the tests jar file
conf.SetJarByClass(typeof(TestMiniMRClasspath));
JobClient.RunJob(conf);
StringBuilder result = new StringBuilder();
Path[] fileList = FileUtil.Stat2Paths(fs.ListStatus(outDir, new Utils.OutputFileUtils.OutputFilesFilter
()));
for (int i = 0; i < fileList.Length; ++i)
{
BufferedReader file = new BufferedReader(new InputStreamReader(fs.Open(fileList[i
])));
string line = file.ReadLine();
while (line != null)
{
result.Append(line);
line = file.ReadLine();
result.Append("\n");
}
file.Close();
}
return(result.ToString());
}
// In the given dir, creates part-0 file with 3 records of same size
/// <exception cref="System.IO.IOException"/>
private void CreateInputFile(Path rootDir)
{
if (fs.Exists(rootDir))
{
fs.Delete(rootDir, true);
}
string str = "The quick brown fox\n" + "The brown quick fox\n" + "The fox brown quick\n";
DataOutputStream inpFile = fs.Create(new Path(rootDir, "part-0"));
inpFile.WriteBytes(str);
inpFile.Close();
}
/// <exception cref="System.IO.IOException"/>
public static bool LaunchJob(URI fileSys, JobConf conf, int numMaps, int numReduces
)
{
Path inDir = new Path("/testing/input");
Path outDir = new Path("/testing/output");
FileSystem fs = FileSystem.Get(fileSys, conf);
fs.Delete(outDir, true);
if (!fs.Mkdirs(inDir))
{
Log.Warn("Can't create " + inDir);
return(false);
}
// generate an input file
DataOutputStream file = fs.Create(new Path(inDir, "part-0"));
file.WriteBytes("foo foo2 foo3");
file.Close();
// use WordCount example
FileSystem.SetDefaultUri(conf, fileSys);
conf.SetJobName("foo");
conf.SetInputFormat(typeof(TextInputFormat));
conf.SetOutputFormat(typeof(TestSpecialCharactersInOutputPath.SpecialTextOutputFormat
));
conf.SetOutputKeyClass(typeof(LongWritable));
conf.SetOutputValueClass(typeof(Text));
conf.SetMapperClass(typeof(IdentityMapper));
conf.SetReducerClass(typeof(IdentityReducer));
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
conf.SetNumMapTasks(numMaps);
conf.SetNumReduceTasks(numReduces);
// run job and wait for completion
RunningJob runningJob = JobClient.RunJob(conf);
try
{
NUnit.Framework.Assert.IsTrue(runningJob.IsComplete());
NUnit.Framework.Assert.IsTrue(runningJob.IsSuccessful());
NUnit.Framework.Assert.IsTrue("Output folder not found!", fs.Exists(new Path("/testing/output/"
+ OutputFilename)));
}
catch (ArgumentNullException)
{
// This NPE should no more happens
Fail("A NPE should not have happened.");
}
// return job result
Log.Info("job is complete: " + runningJob.IsSuccessful());
return(runningJob.IsSuccessful());
}
/// <exception cref="System.IO.IOException"/>
private static void WriteToIndexFile(string logLocation, bool isCleanup)
{
lock (typeof(TaskLog))
{
// To ensure atomicity of updates to index file, write to temporary index
// file first and then rename.
FilePath tmpIndexFile = GetTmpIndexFile(currentTaskid, isCleanup);
BufferedOutputStream bos = null;
DataOutputStream dos = null;
try
{
bos = new BufferedOutputStream(SecureIOUtils.CreateForWrite(tmpIndexFile, 0x1a4));
dos = new DataOutputStream(bos);
//the format of the index file is
//LOG_DIR: <the dir where the task logs are really stored>
//STDOUT: <start-offset in the stdout file> <length>
//STDERR: <start-offset in the stderr file> <length>
//SYSLOG: <start-offset in the syslog file> <length>
dos.WriteBytes(TaskLog.LogFileDetail.Location + logLocation + "\n" + TaskLog.LogName
.Stdout.ToString() + ":");
dos.WriteBytes(System.Convert.ToString(prevOutLength) + " ");
dos.WriteBytes(System.Convert.ToString(new FilePath(logLocation, TaskLog.LogName.
Stdout.ToString()).Length() - prevOutLength) + "\n" + TaskLog.LogName.Stderr + ":"
);
dos.WriteBytes(System.Convert.ToString(prevErrLength) + " ");
dos.WriteBytes(System.Convert.ToString(new FilePath(logLocation, TaskLog.LogName.
Stderr.ToString()).Length() - prevErrLength) + "\n" + TaskLog.LogName.Syslog.ToString
() + ":");
dos.WriteBytes(System.Convert.ToString(prevLogLength) + " ");
dos.WriteBytes(System.Convert.ToString(new FilePath(logLocation, TaskLog.LogName.
Syslog.ToString()).Length() - prevLogLength) + "\n");
dos.Close();
dos = null;
bos.Close();
bos = null;
}
finally
{
IOUtils.Cleanup(Log, dos, bos);
}
FilePath indexFile = GetIndexFile(currentTaskid, isCleanup);
Path indexFilePath = new Path(indexFile.GetAbsolutePath());
Path tmpIndexFilePath = new Path(tmpIndexFile.GetAbsolutePath());
if (localFS == null)
{
// set localFS once
localFS = FileSystem.GetLocal(new Configuration());
}
localFS.Rename(tmpIndexFilePath, indexFilePath);
}
}
/// <exception cref="System.IO.IOException"/>
internal static Path WriteFile(FileSystem fs, Path f)
{
DataOutputStream @out = fs.Create(f);
try
{
@out.WriteBytes("test");
}
finally
{
@out.Close();
}
NUnit.Framework.Assert.IsTrue(fs.Exists(f));
return(f);
}
/*
* Writes the current attributes to the specified data output stream.
* XXX Need to handle UTF8 values and break up lines longer than 72 bytes
*/
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: void write(java.io.DataOutputStream os) throws java.io.IOException
internal virtual void Write(DataOutputStream os)
{
IEnumerator <java.util.Map_Entry <Object, Object> > it = EntrySet().Iterator();
while (it.MoveNext())
{
java.util.Map_Entry <Object, Object> e = it.Current;
StringBuffer buffer = new StringBuffer(((Name)e.Key).ToString());
buffer.Append(": ");
String value = (String)e.Value;
if (value != java.util.Map_Fields.Null)
{
sbyte[] vb = value.GetBytes("UTF8");
value = StringHelperClass.NewString(vb, 0, 0, vb.Length);
}
buffer.Append(value);
buffer.Append("\r\n");
Manifest.Make72Safe(buffer);
os.WriteBytes(buffer.ToString());
}
os.WriteBytes("\r\n");
}
/// <exception cref="System.IO.IOException"/>
internal static void WriteInputFile(FileSystem fs, Path dir)
{
DataOutputStream @out = fs.Create(new Path(dir, "part0"));
@out.WriteBytes("Alice was beginning to get very tired of sitting by her\n");
@out.WriteBytes("sister on the bank, and of having nothing to do: once\n");
@out.WriteBytes("or twice she had peeped into the book her sister was\n");
@out.WriteBytes("reading, but it had no pictures or conversations in\n");
@out.WriteBytes("it, `and what is the use of a book,' thought Alice\n");
@out.WriteBytes("`without pictures or conversation?'\n");
@out.Close();
}
/// <exception cref="System.IO.IOException"/>
public static TestJobSysDirWithDFS.TestResult LaunchWordCount(JobConf conf, Path
inDir, Path outDir, string input, int numMaps, int numReduces, string sysDir)
{
FileSystem inFs = inDir.GetFileSystem(conf);
FileSystem outFs = outDir.GetFileSystem(conf);
outFs.Delete(outDir, true);
if (!inFs.Mkdirs(inDir))
{
throw new IOException("Mkdirs failed to create " + inDir.ToString());
}
{
DataOutputStream file = inFs.Create(new Path(inDir, "part-0"));
file.WriteBytes(input);
file.Close();
}
conf.SetJobName("wordcount");
conf.SetInputFormat(typeof(TextInputFormat));
// the keys are words (strings)
conf.SetOutputKeyClass(typeof(Text));
// the values are counts (ints)
conf.SetOutputValueClass(typeof(IntWritable));
conf.SetMapperClass(typeof(WordCount.MapClass));
conf.SetCombinerClass(typeof(WordCount.Reduce));
conf.SetReducerClass(typeof(WordCount.Reduce));
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
conf.SetNumMapTasks(numMaps);
conf.SetNumReduceTasks(numReduces);
conf.Set(JTConfig.JtSystemDir, "/tmp/subru/mapred/system");
JobClient jobClient = new JobClient(conf);
RunningJob job = JobClient.RunJob(conf);
// Checking that the Job Client system dir is not used
NUnit.Framework.Assert.IsFalse(FileSystem.Get(conf).Exists(new Path(conf.Get(JTConfig
.JtSystemDir))));
// Check if the Job Tracker system dir is propogated to client
NUnit.Framework.Assert.IsFalse(sysDir.Contains("/tmp/subru/mapred/system"));
NUnit.Framework.Assert.IsTrue(sysDir.Contains("custom"));
return(new TestJobSysDirWithDFS.TestResult(job, MapReduceTestUtil.ReadOutput(outDir
, conf)));
}
/// <exception cref="System.IO.IOException"/>
private string LaunchWordCount(JobConf conf, string input, int numMaps, int numReduces
)
{
Path inDir = new Path("testing/wc/input");
Path outDir = new Path("testing/wc/output");
// Hack for local FS that does not have the concept of a 'mounting point'
if (IsLocalFS())
{
string localPathRoot = Runtime.GetProperty("test.build.data", "/tmp").ToString().
Replace(' ', '+');
inDir = new Path(localPathRoot, inDir);
outDir = new Path(localPathRoot, outDir);
}
FileSystem fs = FileSystem.Get(conf);
fs.Delete(outDir, true);
if (!fs.Mkdirs(inDir))
{
throw new IOException("Mkdirs failed to create " + inDir.ToString());
}
{
DataOutputStream file = fs.Create(new Path(inDir, "part-0"));
file.WriteBytes(input);
file.Close();
}
conf.SetJobName("wordcount");
conf.SetInputFormat(typeof(TextInputFormat));
// the keys are words (strings)
conf.SetOutputKeyClass(typeof(Text));
// the values are counts (ints)
conf.SetOutputValueClass(typeof(IntWritable));
conf.SetMapperClass(typeof(WordCount.MapClass));
conf.SetCombinerClass(typeof(WordCount.Reduce));
conf.SetReducerClass(typeof(WordCount.Reduce));
FileInputFormat.SetInputPaths(conf, inDir);
FileOutputFormat.SetOutputPath(conf, outDir);
conf.SetNumMapTasks(numMaps);
conf.SetNumReduceTasks(numReduces);
JobClient.RunJob(conf);
return(MapReduceTestUtil.ReadOutput(outDir, conf));
}
/// <exception cref="System.Exception"/>
internal static void CreateInputOutPutFolder(Path inDir, Path outDir, int numMaps
)
{
FileSystem fs = FileSystem.Get(conf);
if (fs.Exists(outDir))
{
fs.Delete(outDir, true);
}
if (!fs.Exists(inDir))
{
fs.Mkdirs(inDir);
}
string input = "The quick brown fox\n" + "has many silly\n" + "red fox sox\n";
for (int i = 0; i < numMaps; ++i)
{
DataOutputStream file = fs.Create(new Path(inDir, "part-" + i));
file.WriteBytes(input);
file.Close();
}
}
public static void SetUp()
{
JobConf conf = new JobConf();
fileSys = FileSystem.Get(conf);
fileSys.Delete(new Path(TestRootDir), true);
conf.Set("mapred.job.tracker.handler.count", "1");
conf.Set("mapred.job.tracker", "127.0.0.1:0");
conf.Set("mapred.job.tracker.http.address", "127.0.0.1:0");
conf.Set("mapred.task.tracker.http.address", "127.0.0.1:0");
conf.Set(JHAdminConfig.MrHistoryIntermediateDoneDir, TestRootDir + "/intermediate"
);
conf.Set(FileOutputCommitter.SuccessfulJobOutputDirMarker, "true");
mr = new MiniMRCluster(1, "file:///", 1, null, null, conf);
inDir = new Path(TestRootDir, "test-input");
string input = "The quick brown fox\n" + "has many silly\n" + "red fox sox\n";
DataOutputStream file = fileSys.Create(new Path(inDir, "part-" + 0));
file.WriteBytes(input);
file.Close();
emptyInDir = new Path(TestRootDir, "empty-input");
fileSys.Mkdirs(emptyInDir);
}