/// <summary>Generate a text file on the given filesystem with the given path name.</summary>
/// <remarks>
/// Generate a text file on the given filesystem with the given path name.
/// The text file will contain the given number of lines of generated data.
/// The generated data are string representations of numbers. Each line
/// is the same length, which is achieved by padding each number with
/// an appropriate number of leading '0' (zero) characters. The order of
/// generated data is one of ascending, descending, or random.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void GenerateTextFile(FileSystem fs, Path inputFile, long numLines
, MRBench.Order sortOrder)
{
Log.Info("creating control file: " + numLines + " numLines, " + sortOrder + " sortOrder"
);
TextWriter output = null;
try
{
output = new TextWriter(fs.Create(inputFile));
int padding = numLines.ToString().Length;
switch (sortOrder)
{
case MRBench.Order.Random:
{
for (long l = 0; l < numLines; l++)
{
output.WriteLine(Pad((new Random()).NextLong(), padding));
}
break;
}
case MRBench.Order.Ascending:
{
for (long l_1 = 0; l_1 < numLines; l_1++)
{
output.WriteLine(Pad(l_1, padding));
}
break;
}
case MRBench.Order.Descending:
{
for (long l_2 = numLines; l_2 > 0; l_2--)
{
output.WriteLine(Pad(l_2, padding));
}
break;
}
}
}
finally
{
if (output != null)
{
output.Close();
}
}
Log.Info("created control file: " + inputFile);
}
/// <exception cref="System.Exception"/>
public virtual int Run(string[] args)
{
string version = "MRBenchmark.0.0.2";
System.Console.Out.WriteLine(version);
string usage = "Usage: mrbench " + "[-baseDir <base DFS path for output/input, default is /benchmarks/MRBench>] "
+ "[-jar <local path to job jar file containing Mapper and Reducer implementations, default is current jar file>] "
+ "[-numRuns <number of times to run the job, default is 1>] " + "[-maps <number of maps for each run, default is 2>] "
+ "[-reduces <number of reduces for each run, default is 1>] " + "[-inputLines <number of input lines to generate, default is 1>] "
+ "[-inputType <type of input to generate, one of ascending (default), descending, random>] "
+ "[-verbose]";
string jarFile = null;
int inputLines = 1;
int numRuns = 1;
int numMaps = 2;
int numReduces = 1;
bool verbose = false;
MRBench.Order inputSortOrder = MRBench.Order.Ascending;
for (int i = 0; i < args.Length; i++)
{
// parse command line
if (args[i].Equals("-jar"))
{
jarFile = args[++i];
}
else
{
if (args[i].Equals("-numRuns"))
{
numRuns = System.Convert.ToInt32(args[++i]);
}
else
{
if (args[i].Equals("-baseDir"))
{
BaseDir = new Path(args[++i]);
}
else
{
if (args[i].Equals("-maps"))
{
numMaps = System.Convert.ToInt32(args[++i]);
}
else
{
if (args[i].Equals("-reduces"))
{
numReduces = System.Convert.ToInt32(args[++i]);
}
else
{
if (args[i].Equals("-inputLines"))
{
inputLines = System.Convert.ToInt32(args[++i]);
}
else
{
if (args[i].Equals("-inputType"))
{
string s = args[++i];
if (Sharpen.Runtime.EqualsIgnoreCase(s, "ascending"))
{
inputSortOrder = MRBench.Order.Ascending;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(s, "descending"))
{
inputSortOrder = MRBench.Order.Descending;
}
else
{
if (Sharpen.Runtime.EqualsIgnoreCase(s, "random"))
{
inputSortOrder = MRBench.Order.Random;
}
else
{
inputSortOrder = null;
}
}
}
}
else
{
if (args[i].Equals("-verbose"))
{
verbose = true;
}
else
{
System.Console.Error.WriteLine(usage);
System.Environment.Exit(-1);
}
}
}
}
}
}
}
}
}
if (numRuns < 1 || numMaps < 1 || numReduces < 1 || inputLines < 0 || inputSortOrder
== null)
{
// verify args
System.Console.Error.WriteLine(usage);
return -1;
}
JobConf jobConf = SetupJob(numMaps, numReduces, jarFile);
FileSystem fs = FileSystem.Get(jobConf);
Path inputFile = new Path(InputDir, "input_" + (new Random()).Next() + ".txt");
GenerateTextFile(fs, inputFile, inputLines, inputSortOrder);
// setup test output directory
fs.Mkdirs(BaseDir);
AList<long> execTimes = new AList<long>();
try
{
execTimes = RunJobInSequence(jobConf, numRuns);
}
finally
{
// delete output -- should we really do this?
fs.Delete(BaseDir, true);
}
if (verbose)
{
// Print out a report
System.Console.Out.WriteLine("Total MapReduce jobs executed: " + numRuns);
System.Console.Out.WriteLine("Total lines of data per job: " + inputLines);
System.Console.Out.WriteLine("Maps per job: " + numMaps);
System.Console.Out.WriteLine("Reduces per job: " + numReduces);
}
int i_1 = 0;
long totalTime = 0;
foreach (long time in execTimes)
{
totalTime += time;
if (verbose)
{
System.Console.Out.WriteLine("Total milliseconds for task: " + (++i_1) + " = " +
time);
}
}
long avgTime = totalTime / numRuns;
System.Console.Out.WriteLine("DataLines\tMaps\tReduces\tAvgTime (milliseconds)");
System.Console.Out.WriteLine(inputLines + "\t\t" + numMaps + "\t" + numReduces +
"\t" + avgTime);
return 0;
}
