public virtual void TestArguments()
{
ConfigExtractor extractor = GetTestConfig(true);
NUnit.Framework.Assert.AreEqual(extractor.GetOpCount(), Constants.OperationType.Values
().Length);
NUnit.Framework.Assert.AreEqual(extractor.GetMapAmount(), 2);
NUnit.Framework.Assert.AreEqual(extractor.GetReducerAmount(), 2);
Range <long> apRange = extractor.GetAppendSize();
NUnit.Framework.Assert.AreEqual(apRange.GetLower(), Constants.Megabytes * 1);
NUnit.Framework.Assert.AreEqual(apRange.GetUpper(), Constants.Megabytes * 2);
Range <long> wRange = extractor.GetWriteSize();
NUnit.Framework.Assert.AreEqual(wRange.GetLower(), Constants.Megabytes * 1);
NUnit.Framework.Assert.AreEqual(wRange.GetUpper(), Constants.Megabytes * 2);
Range <long> trRange = extractor.GetTruncateSize();
NUnit.Framework.Assert.AreEqual(trRange.GetLower(), 0);
NUnit.Framework.Assert.AreEqual(trRange.GetUpper(), Constants.Megabytes * 1);
Range <long> bRange = extractor.GetBlockSize();
NUnit.Framework.Assert.AreEqual(bRange.GetLower(), Constants.Megabytes * 1);
NUnit.Framework.Assert.AreEqual(bRange.GetUpper(), Constants.Megabytes * 2);
string resfile = extractor.GetResultFile();
NUnit.Framework.Assert.AreEqual(resfile, GetResultFile().ToString());
int durationMs = extractor.GetDurationMilliseconds();
NUnit.Framework.Assert.AreEqual(durationMs, 10 * 1000);
}
public virtual void TestMRFlow()
{
ConfigExtractor extractor = GetTestConfig(false);
SliveTest s = new SliveTest(GetBaseConfig());
int ec = ToolRunner.Run(s, GetTestArgs(false));
NUnit.Framework.Assert.IsTrue(ec == 0);
string resFile = extractor.GetResultFile();
FilePath fn = new FilePath(resFile);
NUnit.Framework.Assert.IsTrue(fn.Exists());
}
/// <summary>
/// Handles merging all options and verifying from the given command line
/// output and the given base configuration and returns the merged
/// configuration
/// </summary>
/// <param name="opts">the parsed command line option output</param>
/// <param name="base">the base configuration to merge with</param>
/// <returns>the merged configuration</returns>
/// <exception cref="ConfigException"/>
/// <exception cref="Org.Apache.Hadoop.FS.Slive.ConfigMerger.ConfigException"/>
private Configuration HandleOptions(ArgumentParser.ParsedOutput opts, Configuration
@base)
{
// ensure variables are overwritten and verified
ConfigExtractor extractor = new ConfigExtractor(@base);
{
// overwrite the map amount and check to ensure > 0
int mapAmount = null;
try
{
mapAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Maps.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging map amount", e
);
}
if (mapAmount != null)
{
if (mapAmount <= 0)
{
throw new ConfigMerger.ConfigException("Map amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Maps.GetCfgOption(), mapAmount.ToString());
}
}
{
// overwrite the reducer amount and check to ensure > 0
int reduceAmount = null;
try
{
reduceAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Reduces.GetOpt()
));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging reducer amount"
, e);
}
if (reduceAmount != null)
{
if (reduceAmount <= 0)
{
throw new ConfigMerger.ConfigException("Reducer amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Reduces.GetCfgOption(), reduceAmount.ToString());
}
}
{
// overwrite the duration amount and ensure > 0
int duration = null;
try
{
duration = extractor.GetDuration(opts.GetValue(ConfigOption.Duration.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging duration", e);
}
if (duration != null)
{
if (duration <= 0)
{
throw new ConfigMerger.ConfigException("Duration can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Duration.GetCfgOption(), duration.ToString());
}
}
{
// overwrite the operation amount and ensure > 0
int operationAmount = null;
try
{
operationAmount = extractor.GetOpCount(opts.GetValue(ConfigOption.Ops.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging operation amount"
, e);
}
if (operationAmount != null)
{
if (operationAmount <= 0)
{
throw new ConfigMerger.ConfigException("Operation amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Ops.GetCfgOption(), operationAmount.ToString());
}
}
{
// overwrite the exit on error setting
try
{
bool exitOnError = extractor.ShouldExitOnFirstError(opts.GetValue(ConfigOption.ExitOnError
.GetOpt()));
@base.SetBoolean(ConfigOption.ExitOnError.GetCfgOption(), exitOnError);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging exit on error value"
, e);
}
}
{
// overwrite the truncate wait setting
try
{
bool waitOnTruncate = extractor.ShouldWaitOnTruncate(opts.GetValue(ConfigOption.TruncateWait
.GetOpt()));
@base.SetBoolean(ConfigOption.TruncateWait.GetCfgOption(), waitOnTruncate);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging wait on truncate value"
, e);
}
}
{
// verify and set file limit and ensure > 0
int fileAm = null;
try
{
fileAm = extractor.GetTotalFiles(opts.GetValue(ConfigOption.Files.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging total file limit amount"
, e);
}
if (fileAm != null)
{
if (fileAm <= 0)
{
throw new ConfigMerger.ConfigException("File amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Files.GetCfgOption(), fileAm.ToString());
}
}
{
// set the grid queue to run on
try
{
string qname = extractor.GetQueueName(opts.GetValue(ConfigOption.QueueName.GetOpt
()));
if (qname != null)
{
@base.Set(ConfigOption.QueueName.GetCfgOption(), qname);
}
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging queue name", e
);
}
}
{
// verify and set the directory limit and ensure > 0
int directoryLimit = null;
try
{
directoryLimit = extractor.GetDirSize(opts.GetValue(ConfigOption.DirSize.GetOpt()
));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging directory file limit"
, e);
}
if (directoryLimit != null)
{
if (directoryLimit <= 0)
{
throw new ConfigMerger.ConfigException("Directory file limit can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.DirSize.GetCfgOption(), directoryLimit.ToString());
}
}
{
// set the base directory
Path basedir = null;
try
{
basedir = extractor.GetBaseDirectory(opts.GetValue(ConfigOption.BaseDir.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging base directory"
, e);
}
if (basedir != null)
{
// always ensure in slive dir
basedir = new Path(basedir, Constants.BaseDir);
@base.Set(ConfigOption.BaseDir.GetCfgOption(), basedir.ToString());
}
}
{
// set the result file
string fn = null;
try
{
fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging result file",
e);
}
if (fn != null)
{
@base.Set(ConfigOption.ResultFile.GetCfgOption(), fn);
}
}
{
string fn = null;
try
{
fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging result file",
e);
}
if (fn != null)
{
@base.Set(ConfigOption.ResultFile.GetCfgOption(), fn);
}
}
{
// set the operations
try
{
@base = HandleOperations(opts, @base, extractor);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging operations", e
);
}
}
{
// set the replication amount range
Range <short> replicationAm = null;
try
{
replicationAm = extractor.GetReplication(opts.GetValue(ConfigOption.ReplicationAm
.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging replication amount range"
, e);
}
if (replicationAm != null)
{
int minRepl = @base.GetInt(Constants.MinReplication, 1);
if (replicationAm.GetLower() < minRepl)
{
throw new ConfigMerger.ConfigException("Replication amount minimum is less than property configured minimum "
+ minRepl);
}
if (replicationAm.GetLower() > replicationAm.GetUpper())
{
throw new ConfigMerger.ConfigException("Replication amount minimum is greater than its maximum"
);
}
if (replicationAm.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Replication amount minimum must be greater than zero"
);
}
@base.Set(ConfigOption.ReplicationAm.GetCfgOption(), replicationAm.ToString());
}
}
{
// set the sleep range
Range <long> sleepRange = null;
try
{
sleepRange = extractor.GetSleepRange(opts.GetValue(ConfigOption.SleepTime.GetOpt(
)));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging sleep size range"
, e);
}
if (sleepRange != null)
{
if (sleepRange.GetLower() > sleepRange.GetUpper())
{
throw new ConfigMerger.ConfigException("Sleep range minimum is greater than its maximum"
);
}
if (sleepRange.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Sleep range minimum must be greater than zero"
);
}
@base.Set(ConfigOption.SleepTime.GetCfgOption(), sleepRange.ToString());
}
}
{
// set the packet size if given
string pSize = opts.GetValue(ConfigOption.PacketSize.GetOpt());
if (pSize == null)
{
pSize = ConfigOption.PacketSize.GetDefault();
}
if (pSize != null)
{
try
{
long packetSize = StringUtils.TraditionalBinaryPrefix.String2long(pSize);
@base.Set(ConfigOption.PacketSize.GetCfgOption(), packetSize.ToString());
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging write packet size"
, e);
}
}
}
{
// set the block size range
Range <long> blockSize = null;
try
{
blockSize = extractor.GetBlockSize(opts.GetValue(ConfigOption.BlockSize.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging block size range"
, e);
}
if (blockSize != null)
{
if (blockSize.GetLower() > blockSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Block size minimum is greater than its maximum"
);
}
if (blockSize.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Block size minimum must be greater than zero"
);
}
// ensure block size is a multiple of BYTES_PER_CHECKSUM
// if a value is set in the configuration
long bytesPerChecksum = extractor.GetByteCheckSum();
if (bytesPerChecksum != null)
{
if ((blockSize.GetLower() % bytesPerChecksum) != 0)
{
throw new ConfigMerger.ConfigException("Blocksize lower bound must be a multiple of "
+ bytesPerChecksum);
}
if ((blockSize.GetUpper() % bytesPerChecksum) != 0)
{
throw new ConfigMerger.ConfigException("Blocksize upper bound must be a multiple of "
+ bytesPerChecksum);
}
}
@base.Set(ConfigOption.BlockSize.GetCfgOption(), blockSize.ToString());
}
}
{
// set the read size range
Range <long> readSize = null;
try
{
readSize = extractor.GetReadSize(opts.GetValue(ConfigOption.ReadSize.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging read size range"
, e);
}
if (readSize != null)
{
if (readSize.GetLower() > readSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Read size minimum is greater than its maximum"
);
}
if (readSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Read size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.ReadSize.GetCfgOption(), readSize.ToString());
}
}
{
// set the write size range
Range <long> writeSize = null;
try
{
writeSize = extractor.GetWriteSize(opts.GetValue(ConfigOption.WriteSize.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging write size range"
, e);
}
if (writeSize != null)
{
if (writeSize.GetLower() > writeSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Write size minimum is greater than its maximum"
);
}
if (writeSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Write size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.WriteSize.GetCfgOption(), writeSize.ToString());
}
}
{
// set the append size range
Range <long> appendSize = null;
try
{
appendSize = extractor.GetAppendSize(opts.GetValue(ConfigOption.AppendSize.GetOpt
()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging append size range"
, e);
}
if (appendSize != null)
{
if (appendSize.GetLower() > appendSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Append size minimum is greater than its maximum"
);
}
if (appendSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Append size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.AppendSize.GetCfgOption(), appendSize.ToString());
}
}
{
// set the truncate size range
Range <long> truncateSize = null;
try
{
truncateSize = extractor.GetTruncateSize(opts.GetValue(ConfigOption.TruncateSize.
GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging truncate size range"
, e);
}
if (truncateSize != null)
{
if (truncateSize.GetLower() > truncateSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Truncate size minimum is greater than its maximum"
);
}
if (truncateSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Truncate size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.TruncateSize.GetCfgOption(), truncateSize.ToString());
}
}
{
// set the seed
long seed = null;
try
{
seed = extractor.GetRandomSeed(opts.GetValue(ConfigOption.RandomSeed.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging random number seed"
, e);
}
if (seed != null)
{
@base.Set(ConfigOption.RandomSeed.GetCfgOption(), seed.ToString());
}
}
return(@base);
}
/// <summary>
/// Attempts to write the report to the given output using the specified
/// config.
/// </summary>
/// <remarks>
/// Attempts to write the report to the given output using the specified
/// config. It will open up the expected reducer output file and read in its
/// contents and then split up by operation output and sort by operation type
/// and then for each operation type it will generate a report to the specified
/// result file and the console.
/// </remarks>
/// <param name="cfg">the config specifying the files and output</param>
/// <exception cref="System.Exception">if files can not be opened/closed/read or invalid format
/// </exception>
private void WriteReport(ConfigExtractor cfg)
{
Path dn = cfg.GetOutputPath();
Log.Info("Writing report using contents of " + dn);
FileSystem fs = dn.GetFileSystem(cfg.GetConfig());
FileStatus[] reduceFiles = fs.ListStatus(dn);
BufferedReader fileReader = null;
PrintWriter reportWriter = null;
try
{
IList <OperationOutput> noOperations = new AList <OperationOutput>();
IDictionary <string, IList <OperationOutput> > splitTypes = new SortedDictionary <string
, IList <OperationOutput> >();
foreach (FileStatus fn in reduceFiles)
{
if (!fn.GetPath().GetName().StartsWith("part"))
{
continue;
}
fileReader = new BufferedReader(new InputStreamReader(new DataInputStream(fs.Open
(fn.GetPath()))));
string line;
while ((line = fileReader.ReadLine()) != null)
{
string[] pieces = line.Split("\t", 2);
if (pieces.Length == 2)
{
OperationOutput data = new OperationOutput(pieces[0], pieces[1]);
string op = (data.GetOperationType());
if (op != null)
{
IList <OperationOutput> opList = splitTypes[op];
if (opList == null)
{
opList = new AList <OperationOutput>();
}
opList.AddItem(data);
splitTypes[op] = opList;
}
else
{
noOperations.AddItem(data);
}
}
else
{
throw new IOException("Unparseable line " + line);
}
}
fileReader.Close();
fileReader = null;
}
FilePath resFile = null;
if (cfg.GetResultFile() != null)
{
resFile = new FilePath(cfg.GetResultFile());
}
if (resFile != null)
{
Log.Info("Report results being placed to logging output and to file " + resFile.GetCanonicalPath
());
reportWriter = new PrintWriter(new FileOutputStream(resFile));
}
else
{
Log.Info("Report results being placed to logging output");
}
ReportWriter reporter = new ReportWriter();
if (!noOperations.IsEmpty())
{
reporter.BasicReport(noOperations, reportWriter);
}
foreach (string opType in splitTypes.Keys)
{
reporter.OpReport(opType, splitTypes[opType], reportWriter);
}
}
finally
{
if (fileReader != null)
{
fileReader.Close();
}
if (reportWriter != null)
{
reportWriter.Close();
}
}
}
