internal ArgumentParser(string[] args)
{
optList = GetOptions();
if (args == null)
{
args = new string[] { };
}
argumentList = args;
parsed = null;
}
// gets the config merged with the arguments
/// <exception cref="System.Exception"/>
private ConfigExtractor GetTestConfig(bool sleep)
{
ArgumentParser parser = new ArgumentParser(GetTestArgs(sleep));
ArgumentParser.ParsedOutput @out = parser.Parse();
NUnit.Framework.Assert.IsTrue([email protected]());
ConfigMerger merge = new ConfigMerger();
Configuration cfg = merge.GetMerged(@out, GetBaseConfig());
ConfigExtractor extractor = new ConfigExtractor(cfg);
return(extractor);
}
/// <summary>Parses the command line options</summary>
/// <returns>false if need to print help output</returns>
/// <exception cref="System.Exception">when parsing fails</exception>
internal virtual ArgumentParser.ParsedOutput Parse()
{
if (parsed == null)
{
PosixParser parser = new PosixParser();
CommandLine popts = parser.Parse(GetOptionList(), argumentList, true);
if (popts.HasOption(ConfigOption.Help.GetOpt()))
{
parsed = new ArgumentParser.ParsedOutput(null, this, true);
}
else
{
parsed = new ArgumentParser.ParsedOutput(popts, this, false);
}
}
return(parsed);
}
/// <summary>
/// Handles the specific task of merging operations from the command line or
/// extractor object into the base configuration provided
/// </summary>
/// <param name="opts">the parsed command line option output</param>
/// <param name="base">the base configuration to merge with</param>
/// <param name="extractor">
/// the access object to fetch operations from if none from the
/// command line
/// </param>
/// <returns>merged configuration object</returns>
/// <exception cref="ConfigException">when verification fails</exception>
/// <exception cref="Org.Apache.Hadoop.FS.Slive.ConfigMerger.ConfigException"/>
private Configuration HandleOperations(ArgumentParser.ParsedOutput opts, Configuration
@base, ConfigExtractor extractor)
{
// get the base set to start off with
IDictionary <Constants.OperationType, OperationData> operations = GetBaseOperations
();
// merge with what is coming from config
IDictionary <Constants.OperationType, OperationData> cfgOperations = extractor.GetOperations
();
foreach (Constants.OperationType opType in cfgOperations.Keys)
{
operations[opType] = cfgOperations[opType];
}
// see if any coming in from the command line
foreach (Constants.OperationType opType_1 in Constants.OperationType.Values())
{
string opName = opType_1.LowerName();
string opVal = opts.GetValue(opName);
if (opVal != null)
{
operations[opType_1] = new OperationData(opVal);
}
}
{
// remove those with <= zero percent
IDictionary <Constants.OperationType, OperationData> cleanedOps = new Dictionary <Constants.OperationType
, OperationData>();
foreach (Constants.OperationType opType_2 in operations.Keys)
{
OperationData data = operations[opType_2];
if (data.GetPercent() == null || data.GetPercent() > 0.0d)
{
cleanedOps[opType_2] = data;
}
}
operations = cleanedOps;
}
if (operations.IsEmpty())
{
throw new ConfigMerger.ConfigException("No operations provided!");
}
// verify and adjust
double currPct = 0;
int needFill = 0;
foreach (Constants.OperationType type in operations.Keys)
{
OperationData op = operations[type];
if (op.GetPercent() != null)
{
currPct += op.GetPercent();
}
else
{
needFill++;
}
}
if (currPct > 1)
{
throw new ConfigMerger.ConfigException("Unable to have accumlative percent greater than 100%"
);
}
if (needFill > 0 && currPct < 1)
{
double leftOver = 1.0 - currPct;
IDictionary <Constants.OperationType, OperationData> mpcp = new Dictionary <Constants.OperationType
, OperationData>();
foreach (Constants.OperationType type_1 in operations.Keys)
{
OperationData op = operations[type_1];
if (op.GetPercent() == null)
{
op = new OperationData(op.GetDistribution(), (leftOver / needFill));
}
mpcp[type_1] = op;
}
operations = mpcp;
}
else
{
if (needFill == 0 && currPct < 1)
{
// redistribute
double leftOver = 1.0 - currPct;
IDictionary <Constants.OperationType, OperationData> mpcp = new Dictionary <Constants.OperationType
, OperationData>();
double each = leftOver / operations.Keys.Count;
foreach (Constants.OperationType t in operations.Keys)
{
OperationData op = operations[t];
op = new OperationData(op.GetDistribution(), (op.GetPercent() + each));
mpcp[t] = op;
}
operations = mpcp;
}
else
{
if (needFill > 0 && currPct >= 1)
{
throw new ConfigMerger.ConfigException(needFill + " unfilled operations but no percentage left to fill with"
);
}
}
}
// save into base
foreach (Constants.OperationType opType_3 in operations.Keys)
{
string opName = opType_3.LowerName();
OperationData opData = operations[opType_3];
string distr = opData.GetDistribution().LowerName();
string ratio = opData.GetPercent() * 100.0d.ToString();
@base.Set(string.Format(Constants.Op, opName), opData.ToString());
@base.Set(string.Format(Constants.OpDistr, opName), distr);
@base.Set(string.Format(Constants.OpPercent, opName), ratio);
}
return(@base);
}
/// <summary>
/// Merges the given command line parsed output with the given configuration
/// object and returns the new configuration object with the correct options
/// overwritten
/// </summary>
/// <param name="opts">the parsed command line option output</param>
/// <param name="base">the base configuration to merge with</param>
/// <returns>merged configuration object</returns>
/// <exception cref="ConfigException">when configuration errors or verification occur
/// </exception>
/// <exception cref="Org.Apache.Hadoop.FS.Slive.ConfigMerger.ConfigException"/>
internal virtual Configuration GetMerged(ArgumentParser.ParsedOutput opts, Configuration
@base)
{
return(HandleOptions(opts, @base));
}
/// <summary>
/// Handles merging all options and verifying from the given command line
/// output and the given base configuration and returns the merged
/// configuration
/// </summary>
/// <param name="opts">the parsed command line option output</param>
/// <param name="base">the base configuration to merge with</param>
/// <returns>the merged configuration</returns>
/// <exception cref="ConfigException"/>
/// <exception cref="Org.Apache.Hadoop.FS.Slive.ConfigMerger.ConfigException"/>
private Configuration HandleOptions(ArgumentParser.ParsedOutput opts, Configuration
@base)
{
// ensure variables are overwritten and verified
ConfigExtractor extractor = new ConfigExtractor(@base);
{
// overwrite the map amount and check to ensure > 0
int mapAmount = null;
try
{
mapAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Maps.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging map amount", e
);
}
if (mapAmount != null)
{
if (mapAmount <= 0)
{
throw new ConfigMerger.ConfigException("Map amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Maps.GetCfgOption(), mapAmount.ToString());
}
}
{
// overwrite the reducer amount and check to ensure > 0
int reduceAmount = null;
try
{
reduceAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Reduces.GetOpt()
));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging reducer amount"
, e);
}
if (reduceAmount != null)
{
if (reduceAmount <= 0)
{
throw new ConfigMerger.ConfigException("Reducer amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Reduces.GetCfgOption(), reduceAmount.ToString());
}
}
{
// overwrite the duration amount and ensure > 0
int duration = null;
try
{
duration = extractor.GetDuration(opts.GetValue(ConfigOption.Duration.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging duration", e);
}
if (duration != null)
{
if (duration <= 0)
{
throw new ConfigMerger.ConfigException("Duration can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Duration.GetCfgOption(), duration.ToString());
}
}
{
// overwrite the operation amount and ensure > 0
int operationAmount = null;
try
{
operationAmount = extractor.GetOpCount(opts.GetValue(ConfigOption.Ops.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging operation amount"
, e);
}
if (operationAmount != null)
{
if (operationAmount <= 0)
{
throw new ConfigMerger.ConfigException("Operation amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Ops.GetCfgOption(), operationAmount.ToString());
}
}
{
// overwrite the exit on error setting
try
{
bool exitOnError = extractor.ShouldExitOnFirstError(opts.GetValue(ConfigOption.ExitOnError
.GetOpt()));
@base.SetBoolean(ConfigOption.ExitOnError.GetCfgOption(), exitOnError);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging exit on error value"
, e);
}
}
{
// overwrite the truncate wait setting
try
{
bool waitOnTruncate = extractor.ShouldWaitOnTruncate(opts.GetValue(ConfigOption.TruncateWait
.GetOpt()));
@base.SetBoolean(ConfigOption.TruncateWait.GetCfgOption(), waitOnTruncate);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging wait on truncate value"
, e);
}
}
{
// verify and set file limit and ensure > 0
int fileAm = null;
try
{
fileAm = extractor.GetTotalFiles(opts.GetValue(ConfigOption.Files.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging total file limit amount"
, e);
}
if (fileAm != null)
{
if (fileAm <= 0)
{
throw new ConfigMerger.ConfigException("File amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Files.GetCfgOption(), fileAm.ToString());
}
}
{
// set the grid queue to run on
try
{
string qname = extractor.GetQueueName(opts.GetValue(ConfigOption.QueueName.GetOpt
()));
if (qname != null)
{
@base.Set(ConfigOption.QueueName.GetCfgOption(), qname);
}
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging queue name", e
);
}
}
{
// verify and set the directory limit and ensure > 0
int directoryLimit = null;
try
{
directoryLimit = extractor.GetDirSize(opts.GetValue(ConfigOption.DirSize.GetOpt()
));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging directory file limit"
, e);
}
if (directoryLimit != null)
{
if (directoryLimit <= 0)
{
throw new ConfigMerger.ConfigException("Directory file limit can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.DirSize.GetCfgOption(), directoryLimit.ToString());
}
}
{
// set the base directory
Path basedir = null;
try
{
basedir = extractor.GetBaseDirectory(opts.GetValue(ConfigOption.BaseDir.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging base directory"
, e);
}
if (basedir != null)
{
// always ensure in slive dir
basedir = new Path(basedir, Constants.BaseDir);
@base.Set(ConfigOption.BaseDir.GetCfgOption(), basedir.ToString());
}
}
{
// set the result file
string fn = null;
try
{
fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging result file",
e);
}
if (fn != null)
{
@base.Set(ConfigOption.ResultFile.GetCfgOption(), fn);
}
}
{
string fn = null;
try
{
fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging result file",
e);
}
if (fn != null)
{
@base.Set(ConfigOption.ResultFile.GetCfgOption(), fn);
}
}
{
// set the operations
try
{
@base = HandleOperations(opts, @base, extractor);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging operations", e
);
}
}
{
// set the replication amount range
Range <short> replicationAm = null;
try
{
replicationAm = extractor.GetReplication(opts.GetValue(ConfigOption.ReplicationAm
.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging replication amount range"
, e);
}
if (replicationAm != null)
{
int minRepl = @base.GetInt(Constants.MinReplication, 1);
if (replicationAm.GetLower() < minRepl)
{
throw new ConfigMerger.ConfigException("Replication amount minimum is less than property configured minimum "
+ minRepl);
}
if (replicationAm.GetLower() > replicationAm.GetUpper())
{
throw new ConfigMerger.ConfigException("Replication amount minimum is greater than its maximum"
);
}
if (replicationAm.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Replication amount minimum must be greater than zero"
);
}
@base.Set(ConfigOption.ReplicationAm.GetCfgOption(), replicationAm.ToString());
}
}
{
// set the sleep range
Range <long> sleepRange = null;
try
{
sleepRange = extractor.GetSleepRange(opts.GetValue(ConfigOption.SleepTime.GetOpt(
)));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging sleep size range"
, e);
}
if (sleepRange != null)
{
if (sleepRange.GetLower() > sleepRange.GetUpper())
{
throw new ConfigMerger.ConfigException("Sleep range minimum is greater than its maximum"
);
}
if (sleepRange.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Sleep range minimum must be greater than zero"
);
}
@base.Set(ConfigOption.SleepTime.GetCfgOption(), sleepRange.ToString());
}
}
{
// set the packet size if given
string pSize = opts.GetValue(ConfigOption.PacketSize.GetOpt());
if (pSize == null)
{
pSize = ConfigOption.PacketSize.GetDefault();
}
if (pSize != null)
{
try
{
long packetSize = StringUtils.TraditionalBinaryPrefix.String2long(pSize);
@base.Set(ConfigOption.PacketSize.GetCfgOption(), packetSize.ToString());
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging write packet size"
, e);
}
}
}
{
// set the block size range
Range <long> blockSize = null;
try
{
blockSize = extractor.GetBlockSize(opts.GetValue(ConfigOption.BlockSize.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging block size range"
, e);
}
if (blockSize != null)
{
if (blockSize.GetLower() > blockSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Block size minimum is greater than its maximum"
);
}
if (blockSize.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Block size minimum must be greater than zero"
);
}
// ensure block size is a multiple of BYTES_PER_CHECKSUM
// if a value is set in the configuration
long bytesPerChecksum = extractor.GetByteCheckSum();
if (bytesPerChecksum != null)
{
if ((blockSize.GetLower() % bytesPerChecksum) != 0)
{
throw new ConfigMerger.ConfigException("Blocksize lower bound must be a multiple of "
+ bytesPerChecksum);
}
if ((blockSize.GetUpper() % bytesPerChecksum) != 0)
{
throw new ConfigMerger.ConfigException("Blocksize upper bound must be a multiple of "
+ bytesPerChecksum);
}
}
@base.Set(ConfigOption.BlockSize.GetCfgOption(), blockSize.ToString());
}
}
{
// set the read size range
Range <long> readSize = null;
try
{
readSize = extractor.GetReadSize(opts.GetValue(ConfigOption.ReadSize.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging read size range"
, e);
}
if (readSize != null)
{
if (readSize.GetLower() > readSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Read size minimum is greater than its maximum"
);
}
if (readSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Read size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.ReadSize.GetCfgOption(), readSize.ToString());
}
}
{
// set the write size range
Range <long> writeSize = null;
try
{
writeSize = extractor.GetWriteSize(opts.GetValue(ConfigOption.WriteSize.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging write size range"
, e);
}
if (writeSize != null)
{
if (writeSize.GetLower() > writeSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Write size minimum is greater than its maximum"
);
}
if (writeSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Write size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.WriteSize.GetCfgOption(), writeSize.ToString());
}
}
{
// set the append size range
Range <long> appendSize = null;
try
{
appendSize = extractor.GetAppendSize(opts.GetValue(ConfigOption.AppendSize.GetOpt
()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging append size range"
, e);
}
if (appendSize != null)
{
if (appendSize.GetLower() > appendSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Append size minimum is greater than its maximum"
);
}
if (appendSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Append size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.AppendSize.GetCfgOption(), appendSize.ToString());
}
}
{
// set the truncate size range
Range <long> truncateSize = null;
try
{
truncateSize = extractor.GetTruncateSize(opts.GetValue(ConfigOption.TruncateSize.
GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging truncate size range"
, e);
}
if (truncateSize != null)
{
if (truncateSize.GetLower() > truncateSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Truncate size minimum is greater than its maximum"
);
}
if (truncateSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Truncate size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.TruncateSize.GetCfgOption(), truncateSize.ToString());
}
}
{
// set the seed
long seed = null;
try
{
seed = extractor.GetRandomSeed(opts.GetValue(ConfigOption.RandomSeed.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging random number seed"
, e);
}
if (seed != null)
{
@base.Set(ConfigOption.RandomSeed.GetCfgOption(), seed.ToString());
}
}
return(@base);
}
public virtual int Run(string[] args)
{
ArgumentParser.ParsedOutput parsedOpts = null;
try
{
ArgumentParser argHolder = new ArgumentParser(args);
parsedOpts = argHolder.Parse();
if (parsedOpts.ShouldOutputHelp())
{
parsedOpts.OutputHelp();
return(1);
}
}
catch (Exception e)
{
Log.Error("Unable to parse arguments due to error: ", e);
return(1);
}
Log.Info("Running with option list " + Helper.StringifyArray(args, " "));
ConfigExtractor config = null;
try
{
ConfigMerger cfgMerger = new ConfigMerger();
Configuration cfg = cfgMerger.GetMerged(parsedOpts, new Configuration(@base));
if (cfg != null)
{
config = new ConfigExtractor(cfg);
}
}
catch (Exception e)
{
Log.Error("Unable to merge config due to error: ", e);
return(1);
}
if (config == null)
{
Log.Error("Unable to merge config & options!");
return(1);
}
try
{
Log.Info("Options are:");
ConfigExtractor.DumpOptions(config);
}
catch (Exception e)
{
Log.Error("Unable to dump options due to error: ", e);
return(1);
}
bool jobOk = false;
try
{
Log.Info("Running job:");
RunJob(config);
jobOk = true;
}
catch (Exception e)
{
Log.Error("Unable to run job due to error: ", e);
}
if (jobOk)
{
try
{
Log.Info("Reporting on job:");
WriteReport(config);
}
catch (Exception e)
{
Log.Error("Unable to report on job due to error: ", e);
}
}
// attempt cleanup (not critical)
bool cleanUp = GetBool(parsedOpts.GetValue(ConfigOption.Cleanup.GetOpt()));
if (cleanUp)
{
try
{
Log.Info("Cleaning up job:");
Cleanup(config);
}
catch (Exception e)
{
Log.Error("Unable to cleanup job due to error: ", e);
}
}
// all mostly worked
if (jobOk)
{
return(0);
}
// maybe didn't work
return(1);
}