public virtual void TestArguments()
{
ConfigExtractor extractor = GetTestConfig(true);
NUnit.Framework.Assert.AreEqual(extractor.GetOpCount(), Constants.OperationType.Values
().Length);
NUnit.Framework.Assert.AreEqual(extractor.GetMapAmount(), 2);
NUnit.Framework.Assert.AreEqual(extractor.GetReducerAmount(), 2);
Range <long> apRange = extractor.GetAppendSize();
NUnit.Framework.Assert.AreEqual(apRange.GetLower(), Constants.Megabytes * 1);
NUnit.Framework.Assert.AreEqual(apRange.GetUpper(), Constants.Megabytes * 2);
Range <long> wRange = extractor.GetWriteSize();
NUnit.Framework.Assert.AreEqual(wRange.GetLower(), Constants.Megabytes * 1);
NUnit.Framework.Assert.AreEqual(wRange.GetUpper(), Constants.Megabytes * 2);
Range <long> trRange = extractor.GetTruncateSize();
NUnit.Framework.Assert.AreEqual(trRange.GetLower(), 0);
NUnit.Framework.Assert.AreEqual(trRange.GetUpper(), Constants.Megabytes * 1);
Range <long> bRange = extractor.GetBlockSize();
NUnit.Framework.Assert.AreEqual(bRange.GetLower(), Constants.Megabytes * 1);
NUnit.Framework.Assert.AreEqual(bRange.GetUpper(), Constants.Megabytes * 2);
string resfile = extractor.GetResultFile();
NUnit.Framework.Assert.AreEqual(resfile, GetResultFile().ToString());
int durationMs = extractor.GetDurationMilliseconds();
NUnit.Framework.Assert.AreEqual(durationMs, 10 * 1000);
}
/// <summary>
/// Sets up the operation using the given configuration by setting up the
/// number of operations to perform (and how many are left) and setting up the
/// operation objects to be used throughout selection.
/// </summary>
/// <param name="cfg">ConfigExtractor.</param>
private void ConfigureOperations(ConfigExtractor cfg)
{
operations = new SortedDictionary <Constants.OperationType, WeightSelector.OperationInfo
>();
IDictionary <Constants.OperationType, OperationData> opinfo = cfg.GetOperations();
int totalAm = cfg.GetOpCount();
int opsLeft = totalAm;
NumberFormat formatter = Formatter.GetPercentFormatter();
foreach (Constants.OperationType type in opinfo.Keys)
{
OperationData opData = opinfo[type];
WeightSelector.OperationInfo info = new WeightSelector.OperationInfo();
info.distribution = opData.GetDistribution();
int amLeft = DetermineHowMany(totalAm, opData, type);
opsLeft -= amLeft;
Log.Info(type.ToString() + " has " + amLeft + " initial operations out of " + totalAm
+ " for its ratio " + formatter.Format(opData.GetPercent()));
info.amountLeft = amLeft;
Operation op = factory.GetOperation(type);
// wrap operation in finalizer so that amount left gets decrements when
// its done
if (op != null)
{
ObserveableOp.Observer fn = new _Observer_138(this, type);
info.operation = new ObserveableOp(op, fn);
operations[type] = info;
}
}
if (opsLeft > 0)
{
Log.Info(opsLeft + " left over operations found (due to inability to support partial operations)"
);
}
}
/// <summary>
/// Handles merging all options and verifying from the given command line
/// output and the given base configuration and returns the merged
/// configuration
/// </summary>
/// <param name="opts">the parsed command line option output</param>
/// <param name="base">the base configuration to merge with</param>
/// <returns>the merged configuration</returns>
/// <exception cref="ConfigException"/>
/// <exception cref="Org.Apache.Hadoop.FS.Slive.ConfigMerger.ConfigException"/>
private Configuration HandleOptions(ArgumentParser.ParsedOutput opts, Configuration
@base)
{
// ensure variables are overwritten and verified
ConfigExtractor extractor = new ConfigExtractor(@base);
{
// overwrite the map amount and check to ensure > 0
int mapAmount = null;
try
{
mapAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Maps.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging map amount", e
);
}
if (mapAmount != null)
{
if (mapAmount <= 0)
{
throw new ConfigMerger.ConfigException("Map amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Maps.GetCfgOption(), mapAmount.ToString());
}
}
{
// overwrite the reducer amount and check to ensure > 0
int reduceAmount = null;
try
{
reduceAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Reduces.GetOpt()
));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging reducer amount"
, e);
}
if (reduceAmount != null)
{
if (reduceAmount <= 0)
{
throw new ConfigMerger.ConfigException("Reducer amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Reduces.GetCfgOption(), reduceAmount.ToString());
}
}
{
// overwrite the duration amount and ensure > 0
int duration = null;
try
{
duration = extractor.GetDuration(opts.GetValue(ConfigOption.Duration.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging duration", e);
}
if (duration != null)
{
if (duration <= 0)
{
throw new ConfigMerger.ConfigException("Duration can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Duration.GetCfgOption(), duration.ToString());
}
}
{
// overwrite the operation amount and ensure > 0
int operationAmount = null;
try
{
operationAmount = extractor.GetOpCount(opts.GetValue(ConfigOption.Ops.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging operation amount"
, e);
}
if (operationAmount != null)
{
if (operationAmount <= 0)
{
throw new ConfigMerger.ConfigException("Operation amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Ops.GetCfgOption(), operationAmount.ToString());
}
}
{
// overwrite the exit on error setting
try
{
bool exitOnError = extractor.ShouldExitOnFirstError(opts.GetValue(ConfigOption.ExitOnError
.GetOpt()));
@base.SetBoolean(ConfigOption.ExitOnError.GetCfgOption(), exitOnError);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging exit on error value"
, e);
}
}
{
// overwrite the truncate wait setting
try
{
bool waitOnTruncate = extractor.ShouldWaitOnTruncate(opts.GetValue(ConfigOption.TruncateWait
.GetOpt()));
@base.SetBoolean(ConfigOption.TruncateWait.GetCfgOption(), waitOnTruncate);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging wait on truncate value"
, e);
}
}
{
// verify and set file limit and ensure > 0
int fileAm = null;
try
{
fileAm = extractor.GetTotalFiles(opts.GetValue(ConfigOption.Files.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging total file limit amount"
, e);
}
if (fileAm != null)
{
if (fileAm <= 0)
{
throw new ConfigMerger.ConfigException("File amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Files.GetCfgOption(), fileAm.ToString());
}
}
{
// set the grid queue to run on
try
{
string qname = extractor.GetQueueName(opts.GetValue(ConfigOption.QueueName.GetOpt
()));
if (qname != null)
{
@base.Set(ConfigOption.QueueName.GetCfgOption(), qname);
}
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging queue name", e
);
}
}
{
// verify and set the directory limit and ensure > 0
int directoryLimit = null;
try
{
directoryLimit = extractor.GetDirSize(opts.GetValue(ConfigOption.DirSize.GetOpt()
));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging directory file limit"
, e);
}
if (directoryLimit != null)
{
if (directoryLimit <= 0)
{
throw new ConfigMerger.ConfigException("Directory file limit can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.DirSize.GetCfgOption(), directoryLimit.ToString());
}
}
{
// set the base directory
Path basedir = null;
try
{
basedir = extractor.GetBaseDirectory(opts.GetValue(ConfigOption.BaseDir.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging base directory"
, e);
}
if (basedir != null)
{
// always ensure in slive dir
basedir = new Path(basedir, Constants.BaseDir);
@base.Set(ConfigOption.BaseDir.GetCfgOption(), basedir.ToString());
}
}
{
// set the result file
string fn = null;
try
{
fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging result file",
e);
}
if (fn != null)
{
@base.Set(ConfigOption.ResultFile.GetCfgOption(), fn);
}
}
{
string fn = null;
try
{
fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging result file",
e);
}
if (fn != null)
{
@base.Set(ConfigOption.ResultFile.GetCfgOption(), fn);
}
}
{
// set the operations
try
{
@base = HandleOperations(opts, @base, extractor);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging operations", e
);
}
}
{
// set the replication amount range
Range <short> replicationAm = null;
try
{
replicationAm = extractor.GetReplication(opts.GetValue(ConfigOption.ReplicationAm
.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging replication amount range"
, e);
}
if (replicationAm != null)
{
int minRepl = @base.GetInt(Constants.MinReplication, 1);
if (replicationAm.GetLower() < minRepl)
{
throw new ConfigMerger.ConfigException("Replication amount minimum is less than property configured minimum "
+ minRepl);
}
if (replicationAm.GetLower() > replicationAm.GetUpper())
{
throw new ConfigMerger.ConfigException("Replication amount minimum is greater than its maximum"
);
}
if (replicationAm.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Replication amount minimum must be greater than zero"
);
}
@base.Set(ConfigOption.ReplicationAm.GetCfgOption(), replicationAm.ToString());
}
}
{
// set the sleep range
Range <long> sleepRange = null;
try
{
sleepRange = extractor.GetSleepRange(opts.GetValue(ConfigOption.SleepTime.GetOpt(
)));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging sleep size range"
, e);
}
if (sleepRange != null)
{
if (sleepRange.GetLower() > sleepRange.GetUpper())
{
throw new ConfigMerger.ConfigException("Sleep range minimum is greater than its maximum"
);
}
if (sleepRange.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Sleep range minimum must be greater than zero"
);
}
@base.Set(ConfigOption.SleepTime.GetCfgOption(), sleepRange.ToString());
}
}
{
// set the packet size if given
string pSize = opts.GetValue(ConfigOption.PacketSize.GetOpt());
if (pSize == null)
{
pSize = ConfigOption.PacketSize.GetDefault();
}
if (pSize != null)
{
try
{
long packetSize = StringUtils.TraditionalBinaryPrefix.String2long(pSize);
@base.Set(ConfigOption.PacketSize.GetCfgOption(), packetSize.ToString());
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging write packet size"
, e);
}
}
}
{
// set the block size range
Range <long> blockSize = null;
try
{
blockSize = extractor.GetBlockSize(opts.GetValue(ConfigOption.BlockSize.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging block size range"
, e);
}
if (blockSize != null)
{
if (blockSize.GetLower() > blockSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Block size minimum is greater than its maximum"
);
}
if (blockSize.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Block size minimum must be greater than zero"
);
}
// ensure block size is a multiple of BYTES_PER_CHECKSUM
// if a value is set in the configuration
long bytesPerChecksum = extractor.GetByteCheckSum();
if (bytesPerChecksum != null)
{
if ((blockSize.GetLower() % bytesPerChecksum) != 0)
{
throw new ConfigMerger.ConfigException("Blocksize lower bound must be a multiple of "
+ bytesPerChecksum);
}
if ((blockSize.GetUpper() % bytesPerChecksum) != 0)
{
throw new ConfigMerger.ConfigException("Blocksize upper bound must be a multiple of "
+ bytesPerChecksum);
}
}
@base.Set(ConfigOption.BlockSize.GetCfgOption(), blockSize.ToString());
}
}
{
// set the read size range
Range <long> readSize = null;
try
{
readSize = extractor.GetReadSize(opts.GetValue(ConfigOption.ReadSize.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging read size range"
, e);
}
if (readSize != null)
{
if (readSize.GetLower() > readSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Read size minimum is greater than its maximum"
);
}
if (readSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Read size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.ReadSize.GetCfgOption(), readSize.ToString());
}
}
{
// set the write size range
Range <long> writeSize = null;
try
{
writeSize = extractor.GetWriteSize(opts.GetValue(ConfigOption.WriteSize.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging write size range"
, e);
}
if (writeSize != null)
{
if (writeSize.GetLower() > writeSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Write size minimum is greater than its maximum"
);
}
if (writeSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Write size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.WriteSize.GetCfgOption(), writeSize.ToString());
}
}
{
// set the append size range
Range <long> appendSize = null;
try
{
appendSize = extractor.GetAppendSize(opts.GetValue(ConfigOption.AppendSize.GetOpt
()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging append size range"
, e);
}
if (appendSize != null)
{
if (appendSize.GetLower() > appendSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Append size minimum is greater than its maximum"
);
}
if (appendSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Append size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.AppendSize.GetCfgOption(), appendSize.ToString());
}
}
{
// set the truncate size range
Range <long> truncateSize = null;
try
{
truncateSize = extractor.GetTruncateSize(opts.GetValue(ConfigOption.TruncateSize.
GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging truncate size range"
, e);
}
if (truncateSize != null)
{
if (truncateSize.GetLower() > truncateSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Truncate size minimum is greater than its maximum"
);
}
if (truncateSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Truncate size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.TruncateSize.GetCfgOption(), truncateSize.ToString());
}
}
{
// set the seed
long seed = null;
try
{
seed = extractor.GetRandomSeed(opts.GetValue(ConfigOption.RandomSeed.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging random number seed"
, e);
}
if (seed != null)
{
@base.Set(ConfigOption.RandomSeed.GetCfgOption(), seed.ToString());
}
}
return(@base);
}