public virtual void TestArguments()
{
ConfigExtractor extractor = GetTestConfig(true);
NUnit.Framework.Assert.AreEqual(extractor.GetOpCount(), Constants.OperationType.Values
().Length);
NUnit.Framework.Assert.AreEqual(extractor.GetMapAmount(), 2);
NUnit.Framework.Assert.AreEqual(extractor.GetReducerAmount(), 2);
Range <long> apRange = extractor.GetAppendSize();
NUnit.Framework.Assert.AreEqual(apRange.GetLower(), Constants.Megabytes * 1);
NUnit.Framework.Assert.AreEqual(apRange.GetUpper(), Constants.Megabytes * 2);
Range <long> wRange = extractor.GetWriteSize();
NUnit.Framework.Assert.AreEqual(wRange.GetLower(), Constants.Megabytes * 1);
NUnit.Framework.Assert.AreEqual(wRange.GetUpper(), Constants.Megabytes * 2);
Range <long> trRange = extractor.GetTruncateSize();
NUnit.Framework.Assert.AreEqual(trRange.GetLower(), 0);
NUnit.Framework.Assert.AreEqual(trRange.GetUpper(), Constants.Megabytes * 1);
Range <long> bRange = extractor.GetBlockSize();
NUnit.Framework.Assert.AreEqual(bRange.GetLower(), Constants.Megabytes * 1);
NUnit.Framework.Assert.AreEqual(bRange.GetUpper(), Constants.Megabytes * 2);
string resfile = extractor.GetResultFile();
NUnit.Framework.Assert.AreEqual(resfile, GetResultFile().ToString());
int durationMs = extractor.GetDurationMilliseconds();
NUnit.Framework.Assert.AreEqual(durationMs, 10 * 1000);
}
/// <summary>Sets up a job conf for the given job using the given config object.</summary>
/// <remarks>
/// Sets up a job conf for the given job using the given config object. Ensures
/// that the correct input format is set, the mapper and and reducer class and
/// the input and output keys and value classes along with any other job
/// configuration.
/// </remarks>
/// <param name="config"/>
/// <returns>JobConf representing the job to be ran</returns>
/// <exception cref="System.IO.IOException"/>
private JobConf GetJob(ConfigExtractor config)
{
JobConf job = new JobConf(config.GetConfig(), typeof(Org.Apache.Hadoop.FS.Slive.SliveTest
));
job.SetInputFormat(typeof(DummyInputFormat));
FileOutputFormat.SetOutputPath(job, config.GetOutputPath());
job.SetMapperClass(typeof(SliveMapper));
job.SetPartitionerClass(typeof(SlivePartitioner));
job.SetReducerClass(typeof(SliveReducer));
job.SetOutputKeyClass(typeof(Text));
job.SetOutputValueClass(typeof(Text));
job.SetOutputFormat(typeof(TextOutputFormat));
TextOutputFormat.SetCompressOutput(job, false);
job.SetNumReduceTasks(config.GetReducerAmount());
job.SetNumMapTasks(config.GetMapAmount());
return(job);
}
/// <summary>
/// Handles merging all options and verifying from the given command line
/// output and the given base configuration and returns the merged
/// configuration
/// </summary>
/// <param name="opts">the parsed command line option output</param>
/// <param name="base">the base configuration to merge with</param>
/// <returns>the merged configuration</returns>
/// <exception cref="ConfigException"/>
/// <exception cref="Org.Apache.Hadoop.FS.Slive.ConfigMerger.ConfigException"/>
private Configuration HandleOptions(ArgumentParser.ParsedOutput opts, Configuration
@base)
{
// ensure variables are overwritten and verified
ConfigExtractor extractor = new ConfigExtractor(@base);
{
// overwrite the map amount and check to ensure > 0
int mapAmount = null;
try
{
mapAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Maps.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging map amount", e
);
}
if (mapAmount != null)
{
if (mapAmount <= 0)
{
throw new ConfigMerger.ConfigException("Map amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Maps.GetCfgOption(), mapAmount.ToString());
}
}
{
// overwrite the reducer amount and check to ensure > 0
int reduceAmount = null;
try
{
reduceAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Reduces.GetOpt()
));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging reducer amount"
, e);
}
if (reduceAmount != null)
{
if (reduceAmount <= 0)
{
throw new ConfigMerger.ConfigException("Reducer amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Reduces.GetCfgOption(), reduceAmount.ToString());
}
}
{
// overwrite the duration amount and ensure > 0
int duration = null;
try
{
duration = extractor.GetDuration(opts.GetValue(ConfigOption.Duration.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging duration", e);
}
if (duration != null)
{
if (duration <= 0)
{
throw new ConfigMerger.ConfigException("Duration can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Duration.GetCfgOption(), duration.ToString());
}
}
{
// overwrite the operation amount and ensure > 0
int operationAmount = null;
try
{
operationAmount = extractor.GetOpCount(opts.GetValue(ConfigOption.Ops.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging operation amount"
, e);
}
if (operationAmount != null)
{
if (operationAmount <= 0)
{
throw new ConfigMerger.ConfigException("Operation amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Ops.GetCfgOption(), operationAmount.ToString());
}
}
{
// overwrite the exit on error setting
try
{
bool exitOnError = extractor.ShouldExitOnFirstError(opts.GetValue(ConfigOption.ExitOnError
.GetOpt()));
@base.SetBoolean(ConfigOption.ExitOnError.GetCfgOption(), exitOnError);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging exit on error value"
, e);
}
}
{
// overwrite the truncate wait setting
try
{
bool waitOnTruncate = extractor.ShouldWaitOnTruncate(opts.GetValue(ConfigOption.TruncateWait
.GetOpt()));
@base.SetBoolean(ConfigOption.TruncateWait.GetCfgOption(), waitOnTruncate);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging wait on truncate value"
, e);
}
}
{
// verify and set file limit and ensure > 0
int fileAm = null;
try
{
fileAm = extractor.GetTotalFiles(opts.GetValue(ConfigOption.Files.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging total file limit amount"
, e);
}
if (fileAm != null)
{
if (fileAm <= 0)
{
throw new ConfigMerger.ConfigException("File amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Files.GetCfgOption(), fileAm.ToString());
}
}
{
// set the grid queue to run on
try
{
string qname = extractor.GetQueueName(opts.GetValue(ConfigOption.QueueName.GetOpt
()));
if (qname != null)
{
@base.Set(ConfigOption.QueueName.GetCfgOption(), qname);
}
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging queue name", e
);
}
}
{
// verify and set the directory limit and ensure > 0
int directoryLimit = null;
try
{
directoryLimit = extractor.GetDirSize(opts.GetValue(ConfigOption.DirSize.GetOpt()
));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging directory file limit"
, e);
}
if (directoryLimit != null)
{
if (directoryLimit <= 0)
{
throw new ConfigMerger.ConfigException("Directory file limit can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.DirSize.GetCfgOption(), directoryLimit.ToString());
}
}
{
// set the base directory
Path basedir = null;
try
{
basedir = extractor.GetBaseDirectory(opts.GetValue(ConfigOption.BaseDir.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging base directory"
, e);
}
if (basedir != null)
{
// always ensure in slive dir
basedir = new Path(basedir, Constants.BaseDir);
@base.Set(ConfigOption.BaseDir.GetCfgOption(), basedir.ToString());
}
}
{
// set the result file
string fn = null;
try
{
fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging result file",
e);
}
if (fn != null)
{
@base.Set(ConfigOption.ResultFile.GetCfgOption(), fn);
}
}
{
string fn = null;
try
{
fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging result file",
e);
}
if (fn != null)
{
@base.Set(ConfigOption.ResultFile.GetCfgOption(), fn);
}
}
{
// set the operations
try
{
@base = HandleOperations(opts, @base, extractor);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging operations", e
);
}
}
{
// set the replication amount range
Range <short> replicationAm = null;
try
{
replicationAm = extractor.GetReplication(opts.GetValue(ConfigOption.ReplicationAm
.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging replication amount range"
, e);
}
if (replicationAm != null)
{
int minRepl = @base.GetInt(Constants.MinReplication, 1);
if (replicationAm.GetLower() < minRepl)
{
throw new ConfigMerger.ConfigException("Replication amount minimum is less than property configured minimum "
+ minRepl);
}
if (replicationAm.GetLower() > replicationAm.GetUpper())
{
throw new ConfigMerger.ConfigException("Replication amount minimum is greater than its maximum"
);
}
if (replicationAm.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Replication amount minimum must be greater than zero"
);
}
@base.Set(ConfigOption.ReplicationAm.GetCfgOption(), replicationAm.ToString());
}
}
{
// set the sleep range
Range <long> sleepRange = null;
try
{
sleepRange = extractor.GetSleepRange(opts.GetValue(ConfigOption.SleepTime.GetOpt(
)));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging sleep size range"
, e);
}
if (sleepRange != null)
{
if (sleepRange.GetLower() > sleepRange.GetUpper())
{
throw new ConfigMerger.ConfigException("Sleep range minimum is greater than its maximum"
);
}
if (sleepRange.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Sleep range minimum must be greater than zero"
);
}
@base.Set(ConfigOption.SleepTime.GetCfgOption(), sleepRange.ToString());
}
}
{
// set the packet size if given
string pSize = opts.GetValue(ConfigOption.PacketSize.GetOpt());
if (pSize == null)
{
pSize = ConfigOption.PacketSize.GetDefault();
}
if (pSize != null)
{
try
{
long packetSize = StringUtils.TraditionalBinaryPrefix.String2long(pSize);
@base.Set(ConfigOption.PacketSize.GetCfgOption(), packetSize.ToString());
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging write packet size"
, e);
}
}
}
{
// set the block size range
Range <long> blockSize = null;
try
{
blockSize = extractor.GetBlockSize(opts.GetValue(ConfigOption.BlockSize.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging block size range"
, e);
}
if (blockSize != null)
{
if (blockSize.GetLower() > blockSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Block size minimum is greater than its maximum"
);
}
if (blockSize.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Block size minimum must be greater than zero"
);
}
// ensure block size is a multiple of BYTES_PER_CHECKSUM
// if a value is set in the configuration
long bytesPerChecksum = extractor.GetByteCheckSum();
if (bytesPerChecksum != null)
{
if ((blockSize.GetLower() % bytesPerChecksum) != 0)
{
throw new ConfigMerger.ConfigException("Blocksize lower bound must be a multiple of "
+ bytesPerChecksum);
}
if ((blockSize.GetUpper() % bytesPerChecksum) != 0)
{
throw new ConfigMerger.ConfigException("Blocksize upper bound must be a multiple of "
+ bytesPerChecksum);
}
}
@base.Set(ConfigOption.BlockSize.GetCfgOption(), blockSize.ToString());
}
}
{
// set the read size range
Range <long> readSize = null;
try
{
readSize = extractor.GetReadSize(opts.GetValue(ConfigOption.ReadSize.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging read size range"
, e);
}
if (readSize != null)
{
if (readSize.GetLower() > readSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Read size minimum is greater than its maximum"
);
}
if (readSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Read size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.ReadSize.GetCfgOption(), readSize.ToString());
}
}
{
// set the write size range
Range <long> writeSize = null;
try
{
writeSize = extractor.GetWriteSize(opts.GetValue(ConfigOption.WriteSize.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging write size range"
, e);
}
if (writeSize != null)
{
if (writeSize.GetLower() > writeSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Write size minimum is greater than its maximum"
);
}
if (writeSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Write size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.WriteSize.GetCfgOption(), writeSize.ToString());
}
}
{
// set the append size range
Range <long> appendSize = null;
try
{
appendSize = extractor.GetAppendSize(opts.GetValue(ConfigOption.AppendSize.GetOpt
()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging append size range"
, e);
}
if (appendSize != null)
{
if (appendSize.GetLower() > appendSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Append size minimum is greater than its maximum"
);
}
if (appendSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Append size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.AppendSize.GetCfgOption(), appendSize.ToString());
}
}
{
// set the truncate size range
Range <long> truncateSize = null;
try
{
truncateSize = extractor.GetTruncateSize(opts.GetValue(ConfigOption.TruncateSize.
GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging truncate size range"
, e);
}
if (truncateSize != null)
{
if (truncateSize.GetLower() > truncateSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Truncate size minimum is greater than its maximum"
);
}
if (truncateSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Truncate size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.TruncateSize.GetCfgOption(), truncateSize.ToString());
}
}
{
// set the seed
long seed = null;
try
{
seed = extractor.GetRandomSeed(opts.GetValue(ConfigOption.RandomSeed.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging random number seed"
, e);
}
if (seed != null)
{
@base.Set(ConfigOption.RandomSeed.GetCfgOption(), seed.ToString());
}
}
return(@base);
}