/// <summary>
/// Handles merging all options and verifying from the given command line
/// output and the given base configuration and returns the merged
/// configuration
/// </summary>
/// <param name="opts">the parsed command line option output</param>
/// <param name="base">the base configuration to merge with</param>
/// <returns>the merged configuration</returns>
/// <exception cref="ConfigException"/>
/// <exception cref="Org.Apache.Hadoop.FS.Slive.ConfigMerger.ConfigException"/>
private Configuration HandleOptions(ArgumentParser.ParsedOutput opts, Configuration
@base)
{
// ensure variables are overwritten and verified
ConfigExtractor extractor = new ConfigExtractor(@base);
{
// overwrite the map amount and check to ensure > 0
int mapAmount = null;
try
{
mapAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Maps.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging map amount", e
);
}
if (mapAmount != null)
{
if (mapAmount <= 0)
{
throw new ConfigMerger.ConfigException("Map amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Maps.GetCfgOption(), mapAmount.ToString());
}
}
{
// overwrite the reducer amount and check to ensure > 0
int reduceAmount = null;
try
{
reduceAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Reduces.GetOpt()
));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging reducer amount"
, e);
}
if (reduceAmount != null)
{
if (reduceAmount <= 0)
{
throw new ConfigMerger.ConfigException("Reducer amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Reduces.GetCfgOption(), reduceAmount.ToString());
}
}
{
// overwrite the duration amount and ensure > 0
int duration = null;
try
{
duration = extractor.GetDuration(opts.GetValue(ConfigOption.Duration.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging duration", e);
}
if (duration != null)
{
if (duration <= 0)
{
throw new ConfigMerger.ConfigException("Duration can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Duration.GetCfgOption(), duration.ToString());
}
}
{
// overwrite the operation amount and ensure > 0
int operationAmount = null;
try
{
operationAmount = extractor.GetOpCount(opts.GetValue(ConfigOption.Ops.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging operation amount"
, e);
}
if (operationAmount != null)
{
if (operationAmount <= 0)
{
throw new ConfigMerger.ConfigException("Operation amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Ops.GetCfgOption(), operationAmount.ToString());
}
}
{
// overwrite the exit on error setting
try
{
bool exitOnError = extractor.ShouldExitOnFirstError(opts.GetValue(ConfigOption.ExitOnError
.GetOpt()));
@base.SetBoolean(ConfigOption.ExitOnError.GetCfgOption(), exitOnError);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging exit on error value"
, e);
}
}
{
// overwrite the truncate wait setting
try
{
bool waitOnTruncate = extractor.ShouldWaitOnTruncate(opts.GetValue(ConfigOption.TruncateWait
.GetOpt()));
@base.SetBoolean(ConfigOption.TruncateWait.GetCfgOption(), waitOnTruncate);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging wait on truncate value"
, e);
}
}
{
// verify and set file limit and ensure > 0
int fileAm = null;
try
{
fileAm = extractor.GetTotalFiles(opts.GetValue(ConfigOption.Files.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging total file limit amount"
, e);
}
if (fileAm != null)
{
if (fileAm <= 0)
{
throw new ConfigMerger.ConfigException("File amount can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.Files.GetCfgOption(), fileAm.ToString());
}
}
{
// set the grid queue to run on
try
{
string qname = extractor.GetQueueName(opts.GetValue(ConfigOption.QueueName.GetOpt
()));
if (qname != null)
{
@base.Set(ConfigOption.QueueName.GetCfgOption(), qname);
}
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging queue name", e
);
}
}
{
// verify and set the directory limit and ensure > 0
int directoryLimit = null;
try
{
directoryLimit = extractor.GetDirSize(opts.GetValue(ConfigOption.DirSize.GetOpt()
));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging directory file limit"
, e);
}
if (directoryLimit != null)
{
if (directoryLimit <= 0)
{
throw new ConfigMerger.ConfigException("Directory file limit can not be less than or equal to zero"
);
}
@base.Set(ConfigOption.DirSize.GetCfgOption(), directoryLimit.ToString());
}
}
{
// set the base directory
Path basedir = null;
try
{
basedir = extractor.GetBaseDirectory(opts.GetValue(ConfigOption.BaseDir.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging base directory"
, e);
}
if (basedir != null)
{
// always ensure in slive dir
basedir = new Path(basedir, Constants.BaseDir);
@base.Set(ConfigOption.BaseDir.GetCfgOption(), basedir.ToString());
}
}
{
// set the result file
string fn = null;
try
{
fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging result file",
e);
}
if (fn != null)
{
@base.Set(ConfigOption.ResultFile.GetCfgOption(), fn);
}
}
{
string fn = null;
try
{
fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging result file",
e);
}
if (fn != null)
{
@base.Set(ConfigOption.ResultFile.GetCfgOption(), fn);
}
}
{
// set the operations
try
{
@base = HandleOperations(opts, @base, extractor);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging operations", e
);
}
}
{
// set the replication amount range
Range <short> replicationAm = null;
try
{
replicationAm = extractor.GetReplication(opts.GetValue(ConfigOption.ReplicationAm
.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging replication amount range"
, e);
}
if (replicationAm != null)
{
int minRepl = @base.GetInt(Constants.MinReplication, 1);
if (replicationAm.GetLower() < minRepl)
{
throw new ConfigMerger.ConfigException("Replication amount minimum is less than property configured minimum "
+ minRepl);
}
if (replicationAm.GetLower() > replicationAm.GetUpper())
{
throw new ConfigMerger.ConfigException("Replication amount minimum is greater than its maximum"
);
}
if (replicationAm.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Replication amount minimum must be greater than zero"
);
}
@base.Set(ConfigOption.ReplicationAm.GetCfgOption(), replicationAm.ToString());
}
}
{
// set the sleep range
Range <long> sleepRange = null;
try
{
sleepRange = extractor.GetSleepRange(opts.GetValue(ConfigOption.SleepTime.GetOpt(
)));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging sleep size range"
, e);
}
if (sleepRange != null)
{
if (sleepRange.GetLower() > sleepRange.GetUpper())
{
throw new ConfigMerger.ConfigException("Sleep range minimum is greater than its maximum"
);
}
if (sleepRange.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Sleep range minimum must be greater than zero"
);
}
@base.Set(ConfigOption.SleepTime.GetCfgOption(), sleepRange.ToString());
}
}
{
// set the packet size if given
string pSize = opts.GetValue(ConfigOption.PacketSize.GetOpt());
if (pSize == null)
{
pSize = ConfigOption.PacketSize.GetDefault();
}
if (pSize != null)
{
try
{
long packetSize = StringUtils.TraditionalBinaryPrefix.String2long(pSize);
@base.Set(ConfigOption.PacketSize.GetCfgOption(), packetSize.ToString());
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging write packet size"
, e);
}
}
}
{
// set the block size range
Range <long> blockSize = null;
try
{
blockSize = extractor.GetBlockSize(opts.GetValue(ConfigOption.BlockSize.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging block size range"
, e);
}
if (blockSize != null)
{
if (blockSize.GetLower() > blockSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Block size minimum is greater than its maximum"
);
}
if (blockSize.GetLower() <= 0)
{
throw new ConfigMerger.ConfigException("Block size minimum must be greater than zero"
);
}
// ensure block size is a multiple of BYTES_PER_CHECKSUM
// if a value is set in the configuration
long bytesPerChecksum = extractor.GetByteCheckSum();
if (bytesPerChecksum != null)
{
if ((blockSize.GetLower() % bytesPerChecksum) != 0)
{
throw new ConfigMerger.ConfigException("Blocksize lower bound must be a multiple of "
+ bytesPerChecksum);
}
if ((blockSize.GetUpper() % bytesPerChecksum) != 0)
{
throw new ConfigMerger.ConfigException("Blocksize upper bound must be a multiple of "
+ bytesPerChecksum);
}
}
@base.Set(ConfigOption.BlockSize.GetCfgOption(), blockSize.ToString());
}
}
{
// set the read size range
Range <long> readSize = null;
try
{
readSize = extractor.GetReadSize(opts.GetValue(ConfigOption.ReadSize.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging read size range"
, e);
}
if (readSize != null)
{
if (readSize.GetLower() > readSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Read size minimum is greater than its maximum"
);
}
if (readSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Read size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.ReadSize.GetCfgOption(), readSize.ToString());
}
}
{
// set the write size range
Range <long> writeSize = null;
try
{
writeSize = extractor.GetWriteSize(opts.GetValue(ConfigOption.WriteSize.GetOpt())
);
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging write size range"
, e);
}
if (writeSize != null)
{
if (writeSize.GetLower() > writeSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Write size minimum is greater than its maximum"
);
}
if (writeSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Write size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.WriteSize.GetCfgOption(), writeSize.ToString());
}
}
{
// set the append size range
Range <long> appendSize = null;
try
{
appendSize = extractor.GetAppendSize(opts.GetValue(ConfigOption.AppendSize.GetOpt
()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging append size range"
, e);
}
if (appendSize != null)
{
if (appendSize.GetLower() > appendSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Append size minimum is greater than its maximum"
);
}
if (appendSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Append size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.AppendSize.GetCfgOption(), appendSize.ToString());
}
}
{
// set the truncate size range
Range <long> truncateSize = null;
try
{
truncateSize = extractor.GetTruncateSize(opts.GetValue(ConfigOption.TruncateSize.
GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging truncate size range"
, e);
}
if (truncateSize != null)
{
if (truncateSize.GetLower() > truncateSize.GetUpper())
{
throw new ConfigMerger.ConfigException("Truncate size minimum is greater than its maximum"
);
}
if (truncateSize.GetLower() < 0)
{
throw new ConfigMerger.ConfigException("Truncate size minimum must be greater than or equal to zero"
);
}
@base.Set(ConfigOption.TruncateSize.GetCfgOption(), truncateSize.ToString());
}
}
{
// set the seed
long seed = null;
try
{
seed = extractor.GetRandomSeed(opts.GetValue(ConfigOption.RandomSeed.GetOpt()));
}
catch (Exception e)
{
throw new ConfigMerger.ConfigException("Error extracting & merging random number seed"
, e);
}
if (seed != null)
{
@base.Set(ConfigOption.RandomSeed.GetCfgOption(), seed.ToString());
}
}
return(@base);
}
/*
* (non-Javadoc)
*
* @see org.apache.hadoop.mapred.Mapper#map(java.lang.Object,
* java.lang.Object, org.apache.hadoop.mapred.OutputCollector,
* org.apache.hadoop.mapred.Reporter)
*/
/// <exception cref="System.IO.IOException"/>
public virtual void Map(object key, object value, OutputCollector <Text, Text> output
, Reporter reporter)
{
// Mapper
LogAndSetStatus(reporter, "Running slive mapper for dummy key " + key + " and dummy value "
+ value);
//Add taskID to randomSeed to deterministically seed rnd.
Random rnd = config.GetRandomSeed() != null ? new Random(this.taskId + config.GetRandomSeed
()) : new Random();
WeightSelector selector = new WeightSelector(config, rnd);
long startTime = Timer.Now();
long opAm = 0;
long sleepOps = 0;
int duration = GetConfig().GetDurationMilliseconds();
Range <long> sleepRange = GetConfig().GetSleepRange();
Operation sleeper = null;
if (sleepRange != null)
{
sleeper = new SleepOp(GetConfig(), rnd);
}
while (Timer.Elapsed(startTime) < duration)
{
try
{
LogAndSetStatus(reporter, "Attempting to select operation #" + (opAm + 1));
int currElapsed = (int)(Timer.Elapsed(startTime));
Operation op = selector.Select(currElapsed, duration);
if (op == null)
{
// no ops left
break;
}
else
{
// got a good op
++opAm;
RunOperation(op, reporter, output, opAm);
}
// do a sleep??
if (sleeper != null)
{
// these don't count against the number of operations
++sleepOps;
RunOperation(sleeper, reporter, output, sleepOps);
}
}
catch (Exception e)
{
LogAndSetStatus(reporter, "Failed at running due to " + StringUtils.StringifyException
(e));
if (GetConfig().ShouldExitOnFirstError())
{
break;
}
}
}
{
// write out any accumulated mapper stats
long timeTaken = Timer.Elapsed(startTime);
OperationOutput opCount = new OperationOutput(OperationOutput.OutputType.Long, OpType
, ReportWriter.OpCount, opAm);
output.Collect(opCount.GetKey(), opCount.GetOutputValue());
OperationOutput overallTime = new OperationOutput(OperationOutput.OutputType.Long
, OpType, ReportWriter.OkTimeTaken, timeTaken);
output.Collect(overallTime.GetKey(), overallTime.GetOutputValue());
LogAndSetStatus(reporter, "Finished " + opAm + " operations in " + timeTaken + " milliseconds"
);
}
}