/// <summary> /// Handles merging all options and verifying from the given command line /// output and the given base configuration and returns the merged /// configuration /// </summary> /// <param name="opts">the parsed command line option output</param> /// <param name="base">the base configuration to merge with</param> /// <returns>the merged configuration</returns> /// <exception cref="ConfigException"/> /// <exception cref="Org.Apache.Hadoop.FS.Slive.ConfigMerger.ConfigException"/> private Configuration HandleOptions(ArgumentParser.ParsedOutput opts, Configuration @base) { // ensure variables are overwritten and verified ConfigExtractor extractor = new ConfigExtractor(@base); { // overwrite the map amount and check to ensure > 0 int mapAmount = null; try { mapAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Maps.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging map amount", e ); } if (mapAmount != null) { if (mapAmount <= 0) { throw new ConfigMerger.ConfigException("Map amount can not be less than or equal to zero" ); } @base.Set(ConfigOption.Maps.GetCfgOption(), mapAmount.ToString()); } } { // overwrite the reducer amount and check to ensure > 0 int reduceAmount = null; try { reduceAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Reduces.GetOpt() )); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging reducer amount" , e); } if (reduceAmount != null) { if (reduceAmount <= 0) { throw new ConfigMerger.ConfigException("Reducer amount can not be less than or equal to zero" ); } @base.Set(ConfigOption.Reduces.GetCfgOption(), reduceAmount.ToString()); } } { // overwrite the duration amount and ensure > 0 int duration = null; try { duration = extractor.GetDuration(opts.GetValue(ConfigOption.Duration.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging duration", e); } if (duration != null) { if (duration <= 0) { throw new ConfigMerger.ConfigException("Duration can not be less than or equal to zero" ); } @base.Set(ConfigOption.Duration.GetCfgOption(), duration.ToString()); } } { // overwrite the operation amount and ensure > 0 int operationAmount = null; try { operationAmount = extractor.GetOpCount(opts.GetValue(ConfigOption.Ops.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging operation amount" , e); } if (operationAmount != null) { if (operationAmount <= 0) { throw new ConfigMerger.ConfigException("Operation amount can not be less than or equal to zero" ); } @base.Set(ConfigOption.Ops.GetCfgOption(), operationAmount.ToString()); } } { // overwrite the exit on error setting try { bool exitOnError = extractor.ShouldExitOnFirstError(opts.GetValue(ConfigOption.ExitOnError .GetOpt())); @base.SetBoolean(ConfigOption.ExitOnError.GetCfgOption(), exitOnError); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging exit on error value" , e); } } { // overwrite the truncate wait setting try { bool waitOnTruncate = extractor.ShouldWaitOnTruncate(opts.GetValue(ConfigOption.TruncateWait .GetOpt())); @base.SetBoolean(ConfigOption.TruncateWait.GetCfgOption(), waitOnTruncate); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging wait on truncate value" , e); } } { // verify and set file limit and ensure > 0 int fileAm = null; try { fileAm = extractor.GetTotalFiles(opts.GetValue(ConfigOption.Files.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging total file limit amount" , e); } if (fileAm != null) { if (fileAm <= 0) { throw new ConfigMerger.ConfigException("File amount can not be less than or equal to zero" ); } @base.Set(ConfigOption.Files.GetCfgOption(), fileAm.ToString()); } } { // set the grid queue to run on try { string qname = extractor.GetQueueName(opts.GetValue(ConfigOption.QueueName.GetOpt ())); if (qname != null) { @base.Set(ConfigOption.QueueName.GetCfgOption(), qname); } } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging queue name", e ); } } { // verify and set the directory limit and ensure > 0 int directoryLimit = null; try { directoryLimit = extractor.GetDirSize(opts.GetValue(ConfigOption.DirSize.GetOpt() )); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging directory file limit" , e); } if (directoryLimit != null) { if (directoryLimit <= 0) { throw new ConfigMerger.ConfigException("Directory file limit can not be less than or equal to zero" ); } @base.Set(ConfigOption.DirSize.GetCfgOption(), directoryLimit.ToString()); } } { // set the base directory Path basedir = null; try { basedir = extractor.GetBaseDirectory(opts.GetValue(ConfigOption.BaseDir.GetOpt()) ); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging base directory" , e); } if (basedir != null) { // always ensure in slive dir basedir = new Path(basedir, Constants.BaseDir); @base.Set(ConfigOption.BaseDir.GetCfgOption(), basedir.ToString()); } } { // set the result file string fn = null; try { fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging result file", e); } if (fn != null) { @base.Set(ConfigOption.ResultFile.GetCfgOption(), fn); } } { string fn = null; try { fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging result file", e); } if (fn != null) { @base.Set(ConfigOption.ResultFile.GetCfgOption(), fn); } } { // set the operations try { @base = HandleOperations(opts, @base, extractor); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging operations", e ); } } { // set the replication amount range Range <short> replicationAm = null; try { replicationAm = extractor.GetReplication(opts.GetValue(ConfigOption.ReplicationAm .GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging replication amount range" , e); } if (replicationAm != null) { int minRepl = @base.GetInt(Constants.MinReplication, 1); if (replicationAm.GetLower() < minRepl) { throw new ConfigMerger.ConfigException("Replication amount minimum is less than property configured minimum " + minRepl); } if (replicationAm.GetLower() > replicationAm.GetUpper()) { throw new ConfigMerger.ConfigException("Replication amount minimum is greater than its maximum" ); } if (replicationAm.GetLower() <= 0) { throw new ConfigMerger.ConfigException("Replication amount minimum must be greater than zero" ); } @base.Set(ConfigOption.ReplicationAm.GetCfgOption(), replicationAm.ToString()); } } { // set the sleep range Range <long> sleepRange = null; try { sleepRange = extractor.GetSleepRange(opts.GetValue(ConfigOption.SleepTime.GetOpt( ))); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging sleep size range" , e); } if (sleepRange != null) { if (sleepRange.GetLower() > sleepRange.GetUpper()) { throw new ConfigMerger.ConfigException("Sleep range minimum is greater than its maximum" ); } if (sleepRange.GetLower() <= 0) { throw new ConfigMerger.ConfigException("Sleep range minimum must be greater than zero" ); } @base.Set(ConfigOption.SleepTime.GetCfgOption(), sleepRange.ToString()); } } { // set the packet size if given string pSize = opts.GetValue(ConfigOption.PacketSize.GetOpt()); if (pSize == null) { pSize = ConfigOption.PacketSize.GetDefault(); } if (pSize != null) { try { long packetSize = StringUtils.TraditionalBinaryPrefix.String2long(pSize); @base.Set(ConfigOption.PacketSize.GetCfgOption(), packetSize.ToString()); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging write packet size" , e); } } } { // set the block size range Range <long> blockSize = null; try { blockSize = extractor.GetBlockSize(opts.GetValue(ConfigOption.BlockSize.GetOpt()) ); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging block size range" , e); } if (blockSize != null) { if (blockSize.GetLower() > blockSize.GetUpper()) { throw new ConfigMerger.ConfigException("Block size minimum is greater than its maximum" ); } if (blockSize.GetLower() <= 0) { throw new ConfigMerger.ConfigException("Block size minimum must be greater than zero" ); } // ensure block size is a multiple of BYTES_PER_CHECKSUM // if a value is set in the configuration long bytesPerChecksum = extractor.GetByteCheckSum(); if (bytesPerChecksum != null) { if ((blockSize.GetLower() % bytesPerChecksum) != 0) { throw new ConfigMerger.ConfigException("Blocksize lower bound must be a multiple of " + bytesPerChecksum); } if ((blockSize.GetUpper() % bytesPerChecksum) != 0) { throw new ConfigMerger.ConfigException("Blocksize upper bound must be a multiple of " + bytesPerChecksum); } } @base.Set(ConfigOption.BlockSize.GetCfgOption(), blockSize.ToString()); } } { // set the read size range Range <long> readSize = null; try { readSize = extractor.GetReadSize(opts.GetValue(ConfigOption.ReadSize.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging read size range" , e); } if (readSize != null) { if (readSize.GetLower() > readSize.GetUpper()) { throw new ConfigMerger.ConfigException("Read size minimum is greater than its maximum" ); } if (readSize.GetLower() < 0) { throw new ConfigMerger.ConfigException("Read size minimum must be greater than or equal to zero" ); } @base.Set(ConfigOption.ReadSize.GetCfgOption(), readSize.ToString()); } } { // set the write size range Range <long> writeSize = null; try { writeSize = extractor.GetWriteSize(opts.GetValue(ConfigOption.WriteSize.GetOpt()) ); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging write size range" , e); } if (writeSize != null) { if (writeSize.GetLower() > writeSize.GetUpper()) { throw new ConfigMerger.ConfigException("Write size minimum is greater than its maximum" ); } if (writeSize.GetLower() < 0) { throw new ConfigMerger.ConfigException("Write size minimum must be greater than or equal to zero" ); } @base.Set(ConfigOption.WriteSize.GetCfgOption(), writeSize.ToString()); } } { // set the append size range Range <long> appendSize = null; try { appendSize = extractor.GetAppendSize(opts.GetValue(ConfigOption.AppendSize.GetOpt ())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging append size range" , e); } if (appendSize != null) { if (appendSize.GetLower() > appendSize.GetUpper()) { throw new ConfigMerger.ConfigException("Append size minimum is greater than its maximum" ); } if (appendSize.GetLower() < 0) { throw new ConfigMerger.ConfigException("Append size minimum must be greater than or equal to zero" ); } @base.Set(ConfigOption.AppendSize.GetCfgOption(), appendSize.ToString()); } } { // set the truncate size range Range <long> truncateSize = null; try { truncateSize = extractor.GetTruncateSize(opts.GetValue(ConfigOption.TruncateSize. GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging truncate size range" , e); } if (truncateSize != null) { if (truncateSize.GetLower() > truncateSize.GetUpper()) { throw new ConfigMerger.ConfigException("Truncate size minimum is greater than its maximum" ); } if (truncateSize.GetLower() < 0) { throw new ConfigMerger.ConfigException("Truncate size minimum must be greater than or equal to zero" ); } @base.Set(ConfigOption.TruncateSize.GetCfgOption(), truncateSize.ToString()); } } { // set the seed long seed = null; try { seed = extractor.GetRandomSeed(opts.GetValue(ConfigOption.RandomSeed.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging random number seed" , e); } if (seed != null) { @base.Set(ConfigOption.RandomSeed.GetCfgOption(), seed.ToString()); } } return(@base); }
/* * (non-Javadoc) * * @see org.apache.hadoop.mapred.Mapper#map(java.lang.Object, * java.lang.Object, org.apache.hadoop.mapred.OutputCollector, * org.apache.hadoop.mapred.Reporter) */ /// <exception cref="System.IO.IOException"/> public virtual void Map(object key, object value, OutputCollector <Text, Text> output , Reporter reporter) { // Mapper LogAndSetStatus(reporter, "Running slive mapper for dummy key " + key + " and dummy value " + value); //Add taskID to randomSeed to deterministically seed rnd. Random rnd = config.GetRandomSeed() != null ? new Random(this.taskId + config.GetRandomSeed ()) : new Random(); WeightSelector selector = new WeightSelector(config, rnd); long startTime = Timer.Now(); long opAm = 0; long sleepOps = 0; int duration = GetConfig().GetDurationMilliseconds(); Range <long> sleepRange = GetConfig().GetSleepRange(); Operation sleeper = null; if (sleepRange != null) { sleeper = new SleepOp(GetConfig(), rnd); } while (Timer.Elapsed(startTime) < duration) { try { LogAndSetStatus(reporter, "Attempting to select operation #" + (opAm + 1)); int currElapsed = (int)(Timer.Elapsed(startTime)); Operation op = selector.Select(currElapsed, duration); if (op == null) { // no ops left break; } else { // got a good op ++opAm; RunOperation(op, reporter, output, opAm); } // do a sleep?? if (sleeper != null) { // these don't count against the number of operations ++sleepOps; RunOperation(sleeper, reporter, output, sleepOps); } } catch (Exception e) { LogAndSetStatus(reporter, "Failed at running due to " + StringUtils.StringifyException (e)); if (GetConfig().ShouldExitOnFirstError()) { break; } } } { // write out any accumulated mapper stats long timeTaken = Timer.Elapsed(startTime); OperationOutput opCount = new OperationOutput(OperationOutput.OutputType.Long, OpType , ReportWriter.OpCount, opAm); output.Collect(opCount.GetKey(), opCount.GetOutputValue()); OperationOutput overallTime = new OperationOutput(OperationOutput.OutputType.Long , OpType, ReportWriter.OkTimeTaken, timeTaken); output.Collect(overallTime.GetKey(), overallTime.GetOutputValue()); LogAndSetStatus(reporter, "Finished " + opAm + " operations in " + timeTaken + " milliseconds" ); } }