public virtual void TestArguments() { ConfigExtractor extractor = GetTestConfig(true); NUnit.Framework.Assert.AreEqual(extractor.GetOpCount(), Constants.OperationType.Values ().Length); NUnit.Framework.Assert.AreEqual(extractor.GetMapAmount(), 2); NUnit.Framework.Assert.AreEqual(extractor.GetReducerAmount(), 2); Range <long> apRange = extractor.GetAppendSize(); NUnit.Framework.Assert.AreEqual(apRange.GetLower(), Constants.Megabytes * 1); NUnit.Framework.Assert.AreEqual(apRange.GetUpper(), Constants.Megabytes * 2); Range <long> wRange = extractor.GetWriteSize(); NUnit.Framework.Assert.AreEqual(wRange.GetLower(), Constants.Megabytes * 1); NUnit.Framework.Assert.AreEqual(wRange.GetUpper(), Constants.Megabytes * 2); Range <long> trRange = extractor.GetTruncateSize(); NUnit.Framework.Assert.AreEqual(trRange.GetLower(), 0); NUnit.Framework.Assert.AreEqual(trRange.GetUpper(), Constants.Megabytes * 1); Range <long> bRange = extractor.GetBlockSize(); NUnit.Framework.Assert.AreEqual(bRange.GetLower(), Constants.Megabytes * 1); NUnit.Framework.Assert.AreEqual(bRange.GetUpper(), Constants.Megabytes * 2); string resfile = extractor.GetResultFile(); NUnit.Framework.Assert.AreEqual(resfile, GetResultFile().ToString()); int durationMs = extractor.GetDurationMilliseconds(); NUnit.Framework.Assert.AreEqual(durationMs, 10 * 1000); }
/// <summary> /// Sets up the operation using the given configuration by setting up the /// number of operations to perform (and how many are left) and setting up the /// operation objects to be used throughout selection. /// </summary> /// <param name="cfg">ConfigExtractor.</param> private void ConfigureOperations(ConfigExtractor cfg) { operations = new SortedDictionary <Constants.OperationType, WeightSelector.OperationInfo >(); IDictionary <Constants.OperationType, OperationData> opinfo = cfg.GetOperations(); int totalAm = cfg.GetOpCount(); int opsLeft = totalAm; NumberFormat formatter = Formatter.GetPercentFormatter(); foreach (Constants.OperationType type in opinfo.Keys) { OperationData opData = opinfo[type]; WeightSelector.OperationInfo info = new WeightSelector.OperationInfo(); info.distribution = opData.GetDistribution(); int amLeft = DetermineHowMany(totalAm, opData, type); opsLeft -= amLeft; Log.Info(type.ToString() + " has " + amLeft + " initial operations out of " + totalAm + " for its ratio " + formatter.Format(opData.GetPercent())); info.amountLeft = amLeft; Operation op = factory.GetOperation(type); // wrap operation in finalizer so that amount left gets decrements when // its done if (op != null) { ObserveableOp.Observer fn = new _Observer_138(this, type); info.operation = new ObserveableOp(op, fn); operations[type] = info; } } if (opsLeft > 0) { Log.Info(opsLeft + " left over operations found (due to inability to support partial operations)" ); } }
/// <summary> /// Handles merging all options and verifying from the given command line /// output and the given base configuration and returns the merged /// configuration /// </summary> /// <param name="opts">the parsed command line option output</param> /// <param name="base">the base configuration to merge with</param> /// <returns>the merged configuration</returns> /// <exception cref="ConfigException"/> /// <exception cref="Org.Apache.Hadoop.FS.Slive.ConfigMerger.ConfigException"/> private Configuration HandleOptions(ArgumentParser.ParsedOutput opts, Configuration @base) { // ensure variables are overwritten and verified ConfigExtractor extractor = new ConfigExtractor(@base); { // overwrite the map amount and check to ensure > 0 int mapAmount = null; try { mapAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Maps.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging map amount", e ); } if (mapAmount != null) { if (mapAmount <= 0) { throw new ConfigMerger.ConfigException("Map amount can not be less than or equal to zero" ); } @base.Set(ConfigOption.Maps.GetCfgOption(), mapAmount.ToString()); } } { // overwrite the reducer amount and check to ensure > 0 int reduceAmount = null; try { reduceAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Reduces.GetOpt() )); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging reducer amount" , e); } if (reduceAmount != null) { if (reduceAmount <= 0) { throw new ConfigMerger.ConfigException("Reducer amount can not be less than or equal to zero" ); } @base.Set(ConfigOption.Reduces.GetCfgOption(), reduceAmount.ToString()); } } { // overwrite the duration amount and ensure > 0 int duration = null; try { duration = extractor.GetDuration(opts.GetValue(ConfigOption.Duration.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging duration", e); } if (duration != null) { if (duration <= 0) { throw new ConfigMerger.ConfigException("Duration can not be less than or equal to zero" ); } @base.Set(ConfigOption.Duration.GetCfgOption(), duration.ToString()); } } { // overwrite the operation amount and ensure > 0 int operationAmount = null; try { operationAmount = extractor.GetOpCount(opts.GetValue(ConfigOption.Ops.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging operation amount" , e); } if (operationAmount != null) { if (operationAmount <= 0) { throw new ConfigMerger.ConfigException("Operation amount can not be less than or equal to zero" ); } @base.Set(ConfigOption.Ops.GetCfgOption(), operationAmount.ToString()); } } { // overwrite the exit on error setting try { bool exitOnError = extractor.ShouldExitOnFirstError(opts.GetValue(ConfigOption.ExitOnError .GetOpt())); @base.SetBoolean(ConfigOption.ExitOnError.GetCfgOption(), exitOnError); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging exit on error value" , e); } } { // overwrite the truncate wait setting try { bool waitOnTruncate = extractor.ShouldWaitOnTruncate(opts.GetValue(ConfigOption.TruncateWait .GetOpt())); @base.SetBoolean(ConfigOption.TruncateWait.GetCfgOption(), waitOnTruncate); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging wait on truncate value" , e); } } { // verify and set file limit and ensure > 0 int fileAm = null; try { fileAm = extractor.GetTotalFiles(opts.GetValue(ConfigOption.Files.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging total file limit amount" , e); } if (fileAm != null) { if (fileAm <= 0) { throw new ConfigMerger.ConfigException("File amount can not be less than or equal to zero" ); } @base.Set(ConfigOption.Files.GetCfgOption(), fileAm.ToString()); } } { // set the grid queue to run on try { string qname = extractor.GetQueueName(opts.GetValue(ConfigOption.QueueName.GetOpt ())); if (qname != null) { @base.Set(ConfigOption.QueueName.GetCfgOption(), qname); } } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging queue name", e ); } } { // verify and set the directory limit and ensure > 0 int directoryLimit = null; try { directoryLimit = extractor.GetDirSize(opts.GetValue(ConfigOption.DirSize.GetOpt() )); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging directory file limit" , e); } if (directoryLimit != null) { if (directoryLimit <= 0) { throw new ConfigMerger.ConfigException("Directory file limit can not be less than or equal to zero" ); } @base.Set(ConfigOption.DirSize.GetCfgOption(), directoryLimit.ToString()); } } { // set the base directory Path basedir = null; try { basedir = extractor.GetBaseDirectory(opts.GetValue(ConfigOption.BaseDir.GetOpt()) ); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging base directory" , e); } if (basedir != null) { // always ensure in slive dir basedir = new Path(basedir, Constants.BaseDir); @base.Set(ConfigOption.BaseDir.GetCfgOption(), basedir.ToString()); } } { // set the result file string fn = null; try { fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging result file", e); } if (fn != null) { @base.Set(ConfigOption.ResultFile.GetCfgOption(), fn); } } { string fn = null; try { fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging result file", e); } if (fn != null) { @base.Set(ConfigOption.ResultFile.GetCfgOption(), fn); } } { // set the operations try { @base = HandleOperations(opts, @base, extractor); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging operations", e ); } } { // set the replication amount range Range <short> replicationAm = null; try { replicationAm = extractor.GetReplication(opts.GetValue(ConfigOption.ReplicationAm .GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging replication amount range" , e); } if (replicationAm != null) { int minRepl = @base.GetInt(Constants.MinReplication, 1); if (replicationAm.GetLower() < minRepl) { throw new ConfigMerger.ConfigException("Replication amount minimum is less than property configured minimum " + minRepl); } if (replicationAm.GetLower() > replicationAm.GetUpper()) { throw new ConfigMerger.ConfigException("Replication amount minimum is greater than its maximum" ); } if (replicationAm.GetLower() <= 0) { throw new ConfigMerger.ConfigException("Replication amount minimum must be greater than zero" ); } @base.Set(ConfigOption.ReplicationAm.GetCfgOption(), replicationAm.ToString()); } } { // set the sleep range Range <long> sleepRange = null; try { sleepRange = extractor.GetSleepRange(opts.GetValue(ConfigOption.SleepTime.GetOpt( ))); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging sleep size range" , e); } if (sleepRange != null) { if (sleepRange.GetLower() > sleepRange.GetUpper()) { throw new ConfigMerger.ConfigException("Sleep range minimum is greater than its maximum" ); } if (sleepRange.GetLower() <= 0) { throw new ConfigMerger.ConfigException("Sleep range minimum must be greater than zero" ); } @base.Set(ConfigOption.SleepTime.GetCfgOption(), sleepRange.ToString()); } } { // set the packet size if given string pSize = opts.GetValue(ConfigOption.PacketSize.GetOpt()); if (pSize == null) { pSize = ConfigOption.PacketSize.GetDefault(); } if (pSize != null) { try { long packetSize = StringUtils.TraditionalBinaryPrefix.String2long(pSize); @base.Set(ConfigOption.PacketSize.GetCfgOption(), packetSize.ToString()); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging write packet size" , e); } } } { // set the block size range Range <long> blockSize = null; try { blockSize = extractor.GetBlockSize(opts.GetValue(ConfigOption.BlockSize.GetOpt()) ); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging block size range" , e); } if (blockSize != null) { if (blockSize.GetLower() > blockSize.GetUpper()) { throw new ConfigMerger.ConfigException("Block size minimum is greater than its maximum" ); } if (blockSize.GetLower() <= 0) { throw new ConfigMerger.ConfigException("Block size minimum must be greater than zero" ); } // ensure block size is a multiple of BYTES_PER_CHECKSUM // if a value is set in the configuration long bytesPerChecksum = extractor.GetByteCheckSum(); if (bytesPerChecksum != null) { if ((blockSize.GetLower() % bytesPerChecksum) != 0) { throw new ConfigMerger.ConfigException("Blocksize lower bound must be a multiple of " + bytesPerChecksum); } if ((blockSize.GetUpper() % bytesPerChecksum) != 0) { throw new ConfigMerger.ConfigException("Blocksize upper bound must be a multiple of " + bytesPerChecksum); } } @base.Set(ConfigOption.BlockSize.GetCfgOption(), blockSize.ToString()); } } { // set the read size range Range <long> readSize = null; try { readSize = extractor.GetReadSize(opts.GetValue(ConfigOption.ReadSize.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging read size range" , e); } if (readSize != null) { if (readSize.GetLower() > readSize.GetUpper()) { throw new ConfigMerger.ConfigException("Read size minimum is greater than its maximum" ); } if (readSize.GetLower() < 0) { throw new ConfigMerger.ConfigException("Read size minimum must be greater than or equal to zero" ); } @base.Set(ConfigOption.ReadSize.GetCfgOption(), readSize.ToString()); } } { // set the write size range Range <long> writeSize = null; try { writeSize = extractor.GetWriteSize(opts.GetValue(ConfigOption.WriteSize.GetOpt()) ); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging write size range" , e); } if (writeSize != null) { if (writeSize.GetLower() > writeSize.GetUpper()) { throw new ConfigMerger.ConfigException("Write size minimum is greater than its maximum" ); } if (writeSize.GetLower() < 0) { throw new ConfigMerger.ConfigException("Write size minimum must be greater than or equal to zero" ); } @base.Set(ConfigOption.WriteSize.GetCfgOption(), writeSize.ToString()); } } { // set the append size range Range <long> appendSize = null; try { appendSize = extractor.GetAppendSize(opts.GetValue(ConfigOption.AppendSize.GetOpt ())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging append size range" , e); } if (appendSize != null) { if (appendSize.GetLower() > appendSize.GetUpper()) { throw new ConfigMerger.ConfigException("Append size minimum is greater than its maximum" ); } if (appendSize.GetLower() < 0) { throw new ConfigMerger.ConfigException("Append size minimum must be greater than or equal to zero" ); } @base.Set(ConfigOption.AppendSize.GetCfgOption(), appendSize.ToString()); } } { // set the truncate size range Range <long> truncateSize = null; try { truncateSize = extractor.GetTruncateSize(opts.GetValue(ConfigOption.TruncateSize. GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging truncate size range" , e); } if (truncateSize != null) { if (truncateSize.GetLower() > truncateSize.GetUpper()) { throw new ConfigMerger.ConfigException("Truncate size minimum is greater than its maximum" ); } if (truncateSize.GetLower() < 0) { throw new ConfigMerger.ConfigException("Truncate size minimum must be greater than or equal to zero" ); } @base.Set(ConfigOption.TruncateSize.GetCfgOption(), truncateSize.ToString()); } } { // set the seed long seed = null; try { seed = extractor.GetRandomSeed(opts.GetValue(ConfigOption.RandomSeed.GetOpt())); } catch (Exception e) { throw new ConfigMerger.ConfigException("Error extracting & merging random number seed" , e); } if (seed != null) { @base.Set(ConfigOption.RandomSeed.GetCfgOption(), seed.ToString()); } } return(@base); }