Exemple #1
0
        /// <summary>
        /// Handles merging all options and verifying from the given command line
        /// output and the given base configuration and returns the merged
        /// configuration
        /// </summary>
        /// <param name="opts">the parsed command line option output</param>
        /// <param name="base">the base configuration to merge with</param>
        /// <returns>the merged configuration</returns>
        /// <exception cref="ConfigException"/>
        /// <exception cref="Org.Apache.Hadoop.FS.Slive.ConfigMerger.ConfigException"/>
        private Configuration HandleOptions(ArgumentParser.ParsedOutput opts, Configuration
                                            @base)
        {
            // ensure variables are overwritten and verified
            ConfigExtractor extractor = new ConfigExtractor(@base);

            {
                // overwrite the map amount and check to ensure > 0
                int mapAmount = null;
                try
                {
                    mapAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Maps.GetOpt()));
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging map amount", e
                                                           );
                }
                if (mapAmount != null)
                {
                    if (mapAmount <= 0)
                    {
                        throw new ConfigMerger.ConfigException("Map amount can not be less than or equal to zero"
                                                               );
                    }
                    @base.Set(ConfigOption.Maps.GetCfgOption(), mapAmount.ToString());
                }
            }
            {
                // overwrite the reducer amount and check to ensure > 0
                int reduceAmount = null;
                try
                {
                    reduceAmount = extractor.GetMapAmount(opts.GetValue(ConfigOption.Reduces.GetOpt()
                                                                        ));
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging reducer amount"
                                                           , e);
                }
                if (reduceAmount != null)
                {
                    if (reduceAmount <= 0)
                    {
                        throw new ConfigMerger.ConfigException("Reducer amount can not be less than or equal to zero"
                                                               );
                    }
                    @base.Set(ConfigOption.Reduces.GetCfgOption(), reduceAmount.ToString());
                }
            }
            {
                // overwrite the duration amount and ensure > 0
                int duration = null;
                try
                {
                    duration = extractor.GetDuration(opts.GetValue(ConfigOption.Duration.GetOpt()));
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging duration", e);
                }
                if (duration != null)
                {
                    if (duration <= 0)
                    {
                        throw new ConfigMerger.ConfigException("Duration can not be less than or equal to zero"
                                                               );
                    }
                    @base.Set(ConfigOption.Duration.GetCfgOption(), duration.ToString());
                }
            }
            {
                // overwrite the operation amount and ensure > 0
                int operationAmount = null;
                try
                {
                    operationAmount = extractor.GetOpCount(opts.GetValue(ConfigOption.Ops.GetOpt()));
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging operation amount"
                                                           , e);
                }
                if (operationAmount != null)
                {
                    if (operationAmount <= 0)
                    {
                        throw new ConfigMerger.ConfigException("Operation amount can not be less than or equal to zero"
                                                               );
                    }
                    @base.Set(ConfigOption.Ops.GetCfgOption(), operationAmount.ToString());
                }
            }
            {
                // overwrite the exit on error setting
                try
                {
                    bool exitOnError = extractor.ShouldExitOnFirstError(opts.GetValue(ConfigOption.ExitOnError
                                                                                      .GetOpt()));
                    @base.SetBoolean(ConfigOption.ExitOnError.GetCfgOption(), exitOnError);
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging exit on error value"
                                                           , e);
                }
            }
            {
                // overwrite the truncate wait setting
                try
                {
                    bool waitOnTruncate = extractor.ShouldWaitOnTruncate(opts.GetValue(ConfigOption.TruncateWait
                                                                                       .GetOpt()));
                    @base.SetBoolean(ConfigOption.TruncateWait.GetCfgOption(), waitOnTruncate);
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging wait on truncate value"
                                                           , e);
                }
            }
            {
                // verify and set file limit and ensure > 0
                int fileAm = null;
                try
                {
                    fileAm = extractor.GetTotalFiles(opts.GetValue(ConfigOption.Files.GetOpt()));
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging total file limit amount"
                                                           , e);
                }
                if (fileAm != null)
                {
                    if (fileAm <= 0)
                    {
                        throw new ConfigMerger.ConfigException("File amount can not be less than or equal to zero"
                                                               );
                    }
                    @base.Set(ConfigOption.Files.GetCfgOption(), fileAm.ToString());
                }
            }
            {
                // set the grid queue to run on
                try
                {
                    string qname = extractor.GetQueueName(opts.GetValue(ConfigOption.QueueName.GetOpt
                                                                            ()));
                    if (qname != null)
                    {
                        @base.Set(ConfigOption.QueueName.GetCfgOption(), qname);
                    }
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging queue name", e
                                                           );
                }
            }
            {
                // verify and set the directory limit and ensure > 0
                int directoryLimit = null;
                try
                {
                    directoryLimit = extractor.GetDirSize(opts.GetValue(ConfigOption.DirSize.GetOpt()
                                                                        ));
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging directory file limit"
                                                           , e);
                }
                if (directoryLimit != null)
                {
                    if (directoryLimit <= 0)
                    {
                        throw new ConfigMerger.ConfigException("Directory file limit can not be less than or equal to zero"
                                                               );
                    }
                    @base.Set(ConfigOption.DirSize.GetCfgOption(), directoryLimit.ToString());
                }
            }
            {
                // set the base directory
                Path basedir = null;
                try
                {
                    basedir = extractor.GetBaseDirectory(opts.GetValue(ConfigOption.BaseDir.GetOpt())
                                                         );
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging base directory"
                                                           , e);
                }
                if (basedir != null)
                {
                    // always ensure in slive dir
                    basedir = new Path(basedir, Constants.BaseDir);
                    @base.Set(ConfigOption.BaseDir.GetCfgOption(), basedir.ToString());
                }
            }
            {
                // set the result file
                string fn = null;
                try
                {
                    fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt()));
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging result file",
                                                           e);
                }
                if (fn != null)
                {
                    @base.Set(ConfigOption.ResultFile.GetCfgOption(), fn);
                }
            }
            {
                string fn = null;
                try
                {
                    fn = extractor.GetResultFile(opts.GetValue(ConfigOption.ResultFile.GetOpt()));
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging result file",
                                                           e);
                }
                if (fn != null)
                {
                    @base.Set(ConfigOption.ResultFile.GetCfgOption(), fn);
                }
            }
            {
                // set the operations
                try
                {
                    @base = HandleOperations(opts, @base, extractor);
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging operations", e
                                                           );
                }
            }
            {
                // set the replication amount range
                Range <short> replicationAm = null;
                try
                {
                    replicationAm = extractor.GetReplication(opts.GetValue(ConfigOption.ReplicationAm
                                                                           .GetOpt()));
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging replication amount range"
                                                           , e);
                }
                if (replicationAm != null)
                {
                    int minRepl = @base.GetInt(Constants.MinReplication, 1);
                    if (replicationAm.GetLower() < minRepl)
                    {
                        throw new ConfigMerger.ConfigException("Replication amount minimum is less than property configured minimum "
                                                               + minRepl);
                    }
                    if (replicationAm.GetLower() > replicationAm.GetUpper())
                    {
                        throw new ConfigMerger.ConfigException("Replication amount minimum is greater than its maximum"
                                                               );
                    }
                    if (replicationAm.GetLower() <= 0)
                    {
                        throw new ConfigMerger.ConfigException("Replication amount minimum must be greater than zero"
                                                               );
                    }
                    @base.Set(ConfigOption.ReplicationAm.GetCfgOption(), replicationAm.ToString());
                }
            }
            {
                // set the sleep range
                Range <long> sleepRange = null;
                try
                {
                    sleepRange = extractor.GetSleepRange(opts.GetValue(ConfigOption.SleepTime.GetOpt(
                                                                           )));
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging sleep size range"
                                                           , e);
                }
                if (sleepRange != null)
                {
                    if (sleepRange.GetLower() > sleepRange.GetUpper())
                    {
                        throw new ConfigMerger.ConfigException("Sleep range minimum is greater than its maximum"
                                                               );
                    }
                    if (sleepRange.GetLower() <= 0)
                    {
                        throw new ConfigMerger.ConfigException("Sleep range minimum must be greater than zero"
                                                               );
                    }
                    @base.Set(ConfigOption.SleepTime.GetCfgOption(), sleepRange.ToString());
                }
            }
            {
                // set the packet size if given
                string pSize = opts.GetValue(ConfigOption.PacketSize.GetOpt());
                if (pSize == null)
                {
                    pSize = ConfigOption.PacketSize.GetDefault();
                }
                if (pSize != null)
                {
                    try
                    {
                        long packetSize = StringUtils.TraditionalBinaryPrefix.String2long(pSize);
                        @base.Set(ConfigOption.PacketSize.GetCfgOption(), packetSize.ToString());
                    }
                    catch (Exception e)
                    {
                        throw new ConfigMerger.ConfigException("Error extracting & merging write packet size"
                                                               , e);
                    }
                }
            }
            {
                // set the block size range
                Range <long> blockSize = null;
                try
                {
                    blockSize = extractor.GetBlockSize(opts.GetValue(ConfigOption.BlockSize.GetOpt())
                                                       );
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging block size range"
                                                           , e);
                }
                if (blockSize != null)
                {
                    if (blockSize.GetLower() > blockSize.GetUpper())
                    {
                        throw new ConfigMerger.ConfigException("Block size minimum is greater than its maximum"
                                                               );
                    }
                    if (blockSize.GetLower() <= 0)
                    {
                        throw new ConfigMerger.ConfigException("Block size minimum must be greater than zero"
                                                               );
                    }
                    // ensure block size is a multiple of BYTES_PER_CHECKSUM
                    // if a value is set in the configuration
                    long bytesPerChecksum = extractor.GetByteCheckSum();
                    if (bytesPerChecksum != null)
                    {
                        if ((blockSize.GetLower() % bytesPerChecksum) != 0)
                        {
                            throw new ConfigMerger.ConfigException("Blocksize lower bound must be a multiple of "
                                                                   + bytesPerChecksum);
                        }
                        if ((blockSize.GetUpper() % bytesPerChecksum) != 0)
                        {
                            throw new ConfigMerger.ConfigException("Blocksize upper bound must be a multiple of "
                                                                   + bytesPerChecksum);
                        }
                    }
                    @base.Set(ConfigOption.BlockSize.GetCfgOption(), blockSize.ToString());
                }
            }
            {
                // set the read size range
                Range <long> readSize = null;
                try
                {
                    readSize = extractor.GetReadSize(opts.GetValue(ConfigOption.ReadSize.GetOpt()));
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging read size range"
                                                           , e);
                }
                if (readSize != null)
                {
                    if (readSize.GetLower() > readSize.GetUpper())
                    {
                        throw new ConfigMerger.ConfigException("Read size minimum is greater than its maximum"
                                                               );
                    }
                    if (readSize.GetLower() < 0)
                    {
                        throw new ConfigMerger.ConfigException("Read size minimum must be greater than or equal to zero"
                                                               );
                    }
                    @base.Set(ConfigOption.ReadSize.GetCfgOption(), readSize.ToString());
                }
            }
            {
                // set the write size range
                Range <long> writeSize = null;
                try
                {
                    writeSize = extractor.GetWriteSize(opts.GetValue(ConfigOption.WriteSize.GetOpt())
                                                       );
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging write size range"
                                                           , e);
                }
                if (writeSize != null)
                {
                    if (writeSize.GetLower() > writeSize.GetUpper())
                    {
                        throw new ConfigMerger.ConfigException("Write size minimum is greater than its maximum"
                                                               );
                    }
                    if (writeSize.GetLower() < 0)
                    {
                        throw new ConfigMerger.ConfigException("Write size minimum must be greater than or equal to zero"
                                                               );
                    }
                    @base.Set(ConfigOption.WriteSize.GetCfgOption(), writeSize.ToString());
                }
            }
            {
                // set the append size range
                Range <long> appendSize = null;
                try
                {
                    appendSize = extractor.GetAppendSize(opts.GetValue(ConfigOption.AppendSize.GetOpt
                                                                           ()));
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging append size range"
                                                           , e);
                }
                if (appendSize != null)
                {
                    if (appendSize.GetLower() > appendSize.GetUpper())
                    {
                        throw new ConfigMerger.ConfigException("Append size minimum is greater than its maximum"
                                                               );
                    }
                    if (appendSize.GetLower() < 0)
                    {
                        throw new ConfigMerger.ConfigException("Append size minimum must be greater than or equal to zero"
                                                               );
                    }
                    @base.Set(ConfigOption.AppendSize.GetCfgOption(), appendSize.ToString());
                }
            }
            {
                // set the truncate size range
                Range <long> truncateSize = null;
                try
                {
                    truncateSize = extractor.GetTruncateSize(opts.GetValue(ConfigOption.TruncateSize.
                                                                           GetOpt()));
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging truncate size range"
                                                           , e);
                }
                if (truncateSize != null)
                {
                    if (truncateSize.GetLower() > truncateSize.GetUpper())
                    {
                        throw new ConfigMerger.ConfigException("Truncate size minimum is greater than its maximum"
                                                               );
                    }
                    if (truncateSize.GetLower() < 0)
                    {
                        throw new ConfigMerger.ConfigException("Truncate size minimum must be greater than or equal to zero"
                                                               );
                    }
                    @base.Set(ConfigOption.TruncateSize.GetCfgOption(), truncateSize.ToString());
                }
            }
            {
                // set the seed
                long seed = null;
                try
                {
                    seed = extractor.GetRandomSeed(opts.GetValue(ConfigOption.RandomSeed.GetOpt()));
                }
                catch (Exception e)
                {
                    throw new ConfigMerger.ConfigException("Error extracting & merging random number seed"
                                                           , e);
                }
                if (seed != null)
                {
                    @base.Set(ConfigOption.RandomSeed.GetCfgOption(), seed.ToString());
                }
            }
            return(@base);
        }
Exemple #2
0
        /*
         * (non-Javadoc)
         *
         * @see org.apache.hadoop.mapred.Mapper#map(java.lang.Object,
         * java.lang.Object, org.apache.hadoop.mapred.OutputCollector,
         * org.apache.hadoop.mapred.Reporter)
         */
        /// <exception cref="System.IO.IOException"/>
        public virtual void Map(object key, object value, OutputCollector <Text, Text> output
                                , Reporter reporter)
        {
            // Mapper
            LogAndSetStatus(reporter, "Running slive mapper for dummy key " + key + " and dummy value "
                            + value);
            //Add taskID to randomSeed to deterministically seed rnd.
            Random rnd = config.GetRandomSeed() != null ? new Random(this.taskId + config.GetRandomSeed
                                                                         ()) : new Random();
            WeightSelector selector   = new WeightSelector(config, rnd);
            long           startTime  = Timer.Now();
            long           opAm       = 0;
            long           sleepOps   = 0;
            int            duration   = GetConfig().GetDurationMilliseconds();
            Range <long>   sleepRange = GetConfig().GetSleepRange();
            Operation      sleeper    = null;

            if (sleepRange != null)
            {
                sleeper = new SleepOp(GetConfig(), rnd);
            }
            while (Timer.Elapsed(startTime) < duration)
            {
                try
                {
                    LogAndSetStatus(reporter, "Attempting to select operation #" + (opAm + 1));
                    int       currElapsed = (int)(Timer.Elapsed(startTime));
                    Operation op          = selector.Select(currElapsed, duration);
                    if (op == null)
                    {
                        // no ops left
                        break;
                    }
                    else
                    {
                        // got a good op
                        ++opAm;
                        RunOperation(op, reporter, output, opAm);
                    }
                    // do a sleep??
                    if (sleeper != null)
                    {
                        // these don't count against the number of operations
                        ++sleepOps;
                        RunOperation(sleeper, reporter, output, sleepOps);
                    }
                }
                catch (Exception e)
                {
                    LogAndSetStatus(reporter, "Failed at running due to " + StringUtils.StringifyException
                                        (e));
                    if (GetConfig().ShouldExitOnFirstError())
                    {
                        break;
                    }
                }
            }
            {
                // write out any accumulated mapper stats
                long            timeTaken = Timer.Elapsed(startTime);
                OperationOutput opCount   = new OperationOutput(OperationOutput.OutputType.Long, OpType
                                                                , ReportWriter.OpCount, opAm);
                output.Collect(opCount.GetKey(), opCount.GetOutputValue());
                OperationOutput overallTime = new OperationOutput(OperationOutput.OutputType.Long
                                                                  , OpType, ReportWriter.OkTimeTaken, timeTaken);
                output.Collect(overallTime.GetKey(), overallTime.GetOutputValue());
                LogAndSetStatus(reporter, "Finished " + opAm + " operations in " + timeTaken + " milliseconds"
                                );
            }
        }