Exemplo n.º 1
0
        // weights.put(Distribution.BEG, new BeginWeight());
        // weights.put(Distribution.END, new EndWeight());
        // weights.put(Distribution.MID, new MidWeight());
        /// <summary>Determines how many initial operations a given operation data should have
        ///     </summary>
        /// <param name="totalAm">the total amount of operations allowed</param>
        /// <param name="opData">the given operation information (with a valid percentage &gt;= 0)
        ///     </param>
        /// <returns>the number of items to allow to run</returns>
        /// <exception cref="System.ArgumentException">if negative operations are determined</exception>
        internal static int DetermineHowMany(int totalAm, OperationData opData, Constants.OperationType
                                             type)
        {
            if (totalAm <= 0)
            {
                return(0);
            }
            int amLeft = (int)Math.Floor(opData.GetPercent() * totalAm);

            if (amLeft < 0)
            {
                throw new ArgumentException("Invalid amount " + amLeft + " determined for operation type "
                                            + type.ToString());
            }
            return(amLeft);
        }
Exemplo n.º 2
0
        /// <returns>
        /// the map of operations to perform using config (percent may be null
        /// if unspecified)
        /// </returns>
        internal virtual IDictionary <Constants.OperationType, OperationData> GetOperations
            ()
        {
            IDictionary <Constants.OperationType, OperationData> operations = new Dictionary <Constants.OperationType
                                                                                              , OperationData>();

            foreach (Constants.OperationType type in Constants.OperationType.Values())
            {
                string opname  = type.LowerName();
                string keyname = string.Format(Constants.Op, opname);
                string kval    = config.Get(keyname);
                if (kval == null)
                {
                    continue;
                }
                operations[type] = new OperationData(kval);
            }
            return(operations);
        }
Exemplo n.º 3
0
        /// <summary>Dumps out the given options for the given config extractor</summary>
        /// <param name="cfg">the config to write to the log</param>
        internal static void DumpOptions(Org.Apache.Hadoop.FS.Slive.ConfigExtractor cfg)
        {
            if (cfg == null)
            {
                return;
            }
            Log.Info("Base directory = " + cfg.GetBaseDirectory());
            Log.Info("Data directory = " + cfg.GetDataPath());
            Log.Info("Output directory = " + cfg.GetOutputPath());
            Log.Info("Result file = " + cfg.GetResultFile());
            Log.Info("Grid queue = " + cfg.GetQueueName());
            Log.Info("Should exit on first error = " + cfg.ShouldExitOnFirstError());
            {
                string duration = "Duration = ";
                if (cfg.GetDurationMilliseconds() == int.MaxValue)
                {
                    duration += "unlimited";
                }
                else
                {
                    duration += cfg.GetDurationMilliseconds() + " milliseconds";
                }
                Log.Info(duration);
            }
            Log.Info("Map amount = " + cfg.GetMapAmount());
            Log.Info("Reducer amount = " + cfg.GetReducerAmount());
            Log.Info("Operation amount = " + cfg.GetOpCount());
            Log.Info("Total file limit = " + cfg.GetTotalFiles());
            Log.Info("Total dir file limit = " + cfg.GetDirSize());
            {
                string read = "Read size = ";
                if (cfg.ShouldReadFullFile())
                {
                    read += "entire file";
                }
                else
                {
                    read += cfg.GetReadSize() + " bytes";
                }
                Log.Info(read);
            }
            {
                string write = "Write size = ";
                if (cfg.ShouldWriteUseBlockSize())
                {
                    write += "blocksize";
                }
                else
                {
                    write += cfg.GetWriteSize() + " bytes";
                }
                Log.Info(write);
            }
            {
                string append = "Append size = ";
                if (cfg.ShouldAppendUseBlockSize())
                {
                    append += "blocksize";
                }
                else
                {
                    append += cfg.GetAppendSize() + " bytes";
                }
                Log.Info(append);
            }
            {
                string bsize = "Block size = ";
                bsize += cfg.GetBlockSize() + " bytes";
                Log.Info(bsize);
            }
            if (cfg.GetRandomSeed() != null)
            {
                Log.Info("Random seed = " + cfg.GetRandomSeed());
            }
            if (cfg.GetSleepRange() != null)
            {
                Log.Info("Sleep range = " + cfg.GetSleepRange() + " milliseconds");
            }
            Log.Info("Replication amount = " + cfg.GetReplication());
            Log.Info("Operations are:");
            NumberFormat percFormatter = Formatter.GetPercentFormatter();
            IDictionary <Constants.OperationType, OperationData> operations = cfg.GetOperations
                                                                                  ();

            foreach (Constants.OperationType type in operations.Keys)
            {
                string name = type.ToString();
                Log.Info(name);
                OperationData opInfo = operations[type];
                Log.Info(" " + opInfo.GetDistribution().ToString());
                if (opInfo.GetPercent() != null)
                {
                    Log.Info(" " + percFormatter.Format(opInfo.GetPercent()));
                }
                else
                {
                    Log.Info(" ???");
                }
            }
        }
Exemplo n.º 4
0
        /// <summary>
        /// Handles the specific task of merging operations from the command line or
        /// extractor object into the base configuration provided
        /// </summary>
        /// <param name="opts">the parsed command line option output</param>
        /// <param name="base">the base configuration to merge with</param>
        /// <param name="extractor">
        /// the access object to fetch operations from if none from the
        /// command line
        /// </param>
        /// <returns>merged configuration object</returns>
        /// <exception cref="ConfigException">when verification fails</exception>
        /// <exception cref="Org.Apache.Hadoop.FS.Slive.ConfigMerger.ConfigException"/>
        private Configuration HandleOperations(ArgumentParser.ParsedOutput opts, Configuration
                                               @base, ConfigExtractor extractor)
        {
            // get the base set to start off with
            IDictionary <Constants.OperationType, OperationData> operations = GetBaseOperations
                                                                                  ();
            // merge with what is coming from config
            IDictionary <Constants.OperationType, OperationData> cfgOperations = extractor.GetOperations
                                                                                     ();

            foreach (Constants.OperationType opType in cfgOperations.Keys)
            {
                operations[opType] = cfgOperations[opType];
            }
            // see if any coming in from the command line
            foreach (Constants.OperationType opType_1 in Constants.OperationType.Values())
            {
                string opName = opType_1.LowerName();
                string opVal  = opts.GetValue(opName);
                if (opVal != null)
                {
                    operations[opType_1] = new OperationData(opVal);
                }
            }
            {
                // remove those with <= zero percent
                IDictionary <Constants.OperationType, OperationData> cleanedOps = new Dictionary <Constants.OperationType
                                                                                                  , OperationData>();
                foreach (Constants.OperationType opType_2 in operations.Keys)
                {
                    OperationData data = operations[opType_2];
                    if (data.GetPercent() == null || data.GetPercent() > 0.0d)
                    {
                        cleanedOps[opType_2] = data;
                    }
                }
                operations = cleanedOps;
            }
            if (operations.IsEmpty())
            {
                throw new ConfigMerger.ConfigException("No operations provided!");
            }
            // verify and adjust
            double currPct  = 0;
            int    needFill = 0;

            foreach (Constants.OperationType type in operations.Keys)
            {
                OperationData op = operations[type];
                if (op.GetPercent() != null)
                {
                    currPct += op.GetPercent();
                }
                else
                {
                    needFill++;
                }
            }
            if (currPct > 1)
            {
                throw new ConfigMerger.ConfigException("Unable to have accumlative percent greater than 100%"
                                                       );
            }
            if (needFill > 0 && currPct < 1)
            {
                double leftOver = 1.0 - currPct;
                IDictionary <Constants.OperationType, OperationData> mpcp = new Dictionary <Constants.OperationType
                                                                                            , OperationData>();
                foreach (Constants.OperationType type_1 in operations.Keys)
                {
                    OperationData op = operations[type_1];
                    if (op.GetPercent() == null)
                    {
                        op = new OperationData(op.GetDistribution(), (leftOver / needFill));
                    }
                    mpcp[type_1] = op;
                }
                operations = mpcp;
            }
            else
            {
                if (needFill == 0 && currPct < 1)
                {
                    // redistribute
                    double leftOver = 1.0 - currPct;
                    IDictionary <Constants.OperationType, OperationData> mpcp = new Dictionary <Constants.OperationType
                                                                                                , OperationData>();
                    double each = leftOver / operations.Keys.Count;
                    foreach (Constants.OperationType t in operations.Keys)
                    {
                        OperationData op = operations[t];
                        op      = new OperationData(op.GetDistribution(), (op.GetPercent() + each));
                        mpcp[t] = op;
                    }
                    operations = mpcp;
                }
                else
                {
                    if (needFill > 0 && currPct >= 1)
                    {
                        throw new ConfigMerger.ConfigException(needFill + " unfilled operations but no percentage left to fill with"
                                                               );
                    }
                }
            }
            // save into base
            foreach (Constants.OperationType opType_3 in operations.Keys)
            {
                string        opName = opType_3.LowerName();
                OperationData opData = operations[opType_3];
                string        distr  = opData.GetDistribution().LowerName();
                string        ratio  = opData.GetPercent() * 100.0d.ToString();
                @base.Set(string.Format(Constants.Op, opName), opData.ToString());
                @base.Set(string.Format(Constants.OpDistr, opName), distr);
                @base.Set(string.Format(Constants.OpPercent, opName), ratio);
            }
            return(@base);
        }