/// <summary>
/// Sets up the operation using the given configuration by setting up the
/// number of operations to perform (and how many are left) and setting up the
/// operation objects to be used throughout selection.
/// </summary>
/// <param name="cfg">ConfigExtractor.</param>
private void ConfigureOperations(ConfigExtractor cfg)
{
operations = new SortedDictionary <Constants.OperationType, WeightSelector.OperationInfo
>();
IDictionary <Constants.OperationType, OperationData> opinfo = cfg.GetOperations();
int totalAm = cfg.GetOpCount();
int opsLeft = totalAm;
NumberFormat formatter = Formatter.GetPercentFormatter();
foreach (Constants.OperationType type in opinfo.Keys)
{
OperationData opData = opinfo[type];
WeightSelector.OperationInfo info = new WeightSelector.OperationInfo();
info.distribution = opData.GetDistribution();
int amLeft = DetermineHowMany(totalAm, opData, type);
opsLeft -= amLeft;
Log.Info(type.ToString() + " has " + amLeft + " initial operations out of " + totalAm
+ " for its ratio " + formatter.Format(opData.GetPercent()));
info.amountLeft = amLeft;
Operation op = factory.GetOperation(type);
// wrap operation in finalizer so that amount left gets decrements when
// its done
if (op != null)
{
ObserveableOp.Observer fn = new _Observer_138(this, type);
info.operation = new ObserveableOp(op, fn);
operations[type] = info;
}
}
if (opsLeft > 0)
{
Log.Info(opsLeft + " left over operations found (due to inability to support partial operations)"
);
}
}
// weights.put(Distribution.BEG, new BeginWeight());
// weights.put(Distribution.END, new EndWeight());
// weights.put(Distribution.MID, new MidWeight());
/// <summary>Determines how many initial operations a given operation data should have
/// </summary>
/// <param name="totalAm">the total amount of operations allowed</param>
/// <param name="opData">the given operation information (with a valid percentage >= 0)
/// </param>
/// <returns>the number of items to allow to run</returns>
/// <exception cref="System.ArgumentException">if negative operations are determined</exception>
internal static int DetermineHowMany(int totalAm, OperationData opData, Constants.OperationType
type)
{
if (totalAm <= 0)
{
return(0);
}
int amLeft = (int)Math.Floor(opData.GetPercent() * totalAm);
if (amLeft < 0)
{
throw new ArgumentException("Invalid amount " + amLeft + " determined for operation type "
+ type.ToString());
}
return(amLeft);
}
/// <summary>Dumps out the given options for the given config extractor</summary>
/// <param name="cfg">the config to write to the log</param>
internal static void DumpOptions(Org.Apache.Hadoop.FS.Slive.ConfigExtractor cfg)
{
if (cfg == null)
{
return;
}
Log.Info("Base directory = " + cfg.GetBaseDirectory());
Log.Info("Data directory = " + cfg.GetDataPath());
Log.Info("Output directory = " + cfg.GetOutputPath());
Log.Info("Result file = " + cfg.GetResultFile());
Log.Info("Grid queue = " + cfg.GetQueueName());
Log.Info("Should exit on first error = " + cfg.ShouldExitOnFirstError());
{
string duration = "Duration = ";
if (cfg.GetDurationMilliseconds() == int.MaxValue)
{
duration += "unlimited";
}
else
{
duration += cfg.GetDurationMilliseconds() + " milliseconds";
}
Log.Info(duration);
}
Log.Info("Map amount = " + cfg.GetMapAmount());
Log.Info("Reducer amount = " + cfg.GetReducerAmount());
Log.Info("Operation amount = " + cfg.GetOpCount());
Log.Info("Total file limit = " + cfg.GetTotalFiles());
Log.Info("Total dir file limit = " + cfg.GetDirSize());
{
string read = "Read size = ";
if (cfg.ShouldReadFullFile())
{
read += "entire file";
}
else
{
read += cfg.GetReadSize() + " bytes";
}
Log.Info(read);
}
{
string write = "Write size = ";
if (cfg.ShouldWriteUseBlockSize())
{
write += "blocksize";
}
else
{
write += cfg.GetWriteSize() + " bytes";
}
Log.Info(write);
}
{
string append = "Append size = ";
if (cfg.ShouldAppendUseBlockSize())
{
append += "blocksize";
}
else
{
append += cfg.GetAppendSize() + " bytes";
}
Log.Info(append);
}
{
string bsize = "Block size = ";
bsize += cfg.GetBlockSize() + " bytes";
Log.Info(bsize);
}
if (cfg.GetRandomSeed() != null)
{
Log.Info("Random seed = " + cfg.GetRandomSeed());
}
if (cfg.GetSleepRange() != null)
{
Log.Info("Sleep range = " + cfg.GetSleepRange() + " milliseconds");
}
Log.Info("Replication amount = " + cfg.GetReplication());
Log.Info("Operations are:");
NumberFormat percFormatter = Formatter.GetPercentFormatter();
IDictionary <Constants.OperationType, OperationData> operations = cfg.GetOperations
();
foreach (Constants.OperationType type in operations.Keys)
{
string name = type.ToString();
Log.Info(name);
OperationData opInfo = operations[type];
Log.Info(" " + opInfo.GetDistribution().ToString());
if (opInfo.GetPercent() != null)
{
Log.Info(" " + percFormatter.Format(opInfo.GetPercent()));
}
else
{
Log.Info(" ???");
}
}
}
/// <summary>
/// Handles the specific task of merging operations from the command line or
/// extractor object into the base configuration provided
/// </summary>
/// <param name="opts">the parsed command line option output</param>
/// <param name="base">the base configuration to merge with</param>
/// <param name="extractor">
/// the access object to fetch operations from if none from the
/// command line
/// </param>
/// <returns>merged configuration object</returns>
/// <exception cref="ConfigException">when verification fails</exception>
/// <exception cref="Org.Apache.Hadoop.FS.Slive.ConfigMerger.ConfigException"/>
private Configuration HandleOperations(ArgumentParser.ParsedOutput opts, Configuration
@base, ConfigExtractor extractor)
{
// get the base set to start off with
IDictionary <Constants.OperationType, OperationData> operations = GetBaseOperations
();
// merge with what is coming from config
IDictionary <Constants.OperationType, OperationData> cfgOperations = extractor.GetOperations
();
foreach (Constants.OperationType opType in cfgOperations.Keys)
{
operations[opType] = cfgOperations[opType];
}
// see if any coming in from the command line
foreach (Constants.OperationType opType_1 in Constants.OperationType.Values())
{
string opName = opType_1.LowerName();
string opVal = opts.GetValue(opName);
if (opVal != null)
{
operations[opType_1] = new OperationData(opVal);
}
}
{
// remove those with <= zero percent
IDictionary <Constants.OperationType, OperationData> cleanedOps = new Dictionary <Constants.OperationType
, OperationData>();
foreach (Constants.OperationType opType_2 in operations.Keys)
{
OperationData data = operations[opType_2];
if (data.GetPercent() == null || data.GetPercent() > 0.0d)
{
cleanedOps[opType_2] = data;
}
}
operations = cleanedOps;
}
if (operations.IsEmpty())
{
throw new ConfigMerger.ConfigException("No operations provided!");
}
// verify and adjust
double currPct = 0;
int needFill = 0;
foreach (Constants.OperationType type in operations.Keys)
{
OperationData op = operations[type];
if (op.GetPercent() != null)
{
currPct += op.GetPercent();
}
else
{
needFill++;
}
}
if (currPct > 1)
{
throw new ConfigMerger.ConfigException("Unable to have accumlative percent greater than 100%"
);
}
if (needFill > 0 && currPct < 1)
{
double leftOver = 1.0 - currPct;
IDictionary <Constants.OperationType, OperationData> mpcp = new Dictionary <Constants.OperationType
, OperationData>();
foreach (Constants.OperationType type_1 in operations.Keys)
{
OperationData op = operations[type_1];
if (op.GetPercent() == null)
{
op = new OperationData(op.GetDistribution(), (leftOver / needFill));
}
mpcp[type_1] = op;
}
operations = mpcp;
}
else
{
if (needFill == 0 && currPct < 1)
{
// redistribute
double leftOver = 1.0 - currPct;
IDictionary <Constants.OperationType, OperationData> mpcp = new Dictionary <Constants.OperationType
, OperationData>();
double each = leftOver / operations.Keys.Count;
foreach (Constants.OperationType t in operations.Keys)
{
OperationData op = operations[t];
op = new OperationData(op.GetDistribution(), (op.GetPercent() + each));
mpcp[t] = op;
}
operations = mpcp;
}
else
{
if (needFill > 0 && currPct >= 1)
{
throw new ConfigMerger.ConfigException(needFill + " unfilled operations but no percentage left to fill with"
);
}
}
}
// save into base
foreach (Constants.OperationType opType_3 in operations.Keys)
{
string opName = opType_3.LowerName();
OperationData opData = operations[opType_3];
string distr = opData.GetDistribution().LowerName();
string ratio = opData.GetPercent() * 100.0d.ToString();
@base.Set(string.Format(Constants.Op, opName), opData.ToString());
@base.Set(string.Format(Constants.OpDistr, opName), distr);
@base.Set(string.Format(Constants.OpPercent, opName), ratio);
}
return(@base);
}