/// <exception cref="System.IO.IOException"/>
protected internal override Configuration CreateConfiguration()
{
FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration();
string TestDir = new FilePath(Runtime.GetProperty("test.build.data", "/tmp")).GetAbsolutePath
();
string AllocFile = new FilePath(TestDir, "test-queues.xml").GetAbsolutePath();
PrintWriter @out = new PrintWriter(new FileWriter(AllocFile));
@out.WriteLine("<?xml version=\"1.0\"?>");
@out.WriteLine("<allocations>");
@out.WriteLine("<queue name=\"root\">");
@out.WriteLine(" <aclSubmitApps> </aclSubmitApps>");
@out.WriteLine(" <aclAdministerApps>root_admin </aclAdministerApps>");
@out.WriteLine(" <queue name=\"queueA\">");
@out.WriteLine(" <aclSubmitApps>queueA_user,common_user </aclSubmitApps>");
@out.WriteLine(" <aclAdministerApps>queueA_admin </aclAdministerApps>");
@out.WriteLine(" </queue>");
@out.WriteLine(" <queue name=\"queueB\">");
@out.WriteLine(" <aclSubmitApps>queueB_user,common_user </aclSubmitApps>");
@out.WriteLine(" <aclAdministerApps>queueB_admin </aclAdministerApps>");
@out.WriteLine(" </queue>");
@out.WriteLine("</queue>");
@out.WriteLine("</allocations>");
@out.Close();
fsConf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile);
fsConf.SetBoolean(YarnConfiguration.YarnAclEnable, true);
fsConf.Set("yarn.resourcemanager.scheduler.class", typeof(FairScheduler).FullName
);
return(fsConf);
}
public virtual void TestParseResourceConfigValue()
{
NUnit.Framework.Assert.AreEqual(BuilderUtils.NewResource(1024, 2), FairSchedulerConfiguration.ParseResourceConfigValue
("2 vcores, 1024 mb"));
NUnit.Framework.Assert.AreEqual(BuilderUtils.NewResource(1024, 2), FairSchedulerConfiguration.ParseResourceConfigValue
("1024 mb, 2 vcores"));
NUnit.Framework.Assert.AreEqual(BuilderUtils.NewResource(1024, 2), FairSchedulerConfiguration.ParseResourceConfigValue
("2vcores,1024mb"));
NUnit.Framework.Assert.AreEqual(BuilderUtils.NewResource(1024, 2), FairSchedulerConfiguration.ParseResourceConfigValue
("1024mb,2vcores"));
NUnit.Framework.Assert.AreEqual(BuilderUtils.NewResource(1024, 2), FairSchedulerConfiguration.ParseResourceConfigValue
("1024 mb, 2 vcores"));
NUnit.Framework.Assert.AreEqual(BuilderUtils.NewResource(1024, 2), FairSchedulerConfiguration.ParseResourceConfigValue
("1024 Mb, 2 vCores"));
NUnit.Framework.Assert.AreEqual(BuilderUtils.NewResource(1024, 2), FairSchedulerConfiguration.ParseResourceConfigValue
(" 1024 mb, 2 vcores "));
}
public virtual void SetUp()
{
conf = new FairSchedulerConfiguration();
FairScheduler scheduler = Org.Mockito.Mockito.Mock <FairScheduler>();
AllocationConfiguration allocConf = new AllocationConfiguration(conf);
Org.Mockito.Mockito.When(scheduler.GetAllocationConfiguration()).ThenReturn(allocConf
);
Org.Mockito.Mockito.When(scheduler.GetConf()).ThenReturn(conf);
SystemClock clock = new SystemClock();
Org.Mockito.Mockito.When(scheduler.GetClock()).ThenReturn(clock);
notEmptyQueues = new HashSet <FSQueue>();
queueManager = new _QueueManager_47(this, scheduler);
FSQueueMetrics.ForQueue("root", null, true, conf);
queueManager.Initialize(conf);
}
internal virtual bool Init(FairSchedulerConfiguration conf)
{
if (conf.IsEventLogEnabled())
{
try
{
logDir = conf.GetEventlogDir();
FilePath logDirFile = new FilePath(logDir);
if (!logDirFile.Exists())
{
if (!logDirFile.Mkdirs())
{
throw new IOException("Mkdirs failed to create " + logDirFile.ToString());
}
}
string username = Runtime.GetProperty("user.name");
logFile = string.Format("%s%shadoop-%s-fairscheduler.log", logDir, FilePath.separator
, username);
logDisabled = false;
PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n");
appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd");
appender.ActivateOptions();
Log.Info("Initialized fair scheduler event log, logging to " + logFile);
}
catch (IOException e)
{
Log.Error("Failed to initialize fair scheduler event log. Disabling it.", e);
logDisabled = true;
}
}
else
{
logDisabled = true;
}
return(!(logDisabled));
}
/// <summary>Loads a queue from a queue element in the configuration file</summary>
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Scheduler.Fair.AllocationConfigurationException
/// "/>
private void LoadQueue(string parentName, Element element, IDictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource
> minQueueResources, IDictionary <string, Org.Apache.Hadoop.Yarn.Api.Records.Resource
> maxQueueResources, IDictionary <string, int> queueMaxApps, IDictionary <string,
int> userMaxApps, IDictionary <string, float> queueMaxAMShares, IDictionary <string
, ResourceWeights> queueWeights, IDictionary <string, SchedulingPolicy> queuePolicies
, IDictionary <string, long> minSharePreemptionTimeouts, IDictionary <string, long
> fairSharePreemptionTimeouts, IDictionary <string, float> fairSharePreemptionThresholds
, IDictionary <string, IDictionary <QueueACL, AccessControlList> > queueAcls, IDictionary
<FSQueueType, ICollection <string> > configuredQueues, ICollection <string> reservableQueues
)
{
string queueName = element.GetAttribute("name");
if (queueName.Contains("."))
{
throw new AllocationConfigurationException("Bad fair scheduler config " + "file: queue name ("
+ queueName + ") shouldn't contain period.");
}
if (parentName != null)
{
queueName = parentName + "." + queueName;
}
IDictionary <QueueACL, AccessControlList> acls = new Dictionary <QueueACL, AccessControlList
>();
NodeList fields = element.GetChildNodes();
bool isLeaf = true;
for (int j = 0; j < fields.GetLength(); j++)
{
Node fieldNode = fields.Item(j);
if (!(fieldNode is Element))
{
continue;
}
Element field = (Element)fieldNode;
if ("minResources".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
Org.Apache.Hadoop.Yarn.Api.Records.Resource val = FairSchedulerConfiguration.ParseResourceConfigValue
(text);
minQueueResources[queueName] = val;
}
else
{
if ("maxResources".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
Org.Apache.Hadoop.Yarn.Api.Records.Resource val = FairSchedulerConfiguration.ParseResourceConfigValue
(text);
maxQueueResources[queueName] = val;
}
else
{
if ("maxRunningApps".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
int val = System.Convert.ToInt32(text);
queueMaxApps[queueName] = val;
}
else
{
if ("maxAMShare".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
float val = float.ParseFloat(text);
val = Math.Min(val, 1.0f);
queueMaxAMShares[queueName] = val;
}
else
{
if ("weight".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
double val = double.ParseDouble(text);
queueWeights[queueName] = new ResourceWeights((float)val);
}
else
{
if ("minSharePreemptionTimeout".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
minSharePreemptionTimeouts[queueName] = val;
}
else
{
if ("fairSharePreemptionTimeout".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
long val = long.Parse(text) * 1000L;
fairSharePreemptionTimeouts[queueName] = val;
}
else
{
if ("fairSharePreemptionThreshold".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
float val = float.ParseFloat(text);
val = Math.Max(Math.Min(val, 1.0f), 0.0f);
fairSharePreemptionThresholds[queueName] = val;
}
else
{
if ("schedulingPolicy".Equals(field.GetTagName()) || "schedulingMode".Equals(field
.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData().Trim();
SchedulingPolicy policy = SchedulingPolicy.Parse(text);
queuePolicies[queueName] = policy;
}
else
{
if ("aclSubmitApps".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData();
acls[QueueACL.SubmitApplications] = new AccessControlList(text);
}
else
{
if ("aclAdministerApps".Equals(field.GetTagName()))
{
string text = ((Text)field.GetFirstChild()).GetData();
acls[QueueACL.AdministerQueue] = new AccessControlList(text);
}
else
{
if ("reservation".Equals(field.GetTagName()))
{
isLeaf = false;
reservableQueues.AddItem(queueName);
configuredQueues[FSQueueType.Parent].AddItem(queueName);
}
else
{
if ("queue".EndsWith(field.GetTagName()) || "pool".Equals(field.GetTagName()))
{
LoadQueue(queueName, field, minQueueResources, maxQueueResources, queueMaxApps, userMaxApps
, queueMaxAMShares, queueWeights, queuePolicies, minSharePreemptionTimeouts, fairSharePreemptionTimeouts
, fairSharePreemptionThresholds, queueAcls, configuredQueues, reservableQueues);
isLeaf = false;
}
}
}
}
}
}
}
}
}
}
}
}
}
}
if (isLeaf)
{
// if a leaf in the alloc file is marked as type='parent'
// then store it under 'parent'
if ("parent".Equals(element.GetAttribute("type")))
{
configuredQueues[FSQueueType.Parent].AddItem(queueName);
}
else
{
configuredQueues[FSQueueType.Leaf].AddItem(queueName);
}
}
else
{
if ("parent".Equals(element.GetAttribute("type")))
{
throw new AllocationConfigurationException("Both <reservation> and " + "type=\"parent\" found for queue "
+ queueName + " which is " + "unsupported");
}
configuredQueues[FSQueueType.Parent].AddItem(queueName);
}
queueAcls[queueName] = acls;
if (maxQueueResources.Contains(queueName) && minQueueResources.Contains(queueName
) && !Resources.FitsIn(minQueueResources[queueName], maxQueueResources[queueName
]))
{
Log.Warn(string.Format("Queue %s has max resources %s less than min resources %s"
, queueName, maxQueueResources[queueName], minQueueResources[queueName]));
}
}
/// <exception cref="System.Exception"/>
public virtual void TestGibberish()
{
FairSchedulerConfiguration.ParseResourceConfigValue("1o24vc0res");
}
/// <exception cref="System.Exception"/>
public virtual void TestOnlyCPU()
{
FairSchedulerConfiguration.ParseResourceConfigValue("1024vcores");
}
/// <exception cref="System.Exception"/>
public virtual void TestOnlyMemory()
{
FairSchedulerConfiguration.ParseResourceConfigValue("1024mb");
}
/// <exception cref="System.Exception"/>
public virtual void TestNoUnits()
{
FairSchedulerConfiguration.ParseResourceConfigValue("1024");
}
