public virtual void TestAllocationFileParsing() { Configuration conf = new Configuration(); conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile); AllocationFileLoaderService allocLoader = new AllocationFileLoaderService(); PrintWriter @out = new PrintWriter(new FileWriter(AllocFile)); @out.WriteLine("<?xml version=\"1.0\"?>"); @out.WriteLine("<allocations>"); // Give queue A a minimum of 1024 M @out.WriteLine("<queue name=\"queueA\">"); @out.WriteLine("<minResources>1024mb,0vcores</minResources>"); @out.WriteLine("</queue>"); // Give queue B a minimum of 2048 M @out.WriteLine("<queue name=\"queueB\">"); @out.WriteLine("<minResources>2048mb,0vcores</minResources>"); @out.WriteLine("<aclAdministerApps>alice,bob admins</aclAdministerApps>"); @out.WriteLine("<schedulingPolicy>fair</schedulingPolicy>"); @out.WriteLine("</queue>"); // Give queue C no minimum @out.WriteLine("<queue name=\"queueC\">"); @out.WriteLine("<aclSubmitApps>alice,bob admins</aclSubmitApps>"); @out.WriteLine("</queue>"); // Give queue D a limit of 3 running apps and 0.4f maxAMShare @out.WriteLine("<queue name=\"queueD\">"); @out.WriteLine("<maxRunningApps>3</maxRunningApps>"); @out.WriteLine("<maxAMShare>0.4</maxAMShare>"); @out.WriteLine("</queue>"); // Give queue E a preemption timeout of one minute @out.WriteLine("<queue name=\"queueE\">"); @out.WriteLine("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>"); @out.WriteLine("</queue>"); //Make queue F a parent queue without configured leaf queues using the 'type' attribute @out.WriteLine("<queue name=\"queueF\" type=\"parent\" >"); @out.WriteLine("</queue>"); // Create hierarchical queues G,H, with different min/fair share preemption // timeouts and preemption thresholds @out.WriteLine("<queue name=\"queueG\">"); @out.WriteLine("<fairSharePreemptionTimeout>120</fairSharePreemptionTimeout>"); @out.WriteLine("<minSharePreemptionTimeout>50</minSharePreemptionTimeout>"); @out.WriteLine("<fairSharePreemptionThreshold>0.6</fairSharePreemptionThreshold>" ); @out.WriteLine(" <queue name=\"queueH\">"); @out.WriteLine(" <fairSharePreemptionTimeout>180</fairSharePreemptionTimeout>"); @out.WriteLine(" <minSharePreemptionTimeout>40</minSharePreemptionTimeout>"); @out.WriteLine(" <fairSharePreemptionThreshold>0.7</fairSharePreemptionThreshold>" ); @out.WriteLine(" </queue>"); @out.WriteLine("</queue>"); // Set default limit of apps per queue to 15 @out.WriteLine("<queueMaxAppsDefault>15</queueMaxAppsDefault>"); // Set default limit of apps per user to 5 @out.WriteLine("<userMaxAppsDefault>5</userMaxAppsDefault>"); // Set default limit of AMResourceShare to 0.5f @out.WriteLine("<queueMaxAMShareDefault>0.5f</queueMaxAMShareDefault>"); // Give user1 a limit of 10 jobs @out.WriteLine("<user name=\"user1\">"); @out.WriteLine("<maxRunningApps>10</maxRunningApps>"); @out.WriteLine("</user>"); // Set default min share preemption timeout to 2 minutes @out.WriteLine("<defaultMinSharePreemptionTimeout>120" + "</defaultMinSharePreemptionTimeout>" ); // Set default fair share preemption timeout to 5 minutes @out.WriteLine("<defaultFairSharePreemptionTimeout>300</defaultFairSharePreemptionTimeout>" ); // Set default fair share preemption threshold to 0.4 @out.WriteLine("<defaultFairSharePreemptionThreshold>0.4</defaultFairSharePreemptionThreshold>" ); // Set default scheduling policy to DRF @out.WriteLine("<defaultQueueSchedulingPolicy>drf</defaultQueueSchedulingPolicy>" ); @out.WriteLine("</allocations>"); @out.Close(); allocLoader.Init(conf); TestAllocationFileLoaderService.ReloadListener confHolder = new TestAllocationFileLoaderService.ReloadListener (this); allocLoader.SetReloadListener(confHolder); allocLoader.ReloadAllocations(); AllocationConfiguration queueConf = confHolder.allocConf; NUnit.Framework.Assert.AreEqual(6, queueConf.GetConfiguredQueues()[FSQueueType.Leaf ].Count); NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources ("root." + YarnConfiguration.DefaultQueueName)); NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources ("root." + YarnConfiguration.DefaultQueueName)); NUnit.Framework.Assert.AreEqual(Resources.CreateResource(1024, 0), queueConf.GetMinResources ("root.queueA")); NUnit.Framework.Assert.AreEqual(Resources.CreateResource(2048, 0), queueConf.GetMinResources ("root.queueB")); NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources ("root.queueC")); NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources ("root.queueD")); NUnit.Framework.Assert.AreEqual(Resources.CreateResource(0), queueConf.GetMinResources ("root.queueE")); NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root." + YarnConfiguration .DefaultQueueName)); NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root.queueA")); NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root.queueB")); NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root.queueC")); NUnit.Framework.Assert.AreEqual(3, queueConf.GetQueueMaxApps("root.queueD")); NUnit.Framework.Assert.AreEqual(15, queueConf.GetQueueMaxApps("root.queueE")); NUnit.Framework.Assert.AreEqual(10, queueConf.GetUserMaxApps("user1")); NUnit.Framework.Assert.AreEqual(5, queueConf.GetUserMaxApps("user2")); NUnit.Framework.Assert.AreEqual(.5f, queueConf.GetQueueMaxAMShare("root." + YarnConfiguration .DefaultQueueName), 0.01); NUnit.Framework.Assert.AreEqual(.5f, queueConf.GetQueueMaxAMShare("root.queueA"), 0.01); NUnit.Framework.Assert.AreEqual(.5f, queueConf.GetQueueMaxAMShare("root.queueB"), 0.01); NUnit.Framework.Assert.AreEqual(.5f, queueConf.GetQueueMaxAMShare("root.queueC"), 0.01); NUnit.Framework.Assert.AreEqual(.4f, queueConf.GetQueueMaxAMShare("root.queueD"), 0.01); NUnit.Framework.Assert.AreEqual(.5f, queueConf.GetQueueMaxAMShare("root.queueE"), 0.01); // Root should get * ACL NUnit.Framework.Assert.AreEqual("*", queueConf.GetQueueAcl("root", QueueACL.AdministerQueue ).GetAclString()); NUnit.Framework.Assert.AreEqual("*", queueConf.GetQueueAcl("root", QueueACL.SubmitApplications ).GetAclString()); // Unspecified queues should get default ACL NUnit.Framework.Assert.AreEqual(" ", queueConf.GetQueueAcl("root.queueA", QueueACL .AdministerQueue).GetAclString()); NUnit.Framework.Assert.AreEqual(" ", queueConf.GetQueueAcl("root.queueA", QueueACL .SubmitApplications).GetAclString()); // Queue B ACL NUnit.Framework.Assert.AreEqual("alice,bob admins", queueConf.GetQueueAcl("root.queueB" , QueueACL.AdministerQueue).GetAclString()); // Queue C ACL NUnit.Framework.Assert.AreEqual("alice,bob admins", queueConf.GetQueueAcl("root.queueC" , QueueACL.SubmitApplications).GetAclString()); NUnit.Framework.Assert.AreEqual(120000, queueConf.GetMinSharePreemptionTimeout("root" )); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root." + YarnConfiguration.DefaultQueueName)); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root.queueA" )); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root.queueB" )); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root.queueC" )); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root.queueD" )); NUnit.Framework.Assert.AreEqual(60000, queueConf.GetMinSharePreemptionTimeout("root.queueE" )); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetMinSharePreemptionTimeout("root.queueF" )); NUnit.Framework.Assert.AreEqual(50000, queueConf.GetMinSharePreemptionTimeout("root.queueG" )); NUnit.Framework.Assert.AreEqual(40000, queueConf.GetMinSharePreemptionTimeout("root.queueG.queueH" )); NUnit.Framework.Assert.AreEqual(300000, queueConf.GetFairSharePreemptionTimeout("root" )); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root." + YarnConfiguration.DefaultQueueName)); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueA" )); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueB" )); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueC" )); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueD" )); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueE" )); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionTimeout("root.queueF" )); NUnit.Framework.Assert.AreEqual(120000, queueConf.GetFairSharePreemptionTimeout("root.queueG" )); NUnit.Framework.Assert.AreEqual(180000, queueConf.GetFairSharePreemptionTimeout("root.queueG.queueH" )); NUnit.Framework.Assert.AreEqual(.4f, queueConf.GetFairSharePreemptionThreshold("root" ), 0.01); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root." + YarnConfiguration.DefaultQueueName), 0.01); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueA" ), 0.01); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueB" ), 0.01); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueC" ), 0.01); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueD" ), 0.01); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueE" ), 0.01); NUnit.Framework.Assert.AreEqual(-1, queueConf.GetFairSharePreemptionThreshold("root.queueF" ), 0.01); NUnit.Framework.Assert.AreEqual(.6f, queueConf.GetFairSharePreemptionThreshold("root.queueG" ), 0.01); NUnit.Framework.Assert.AreEqual(.7f, queueConf.GetFairSharePreemptionThreshold("root.queueG.queueH" ), 0.01); NUnit.Framework.Assert.IsTrue(queueConf.GetConfiguredQueues()[FSQueueType.Parent] .Contains("root.queueF")); NUnit.Framework.Assert.IsTrue(queueConf.GetConfiguredQueues()[FSQueueType.Parent] .Contains("root.queueG")); NUnit.Framework.Assert.IsTrue(queueConf.GetConfiguredQueues()[FSQueueType.Leaf].Contains ("root.queueG.queueH")); // Verify existing queues have default scheduling policy NUnit.Framework.Assert.AreEqual(DominantResourceFairnessPolicy.Name, queueConf.GetSchedulingPolicy ("root").GetName()); NUnit.Framework.Assert.AreEqual(DominantResourceFairnessPolicy.Name, queueConf.GetSchedulingPolicy ("root.queueA").GetName()); // Verify default is overriden if specified explicitly NUnit.Framework.Assert.AreEqual(FairSharePolicy.Name, queueConf.GetSchedulingPolicy ("root.queueB").GetName()); // Verify new queue gets default scheduling policy NUnit.Framework.Assert.AreEqual(DominantResourceFairnessPolicy.Name, queueConf.GetSchedulingPolicy ("root.newqueue").GetName()); }