/// <exception cref="System.Exception"/>
public virtual void TestReconnectedNode()
{
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
conf.SetQueues("default", new string[] { "default" });
conf.SetCapacity("default", 100);
FifoScheduler fs = new FifoScheduler();
fs.Init(conf);
fs.Start();
// mock rmContext to avoid NPE.
RMContext context = Org.Mockito.Mockito.Mock <RMContext>();
fs.Reinitialize(conf, null);
fs.SetRMContext(context);
RMNode n1 = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, "127.0.0.2"
);
RMNode n2 = MockNodes.NewNodeInfo(0, MockNodes.NewResource(2 * Gb), 2, "127.0.0.3"
);
fs.Handle(new NodeAddedSchedulerEvent(n1));
fs.Handle(new NodeAddedSchedulerEvent(n2));
fs.Handle(new NodeUpdateSchedulerEvent(n1));
NUnit.Framework.Assert.AreEqual(6 * Gb, fs.GetRootQueueMetrics().GetAvailableMB()
);
// reconnect n1 with downgraded memory
n1 = MockNodes.NewNodeInfo(0, MockNodes.NewResource(2 * Gb), 1, "127.0.0.2");
fs.Handle(new NodeRemovedSchedulerEvent(n1));
fs.Handle(new NodeAddedSchedulerEvent(n1));
fs.Handle(new NodeUpdateSchedulerEvent(n1));
NUnit.Framework.Assert.AreEqual(4 * Gb, fs.GetRootQueueMetrics().GetAvailableMB()
);
fs.Stop();
}
public virtual void TestNodeUpdateBeforeAppAttemptInit()
{
FifoScheduler scheduler = new FifoScheduler();
MockRM rm = new MockRM(conf);
scheduler.SetRMContext(rm.GetRMContext());
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(conf, rm.GetRMContext());
RMNode node = MockNodes.NewNodeInfo(1, Resources.CreateResource(1024, 4), 1, "127.0.0.1"
);
scheduler.Handle(new NodeAddedSchedulerEvent(node));
ApplicationId appId = ApplicationId.NewInstance(0, 1);
scheduler.AddApplication(appId, "queue1", "user1", false);
NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
try
{
scheduler.Handle(updateEvent);
}
catch (ArgumentNullException)
{
NUnit.Framework.Assert.Fail();
}
ApplicationAttemptId attId = ApplicationAttemptId.NewInstance(appId, 1);
scheduler.AddApplicationAttempt(attId, false, false);
rm.Stop();
}
public FifoSchedulerInfo(ResourceManager rm)
{
// JAXB needs this
RMContext rmContext = rm.GetRMContext();
FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler();
qName = fs.GetQueueInfo(string.Empty, false, false).GetQueueName();
QueueInfo qInfo = fs.GetQueueInfo(qName, true, true);
this.usedCapacity = qInfo.GetCurrentCapacity();
this.capacity = qInfo.GetCapacity();
this.minQueueMemoryCapacity = fs.GetMinimumResourceCapability().GetMemory();
this.maxQueueMemoryCapacity = fs.GetMaximumResourceCapability().GetMemory();
this.qstate = qInfo.GetQueueState();
this.numNodes = rmContext.GetRMNodes().Count;
this.usedNodeCapacity = 0;
this.availNodeCapacity = 0;
this.totalNodeCapacity = 0;
this.numContainers = 0;
foreach (RMNode ni in rmContext.GetRMNodes().Values)
{
SchedulerNodeReport report = fs.GetNodeReport(ni.GetNodeID());
this.usedNodeCapacity += report.GetUsedResource().GetMemory();
this.availNodeCapacity += report.GetAvailableResource().GetMemory();
this.totalNodeCapacity += ni.GetTotalCapability().GetMemory();
this.numContainers += fs.GetNodeReport(ni.GetNodeID()).GetNumContainers();
}
}
public void Add_IsUriRecrawlingIsFalse_DuplicateNotAdded()
{
_unitUnderTest = new FifoScheduler(false);//this is the default
_unitUnderTest.Add(new PageToCrawl(new Uri("http://a.com/")));
_unitUnderTest.Add(new PageToCrawl(new Uri("http://a.com/")));
_unitUnderTest.Add(new PageToCrawl(new Uri("http://a.com/")));
Assert.AreEqual(1, _unitUnderTest.Count);
}
public void Add_IEnumerableParam_IsUriRecrawlingIsTrue_DuplicateAdded()
{
_unitUnderTest = new FifoScheduler(true);//this is the default
_unitUnderTest.Add(new List <PageToCrawl> {
new PageToCrawl(new Uri("http://a.com/")), new PageToCrawl(new Uri("http://a.com/")), new PageToCrawl(new Uri("http://a.com/"))
});
Assert.AreEqual(3, _unitUnderTest.Count);
}
public void Add_IsUriRecrawlingIsTrue_DuplicateAdded()
{
_unitUnderTest = new FifoScheduler(true);
_unitUnderTest.Add(new PageToCrawl(new Uri("http://a.com/")));
_unitUnderTest.Add(new PageToCrawl(new Uri("http://a.com/")));
_unitUnderTest.Add(new PageToCrawl(new Uri("http://a.com/")));
Assert.AreEqual(3, _unitUnderTest.Count);
}
public void Clear_RemovesAllPrevious()
{
_unitUnderTest = new FifoScheduler();
_unitUnderTest.Add(new PageToCrawl(new Uri("http://a.com/")));
_unitUnderTest.Add(new PageToCrawl(new Uri("http://b.com/")));
_unitUnderTest.Add(new PageToCrawl(new Uri("http://c.com/")));
_unitUnderTest.Clear();
Assert.AreEqual(0, _unitUnderTest.Count);
}
/// <exception cref="System.Exception"/>
public static FifoScheduler MockFifoScheduler(RMContext rmContext)
{
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
SetupFifoQueueConfiguration(conf);
FifoScheduler fs = new FifoScheduler();
fs.SetConf(new YarnConfiguration());
fs.SetRMContext(rmContext);
fs.Init(conf);
return(fs);
}
public void SetUp()
{
_fakeHyperLinkParser = new Mock <IHyperLinkParser>();
_fakeHttpRequester = new Mock <IPageRequester>();
_fakeCrawlDecisionMaker = new Mock <ICrawlDecisionMaker>();
_fakeDomainRateLimiter = new Mock <IDomainRateLimiter>();
_fakeMemoryManager = new Mock <IMemoryManager>();
_fakeRobotsDotTextFinder = new Mock <IRobotsDotTextFinder>();
_fakeRobotsDotText = new Mock <IRobotsDotText>();
_dummyScheduler = new FifoScheduler();
_dummyThreadManager = new ProducerConsumerThreadManager(1);
_dummyConfiguration = new CrawlConfiguration();
_dummyConfiguration.ConfigurationExtensions.Add("somekey", "someval");
_rootUri = new Uri("http://a.com/");
}
/// <exception cref="System.Exception"/>
public virtual void TestConfValidation()
{
FifoScheduler scheduler = new FifoScheduler();
Configuration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, 2048);
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, 1024);
try
{
scheduler.ServiceInit(conf);
NUnit.Framework.Assert.Fail("Exception is expected because the min memory allocation is"
+ " larger than the max memory allocation.");
}
catch (YarnRuntimeException e)
{
// Exception is expected.
NUnit.Framework.Assert.IsTrue("The thrown exception is not the expected one.", e.
Message.StartsWith("Invalid resource scheduler memory"));
}
}
public void SetUp()
{
_unitUnderTest = new FifoScheduler();
}
/// <exception cref="System.Exception"/>
public virtual void TestHeadroom()
{
Configuration conf = new Configuration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
));
MockRM rm = new MockRM(conf);
rm.Start();
FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler();
// Add a node
RMNode n1 = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, "127.0.0.2"
);
fs.Handle(new NodeAddedSchedulerEvent(n1));
// Add two applications
ApplicationId appId1 = BuilderUtils.NewApplicationId(100, 1);
ApplicationAttemptId appAttemptId1 = BuilderUtils.NewApplicationAttemptId(appId1,
1);
CreateMockRMApp(appAttemptId1, rm.GetRMContext());
SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");
fs.Handle(appEvent);
SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false
);
fs.Handle(attemptEvent);
ApplicationId appId2 = BuilderUtils.NewApplicationId(200, 2);
ApplicationAttemptId appAttemptId2 = BuilderUtils.NewApplicationAttemptId(appId2,
1);
CreateMockRMApp(appAttemptId2, rm.GetRMContext());
SchedulerEvent appEvent2 = new AppAddedSchedulerEvent(appId2, "queue", "user");
fs.Handle(appEvent2);
SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId2, false
);
fs.Handle(attemptEvent2);
IList <ContainerId> emptyId = new AList <ContainerId>();
IList <ResourceRequest> emptyAsk = new AList <ResourceRequest>();
// Set up resource requests
// Ask for a 1 GB container for app 1
IList <ResourceRequest> ask1 = new AList <ResourceRequest>();
ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
.Any, BuilderUtils.NewResource(Gb, 1), 1));
fs.Allocate(appAttemptId1, ask1, emptyId, null, null);
// Ask for a 2 GB container for app 2
IList <ResourceRequest> ask2 = new AList <ResourceRequest>();
ask2.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
.Any, BuilderUtils.NewResource(2 * Gb, 1), 1));
fs.Allocate(appAttemptId2, ask2, emptyId, null, null);
// Trigger container assignment
fs.Handle(new NodeUpdateSchedulerEvent(n1));
// Get the allocation for the applications and verify headroom
Allocation allocation1 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("Allocation headroom", 1 * Gb, allocation1.GetResourceLimit
().GetMemory());
Allocation allocation2 = fs.Allocate(appAttemptId2, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("Allocation headroom", 1 * Gb, allocation2.GetResourceLimit
().GetMemory());
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestBlackListNodes()
{
Configuration conf = new Configuration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
));
MockRM rm = new MockRM(conf);
rm.Start();
FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler();
int rack_num_0 = 0;
int rack_num_1 = 1;
// Add 4 nodes in 2 racks
// host_0_0 in rack0
string host_0_0 = "127.0.0.1";
RMNode n1 = MockNodes.NewNodeInfo(rack_num_0, MockNodes.NewResource(4 * Gb), 1, host_0_0
);
fs.Handle(new NodeAddedSchedulerEvent(n1));
// host_0_1 in rack0
string host_0_1 = "127.0.0.2";
RMNode n2 = MockNodes.NewNodeInfo(rack_num_0, MockNodes.NewResource(4 * Gb), 1, host_0_1
);
fs.Handle(new NodeAddedSchedulerEvent(n2));
// host_1_0 in rack1
string host_1_0 = "127.0.0.3";
RMNode n3 = MockNodes.NewNodeInfo(rack_num_1, MockNodes.NewResource(4 * Gb), 1, host_1_0
);
fs.Handle(new NodeAddedSchedulerEvent(n3));
// host_1_1 in rack1
string host_1_1 = "127.0.0.4";
RMNode n4 = MockNodes.NewNodeInfo(rack_num_1, MockNodes.NewResource(4 * Gb), 1, host_1_1
);
fs.Handle(new NodeAddedSchedulerEvent(n4));
// Add one application
ApplicationId appId1 = BuilderUtils.NewApplicationId(100, 1);
ApplicationAttemptId appAttemptId1 = BuilderUtils.NewApplicationAttemptId(appId1,
1);
CreateMockRMApp(appAttemptId1, rm.GetRMContext());
SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");
fs.Handle(appEvent);
SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false
);
fs.Handle(attemptEvent);
IList <ContainerId> emptyId = new AList <ContainerId>();
IList <ResourceRequest> emptyAsk = new AList <ResourceRequest>();
// Allow rack-locality for rack_1, but blacklist host_1_0
// Set up resource requests
// Ask for a 1 GB container for app 1
IList <ResourceRequest> ask1 = new AList <ResourceRequest>();
ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), "rack1"
, BuilderUtils.NewResource(Gb, 1), 1));
ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
.Any, BuilderUtils.NewResource(Gb, 1), 1));
fs.Allocate(appAttemptId1, ask1, emptyId, Sharpen.Collections.SingletonList(host_1_0
), null);
// Trigger container assignment
fs.Handle(new NodeUpdateSchedulerEvent(n3));
// Get the allocation for the application and verify no allocation on blacklist node
Allocation allocation1 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation1", 0, allocation1.GetContainers().Count
);
// verify host_1_1 can get allocated as not in blacklist
fs.Handle(new NodeUpdateSchedulerEvent(n4));
Allocation allocation2 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation2", 1, allocation2.GetContainers().Count
);
IList <Container> containerList = allocation2.GetContainers();
foreach (Container container in containerList)
{
NUnit.Framework.Assert.AreEqual("Container is allocated on n4", container.GetNodeId
(), n4.GetNodeID());
}
// Ask for a 1 GB container again for app 1
IList <ResourceRequest> ask2 = new AList <ResourceRequest>();
// this time, rack0 is also in blacklist, so only host_1_1 is available to
// be assigned
ask2.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
.Any, BuilderUtils.NewResource(Gb, 1), 1));
fs.Allocate(appAttemptId1, ask2, emptyId, Sharpen.Collections.SingletonList("rack0"
), null);
// verify n1 is not qualified to be allocated
fs.Handle(new NodeUpdateSchedulerEvent(n1));
Allocation allocation3 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation3", 0, allocation3.GetContainers().Count
);
// verify n2 is not qualified to be allocated
fs.Handle(new NodeUpdateSchedulerEvent(n2));
Allocation allocation4 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation4", 0, allocation4.GetContainers().Count
);
// verify n3 is not qualified to be allocated
fs.Handle(new NodeUpdateSchedulerEvent(n3));
Allocation allocation5 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation5", 0, allocation5.GetContainers().Count
);
fs.Handle(new NodeUpdateSchedulerEvent(n4));
Allocation allocation6 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation6", 1, allocation6.GetContainers().Count
);
containerList = allocation6.GetContainers();
foreach (Container container_1 in containerList)
{
NUnit.Framework.Assert.AreEqual("Container is allocated on n4", container_1.GetNodeId
(), n4.GetNodeID());
}
rm.Stop();
}
internal QueuesBlock(ResourceManager rm)
{
sinfo = new FifoSchedulerInfo(rm);
fs = (FifoScheduler)rm.GetResourceScheduler();
}
