public IHttpActionResult GetResourceCount(string connectionInfo, string query)
{
try
{
Lazy <IOCGDSRepository> repo = RepositoryManager.GetRepository(repos, "MIMResource");
if (repo != null)
{
ConnectionInfo ci = ConnectionInfo.BuildConnectionInfo(connectionInfo);
ResourceOption ro = new ResourceOption(ci);
int count = repo.Value.GetResourceCount(query, ro);
return(Ok(count));
}
else
{
return(NotFound());
}
}
catch (Exception exp)
{
return(InternalServerError(exp));
}
}
public IHttpActionResult RemoveValuesFromResource(string connectionInfo, string objectID, string attributeName, [FromUri] string[] valuesToRemove)
{
try
{
Lazy <IOCGDSRepository> repo = RepositoryManager.GetRepository(repos, "MIMResource");
if (repo != null)
{
ConnectionInfo ci = ConnectionInfo.BuildConnectionInfo(connectionInfo);
ResourceOption ro = new ResourceOption(ci);
string id = repo.Value.RemoveValuesFromResource(objectID, attributeName, valuesToRemove, ro);
if (id.Equals("AuthorizationRequired"))
{
return(Content(HttpStatusCode.PartialContent, id));
}
return(Ok(id));
}
else
{
return(NotFound());
}
}
catch (Exception exp)
{
return(InternalServerError(exp));
}
}
public IHttpActionResult CreateResource(string connectionInfo, DSResource resource)
{
try
{
Lazy <IOCGDSRepository> repo = RepositoryManager.GetRepository(repos, "MIMResource");
if (repo != null)
{
ConnectionInfo ci = ConnectionInfo.BuildConnectionInfo(connectionInfo);
ResourceOption ro = new ResourceOption(ci);
string id = repo.Value.CreateResource(resource, ro);
return(Ok(id));
}
else
{
return(NotFound());
}
}
catch (Exception exp)
{
return(InternalServerError(exp));
}
}
public IHttpActionResult UpdateResource(string connectionInfo, DSResource resource, bool isDelta = false)
{
try
{
Lazy <IOCGDSRepository> repo = RepositoryManager.GetRepository(repos, "MIMResource");
if (repo != null)
{
ConnectionInfo ci = ConnectionInfo.BuildConnectionInfo(connectionInfo);
ResourceOption ro = new ResourceOption(ci);
string id = repo.Value.UpdateResource(resource, isDelta, ro);
if (id.Equals("AuthorizationRequired"))
{
return(Content(HttpStatusCode.PartialContent, id));
}
return(Ok(id));
}
else
{
return(NotFound());
}
}
catch (Exception exp)
{
return(InternalServerError(exp));
}
}
/// <summary>Process resource update on a node.</summary>
public virtual void UpdateNodeResource(RMNode nm, ResourceOption resourceOption)
{
lock (this)
{
SchedulerNode node = GetSchedulerNode(nm.GetNodeID());
Org.Apache.Hadoop.Yarn.Api.Records.Resource newResource = resourceOption.GetResource
();
Org.Apache.Hadoop.Yarn.Api.Records.Resource oldResource = node.GetTotalResource();
if (!oldResource.Equals(newResource))
{
// Log resource change
Log.Info("Update resource on node: " + node.GetNodeName() + " from: " + oldResource
+ ", to: " + newResource);
Sharpen.Collections.Remove(nodes, nm.GetNodeID());
UpdateMaximumAllocation(node, false);
// update resource to node
node.SetTotalResource(newResource);
nodes[nm.GetNodeID()] = (N)node;
UpdateMaximumAllocation(node, true);
// update resource to clusterResource
Resources.SubtractFrom(clusterResource, oldResource);
Resources.AddTo(clusterResource, newResource);
}
else
{
// Log resource change
Log.Warn("Update resource on node: " + node.GetNodeName() + " with the same resource: "
+ newResource);
}
}
}
public IHttpActionResult GetResourceByQuery(
string connectionInfo, string query, [FromUri] string[] attributesToGet = null,
int pageSize = 0, int index = 0, int cultureKey = 127, bool resolveID = false, bool deepResolve = false,
[FromUri] string[] attributesToResolve = null, [FromUri] string[] attributesToSort = null)
{
try
{
Lazy <IOCGDSRepository> repo = RepositoryManager.GetRepository(repos, "MIMResource");
if (repo != null)
{
ConnectionInfo ci = ConnectionInfo.BuildConnectionInfo(connectionInfo);
ResourceOption ro = new ResourceOption(
ci, cultureKey, resolveID, deepResolve, attributesToResolve, attributesToSort);
DSResourceSet rss = repo.Value.GetResourceByQuery(
query,
(attributesToGet == null || attributesToGet.Length == 0) ? new string[] { "DisplayName" } : attributesToGet,
pageSize,
index,
ro);
return(Ok(rss));
}
else
{
return(NotFound());
}
}
catch (Exception exp)
{
return(InternalServerError(exp));
}
}
public IHttpActionResult GetResourceByID(
string encryptionKey, string id, [FromUri] string[] attributesToGet = null,
bool includePermission = false, int cultureKey = 127, bool resolveID = false, bool deepResolve = false,
[FromUri] string[] attributesToResolve = null)
{
try
{
Lazy <IOCGDSRepository> repo = RepositoryManager.GetRepository(repos, "MIMResource");
if (!encryptionKey.Equals(this.encryptionKey))
{
return(InternalServerError(new Exception("Invalid Encryption Key")));
}
if (repo != null)
{
ConnectionInfo ci = ConnectionInfo.BuildConnectionInfo(this.adminConnection);
ResourceOption ro = new ResourceOption(ci, cultureKey, resolveID, deepResolve, attributesToResolve);
DSResource rs = repo.Value.GetResourceByID(id, (attributesToGet == null || attributesToGet.Length == 0) ? new string[] { "DisplayName" } : attributesToGet, includePermission, ro);
return(Ok(rs));
}
else
{
return(NotFound());
}
}
catch (Exception exp)
{
return(InternalServerError(exp));
}
}
public IHttpActionResult AddValuesToResource(string encryptionKey, string objectID, string attributeName, [FromUri] string[] valuesToAdd)
{
try
{
Lazy <IOCGDSRepository> repo = RepositoryManager.GetRepository(repos, "MIMResource");
if (!encryptionKey.Equals(this.encryptionKey))
{
return(InternalServerError(new Exception("Invalid Encryption Key")));
}
if (repo != null)
{
ConnectionInfo ci = ConnectionInfo.BuildConnectionInfo(this.adminConnection);
ResourceOption ro = new ResourceOption(ci);
string id = repo.Value.AddValuesToResource(objectID, attributeName, valuesToAdd, ro);
if (id.Equals("AuthorizationRequired"))
{
return(Content(HttpStatusCode.PartialContent, id));
}
return(Ok(id));
}
else
{
return(NotFound());
}
}
catch (Exception exp)
{
return(InternalServerError(exp));
}
}
public IHttpActionResult CreateResource(string encryptionKey, DSResource resource)
{
try
{
Lazy <IOCGDSRepository> repo = RepositoryManager.GetRepository(repos, "MIMResource");
if (!encryptionKey.Equals(this.encryptionKey))
{
return(InternalServerError(new Exception("Invalid Encryption Key")));
}
if (repo != null)
{
ConnectionInfo ci = ConnectionInfo.BuildConnectionInfo(this.adminConnection);
ResourceOption ro = new ResourceOption(ci);
string id = repo.Value.CreateResource(resource, ro);
return(Ok(id));
}
else
{
return(NotFound());
}
}
catch (Exception exp)
{
return(InternalServerError(exp));
}
}
public NodeResourceUpdateSchedulerEvent(RMNode rmNode, ResourceOption resourceOption
)
: base(SchedulerEventType.NodeResourceUpdate)
{
this.rmNode = rmNode;
this.resourceOption = resourceOption;
}
public void DeleteResource(string id, ResourceOption resourceOption = null)
{
ResourceOption option = resourceOption == null ? new ResourceOption() : resourceOption;
ResourceManagementClient client = getClient(option.ConnectionInfo);
client.DeleteResource(id);
}
public int GetResourceCount(string query, ResourceOption resourceOption = null)
{
ResourceOption option = resourceOption == null ? new ResourceOption() : resourceOption;
ResourceManagementClient client = getClient(option.ConnectionInfo);
return(client.GetResourceCount(query));
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual UpdateNodeResourceResponse UpdateNodeResource(UpdateNodeResourceRequest
request)
{
string argName = "updateNodeResource";
UserGroupInformation user = CheckAcls(argName);
CheckRMStatus(user.GetShortUserName(), argName, "update node resource.");
IDictionary <NodeId, ResourceOption> nodeResourceMap = request.GetNodeResourceMap(
);
ICollection <NodeId> nodeIds = nodeResourceMap.Keys;
// verify nodes are all valid first.
// if any invalid nodes, throw exception instead of partially updating
// valid nodes.
foreach (NodeId nodeId in nodeIds)
{
RMNode node = this.rmContext.GetRMNodes()[nodeId];
if (node == null)
{
Log.Error("Resource update get failed on all nodes due to change " + "resource on an unrecognized node: "
+ nodeId);
throw RPCUtil.GetRemoteException("Resource update get failed on all nodes due to change resource "
+ "on an unrecognized node: " + nodeId);
}
}
// do resource update on each node.
// Notice: it is still possible to have invalid NodeIDs as nodes decommission
// may happen just at the same time. This time, only log and skip absent
// nodes without throwing any exceptions.
bool allSuccess = true;
foreach (KeyValuePair <NodeId, ResourceOption> entry in nodeResourceMap)
{
ResourceOption newResourceOption = entry.Value;
NodeId nodeId_1 = entry.Key;
RMNode node = this.rmContext.GetRMNodes()[nodeId_1];
if (node == null)
{
Log.Warn("Resource update get failed on an unrecognized node: " + nodeId_1);
allSuccess = false;
}
else
{
// update resource to RMNode
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeResourceUpdateEvent
(nodeId_1, newResourceOption));
Log.Info("Update resource on node(" + node.GetNodeID() + ") with resource(" + newResourceOption
.ToString() + ")");
}
}
if (allSuccess)
{
RMAuditLogger.LogSuccess(user.GetShortUserName(), argName, "AdminService");
}
UpdateNodeResourceResponse response = UpdateNodeResourceResponse.NewInstance();
return(response);
}
public virtual void TestMaxAllocationAfterUpdateNodeResource()
{
int configuredMaxVCores = 20;
int configuredMaxMemory = 10 * 1024;
Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(configuredMaxMemory, configuredMaxVCores);
ConfigureScheduler();
YarnConfiguration conf = GetConf();
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores
);
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory
);
conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0);
MockRM rm = new MockRM(conf);
try
{
rm.Start();
AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler(
);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 5);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(4096, 10);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource3 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(512, 1);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource4 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 2);
RMNode node1 = MockNodes.NewNodeInfo(0, resource1, 1, "127.0.0.2");
scheduler.Handle(new NodeAddedSchedulerEvent(node1));
RMNode node2 = MockNodes.NewNodeInfo(0, resource3, 2, "127.0.0.3");
scheduler.Handle(new NodeAddedSchedulerEvent(node2));
VerifyMaximumResourceCapability(resource1, scheduler);
// increase node1 resource
scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource2, 0));
VerifyMaximumResourceCapability(resource2, scheduler);
// decrease node1 resource
scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource1, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
// increase node2 resource
scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource4, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
// decrease node2 resource
scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource3, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
}
finally
{
rm.Stop();
}
}
public DSResourceSet GetResourceByQuery(string query, string[] attributes,
int pageSize = 0, int index = 0, ResourceOption resourceOption = null)
{
ResourceOption option = resourceOption == null ? new ResourceOption() : resourceOption;
ResourceManagementClient client = getClient(option.ConnectionInfo);
client.RefreshSchema();
DSResourceSet retVal = new DSResourceSet();
List <SortingAttribute> sortingAttributes = getSortingAttributes(option.SortingAttributes);
if (pageSize == 0)
{
SearchResultCollection src = sortingAttributes.Count == 0 ?
client.GetResources(query, attributes) as SearchResultCollection :
client.GetResources(query, attributes, sortingAttributes) as SearchResultCollection;
if (src != null)
{
retVal.TotalCount = src.Count;
foreach (ResourceObject resource in src)
{
retVal.Resources.Add(convertToDSResource(client, resource, attributes, false, option));
}
}
}
else
{
SearchResultPager srp = sortingAttributes.Count == 0 ?
client.GetResourcesPaged(query, pageSize, attributes) :
client.GetResourcesPaged(query, pageSize, attributes, sortingAttributes);
if (index >= 0)
{
srp.CurrentIndex = index;
}
srp.PageSize = pageSize;
foreach (ResourceObject resource in srp.GetNextPage())
{
retVal.Resources.Add(convertToDSResource(client, resource, attributes, false, option));
}
retVal.TotalCount = srp.TotalCount;
retVal.HasMoreItems = srp.HasMoreItems;
}
return(retVal);
}
public DSResource GetResourceByID(
string id, string[] attributes, bool includePermission = false, ResourceOption resourceOption = null)
{
ResourceOption option = resourceOption == null ? new ResourceOption() : resourceOption;
ResourceManagementClient client = getClient(option.ConnectionInfo);
client.RefreshSchema();
ResourceObject resource = client.GetResource(id, attributes, includePermission);
return(convertToDSResource(client, resource, attributes, includePermission, resourceOption));
}
public string CreateResource(DSResource resource, ResourceOption resourceOption = null)
{
ResourceOption option = resourceOption == null ? new ResourceOption() : resourceOption;
ResourceManagementClient client = getClient(option.ConnectionInfo);
ResourceObject objResource = client.CreateResource(resource.ObjectType);
convertToResourceObject(resource, ref objResource);
objResource.Save();
return(objResource.ObjectID.Value);
}
public IHttpActionResult GetCurrentUser([FromUri] string[] attributesToGet = null)
{
WindowsImpersonationContext wic = null;
try
{
wic = ((WindowsIdentity)User.Identity).Impersonate();
Lazy <IOCGDSRepository> repo = RepositoryManager.GetRepository(repos, "MIMResource");
if (repo != null)
{
ResourceOption ro = new ResourceOption();
string userName = HttpContext.Current.User.Identity.Name;
int pos = userName.IndexOf(@"\");
string accountName = userName.Substring(pos + 1);
string query = string.Format(@"/Person[AccountName='{0}']", accountName);
DSResourceSet rss = repo.Value.GetResourceByQuery(query,
(attributesToGet == null || attributesToGet.Length == 0) ?
new string[] { "DisplayName" } : attributesToGet);
if (rss != null && rss.TotalCount == 1)
{
return(Ok(rss.Resources.First()));
}
else
{
return(InternalServerError(new Exception($"account not found: {accountName}")));
}
}
else
{
return(NotFound());
}
}
catch (Exception exp)
{
return(InternalServerError(exp));
}
finally
{
if (wic != null)
{
wic.Undo();
}
}
}
private void LoadBuildParamaters()
{
ResourceOption resourceOps = new ResourceOption();
foreach (DataItemModel arg in resourceOps.ScriptParameterOptions())
{
TextBoxItem item = new TextBoxItem
{
Text = arg.ItemKey.ToString(),
ToolTip = arg.DisplayValue,
IsReadOnly = true,
BorderThickness = new Thickness(0)
};
this.uxParametersList.Children.Add(item);
}
}
public IHttpActionResult GetResourceByQuery(
string query, [FromUri] string[] attributesToGet = null, int pageSize = 0, int index = 0,
int cultureKey = 127, bool resolveID = false, bool deepResolve = false,
[FromUri] string[] attributesToResolve = null, [FromUri] string[] attributesToSort = null)
{
WindowsImpersonationContext wic = null;
try
{
wic = ((WindowsIdentity)User.Identity).Impersonate();
Lazy <IOCGDSRepository> repo = RepositoryManager.GetRepository(repos, "MIMResource");
if (repo != null)
{
ResourceOption ro = new ResourceOption(
new ConnectionInfo(), cultureKey, resolveID, deepResolve, attributesToResolve, attributesToSort);
DSResourceSet rss = repo.Value.GetResourceByQuery(
query,
(attributesToGet == null || attributesToGet.Length == 0) ? new string[] { "DisplayName" } : attributesToGet,
pageSize,
index,
ro);
return(Ok(rss));
}
else
{
return(NotFound());
}
}
catch (Exception exp)
{
return(InternalServerError(exp));
}
finally
{
if (wic != null)
{
wic.Undo();
}
}
}
public virtual void TestResourceUpdateOnRebootedNode()
{
RMNodeImpl node = GetRebootedNode();
Resource oldCapacity = node.GetTotalCapability();
NUnit.Framework.Assert.AreEqual("Memory resource is not match.", oldCapacity.GetMemory
(), 4096);
NUnit.Framework.Assert.AreEqual("CPU resource is not match.", oldCapacity.GetVirtualCores
(), 4);
node.Handle(new RMNodeResourceUpdateEvent(node.GetNodeID(), ResourceOption.NewInstance
(Resource.NewInstance(2048, 2), RMNode.OverCommitTimeoutMillisDefault)));
Resource newCapacity = node.GetTotalCapability();
NUnit.Framework.Assert.AreEqual("Memory resource is not match.", newCapacity.GetMemory
(), 2048);
NUnit.Framework.Assert.AreEqual("CPU resource is not match.", newCapacity.GetVirtualCores
(), 2);
NUnit.Framework.Assert.AreEqual(NodeState.Rebooted, node.GetState());
}
public string UpdateResource(DSResource resource, bool isDelta = false, ResourceOption resourceOption = null)
{
ResourceOption option = resourceOption == null ? new ResourceOption() : resourceOption;
ResourceManagementClient client = getClient(option.ConnectionInfo);
ResourceObject objResource = client.CreateResourceTemplateForUpdate(
resource.ObjectType, new UniqueIdentifier(resource.ObjectID));
convertToResourceObject(resource, ref objResource, isDelta);
try
{
objResource.Save();
}
catch (AuthorizationRequiredException)
{
return("AuthorizationRequired");
}
return(objResource.ObjectID.Value);
}
public string RemoveValuesFromResource(string id, string attributeName, string[] valuesToRemove, ResourceOption resourceOption = null)
{
if (valuesToRemove == null || valuesToRemove.Length == 0)
{
return(id);
}
ResourceOption option = resourceOption == null ? new ResourceOption() : resourceOption;
ResourceManagementClient client = getClient(option.ConnectionInfo);
client.RefreshSchema();
ResourceObject objResource = client.GetResource(id, new string[] { attributeName });
if (objResource == null)
{
throw new Exception($"No Resource was found with ObjectID: {id}");
}
foreach (string value in valuesToRemove)
{
objResource.RemoveValue(attributeName, value);
}
try
{
objResource.Save();
}
catch (AuthorizationRequiredException)
{
return("AuthorizationRequired");
}
return(objResource.ObjectID.Value);
}
public void DeleteResource(string id, ResourceOption resourceOption = null)
{
throw new NotImplementedException();
}
private YarnProtos.ResourceOptionProto ConvertToProtoFormat(ResourceOption c)
{
return(((ResourceOptionPBImpl)c).GetProto());
}
public string CreateResource(DSResource resource, ResourceOption resourceOption = null)
{
throw new NotImplementedException();
}
public string UpdateResource(DSResource resource, bool isDelta = false, ResourceOption resourceOption = null)
{
throw new NotImplementedException();
}
public string RemoveValuesFromResource(string id, string attributeName, string[] valuesToRemove, ResourceOption resourceOption = null)
{
throw new NotImplementedException();
}
public DSResource convertToDSResource(ResourceManagementClient client, ResourceObject resource,
string[] loadedAttributes, bool includePermission, ResourceOption option, bool deepResolve = true)
{
DSResource dsResource = new DSResource
{
DisplayName = resource.DisplayName,
ObjectID = resource.ObjectID.Value,
ObjectType = resource.ObjectTypeName,
HasPermissionHints = resource.HasPermissionHints
};
List <RmAttribute> attributeDef = getAttributeDefinition(resource.ObjectTypeName, option.CultureKey, option.ConnectionInfo.EncryptionKey);
Dictionary <string, DSAttribute> attributes = new Dictionary <string, DSAttribute>();
foreach (string attributeName in loadedAttributes)
{
if (resource.Attributes.ContainsAttribute(attributeName))
{
AttributeValue attValue = resource.Attributes[attributeName];
if (attValue.Attribute.SystemName.Equals("ObjectID") || attValue.Attribute.SystemName.Equals("ObjectType"))
{
continue;
}
DSAttribute dsAttribute = new DSAttribute
{
Description = attValue.Attribute.Description,
DisplayName = attValue.Attribute.DisplayName,
IsMultivalued = attValue.Attribute.IsMultivalued,
IsReadOnly = attValue.Attribute.IsReadOnly,
IsRequired = attValue.Attribute.IsRequired,
Regex = attValue.Attribute.Regex,
SystemName = attValue.Attribute.SystemName,
Type = attValue.Attribute.Type.ToString(),
IsNull = attValue.IsNull,
PermissionHint = attValue.PermissionHint.ToString(),
Value = attValue.Value,
Values = attValue.Values.ToList()
};
if (attributeDef != null)
{
RmAttribute attr = attributeDef.FirstOrDefault(a => a.Name.Equals(attValue.AttributeName));
if (attr != null)
{
dsAttribute.DisplayName = attr.DisplayName;
dsAttribute.Description = attr.Description;
}
}
if (!dsAttribute.IsNull && dsAttribute.Type.Equals("Reference"))
{
dsAttribute.Value = attValue.Attribute.IsMultivalued ?
attValue.StringValues.FirstOrDefault() : attValue.StringValue;
dsAttribute.Values = attValue.Attribute.IsMultivalued ?
attValue.StringValues.ToList <object>() : new List <object>()
{
attValue.StringValue
};
if (!string.IsNullOrEmpty(dsAttribute.Value.ToString()) && deepResolve && option.ResolveID)
{
if (dsAttribute.IsMultivalued)
{
foreach (string value in attValue.StringValues)
{
ResourceObject resolvedObject = client.GetResource(
value, option.AttributesToResolve, includePermission);
dsAttribute.ResolvedValues.Add(convertToDSResource(client,
resolvedObject, option.AttributesToResolve, includePermission, option, option.DeepResolve));
}
}
else
{
ResourceObject resolvedObject = client.GetResource(
attValue.StringValue, option.AttributesToResolve, includePermission);
dsAttribute.ResolvedValue = convertToDSResource(client,
resolvedObject, option.AttributesToResolve, includePermission, option, option.DeepResolve);
}
}
}
attributes.Add(attValue.AttributeName, dsAttribute);
}
}
dsResource.Attributes = attributes;
return(dsResource);
}
public virtual void TestResourceOverCommit()
{
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 4 * Gb);
RMApp app1 = rm.SubmitApp(2048);
// kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId
());
// check node report, 2 GB used and 2 GB available
NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm1.GetUsedResource().GetMemory());
NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm1.GetAvailableResource().GetMemory
());
// add request for containers
am1.AddRequests(new string[] { "127.0.0.1", "127.0.0.2" }, 2 * Gb, 1, 1);
AllocateResponse alloc1Response = am1.Schedule();
// send the request
// kick the scheduler, 2 GB given to AM1, resource remaining 0
nm1.NodeHeartbeat(true);
while (alloc1Response.GetAllocatedContainers().Count < 1)
{
Log.Info("Waiting for containers to be created for app 1...");
Sharpen.Thread.Sleep(1000);
alloc1Response = am1.Schedule();
}
IList <Container> allocated1 = alloc1Response.GetAllocatedContainers();
NUnit.Framework.Assert.AreEqual(1, allocated1.Count);
NUnit.Framework.Assert.AreEqual(2 * Gb, allocated1[0].GetResource().GetMemory());
NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), allocated1[0].GetNodeId());
report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId());
// check node report, 4 GB used and 0 GB available
NUnit.Framework.Assert.AreEqual(0, report_nm1.GetAvailableResource().GetMemory());
NUnit.Framework.Assert.AreEqual(4 * Gb, report_nm1.GetUsedResource().GetMemory());
// check container is assigned with 2 GB.
Container c1 = allocated1[0];
NUnit.Framework.Assert.AreEqual(2 * Gb, c1.GetResource().GetMemory());
// update node resource to 2 GB, so resource is over-consumed.
IDictionary <NodeId, ResourceOption> nodeResourceMap = new Dictionary <NodeId, ResourceOption
>();
nodeResourceMap[nm1.GetNodeId()] = ResourceOption.NewInstance(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2 * Gb, 1), -1);
UpdateNodeResourceRequest request = UpdateNodeResourceRequest.NewInstance(nodeResourceMap
);
AdminService @as = rm.adminService;
@as.UpdateNodeResource(request);
// Now, the used resource is still 4 GB, and available resource is minus value.
report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId());
NUnit.Framework.Assert.AreEqual(4 * Gb, report_nm1.GetUsedResource().GetMemory());
NUnit.Framework.Assert.AreEqual(-2 * Gb, report_nm1.GetAvailableResource().GetMemory
());
// Check container can complete successfully in case of resource over-commitment.
ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(c1.GetId(), ContainerState
.Complete, string.Empty, 0);
nm1.ContainerStatus(containerStatus);
int waitCount = 0;
while (attempt1.GetJustFinishedContainers().Count < 1 && waitCount++ != 20)
{
Log.Info("Waiting for containers to be finished for app 1... Tried " + waitCount
+ " times already..");
Sharpen.Thread.Sleep(100);
}
NUnit.Framework.Assert.AreEqual(1, attempt1.GetJustFinishedContainers().Count);
NUnit.Framework.Assert.AreEqual(1, am1.Schedule().GetCompletedContainersStatuses(
).Count);
report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId());
NUnit.Framework.Assert.AreEqual(2 * Gb, report_nm1.GetUsedResource().GetMemory());
// As container return 2 GB back, the available resource becomes 0 again.
NUnit.Framework.Assert.AreEqual(0 * Gb, report_nm1.GetAvailableResource().GetMemory
());
rm.Stop();
}
