public virtual void TestUpdatingOldEntityWithoutDomainId()
{
// Set the domain to the default domain when updating
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType("OLD_ENTITY_TYPE_1");
entity.SetEntityId("OLD_ENTITY_ID_1");
entity.SetDomainId(TimelineDataManager.DefaultDomainId);
entity.AddOtherInfo("NEW_OTHER_INFO_KEY", "NEW_OTHER_INFO_VALUE");
TimelineEntities entities = new TimelineEntities();
entities.AddEntity(entity);
TimelinePutResponse response = dataManaer.PostEntities(entities, UserGroupInformation
.GetCurrentUser());
NUnit.Framework.Assert.AreEqual(0, response.GetErrors().Count);
entity = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
NUnit.Framework.Assert.IsNotNull(entity);
// Even in leveldb, the domain is updated to the default domain Id
NUnit.Framework.Assert.AreEqual(TimelineDataManager.DefaultDomainId, entity.GetDomainId
());
NUnit.Framework.Assert.AreEqual(1, entity.GetOtherInfo().Count);
NUnit.Framework.Assert.AreEqual("NEW_OTHER_INFO_KEY", entity.GetOtherInfo().Keys.
GetEnumerator().Next());
NUnit.Framework.Assert.AreEqual("NEW_OTHER_INFO_VALUE", entity.GetOtherInfo().Values
.GetEnumerator().Next());
// Set the domain to the non-default domain when updating
entity = new TimelineEntity();
entity.SetEntityType("OLD_ENTITY_TYPE_1");
entity.SetEntityId("OLD_ENTITY_ID_2");
entity.SetDomainId("NON_DEFAULT");
entity.AddOtherInfo("NEW_OTHER_INFO_KEY", "NEW_OTHER_INFO_VALUE");
entities = new TimelineEntities();
entities.AddEntity(entity);
response = dataManaer.PostEntities(entities, UserGroupInformation.GetCurrentUser(
));
NUnit.Framework.Assert.AreEqual(1, response.GetErrors().Count);
NUnit.Framework.Assert.AreEqual(TimelinePutResponse.TimelinePutError.AccessDenied
, response.GetErrors()[0].GetErrorCode());
entity = store.GetEntity("OLD_ENTITY_ID_2", "OLD_ENTITY_TYPE_1", null);
NUnit.Framework.Assert.IsNotNull(entity);
// In leveldb, the domain Id is still null
NUnit.Framework.Assert.IsNull(entity.GetDomainId());
// Updating is not executed
NUnit.Framework.Assert.AreEqual(0, entity.GetOtherInfo().Count);
}
private static TimelineEntity MaskFields(TimelineEntity entity, EnumSet <TimelineReader.Field
> fields)
{
// Conceal the fields that are not going to be exposed
TimelineEntity entityToReturn = new TimelineEntity();
entityToReturn.SetEntityId(entity.GetEntityId());
entityToReturn.SetEntityType(entity.GetEntityType());
entityToReturn.SetStartTime(entity.GetStartTime());
entityToReturn.SetDomainId(entity.GetDomainId());
// Deep copy
if (fields.Contains(TimelineReader.Field.Events))
{
entityToReturn.AddEvents(entity.GetEvents());
}
else
{
if (fields.Contains(TimelineReader.Field.LastEventOnly))
{
entityToReturn.AddEvent(entity.GetEvents()[0]);
}
else
{
entityToReturn.SetEvents(null);
}
}
if (fields.Contains(TimelineReader.Field.RelatedEntities))
{
entityToReturn.AddRelatedEntities(entity.GetRelatedEntities());
}
else
{
entityToReturn.SetRelatedEntities(null);
}
if (fields.Contains(TimelineReader.Field.PrimaryFilters))
{
entityToReturn.AddPrimaryFilters(entity.GetPrimaryFilters());
}
else
{
entityToReturn.SetPrimaryFilters(null);
}
if (fields.Contains(TimelineReader.Field.OtherInfo))
{
entityToReturn.AddOtherInfo(entity.GetOtherInfo());
}
else
{
entityToReturn.SetOtherInfo(null);
}
return(entityToReturn);
}
/// <exception cref="System.IO.IOException"/>
public virtual TimelineEntities GetEntities(string entityType, long limit, long windowStart
, long windowEnd, string fromId, long fromTs, NameValuePair primaryFilter, ICollection
<NameValuePair> secondaryFilters, EnumSet <TimelineReader.Field> fields, TimelineDataManager.CheckAcl
checkAcl)
{
lock (this)
{
if (limit == null)
{
limit = DefaultLimit;
}
if (windowStart == null)
{
windowStart = long.MinValue;
}
if (windowEnd == null)
{
windowEnd = long.MaxValue;
}
if (fields == null)
{
fields = EnumSet.AllOf <TimelineReader.Field>();
}
IEnumerator <TimelineEntity> entityIterator = null;
if (fromId != null)
{
TimelineEntity firstEntity = entities[new EntityIdentifier(fromId, entityType)];
if (firstEntity == null)
{
return(new TimelineEntities());
}
else
{
entityIterator = new TreeSet <TimelineEntity>(entities.Values).TailSet(firstEntity
, true).GetEnumerator();
}
}
if (entityIterator == null)
{
entityIterator = new PriorityQueue <TimelineEntity>(entities.Values).GetEnumerator
();
}
IList <TimelineEntity> entitiesSelected = new AList <TimelineEntity>();
while (entityIterator.HasNext())
{
TimelineEntity entity = entityIterator.Next();
if (entitiesSelected.Count >= limit)
{
break;
}
if (!entity.GetEntityType().Equals(entityType))
{
continue;
}
if (entity.GetStartTime() <= windowStart)
{
continue;
}
if (entity.GetStartTime() > windowEnd)
{
continue;
}
if (fromTs != null && entityInsertTimes[new EntityIdentifier(entity.GetEntityId()
, entity.GetEntityType())] > fromTs)
{
continue;
}
if (primaryFilter != null && !MatchPrimaryFilter(entity.GetPrimaryFilters(), primaryFilter
))
{
continue;
}
if (secondaryFilters != null)
{
// AND logic
bool flag = true;
foreach (NameValuePair secondaryFilter in secondaryFilters)
{
if (secondaryFilter != null && !MatchPrimaryFilter(entity.GetPrimaryFilters(), secondaryFilter
) && !MatchFilter(entity.GetOtherInfo(), secondaryFilter))
{
flag = false;
break;
}
}
if (!flag)
{
continue;
}
}
if (entity.GetDomainId() == null)
{
entity.SetDomainId(TimelineDataManager.DefaultDomainId);
}
if (checkAcl == null || checkAcl.Check(entity))
{
entitiesSelected.AddItem(entity);
}
}
IList <TimelineEntity> entitiesToReturn = new AList <TimelineEntity>();
foreach (TimelineEntity entitySelected in entitiesSelected)
{
entitiesToReturn.AddItem(MaskFields(entitySelected, fields));
}
entitiesToReturn.Sort();
TimelineEntities entitiesWrapper = new TimelineEntities();
entitiesWrapper.SetEntities(entitiesToReturn);
return(entitiesWrapper);
}
}
private static ContainerReport ConvertToContainerReport(TimelineEntity entity, string
serverHttpAddress, string user)
{
int allocatedMem = 0;
int allocatedVcore = 0;
string allocatedHost = null;
int allocatedPort = -1;
int allocatedPriority = 0;
long createdTime = 0;
long finishedTime = 0;
string diagnosticsInfo = null;
int exitStatus = ContainerExitStatus.Invalid;
ContainerState state = null;
string nodeHttpAddress = null;
IDictionary <string, object> entityInfo = entity.GetOtherInfo();
if (entityInfo != null)
{
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedMemoryEntityInfo))
{
allocatedMem = (int)entityInfo[ContainerMetricsConstants.AllocatedMemoryEntityInfo
];
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedVcoreEntityInfo))
{
allocatedVcore = (int)entityInfo[ContainerMetricsConstants.AllocatedVcoreEntityInfo
];
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedHostEntityInfo))
{
allocatedHost = entityInfo[ContainerMetricsConstants.AllocatedHostEntityInfo].ToString
();
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedPortEntityInfo))
{
allocatedPort = (int)entityInfo[ContainerMetricsConstants.AllocatedPortEntityInfo
];
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedPriorityEntityInfo))
{
allocatedPriority = (int)entityInfo[ContainerMetricsConstants.AllocatedPriorityEntityInfo
];
}
if (entityInfo.Contains(ContainerMetricsConstants.AllocatedHostHttpAddressEntityInfo
))
{
nodeHttpAddress = (string)entityInfo[ContainerMetricsConstants.AllocatedHostHttpAddressEntityInfo
];
}
}
IList <TimelineEvent> events = entity.GetEvents();
if (events != null)
{
foreach (TimelineEvent @event in events)
{
if (@event.GetEventType().Equals(ContainerMetricsConstants.CreatedEventType))
{
createdTime = @event.GetTimestamp();
}
else
{
if (@event.GetEventType().Equals(ContainerMetricsConstants.FinishedEventType))
{
finishedTime = @event.GetTimestamp();
IDictionary <string, object> eventInfo = @event.GetEventInfo();
if (eventInfo == null)
{
continue;
}
if (eventInfo.Contains(ContainerMetricsConstants.DiagnosticsInfoEventInfo))
{
diagnosticsInfo = eventInfo[ContainerMetricsConstants.DiagnosticsInfoEventInfo].ToString
();
}
if (eventInfo.Contains(ContainerMetricsConstants.ExitStatusEventInfo))
{
exitStatus = (int)eventInfo[ContainerMetricsConstants.ExitStatusEventInfo];
}
if (eventInfo.Contains(ContainerMetricsConstants.StateEventInfo))
{
state = ContainerState.ValueOf(eventInfo[ContainerMetricsConstants.StateEventInfo
].ToString());
}
}
}
}
}
NodeId allocatedNode = NodeId.NewInstance(allocatedHost, allocatedPort);
ContainerId containerId = ConverterUtils.ToContainerId(entity.GetEntityId());
string logUrl = WebAppUtils.GetAggregatedLogURL(serverHttpAddress, allocatedNode.
ToString(), containerId.ToString(), containerId.ToString(), user);
return(ContainerReport.NewInstance(ConverterUtils.ToContainerId(entity.GetEntityId
()), Resource.NewInstance(allocatedMem, allocatedVcore), NodeId.NewInstance(allocatedHost
, allocatedPort), Priority.NewInstance(allocatedPriority), createdTime, finishedTime
, diagnosticsInfo, logUrl, exitStatus, state, nodeHttpAddress));
}
private static ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt ConvertToApplicationReport
(TimelineEntity entity, ApplicationHistoryManagerOnTimelineStore.ApplicationReportField
field)
{
string user = null;
string queue = null;
string name = null;
string type = null;
long createdTime = 0;
long finishedTime = 0;
ApplicationAttemptId latestApplicationAttemptId = null;
string diagnosticsInfo = null;
FinalApplicationStatus finalStatus = FinalApplicationStatus.Undefined;
YarnApplicationState state = null;
ApplicationResourceUsageReport appResources = null;
IDictionary <ApplicationAccessType, string> appViewACLs = new Dictionary <ApplicationAccessType
, string>();
IDictionary <string, object> entityInfo = entity.GetOtherInfo();
if (entityInfo != null)
{
if (entityInfo.Contains(ApplicationMetricsConstants.UserEntityInfo))
{
user = entityInfo[ApplicationMetricsConstants.UserEntityInfo].ToString();
}
if (entityInfo.Contains(ApplicationMetricsConstants.AppViewAclsEntityInfo))
{
string appViewACLsStr = entityInfo[ApplicationMetricsConstants.AppViewAclsEntityInfo
].ToString();
if (appViewACLsStr.Length > 0)
{
appViewACLs[ApplicationAccessType.ViewApp] = appViewACLsStr;
}
}
if (field == ApplicationHistoryManagerOnTimelineStore.ApplicationReportField.UserAndAcls)
{
return(new ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt(ApplicationReport
.NewInstance(ConverterUtils.ToApplicationId(entity.GetEntityId()), latestApplicationAttemptId
, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime,
finishedTime, finalStatus, null, null, 1.0F, type, null), appViewACLs));
}
if (entityInfo.Contains(ApplicationMetricsConstants.QueueEntityInfo))
{
queue = entityInfo[ApplicationMetricsConstants.QueueEntityInfo].ToString();
}
if (entityInfo.Contains(ApplicationMetricsConstants.NameEntityInfo))
{
name = entityInfo[ApplicationMetricsConstants.NameEntityInfo].ToString();
}
if (entityInfo.Contains(ApplicationMetricsConstants.TypeEntityInfo))
{
type = entityInfo[ApplicationMetricsConstants.TypeEntityInfo].ToString();
}
if (entityInfo.Contains(ApplicationMetricsConstants.AppCpuMetrics))
{
long vcoreSeconds = long.Parse(entityInfo[ApplicationMetricsConstants.AppCpuMetrics
].ToString());
long memorySeconds = long.Parse(entityInfo[ApplicationMetricsConstants.AppMemMetrics
].ToString());
appResources = ApplicationResourceUsageReport.NewInstance(0, 0, null, null, null,
memorySeconds, vcoreSeconds);
}
}
IList <TimelineEvent> events = entity.GetEvents();
if (events != null)
{
foreach (TimelineEvent @event in events)
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType))
{
createdTime = @event.GetTimestamp();
}
else
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType))
{
finishedTime = @event.GetTimestamp();
IDictionary <string, object> eventInfo = @event.GetEventInfo();
if (eventInfo == null)
{
continue;
}
if (eventInfo.Contains(ApplicationMetricsConstants.LatestAppAttemptEventInfo))
{
latestApplicationAttemptId = ConverterUtils.ToApplicationAttemptId(eventInfo[ApplicationMetricsConstants
.LatestAppAttemptEventInfo].ToString());
}
if (eventInfo.Contains(ApplicationMetricsConstants.DiagnosticsInfoEventInfo))
{
diagnosticsInfo = eventInfo[ApplicationMetricsConstants.DiagnosticsInfoEventInfo]
.ToString();
}
if (eventInfo.Contains(ApplicationMetricsConstants.FinalStatusEventInfo))
{
finalStatus = FinalApplicationStatus.ValueOf(eventInfo[ApplicationMetricsConstants
.FinalStatusEventInfo].ToString());
}
if (eventInfo.Contains(ApplicationMetricsConstants.StateEventInfo))
{
state = YarnApplicationState.ValueOf(eventInfo[ApplicationMetricsConstants.StateEventInfo
].ToString());
}
}
}
}
}
return(new ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt(ApplicationReport
.NewInstance(ConverterUtils.ToApplicationId(entity.GetEntityId()), latestApplicationAttemptId
, user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime,
finishedTime, finalStatus, appResources, null, 1.0F, type, null), appViewACLs));
}
/// <exception cref="System.Exception"/>
public virtual void TestPublishApplicationMetrics()
{
for (int i = 1; i <= 2; ++i)
{
ApplicationId appId = ApplicationId.NewInstance(0, i);
RMApp app = CreateRMApp(appId);
metricsPublisher.AppCreated(app, app.GetStartTime());
metricsPublisher.AppFinished(app, RMAppState.Finished, app.GetFinishTime());
if (i == 1)
{
metricsPublisher.AppACLsUpdated(app, "uers1,user2", 4L);
}
else
{
// in case user doesn't specify the ACLs
metricsPublisher.AppACLsUpdated(app, null, 4L);
}
TimelineEntity entity = null;
do
{
entity = store.GetEntity(appId.ToString(), ApplicationMetricsConstants.EntityType
, EnumSet.AllOf <TimelineReader.Field>());
}while (entity == null || entity.GetEvents().Count < 3);
// ensure three events are both published before leaving the loop
// verify all the fields
NUnit.Framework.Assert.AreEqual(ApplicationMetricsConstants.EntityType, entity.GetEntityType
());
NUnit.Framework.Assert.AreEqual(app.GetApplicationId().ToString(), entity.GetEntityId
());
NUnit.Framework.Assert.AreEqual(app.GetName(), entity.GetOtherInfo()[ApplicationMetricsConstants
.NameEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetQueue(), entity.GetOtherInfo()[ApplicationMetricsConstants
.QueueEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetUser(), entity.GetOtherInfo()[ApplicationMetricsConstants
.UserEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetApplicationType(), entity.GetOtherInfo()[ApplicationMetricsConstants
.TypeEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), entity.GetOtherInfo()[ApplicationMetricsConstants
.SubmittedTimeEntityInfo]);
if (i == 1)
{
NUnit.Framework.Assert.AreEqual("uers1,user2", entity.GetOtherInfo()[ApplicationMetricsConstants
.AppViewAclsEntityInfo]);
}
else
{
NUnit.Framework.Assert.AreEqual(string.Empty, entity.GetOtherInfo()[ApplicationMetricsConstants
.AppViewAclsEntityInfo]);
NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetMemorySeconds(), long.Parse
(entity.GetOtherInfo()[ApplicationMetricsConstants.AppMemMetrics].ToString()));
NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetVcoreSeconds(), long.Parse
(entity.GetOtherInfo()[ApplicationMetricsConstants.AppCpuMetrics].ToString()));
}
bool hasCreatedEvent = false;
bool hasFinishedEvent = false;
bool hasACLsUpdatedEvent = false;
foreach (TimelineEvent @event in entity.GetEvents())
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType))
{
hasCreatedEvent = true;
NUnit.Framework.Assert.AreEqual(app.GetStartTime(), @event.GetTimestamp());
}
else
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType))
{
hasFinishedEvent = true;
NUnit.Framework.Assert.AreEqual(app.GetFinishTime(), @event.GetTimestamp());
NUnit.Framework.Assert.AreEqual(app.GetDiagnostics().ToString(), @event.GetEventInfo
()[ApplicationMetricsConstants.DiagnosticsInfoEventInfo]);
NUnit.Framework.Assert.AreEqual(app.GetFinalApplicationStatus().ToString(), @event
.GetEventInfo()[ApplicationMetricsConstants.FinalStatusEventInfo]);
NUnit.Framework.Assert.AreEqual(YarnApplicationState.Finished.ToString(), @event.
GetEventInfo()[ApplicationMetricsConstants.StateEventInfo]);
}
else
{
if (@event.GetEventType().Equals(ApplicationMetricsConstants.AclsUpdatedEventType
))
{
hasACLsUpdatedEvent = true;
NUnit.Framework.Assert.AreEqual(4L, @event.GetTimestamp());
}
}
}
}
NUnit.Framework.Assert.IsTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent
);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestPublishContainerMetrics()
{
ContainerId containerId = ContainerId.NewContainerId(ApplicationAttemptId.NewInstance
(ApplicationId.NewInstance(0, 1), 1), 1);
RMContainer container = CreateRMContainer(containerId);
metricsPublisher.ContainerCreated(container, container.GetCreationTime());
metricsPublisher.ContainerFinished(container, container.GetFinishTime());
TimelineEntity entity = null;
do
{
entity = store.GetEntity(containerId.ToString(), ContainerMetricsConstants.EntityType
, EnumSet.AllOf <TimelineReader.Field>());
}while (entity == null || entity.GetEvents().Count < 2);
// ensure two events are both published before leaving the loop
// verify all the fields
NUnit.Framework.Assert.AreEqual(ContainerMetricsConstants.EntityType, entity.GetEntityType
());
NUnit.Framework.Assert.AreEqual(containerId.ToString(), entity.GetEntityId());
NUnit.Framework.Assert.AreEqual(containerId.GetApplicationAttemptId().ToString(),
entity.GetPrimaryFilters()[ContainerMetricsConstants.ParentPrimariyFilter].GetEnumerator
().Next());
NUnit.Framework.Assert.AreEqual(container.GetAllocatedNode().GetHost(), entity.GetOtherInfo
()[ContainerMetricsConstants.AllocatedHostEntityInfo]);
NUnit.Framework.Assert.AreEqual(container.GetAllocatedNode().GetPort(), entity.GetOtherInfo
()[ContainerMetricsConstants.AllocatedPortEntityInfo]);
NUnit.Framework.Assert.AreEqual(container.GetAllocatedResource().GetMemory(), entity
.GetOtherInfo()[ContainerMetricsConstants.AllocatedMemoryEntityInfo]);
NUnit.Framework.Assert.AreEqual(container.GetAllocatedResource().GetVirtualCores(
), entity.GetOtherInfo()[ContainerMetricsConstants.AllocatedVcoreEntityInfo]);
NUnit.Framework.Assert.AreEqual(container.GetAllocatedPriority().GetPriority(), entity
.GetOtherInfo()[ContainerMetricsConstants.AllocatedPriorityEntityInfo]);
bool hasCreatedEvent = false;
bool hasFinishedEvent = false;
foreach (TimelineEvent @event in entity.GetEvents())
{
if (@event.GetEventType().Equals(ContainerMetricsConstants.CreatedEventType))
{
hasCreatedEvent = true;
NUnit.Framework.Assert.AreEqual(container.GetCreationTime(), @event.GetTimestamp(
));
}
else
{
if (@event.GetEventType().Equals(ContainerMetricsConstants.FinishedEventType))
{
hasFinishedEvent = true;
NUnit.Framework.Assert.AreEqual(container.GetFinishTime(), @event.GetTimestamp());
NUnit.Framework.Assert.AreEqual(container.GetDiagnosticsInfo(), @event.GetEventInfo
()[ContainerMetricsConstants.DiagnosticsInfoEventInfo]);
NUnit.Framework.Assert.AreEqual(container.GetContainerExitStatus(), @event.GetEventInfo
()[ContainerMetricsConstants.ExitStatusEventInfo]);
NUnit.Framework.Assert.AreEqual(container.GetContainerState().ToString(), @event.
GetEventInfo()[ContainerMetricsConstants.StateEventInfo]);
}
}
}
NUnit.Framework.Assert.IsTrue(hasCreatedEvent && hasFinishedEvent);
}
