public virtual void TestUpdatingOldEntityWithoutDomainId() { // Set the domain to the default domain when updating TimelineEntity entity = new TimelineEntity(); entity.SetEntityType("OLD_ENTITY_TYPE_1"); entity.SetEntityId("OLD_ENTITY_ID_1"); entity.SetDomainId(TimelineDataManager.DefaultDomainId); entity.AddOtherInfo("NEW_OTHER_INFO_KEY", "NEW_OTHER_INFO_VALUE"); TimelineEntities entities = new TimelineEntities(); entities.AddEntity(entity); TimelinePutResponse response = dataManaer.PostEntities(entities, UserGroupInformation .GetCurrentUser()); NUnit.Framework.Assert.AreEqual(0, response.GetErrors().Count); entity = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null); NUnit.Framework.Assert.IsNotNull(entity); // Even in leveldb, the domain is updated to the default domain Id NUnit.Framework.Assert.AreEqual(TimelineDataManager.DefaultDomainId, entity.GetDomainId ()); NUnit.Framework.Assert.AreEqual(1, entity.GetOtherInfo().Count); NUnit.Framework.Assert.AreEqual("NEW_OTHER_INFO_KEY", entity.GetOtherInfo().Keys. GetEnumerator().Next()); NUnit.Framework.Assert.AreEqual("NEW_OTHER_INFO_VALUE", entity.GetOtherInfo().Values .GetEnumerator().Next()); // Set the domain to the non-default domain when updating entity = new TimelineEntity(); entity.SetEntityType("OLD_ENTITY_TYPE_1"); entity.SetEntityId("OLD_ENTITY_ID_2"); entity.SetDomainId("NON_DEFAULT"); entity.AddOtherInfo("NEW_OTHER_INFO_KEY", "NEW_OTHER_INFO_VALUE"); entities = new TimelineEntities(); entities.AddEntity(entity); response = dataManaer.PostEntities(entities, UserGroupInformation.GetCurrentUser( )); NUnit.Framework.Assert.AreEqual(1, response.GetErrors().Count); NUnit.Framework.Assert.AreEqual(TimelinePutResponse.TimelinePutError.AccessDenied , response.GetErrors()[0].GetErrorCode()); entity = store.GetEntity("OLD_ENTITY_ID_2", "OLD_ENTITY_TYPE_1", null); NUnit.Framework.Assert.IsNotNull(entity); // In leveldb, the domain Id is still null NUnit.Framework.Assert.IsNull(entity.GetDomainId()); // Updating is not executed NUnit.Framework.Assert.AreEqual(0, entity.GetOtherInfo().Count); }
private static TimelineEntity MaskFields(TimelineEntity entity, EnumSet <TimelineReader.Field > fields) { // Conceal the fields that are not going to be exposed TimelineEntity entityToReturn = new TimelineEntity(); entityToReturn.SetEntityId(entity.GetEntityId()); entityToReturn.SetEntityType(entity.GetEntityType()); entityToReturn.SetStartTime(entity.GetStartTime()); entityToReturn.SetDomainId(entity.GetDomainId()); // Deep copy if (fields.Contains(TimelineReader.Field.Events)) { entityToReturn.AddEvents(entity.GetEvents()); } else { if (fields.Contains(TimelineReader.Field.LastEventOnly)) { entityToReturn.AddEvent(entity.GetEvents()[0]); } else { entityToReturn.SetEvents(null); } } if (fields.Contains(TimelineReader.Field.RelatedEntities)) { entityToReturn.AddRelatedEntities(entity.GetRelatedEntities()); } else { entityToReturn.SetRelatedEntities(null); } if (fields.Contains(TimelineReader.Field.PrimaryFilters)) { entityToReturn.AddPrimaryFilters(entity.GetPrimaryFilters()); } else { entityToReturn.SetPrimaryFilters(null); } if (fields.Contains(TimelineReader.Field.OtherInfo)) { entityToReturn.AddOtherInfo(entity.GetOtherInfo()); } else { entityToReturn.SetOtherInfo(null); } return(entityToReturn); }
/// <exception cref="System.IO.IOException"/> public virtual TimelineEntities GetEntities(string entityType, long limit, long windowStart , long windowEnd, string fromId, long fromTs, NameValuePair primaryFilter, ICollection <NameValuePair> secondaryFilters, EnumSet <TimelineReader.Field> fields, TimelineDataManager.CheckAcl checkAcl) { lock (this) { if (limit == null) { limit = DefaultLimit; } if (windowStart == null) { windowStart = long.MinValue; } if (windowEnd == null) { windowEnd = long.MaxValue; } if (fields == null) { fields = EnumSet.AllOf <TimelineReader.Field>(); } IEnumerator <TimelineEntity> entityIterator = null; if (fromId != null) { TimelineEntity firstEntity = entities[new EntityIdentifier(fromId, entityType)]; if (firstEntity == null) { return(new TimelineEntities()); } else { entityIterator = new TreeSet <TimelineEntity>(entities.Values).TailSet(firstEntity , true).GetEnumerator(); } } if (entityIterator == null) { entityIterator = new PriorityQueue <TimelineEntity>(entities.Values).GetEnumerator (); } IList <TimelineEntity> entitiesSelected = new AList <TimelineEntity>(); while (entityIterator.HasNext()) { TimelineEntity entity = entityIterator.Next(); if (entitiesSelected.Count >= limit) { break; } if (!entity.GetEntityType().Equals(entityType)) { continue; } if (entity.GetStartTime() <= windowStart) { continue; } if (entity.GetStartTime() > windowEnd) { continue; } if (fromTs != null && entityInsertTimes[new EntityIdentifier(entity.GetEntityId() , entity.GetEntityType())] > fromTs) { continue; } if (primaryFilter != null && !MatchPrimaryFilter(entity.GetPrimaryFilters(), primaryFilter )) { continue; } if (secondaryFilters != null) { // AND logic bool flag = true; foreach (NameValuePair secondaryFilter in secondaryFilters) { if (secondaryFilter != null && !MatchPrimaryFilter(entity.GetPrimaryFilters(), secondaryFilter ) && !MatchFilter(entity.GetOtherInfo(), secondaryFilter)) { flag = false; break; } } if (!flag) { continue; } } if (entity.GetDomainId() == null) { entity.SetDomainId(TimelineDataManager.DefaultDomainId); } if (checkAcl == null || checkAcl.Check(entity)) { entitiesSelected.AddItem(entity); } } IList <TimelineEntity> entitiesToReturn = new AList <TimelineEntity>(); foreach (TimelineEntity entitySelected in entitiesSelected) { entitiesToReturn.AddItem(MaskFields(entitySelected, fields)); } entitiesToReturn.Sort(); TimelineEntities entitiesWrapper = new TimelineEntities(); entitiesWrapper.SetEntities(entitiesToReturn); return(entitiesWrapper); } }
private static ContainerReport ConvertToContainerReport(TimelineEntity entity, string serverHttpAddress, string user) { int allocatedMem = 0; int allocatedVcore = 0; string allocatedHost = null; int allocatedPort = -1; int allocatedPriority = 0; long createdTime = 0; long finishedTime = 0; string diagnosticsInfo = null; int exitStatus = ContainerExitStatus.Invalid; ContainerState state = null; string nodeHttpAddress = null; IDictionary <string, object> entityInfo = entity.GetOtherInfo(); if (entityInfo != null) { if (entityInfo.Contains(ContainerMetricsConstants.AllocatedMemoryEntityInfo)) { allocatedMem = (int)entityInfo[ContainerMetricsConstants.AllocatedMemoryEntityInfo ]; } if (entityInfo.Contains(ContainerMetricsConstants.AllocatedVcoreEntityInfo)) { allocatedVcore = (int)entityInfo[ContainerMetricsConstants.AllocatedVcoreEntityInfo ]; } if (entityInfo.Contains(ContainerMetricsConstants.AllocatedHostEntityInfo)) { allocatedHost = entityInfo[ContainerMetricsConstants.AllocatedHostEntityInfo].ToString (); } if (entityInfo.Contains(ContainerMetricsConstants.AllocatedPortEntityInfo)) { allocatedPort = (int)entityInfo[ContainerMetricsConstants.AllocatedPortEntityInfo ]; } if (entityInfo.Contains(ContainerMetricsConstants.AllocatedPriorityEntityInfo)) { allocatedPriority = (int)entityInfo[ContainerMetricsConstants.AllocatedPriorityEntityInfo ]; } if (entityInfo.Contains(ContainerMetricsConstants.AllocatedHostHttpAddressEntityInfo )) { nodeHttpAddress = (string)entityInfo[ContainerMetricsConstants.AllocatedHostHttpAddressEntityInfo ]; } } IList <TimelineEvent> events = entity.GetEvents(); if (events != null) { foreach (TimelineEvent @event in events) { if (@event.GetEventType().Equals(ContainerMetricsConstants.CreatedEventType)) { createdTime = @event.GetTimestamp(); } else { if (@event.GetEventType().Equals(ContainerMetricsConstants.FinishedEventType)) { finishedTime = @event.GetTimestamp(); IDictionary <string, object> eventInfo = @event.GetEventInfo(); if (eventInfo == null) { continue; } if (eventInfo.Contains(ContainerMetricsConstants.DiagnosticsInfoEventInfo)) { diagnosticsInfo = eventInfo[ContainerMetricsConstants.DiagnosticsInfoEventInfo].ToString (); } if (eventInfo.Contains(ContainerMetricsConstants.ExitStatusEventInfo)) { exitStatus = (int)eventInfo[ContainerMetricsConstants.ExitStatusEventInfo]; } if (eventInfo.Contains(ContainerMetricsConstants.StateEventInfo)) { state = ContainerState.ValueOf(eventInfo[ContainerMetricsConstants.StateEventInfo ].ToString()); } } } } } NodeId allocatedNode = NodeId.NewInstance(allocatedHost, allocatedPort); ContainerId containerId = ConverterUtils.ToContainerId(entity.GetEntityId()); string logUrl = WebAppUtils.GetAggregatedLogURL(serverHttpAddress, allocatedNode. ToString(), containerId.ToString(), containerId.ToString(), user); return(ContainerReport.NewInstance(ConverterUtils.ToContainerId(entity.GetEntityId ()), Resource.NewInstance(allocatedMem, allocatedVcore), NodeId.NewInstance(allocatedHost , allocatedPort), Priority.NewInstance(allocatedPriority), createdTime, finishedTime , diagnosticsInfo, logUrl, exitStatus, state, nodeHttpAddress)); }
private static ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt ConvertToApplicationReport (TimelineEntity entity, ApplicationHistoryManagerOnTimelineStore.ApplicationReportField field) { string user = null; string queue = null; string name = null; string type = null; long createdTime = 0; long finishedTime = 0; ApplicationAttemptId latestApplicationAttemptId = null; string diagnosticsInfo = null; FinalApplicationStatus finalStatus = FinalApplicationStatus.Undefined; YarnApplicationState state = null; ApplicationResourceUsageReport appResources = null; IDictionary <ApplicationAccessType, string> appViewACLs = new Dictionary <ApplicationAccessType , string>(); IDictionary <string, object> entityInfo = entity.GetOtherInfo(); if (entityInfo != null) { if (entityInfo.Contains(ApplicationMetricsConstants.UserEntityInfo)) { user = entityInfo[ApplicationMetricsConstants.UserEntityInfo].ToString(); } if (entityInfo.Contains(ApplicationMetricsConstants.AppViewAclsEntityInfo)) { string appViewACLsStr = entityInfo[ApplicationMetricsConstants.AppViewAclsEntityInfo ].ToString(); if (appViewACLsStr.Length > 0) { appViewACLs[ApplicationAccessType.ViewApp] = appViewACLsStr; } } if (field == ApplicationHistoryManagerOnTimelineStore.ApplicationReportField.UserAndAcls) { return(new ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt(ApplicationReport .NewInstance(ConverterUtils.ToApplicationId(entity.GetEntityId()), latestApplicationAttemptId , user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null, null, 1.0F, type, null), appViewACLs)); } if (entityInfo.Contains(ApplicationMetricsConstants.QueueEntityInfo)) { queue = entityInfo[ApplicationMetricsConstants.QueueEntityInfo].ToString(); } if (entityInfo.Contains(ApplicationMetricsConstants.NameEntityInfo)) { name = entityInfo[ApplicationMetricsConstants.NameEntityInfo].ToString(); } if (entityInfo.Contains(ApplicationMetricsConstants.TypeEntityInfo)) { type = entityInfo[ApplicationMetricsConstants.TypeEntityInfo].ToString(); } if (entityInfo.Contains(ApplicationMetricsConstants.AppCpuMetrics)) { long vcoreSeconds = long.Parse(entityInfo[ApplicationMetricsConstants.AppCpuMetrics ].ToString()); long memorySeconds = long.Parse(entityInfo[ApplicationMetricsConstants.AppMemMetrics ].ToString()); appResources = ApplicationResourceUsageReport.NewInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds); } } IList <TimelineEvent> events = entity.GetEvents(); if (events != null) { foreach (TimelineEvent @event in events) { if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType)) { createdTime = @event.GetTimestamp(); } else { if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType)) { finishedTime = @event.GetTimestamp(); IDictionary <string, object> eventInfo = @event.GetEventInfo(); if (eventInfo == null) { continue; } if (eventInfo.Contains(ApplicationMetricsConstants.LatestAppAttemptEventInfo)) { latestApplicationAttemptId = ConverterUtils.ToApplicationAttemptId(eventInfo[ApplicationMetricsConstants .LatestAppAttemptEventInfo].ToString()); } if (eventInfo.Contains(ApplicationMetricsConstants.DiagnosticsInfoEventInfo)) { diagnosticsInfo = eventInfo[ApplicationMetricsConstants.DiagnosticsInfoEventInfo] .ToString(); } if (eventInfo.Contains(ApplicationMetricsConstants.FinalStatusEventInfo)) { finalStatus = FinalApplicationStatus.ValueOf(eventInfo[ApplicationMetricsConstants .FinalStatusEventInfo].ToString()); } if (eventInfo.Contains(ApplicationMetricsConstants.StateEventInfo)) { state = YarnApplicationState.ValueOf(eventInfo[ApplicationMetricsConstants.StateEventInfo ].ToString()); } } } } } return(new ApplicationHistoryManagerOnTimelineStore.ApplicationReportExt(ApplicationReport .NewInstance(ConverterUtils.ToApplicationId(entity.GetEntityId()), latestApplicationAttemptId , user, queue, name, null, -1, null, state, diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources, null, 1.0F, type, null), appViewACLs)); }
/// <exception cref="System.Exception"/> public virtual void TestPublishApplicationMetrics() { for (int i = 1; i <= 2; ++i) { ApplicationId appId = ApplicationId.NewInstance(0, i); RMApp app = CreateRMApp(appId); metricsPublisher.AppCreated(app, app.GetStartTime()); metricsPublisher.AppFinished(app, RMAppState.Finished, app.GetFinishTime()); if (i == 1) { metricsPublisher.AppACLsUpdated(app, "uers1,user2", 4L); } else { // in case user doesn't specify the ACLs metricsPublisher.AppACLsUpdated(app, null, 4L); } TimelineEntity entity = null; do { entity = store.GetEntity(appId.ToString(), ApplicationMetricsConstants.EntityType , EnumSet.AllOf <TimelineReader.Field>()); }while (entity == null || entity.GetEvents().Count < 3); // ensure three events are both published before leaving the loop // verify all the fields NUnit.Framework.Assert.AreEqual(ApplicationMetricsConstants.EntityType, entity.GetEntityType ()); NUnit.Framework.Assert.AreEqual(app.GetApplicationId().ToString(), entity.GetEntityId ()); NUnit.Framework.Assert.AreEqual(app.GetName(), entity.GetOtherInfo()[ApplicationMetricsConstants .NameEntityInfo]); NUnit.Framework.Assert.AreEqual(app.GetQueue(), entity.GetOtherInfo()[ApplicationMetricsConstants .QueueEntityInfo]); NUnit.Framework.Assert.AreEqual(app.GetUser(), entity.GetOtherInfo()[ApplicationMetricsConstants .UserEntityInfo]); NUnit.Framework.Assert.AreEqual(app.GetApplicationType(), entity.GetOtherInfo()[ApplicationMetricsConstants .TypeEntityInfo]); NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), entity.GetOtherInfo()[ApplicationMetricsConstants .SubmittedTimeEntityInfo]); if (i == 1) { NUnit.Framework.Assert.AreEqual("uers1,user2", entity.GetOtherInfo()[ApplicationMetricsConstants .AppViewAclsEntityInfo]); } else { NUnit.Framework.Assert.AreEqual(string.Empty, entity.GetOtherInfo()[ApplicationMetricsConstants .AppViewAclsEntityInfo]); NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetMemorySeconds(), long.Parse (entity.GetOtherInfo()[ApplicationMetricsConstants.AppMemMetrics].ToString())); NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetVcoreSeconds(), long.Parse (entity.GetOtherInfo()[ApplicationMetricsConstants.AppCpuMetrics].ToString())); } bool hasCreatedEvent = false; bool hasFinishedEvent = false; bool hasACLsUpdatedEvent = false; foreach (TimelineEvent @event in entity.GetEvents()) { if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType)) { hasCreatedEvent = true; NUnit.Framework.Assert.AreEqual(app.GetStartTime(), @event.GetTimestamp()); } else { if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType)) { hasFinishedEvent = true; NUnit.Framework.Assert.AreEqual(app.GetFinishTime(), @event.GetTimestamp()); NUnit.Framework.Assert.AreEqual(app.GetDiagnostics().ToString(), @event.GetEventInfo ()[ApplicationMetricsConstants.DiagnosticsInfoEventInfo]); NUnit.Framework.Assert.AreEqual(app.GetFinalApplicationStatus().ToString(), @event .GetEventInfo()[ApplicationMetricsConstants.FinalStatusEventInfo]); NUnit.Framework.Assert.AreEqual(YarnApplicationState.Finished.ToString(), @event. GetEventInfo()[ApplicationMetricsConstants.StateEventInfo]); } else { if (@event.GetEventType().Equals(ApplicationMetricsConstants.AclsUpdatedEventType )) { hasACLsUpdatedEvent = true; NUnit.Framework.Assert.AreEqual(4L, @event.GetTimestamp()); } } } } NUnit.Framework.Assert.IsTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent ); } }
/// <exception cref="System.Exception"/> public virtual void TestPublishContainerMetrics() { ContainerId containerId = ContainerId.NewContainerId(ApplicationAttemptId.NewInstance (ApplicationId.NewInstance(0, 1), 1), 1); RMContainer container = CreateRMContainer(containerId); metricsPublisher.ContainerCreated(container, container.GetCreationTime()); metricsPublisher.ContainerFinished(container, container.GetFinishTime()); TimelineEntity entity = null; do { entity = store.GetEntity(containerId.ToString(), ContainerMetricsConstants.EntityType , EnumSet.AllOf <TimelineReader.Field>()); }while (entity == null || entity.GetEvents().Count < 2); // ensure two events are both published before leaving the loop // verify all the fields NUnit.Framework.Assert.AreEqual(ContainerMetricsConstants.EntityType, entity.GetEntityType ()); NUnit.Framework.Assert.AreEqual(containerId.ToString(), entity.GetEntityId()); NUnit.Framework.Assert.AreEqual(containerId.GetApplicationAttemptId().ToString(), entity.GetPrimaryFilters()[ContainerMetricsConstants.ParentPrimariyFilter].GetEnumerator ().Next()); NUnit.Framework.Assert.AreEqual(container.GetAllocatedNode().GetHost(), entity.GetOtherInfo ()[ContainerMetricsConstants.AllocatedHostEntityInfo]); NUnit.Framework.Assert.AreEqual(container.GetAllocatedNode().GetPort(), entity.GetOtherInfo ()[ContainerMetricsConstants.AllocatedPortEntityInfo]); NUnit.Framework.Assert.AreEqual(container.GetAllocatedResource().GetMemory(), entity .GetOtherInfo()[ContainerMetricsConstants.AllocatedMemoryEntityInfo]); NUnit.Framework.Assert.AreEqual(container.GetAllocatedResource().GetVirtualCores( ), entity.GetOtherInfo()[ContainerMetricsConstants.AllocatedVcoreEntityInfo]); NUnit.Framework.Assert.AreEqual(container.GetAllocatedPriority().GetPriority(), entity .GetOtherInfo()[ContainerMetricsConstants.AllocatedPriorityEntityInfo]); bool hasCreatedEvent = false; bool hasFinishedEvent = false; foreach (TimelineEvent @event in entity.GetEvents()) { if (@event.GetEventType().Equals(ContainerMetricsConstants.CreatedEventType)) { hasCreatedEvent = true; NUnit.Framework.Assert.AreEqual(container.GetCreationTime(), @event.GetTimestamp( )); } else { if (@event.GetEventType().Equals(ContainerMetricsConstants.FinishedEventType)) { hasFinishedEvent = true; NUnit.Framework.Assert.AreEqual(container.GetFinishTime(), @event.GetTimestamp()); NUnit.Framework.Assert.AreEqual(container.GetDiagnosticsInfo(), @event.GetEventInfo ()[ContainerMetricsConstants.DiagnosticsInfoEventInfo]); NUnit.Framework.Assert.AreEqual(container.GetContainerExitStatus(), @event.GetEventInfo ()[ContainerMetricsConstants.ExitStatusEventInfo]); NUnit.Framework.Assert.AreEqual(container.GetContainerState().ToString(), @event. GetEventInfo()[ContainerMetricsConstants.StateEventInfo]); } } } NUnit.Framework.Assert.IsTrue(hasCreatedEvent && hasFinishedEvent); }