private static TimelineEntity CreateApplicationTimelineEntity(ApplicationId appId , bool emptyACLs, bool noAttemptId, bool wrongAppId) { TimelineEntity entity = new TimelineEntity(); entity.SetEntityType(ApplicationMetricsConstants.EntityType); if (wrongAppId) { entity.SetEntityId("wrong_app_id"); } else { entity.SetEntityId(appId.ToString()); } entity.SetDomainId(TimelineDataManager.DefaultDomainId); entity.AddPrimaryFilter(TimelineStore.SystemFilter.EntityOwner.ToString(), "yarn" ); IDictionary <string, object> entityInfo = new Dictionary <string, object>(); entityInfo[ApplicationMetricsConstants.NameEntityInfo] = "test app"; entityInfo[ApplicationMetricsConstants.TypeEntityInfo] = "test app type"; entityInfo[ApplicationMetricsConstants.UserEntityInfo] = "user1"; entityInfo[ApplicationMetricsConstants.QueueEntityInfo] = "test queue"; entityInfo[ApplicationMetricsConstants.SubmittedTimeEntityInfo] = int.MaxValue + 1L; entityInfo[ApplicationMetricsConstants.AppMemMetrics] = 123; entityInfo[ApplicationMetricsConstants.AppCpuMetrics] = 345; if (emptyACLs) { entityInfo[ApplicationMetricsConstants.AppViewAclsEntityInfo] = string.Empty; } else { entityInfo[ApplicationMetricsConstants.AppViewAclsEntityInfo] = "user2"; } entity.SetOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.SetEventType(ApplicationMetricsConstants.CreatedEventType); tEvent.SetTimestamp(int.MaxValue + 2L + appId.GetId()); entity.AddEvent(tEvent); tEvent = new TimelineEvent(); tEvent.SetEventType(ApplicationMetricsConstants.FinishedEventType); tEvent.SetTimestamp(int.MaxValue + 3L + appId.GetId()); IDictionary <string, object> eventInfo = new Dictionary <string, object>(); eventInfo[ApplicationMetricsConstants.DiagnosticsInfoEventInfo] = "test diagnostics info"; eventInfo[ApplicationMetricsConstants.FinalStatusEventInfo] = FinalApplicationStatus .Undefined.ToString(); eventInfo[ApplicationMetricsConstants.StateEventInfo] = YarnApplicationState.Finished .ToString(); if (!noAttemptId) { eventInfo[ApplicationMetricsConstants.LatestAppAttemptEventInfo] = ApplicationAttemptId .NewInstance(appId, 1); } tEvent.SetEventInfo(eventInfo); entity.AddEvent(tEvent); return(entity); }
private static TimelineEntity GenerateEntity() { TimelineEntity entity = new TimelineEntity(); entity.SetEntityId("entity id"); entity.SetEntityType("entity type"); entity.SetStartTime(Runtime.CurrentTimeMillis()); for (int i = 0; i < 2; ++i) { TimelineEvent @event = new TimelineEvent(); @event.SetTimestamp(Runtime.CurrentTimeMillis()); @event.SetEventType("test event type " + i); @event.AddEventInfo("key1", "val1"); @event.AddEventInfo("key2", "val2"); entity.AddEvent(@event); } entity.AddRelatedEntity("test ref type 1", "test ref id 1"); entity.AddRelatedEntity("test ref type 2", "test ref id 2"); entity.AddPrimaryFilter("pkey1", "pval1"); entity.AddPrimaryFilter("pkey2", "pval2"); entity.AddOtherInfo("okey1", "oval1"); entity.AddOtherInfo("okey2", "oval2"); entity.SetDomainId("domain id 1"); return(entity); }
public virtual void TestPutEntities() { TestTimelineWebServicesWithSSL.TestTimelineClient client = new TestTimelineWebServicesWithSSL.TestTimelineClient (); try { client.Init(conf); client.Start(); TimelineEntity expectedEntity = new TimelineEntity(); expectedEntity.SetEntityType("test entity type"); expectedEntity.SetEntityId("test entity id"); expectedEntity.SetDomainId("test domain id"); TimelineEvent @event = new TimelineEvent(); @event.SetEventType("test event type"); @event.SetTimestamp(0L); expectedEntity.AddEvent(@event); TimelinePutResponse response = client.PutEntities(expectedEntity); NUnit.Framework.Assert.AreEqual(0, response.GetErrors().Count); NUnit.Framework.Assert.IsTrue(client.resp.ToString().Contains("https")); TimelineEntity actualEntity = store.GetEntity(expectedEntity.GetEntityId(), expectedEntity .GetEntityType(), EnumSet.AllOf <TimelineReader.Field>()); NUnit.Framework.Assert.IsNotNull(actualEntity); NUnit.Framework.Assert.AreEqual(expectedEntity.GetEntityId(), actualEntity.GetEntityId ()); NUnit.Framework.Assert.AreEqual(expectedEntity.GetEntityType(), actualEntity.GetEntityType ()); } finally { client.Stop(); client.Close(); } }
private static TimelineEntity CreateApplicationEntity(ApplicationId applicationId ) { TimelineEntity entity = new TimelineEntity(); entity.SetEntityType(ApplicationMetricsConstants.EntityType); entity.SetEntityId(applicationId.ToString()); return(entity); }
private static TimelineEntity CreateContainerEntity(ContainerId containerId) { TimelineEntity entity = new TimelineEntity(); entity.SetEntityType(ContainerMetricsConstants.EntityType); entity.SetEntityId(containerId.ToString()); entity.AddPrimaryFilter(ContainerMetricsConstants.ParentPrimariyFilter, containerId .GetApplicationAttemptId().ToString()); return(entity); }
private static TimelineEntity CreateAppAttemptEntity(ApplicationAttemptId appAttemptId ) { TimelineEntity entity = new TimelineEntity(); entity.SetEntityType(AppAttemptMetricsConstants.EntityType); entity.SetEntityId(appAttemptId.ToString()); entity.AddPrimaryFilter(AppAttemptMetricsConstants.ParentPrimaryFilter, appAttemptId .GetApplicationId().ToString()); return(entity); }
private static TimelineEntity MaskFields(TimelineEntity entity, EnumSet <TimelineReader.Field > fields) { // Conceal the fields that are not going to be exposed TimelineEntity entityToReturn = new TimelineEntity(); entityToReturn.SetEntityId(entity.GetEntityId()); entityToReturn.SetEntityType(entity.GetEntityType()); entityToReturn.SetStartTime(entity.GetStartTime()); entityToReturn.SetDomainId(entity.GetDomainId()); // Deep copy if (fields.Contains(TimelineReader.Field.Events)) { entityToReturn.AddEvents(entity.GetEvents()); } else { if (fields.Contains(TimelineReader.Field.LastEventOnly)) { entityToReturn.AddEvent(entity.GetEvents()[0]); } else { entityToReturn.SetEvents(null); } } if (fields.Contains(TimelineReader.Field.RelatedEntities)) { entityToReturn.AddRelatedEntities(entity.GetRelatedEntities()); } else { entityToReturn.SetRelatedEntities(null); } if (fields.Contains(TimelineReader.Field.PrimaryFilters)) { entityToReturn.AddPrimaryFilters(entity.GetPrimaryFilters()); } else { entityToReturn.SetPrimaryFilters(null); } if (fields.Contains(TimelineReader.Field.OtherInfo)) { entityToReturn.AddOtherInfo(entity.GetOtherInfo()); } else { entityToReturn.SetOtherInfo(null); } return(entityToReturn); }
public virtual void TestUpdatingOldEntityWithoutDomainId() { // Set the domain to the default domain when updating TimelineEntity entity = new TimelineEntity(); entity.SetEntityType("OLD_ENTITY_TYPE_1"); entity.SetEntityId("OLD_ENTITY_ID_1"); entity.SetDomainId(TimelineDataManager.DefaultDomainId); entity.AddOtherInfo("NEW_OTHER_INFO_KEY", "NEW_OTHER_INFO_VALUE"); TimelineEntities entities = new TimelineEntities(); entities.AddEntity(entity); TimelinePutResponse response = dataManaer.PostEntities(entities, UserGroupInformation .GetCurrentUser()); NUnit.Framework.Assert.AreEqual(0, response.GetErrors().Count); entity = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null); NUnit.Framework.Assert.IsNotNull(entity); // Even in leveldb, the domain is updated to the default domain Id NUnit.Framework.Assert.AreEqual(TimelineDataManager.DefaultDomainId, entity.GetDomainId ()); NUnit.Framework.Assert.AreEqual(1, entity.GetOtherInfo().Count); NUnit.Framework.Assert.AreEqual("NEW_OTHER_INFO_KEY", entity.GetOtherInfo().Keys. GetEnumerator().Next()); NUnit.Framework.Assert.AreEqual("NEW_OTHER_INFO_VALUE", entity.GetOtherInfo().Values .GetEnumerator().Next()); // Set the domain to the non-default domain when updating entity = new TimelineEntity(); entity.SetEntityType("OLD_ENTITY_TYPE_1"); entity.SetEntityId("OLD_ENTITY_ID_2"); entity.SetDomainId("NON_DEFAULT"); entity.AddOtherInfo("NEW_OTHER_INFO_KEY", "NEW_OTHER_INFO_VALUE"); entities = new TimelineEntities(); entities.AddEntity(entity); response = dataManaer.PostEntities(entities, UserGroupInformation.GetCurrentUser( )); NUnit.Framework.Assert.AreEqual(1, response.GetErrors().Count); NUnit.Framework.Assert.AreEqual(TimelinePutResponse.TimelinePutError.AccessDenied , response.GetErrors()[0].GetErrorCode()); entity = store.GetEntity("OLD_ENTITY_ID_2", "OLD_ENTITY_TYPE_1", null); NUnit.Framework.Assert.IsNotNull(entity); // In leveldb, the domain Id is still null NUnit.Framework.Assert.IsNull(entity.GetDomainId()); // Updating is not executed NUnit.Framework.Assert.AreEqual(0, entity.GetOtherInfo().Count); }
public virtual void TestRelatingToOldEntityWithoutDomainId() { // New entity is put in the default domain TimelineEntity entityToStore = new TimelineEntity(); entityToStore.SetEntityType("NEW_ENTITY_TYPE_1"); entityToStore.SetEntityId("NEW_ENTITY_ID_1"); entityToStore.SetDomainId(TimelineDataManager.DefaultDomainId); entityToStore.AddRelatedEntity("OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1"); TimelineEntities entities = new TimelineEntities(); entities.AddEntity(entityToStore); store.Put(entities); TimelineEntity entityToGet = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1" , null); NUnit.Framework.Assert.IsNotNull(entityToGet); NUnit.Framework.Assert.IsNull(entityToGet.GetDomainId()); NUnit.Framework.Assert.AreEqual("NEW_ENTITY_TYPE_1", entityToGet.GetRelatedEntities ().Keys.GetEnumerator().Next()); NUnit.Framework.Assert.AreEqual("NEW_ENTITY_ID_1", entityToGet.GetRelatedEntities ().Values.GetEnumerator().Next().GetEnumerator().Next()); // New entity is not put in the default domain entityToStore = new TimelineEntity(); entityToStore.SetEntityType("NEW_ENTITY_TYPE_2"); entityToStore.SetEntityId("NEW_ENTITY_ID_2"); entityToStore.SetDomainId("NON_DEFAULT"); entityToStore.AddRelatedEntity("OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1"); entities = new TimelineEntities(); entities.AddEntity(entityToStore); TimelinePutResponse response = store.Put(entities); NUnit.Framework.Assert.AreEqual(1, response.GetErrors().Count); NUnit.Framework.Assert.AreEqual(TimelinePutResponse.TimelinePutError.ForbiddenRelation , response.GetErrors()[0].GetErrorCode()); entityToGet = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null); NUnit.Framework.Assert.IsNotNull(entityToGet); NUnit.Framework.Assert.IsNull(entityToGet.GetDomainId()); // Still have one related entity NUnit.Framework.Assert.AreEqual(1, entityToGet.GetRelatedEntities().Keys.Count); NUnit.Framework.Assert.AreEqual(1, entityToGet.GetRelatedEntities().Values.GetEnumerator ().Next().Count); }
/// <exception cref="System.Exception"/> public Void Call() { TimelineClient client = this._enclosing.CreateTimelineClientForUGI(); TimelineEntity entityToStore = new TimelineEntity(); entityToStore.SetEntityType(typeof(Org.Apache.Hadoop.Yarn.Server.Timeline.Security.TestTimelineAuthenticationFilter ).FullName); entityToStore.SetEntityId("entity1"); entityToStore.SetStartTime(0L); TimelinePutResponse putResponse = client.PutEntities(entityToStore); NUnit.Framework.Assert.AreEqual(0, putResponse.GetErrors().Count); TimelineEntity entityToRead = Org.Apache.Hadoop.Yarn.Server.Timeline.Security.TestTimelineAuthenticationFilter .testTimelineServer.GetTimelineStore().GetEntity("entity1", typeof(Org.Apache.Hadoop.Yarn.Server.Timeline.Security.TestTimelineAuthenticationFilter ).FullName, null); NUnit.Framework.Assert.IsNotNull(entityToRead); return(null); }
private static TimelineEntity CreateAppAttemptTimelineEntity(ApplicationAttemptId appAttemptId) { TimelineEntity entity = new TimelineEntity(); entity.SetEntityType(AppAttemptMetricsConstants.EntityType); entity.SetEntityId(appAttemptId.ToString()); entity.SetDomainId(TimelineDataManager.DefaultDomainId); entity.AddPrimaryFilter(AppAttemptMetricsConstants.ParentPrimaryFilter, appAttemptId .GetApplicationId().ToString()); entity.AddPrimaryFilter(TimelineStore.SystemFilter.EntityOwner.ToString(), "yarn" ); TimelineEvent tEvent = new TimelineEvent(); tEvent.SetEventType(AppAttemptMetricsConstants.RegisteredEventType); tEvent.SetTimestamp(int.MaxValue + 1L); IDictionary <string, object> eventInfo = new Dictionary <string, object>(); eventInfo[AppAttemptMetricsConstants.TrackingUrlEventInfo] = "test tracking url"; eventInfo[AppAttemptMetricsConstants.OriginalTrackingUrlEventInfo] = "test original tracking url"; eventInfo[AppAttemptMetricsConstants.HostEventInfo] = "test host"; eventInfo[AppAttemptMetricsConstants.RpcPortEventInfo] = 100; eventInfo[AppAttemptMetricsConstants.MasterContainerEventInfo] = ContainerId.NewContainerId (appAttemptId, 1); tEvent.SetEventInfo(eventInfo); entity.AddEvent(tEvent); tEvent = new TimelineEvent(); tEvent.SetEventType(AppAttemptMetricsConstants.FinishedEventType); tEvent.SetTimestamp(int.MaxValue + 2L); eventInfo = new Dictionary <string, object>(); eventInfo[AppAttemptMetricsConstants.TrackingUrlEventInfo] = "test tracking url"; eventInfo[AppAttemptMetricsConstants.OriginalTrackingUrlEventInfo] = "test original tracking url"; eventInfo[AppAttemptMetricsConstants.DiagnosticsInfoEventInfo] = "test diagnostics info"; eventInfo[AppAttemptMetricsConstants.FinalStatusEventInfo] = FinalApplicationStatus .Undefined.ToString(); eventInfo[AppAttemptMetricsConstants.StateEventInfo] = YarnApplicationAttemptState .Finished.ToString(); tEvent.SetEventInfo(eventInfo); entity.AddEvent(tEvent); return(entity); }
public virtual void TestRelatingToNonExistingEntity() { TimelineEntity entityToStore = new TimelineEntity(); entityToStore.SetEntityType("TEST_ENTITY_TYPE_1"); entityToStore.SetEntityId("TEST_ENTITY_ID_1"); entityToStore.SetDomainId(TimelineDataManager.DefaultDomainId); entityToStore.AddRelatedEntity("TEST_ENTITY_TYPE_2", "TEST_ENTITY_ID_2"); TimelineEntities entities = new TimelineEntities(); entities.AddEntity(entityToStore); store.Put(entities); TimelineEntity entityToGet = store.GetEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2" , null); NUnit.Framework.Assert.IsNotNull(entityToGet); NUnit.Framework.Assert.AreEqual("DEFAULT", entityToGet.GetDomainId()); NUnit.Framework.Assert.AreEqual("TEST_ENTITY_TYPE_1", entityToGet.GetRelatedEntities ().Keys.GetEnumerator().Next()); NUnit.Framework.Assert.AreEqual("TEST_ENTITY_ID_1", entityToGet.GetRelatedEntities ().Values.GetEnumerator().Next().GetEnumerator().Next()); }
private static TimelineEntity CreateContainerEntity(ContainerId containerId) { TimelineEntity entity = new TimelineEntity(); entity.SetEntityType(ContainerMetricsConstants.EntityType); entity.SetEntityId(containerId.ToString()); entity.SetDomainId(TimelineDataManager.DefaultDomainId); entity.AddPrimaryFilter(ContainerMetricsConstants.ParentPrimariyFilter, containerId .GetApplicationAttemptId().ToString()); entity.AddPrimaryFilter(TimelineStore.SystemFilter.EntityOwner.ToString(), "yarn" ); IDictionary <string, object> entityInfo = new Dictionary <string, object>(); entityInfo[ContainerMetricsConstants.AllocatedMemoryEntityInfo] = -1; entityInfo[ContainerMetricsConstants.AllocatedVcoreEntityInfo] = -1; entityInfo[ContainerMetricsConstants.AllocatedHostEntityInfo] = "test host"; entityInfo[ContainerMetricsConstants.AllocatedPortEntityInfo] = 100; entityInfo[ContainerMetricsConstants.AllocatedPriorityEntityInfo] = -1; entity.SetOtherInfo(entityInfo); TimelineEvent tEvent = new TimelineEvent(); tEvent.SetEventType(ContainerMetricsConstants.CreatedEventType); tEvent.SetTimestamp(int.MaxValue + 1L); entity.AddEvent(tEvent); tEvent = new TimelineEvent(); tEvent.SetEventType(ContainerMetricsConstants.FinishedEventType); tEvent.SetTimestamp(int.MaxValue + 2L); IDictionary <string, object> eventInfo = new Dictionary <string, object>(); eventInfo[ContainerMetricsConstants.DiagnosticsInfoEventInfo] = "test diagnostics info"; eventInfo[ContainerMetricsConstants.ExitStatusEventInfo] = -1; eventInfo[ContainerMetricsConstants.StateEventInfo] = ContainerState.Complete.ToString (); tEvent.SetEventInfo(eventInfo); entity.AddEvent(tEvent); return(entity); }
public virtual TimelinePutResponse Put(TimelineEntities data) { lock (this) { TimelinePutResponse response = new TimelinePutResponse(); foreach (TimelineEntity entity in data.GetEntities()) { EntityIdentifier entityId = new EntityIdentifier(entity.GetEntityId(), entity.GetEntityType ()); // store entity info in memory TimelineEntity existingEntity = entities[entityId]; if (existingEntity == null) { existingEntity = new TimelineEntity(); existingEntity.SetEntityId(entity.GetEntityId()); existingEntity.SetEntityType(entity.GetEntityType()); existingEntity.SetStartTime(entity.GetStartTime()); if (entity.GetDomainId() == null || entity.GetDomainId().Length == 0) { TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError (); error.SetEntityId(entityId.GetId()); error.SetEntityType(entityId.GetType()); error.SetErrorCode(TimelinePutResponse.TimelinePutError.NoDomain); response.AddError(error); continue; } existingEntity.SetDomainId(entity.GetDomainId()); entities[entityId] = existingEntity; entityInsertTimes[entityId] = Runtime.CurrentTimeMillis(); } if (entity.GetEvents() != null) { if (existingEntity.GetEvents() == null) { existingEntity.SetEvents(entity.GetEvents()); } else { existingEntity.AddEvents(entity.GetEvents()); } existingEntity.GetEvents().Sort(); } // check startTime if (existingEntity.GetStartTime() == null) { if (existingEntity.GetEvents() == null || existingEntity.GetEvents().IsEmpty()) { TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError (); error.SetEntityId(entityId.GetId()); error.SetEntityType(entityId.GetType()); error.SetErrorCode(TimelinePutResponse.TimelinePutError.NoStartTime); response.AddError(error); Sharpen.Collections.Remove(entities, entityId); Sharpen.Collections.Remove(entityInsertTimes, entityId); continue; } else { long min = long.MaxValue; foreach (TimelineEvent e in entity.GetEvents()) { if (min > e.GetTimestamp()) { min = e.GetTimestamp(); } } existingEntity.SetStartTime(min); } } if (entity.GetPrimaryFilters() != null) { if (existingEntity.GetPrimaryFilters() == null) { existingEntity.SetPrimaryFilters(new Dictionary <string, ICollection <object> >()); } foreach (KeyValuePair <string, ICollection <object> > pf in entity.GetPrimaryFilters ()) { foreach (object pfo in pf.Value) { existingEntity.AddPrimaryFilter(pf.Key, MaybeConvert(pfo)); } } } if (entity.GetOtherInfo() != null) { if (existingEntity.GetOtherInfo() == null) { existingEntity.SetOtherInfo(new Dictionary <string, object>()); } foreach (KeyValuePair <string, object> info in entity.GetOtherInfo()) { existingEntity.AddOtherInfo(info.Key, MaybeConvert(info.Value)); } } // relate it to other entities if (entity.GetRelatedEntities() == null) { continue; } foreach (KeyValuePair <string, ICollection <string> > partRelatedEntities in entity. GetRelatedEntities()) { if (partRelatedEntities == null) { continue; } foreach (string idStr in partRelatedEntities.Value) { EntityIdentifier relatedEntityId = new EntityIdentifier(idStr, partRelatedEntities .Key); TimelineEntity relatedEntity = entities[relatedEntityId]; if (relatedEntity != null) { if (relatedEntity.GetDomainId().Equals(existingEntity.GetDomainId())) { relatedEntity.AddRelatedEntity(existingEntity.GetEntityType(), existingEntity.GetEntityId ()); } else { // in this case the entity will be put, but the relation will be // ignored TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError (); error.SetEntityType(existingEntity.GetEntityType()); error.SetEntityId(existingEntity.GetEntityId()); error.SetErrorCode(TimelinePutResponse.TimelinePutError.ForbiddenRelation); response.AddError(error); } } else { relatedEntity = new TimelineEntity(); relatedEntity.SetEntityId(relatedEntityId.GetId()); relatedEntity.SetEntityType(relatedEntityId.GetType()); relatedEntity.SetStartTime(existingEntity.GetStartTime()); relatedEntity.AddRelatedEntity(existingEntity.GetEntityType(), existingEntity.GetEntityId ()); relatedEntity.SetDomainId(existingEntity.GetDomainId()); entities[relatedEntityId] = relatedEntity; entityInsertTimes[relatedEntityId] = Runtime.CurrentTimeMillis(); } } } } return(response); } }