private static TimelineEntity CreateApplicationTimelineEntity(ApplicationId appId
, bool emptyACLs, bool noAttemptId, bool wrongAppId)
{
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType(ApplicationMetricsConstants.EntityType);
if (wrongAppId)
{
entity.SetEntityId("wrong_app_id");
}
else
{
entity.SetEntityId(appId.ToString());
}
entity.SetDomainId(TimelineDataManager.DefaultDomainId);
entity.AddPrimaryFilter(TimelineStore.SystemFilter.EntityOwner.ToString(), "yarn"
);
IDictionary <string, object> entityInfo = new Dictionary <string, object>();
entityInfo[ApplicationMetricsConstants.NameEntityInfo] = "test app";
entityInfo[ApplicationMetricsConstants.TypeEntityInfo] = "test app type";
entityInfo[ApplicationMetricsConstants.UserEntityInfo] = "user1";
entityInfo[ApplicationMetricsConstants.QueueEntityInfo] = "test queue";
entityInfo[ApplicationMetricsConstants.SubmittedTimeEntityInfo] = int.MaxValue +
1L;
entityInfo[ApplicationMetricsConstants.AppMemMetrics] = 123;
entityInfo[ApplicationMetricsConstants.AppCpuMetrics] = 345;
if (emptyACLs)
{
entityInfo[ApplicationMetricsConstants.AppViewAclsEntityInfo] = string.Empty;
}
else
{
entityInfo[ApplicationMetricsConstants.AppViewAclsEntityInfo] = "user2";
}
entity.SetOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.SetEventType(ApplicationMetricsConstants.CreatedEventType);
tEvent.SetTimestamp(int.MaxValue + 2L + appId.GetId());
entity.AddEvent(tEvent);
tEvent = new TimelineEvent();
tEvent.SetEventType(ApplicationMetricsConstants.FinishedEventType);
tEvent.SetTimestamp(int.MaxValue + 3L + appId.GetId());
IDictionary <string, object> eventInfo = new Dictionary <string, object>();
eventInfo[ApplicationMetricsConstants.DiagnosticsInfoEventInfo] = "test diagnostics info";
eventInfo[ApplicationMetricsConstants.FinalStatusEventInfo] = FinalApplicationStatus
.Undefined.ToString();
eventInfo[ApplicationMetricsConstants.StateEventInfo] = YarnApplicationState.Finished
.ToString();
if (!noAttemptId)
{
eventInfo[ApplicationMetricsConstants.LatestAppAttemptEventInfo] = ApplicationAttemptId
.NewInstance(appId, 1);
}
tEvent.SetEventInfo(eventInfo);
entity.AddEvent(tEvent);
return(entity);
}
private static TimelineEntity GenerateEntity()
{
TimelineEntity entity = new TimelineEntity();
entity.SetEntityId("entity id");
entity.SetEntityType("entity type");
entity.SetStartTime(Runtime.CurrentTimeMillis());
for (int i = 0; i < 2; ++i)
{
TimelineEvent @event = new TimelineEvent();
@event.SetTimestamp(Runtime.CurrentTimeMillis());
@event.SetEventType("test event type " + i);
@event.AddEventInfo("key1", "val1");
@event.AddEventInfo("key2", "val2");
entity.AddEvent(@event);
}
entity.AddRelatedEntity("test ref type 1", "test ref id 1");
entity.AddRelatedEntity("test ref type 2", "test ref id 2");
entity.AddPrimaryFilter("pkey1", "pval1");
entity.AddPrimaryFilter("pkey2", "pval2");
entity.AddOtherInfo("okey1", "oval1");
entity.AddOtherInfo("okey2", "oval2");
entity.SetDomainId("domain id 1");
return(entity);
}
public virtual void TestPutEntities()
{
TestTimelineWebServicesWithSSL.TestTimelineClient client = new TestTimelineWebServicesWithSSL.TestTimelineClient
();
try
{
client.Init(conf);
client.Start();
TimelineEntity expectedEntity = new TimelineEntity();
expectedEntity.SetEntityType("test entity type");
expectedEntity.SetEntityId("test entity id");
expectedEntity.SetDomainId("test domain id");
TimelineEvent @event = new TimelineEvent();
@event.SetEventType("test event type");
@event.SetTimestamp(0L);
expectedEntity.AddEvent(@event);
TimelinePutResponse response = client.PutEntities(expectedEntity);
NUnit.Framework.Assert.AreEqual(0, response.GetErrors().Count);
NUnit.Framework.Assert.IsTrue(client.resp.ToString().Contains("https"));
TimelineEntity actualEntity = store.GetEntity(expectedEntity.GetEntityId(), expectedEntity
.GetEntityType(), EnumSet.AllOf <TimelineReader.Field>());
NUnit.Framework.Assert.IsNotNull(actualEntity);
NUnit.Framework.Assert.AreEqual(expectedEntity.GetEntityId(), actualEntity.GetEntityId
());
NUnit.Framework.Assert.AreEqual(expectedEntity.GetEntityType(), actualEntity.GetEntityType
());
}
finally
{
client.Stop();
client.Close();
}
}
private static TimelineEntity CreateApplicationEntity(ApplicationId applicationId
)
{
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType(ApplicationMetricsConstants.EntityType);
entity.SetEntityId(applicationId.ToString());
return(entity);
}
private static TimelineEntity CreateContainerEntity(ContainerId containerId)
{
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType(ContainerMetricsConstants.EntityType);
entity.SetEntityId(containerId.ToString());
entity.AddPrimaryFilter(ContainerMetricsConstants.ParentPrimariyFilter, containerId
.GetApplicationAttemptId().ToString());
return(entity);
}
private static TimelineEntity CreateAppAttemptEntity(ApplicationAttemptId appAttemptId
)
{
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType(AppAttemptMetricsConstants.EntityType);
entity.SetEntityId(appAttemptId.ToString());
entity.AddPrimaryFilter(AppAttemptMetricsConstants.ParentPrimaryFilter, appAttemptId
.GetApplicationId().ToString());
return(entity);
}
private static TimelineEntity MaskFields(TimelineEntity entity, EnumSet <TimelineReader.Field
> fields)
{
// Conceal the fields that are not going to be exposed
TimelineEntity entityToReturn = new TimelineEntity();
entityToReturn.SetEntityId(entity.GetEntityId());
entityToReturn.SetEntityType(entity.GetEntityType());
entityToReturn.SetStartTime(entity.GetStartTime());
entityToReturn.SetDomainId(entity.GetDomainId());
// Deep copy
if (fields.Contains(TimelineReader.Field.Events))
{
entityToReturn.AddEvents(entity.GetEvents());
}
else
{
if (fields.Contains(TimelineReader.Field.LastEventOnly))
{
entityToReturn.AddEvent(entity.GetEvents()[0]);
}
else
{
entityToReturn.SetEvents(null);
}
}
if (fields.Contains(TimelineReader.Field.RelatedEntities))
{
entityToReturn.AddRelatedEntities(entity.GetRelatedEntities());
}
else
{
entityToReturn.SetRelatedEntities(null);
}
if (fields.Contains(TimelineReader.Field.PrimaryFilters))
{
entityToReturn.AddPrimaryFilters(entity.GetPrimaryFilters());
}
else
{
entityToReturn.SetPrimaryFilters(null);
}
if (fields.Contains(TimelineReader.Field.OtherInfo))
{
entityToReturn.AddOtherInfo(entity.GetOtherInfo());
}
else
{
entityToReturn.SetOtherInfo(null);
}
return(entityToReturn);
}
public virtual void TestUpdatingOldEntityWithoutDomainId()
{
// Set the domain to the default domain when updating
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType("OLD_ENTITY_TYPE_1");
entity.SetEntityId("OLD_ENTITY_ID_1");
entity.SetDomainId(TimelineDataManager.DefaultDomainId);
entity.AddOtherInfo("NEW_OTHER_INFO_KEY", "NEW_OTHER_INFO_VALUE");
TimelineEntities entities = new TimelineEntities();
entities.AddEntity(entity);
TimelinePutResponse response = dataManaer.PostEntities(entities, UserGroupInformation
.GetCurrentUser());
NUnit.Framework.Assert.AreEqual(0, response.GetErrors().Count);
entity = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
NUnit.Framework.Assert.IsNotNull(entity);
// Even in leveldb, the domain is updated to the default domain Id
NUnit.Framework.Assert.AreEqual(TimelineDataManager.DefaultDomainId, entity.GetDomainId
());
NUnit.Framework.Assert.AreEqual(1, entity.GetOtherInfo().Count);
NUnit.Framework.Assert.AreEqual("NEW_OTHER_INFO_KEY", entity.GetOtherInfo().Keys.
GetEnumerator().Next());
NUnit.Framework.Assert.AreEqual("NEW_OTHER_INFO_VALUE", entity.GetOtherInfo().Values
.GetEnumerator().Next());
// Set the domain to the non-default domain when updating
entity = new TimelineEntity();
entity.SetEntityType("OLD_ENTITY_TYPE_1");
entity.SetEntityId("OLD_ENTITY_ID_2");
entity.SetDomainId("NON_DEFAULT");
entity.AddOtherInfo("NEW_OTHER_INFO_KEY", "NEW_OTHER_INFO_VALUE");
entities = new TimelineEntities();
entities.AddEntity(entity);
response = dataManaer.PostEntities(entities, UserGroupInformation.GetCurrentUser(
));
NUnit.Framework.Assert.AreEqual(1, response.GetErrors().Count);
NUnit.Framework.Assert.AreEqual(TimelinePutResponse.TimelinePutError.AccessDenied
, response.GetErrors()[0].GetErrorCode());
entity = store.GetEntity("OLD_ENTITY_ID_2", "OLD_ENTITY_TYPE_1", null);
NUnit.Framework.Assert.IsNotNull(entity);
// In leveldb, the domain Id is still null
NUnit.Framework.Assert.IsNull(entity.GetDomainId());
// Updating is not executed
NUnit.Framework.Assert.AreEqual(0, entity.GetOtherInfo().Count);
}
public virtual void TestRelatingToOldEntityWithoutDomainId()
{
// New entity is put in the default domain
TimelineEntity entityToStore = new TimelineEntity();
entityToStore.SetEntityType("NEW_ENTITY_TYPE_1");
entityToStore.SetEntityId("NEW_ENTITY_ID_1");
entityToStore.SetDomainId(TimelineDataManager.DefaultDomainId);
entityToStore.AddRelatedEntity("OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1");
TimelineEntities entities = new TimelineEntities();
entities.AddEntity(entityToStore);
store.Put(entities);
TimelineEntity entityToGet = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1"
, null);
NUnit.Framework.Assert.IsNotNull(entityToGet);
NUnit.Framework.Assert.IsNull(entityToGet.GetDomainId());
NUnit.Framework.Assert.AreEqual("NEW_ENTITY_TYPE_1", entityToGet.GetRelatedEntities
().Keys.GetEnumerator().Next());
NUnit.Framework.Assert.AreEqual("NEW_ENTITY_ID_1", entityToGet.GetRelatedEntities
().Values.GetEnumerator().Next().GetEnumerator().Next());
// New entity is not put in the default domain
entityToStore = new TimelineEntity();
entityToStore.SetEntityType("NEW_ENTITY_TYPE_2");
entityToStore.SetEntityId("NEW_ENTITY_ID_2");
entityToStore.SetDomainId("NON_DEFAULT");
entityToStore.AddRelatedEntity("OLD_ENTITY_TYPE_1", "OLD_ENTITY_ID_1");
entities = new TimelineEntities();
entities.AddEntity(entityToStore);
TimelinePutResponse response = store.Put(entities);
NUnit.Framework.Assert.AreEqual(1, response.GetErrors().Count);
NUnit.Framework.Assert.AreEqual(TimelinePutResponse.TimelinePutError.ForbiddenRelation
, response.GetErrors()[0].GetErrorCode());
entityToGet = store.GetEntity("OLD_ENTITY_ID_1", "OLD_ENTITY_TYPE_1", null);
NUnit.Framework.Assert.IsNotNull(entityToGet);
NUnit.Framework.Assert.IsNull(entityToGet.GetDomainId());
// Still have one related entity
NUnit.Framework.Assert.AreEqual(1, entityToGet.GetRelatedEntities().Keys.Count);
NUnit.Framework.Assert.AreEqual(1, entityToGet.GetRelatedEntities().Values.GetEnumerator
().Next().Count);
}
/// <exception cref="System.Exception"/>
public Void Call()
{
TimelineClient client = this._enclosing.CreateTimelineClientForUGI();
TimelineEntity entityToStore = new TimelineEntity();
entityToStore.SetEntityType(typeof(Org.Apache.Hadoop.Yarn.Server.Timeline.Security.TestTimelineAuthenticationFilter
).FullName);
entityToStore.SetEntityId("entity1");
entityToStore.SetStartTime(0L);
TimelinePutResponse putResponse = client.PutEntities(entityToStore);
NUnit.Framework.Assert.AreEqual(0, putResponse.GetErrors().Count);
TimelineEntity entityToRead = Org.Apache.Hadoop.Yarn.Server.Timeline.Security.TestTimelineAuthenticationFilter
.testTimelineServer.GetTimelineStore().GetEntity("entity1", typeof(Org.Apache.Hadoop.Yarn.Server.Timeline.Security.TestTimelineAuthenticationFilter
).FullName, null);
NUnit.Framework.Assert.IsNotNull(entityToRead);
return(null);
}
private static TimelineEntity CreateAppAttemptTimelineEntity(ApplicationAttemptId
appAttemptId)
{
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType(AppAttemptMetricsConstants.EntityType);
entity.SetEntityId(appAttemptId.ToString());
entity.SetDomainId(TimelineDataManager.DefaultDomainId);
entity.AddPrimaryFilter(AppAttemptMetricsConstants.ParentPrimaryFilter, appAttemptId
.GetApplicationId().ToString());
entity.AddPrimaryFilter(TimelineStore.SystemFilter.EntityOwner.ToString(), "yarn"
);
TimelineEvent tEvent = new TimelineEvent();
tEvent.SetEventType(AppAttemptMetricsConstants.RegisteredEventType);
tEvent.SetTimestamp(int.MaxValue + 1L);
IDictionary <string, object> eventInfo = new Dictionary <string, object>();
eventInfo[AppAttemptMetricsConstants.TrackingUrlEventInfo] = "test tracking url";
eventInfo[AppAttemptMetricsConstants.OriginalTrackingUrlEventInfo] = "test original tracking url";
eventInfo[AppAttemptMetricsConstants.HostEventInfo] = "test host";
eventInfo[AppAttemptMetricsConstants.RpcPortEventInfo] = 100;
eventInfo[AppAttemptMetricsConstants.MasterContainerEventInfo] = ContainerId.NewContainerId
(appAttemptId, 1);
tEvent.SetEventInfo(eventInfo);
entity.AddEvent(tEvent);
tEvent = new TimelineEvent();
tEvent.SetEventType(AppAttemptMetricsConstants.FinishedEventType);
tEvent.SetTimestamp(int.MaxValue + 2L);
eventInfo = new Dictionary <string, object>();
eventInfo[AppAttemptMetricsConstants.TrackingUrlEventInfo] = "test tracking url";
eventInfo[AppAttemptMetricsConstants.OriginalTrackingUrlEventInfo] = "test original tracking url";
eventInfo[AppAttemptMetricsConstants.DiagnosticsInfoEventInfo] = "test diagnostics info";
eventInfo[AppAttemptMetricsConstants.FinalStatusEventInfo] = FinalApplicationStatus
.Undefined.ToString();
eventInfo[AppAttemptMetricsConstants.StateEventInfo] = YarnApplicationAttemptState
.Finished.ToString();
tEvent.SetEventInfo(eventInfo);
entity.AddEvent(tEvent);
return(entity);
}
public virtual void TestRelatingToNonExistingEntity()
{
TimelineEntity entityToStore = new TimelineEntity();
entityToStore.SetEntityType("TEST_ENTITY_TYPE_1");
entityToStore.SetEntityId("TEST_ENTITY_ID_1");
entityToStore.SetDomainId(TimelineDataManager.DefaultDomainId);
entityToStore.AddRelatedEntity("TEST_ENTITY_TYPE_2", "TEST_ENTITY_ID_2");
TimelineEntities entities = new TimelineEntities();
entities.AddEntity(entityToStore);
store.Put(entities);
TimelineEntity entityToGet = store.GetEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2"
, null);
NUnit.Framework.Assert.IsNotNull(entityToGet);
NUnit.Framework.Assert.AreEqual("DEFAULT", entityToGet.GetDomainId());
NUnit.Framework.Assert.AreEqual("TEST_ENTITY_TYPE_1", entityToGet.GetRelatedEntities
().Keys.GetEnumerator().Next());
NUnit.Framework.Assert.AreEqual("TEST_ENTITY_ID_1", entityToGet.GetRelatedEntities
().Values.GetEnumerator().Next().GetEnumerator().Next());
}
private static TimelineEntity CreateContainerEntity(ContainerId containerId)
{
TimelineEntity entity = new TimelineEntity();
entity.SetEntityType(ContainerMetricsConstants.EntityType);
entity.SetEntityId(containerId.ToString());
entity.SetDomainId(TimelineDataManager.DefaultDomainId);
entity.AddPrimaryFilter(ContainerMetricsConstants.ParentPrimariyFilter, containerId
.GetApplicationAttemptId().ToString());
entity.AddPrimaryFilter(TimelineStore.SystemFilter.EntityOwner.ToString(), "yarn"
);
IDictionary <string, object> entityInfo = new Dictionary <string, object>();
entityInfo[ContainerMetricsConstants.AllocatedMemoryEntityInfo] = -1;
entityInfo[ContainerMetricsConstants.AllocatedVcoreEntityInfo] = -1;
entityInfo[ContainerMetricsConstants.AllocatedHostEntityInfo] = "test host";
entityInfo[ContainerMetricsConstants.AllocatedPortEntityInfo] = 100;
entityInfo[ContainerMetricsConstants.AllocatedPriorityEntityInfo] = -1;
entity.SetOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.SetEventType(ContainerMetricsConstants.CreatedEventType);
tEvent.SetTimestamp(int.MaxValue + 1L);
entity.AddEvent(tEvent);
tEvent = new TimelineEvent();
tEvent.SetEventType(ContainerMetricsConstants.FinishedEventType);
tEvent.SetTimestamp(int.MaxValue + 2L);
IDictionary <string, object> eventInfo = new Dictionary <string, object>();
eventInfo[ContainerMetricsConstants.DiagnosticsInfoEventInfo] = "test diagnostics info";
eventInfo[ContainerMetricsConstants.ExitStatusEventInfo] = -1;
eventInfo[ContainerMetricsConstants.StateEventInfo] = ContainerState.Complete.ToString
();
tEvent.SetEventInfo(eventInfo);
entity.AddEvent(tEvent);
return(entity);
}
public virtual TimelinePutResponse Put(TimelineEntities data)
{
lock (this)
{
TimelinePutResponse response = new TimelinePutResponse();
foreach (TimelineEntity entity in data.GetEntities())
{
EntityIdentifier entityId = new EntityIdentifier(entity.GetEntityId(), entity.GetEntityType
());
// store entity info in memory
TimelineEntity existingEntity = entities[entityId];
if (existingEntity == null)
{
existingEntity = new TimelineEntity();
existingEntity.SetEntityId(entity.GetEntityId());
existingEntity.SetEntityType(entity.GetEntityType());
existingEntity.SetStartTime(entity.GetStartTime());
if (entity.GetDomainId() == null || entity.GetDomainId().Length == 0)
{
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityId(entityId.GetId());
error.SetEntityType(entityId.GetType());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.NoDomain);
response.AddError(error);
continue;
}
existingEntity.SetDomainId(entity.GetDomainId());
entities[entityId] = existingEntity;
entityInsertTimes[entityId] = Runtime.CurrentTimeMillis();
}
if (entity.GetEvents() != null)
{
if (existingEntity.GetEvents() == null)
{
existingEntity.SetEvents(entity.GetEvents());
}
else
{
existingEntity.AddEvents(entity.GetEvents());
}
existingEntity.GetEvents().Sort();
}
// check startTime
if (existingEntity.GetStartTime() == null)
{
if (existingEntity.GetEvents() == null || existingEntity.GetEvents().IsEmpty())
{
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityId(entityId.GetId());
error.SetEntityType(entityId.GetType());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.NoStartTime);
response.AddError(error);
Sharpen.Collections.Remove(entities, entityId);
Sharpen.Collections.Remove(entityInsertTimes, entityId);
continue;
}
else
{
long min = long.MaxValue;
foreach (TimelineEvent e in entity.GetEvents())
{
if (min > e.GetTimestamp())
{
min = e.GetTimestamp();
}
}
existingEntity.SetStartTime(min);
}
}
if (entity.GetPrimaryFilters() != null)
{
if (existingEntity.GetPrimaryFilters() == null)
{
existingEntity.SetPrimaryFilters(new Dictionary <string, ICollection <object> >());
}
foreach (KeyValuePair <string, ICollection <object> > pf in entity.GetPrimaryFilters
())
{
foreach (object pfo in pf.Value)
{
existingEntity.AddPrimaryFilter(pf.Key, MaybeConvert(pfo));
}
}
}
if (entity.GetOtherInfo() != null)
{
if (existingEntity.GetOtherInfo() == null)
{
existingEntity.SetOtherInfo(new Dictionary <string, object>());
}
foreach (KeyValuePair <string, object> info in entity.GetOtherInfo())
{
existingEntity.AddOtherInfo(info.Key, MaybeConvert(info.Value));
}
}
// relate it to other entities
if (entity.GetRelatedEntities() == null)
{
continue;
}
foreach (KeyValuePair <string, ICollection <string> > partRelatedEntities in entity.
GetRelatedEntities())
{
if (partRelatedEntities == null)
{
continue;
}
foreach (string idStr in partRelatedEntities.Value)
{
EntityIdentifier relatedEntityId = new EntityIdentifier(idStr, partRelatedEntities
.Key);
TimelineEntity relatedEntity = entities[relatedEntityId];
if (relatedEntity != null)
{
if (relatedEntity.GetDomainId().Equals(existingEntity.GetDomainId()))
{
relatedEntity.AddRelatedEntity(existingEntity.GetEntityType(), existingEntity.GetEntityId
());
}
else
{
// in this case the entity will be put, but the relation will be
// ignored
TimelinePutResponse.TimelinePutError error = new TimelinePutResponse.TimelinePutError
();
error.SetEntityType(existingEntity.GetEntityType());
error.SetEntityId(existingEntity.GetEntityId());
error.SetErrorCode(TimelinePutResponse.TimelinePutError.ForbiddenRelation);
response.AddError(error);
}
}
else
{
relatedEntity = new TimelineEntity();
relatedEntity.SetEntityId(relatedEntityId.GetId());
relatedEntity.SetEntityType(relatedEntityId.GetType());
relatedEntity.SetStartTime(existingEntity.GetStartTime());
relatedEntity.AddRelatedEntity(existingEntity.GetEntityType(), existingEntity.GetEntityId
());
relatedEntity.SetDomainId(existingEntity.GetDomainId());
entities[relatedEntityId] = relatedEntity;
entityInsertTimes[relatedEntityId] = Runtime.CurrentTimeMillis();
}
}
}
}
return(response);
}
}
